aboutsummaryrefslogtreecommitdiff
path: root/examples/redis-unstable/deps/jemalloc/src
diff options
context:
space:
mode:
Diffstat (limited to 'examples/redis-unstable/deps/jemalloc/src')
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/arena.c1891
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/background_thread.c820
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/base.c529
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/bin.c69
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/bin_info.c30
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/bitmap.c120
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/buf_writer.c144
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/cache_bin.c99
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/ckh.c569
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/counter.c30
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/ctl.c4414
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/decay.c295
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/div.c55
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/ecache.c35
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/edata.c6
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/edata_cache.c154
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/ehooks.c275
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/emap.c386
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/eset.c282
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/exp_grow.c8
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/extent.c1326
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/extent_dss.c277
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/extent_mmap.c41
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/fxp.c124
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/hook.c195
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/hpa.c1044
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/hpa_hooks.c63
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/hpdata.c325
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/inspect.c77
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/jemalloc.c4539
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/jemalloc_cpp.cpp254
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/large.c322
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/log.c78
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/malloc_io.c697
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/mutex.c228
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/nstime.c289
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/pa.c277
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/pa_extra.c191
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/pac.c587
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/pages.c824
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/pai.c31
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/peak_event.c82
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/prof.c789
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/prof_data.c1447
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/prof_log.c717
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/prof_recent.c600
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/prof_stats.c57
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/prof_sys.c669
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/psset.c385
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/rtree.c261
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/safety_check.c36
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/san.c208
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/san_bump.c104
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/sc.c306
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/sec.c422
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/stats.c1973
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/sz.c114
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/tcache.c1101
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/test_hooks.c12
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/thread_event.c343
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/ticker.c32
-rwxr-xr-xexamples/redis-unstable/deps/jemalloc/src/ticker.py15
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/tsd.c549
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/witness.c122
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/zone.c469
65 files changed, 0 insertions, 32813 deletions
diff --git a/examples/redis-unstable/deps/jemalloc/src/arena.c b/examples/redis-unstable/deps/jemalloc/src/arena.c
deleted file mode 100644
index 857b27c..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/arena.c
+++ /dev/null
@@ -1,1891 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/decay.h"
6#include "jemalloc/internal/ehooks.h"
7#include "jemalloc/internal/extent_dss.h"
8#include "jemalloc/internal/extent_mmap.h"
9#include "jemalloc/internal/san.h"
10#include "jemalloc/internal/mutex.h"
11#include "jemalloc/internal/rtree.h"
12#include "jemalloc/internal/safety_check.h"
13#include "jemalloc/internal/util.h"
14
15JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
16
17/******************************************************************************/
18/* Data. */
19
20/*
21 * Define names for both unininitialized and initialized phases, so that
22 * options and mallctl processing are straightforward.
23 */
24const char *percpu_arena_mode_names[] = {
25 "percpu",
26 "phycpu",
27 "disabled",
28 "percpu",
29 "phycpu"
30};
31percpu_arena_mode_t opt_percpu_arena = PERCPU_ARENA_DEFAULT;
32
33ssize_t opt_dirty_decay_ms = DIRTY_DECAY_MS_DEFAULT;
34ssize_t opt_muzzy_decay_ms = MUZZY_DECAY_MS_DEFAULT;
35
36static atomic_zd_t dirty_decay_ms_default;
37static atomic_zd_t muzzy_decay_ms_default;
38
39emap_t arena_emap_global;
40pa_central_t arena_pa_central_global;
41
42div_info_t arena_binind_div_info[SC_NBINS];
43
44size_t opt_oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT;
45size_t oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT;
46
47uint32_t arena_bin_offsets[SC_NBINS];
48static unsigned nbins_total;
49
50static unsigned huge_arena_ind;
51
52const arena_config_t arena_config_default = {
53 /* .extent_hooks = */ (extent_hooks_t *)&ehooks_default_extent_hooks,
54 /* .metadata_use_hooks = */ true,
55};
56
57/******************************************************************************/
58/*
59 * Function prototypes for static functions that are referenced prior to
60 * definition.
61 */
62
63static bool arena_decay_dirty(tsdn_t *tsdn, arena_t *arena,
64 bool is_background_thread, bool all);
65static void arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, edata_t *slab,
66 bin_t *bin);
67static void
68arena_maybe_do_deferred_work(tsdn_t *tsdn, arena_t *arena, decay_t *decay,
69 size_t npages_new);
70
71/******************************************************************************/
72
73void
74arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
75 const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
76 size_t *nactive, size_t *ndirty, size_t *nmuzzy) {
77 *nthreads += arena_nthreads_get(arena, false);
78 *dss = dss_prec_names[arena_dss_prec_get(arena)];
79 *dirty_decay_ms = arena_decay_ms_get(arena, extent_state_dirty);
80 *muzzy_decay_ms = arena_decay_ms_get(arena, extent_state_muzzy);
81 pa_shard_basic_stats_merge(&arena->pa_shard, nactive, ndirty, nmuzzy);
82}
83
84void
85arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
86 const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
87 size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
88 bin_stats_data_t *bstats, arena_stats_large_t *lstats,
89 pac_estats_t *estats, hpa_shard_stats_t *hpastats, sec_stats_t *secstats) {
90 cassert(config_stats);
91
92 arena_basic_stats_merge(tsdn, arena, nthreads, dss, dirty_decay_ms,
93 muzzy_decay_ms, nactive, ndirty, nmuzzy);
94
95 size_t base_allocated, base_resident, base_mapped, metadata_thp;
96 base_stats_get(tsdn, arena->base, &base_allocated, &base_resident,
97 &base_mapped, &metadata_thp);
98 size_t pac_mapped_sz = pac_mapped(&arena->pa_shard.pac);
99 astats->mapped += base_mapped + pac_mapped_sz;
100 astats->resident += base_resident;
101
102 LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx);
103
104 astats->base += base_allocated;
105 atomic_load_add_store_zu(&astats->internal, arena_internal_get(arena));
106 astats->metadata_thp += metadata_thp;
107
108 for (szind_t i = 0; i < SC_NSIZES - SC_NBINS; i++) {
109 uint64_t nmalloc = locked_read_u64(tsdn,
110 LOCKEDINT_MTX(arena->stats.mtx),
111 &arena->stats.lstats[i].nmalloc);
112 locked_inc_u64_unsynchronized(&lstats[i].nmalloc, nmalloc);
113 astats->nmalloc_large += nmalloc;
114
115 uint64_t ndalloc = locked_read_u64(tsdn,
116 LOCKEDINT_MTX(arena->stats.mtx),
117 &arena->stats.lstats[i].ndalloc);
118 locked_inc_u64_unsynchronized(&lstats[i].ndalloc, ndalloc);
119 astats->ndalloc_large += ndalloc;
120
121 uint64_t nrequests = locked_read_u64(tsdn,
122 LOCKEDINT_MTX(arena->stats.mtx),
123 &arena->stats.lstats[i].nrequests);
124 locked_inc_u64_unsynchronized(&lstats[i].nrequests,
125 nmalloc + nrequests);
126 astats->nrequests_large += nmalloc + nrequests;
127
128 /* nfill == nmalloc for large currently. */
129 locked_inc_u64_unsynchronized(&lstats[i].nfills, nmalloc);
130 astats->nfills_large += nmalloc;
131
132 uint64_t nflush = locked_read_u64(tsdn,
133 LOCKEDINT_MTX(arena->stats.mtx),
134 &arena->stats.lstats[i].nflushes);
135 locked_inc_u64_unsynchronized(&lstats[i].nflushes, nflush);
136 astats->nflushes_large += nflush;
137
138 assert(nmalloc >= ndalloc);
139 assert(nmalloc - ndalloc <= SIZE_T_MAX);
140 size_t curlextents = (size_t)(nmalloc - ndalloc);
141 lstats[i].curlextents += curlextents;
142 astats->allocated_large +=
143 curlextents * sz_index2size(SC_NBINS + i);
144 }
145
146 pa_shard_stats_merge(tsdn, &arena->pa_shard, &astats->pa_shard_stats,
147 estats, hpastats, secstats, &astats->resident);
148
149 LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx);
150
151 /* Currently cached bytes and sanitizer-stashed bytes in tcache. */
152 astats->tcache_bytes = 0;
153 astats->tcache_stashed_bytes = 0;
154 malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
155 cache_bin_array_descriptor_t *descriptor;
156 ql_foreach(descriptor, &arena->cache_bin_array_descriptor_ql, link) {
157 for (szind_t i = 0; i < nhbins; i++) {
158 cache_bin_t *cache_bin = &descriptor->bins[i];
159 cache_bin_sz_t ncached, nstashed;
160 cache_bin_nitems_get_remote(cache_bin,
161 &tcache_bin_info[i], &ncached, &nstashed);
162
163 astats->tcache_bytes += ncached * sz_index2size(i);
164 astats->tcache_stashed_bytes += nstashed *
165 sz_index2size(i);
166 }
167 }
168 malloc_mutex_prof_read(tsdn,
169 &astats->mutex_prof_data[arena_prof_mutex_tcache_list],
170 &arena->tcache_ql_mtx);
171 malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
172
173#define READ_ARENA_MUTEX_PROF_DATA(mtx, ind) \
174 malloc_mutex_lock(tsdn, &arena->mtx); \
175 malloc_mutex_prof_read(tsdn, &astats->mutex_prof_data[ind], \
176 &arena->mtx); \
177 malloc_mutex_unlock(tsdn, &arena->mtx);
178
179 /* Gather per arena mutex profiling data. */
180 READ_ARENA_MUTEX_PROF_DATA(large_mtx, arena_prof_mutex_large);
181 READ_ARENA_MUTEX_PROF_DATA(base->mtx,
182 arena_prof_mutex_base);
183#undef READ_ARENA_MUTEX_PROF_DATA
184 pa_shard_mtx_stats_read(tsdn, &arena->pa_shard,
185 astats->mutex_prof_data);
186
187 nstime_copy(&astats->uptime, &arena->create_time);
188 nstime_update(&astats->uptime);
189 nstime_subtract(&astats->uptime, &arena->create_time);
190
191 for (szind_t i = 0; i < SC_NBINS; i++) {
192 for (unsigned j = 0; j < bin_infos[i].n_shards; j++) {
193 bin_stats_merge(tsdn, &bstats[i],
194 arena_get_bin(arena, i, j));
195 }
196 }
197}
198
199static void
200arena_background_thread_inactivity_check(tsdn_t *tsdn, arena_t *arena,
201 bool is_background_thread) {
202 if (!background_thread_enabled() || is_background_thread) {
203 return;
204 }
205 background_thread_info_t *info =
206 arena_background_thread_info_get(arena);
207 if (background_thread_indefinite_sleep(info)) {
208 arena_maybe_do_deferred_work(tsdn, arena,
209 &arena->pa_shard.pac.decay_dirty, 0);
210 }
211}
212
213/*
214 * React to deferred work generated by a PAI function.
215 */
216void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena) {
217 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
218 WITNESS_RANK_CORE, 0);
219
220 if (decay_immediately(&arena->pa_shard.pac.decay_dirty)) {
221 arena_decay_dirty(tsdn, arena, false, true);
222 }
223 arena_background_thread_inactivity_check(tsdn, arena, false);
224}
225
226static void *
227arena_slab_reg_alloc(edata_t *slab, const bin_info_t *bin_info) {
228 void *ret;
229 slab_data_t *slab_data = edata_slab_data_get(slab);
230 size_t regind;
231
232 assert(edata_nfree_get(slab) > 0);
233 assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info));
234
235 regind = bitmap_sfu(slab_data->bitmap, &bin_info->bitmap_info);
236 ret = (void *)((uintptr_t)edata_addr_get(slab) +
237 (uintptr_t)(bin_info->reg_size * regind));
238 edata_nfree_dec(slab);
239 return ret;
240}
241
242static void
243arena_slab_reg_alloc_batch(edata_t *slab, const bin_info_t *bin_info,
244 unsigned cnt, void** ptrs) {
245 slab_data_t *slab_data = edata_slab_data_get(slab);
246
247 assert(edata_nfree_get(slab) >= cnt);
248 assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info));
249
250#if (! defined JEMALLOC_INTERNAL_POPCOUNTL) || (defined BITMAP_USE_TREE)
251 for (unsigned i = 0; i < cnt; i++) {
252 size_t regind = bitmap_sfu(slab_data->bitmap,
253 &bin_info->bitmap_info);
254 *(ptrs + i) = (void *)((uintptr_t)edata_addr_get(slab) +
255 (uintptr_t)(bin_info->reg_size * regind));
256 }
257#else
258 unsigned group = 0;
259 bitmap_t g = slab_data->bitmap[group];
260 unsigned i = 0;
261 while (i < cnt) {
262 while (g == 0) {
263 g = slab_data->bitmap[++group];
264 }
265 size_t shift = group << LG_BITMAP_GROUP_NBITS;
266 size_t pop = popcount_lu(g);
267 if (pop > (cnt - i)) {
268 pop = cnt - i;
269 }
270
271 /*
272 * Load from memory locations only once, outside the
273 * hot loop below.
274 */
275 uintptr_t base = (uintptr_t)edata_addr_get(slab);
276 uintptr_t regsize = (uintptr_t)bin_info->reg_size;
277 while (pop--) {
278 size_t bit = cfs_lu(&g);
279 size_t regind = shift + bit;
280 *(ptrs + i) = (void *)(base + regsize * regind);
281
282 i++;
283 }
284 slab_data->bitmap[group] = g;
285 }
286#endif
287 edata_nfree_sub(slab, cnt);
288}
289
290static void
291arena_large_malloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) {
292 szind_t index, hindex;
293
294 cassert(config_stats);
295
296 if (usize < SC_LARGE_MINCLASS) {
297 usize = SC_LARGE_MINCLASS;
298 }
299 index = sz_size2index(usize);
300 hindex = (index >= SC_NBINS) ? index - SC_NBINS : 0;
301
302 locked_inc_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx),
303 &arena->stats.lstats[hindex].nmalloc, 1);
304}
305
306static void
307arena_large_dalloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) {
308 szind_t index, hindex;
309
310 cassert(config_stats);
311
312 if (usize < SC_LARGE_MINCLASS) {
313 usize = SC_LARGE_MINCLASS;
314 }
315 index = sz_size2index(usize);
316 hindex = (index >= SC_NBINS) ? index - SC_NBINS : 0;
317
318 locked_inc_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx),
319 &arena->stats.lstats[hindex].ndalloc, 1);
320}
321
322static void
323arena_large_ralloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t oldusize,
324 size_t usize) {
325 arena_large_malloc_stats_update(tsdn, arena, usize);
326 arena_large_dalloc_stats_update(tsdn, arena, oldusize);
327}
328
329edata_t *
330arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
331 size_t alignment, bool zero) {
332 bool deferred_work_generated = false;
333 szind_t szind = sz_size2index(usize);
334 size_t esize = usize + sz_large_pad;
335
336 bool guarded = san_large_extent_decide_guard(tsdn,
337 arena_get_ehooks(arena), esize, alignment);
338 edata_t *edata = pa_alloc(tsdn, &arena->pa_shard, esize, alignment,
339 /* slab */ false, szind, zero, guarded, &deferred_work_generated);
340 assert(deferred_work_generated == false);
341
342 if (edata != NULL) {
343 if (config_stats) {
344 LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx);
345 arena_large_malloc_stats_update(tsdn, arena, usize);
346 LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx);
347 }
348 }
349
350 if (edata != NULL && sz_large_pad != 0) {
351 arena_cache_oblivious_randomize(tsdn, arena, edata, alignment);
352 }
353
354 return edata;
355}
356
357void
358arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
359 if (config_stats) {
360 LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx);
361 arena_large_dalloc_stats_update(tsdn, arena,
362 edata_usize_get(edata));
363 LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx);
364 }
365}
366
367void
368arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
369 size_t oldusize) {
370 size_t usize = edata_usize_get(edata);
371
372 if (config_stats) {
373 LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx);
374 arena_large_ralloc_stats_update(tsdn, arena, oldusize, usize);
375 LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx);
376 }
377}
378
379void
380arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
381 size_t oldusize) {
382 size_t usize = edata_usize_get(edata);
383
384 if (config_stats) {
385 LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx);
386 arena_large_ralloc_stats_update(tsdn, arena, oldusize, usize);
387 LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx);
388 }
389}
390
391/*
392 * In situations where we're not forcing a decay (i.e. because the user
393 * specifically requested it), should we purge ourselves, or wait for the
394 * background thread to get to it.
395 */
396static pac_purge_eagerness_t
397arena_decide_unforced_purge_eagerness(bool is_background_thread) {
398 if (is_background_thread) {
399 return PAC_PURGE_ALWAYS;
400 } else if (!is_background_thread && background_thread_enabled()) {
401 return PAC_PURGE_NEVER;
402 } else {
403 return PAC_PURGE_ON_EPOCH_ADVANCE;
404 }
405}
406
407bool
408arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, extent_state_t state,
409 ssize_t decay_ms) {
410 pac_purge_eagerness_t eagerness = arena_decide_unforced_purge_eagerness(
411 /* is_background_thread */ false);
412 return pa_decay_ms_set(tsdn, &arena->pa_shard, state, decay_ms,
413 eagerness);
414}
415
416ssize_t
417arena_decay_ms_get(arena_t *arena, extent_state_t state) {
418 return pa_decay_ms_get(&arena->pa_shard, state);
419}
420
421static bool
422arena_decay_impl(tsdn_t *tsdn, arena_t *arena, decay_t *decay,
423 pac_decay_stats_t *decay_stats, ecache_t *ecache,
424 bool is_background_thread, bool all) {
425 if (all) {
426 malloc_mutex_lock(tsdn, &decay->mtx);
427 pac_decay_all(tsdn, &arena->pa_shard.pac, decay, decay_stats,
428 ecache, /* fully_decay */ all);
429 malloc_mutex_unlock(tsdn, &decay->mtx);
430 return false;
431 }
432
433 if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
434 /* No need to wait if another thread is in progress. */
435 return true;
436 }
437 pac_purge_eagerness_t eagerness =
438 arena_decide_unforced_purge_eagerness(is_background_thread);
439 bool epoch_advanced = pac_maybe_decay_purge(tsdn, &arena->pa_shard.pac,
440 decay, decay_stats, ecache, eagerness);
441 size_t npages_new;
442 if (epoch_advanced) {
443 /* Backlog is updated on epoch advance. */
444 npages_new = decay_epoch_npages_delta(decay);
445 }
446 malloc_mutex_unlock(tsdn, &decay->mtx);
447
448 if (have_background_thread && background_thread_enabled() &&
449 epoch_advanced && !is_background_thread) {
450 arena_maybe_do_deferred_work(tsdn, arena, decay, npages_new);
451 }
452
453 return false;
454}
455
456static bool
457arena_decay_dirty(tsdn_t *tsdn, arena_t *arena, bool is_background_thread,
458 bool all) {
459 return arena_decay_impl(tsdn, arena, &arena->pa_shard.pac.decay_dirty,
460 &arena->pa_shard.pac.stats->decay_dirty,
461 &arena->pa_shard.pac.ecache_dirty, is_background_thread, all);
462}
463
464static bool
465arena_decay_muzzy(tsdn_t *tsdn, arena_t *arena, bool is_background_thread,
466 bool all) {
467 if (pa_shard_dont_decay_muzzy(&arena->pa_shard)) {
468 return false;
469 }
470 return arena_decay_impl(tsdn, arena, &arena->pa_shard.pac.decay_muzzy,
471 &arena->pa_shard.pac.stats->decay_muzzy,
472 &arena->pa_shard.pac.ecache_muzzy, is_background_thread, all);
473}
474
475void
476arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) {
477 if (all) {
478 /*
479 * We should take a purge of "all" to mean "save as much memory
480 * as possible", including flushing any caches (for situations
481 * like thread death, or manual purge calls).
482 */
483 sec_flush(tsdn, &arena->pa_shard.hpa_sec);
484 }
485 if (arena_decay_dirty(tsdn, arena, is_background_thread, all)) {
486 return;
487 }
488 arena_decay_muzzy(tsdn, arena, is_background_thread, all);
489}
490
491static bool
492arena_should_decay_early(tsdn_t *tsdn, arena_t *arena, decay_t *decay,
493 background_thread_info_t *info, nstime_t *remaining_sleep,
494 size_t npages_new) {
495 malloc_mutex_assert_owner(tsdn, &info->mtx);
496
497 if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
498 return false;
499 }
500
501 if (!decay_gradually(decay)) {
502 malloc_mutex_unlock(tsdn, &decay->mtx);
503 return false;
504 }
505
506 nstime_init(remaining_sleep, background_thread_wakeup_time_get(info));
507 if (nstime_compare(remaining_sleep, &decay->epoch) <= 0) {
508 malloc_mutex_unlock(tsdn, &decay->mtx);
509 return false;
510 }
511 nstime_subtract(remaining_sleep, &decay->epoch);
512 if (npages_new > 0) {
513 uint64_t npurge_new = decay_npages_purge_in(decay,
514 remaining_sleep, npages_new);
515 info->npages_to_purge_new += npurge_new;
516 }
517 malloc_mutex_unlock(tsdn, &decay->mtx);
518 return info->npages_to_purge_new >
519 ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD;
520}
521
522/*
523 * Check if deferred work needs to be done sooner than planned.
524 * For decay we might want to wake up earlier because of an influx of dirty
525 * pages. Rather than waiting for previously estimated time, we proactively
526 * purge those pages.
527 * If background thread sleeps indefinitely, always wake up because some
528 * deferred work has been generated.
529 */
530static void
531arena_maybe_do_deferred_work(tsdn_t *tsdn, arena_t *arena, decay_t *decay,
532 size_t npages_new) {
533 background_thread_info_t *info = arena_background_thread_info_get(
534 arena);
535 if (malloc_mutex_trylock(tsdn, &info->mtx)) {
536 /*
537 * Background thread may hold the mutex for a long period of
538 * time. We'd like to avoid the variance on application
539 * threads. So keep this non-blocking, and leave the work to a
540 * future epoch.
541 */
542 return;
543 }
544 if (!background_thread_is_started(info)) {
545 goto label_done;
546 }
547
548 nstime_t remaining_sleep;
549 if (background_thread_indefinite_sleep(info)) {
550 background_thread_wakeup_early(info, NULL);
551 } else if (arena_should_decay_early(tsdn, arena, decay, info,
552 &remaining_sleep, npages_new)) {
553 info->npages_to_purge_new = 0;
554 background_thread_wakeup_early(info, &remaining_sleep);
555 }
556label_done:
557 malloc_mutex_unlock(tsdn, &info->mtx);
558}
559
560/* Called from background threads. */
561void
562arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena) {
563 arena_decay(tsdn, arena, true, false);
564 pa_shard_do_deferred_work(tsdn, &arena->pa_shard);
565}
566
567void
568arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab) {
569 bool deferred_work_generated = false;
570 pa_dalloc(tsdn, &arena->pa_shard, slab, &deferred_work_generated);
571 if (deferred_work_generated) {
572 arena_handle_deferred_work(tsdn, arena);
573 }
574}
575
576static void
577arena_bin_slabs_nonfull_insert(bin_t *bin, edata_t *slab) {
578 assert(edata_nfree_get(slab) > 0);
579 edata_heap_insert(&bin->slabs_nonfull, slab);
580 if (config_stats) {
581 bin->stats.nonfull_slabs++;
582 }
583}
584
585static void
586arena_bin_slabs_nonfull_remove(bin_t *bin, edata_t *slab) {
587 edata_heap_remove(&bin->slabs_nonfull, slab);
588 if (config_stats) {
589 bin->stats.nonfull_slabs--;
590 }
591}
592
593static edata_t *
594arena_bin_slabs_nonfull_tryget(bin_t *bin) {
595 edata_t *slab = edata_heap_remove_first(&bin->slabs_nonfull);
596 if (slab == NULL) {
597 return NULL;
598 }
599 if (config_stats) {
600 bin->stats.reslabs++;
601 bin->stats.nonfull_slabs--;
602 }
603 return slab;
604}
605
606static void
607arena_bin_slabs_full_insert(arena_t *arena, bin_t *bin, edata_t *slab) {
608 assert(edata_nfree_get(slab) == 0);
609 /*
610 * Tracking extents is required by arena_reset, which is not allowed
611 * for auto arenas. Bypass this step to avoid touching the edata
612 * linkage (often results in cache misses) for auto arenas.
613 */
614 if (arena_is_auto(arena)) {
615 return;
616 }
617 edata_list_active_append(&bin->slabs_full, slab);
618}
619
620static void
621arena_bin_slabs_full_remove(arena_t *arena, bin_t *bin, edata_t *slab) {
622 if (arena_is_auto(arena)) {
623 return;
624 }
625 edata_list_active_remove(&bin->slabs_full, slab);
626}
627
628static void
629arena_bin_reset(tsd_t *tsd, arena_t *arena, bin_t *bin) {
630 edata_t *slab;
631
632 malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
633 if (bin->slabcur != NULL) {
634 slab = bin->slabcur;
635 bin->slabcur = NULL;
636 malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
637 arena_slab_dalloc(tsd_tsdn(tsd), arena, slab);
638 malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
639 }
640 while ((slab = edata_heap_remove_first(&bin->slabs_nonfull)) != NULL) {
641 malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
642 arena_slab_dalloc(tsd_tsdn(tsd), arena, slab);
643 malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
644 }
645 for (slab = edata_list_active_first(&bin->slabs_full); slab != NULL;
646 slab = edata_list_active_first(&bin->slabs_full)) {
647 arena_bin_slabs_full_remove(arena, bin, slab);
648 malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
649 arena_slab_dalloc(tsd_tsdn(tsd), arena, slab);
650 malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
651 }
652 if (config_stats) {
653 bin->stats.curregs = 0;
654 bin->stats.curslabs = 0;
655 }
656 malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
657}
658
659void
660arena_reset(tsd_t *tsd, arena_t *arena) {
661 /*
662 * Locking in this function is unintuitive. The caller guarantees that
663 * no concurrent operations are happening in this arena, but there are
664 * still reasons that some locking is necessary:
665 *
666 * - Some of the functions in the transitive closure of calls assume
667 * appropriate locks are held, and in some cases these locks are
668 * temporarily dropped to avoid lock order reversal or deadlock due to
669 * reentry.
670 * - mallctl("epoch", ...) may concurrently refresh stats. While
671 * strictly speaking this is a "concurrent operation", disallowing
672 * stats refreshes would impose an inconvenient burden.
673 */
674
675 /* Large allocations. */
676 malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx);
677
678 for (edata_t *edata = edata_list_active_first(&arena->large);
679 edata != NULL; edata = edata_list_active_first(&arena->large)) {
680 void *ptr = edata_base_get(edata);
681 size_t usize;
682
683 malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx);
684 emap_alloc_ctx_t alloc_ctx;
685 emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr,
686 &alloc_ctx);
687 assert(alloc_ctx.szind != SC_NSIZES);
688
689 if (config_stats || (config_prof && opt_prof)) {
690 usize = sz_index2size(alloc_ctx.szind);
691 assert(usize == isalloc(tsd_tsdn(tsd), ptr));
692 }
693 /* Remove large allocation from prof sample set. */
694 if (config_prof && opt_prof) {
695 prof_free(tsd, ptr, usize, &alloc_ctx);
696 }
697 large_dalloc(tsd_tsdn(tsd), edata);
698 malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx);
699 }
700 malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx);
701
702 /* Bins. */
703 for (unsigned i = 0; i < SC_NBINS; i++) {
704 for (unsigned j = 0; j < bin_infos[i].n_shards; j++) {
705 arena_bin_reset(tsd, arena, arena_get_bin(arena, i, j));
706 }
707 }
708 pa_shard_reset(tsd_tsdn(tsd), &arena->pa_shard);
709}
710
711static void
712arena_prepare_base_deletion_sync_finish(tsd_t *tsd, malloc_mutex_t **mutexes,
713 unsigned n_mtx) {
714 for (unsigned i = 0; i < n_mtx; i++) {
715 malloc_mutex_lock(tsd_tsdn(tsd), mutexes[i]);
716 malloc_mutex_unlock(tsd_tsdn(tsd), mutexes[i]);
717 }
718}
719
720#define ARENA_DESTROY_MAX_DELAYED_MTX 32
721static void
722arena_prepare_base_deletion_sync(tsd_t *tsd, malloc_mutex_t *mtx,
723 malloc_mutex_t **delayed_mtx, unsigned *n_delayed) {
724 if (!malloc_mutex_trylock(tsd_tsdn(tsd), mtx)) {
725 /* No contention. */
726 malloc_mutex_unlock(tsd_tsdn(tsd), mtx);
727 return;
728 }
729 unsigned n = *n_delayed;
730 assert(n < ARENA_DESTROY_MAX_DELAYED_MTX);
731 /* Add another to the batch. */
732 delayed_mtx[n++] = mtx;
733
734 if (n == ARENA_DESTROY_MAX_DELAYED_MTX) {
735 arena_prepare_base_deletion_sync_finish(tsd, delayed_mtx, n);
736 n = 0;
737 }
738 *n_delayed = n;
739}
740
741static void
742arena_prepare_base_deletion(tsd_t *tsd, base_t *base_to_destroy) {
743 /*
744 * In order to coalesce, emap_try_acquire_edata_neighbor will attempt to
745 * check neighbor edata's state to determine eligibility. This means
746 * under certain conditions, the metadata from an arena can be accessed
747 * w/o holding any locks from that arena. In order to guarantee safe
748 * memory access, the metadata and the underlying base allocator needs
749 * to be kept alive, until all pending accesses are done.
750 *
751 * 1) with opt_retain, the arena boundary implies the is_head state
752 * (tracked in the rtree leaf), and the coalesce flow will stop at the
753 * head state branch. Therefore no cross arena metadata access
754 * possible.
755 *
756 * 2) w/o opt_retain, the arena id needs to be read from the edata_t,
757 * meaning read only cross-arena metadata access is possible. The
758 * coalesce attempt will stop at the arena_id mismatch, and is always
759 * under one of the ecache locks. To allow safe passthrough of such
760 * metadata accesses, the loop below will iterate through all manual
761 * arenas' ecache locks. As all the metadata from this base allocator
762 * have been unlinked from the rtree, after going through all the
763 * relevant ecache locks, it's safe to say that a) pending accesses are
764 * all finished, and b) no new access will be generated.
765 */
766 if (opt_retain) {
767 return;
768 }
769 unsigned destroy_ind = base_ind_get(base_to_destroy);
770 assert(destroy_ind >= manual_arena_base);
771
772 tsdn_t *tsdn = tsd_tsdn(tsd);
773 malloc_mutex_t *delayed_mtx[ARENA_DESTROY_MAX_DELAYED_MTX];
774 unsigned n_delayed = 0, total = narenas_total_get();
775 for (unsigned i = 0; i < total; i++) {
776 if (i == destroy_ind) {
777 continue;
778 }
779 arena_t *arena = arena_get(tsdn, i, false);
780 if (arena == NULL) {
781 continue;
782 }
783 pac_t *pac = &arena->pa_shard.pac;
784 arena_prepare_base_deletion_sync(tsd, &pac->ecache_dirty.mtx,
785 delayed_mtx, &n_delayed);
786 arena_prepare_base_deletion_sync(tsd, &pac->ecache_muzzy.mtx,
787 delayed_mtx, &n_delayed);
788 arena_prepare_base_deletion_sync(tsd, &pac->ecache_retained.mtx,
789 delayed_mtx, &n_delayed);
790 }
791 arena_prepare_base_deletion_sync_finish(tsd, delayed_mtx, n_delayed);
792}
793#undef ARENA_DESTROY_MAX_DELAYED_MTX
794
795void
796arena_destroy(tsd_t *tsd, arena_t *arena) {
797 assert(base_ind_get(arena->base) >= narenas_auto);
798 assert(arena_nthreads_get(arena, false) == 0);
799 assert(arena_nthreads_get(arena, true) == 0);
800
801 /*
802 * No allocations have occurred since arena_reset() was called.
803 * Furthermore, the caller (arena_i_destroy_ctl()) purged all cached
804 * extents, so only retained extents may remain and it's safe to call
805 * pa_shard_destroy_retained.
806 */
807 pa_shard_destroy(tsd_tsdn(tsd), &arena->pa_shard);
808
809 /*
810 * Remove the arena pointer from the arenas array. We rely on the fact
811 * that there is no way for the application to get a dirty read from the
812 * arenas array unless there is an inherent race in the application
813 * involving access of an arena being concurrently destroyed. The
814 * application must synchronize knowledge of the arena's validity, so as
815 * long as we use an atomic write to update the arenas array, the
816 * application will get a clean read any time after it synchronizes
817 * knowledge that the arena is no longer valid.
818 */
819 arena_set(base_ind_get(arena->base), NULL);
820
821 /*
822 * Destroy the base allocator, which manages all metadata ever mapped by
823 * this arena. The prepare function will make sure no pending access to
824 * the metadata in this base anymore.
825 */
826 arena_prepare_base_deletion(tsd, arena->base);
827 base_delete(tsd_tsdn(tsd), arena->base);
828}
829
830static edata_t *
831arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard,
832 const bin_info_t *bin_info) {
833 bool deferred_work_generated = false;
834 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
835 WITNESS_RANK_CORE, 0);
836
837 bool guarded = san_slab_extent_decide_guard(tsdn,
838 arena_get_ehooks(arena));
839 edata_t *slab = pa_alloc(tsdn, &arena->pa_shard, bin_info->slab_size,
840 /* alignment */ PAGE, /* slab */ true, /* szind */ binind,
841 /* zero */ false, guarded, &deferred_work_generated);
842
843 if (deferred_work_generated) {
844 arena_handle_deferred_work(tsdn, arena);
845 }
846
847 if (slab == NULL) {
848 return NULL;
849 }
850 assert(edata_slab_get(slab));
851
852 /* Initialize slab internals. */
853 slab_data_t *slab_data = edata_slab_data_get(slab);
854 edata_nfree_binshard_set(slab, bin_info->nregs, binshard);
855 bitmap_init(slab_data->bitmap, &bin_info->bitmap_info, false);
856
857 return slab;
858}
859
860/*
861 * Before attempting the _with_fresh_slab approaches below, the _no_fresh_slab
862 * variants (i.e. through slabcur and nonfull) must be tried first.
863 */
864static void
865arena_bin_refill_slabcur_with_fresh_slab(tsdn_t *tsdn, arena_t *arena,
866 bin_t *bin, szind_t binind, edata_t *fresh_slab) {
867 malloc_mutex_assert_owner(tsdn, &bin->lock);
868 /* Only called after slabcur and nonfull both failed. */
869 assert(bin->slabcur == NULL);
870 assert(edata_heap_first(&bin->slabs_nonfull) == NULL);
871 assert(fresh_slab != NULL);
872
873 /* A new slab from arena_slab_alloc() */
874 assert(edata_nfree_get(fresh_slab) == bin_infos[binind].nregs);
875 if (config_stats) {
876 bin->stats.nslabs++;
877 bin->stats.curslabs++;
878 }
879 bin->slabcur = fresh_slab;
880}
881
882/* Refill slabcur and then alloc using the fresh slab */
883static void *
884arena_bin_malloc_with_fresh_slab(tsdn_t *tsdn, arena_t *arena, bin_t *bin,
885 szind_t binind, edata_t *fresh_slab) {
886 malloc_mutex_assert_owner(tsdn, &bin->lock);
887 arena_bin_refill_slabcur_with_fresh_slab(tsdn, arena, bin, binind,
888 fresh_slab);
889
890 return arena_slab_reg_alloc(bin->slabcur, &bin_infos[binind]);
891}
892
893static bool
894arena_bin_refill_slabcur_no_fresh_slab(tsdn_t *tsdn, arena_t *arena,
895 bin_t *bin) {
896 malloc_mutex_assert_owner(tsdn, &bin->lock);
897 /* Only called after arena_slab_reg_alloc[_batch] failed. */
898 assert(bin->slabcur == NULL || edata_nfree_get(bin->slabcur) == 0);
899
900 if (bin->slabcur != NULL) {
901 arena_bin_slabs_full_insert(arena, bin, bin->slabcur);
902 }
903
904 /* Look for a usable slab. */
905 bin->slabcur = arena_bin_slabs_nonfull_tryget(bin);
906 assert(bin->slabcur == NULL || edata_nfree_get(bin->slabcur) > 0);
907
908 return (bin->slabcur == NULL);
909}
910
911bin_t *
912arena_bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind,
913 unsigned *binshard_p) {
914 unsigned binshard;
915 if (tsdn_null(tsdn) || tsd_arena_get(tsdn_tsd(tsdn)) == NULL) {
916 binshard = 0;
917 } else {
918 binshard = tsd_binshardsp_get(tsdn_tsd(tsdn))->binshard[binind];
919 }
920 assert(binshard < bin_infos[binind].n_shards);
921 if (binshard_p != NULL) {
922 *binshard_p = binshard;
923 }
924 return arena_get_bin(arena, binind, binshard);
925}
926
927void
928arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena,
929 cache_bin_t *cache_bin, cache_bin_info_t *cache_bin_info, szind_t binind,
930 const unsigned nfill) {
931 assert(cache_bin_ncached_get_local(cache_bin, cache_bin_info) == 0);
932
933 const bin_info_t *bin_info = &bin_infos[binind];
934
935 CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nfill);
936 cache_bin_init_ptr_array_for_fill(cache_bin, cache_bin_info, &ptrs,
937 nfill);
938 /*
939 * Bin-local resources are used first: 1) bin->slabcur, and 2) nonfull
940 * slabs. After both are exhausted, new slabs will be allocated through
941 * arena_slab_alloc().
942 *
943 * Bin lock is only taken / released right before / after the while(...)
944 * refill loop, with new slab allocation (which has its own locking)
945 * kept outside of the loop. This setup facilitates flat combining, at
946 * the cost of the nested loop (through goto label_refill).
947 *
948 * To optimize for cases with contention and limited resources
949 * (e.g. hugepage-backed or non-overcommit arenas), each fill-iteration
950 * gets one chance of slab_alloc, and a retry of bin local resources
951 * after the slab allocation (regardless if slab_alloc failed, because
952 * the bin lock is dropped during the slab allocation).
953 *
954 * In other words, new slab allocation is allowed, as long as there was
955 * progress since the previous slab_alloc. This is tracked with
956 * made_progress below, initialized to true to jump start the first
957 * iteration.
958 *
959 * In other words (again), the loop will only terminate early (i.e. stop
960 * with filled < nfill) after going through the three steps: a) bin
961 * local exhausted, b) unlock and slab_alloc returns null, c) re-lock
962 * and bin local fails again.
963 */
964 bool made_progress = true;
965 edata_t *fresh_slab = NULL;
966 bool alloc_and_retry = false;
967 unsigned filled = 0;
968 unsigned binshard;
969 bin_t *bin = arena_bin_choose(tsdn, arena, binind, &binshard);
970
971label_refill:
972 malloc_mutex_lock(tsdn, &bin->lock);
973
974 while (filled < nfill) {
975 /* Try batch-fill from slabcur first. */
976 edata_t *slabcur = bin->slabcur;
977 if (slabcur != NULL && edata_nfree_get(slabcur) > 0) {
978 unsigned tofill = nfill - filled;
979 unsigned nfree = edata_nfree_get(slabcur);
980 unsigned cnt = tofill < nfree ? tofill : nfree;
981
982 arena_slab_reg_alloc_batch(slabcur, bin_info, cnt,
983 &ptrs.ptr[filled]);
984 made_progress = true;
985 filled += cnt;
986 continue;
987 }
988 /* Next try refilling slabcur from nonfull slabs. */
989 if (!arena_bin_refill_slabcur_no_fresh_slab(tsdn, arena, bin)) {
990 assert(bin->slabcur != NULL);
991 continue;
992 }
993
994 /* Then see if a new slab was reserved already. */
995 if (fresh_slab != NULL) {
996 arena_bin_refill_slabcur_with_fresh_slab(tsdn, arena,
997 bin, binind, fresh_slab);
998 assert(bin->slabcur != NULL);
999 fresh_slab = NULL;
1000 continue;
1001 }
1002
1003 /* Try slab_alloc if made progress (or never did slab_alloc). */
1004 if (made_progress) {
1005 assert(bin->slabcur == NULL);
1006 assert(fresh_slab == NULL);
1007 alloc_and_retry = true;
1008 /* Alloc a new slab then come back. */
1009 break;
1010 }
1011
1012 /* OOM. */
1013
1014 assert(fresh_slab == NULL);
1015 assert(!alloc_and_retry);
1016 break;
1017 } /* while (filled < nfill) loop. */
1018
1019 if (config_stats && !alloc_and_retry) {
1020 bin->stats.nmalloc += filled;
1021 bin->stats.nrequests += cache_bin->tstats.nrequests;
1022 bin->stats.curregs += filled;
1023 bin->stats.nfills++;
1024 cache_bin->tstats.nrequests = 0;
1025 }
1026
1027 malloc_mutex_unlock(tsdn, &bin->lock);
1028
1029 if (alloc_and_retry) {
1030 assert(fresh_slab == NULL);
1031 assert(filled < nfill);
1032 assert(made_progress);
1033
1034 fresh_slab = arena_slab_alloc(tsdn, arena, binind, binshard,
1035 bin_info);
1036 /* fresh_slab NULL case handled in the for loop. */
1037
1038 alloc_and_retry = false;
1039 made_progress = false;
1040 goto label_refill;
1041 }
1042 assert(filled == nfill || (fresh_slab == NULL && !made_progress));
1043
1044 /* Release if allocated but not used. */
1045 if (fresh_slab != NULL) {
1046 assert(edata_nfree_get(fresh_slab) == bin_info->nregs);
1047 arena_slab_dalloc(tsdn, arena, fresh_slab);
1048 fresh_slab = NULL;
1049 }
1050
1051 cache_bin_finish_fill(cache_bin, cache_bin_info, &ptrs, filled);
1052 arena_decay_tick(tsdn, arena);
1053}
1054
1055size_t
1056arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind,
1057 void **ptrs, size_t nfill, bool zero) {
1058 assert(binind < SC_NBINS);
1059 const bin_info_t *bin_info = &bin_infos[binind];
1060 const size_t nregs = bin_info->nregs;
1061 assert(nregs > 0);
1062 const size_t usize = bin_info->reg_size;
1063
1064 const bool manual_arena = !arena_is_auto(arena);
1065 unsigned binshard;
1066 bin_t *bin = arena_bin_choose(tsdn, arena, binind, &binshard);
1067
1068 size_t nslab = 0;
1069 size_t filled = 0;
1070 edata_t *slab = NULL;
1071 edata_list_active_t fulls;
1072 edata_list_active_init(&fulls);
1073
1074 while (filled < nfill && (slab = arena_slab_alloc(tsdn, arena, binind,
1075 binshard, bin_info)) != NULL) {
1076 assert((size_t)edata_nfree_get(slab) == nregs);
1077 ++nslab;
1078 size_t batch = nfill - filled;
1079 if (batch > nregs) {
1080 batch = nregs;
1081 }
1082 assert(batch > 0);
1083 arena_slab_reg_alloc_batch(slab, bin_info, (unsigned)batch,
1084 &ptrs[filled]);
1085 assert(edata_addr_get(slab) == ptrs[filled]);
1086 if (zero) {
1087 memset(ptrs[filled], 0, batch * usize);
1088 }
1089 filled += batch;
1090 if (batch == nregs) {
1091 if (manual_arena) {
1092 edata_list_active_append(&fulls, slab);
1093 }
1094 slab = NULL;
1095 }
1096 }
1097
1098 malloc_mutex_lock(tsdn, &bin->lock);
1099 /*
1100 * Only the last slab can be non-empty, and the last slab is non-empty
1101 * iff slab != NULL.
1102 */
1103 if (slab != NULL) {
1104 arena_bin_lower_slab(tsdn, arena, slab, bin);
1105 }
1106 if (manual_arena) {
1107 edata_list_active_concat(&bin->slabs_full, &fulls);
1108 }
1109 assert(edata_list_active_empty(&fulls));
1110 if (config_stats) {
1111 bin->stats.nslabs += nslab;
1112 bin->stats.curslabs += nslab;
1113 bin->stats.nmalloc += filled;
1114 bin->stats.nrequests += filled;
1115 bin->stats.curregs += filled;
1116 }
1117 malloc_mutex_unlock(tsdn, &bin->lock);
1118
1119 arena_decay_tick(tsdn, arena);
1120 return filled;
1121}
1122
1123/*
1124 * Without allocating a new slab, try arena_slab_reg_alloc() and re-fill
1125 * bin->slabcur if necessary.
1126 */
1127static void *
1128arena_bin_malloc_no_fresh_slab(tsdn_t *tsdn, arena_t *arena, bin_t *bin,
1129 szind_t binind) {
1130 malloc_mutex_assert_owner(tsdn, &bin->lock);
1131 if (bin->slabcur == NULL || edata_nfree_get(bin->slabcur) == 0) {
1132 if (arena_bin_refill_slabcur_no_fresh_slab(tsdn, arena, bin)) {
1133 return NULL;
1134 }
1135 }
1136
1137 assert(bin->slabcur != NULL && edata_nfree_get(bin->slabcur) > 0);
1138 return arena_slab_reg_alloc(bin->slabcur, &bin_infos[binind]);
1139}
1140
1141static void *
1142arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) {
1143 assert(binind < SC_NBINS);
1144 const bin_info_t *bin_info = &bin_infos[binind];
1145 size_t usize = sz_index2size(binind);
1146 unsigned binshard;
1147 bin_t *bin = arena_bin_choose(tsdn, arena, binind, &binshard);
1148
1149 malloc_mutex_lock(tsdn, &bin->lock);
1150 edata_t *fresh_slab = NULL;
1151 void *ret = arena_bin_malloc_no_fresh_slab(tsdn, arena, bin, binind);
1152 if (ret == NULL) {
1153 malloc_mutex_unlock(tsdn, &bin->lock);
1154 /******************************/
1155 fresh_slab = arena_slab_alloc(tsdn, arena, binind, binshard,
1156 bin_info);
1157 /********************************/
1158 malloc_mutex_lock(tsdn, &bin->lock);
1159 /* Retry since the lock was dropped. */
1160 ret = arena_bin_malloc_no_fresh_slab(tsdn, arena, bin, binind);
1161 if (ret == NULL) {
1162 if (fresh_slab == NULL) {
1163 /* OOM */
1164 malloc_mutex_unlock(tsdn, &bin->lock);
1165 return NULL;
1166 }
1167 ret = arena_bin_malloc_with_fresh_slab(tsdn, arena, bin,
1168 binind, fresh_slab);
1169 fresh_slab = NULL;
1170 }
1171 }
1172 if (config_stats) {
1173 bin->stats.nmalloc++;
1174 bin->stats.nrequests++;
1175 bin->stats.curregs++;
1176 }
1177 malloc_mutex_unlock(tsdn, &bin->lock);
1178
1179 if (fresh_slab != NULL) {
1180 arena_slab_dalloc(tsdn, arena, fresh_slab);
1181 }
1182 if (zero) {
1183 memset(ret, 0, usize);
1184 }
1185 arena_decay_tick(tsdn, arena);
1186
1187 return ret;
1188}
1189
1190void *
1191arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
1192 bool zero) {
1193 assert(!tsdn_null(tsdn) || arena != NULL);
1194
1195 if (likely(!tsdn_null(tsdn))) {
1196 arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, size);
1197 }
1198 if (unlikely(arena == NULL)) {
1199 return NULL;
1200 }
1201
1202 if (likely(size <= SC_SMALL_MAXCLASS)) {
1203 return arena_malloc_small(tsdn, arena, ind, zero);
1204 }
1205 return large_malloc(tsdn, arena, sz_index2size(ind), zero);
1206}
1207
1208void *
1209arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
1210 bool zero, tcache_t *tcache) {
1211 void *ret;
1212
1213 if (usize <= SC_SMALL_MAXCLASS) {
1214 /* Small; alignment doesn't require special slab placement. */
1215
1216 /* usize should be a result of sz_sa2u() */
1217 assert((usize & (alignment - 1)) == 0);
1218
1219 /*
1220 * Small usize can't come from an alignment larger than a page.
1221 */
1222 assert(alignment <= PAGE);
1223
1224 ret = arena_malloc(tsdn, arena, usize, sz_size2index(usize),
1225 zero, tcache, true);
1226 } else {
1227 if (likely(alignment <= CACHELINE)) {
1228 ret = large_malloc(tsdn, arena, usize, zero);
1229 } else {
1230 ret = large_palloc(tsdn, arena, usize, alignment, zero);
1231 }
1232 }
1233 return ret;
1234}
1235
1236void
1237arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize) {
1238 cassert(config_prof);
1239 assert(ptr != NULL);
1240 assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS);
1241 assert(usize <= SC_SMALL_MAXCLASS);
1242
1243 if (config_opt_safety_checks) {
1244 safety_check_set_redzone(ptr, usize, SC_LARGE_MINCLASS);
1245 }
1246
1247 edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
1248
1249 szind_t szind = sz_size2index(usize);
1250 edata_szind_set(edata, szind);
1251 emap_remap(tsdn, &arena_emap_global, edata, szind, /* slab */ false);
1252
1253 assert(isalloc(tsdn, ptr) == usize);
1254}
1255
1256static size_t
1257arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr) {
1258 cassert(config_prof);
1259 assert(ptr != NULL);
1260
1261 edata_szind_set(edata, SC_NBINS);
1262 emap_remap(tsdn, &arena_emap_global, edata, SC_NBINS, /* slab */ false);
1263
1264 assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS);
1265
1266 return SC_LARGE_MINCLASS;
1267}
1268
1269void
1270arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
1271 bool slow_path) {
1272 cassert(config_prof);
1273 assert(opt_prof);
1274
1275 edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
1276 size_t usize = edata_usize_get(edata);
1277 size_t bumped_usize = arena_prof_demote(tsdn, edata, ptr);
1278 if (config_opt_safety_checks && usize < SC_LARGE_MINCLASS) {
1279 /*
1280 * Currently, we only do redzoning for small sampled
1281 * allocations.
1282 */
1283 assert(bumped_usize == SC_LARGE_MINCLASS);
1284 safety_check_verify_redzone(ptr, usize, bumped_usize);
1285 }
1286 if (bumped_usize <= tcache_maxclass && tcache != NULL) {
1287 tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr,
1288 sz_size2index(bumped_usize), slow_path);
1289 } else {
1290 large_dalloc(tsdn, edata);
1291 }
1292}
1293
1294static void
1295arena_dissociate_bin_slab(arena_t *arena, edata_t *slab, bin_t *bin) {
1296 /* Dissociate slab from bin. */
1297 if (slab == bin->slabcur) {
1298 bin->slabcur = NULL;
1299 } else {
1300 szind_t binind = edata_szind_get(slab);
1301 const bin_info_t *bin_info = &bin_infos[binind];
1302
1303 /*
1304 * The following block's conditional is necessary because if the
1305 * slab only contains one region, then it never gets inserted
1306 * into the non-full slabs heap.
1307 */
1308 if (bin_info->nregs == 1) {
1309 arena_bin_slabs_full_remove(arena, bin, slab);
1310 } else {
1311 arena_bin_slabs_nonfull_remove(bin, slab);
1312 }
1313 }
1314}
1315
1316static void
1317arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, edata_t *slab,
1318 bin_t *bin) {
1319 assert(edata_nfree_get(slab) > 0);
1320
1321 /*
1322 * Make sure that if bin->slabcur is non-NULL, it refers to the
1323 * oldest/lowest non-full slab. It is okay to NULL slabcur out rather
1324 * than proactively keeping it pointing at the oldest/lowest non-full
1325 * slab.
1326 */
1327 if (bin->slabcur != NULL && edata_snad_comp(bin->slabcur, slab) > 0) {
1328 /* Switch slabcur. */
1329 if (edata_nfree_get(bin->slabcur) > 0) {
1330 arena_bin_slabs_nonfull_insert(bin, bin->slabcur);
1331 } else {
1332 arena_bin_slabs_full_insert(arena, bin, bin->slabcur);
1333 }
1334 bin->slabcur = slab;
1335 if (config_stats) {
1336 bin->stats.reslabs++;
1337 }
1338 } else {
1339 arena_bin_slabs_nonfull_insert(bin, slab);
1340 }
1341}
1342
1343static void
1344arena_dalloc_bin_slab_prepare(tsdn_t *tsdn, edata_t *slab, bin_t *bin) {
1345 malloc_mutex_assert_owner(tsdn, &bin->lock);
1346
1347 assert(slab != bin->slabcur);
1348 if (config_stats) {
1349 bin->stats.curslabs--;
1350 }
1351}
1352
1353void
1354arena_dalloc_bin_locked_handle_newly_empty(tsdn_t *tsdn, arena_t *arena,
1355 edata_t *slab, bin_t *bin) {
1356 arena_dissociate_bin_slab(arena, slab, bin);
1357 arena_dalloc_bin_slab_prepare(tsdn, slab, bin);
1358}
1359
1360void
1361arena_dalloc_bin_locked_handle_newly_nonempty(tsdn_t *tsdn, arena_t *arena,
1362 edata_t *slab, bin_t *bin) {
1363 arena_bin_slabs_full_remove(arena, bin, slab);
1364 arena_bin_lower_slab(tsdn, arena, slab, bin);
1365}
1366
1367static void
1368arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, edata_t *edata, void *ptr) {
1369 szind_t binind = edata_szind_get(edata);
1370 unsigned binshard = edata_binshard_get(edata);
1371 bin_t *bin = arena_get_bin(arena, binind, binshard);
1372
1373 malloc_mutex_lock(tsdn, &bin->lock);
1374 arena_dalloc_bin_locked_info_t info;
1375 arena_dalloc_bin_locked_begin(&info, binind);
1376 bool ret = arena_dalloc_bin_locked_step(tsdn, arena, bin,
1377 &info, binind, edata, ptr);
1378 arena_dalloc_bin_locked_finish(tsdn, arena, bin, &info);
1379 malloc_mutex_unlock(tsdn, &bin->lock);
1380
1381 if (ret) {
1382 arena_slab_dalloc(tsdn, arena, edata);
1383 }
1384}
1385
1386void
1387arena_dalloc_small(tsdn_t *tsdn, void *ptr) {
1388 edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
1389 arena_t *arena = arena_get_from_edata(edata);
1390
1391 arena_dalloc_bin(tsdn, arena, edata, ptr);
1392 arena_decay_tick(tsdn, arena);
1393}
1394
1395bool
1396arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
1397 size_t extra, bool zero, size_t *newsize) {
1398 bool ret;
1399 /* Calls with non-zero extra had to clamp extra. */
1400 assert(extra == 0 || size + extra <= SC_LARGE_MAXCLASS);
1401
1402 edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
1403 if (unlikely(size > SC_LARGE_MAXCLASS)) {
1404 ret = true;
1405 goto done;
1406 }
1407
1408 size_t usize_min = sz_s2u(size);
1409 size_t usize_max = sz_s2u(size + extra);
1410 if (likely(oldsize <= SC_SMALL_MAXCLASS && usize_min
1411 <= SC_SMALL_MAXCLASS)) {
1412 /*
1413 * Avoid moving the allocation if the size class can be left the
1414 * same.
1415 */
1416 assert(bin_infos[sz_size2index(oldsize)].reg_size ==
1417 oldsize);
1418 if ((usize_max > SC_SMALL_MAXCLASS
1419 || sz_size2index(usize_max) != sz_size2index(oldsize))
1420 && (size > oldsize || usize_max < oldsize)) {
1421 ret = true;
1422 goto done;
1423 }
1424
1425 arena_t *arena = arena_get_from_edata(edata);
1426 arena_decay_tick(tsdn, arena);
1427 ret = false;
1428 } else if (oldsize >= SC_LARGE_MINCLASS
1429 && usize_max >= SC_LARGE_MINCLASS) {
1430 ret = large_ralloc_no_move(tsdn, edata, usize_min, usize_max,
1431 zero);
1432 } else {
1433 ret = true;
1434 }
1435done:
1436 assert(edata == emap_edata_lookup(tsdn, &arena_emap_global, ptr));
1437 *newsize = edata_usize_get(edata);
1438
1439 return ret;
1440}
1441
1442static void *
1443arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
1444 size_t alignment, bool zero, tcache_t *tcache) {
1445 if (alignment == 0) {
1446 return arena_malloc(tsdn, arena, usize, sz_size2index(usize),
1447 zero, tcache, true);
1448 }
1449 usize = sz_sa2u(usize, alignment);
1450 if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
1451 return NULL;
1452 }
1453 return ipalloct(tsdn, usize, alignment, zero, tcache, arena);
1454}
1455
1456void *
1457arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
1458 size_t size, size_t alignment, bool zero, tcache_t *tcache,
1459 hook_ralloc_args_t *hook_args) {
1460 size_t usize = alignment == 0 ? sz_s2u(size) : sz_sa2u(size, alignment);
1461 if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) {
1462 return NULL;
1463 }
1464
1465 if (likely(usize <= SC_SMALL_MAXCLASS)) {
1466 /* Try to avoid moving the allocation. */
1467 UNUSED size_t newsize;
1468 if (!arena_ralloc_no_move(tsdn, ptr, oldsize, usize, 0, zero,
1469 &newsize)) {
1470 hook_invoke_expand(hook_args->is_realloc
1471 ? hook_expand_realloc : hook_expand_rallocx,
1472 ptr, oldsize, usize, (uintptr_t)ptr,
1473 hook_args->args);
1474 return ptr;
1475 }
1476 }
1477
1478 if (oldsize >= SC_LARGE_MINCLASS
1479 && usize >= SC_LARGE_MINCLASS) {
1480 return large_ralloc(tsdn, arena, ptr, usize,
1481 alignment, zero, tcache, hook_args);
1482 }
1483
1484 /*
1485 * size and oldsize are different enough that we need to move the
1486 * object. In that case, fall back to allocating new space and copying.
1487 */
1488 void *ret = arena_ralloc_move_helper(tsdn, arena, usize, alignment,
1489 zero, tcache);
1490 if (ret == NULL) {
1491 return NULL;
1492 }
1493
1494 hook_invoke_alloc(hook_args->is_realloc
1495 ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret,
1496 hook_args->args);
1497 hook_invoke_dalloc(hook_args->is_realloc
1498 ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args);
1499
1500 /*
1501 * Junk/zero-filling were already done by
1502 * ipalloc()/arena_malloc().
1503 */
1504 size_t copysize = (usize < oldsize) ? usize : oldsize;
1505 memcpy(ret, ptr, copysize);
1506 isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
1507 return ret;
1508}
1509
1510ehooks_t *
1511arena_get_ehooks(arena_t *arena) {
1512 return base_ehooks_get(arena->base);
1513}
1514
1515extent_hooks_t *
1516arena_set_extent_hooks(tsd_t *tsd, arena_t *arena,
1517 extent_hooks_t *extent_hooks) {
1518 background_thread_info_t *info;
1519 if (have_background_thread) {
1520 info = arena_background_thread_info_get(arena);
1521 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
1522 }
1523 /* No using the HPA now that we have the custom hooks. */
1524 pa_shard_disable_hpa(tsd_tsdn(tsd), &arena->pa_shard);
1525 extent_hooks_t *ret = base_extent_hooks_set(arena->base, extent_hooks);
1526 if (have_background_thread) {
1527 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
1528 }
1529
1530 return ret;
1531}
1532
1533dss_prec_t
1534arena_dss_prec_get(arena_t *arena) {
1535 return (dss_prec_t)atomic_load_u(&arena->dss_prec, ATOMIC_ACQUIRE);
1536}
1537
1538bool
1539arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) {
1540 if (!have_dss) {
1541 return (dss_prec != dss_prec_disabled);
1542 }
1543 atomic_store_u(&arena->dss_prec, (unsigned)dss_prec, ATOMIC_RELEASE);
1544 return false;
1545}
1546
1547ssize_t
1548arena_dirty_decay_ms_default_get(void) {
1549 return atomic_load_zd(&dirty_decay_ms_default, ATOMIC_RELAXED);
1550}
1551
1552bool
1553arena_dirty_decay_ms_default_set(ssize_t decay_ms) {
1554 if (!decay_ms_valid(decay_ms)) {
1555 return true;
1556 }
1557 atomic_store_zd(&dirty_decay_ms_default, decay_ms, ATOMIC_RELAXED);
1558 return false;
1559}
1560
1561ssize_t
1562arena_muzzy_decay_ms_default_get(void) {
1563 return atomic_load_zd(&muzzy_decay_ms_default, ATOMIC_RELAXED);
1564}
1565
1566bool
1567arena_muzzy_decay_ms_default_set(ssize_t decay_ms) {
1568 if (!decay_ms_valid(decay_ms)) {
1569 return true;
1570 }
1571 atomic_store_zd(&muzzy_decay_ms_default, decay_ms, ATOMIC_RELAXED);
1572 return false;
1573}
1574
1575bool
1576arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, size_t *old_limit,
1577 size_t *new_limit) {
1578 assert(opt_retain);
1579 return pac_retain_grow_limit_get_set(tsd_tsdn(tsd),
1580 &arena->pa_shard.pac, old_limit, new_limit);
1581}
1582
1583unsigned
1584arena_nthreads_get(arena_t *arena, bool internal) {
1585 return atomic_load_u(&arena->nthreads[internal], ATOMIC_RELAXED);
1586}
1587
1588void
1589arena_nthreads_inc(arena_t *arena, bool internal) {
1590 atomic_fetch_add_u(&arena->nthreads[internal], 1, ATOMIC_RELAXED);
1591}
1592
1593void
1594arena_nthreads_dec(arena_t *arena, bool internal) {
1595 atomic_fetch_sub_u(&arena->nthreads[internal], 1, ATOMIC_RELAXED);
1596}
1597
1598arena_t *
1599arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
1600 arena_t *arena;
1601 base_t *base;
1602 unsigned i;
1603
1604 if (ind == 0) {
1605 base = b0get();
1606 } else {
1607 base = base_new(tsdn, ind, config->extent_hooks,
1608 config->metadata_use_hooks);
1609 if (base == NULL) {
1610 return NULL;
1611 }
1612 }
1613
1614 size_t arena_size = sizeof(arena_t) + sizeof(bin_t) * nbins_total;
1615 arena = (arena_t *)base_alloc(tsdn, base, arena_size, CACHELINE);
1616 if (arena == NULL) {
1617 goto label_error;
1618 }
1619
1620 atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED);
1621 atomic_store_u(&arena->nthreads[1], 0, ATOMIC_RELAXED);
1622 arena->last_thd = NULL;
1623
1624 if (config_stats) {
1625 if (arena_stats_init(tsdn, &arena->stats)) {
1626 goto label_error;
1627 }
1628
1629 ql_new(&arena->tcache_ql);
1630 ql_new(&arena->cache_bin_array_descriptor_ql);
1631 if (malloc_mutex_init(&arena->tcache_ql_mtx, "tcache_ql",
1632 WITNESS_RANK_TCACHE_QL, malloc_mutex_rank_exclusive)) {
1633 goto label_error;
1634 }
1635 }
1636
1637 atomic_store_u(&arena->dss_prec, (unsigned)extent_dss_prec_get(),
1638 ATOMIC_RELAXED);
1639
1640 edata_list_active_init(&arena->large);
1641 if (malloc_mutex_init(&arena->large_mtx, "arena_large",
1642 WITNESS_RANK_ARENA_LARGE, malloc_mutex_rank_exclusive)) {
1643 goto label_error;
1644 }
1645
1646 nstime_t cur_time;
1647 nstime_init_update(&cur_time);
1648 if (pa_shard_init(tsdn, &arena->pa_shard, &arena_pa_central_global,
1649 &arena_emap_global, base, ind, &arena->stats.pa_shard_stats,
1650 LOCKEDINT_MTX(arena->stats.mtx), &cur_time, oversize_threshold,
1651 arena_dirty_decay_ms_default_get(),
1652 arena_muzzy_decay_ms_default_get())) {
1653 goto label_error;
1654 }
1655
1656 /* Initialize bins. */
1657 atomic_store_u(&arena->binshard_next, 0, ATOMIC_RELEASE);
1658 for (i = 0; i < nbins_total; i++) {
1659 bool err = bin_init(&arena->bins[i]);
1660 if (err) {
1661 goto label_error;
1662 }
1663 }
1664
1665 arena->base = base;
1666 /* Set arena before creating background threads. */
1667 arena_set(ind, arena);
1668 arena->ind = ind;
1669
1670 nstime_init_update(&arena->create_time);
1671
1672 /*
1673 * We turn on the HPA if set to. There are two exceptions:
1674 * - Custom extent hooks (we should only return memory allocated from
1675 * them in that case).
1676 * - Arena 0 initialization. In this case, we're mid-bootstrapping, and
1677 * so arena_hpa_global is not yet initialized.
1678 */
1679 if (opt_hpa && ehooks_are_default(base_ehooks_get(base)) && ind != 0) {
1680 hpa_shard_opts_t hpa_shard_opts = opt_hpa_opts;
1681 hpa_shard_opts.deferral_allowed = background_thread_enabled();
1682 if (pa_shard_enable_hpa(tsdn, &arena->pa_shard,
1683 &hpa_shard_opts, &opt_hpa_sec_opts)) {
1684 goto label_error;
1685 }
1686 }
1687
1688 /* We don't support reentrancy for arena 0 bootstrapping. */
1689 if (ind != 0) {
1690 /*
1691 * If we're here, then arena 0 already exists, so bootstrapping
1692 * is done enough that we should have tsd.
1693 */
1694 assert(!tsdn_null(tsdn));
1695 pre_reentrancy(tsdn_tsd(tsdn), arena);
1696 if (test_hooks_arena_new_hook) {
1697 test_hooks_arena_new_hook();
1698 }
1699 post_reentrancy(tsdn_tsd(tsdn));
1700 }
1701
1702 return arena;
1703label_error:
1704 if (ind != 0) {
1705 base_delete(tsdn, base);
1706 }
1707 return NULL;
1708}
1709
1710arena_t *
1711arena_choose_huge(tsd_t *tsd) {
1712 /* huge_arena_ind can be 0 during init (will use a0). */
1713 if (huge_arena_ind == 0) {
1714 assert(!malloc_initialized());
1715 }
1716
1717 arena_t *huge_arena = arena_get(tsd_tsdn(tsd), huge_arena_ind, false);
1718 if (huge_arena == NULL) {
1719 /* Create the huge arena on demand. */
1720 assert(huge_arena_ind != 0);
1721 huge_arena = arena_get(tsd_tsdn(tsd), huge_arena_ind, true);
1722 if (huge_arena == NULL) {
1723 return NULL;
1724 }
1725 /*
1726 * Purge eagerly for huge allocations, because: 1) number of
1727 * huge allocations is usually small, which means ticker based
1728 * decay is not reliable; and 2) less immediate reuse is
1729 * expected for huge allocations.
1730 */
1731 if (arena_dirty_decay_ms_default_get() > 0) {
1732 arena_decay_ms_set(tsd_tsdn(tsd), huge_arena,
1733 extent_state_dirty, 0);
1734 }
1735 if (arena_muzzy_decay_ms_default_get() > 0) {
1736 arena_decay_ms_set(tsd_tsdn(tsd), huge_arena,
1737 extent_state_muzzy, 0);
1738 }
1739 }
1740
1741 return huge_arena;
1742}
1743
1744bool
1745arena_init_huge(void) {
1746 bool huge_enabled;
1747
1748 /* The threshold should be large size class. */
1749 if (opt_oversize_threshold > SC_LARGE_MAXCLASS ||
1750 opt_oversize_threshold < SC_LARGE_MINCLASS) {
1751 opt_oversize_threshold = 0;
1752 oversize_threshold = SC_LARGE_MAXCLASS + PAGE;
1753 huge_enabled = false;
1754 } else {
1755 /* Reserve the index for the huge arena. */
1756 huge_arena_ind = narenas_total_get();
1757 oversize_threshold = opt_oversize_threshold;
1758 huge_enabled = true;
1759 }
1760
1761 return huge_enabled;
1762}
1763
1764bool
1765arena_is_huge(unsigned arena_ind) {
1766 if (huge_arena_ind == 0) {
1767 return false;
1768 }
1769 return (arena_ind == huge_arena_ind);
1770}
1771
1772bool
1773arena_boot(sc_data_t *sc_data, base_t *base, bool hpa) {
1774 arena_dirty_decay_ms_default_set(opt_dirty_decay_ms);
1775 arena_muzzy_decay_ms_default_set(opt_muzzy_decay_ms);
1776 for (unsigned i = 0; i < SC_NBINS; i++) {
1777 sc_t *sc = &sc_data->sc[i];
1778 div_init(&arena_binind_div_info[i],
1779 (1U << sc->lg_base) + (sc->ndelta << sc->lg_delta));
1780 }
1781
1782 uint32_t cur_offset = (uint32_t)offsetof(arena_t, bins);
1783 for (szind_t i = 0; i < SC_NBINS; i++) {
1784 arena_bin_offsets[i] = cur_offset;
1785 nbins_total += bin_infos[i].n_shards;
1786 cur_offset += (uint32_t)(bin_infos[i].n_shards * sizeof(bin_t));
1787 }
1788 return pa_central_init(&arena_pa_central_global, base, hpa,
1789 &hpa_hooks_default);
1790}
1791
1792void
1793arena_prefork0(tsdn_t *tsdn, arena_t *arena) {
1794 pa_shard_prefork0(tsdn, &arena->pa_shard);
1795}
1796
1797void
1798arena_prefork1(tsdn_t *tsdn, arena_t *arena) {
1799 if (config_stats) {
1800 malloc_mutex_prefork(tsdn, &arena->tcache_ql_mtx);
1801 }
1802}
1803
1804void
1805arena_prefork2(tsdn_t *tsdn, arena_t *arena) {
1806 pa_shard_prefork2(tsdn, &arena->pa_shard);
1807}
1808
1809void
1810arena_prefork3(tsdn_t *tsdn, arena_t *arena) {
1811 pa_shard_prefork3(tsdn, &arena->pa_shard);
1812}
1813
1814void
1815arena_prefork4(tsdn_t *tsdn, arena_t *arena) {
1816 pa_shard_prefork4(tsdn, &arena->pa_shard);
1817}
1818
1819void
1820arena_prefork5(tsdn_t *tsdn, arena_t *arena) {
1821 pa_shard_prefork5(tsdn, &arena->pa_shard);
1822}
1823
1824void
1825arena_prefork6(tsdn_t *tsdn, arena_t *arena) {
1826 base_prefork(tsdn, arena->base);
1827}
1828
1829void
1830arena_prefork7(tsdn_t *tsdn, arena_t *arena) {
1831 malloc_mutex_prefork(tsdn, &arena->large_mtx);
1832}
1833
1834void
1835arena_prefork8(tsdn_t *tsdn, arena_t *arena) {
1836 for (unsigned i = 0; i < nbins_total; i++) {
1837 bin_prefork(tsdn, &arena->bins[i]);
1838 }
1839}
1840
1841void
1842arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
1843 for (unsigned i = 0; i < nbins_total; i++) {
1844 bin_postfork_parent(tsdn, &arena->bins[i]);
1845 }
1846
1847 malloc_mutex_postfork_parent(tsdn, &arena->large_mtx);
1848 base_postfork_parent(tsdn, arena->base);
1849 pa_shard_postfork_parent(tsdn, &arena->pa_shard);
1850 if (config_stats) {
1851 malloc_mutex_postfork_parent(tsdn, &arena->tcache_ql_mtx);
1852 }
1853}
1854
1855void
1856arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
1857 atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED);
1858 atomic_store_u(&arena->nthreads[1], 0, ATOMIC_RELAXED);
1859 if (tsd_arena_get(tsdn_tsd(tsdn)) == arena) {
1860 arena_nthreads_inc(arena, false);
1861 }
1862 if (tsd_iarena_get(tsdn_tsd(tsdn)) == arena) {
1863 arena_nthreads_inc(arena, true);
1864 }
1865 if (config_stats) {
1866 ql_new(&arena->tcache_ql);
1867 ql_new(&arena->cache_bin_array_descriptor_ql);
1868 tcache_slow_t *tcache_slow = tcache_slow_get(tsdn_tsd(tsdn));
1869 if (tcache_slow != NULL && tcache_slow->arena == arena) {
1870 tcache_t *tcache = tcache_slow->tcache;
1871 ql_elm_new(tcache_slow, link);
1872 ql_tail_insert(&arena->tcache_ql, tcache_slow, link);
1873 cache_bin_array_descriptor_init(
1874 &tcache_slow->cache_bin_array_descriptor,
1875 tcache->bins);
1876 ql_tail_insert(&arena->cache_bin_array_descriptor_ql,
1877 &tcache_slow->cache_bin_array_descriptor, link);
1878 }
1879 }
1880
1881 for (unsigned i = 0; i < nbins_total; i++) {
1882 bin_postfork_child(tsdn, &arena->bins[i]);
1883 }
1884
1885 malloc_mutex_postfork_child(tsdn, &arena->large_mtx);
1886 base_postfork_child(tsdn, arena->base);
1887 pa_shard_postfork_child(tsdn, &arena->pa_shard);
1888 if (config_stats) {
1889 malloc_mutex_postfork_child(tsdn, &arena->tcache_ql_mtx);
1890 }
1891}
diff --git a/examples/redis-unstable/deps/jemalloc/src/background_thread.c b/examples/redis-unstable/deps/jemalloc/src/background_thread.c
deleted file mode 100644
index 3bb8d26..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/background_thread.c
+++ /dev/null
@@ -1,820 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5
6JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
7
8/******************************************************************************/
9/* Data. */
10
11/* This option should be opt-in only. */
12#define BACKGROUND_THREAD_DEFAULT false
13/* Read-only after initialization. */
14bool opt_background_thread = BACKGROUND_THREAD_DEFAULT;
15size_t opt_max_background_threads = MAX_BACKGROUND_THREAD_LIMIT + 1;
16
17/* Used for thread creation, termination and stats. */
18malloc_mutex_t background_thread_lock;
19/* Indicates global state. Atomic because decay reads this w/o locking. */
20atomic_b_t background_thread_enabled_state;
21size_t n_background_threads;
22size_t max_background_threads;
23/* Thread info per-index. */
24background_thread_info_t *background_thread_info;
25
26/******************************************************************************/
27
28#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
29
30static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *,
31 void *(*)(void *), void *__restrict);
32
33static void
34pthread_create_wrapper_init(void) {
35#ifdef JEMALLOC_LAZY_LOCK
36 if (!isthreaded) {
37 isthreaded = true;
38 }
39#endif
40}
41
42int
43pthread_create_wrapper(pthread_t *__restrict thread, const pthread_attr_t *attr,
44 void *(*start_routine)(void *), void *__restrict arg) {
45 pthread_create_wrapper_init();
46
47 return pthread_create_fptr(thread, attr, start_routine, arg);
48}
49#endif /* JEMALLOC_PTHREAD_CREATE_WRAPPER */
50
51#ifndef JEMALLOC_BACKGROUND_THREAD
52#define NOT_REACHED { not_reached(); }
53bool background_thread_create(tsd_t *tsd, unsigned arena_ind) NOT_REACHED
54bool background_threads_enable(tsd_t *tsd) NOT_REACHED
55bool background_threads_disable(tsd_t *tsd) NOT_REACHED
56bool background_thread_is_started(background_thread_info_t *info) NOT_REACHED
57void background_thread_wakeup_early(background_thread_info_t *info,
58 nstime_t *remaining_sleep) NOT_REACHED
59void background_thread_prefork0(tsdn_t *tsdn) NOT_REACHED
60void background_thread_prefork1(tsdn_t *tsdn) NOT_REACHED
61void background_thread_postfork_parent(tsdn_t *tsdn) NOT_REACHED
62void background_thread_postfork_child(tsdn_t *tsdn) NOT_REACHED
63bool background_thread_stats_read(tsdn_t *tsdn,
64 background_thread_stats_t *stats) NOT_REACHED
65void background_thread_ctl_init(tsdn_t *tsdn) NOT_REACHED
66#undef NOT_REACHED
67#else
68
69static bool background_thread_enabled_at_fork;
70
71static void
72background_thread_info_init(tsdn_t *tsdn, background_thread_info_t *info) {
73 background_thread_wakeup_time_set(tsdn, info, 0);
74 info->npages_to_purge_new = 0;
75 if (config_stats) {
76 info->tot_n_runs = 0;
77 nstime_init_zero(&info->tot_sleep_time);
78 }
79}
80
81static inline bool
82set_current_thread_affinity(int cpu) {
83#if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
84 cpu_set_t cpuset;
85#else
86# ifndef __NetBSD__
87 cpuset_t cpuset;
88# else
89 cpuset_t *cpuset;
90# endif
91#endif
92
93#ifndef __NetBSD__
94 CPU_ZERO(&cpuset);
95 CPU_SET(cpu, &cpuset);
96#else
97 cpuset = cpuset_create();
98#endif
99
100#if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
101 return (sched_setaffinity(0, sizeof(cpu_set_t), &cpuset) != 0);
102#else
103# ifndef __NetBSD__
104 int ret = pthread_setaffinity_np(pthread_self(), sizeof(cpuset_t),
105 &cpuset);
106# else
107 int ret = pthread_setaffinity_np(pthread_self(), cpuset_size(cpuset),
108 cpuset);
109 cpuset_destroy(cpuset);
110# endif
111 return ret != 0;
112#endif
113}
114
115#define BILLION UINT64_C(1000000000)
116/* Minimal sleep interval 100 ms. */
117#define BACKGROUND_THREAD_MIN_INTERVAL_NS (BILLION / 10)
118
119static void
120background_thread_sleep(tsdn_t *tsdn, background_thread_info_t *info,
121 uint64_t interval) {
122 if (config_stats) {
123 info->tot_n_runs++;
124 }
125 info->npages_to_purge_new = 0;
126
127 struct timeval tv;
128 /* Specific clock required by timedwait. */
129 gettimeofday(&tv, NULL);
130 nstime_t before_sleep;
131 nstime_init2(&before_sleep, tv.tv_sec, tv.tv_usec * 1000);
132
133 int ret;
134 if (interval == BACKGROUND_THREAD_INDEFINITE_SLEEP) {
135 background_thread_wakeup_time_set(tsdn, info,
136 BACKGROUND_THREAD_INDEFINITE_SLEEP);
137 ret = pthread_cond_wait(&info->cond, &info->mtx.lock);
138 assert(ret == 0);
139 } else {
140 assert(interval >= BACKGROUND_THREAD_MIN_INTERVAL_NS &&
141 interval <= BACKGROUND_THREAD_INDEFINITE_SLEEP);
142 /* We need malloc clock (can be different from tv). */
143 nstime_t next_wakeup;
144 nstime_init_update(&next_wakeup);
145 nstime_iadd(&next_wakeup, interval);
146 assert(nstime_ns(&next_wakeup) <
147 BACKGROUND_THREAD_INDEFINITE_SLEEP);
148 background_thread_wakeup_time_set(tsdn, info,
149 nstime_ns(&next_wakeup));
150
151 nstime_t ts_wakeup;
152 nstime_copy(&ts_wakeup, &before_sleep);
153 nstime_iadd(&ts_wakeup, interval);
154 struct timespec ts;
155 ts.tv_sec = (size_t)nstime_sec(&ts_wakeup);
156 ts.tv_nsec = (size_t)nstime_nsec(&ts_wakeup);
157
158 assert(!background_thread_indefinite_sleep(info));
159 ret = pthread_cond_timedwait(&info->cond, &info->mtx.lock, &ts);
160 assert(ret == ETIMEDOUT || ret == 0);
161 }
162 if (config_stats) {
163 gettimeofday(&tv, NULL);
164 nstime_t after_sleep;
165 nstime_init2(&after_sleep, tv.tv_sec, tv.tv_usec * 1000);
166 if (nstime_compare(&after_sleep, &before_sleep) > 0) {
167 nstime_subtract(&after_sleep, &before_sleep);
168 nstime_add(&info->tot_sleep_time, &after_sleep);
169 }
170 }
171}
172
173static bool
174background_thread_pause_check(tsdn_t *tsdn, background_thread_info_t *info) {
175 if (unlikely(info->state == background_thread_paused)) {
176 malloc_mutex_unlock(tsdn, &info->mtx);
177 /* Wait on global lock to update status. */
178 malloc_mutex_lock(tsdn, &background_thread_lock);
179 malloc_mutex_unlock(tsdn, &background_thread_lock);
180 malloc_mutex_lock(tsdn, &info->mtx);
181 return true;
182 }
183
184 return false;
185}
186
187static inline void
188background_work_sleep_once(tsdn_t *tsdn, background_thread_info_t *info,
189 unsigned ind) {
190 uint64_t ns_until_deferred = BACKGROUND_THREAD_DEFERRED_MAX;
191 unsigned narenas = narenas_total_get();
192 bool slept_indefinitely = background_thread_indefinite_sleep(info);
193
194 for (unsigned i = ind; i < narenas; i += max_background_threads) {
195 arena_t *arena = arena_get(tsdn, i, false);
196 if (!arena) {
197 continue;
198 }
199 /*
200 * If thread was woken up from the indefinite sleep, don't
201 * do the work instantly, but rather check when the deferred
202 * work that caused this thread to wake up is scheduled for.
203 */
204 if (!slept_indefinitely) {
205 arena_do_deferred_work(tsdn, arena);
206 }
207 if (ns_until_deferred <= BACKGROUND_THREAD_MIN_INTERVAL_NS) {
208 /* Min interval will be used. */
209 continue;
210 }
211 uint64_t ns_arena_deferred = pa_shard_time_until_deferred_work(
212 tsdn, &arena->pa_shard);
213 if (ns_arena_deferred < ns_until_deferred) {
214 ns_until_deferred = ns_arena_deferred;
215 }
216 }
217
218 uint64_t sleep_ns;
219 if (ns_until_deferred == BACKGROUND_THREAD_DEFERRED_MAX) {
220 sleep_ns = BACKGROUND_THREAD_INDEFINITE_SLEEP;
221 } else {
222 sleep_ns =
223 (ns_until_deferred < BACKGROUND_THREAD_MIN_INTERVAL_NS)
224 ? BACKGROUND_THREAD_MIN_INTERVAL_NS
225 : ns_until_deferred;
226
227 }
228
229 background_thread_sleep(tsdn, info, sleep_ns);
230}
231
232static bool
233background_threads_disable_single(tsd_t *tsd, background_thread_info_t *info) {
234 if (info == &background_thread_info[0]) {
235 malloc_mutex_assert_owner(tsd_tsdn(tsd),
236 &background_thread_lock);
237 } else {
238 malloc_mutex_assert_not_owner(tsd_tsdn(tsd),
239 &background_thread_lock);
240 }
241
242 pre_reentrancy(tsd, NULL);
243 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
244 bool has_thread;
245 assert(info->state != background_thread_paused);
246 if (info->state == background_thread_started) {
247 has_thread = true;
248 info->state = background_thread_stopped;
249 pthread_cond_signal(&info->cond);
250 } else {
251 has_thread = false;
252 }
253 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
254
255 if (!has_thread) {
256 post_reentrancy(tsd);
257 return false;
258 }
259 void *ret;
260 if (pthread_join(info->thread, &ret)) {
261 post_reentrancy(tsd);
262 return true;
263 }
264 assert(ret == NULL);
265 n_background_threads--;
266 post_reentrancy(tsd);
267
268 return false;
269}
270
271static void *background_thread_entry(void *ind_arg);
272
273static int
274background_thread_create_signals_masked(pthread_t *thread,
275 const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg) {
276 /*
277 * Mask signals during thread creation so that the thread inherits
278 * an empty signal set.
279 */
280 sigset_t set;
281 sigfillset(&set);
282 sigset_t oldset;
283 int mask_err = pthread_sigmask(SIG_SETMASK, &set, &oldset);
284 if (mask_err != 0) {
285 return mask_err;
286 }
287 int create_err = pthread_create_wrapper(thread, attr, start_routine,
288 arg);
289 /*
290 * Restore the signal mask. Failure to restore the signal mask here
291 * changes program behavior.
292 */
293 int restore_err = pthread_sigmask(SIG_SETMASK, &oldset, NULL);
294 if (restore_err != 0) {
295 malloc_printf("<jemalloc>: background thread creation "
296 "failed (%d), and signal mask restoration failed "
297 "(%d)\n", create_err, restore_err);
298 if (opt_abort) {
299 abort();
300 }
301 }
302 return create_err;
303}
304
305static bool
306check_background_thread_creation(tsd_t *tsd, unsigned *n_created,
307 bool *created_threads) {
308 bool ret = false;
309 if (likely(*n_created == n_background_threads)) {
310 return ret;
311 }
312
313 tsdn_t *tsdn = tsd_tsdn(tsd);
314 malloc_mutex_unlock(tsdn, &background_thread_info[0].mtx);
315 for (unsigned i = 1; i < max_background_threads; i++) {
316 if (created_threads[i]) {
317 continue;
318 }
319 background_thread_info_t *info = &background_thread_info[i];
320 malloc_mutex_lock(tsdn, &info->mtx);
321 /*
322 * In case of the background_thread_paused state because of
323 * arena reset, delay the creation.
324 */
325 bool create = (info->state == background_thread_started);
326 malloc_mutex_unlock(tsdn, &info->mtx);
327 if (!create) {
328 continue;
329 }
330
331 pre_reentrancy(tsd, NULL);
332 int err = background_thread_create_signals_masked(&info->thread,
333 NULL, background_thread_entry, (void *)(uintptr_t)i);
334 post_reentrancy(tsd);
335
336 if (err == 0) {
337 (*n_created)++;
338 created_threads[i] = true;
339 } else {
340 malloc_printf("<jemalloc>: background thread "
341 "creation failed (%d)\n", err);
342 if (opt_abort) {
343 abort();
344 }
345 }
346 /* Return to restart the loop since we unlocked. */
347 ret = true;
348 break;
349 }
350 malloc_mutex_lock(tsdn, &background_thread_info[0].mtx);
351
352 return ret;
353}
354
355static void
356background_thread0_work(tsd_t *tsd) {
357 /* Thread0 is also responsible for launching / terminating threads. */
358 VARIABLE_ARRAY(bool, created_threads, max_background_threads);
359 unsigned i;
360 for (i = 1; i < max_background_threads; i++) {
361 created_threads[i] = false;
362 }
363 /* Start working, and create more threads when asked. */
364 unsigned n_created = 1;
365 while (background_thread_info[0].state != background_thread_stopped) {
366 if (background_thread_pause_check(tsd_tsdn(tsd),
367 &background_thread_info[0])) {
368 continue;
369 }
370 if (check_background_thread_creation(tsd, &n_created,
371 (bool *)&created_threads)) {
372 continue;
373 }
374 background_work_sleep_once(tsd_tsdn(tsd),
375 &background_thread_info[0], 0);
376 }
377
378 /*
379 * Shut down other threads at exit. Note that the ctl thread is holding
380 * the global background_thread mutex (and is waiting) for us.
381 */
382 assert(!background_thread_enabled());
383 for (i = 1; i < max_background_threads; i++) {
384 background_thread_info_t *info = &background_thread_info[i];
385 assert(info->state != background_thread_paused);
386 if (created_threads[i]) {
387 background_threads_disable_single(tsd, info);
388 } else {
389 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
390 if (info->state != background_thread_stopped) {
391 /* The thread was not created. */
392 assert(info->state ==
393 background_thread_started);
394 n_background_threads--;
395 info->state = background_thread_stopped;
396 }
397 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
398 }
399 }
400 background_thread_info[0].state = background_thread_stopped;
401 assert(n_background_threads == 1);
402}
403
404static void
405background_work(tsd_t *tsd, unsigned ind) {
406 background_thread_info_t *info = &background_thread_info[ind];
407
408 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
409 background_thread_wakeup_time_set(tsd_tsdn(tsd), info,
410 BACKGROUND_THREAD_INDEFINITE_SLEEP);
411 if (ind == 0) {
412 background_thread0_work(tsd);
413 } else {
414 while (info->state != background_thread_stopped) {
415 if (background_thread_pause_check(tsd_tsdn(tsd),
416 info)) {
417 continue;
418 }
419 background_work_sleep_once(tsd_tsdn(tsd), info, ind);
420 }
421 }
422 assert(info->state == background_thread_stopped);
423 background_thread_wakeup_time_set(tsd_tsdn(tsd), info, 0);
424 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
425}
426
427static void *
428background_thread_entry(void *ind_arg) {
429 unsigned thread_ind = (unsigned)(uintptr_t)ind_arg;
430 assert(thread_ind < max_background_threads);
431#ifdef JEMALLOC_HAVE_PTHREAD_SETNAME_NP
432 pthread_setname_np(pthread_self(), "jemalloc_bg_thd");
433#elif defined(__FreeBSD__) || defined(__DragonFly__)
434 pthread_set_name_np(pthread_self(), "jemalloc_bg_thd");
435#endif
436 if (opt_percpu_arena != percpu_arena_disabled) {
437 set_current_thread_affinity((int)thread_ind);
438 }
439 /*
440 * Start periodic background work. We use internal tsd which avoids
441 * side effects, for example triggering new arena creation (which in
442 * turn triggers another background thread creation).
443 */
444 background_work(tsd_internal_fetch(), thread_ind);
445 assert(pthread_equal(pthread_self(),
446 background_thread_info[thread_ind].thread));
447
448 return NULL;
449}
450
451static void
452background_thread_init(tsd_t *tsd, background_thread_info_t *info) {
453 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
454 info->state = background_thread_started;
455 background_thread_info_init(tsd_tsdn(tsd), info);
456 n_background_threads++;
457}
458
459static bool
460background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) {
461 assert(have_background_thread);
462 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
463
464 /* We create at most NCPUs threads. */
465 size_t thread_ind = arena_ind % max_background_threads;
466 background_thread_info_t *info = &background_thread_info[thread_ind];
467
468 bool need_new_thread;
469 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
470 need_new_thread = background_thread_enabled() &&
471 (info->state == background_thread_stopped);
472 if (need_new_thread) {
473 background_thread_init(tsd, info);
474 }
475 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
476 if (!need_new_thread) {
477 return false;
478 }
479 if (arena_ind != 0) {
480 /* Threads are created asynchronously by Thread 0. */
481 background_thread_info_t *t0 = &background_thread_info[0];
482 malloc_mutex_lock(tsd_tsdn(tsd), &t0->mtx);
483 assert(t0->state == background_thread_started);
484 pthread_cond_signal(&t0->cond);
485 malloc_mutex_unlock(tsd_tsdn(tsd), &t0->mtx);
486
487 return false;
488 }
489
490 pre_reentrancy(tsd, NULL);
491 /*
492 * To avoid complications (besides reentrancy), create internal
493 * background threads with the underlying pthread_create.
494 */
495 int err = background_thread_create_signals_masked(&info->thread, NULL,
496 background_thread_entry, (void *)thread_ind);
497 post_reentrancy(tsd);
498
499 if (err != 0) {
500 malloc_printf("<jemalloc>: arena 0 background thread creation "
501 "failed (%d)\n", err);
502 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
503 info->state = background_thread_stopped;
504 n_background_threads--;
505 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
506
507 return true;
508 }
509
510 return false;
511}
512
513/* Create a new background thread if needed. */
514bool
515background_thread_create(tsd_t *tsd, unsigned arena_ind) {
516 assert(have_background_thread);
517
518 bool ret;
519 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
520 ret = background_thread_create_locked(tsd, arena_ind);
521 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
522
523 return ret;
524}
525
526bool
527background_threads_enable(tsd_t *tsd) {
528 assert(n_background_threads == 0);
529 assert(background_thread_enabled());
530 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
531
532 VARIABLE_ARRAY(bool, marked, max_background_threads);
533 unsigned nmarked;
534 for (unsigned i = 0; i < max_background_threads; i++) {
535 marked[i] = false;
536 }
537 nmarked = 0;
538 /* Thread 0 is required and created at the end. */
539 marked[0] = true;
540 /* Mark the threads we need to create for thread 0. */
541 unsigned narenas = narenas_total_get();
542 for (unsigned i = 1; i < narenas; i++) {
543 if (marked[i % max_background_threads] ||
544 arena_get(tsd_tsdn(tsd), i, false) == NULL) {
545 continue;
546 }
547 background_thread_info_t *info = &background_thread_info[
548 i % max_background_threads];
549 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
550 assert(info->state == background_thread_stopped);
551 background_thread_init(tsd, info);
552 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
553 marked[i % max_background_threads] = true;
554 if (++nmarked == max_background_threads) {
555 break;
556 }
557 }
558
559 bool err = background_thread_create_locked(tsd, 0);
560 if (err) {
561 return true;
562 }
563 for (unsigned i = 0; i < narenas; i++) {
564 arena_t *arena = arena_get(tsd_tsdn(tsd), i, false);
565 if (arena != NULL) {
566 pa_shard_set_deferral_allowed(tsd_tsdn(tsd),
567 &arena->pa_shard, true);
568 }
569 }
570 return false;
571}
572
573bool
574background_threads_disable(tsd_t *tsd) {
575 assert(!background_thread_enabled());
576 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
577
578 /* Thread 0 will be responsible for terminating other threads. */
579 if (background_threads_disable_single(tsd,
580 &background_thread_info[0])) {
581 return true;
582 }
583 assert(n_background_threads == 0);
584 unsigned narenas = narenas_total_get();
585 for (unsigned i = 0; i < narenas; i++) {
586 arena_t *arena = arena_get(tsd_tsdn(tsd), i, false);
587 if (arena != NULL) {
588 pa_shard_set_deferral_allowed(tsd_tsdn(tsd),
589 &arena->pa_shard, false);
590 }
591 }
592
593 return false;
594}
595
596bool
597background_thread_is_started(background_thread_info_t *info) {
598 return info->state == background_thread_started;
599}
600
601void
602background_thread_wakeup_early(background_thread_info_t *info,
603 nstime_t *remaining_sleep) {
604 /*
605 * This is an optimization to increase batching. At this point
606 * we know that background thread wakes up soon, so the time to cache
607 * the just freed memory is bounded and low.
608 */
609 if (remaining_sleep != NULL && nstime_ns(remaining_sleep) <
610 BACKGROUND_THREAD_MIN_INTERVAL_NS) {
611 return;
612 }
613 pthread_cond_signal(&info->cond);
614}
615
616void
617background_thread_prefork0(tsdn_t *tsdn) {
618 malloc_mutex_prefork(tsdn, &background_thread_lock);
619 background_thread_enabled_at_fork = background_thread_enabled();
620}
621
622void
623background_thread_prefork1(tsdn_t *tsdn) {
624 for (unsigned i = 0; i < max_background_threads; i++) {
625 malloc_mutex_prefork(tsdn, &background_thread_info[i].mtx);
626 }
627}
628
629void
630background_thread_postfork_parent(tsdn_t *tsdn) {
631 for (unsigned i = 0; i < max_background_threads; i++) {
632 malloc_mutex_postfork_parent(tsdn,
633 &background_thread_info[i].mtx);
634 }
635 malloc_mutex_postfork_parent(tsdn, &background_thread_lock);
636}
637
638void
639background_thread_postfork_child(tsdn_t *tsdn) {
640 for (unsigned i = 0; i < max_background_threads; i++) {
641 malloc_mutex_postfork_child(tsdn,
642 &background_thread_info[i].mtx);
643 }
644 malloc_mutex_postfork_child(tsdn, &background_thread_lock);
645 if (!background_thread_enabled_at_fork) {
646 return;
647 }
648
649 /* Clear background_thread state (reset to disabled for child). */
650 malloc_mutex_lock(tsdn, &background_thread_lock);
651 n_background_threads = 0;
652 background_thread_enabled_set(tsdn, false);
653 for (unsigned i = 0; i < max_background_threads; i++) {
654 background_thread_info_t *info = &background_thread_info[i];
655 malloc_mutex_lock(tsdn, &info->mtx);
656 info->state = background_thread_stopped;
657 int ret = pthread_cond_init(&info->cond, NULL);
658 assert(ret == 0);
659 background_thread_info_init(tsdn, info);
660 malloc_mutex_unlock(tsdn, &info->mtx);
661 }
662 malloc_mutex_unlock(tsdn, &background_thread_lock);
663}
664
665bool
666background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) {
667 assert(config_stats);
668 malloc_mutex_lock(tsdn, &background_thread_lock);
669 if (!background_thread_enabled()) {
670 malloc_mutex_unlock(tsdn, &background_thread_lock);
671 return true;
672 }
673
674 nstime_init_zero(&stats->run_interval);
675 memset(&stats->max_counter_per_bg_thd, 0, sizeof(mutex_prof_data_t));
676
677 uint64_t num_runs = 0;
678 stats->num_threads = n_background_threads;
679 for (unsigned i = 0; i < max_background_threads; i++) {
680 background_thread_info_t *info = &background_thread_info[i];
681 if (malloc_mutex_trylock(tsdn, &info->mtx)) {
682 /*
683 * Each background thread run may take a long time;
684 * avoid waiting on the stats if the thread is active.
685 */
686 continue;
687 }
688 if (info->state != background_thread_stopped) {
689 num_runs += info->tot_n_runs;
690 nstime_add(&stats->run_interval, &info->tot_sleep_time);
691 malloc_mutex_prof_max_update(tsdn,
692 &stats->max_counter_per_bg_thd, &info->mtx);
693 }
694 malloc_mutex_unlock(tsdn, &info->mtx);
695 }
696 stats->num_runs = num_runs;
697 if (num_runs > 0) {
698 nstime_idivide(&stats->run_interval, num_runs);
699 }
700 malloc_mutex_unlock(tsdn, &background_thread_lock);
701
702 return false;
703}
704
705#undef BACKGROUND_THREAD_NPAGES_THRESHOLD
706#undef BILLION
707#undef BACKGROUND_THREAD_MIN_INTERVAL_NS
708
709#ifdef JEMALLOC_HAVE_DLSYM
710#include <dlfcn.h>
711#endif
712
713static bool
714pthread_create_fptr_init(void) {
715 if (pthread_create_fptr != NULL) {
716 return false;
717 }
718 /*
719 * Try the next symbol first, because 1) when use lazy_lock we have a
720 * wrapper for pthread_create; and 2) application may define its own
721 * wrapper as well (and can call malloc within the wrapper).
722 */
723#ifdef JEMALLOC_HAVE_DLSYM
724 pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create");
725#else
726 pthread_create_fptr = NULL;
727#endif
728 if (pthread_create_fptr == NULL) {
729 if (config_lazy_lock) {
730 malloc_write("<jemalloc>: Error in dlsym(RTLD_NEXT, "
731 "\"pthread_create\")\n");
732 abort();
733 } else {
734 /* Fall back to the default symbol. */
735 pthread_create_fptr = pthread_create;
736 }
737 }
738
739 return false;
740}
741
742/*
743 * When lazy lock is enabled, we need to make sure setting isthreaded before
744 * taking any background_thread locks. This is called early in ctl (instead of
745 * wait for the pthread_create calls to trigger) because the mutex is required
746 * before creating background threads.
747 */
748void
749background_thread_ctl_init(tsdn_t *tsdn) {
750 malloc_mutex_assert_not_owner(tsdn, &background_thread_lock);
751#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
752 pthread_create_fptr_init();
753 pthread_create_wrapper_init();
754#endif
755}
756
757#endif /* defined(JEMALLOC_BACKGROUND_THREAD) */
758
759bool
760background_thread_boot0(void) {
761 if (!have_background_thread && opt_background_thread) {
762 malloc_printf("<jemalloc>: option background_thread currently "
763 "supports pthread only\n");
764 return true;
765 }
766#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
767 if ((config_lazy_lock || opt_background_thread) &&
768 pthread_create_fptr_init()) {
769 return true;
770 }
771#endif
772 return false;
773}
774
775bool
776background_thread_boot1(tsdn_t *tsdn, base_t *base) {
777#ifdef JEMALLOC_BACKGROUND_THREAD
778 assert(have_background_thread);
779 assert(narenas_total_get() > 0);
780
781 if (opt_max_background_threads > MAX_BACKGROUND_THREAD_LIMIT) {
782 opt_max_background_threads = DEFAULT_NUM_BACKGROUND_THREAD;
783 }
784 max_background_threads = opt_max_background_threads;
785
786 background_thread_enabled_set(tsdn, opt_background_thread);
787 if (malloc_mutex_init(&background_thread_lock,
788 "background_thread_global",
789 WITNESS_RANK_BACKGROUND_THREAD_GLOBAL,
790 malloc_mutex_rank_exclusive)) {
791 return true;
792 }
793
794 background_thread_info = (background_thread_info_t *)base_alloc(tsdn,
795 base, opt_max_background_threads *
796 sizeof(background_thread_info_t), CACHELINE);
797 if (background_thread_info == NULL) {
798 return true;
799 }
800
801 for (unsigned i = 0; i < max_background_threads; i++) {
802 background_thread_info_t *info = &background_thread_info[i];
803 /* Thread mutex is rank_inclusive because of thread0. */
804 if (malloc_mutex_init(&info->mtx, "background_thread",
805 WITNESS_RANK_BACKGROUND_THREAD,
806 malloc_mutex_address_ordered)) {
807 return true;
808 }
809 if (pthread_cond_init(&info->cond, NULL)) {
810 return true;
811 }
812 malloc_mutex_lock(tsdn, &info->mtx);
813 info->state = background_thread_stopped;
814 background_thread_info_init(tsdn, info);
815 malloc_mutex_unlock(tsdn, &info->mtx);
816 }
817#endif
818
819 return false;
820}
diff --git a/examples/redis-unstable/deps/jemalloc/src/base.c b/examples/redis-unstable/deps/jemalloc/src/base.c
deleted file mode 100644
index 7f4d675..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/base.c
+++ /dev/null
@@ -1,529 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/extent_mmap.h"
6#include "jemalloc/internal/mutex.h"
7#include "jemalloc/internal/sz.h"
8
9/*
10 * In auto mode, arenas switch to huge pages for the base allocator on the
11 * second base block. a0 switches to thp on the 5th block (after 20 megabytes
12 * of metadata), since more metadata (e.g. rtree nodes) come from a0's base.
13 */
14
15#define BASE_AUTO_THP_THRESHOLD 2
16#define BASE_AUTO_THP_THRESHOLD_A0 5
17
18/******************************************************************************/
19/* Data. */
20
21static base_t *b0;
22
23metadata_thp_mode_t opt_metadata_thp = METADATA_THP_DEFAULT;
24
25const char *metadata_thp_mode_names[] = {
26 "disabled",
27 "auto",
28 "always"
29};
30
31/******************************************************************************/
32
33static inline bool
34metadata_thp_madvise(void) {
35 return (metadata_thp_enabled() &&
36 (init_system_thp_mode == thp_mode_default));
37}
38
39static void *
40base_map(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, size_t size) {
41 void *addr;
42 bool zero = true;
43 bool commit = true;
44
45 /* Use huge page sizes and alignment regardless of opt_metadata_thp. */
46 assert(size == HUGEPAGE_CEILING(size));
47 size_t alignment = HUGEPAGE;
48 if (ehooks_are_default(ehooks)) {
49 addr = extent_alloc_mmap(NULL, size, alignment, &zero, &commit);
50 if (have_madvise_huge && addr) {
51 pages_set_thp_state(addr, size);
52 }
53 } else {
54 addr = ehooks_alloc(tsdn, ehooks, NULL, size, alignment, &zero,
55 &commit);
56 }
57
58 return addr;
59}
60
61static void
62base_unmap(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, void *addr,
63 size_t size) {
64 /*
65 * Cascade through dalloc, decommit, purge_forced, and purge_lazy,
66 * stopping at first success. This cascade is performed for consistency
67 * with the cascade in extent_dalloc_wrapper() because an application's
68 * custom hooks may not support e.g. dalloc. This function is only ever
69 * called as a side effect of arena destruction, so although it might
70 * seem pointless to do anything besides dalloc here, the application
71 * may in fact want the end state of all associated virtual memory to be
72 * in some consistent-but-allocated state.
73 */
74 if (ehooks_are_default(ehooks)) {
75 if (!extent_dalloc_mmap(addr, size)) {
76 goto label_done;
77 }
78 if (!pages_decommit(addr, size)) {
79 goto label_done;
80 }
81 if (!pages_purge_forced(addr, size)) {
82 goto label_done;
83 }
84 if (!pages_purge_lazy(addr, size)) {
85 goto label_done;
86 }
87 /* Nothing worked. This should never happen. */
88 not_reached();
89 } else {
90 if (!ehooks_dalloc(tsdn, ehooks, addr, size, true)) {
91 goto label_done;
92 }
93 if (!ehooks_decommit(tsdn, ehooks, addr, size, 0, size)) {
94 goto label_done;
95 }
96 if (!ehooks_purge_forced(tsdn, ehooks, addr, size, 0, size)) {
97 goto label_done;
98 }
99 if (!ehooks_purge_lazy(tsdn, ehooks, addr, size, 0, size)) {
100 goto label_done;
101 }
102 /* Nothing worked. That's the application's problem. */
103 }
104label_done:
105 if (metadata_thp_madvise()) {
106 /* Set NOHUGEPAGE after unmap to avoid kernel defrag. */
107 assert(((uintptr_t)addr & HUGEPAGE_MASK) == 0 &&
108 (size & HUGEPAGE_MASK) == 0);
109 pages_nohuge(addr, size);
110 }
111}
112
113static void
114base_edata_init(size_t *extent_sn_next, edata_t *edata, void *addr,
115 size_t size) {
116 size_t sn;
117
118 sn = *extent_sn_next;
119 (*extent_sn_next)++;
120
121 edata_binit(edata, addr, size, sn);
122}
123
124static size_t
125base_get_num_blocks(base_t *base, bool with_new_block) {
126 base_block_t *b = base->blocks;
127 assert(b != NULL);
128
129 size_t n_blocks = with_new_block ? 2 : 1;
130 while (b->next != NULL) {
131 n_blocks++;
132 b = b->next;
133 }
134
135 return n_blocks;
136}
137
138static void
139base_auto_thp_switch(tsdn_t *tsdn, base_t *base) {
140 assert(opt_metadata_thp == metadata_thp_auto);
141 malloc_mutex_assert_owner(tsdn, &base->mtx);
142 if (base->auto_thp_switched) {
143 return;
144 }
145 /* Called when adding a new block. */
146 bool should_switch;
147 if (base_ind_get(base) != 0) {
148 should_switch = (base_get_num_blocks(base, true) ==
149 BASE_AUTO_THP_THRESHOLD);
150 } else {
151 should_switch = (base_get_num_blocks(base, true) ==
152 BASE_AUTO_THP_THRESHOLD_A0);
153 }
154 if (!should_switch) {
155 return;
156 }
157
158 base->auto_thp_switched = true;
159 assert(!config_stats || base->n_thp == 0);
160 /* Make the initial blocks THP lazily. */
161 base_block_t *block = base->blocks;
162 while (block != NULL) {
163 assert((block->size & HUGEPAGE_MASK) == 0);
164 pages_huge(block, block->size);
165 if (config_stats) {
166 base->n_thp += HUGEPAGE_CEILING(block->size -
167 edata_bsize_get(&block->edata)) >> LG_HUGEPAGE;
168 }
169 block = block->next;
170 assert(block == NULL || (base_ind_get(base) == 0));
171 }
172}
173
174static void *
175base_extent_bump_alloc_helper(edata_t *edata, size_t *gap_size, size_t size,
176 size_t alignment) {
177 void *ret;
178
179 assert(alignment == ALIGNMENT_CEILING(alignment, QUANTUM));
180 assert(size == ALIGNMENT_CEILING(size, alignment));
181
182 *gap_size = ALIGNMENT_CEILING((uintptr_t)edata_addr_get(edata),
183 alignment) - (uintptr_t)edata_addr_get(edata);
184 ret = (void *)((uintptr_t)edata_addr_get(edata) + *gap_size);
185 assert(edata_bsize_get(edata) >= *gap_size + size);
186 edata_binit(edata, (void *)((uintptr_t)edata_addr_get(edata) +
187 *gap_size + size), edata_bsize_get(edata) - *gap_size - size,
188 edata_sn_get(edata));
189 return ret;
190}
191
192static void
193base_extent_bump_alloc_post(base_t *base, edata_t *edata, size_t gap_size,
194 void *addr, size_t size) {
195 if (edata_bsize_get(edata) > 0) {
196 /*
197 * Compute the index for the largest size class that does not
198 * exceed extent's size.
199 */
200 szind_t index_floor =
201 sz_size2index(edata_bsize_get(edata) + 1) - 1;
202 edata_heap_insert(&base->avail[index_floor], edata);
203 }
204
205 if (config_stats) {
206 base->allocated += size;
207 /*
208 * Add one PAGE to base_resident for every page boundary that is
209 * crossed by the new allocation. Adjust n_thp similarly when
210 * metadata_thp is enabled.
211 */
212 base->resident += PAGE_CEILING((uintptr_t)addr + size) -
213 PAGE_CEILING((uintptr_t)addr - gap_size);
214 assert(base->allocated <= base->resident);
215 assert(base->resident <= base->mapped);
216 if (metadata_thp_madvise() && (opt_metadata_thp ==
217 metadata_thp_always || base->auto_thp_switched)) {
218 base->n_thp += (HUGEPAGE_CEILING((uintptr_t)addr + size)
219 - HUGEPAGE_CEILING((uintptr_t)addr - gap_size)) >>
220 LG_HUGEPAGE;
221 assert(base->mapped >= base->n_thp << LG_HUGEPAGE);
222 }
223 }
224}
225
226static void *
227base_extent_bump_alloc(base_t *base, edata_t *edata, size_t size,
228 size_t alignment) {
229 void *ret;
230 size_t gap_size;
231
232 ret = base_extent_bump_alloc_helper(edata, &gap_size, size, alignment);
233 base_extent_bump_alloc_post(base, edata, gap_size, ret, size);
234 return ret;
235}
236
237/*
238 * Allocate a block of virtual memory that is large enough to start with a
239 * base_block_t header, followed by an object of specified size and alignment.
240 * On success a pointer to the initialized base_block_t header is returned.
241 */
242static base_block_t *
243base_block_alloc(tsdn_t *tsdn, base_t *base, ehooks_t *ehooks, unsigned ind,
244 pszind_t *pind_last, size_t *extent_sn_next, size_t size,
245 size_t alignment) {
246 alignment = ALIGNMENT_CEILING(alignment, QUANTUM);
247 size_t usize = ALIGNMENT_CEILING(size, alignment);
248 size_t header_size = sizeof(base_block_t);
249 size_t gap_size = ALIGNMENT_CEILING(header_size, alignment) -
250 header_size;
251 /*
252 * Create increasingly larger blocks in order to limit the total number
253 * of disjoint virtual memory ranges. Choose the next size in the page
254 * size class series (skipping size classes that are not a multiple of
255 * HUGEPAGE), or a size large enough to satisfy the requested size and
256 * alignment, whichever is larger.
257 */
258 size_t min_block_size = HUGEPAGE_CEILING(sz_psz2u(header_size + gap_size
259 + usize));
260 pszind_t pind_next = (*pind_last + 1 < sz_psz2ind(SC_LARGE_MAXCLASS)) ?
261 *pind_last + 1 : *pind_last;
262 size_t next_block_size = HUGEPAGE_CEILING(sz_pind2sz(pind_next));
263 size_t block_size = (min_block_size > next_block_size) ? min_block_size
264 : next_block_size;
265 base_block_t *block = (base_block_t *)base_map(tsdn, ehooks, ind,
266 block_size);
267 if (block == NULL) {
268 return NULL;
269 }
270
271 if (metadata_thp_madvise()) {
272 void *addr = (void *)block;
273 assert(((uintptr_t)addr & HUGEPAGE_MASK) == 0 &&
274 (block_size & HUGEPAGE_MASK) == 0);
275 if (opt_metadata_thp == metadata_thp_always) {
276 pages_huge(addr, block_size);
277 } else if (opt_metadata_thp == metadata_thp_auto &&
278 base != NULL) {
279 /* base != NULL indicates this is not a new base. */
280 malloc_mutex_lock(tsdn, &base->mtx);
281 base_auto_thp_switch(tsdn, base);
282 if (base->auto_thp_switched) {
283 pages_huge(addr, block_size);
284 }
285 malloc_mutex_unlock(tsdn, &base->mtx);
286 }
287 }
288
289 *pind_last = sz_psz2ind(block_size);
290 block->size = block_size;
291 block->next = NULL;
292 assert(block_size >= header_size);
293 base_edata_init(extent_sn_next, &block->edata,
294 (void *)((uintptr_t)block + header_size), block_size - header_size);
295 return block;
296}
297
298/*
299 * Allocate an extent that is at least as large as specified size, with
300 * specified alignment.
301 */
302static edata_t *
303base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) {
304 malloc_mutex_assert_owner(tsdn, &base->mtx);
305
306 ehooks_t *ehooks = base_ehooks_get_for_metadata(base);
307 /*
308 * Drop mutex during base_block_alloc(), because an extent hook will be
309 * called.
310 */
311 malloc_mutex_unlock(tsdn, &base->mtx);
312 base_block_t *block = base_block_alloc(tsdn, base, ehooks,
313 base_ind_get(base), &base->pind_last, &base->extent_sn_next, size,
314 alignment);
315 malloc_mutex_lock(tsdn, &base->mtx);
316 if (block == NULL) {
317 return NULL;
318 }
319 block->next = base->blocks;
320 base->blocks = block;
321 if (config_stats) {
322 base->allocated += sizeof(base_block_t);
323 base->resident += PAGE_CEILING(sizeof(base_block_t));
324 base->mapped += block->size;
325 if (metadata_thp_madvise() &&
326 !(opt_metadata_thp == metadata_thp_auto
327 && !base->auto_thp_switched)) {
328 assert(base->n_thp > 0);
329 base->n_thp += HUGEPAGE_CEILING(sizeof(base_block_t)) >>
330 LG_HUGEPAGE;
331 }
332 assert(base->allocated <= base->resident);
333 assert(base->resident <= base->mapped);
334 assert(base->n_thp << LG_HUGEPAGE <= base->mapped);
335 }
336 return &block->edata;
337}
338
339base_t *
340b0get(void) {
341 return b0;
342}
343
344base_t *
345base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks,
346 bool metadata_use_hooks) {
347 pszind_t pind_last = 0;
348 size_t extent_sn_next = 0;
349
350 /*
351 * The base will contain the ehooks eventually, but it itself is
352 * allocated using them. So we use some stack ehooks to bootstrap its
353 * memory, and then initialize the ehooks within the base_t.
354 */
355 ehooks_t fake_ehooks;
356 ehooks_init(&fake_ehooks, metadata_use_hooks ?
357 (extent_hooks_t *)extent_hooks :
358 (extent_hooks_t *)&ehooks_default_extent_hooks, ind);
359
360 base_block_t *block = base_block_alloc(tsdn, NULL, &fake_ehooks, ind,
361 &pind_last, &extent_sn_next, sizeof(base_t), QUANTUM);
362 if (block == NULL) {
363 return NULL;
364 }
365
366 size_t gap_size;
367 size_t base_alignment = CACHELINE;
368 size_t base_size = ALIGNMENT_CEILING(sizeof(base_t), base_alignment);
369 base_t *base = (base_t *)base_extent_bump_alloc_helper(&block->edata,
370 &gap_size, base_size, base_alignment);
371 ehooks_init(&base->ehooks, (extent_hooks_t *)extent_hooks, ind);
372 ehooks_init(&base->ehooks_base, metadata_use_hooks ?
373 (extent_hooks_t *)extent_hooks :
374 (extent_hooks_t *)&ehooks_default_extent_hooks, ind);
375 if (malloc_mutex_init(&base->mtx, "base", WITNESS_RANK_BASE,
376 malloc_mutex_rank_exclusive)) {
377 base_unmap(tsdn, &fake_ehooks, ind, block, block->size);
378 return NULL;
379 }
380 base->pind_last = pind_last;
381 base->extent_sn_next = extent_sn_next;
382 base->blocks = block;
383 base->auto_thp_switched = false;
384 for (szind_t i = 0; i < SC_NSIZES; i++) {
385 edata_heap_new(&base->avail[i]);
386 }
387 if (config_stats) {
388 base->allocated = sizeof(base_block_t);
389 base->resident = PAGE_CEILING(sizeof(base_block_t));
390 base->mapped = block->size;
391 base->n_thp = (opt_metadata_thp == metadata_thp_always) &&
392 metadata_thp_madvise() ? HUGEPAGE_CEILING(sizeof(base_block_t))
393 >> LG_HUGEPAGE : 0;
394 assert(base->allocated <= base->resident);
395 assert(base->resident <= base->mapped);
396 assert(base->n_thp << LG_HUGEPAGE <= base->mapped);
397 }
398 base_extent_bump_alloc_post(base, &block->edata, gap_size, base,
399 base_size);
400
401 return base;
402}
403
404void
405base_delete(tsdn_t *tsdn, base_t *base) {
406 ehooks_t *ehooks = base_ehooks_get_for_metadata(base);
407 base_block_t *next = base->blocks;
408 do {
409 base_block_t *block = next;
410 next = block->next;
411 base_unmap(tsdn, ehooks, base_ind_get(base), block,
412 block->size);
413 } while (next != NULL);
414}
415
416ehooks_t *
417base_ehooks_get(base_t *base) {
418 return &base->ehooks;
419}
420
421ehooks_t *
422base_ehooks_get_for_metadata(base_t *base) {
423 return &base->ehooks_base;
424}
425
426extent_hooks_t *
427base_extent_hooks_set(base_t *base, extent_hooks_t *extent_hooks) {
428 extent_hooks_t *old_extent_hooks =
429 ehooks_get_extent_hooks_ptr(&base->ehooks);
430 ehooks_init(&base->ehooks, extent_hooks, ehooks_ind_get(&base->ehooks));
431 return old_extent_hooks;
432}
433
434static void *
435base_alloc_impl(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment,
436 size_t *esn) {
437 alignment = QUANTUM_CEILING(alignment);
438 size_t usize = ALIGNMENT_CEILING(size, alignment);
439 size_t asize = usize + alignment - QUANTUM;
440
441 edata_t *edata = NULL;
442 malloc_mutex_lock(tsdn, &base->mtx);
443 for (szind_t i = sz_size2index(asize); i < SC_NSIZES; i++) {
444 edata = edata_heap_remove_first(&base->avail[i]);
445 if (edata != NULL) {
446 /* Use existing space. */
447 break;
448 }
449 }
450 if (edata == NULL) {
451 /* Try to allocate more space. */
452 edata = base_extent_alloc(tsdn, base, usize, alignment);
453 }
454 void *ret;
455 if (edata == NULL) {
456 ret = NULL;
457 goto label_return;
458 }
459
460 ret = base_extent_bump_alloc(base, edata, usize, alignment);
461 if (esn != NULL) {
462 *esn = (size_t)edata_sn_get(edata);
463 }
464label_return:
465 malloc_mutex_unlock(tsdn, &base->mtx);
466 return ret;
467}
468
469/*
470 * base_alloc() returns zeroed memory, which is always demand-zeroed for the
471 * auto arenas, in order to make multi-page sparse data structures such as radix
472 * tree nodes efficient with respect to physical memory usage. Upon success a
473 * pointer to at least size bytes with specified alignment is returned. Note
474 * that size is rounded up to the nearest multiple of alignment to avoid false
475 * sharing.
476 */
477void *
478base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) {
479 return base_alloc_impl(tsdn, base, size, alignment, NULL);
480}
481
482edata_t *
483base_alloc_edata(tsdn_t *tsdn, base_t *base) {
484 size_t esn;
485 edata_t *edata = base_alloc_impl(tsdn, base, sizeof(edata_t),
486 EDATA_ALIGNMENT, &esn);
487 if (edata == NULL) {
488 return NULL;
489 }
490 edata_esn_set(edata, esn);
491 return edata;
492}
493
494void
495base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated, size_t *resident,
496 size_t *mapped, size_t *n_thp) {
497 cassert(config_stats);
498
499 malloc_mutex_lock(tsdn, &base->mtx);
500 assert(base->allocated <= base->resident);
501 assert(base->resident <= base->mapped);
502 *allocated = base->allocated;
503 *resident = base->resident;
504 *mapped = base->mapped;
505 *n_thp = base->n_thp;
506 malloc_mutex_unlock(tsdn, &base->mtx);
507}
508
509void
510base_prefork(tsdn_t *tsdn, base_t *base) {
511 malloc_mutex_prefork(tsdn, &base->mtx);
512}
513
514void
515base_postfork_parent(tsdn_t *tsdn, base_t *base) {
516 malloc_mutex_postfork_parent(tsdn, &base->mtx);
517}
518
519void
520base_postfork_child(tsdn_t *tsdn, base_t *base) {
521 malloc_mutex_postfork_child(tsdn, &base->mtx);
522}
523
524bool
525base_boot(tsdn_t *tsdn) {
526 b0 = base_new(tsdn, 0, (extent_hooks_t *)&ehooks_default_extent_hooks,
527 /* metadata_use_hooks */ true);
528 return (b0 == NULL);
529}
diff --git a/examples/redis-unstable/deps/jemalloc/src/bin.c b/examples/redis-unstable/deps/jemalloc/src/bin.c
deleted file mode 100644
index fa20458..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/bin.c
+++ /dev/null
@@ -1,69 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/bin.h"
6#include "jemalloc/internal/sc.h"
7#include "jemalloc/internal/witness.h"
8
9bool
10bin_update_shard_size(unsigned bin_shard_sizes[SC_NBINS], size_t start_size,
11 size_t end_size, size_t nshards) {
12 if (nshards > BIN_SHARDS_MAX || nshards == 0) {
13 return true;
14 }
15
16 if (start_size > SC_SMALL_MAXCLASS) {
17 return false;
18 }
19 if (end_size > SC_SMALL_MAXCLASS) {
20 end_size = SC_SMALL_MAXCLASS;
21 }
22
23 /* Compute the index since this may happen before sz init. */
24 szind_t ind1 = sz_size2index_compute(start_size);
25 szind_t ind2 = sz_size2index_compute(end_size);
26 for (unsigned i = ind1; i <= ind2; i++) {
27 bin_shard_sizes[i] = (unsigned)nshards;
28 }
29
30 return false;
31}
32
33void
34bin_shard_sizes_boot(unsigned bin_shard_sizes[SC_NBINS]) {
35 /* Load the default number of shards. */
36 for (unsigned i = 0; i < SC_NBINS; i++) {
37 bin_shard_sizes[i] = N_BIN_SHARDS_DEFAULT;
38 }
39}
40
41bool
42bin_init(bin_t *bin) {
43 if (malloc_mutex_init(&bin->lock, "bin", WITNESS_RANK_BIN,
44 malloc_mutex_rank_exclusive)) {
45 return true;
46 }
47 bin->slabcur = NULL;
48 edata_heap_new(&bin->slabs_nonfull);
49 edata_list_active_init(&bin->slabs_full);
50 if (config_stats) {
51 memset(&bin->stats, 0, sizeof(bin_stats_t));
52 }
53 return false;
54}
55
56void
57bin_prefork(tsdn_t *tsdn, bin_t *bin) {
58 malloc_mutex_prefork(tsdn, &bin->lock);
59}
60
61void
62bin_postfork_parent(tsdn_t *tsdn, bin_t *bin) {
63 malloc_mutex_postfork_parent(tsdn, &bin->lock);
64}
65
66void
67bin_postfork_child(tsdn_t *tsdn, bin_t *bin) {
68 malloc_mutex_postfork_child(tsdn, &bin->lock);
69}
diff --git a/examples/redis-unstable/deps/jemalloc/src/bin_info.c b/examples/redis-unstable/deps/jemalloc/src/bin_info.c
deleted file mode 100644
index 8629ef8..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/bin_info.c
+++ /dev/null
@@ -1,30 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/bin_info.h"
5
6bin_info_t bin_infos[SC_NBINS];
7
8static void
9bin_infos_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
10 bin_info_t infos[SC_NBINS]) {
11 for (unsigned i = 0; i < SC_NBINS; i++) {
12 bin_info_t *bin_info = &infos[i];
13 sc_t *sc = &sc_data->sc[i];
14 bin_info->reg_size = ((size_t)1U << sc->lg_base)
15 + ((size_t)sc->ndelta << sc->lg_delta);
16 bin_info->slab_size = (sc->pgs << LG_PAGE);
17 bin_info->nregs =
18 (uint32_t)(bin_info->slab_size / bin_info->reg_size);
19 bin_info->n_shards = bin_shard_sizes[i];
20 bitmap_info_t bitmap_info = BITMAP_INFO_INITIALIZER(
21 bin_info->nregs);
22 bin_info->bitmap_info = bitmap_info;
23 }
24}
25
26void
27bin_info_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) {
28 assert(sc_data->initialized);
29 bin_infos_init(sc_data, bin_shard_sizes, bin_infos);
30}
diff --git a/examples/redis-unstable/deps/jemalloc/src/bitmap.c b/examples/redis-unstable/deps/jemalloc/src/bitmap.c
deleted file mode 100644
index 0ccedc5..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/bitmap.c
+++ /dev/null
@@ -1,120 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5
6/******************************************************************************/
7
8#ifdef BITMAP_USE_TREE
9
10void
11bitmap_info_init(bitmap_info_t *binfo, size_t nbits) {
12 unsigned i;
13 size_t group_count;
14
15 assert(nbits > 0);
16 assert(nbits <= (ZU(1) << LG_BITMAP_MAXBITS));
17
18 /*
19 * Compute the number of groups necessary to store nbits bits, and
20 * progressively work upward through the levels until reaching a level
21 * that requires only one group.
22 */
23 binfo->levels[0].group_offset = 0;
24 group_count = BITMAP_BITS2GROUPS(nbits);
25 for (i = 1; group_count > 1; i++) {
26 assert(i < BITMAP_MAX_LEVELS);
27 binfo->levels[i].group_offset = binfo->levels[i-1].group_offset
28 + group_count;
29 group_count = BITMAP_BITS2GROUPS(group_count);
30 }
31 binfo->levels[i].group_offset = binfo->levels[i-1].group_offset
32 + group_count;
33 assert(binfo->levels[i].group_offset <= BITMAP_GROUPS_MAX);
34 binfo->nlevels = i;
35 binfo->nbits = nbits;
36}
37
38static size_t
39bitmap_info_ngroups(const bitmap_info_t *binfo) {
40 return binfo->levels[binfo->nlevels].group_offset;
41}
42
43void
44bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill) {
45 size_t extra;
46 unsigned i;
47
48 /*
49 * Bits are actually inverted with regard to the external bitmap
50 * interface.
51 */
52
53 if (fill) {
54 /* The "filled" bitmap starts out with all 0 bits. */
55 memset(bitmap, 0, bitmap_size(binfo));
56 return;
57 }
58
59 /*
60 * The "empty" bitmap starts out with all 1 bits, except for trailing
61 * unused bits (if any). Note that each group uses bit 0 to correspond
62 * to the first logical bit in the group, so extra bits are the most
63 * significant bits of the last group.
64 */
65 memset(bitmap, 0xffU, bitmap_size(binfo));
66 extra = (BITMAP_GROUP_NBITS - (binfo->nbits & BITMAP_GROUP_NBITS_MASK))
67 & BITMAP_GROUP_NBITS_MASK;
68 if (extra != 0) {
69 bitmap[binfo->levels[1].group_offset - 1] >>= extra;
70 }
71 for (i = 1; i < binfo->nlevels; i++) {
72 size_t group_count = binfo->levels[i].group_offset -
73 binfo->levels[i-1].group_offset;
74 extra = (BITMAP_GROUP_NBITS - (group_count &
75 BITMAP_GROUP_NBITS_MASK)) & BITMAP_GROUP_NBITS_MASK;
76 if (extra != 0) {
77 bitmap[binfo->levels[i+1].group_offset - 1] >>= extra;
78 }
79 }
80}
81
82#else /* BITMAP_USE_TREE */
83
84void
85bitmap_info_init(bitmap_info_t *binfo, size_t nbits) {
86 assert(nbits > 0);
87 assert(nbits <= (ZU(1) << LG_BITMAP_MAXBITS));
88
89 binfo->ngroups = BITMAP_BITS2GROUPS(nbits);
90 binfo->nbits = nbits;
91}
92
93static size_t
94bitmap_info_ngroups(const bitmap_info_t *binfo) {
95 return binfo->ngroups;
96}
97
98void
99bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill) {
100 size_t extra;
101
102 if (fill) {
103 memset(bitmap, 0, bitmap_size(binfo));
104 return;
105 }
106
107 memset(bitmap, 0xffU, bitmap_size(binfo));
108 extra = (BITMAP_GROUP_NBITS - (binfo->nbits & BITMAP_GROUP_NBITS_MASK))
109 & BITMAP_GROUP_NBITS_MASK;
110 if (extra != 0) {
111 bitmap[binfo->ngroups - 1] >>= extra;
112 }
113}
114
115#endif /* BITMAP_USE_TREE */
116
117size_t
118bitmap_size(const bitmap_info_t *binfo) {
119 return (bitmap_info_ngroups(binfo) << LG_SIZEOF_BITMAP);
120}
diff --git a/examples/redis-unstable/deps/jemalloc/src/buf_writer.c b/examples/redis-unstable/deps/jemalloc/src/buf_writer.c
deleted file mode 100644
index 7c6f794..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/buf_writer.c
+++ /dev/null
@@ -1,144 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/buf_writer.h"
5#include "jemalloc/internal/malloc_io.h"
6
7static void *
8buf_writer_allocate_internal_buf(tsdn_t *tsdn, size_t buf_len) {
9#ifdef JEMALLOC_JET
10 if (buf_len > SC_LARGE_MAXCLASS) {
11 return NULL;
12 }
13#else
14 assert(buf_len <= SC_LARGE_MAXCLASS);
15#endif
16 return iallocztm(tsdn, buf_len, sz_size2index(buf_len), false, NULL,
17 true, arena_get(tsdn, 0, false), true);
18}
19
20static void
21buf_writer_free_internal_buf(tsdn_t *tsdn, void *buf) {
22 if (buf != NULL) {
23 idalloctm(tsdn, buf, NULL, NULL, true, true);
24 }
25}
26
27static void
28buf_writer_assert(buf_writer_t *buf_writer) {
29 assert(buf_writer != NULL);
30 assert(buf_writer->write_cb != NULL);
31 if (buf_writer->buf != NULL) {
32 assert(buf_writer->buf_size > 0);
33 } else {
34 assert(buf_writer->buf_size == 0);
35 assert(buf_writer->internal_buf);
36 }
37 assert(buf_writer->buf_end <= buf_writer->buf_size);
38}
39
40bool
41buf_writer_init(tsdn_t *tsdn, buf_writer_t *buf_writer, write_cb_t *write_cb,
42 void *cbopaque, char *buf, size_t buf_len) {
43 if (write_cb != NULL) {
44 buf_writer->write_cb = write_cb;
45 } else {
46 buf_writer->write_cb = je_malloc_message != NULL ?
47 je_malloc_message : wrtmessage;
48 }
49 buf_writer->cbopaque = cbopaque;
50 assert(buf_len >= 2);
51 if (buf != NULL) {
52 buf_writer->buf = buf;
53 buf_writer->internal_buf = false;
54 } else {
55 buf_writer->buf = buf_writer_allocate_internal_buf(tsdn,
56 buf_len);
57 buf_writer->internal_buf = true;
58 }
59 if (buf_writer->buf != NULL) {
60 buf_writer->buf_size = buf_len - 1; /* Allowing for '\0'. */
61 } else {
62 buf_writer->buf_size = 0;
63 }
64 buf_writer->buf_end = 0;
65 buf_writer_assert(buf_writer);
66 return buf_writer->buf == NULL;
67}
68
69void
70buf_writer_flush(buf_writer_t *buf_writer) {
71 buf_writer_assert(buf_writer);
72 if (buf_writer->buf == NULL) {
73 return;
74 }
75 buf_writer->buf[buf_writer->buf_end] = '\0';
76 buf_writer->write_cb(buf_writer->cbopaque, buf_writer->buf);
77 buf_writer->buf_end = 0;
78 buf_writer_assert(buf_writer);
79}
80
81void
82buf_writer_cb(void *buf_writer_arg, const char *s) {
83 buf_writer_t *buf_writer = (buf_writer_t *)buf_writer_arg;
84 buf_writer_assert(buf_writer);
85 if (buf_writer->buf == NULL) {
86 buf_writer->write_cb(buf_writer->cbopaque, s);
87 return;
88 }
89 size_t i, slen, n;
90 for (i = 0, slen = strlen(s); i < slen; i += n) {
91 if (buf_writer->buf_end == buf_writer->buf_size) {
92 buf_writer_flush(buf_writer);
93 }
94 size_t s_remain = slen - i;
95 size_t buf_remain = buf_writer->buf_size - buf_writer->buf_end;
96 n = s_remain < buf_remain ? s_remain : buf_remain;
97 memcpy(buf_writer->buf + buf_writer->buf_end, s + i, n);
98 buf_writer->buf_end += n;
99 buf_writer_assert(buf_writer);
100 }
101 assert(i == slen);
102}
103
104void
105buf_writer_terminate(tsdn_t *tsdn, buf_writer_t *buf_writer) {
106 buf_writer_assert(buf_writer);
107 buf_writer_flush(buf_writer);
108 if (buf_writer->internal_buf) {
109 buf_writer_free_internal_buf(tsdn, buf_writer->buf);
110 }
111}
112
113void
114buf_writer_pipe(buf_writer_t *buf_writer, read_cb_t *read_cb,
115 void *read_cbopaque) {
116 /*
117 * A tiny local buffer in case the buffered writer failed to allocate
118 * at init.
119 */
120 static char backup_buf[16];
121 static buf_writer_t backup_buf_writer;
122
123 buf_writer_assert(buf_writer);
124 assert(read_cb != NULL);
125 if (buf_writer->buf == NULL) {
126 buf_writer_init(TSDN_NULL, &backup_buf_writer,
127 buf_writer->write_cb, buf_writer->cbopaque, backup_buf,
128 sizeof(backup_buf));
129 buf_writer = &backup_buf_writer;
130 }
131 assert(buf_writer->buf != NULL);
132 ssize_t nread = 0;
133 do {
134 buf_writer->buf_end += nread;
135 buf_writer_assert(buf_writer);
136 if (buf_writer->buf_end == buf_writer->buf_size) {
137 buf_writer_flush(buf_writer);
138 }
139 nread = read_cb(read_cbopaque,
140 buf_writer->buf + buf_writer->buf_end,
141 buf_writer->buf_size - buf_writer->buf_end);
142 } while (nread > 0);
143 buf_writer_flush(buf_writer);
144}
diff --git a/examples/redis-unstable/deps/jemalloc/src/cache_bin.c b/examples/redis-unstable/deps/jemalloc/src/cache_bin.c
deleted file mode 100644
index 9ae072a..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/cache_bin.c
+++ /dev/null
@@ -1,99 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/bit_util.h"
5#include "jemalloc/internal/cache_bin.h"
6#include "jemalloc/internal/safety_check.h"
7
8void
9cache_bin_info_init(cache_bin_info_t *info,
10 cache_bin_sz_t ncached_max) {
11 assert(ncached_max <= CACHE_BIN_NCACHED_MAX);
12 size_t stack_size = (size_t)ncached_max * sizeof(void *);
13 assert(stack_size < ((size_t)1 << (sizeof(cache_bin_sz_t) * 8)));
14 info->ncached_max = (cache_bin_sz_t)ncached_max;
15}
16
17void
18cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos,
19 size_t *size, size_t *alignment) {
20 /* For the total bin stack region (per tcache), reserve 2 more slots so
21 * that
22 * 1) the empty position can be safely read on the fast path before
23 * checking "is_empty"; and
24 * 2) the cur_ptr can go beyond the empty position by 1 step safely on
25 * the fast path (i.e. no overflow).
26 */
27 *size = sizeof(void *) * 2;
28 for (szind_t i = 0; i < ninfos; i++) {
29 assert(infos[i].ncached_max > 0);
30 *size += infos[i].ncached_max * sizeof(void *);
31 }
32
33 /*
34 * Align to at least PAGE, to minimize the # of TLBs needed by the
35 * smaller sizes; also helps if the larger sizes don't get used at all.
36 */
37 *alignment = PAGE;
38}
39
40void
41cache_bin_preincrement(cache_bin_info_t *infos, szind_t ninfos, void *alloc,
42 size_t *cur_offset) {
43 if (config_debug) {
44 size_t computed_size;
45 size_t computed_alignment;
46
47 /* Pointer should be as aligned as we asked for. */
48 cache_bin_info_compute_alloc(infos, ninfos, &computed_size,
49 &computed_alignment);
50 assert(((uintptr_t)alloc & (computed_alignment - 1)) == 0);
51 }
52
53 *(uintptr_t *)((uintptr_t)alloc + *cur_offset) =
54 cache_bin_preceding_junk;
55 *cur_offset += sizeof(void *);
56}
57
58void
59cache_bin_postincrement(cache_bin_info_t *infos, szind_t ninfos, void *alloc,
60 size_t *cur_offset) {
61 *(uintptr_t *)((uintptr_t)alloc + *cur_offset) =
62 cache_bin_trailing_junk;
63 *cur_offset += sizeof(void *);
64}
65
66void
67cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc,
68 size_t *cur_offset) {
69 /*
70 * The full_position points to the lowest available space. Allocations
71 * will access the slots toward higher addresses (for the benefit of
72 * adjacent prefetch).
73 */
74 void *stack_cur = (void *)((uintptr_t)alloc + *cur_offset);
75 void *full_position = stack_cur;
76 uint16_t bin_stack_size = info->ncached_max * sizeof(void *);
77
78 *cur_offset += bin_stack_size;
79 void *empty_position = (void *)((uintptr_t)alloc + *cur_offset);
80
81 /* Init to the empty position. */
82 bin->stack_head = (void **)empty_position;
83 bin->low_bits_low_water = (uint16_t)(uintptr_t)bin->stack_head;
84 bin->low_bits_full = (uint16_t)(uintptr_t)full_position;
85 bin->low_bits_empty = (uint16_t)(uintptr_t)empty_position;
86 cache_bin_sz_t free_spots = cache_bin_diff(bin,
87 bin->low_bits_full, (uint16_t)(uintptr_t)bin->stack_head,
88 /* racy */ false);
89 assert(free_spots == bin_stack_size);
90 assert(cache_bin_ncached_get_local(bin, info) == 0);
91 assert(cache_bin_empty_position_get(bin) == empty_position);
92
93 assert(bin_stack_size > 0 || empty_position == full_position);
94}
95
96bool
97cache_bin_still_zero_initialized(cache_bin_t *bin) {
98 return bin->stack_head == NULL;
99}
diff --git a/examples/redis-unstable/deps/jemalloc/src/ckh.c b/examples/redis-unstable/deps/jemalloc/src/ckh.c
deleted file mode 100644
index 8db4319..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/ckh.c
+++ /dev/null
@@ -1,569 +0,0 @@
1/*
2 *******************************************************************************
3 * Implementation of (2^1+,2) cuckoo hashing, where 2^1+ indicates that each
4 * hash bucket contains 2^n cells, for n >= 1, and 2 indicates that two hash
5 * functions are employed. The original cuckoo hashing algorithm was described
6 * in:
7 *
8 * Pagh, R., F.F. Rodler (2004) Cuckoo Hashing. Journal of Algorithms
9 * 51(2):122-144.
10 *
11 * Generalization of cuckoo hashing was discussed in:
12 *
13 * Erlingsson, U., M. Manasse, F. McSherry (2006) A cool and practical
14 * alternative to traditional hash tables. In Proceedings of the 7th
15 * Workshop on Distributed Data and Structures (WDAS'06), Santa Clara, CA,
16 * January 2006.
17 *
18 * This implementation uses precisely two hash functions because that is the
19 * fewest that can work, and supporting multiple hashes is an implementation
20 * burden. Here is a reproduction of Figure 1 from Erlingsson et al. (2006)
21 * that shows approximate expected maximum load factors for various
22 * configurations:
23 *
24 * | #cells/bucket |
25 * #hashes | 1 | 2 | 4 | 8 |
26 * --------+-------+-------+-------+-------+
27 * 1 | 0.006 | 0.006 | 0.03 | 0.12 |
28 * 2 | 0.49 | 0.86 |>0.93< |>0.96< |
29 * 3 | 0.91 | 0.97 | 0.98 | 0.999 |
30 * 4 | 0.97 | 0.99 | 0.999 | |
31 *
32 * The number of cells per bucket is chosen such that a bucket fits in one cache
33 * line. So, on 32- and 64-bit systems, we use (8,2) and (4,2) cuckoo hashing,
34 * respectively.
35 *
36 ******************************************************************************/
37#include "jemalloc/internal/jemalloc_preamble.h"
38
39#include "jemalloc/internal/ckh.h"
40
41#include "jemalloc/internal/jemalloc_internal_includes.h"
42
43#include "jemalloc/internal/assert.h"
44#include "jemalloc/internal/hash.h"
45#include "jemalloc/internal/malloc_io.h"
46#include "jemalloc/internal/prng.h"
47#include "jemalloc/internal/util.h"
48
49/******************************************************************************/
50/* Function prototypes for non-inline static functions. */
51
52static bool ckh_grow(tsd_t *tsd, ckh_t *ckh);
53static void ckh_shrink(tsd_t *tsd, ckh_t *ckh);
54
55/******************************************************************************/
56
57/*
58 * Search bucket for key and return the cell number if found; SIZE_T_MAX
59 * otherwise.
60 */
61static size_t
62ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key) {
63 ckhc_t *cell;
64 unsigned i;
65
66 for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) {
67 cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i];
68 if (cell->key != NULL && ckh->keycomp(key, cell->key)) {
69 return (bucket << LG_CKH_BUCKET_CELLS) + i;
70 }
71 }
72
73 return SIZE_T_MAX;
74}
75
76/*
77 * Search table for key and return cell number if found; SIZE_T_MAX otherwise.
78 */
79static size_t
80ckh_isearch(ckh_t *ckh, const void *key) {
81 size_t hashes[2], bucket, cell;
82
83 assert(ckh != NULL);
84
85 ckh->hash(key, hashes);
86
87 /* Search primary bucket. */
88 bucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - 1);
89 cell = ckh_bucket_search(ckh, bucket, key);
90 if (cell != SIZE_T_MAX) {
91 return cell;
92 }
93
94 /* Search secondary bucket. */
95 bucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1);
96 cell = ckh_bucket_search(ckh, bucket, key);
97 return cell;
98}
99
100static bool
101ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key,
102 const void *data) {
103 ckhc_t *cell;
104 unsigned offset, i;
105
106 /*
107 * Cycle through the cells in the bucket, starting at a random position.
108 * The randomness avoids worst-case search overhead as buckets fill up.
109 */
110 offset = (unsigned)prng_lg_range_u64(&ckh->prng_state,
111 LG_CKH_BUCKET_CELLS);
112 for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) {
113 cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) +
114 ((i + offset) & ((ZU(1) << LG_CKH_BUCKET_CELLS) - 1))];
115 if (cell->key == NULL) {
116 cell->key = key;
117 cell->data = data;
118 ckh->count++;
119 return false;
120 }
121 }
122
123 return true;
124}
125
126/*
127 * No space is available in bucket. Randomly evict an item, then try to find an
128 * alternate location for that item. Iteratively repeat this
129 * eviction/relocation procedure until either success or detection of an
130 * eviction/relocation bucket cycle.
131 */
132static bool
133ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
134 void const **argdata) {
135 const void *key, *data, *tkey, *tdata;
136 ckhc_t *cell;
137 size_t hashes[2], bucket, tbucket;
138 unsigned i;
139
140 bucket = argbucket;
141 key = *argkey;
142 data = *argdata;
143 while (true) {
144 /*
145 * Choose a random item within the bucket to evict. This is
146 * critical to correct function, because without (eventually)
147 * evicting all items within a bucket during iteration, it
148 * would be possible to get stuck in an infinite loop if there
149 * were an item for which both hashes indicated the same
150 * bucket.
151 */
152 i = (unsigned)prng_lg_range_u64(&ckh->prng_state,
153 LG_CKH_BUCKET_CELLS);
154 cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i];
155 assert(cell->key != NULL);
156
157 /* Swap cell->{key,data} and {key,data} (evict). */
158 tkey = cell->key; tdata = cell->data;
159 cell->key = key; cell->data = data;
160 key = tkey; data = tdata;
161
162#ifdef CKH_COUNT
163 ckh->nrelocs++;
164#endif
165
166 /* Find the alternate bucket for the evicted item. */
167 ckh->hash(key, hashes);
168 tbucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1);
169 if (tbucket == bucket) {
170 tbucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets)
171 - 1);
172 /*
173 * It may be that (tbucket == bucket) still, if the
174 * item's hashes both indicate this bucket. However,
175 * we are guaranteed to eventually escape this bucket
176 * during iteration, assuming pseudo-random item
177 * selection (true randomness would make infinite
178 * looping a remote possibility). The reason we can
179 * never get trapped forever is that there are two
180 * cases:
181 *
182 * 1) This bucket == argbucket, so we will quickly
183 * detect an eviction cycle and terminate.
184 * 2) An item was evicted to this bucket from another,
185 * which means that at least one item in this bucket
186 * has hashes that indicate distinct buckets.
187 */
188 }
189 /* Check for a cycle. */
190 if (tbucket == argbucket) {
191 *argkey = key;
192 *argdata = data;
193 return true;
194 }
195
196 bucket = tbucket;
197 if (!ckh_try_bucket_insert(ckh, bucket, key, data)) {
198 return false;
199 }
200 }
201}
202
203static bool
204ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata) {
205 size_t hashes[2], bucket;
206 const void *key = *argkey;
207 const void *data = *argdata;
208
209 ckh->hash(key, hashes);
210
211 /* Try to insert in primary bucket. */
212 bucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - 1);
213 if (!ckh_try_bucket_insert(ckh, bucket, key, data)) {
214 return false;
215 }
216
217 /* Try to insert in secondary bucket. */
218 bucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1);
219 if (!ckh_try_bucket_insert(ckh, bucket, key, data)) {
220 return false;
221 }
222
223 /*
224 * Try to find a place for this item via iterative eviction/relocation.
225 */
226 return ckh_evict_reloc_insert(ckh, bucket, argkey, argdata);
227}
228
229/*
230 * Try to rebuild the hash table from scratch by inserting all items from the
231 * old table into the new.
232 */
233static bool
234ckh_rebuild(ckh_t *ckh, ckhc_t *aTab) {
235 size_t count, i, nins;
236 const void *key, *data;
237
238 count = ckh->count;
239 ckh->count = 0;
240 for (i = nins = 0; nins < count; i++) {
241 if (aTab[i].key != NULL) {
242 key = aTab[i].key;
243 data = aTab[i].data;
244 if (ckh_try_insert(ckh, &key, &data)) {
245 ckh->count = count;
246 return true;
247 }
248 nins++;
249 }
250 }
251
252 return false;
253}
254
255static bool
256ckh_grow(tsd_t *tsd, ckh_t *ckh) {
257 bool ret;
258 ckhc_t *tab, *ttab;
259 unsigned lg_prevbuckets, lg_curcells;
260
261#ifdef CKH_COUNT
262 ckh->ngrows++;
263#endif
264
265 /*
266 * It is possible (though unlikely, given well behaved hashes) that the
267 * table will have to be doubled more than once in order to create a
268 * usable table.
269 */
270 lg_prevbuckets = ckh->lg_curbuckets;
271 lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS;
272 while (true) {
273 size_t usize;
274
275 lg_curcells++;
276 usize = sz_sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
277 if (unlikely(usize == 0
278 || usize > SC_LARGE_MAXCLASS)) {
279 ret = true;
280 goto label_return;
281 }
282 tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE,
283 true, NULL, true, arena_ichoose(tsd, NULL));
284 if (tab == NULL) {
285 ret = true;
286 goto label_return;
287 }
288 /* Swap in new table. */
289 ttab = ckh->tab;
290 ckh->tab = tab;
291 tab = ttab;
292 ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS;
293
294 if (!ckh_rebuild(ckh, tab)) {
295 idalloctm(tsd_tsdn(tsd), tab, NULL, NULL, true, true);
296 break;
297 }
298
299 /* Rebuilding failed, so back out partially rebuilt table. */
300 idalloctm(tsd_tsdn(tsd), ckh->tab, NULL, NULL, true, true);
301 ckh->tab = tab;
302 ckh->lg_curbuckets = lg_prevbuckets;
303 }
304
305 ret = false;
306label_return:
307 return ret;
308}
309
310static void
311ckh_shrink(tsd_t *tsd, ckh_t *ckh) {
312 ckhc_t *tab, *ttab;
313 size_t usize;
314 unsigned lg_prevbuckets, lg_curcells;
315
316 /*
317 * It is possible (though unlikely, given well behaved hashes) that the
318 * table rebuild will fail.
319 */
320 lg_prevbuckets = ckh->lg_curbuckets;
321 lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1;
322 usize = sz_sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
323 if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
324 return;
325 }
326 tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, true, NULL,
327 true, arena_ichoose(tsd, NULL));
328 if (tab == NULL) {
329 /*
330 * An OOM error isn't worth propagating, since it doesn't
331 * prevent this or future operations from proceeding.
332 */
333 return;
334 }
335 /* Swap in new table. */
336 ttab = ckh->tab;
337 ckh->tab = tab;
338 tab = ttab;
339 ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS;
340
341 if (!ckh_rebuild(ckh, tab)) {
342 idalloctm(tsd_tsdn(tsd), tab, NULL, NULL, true, true);
343#ifdef CKH_COUNT
344 ckh->nshrinks++;
345#endif
346 return;
347 }
348
349 /* Rebuilding failed, so back out partially rebuilt table. */
350 idalloctm(tsd_tsdn(tsd), ckh->tab, NULL, NULL, true, true);
351 ckh->tab = tab;
352 ckh->lg_curbuckets = lg_prevbuckets;
353#ifdef CKH_COUNT
354 ckh->nshrinkfails++;
355#endif
356}
357
358bool
359ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *ckh_hash,
360 ckh_keycomp_t *keycomp) {
361 bool ret;
362 size_t mincells, usize;
363 unsigned lg_mincells;
364
365 assert(minitems > 0);
366 assert(ckh_hash != NULL);
367 assert(keycomp != NULL);
368
369#ifdef CKH_COUNT
370 ckh->ngrows = 0;
371 ckh->nshrinks = 0;
372 ckh->nshrinkfails = 0;
373 ckh->ninserts = 0;
374 ckh->nrelocs = 0;
375#endif
376 ckh->prng_state = 42; /* Value doesn't really matter. */
377 ckh->count = 0;
378
379 /*
380 * Find the minimum power of 2 that is large enough to fit minitems
381 * entries. We are using (2+,2) cuckoo hashing, which has an expected
382 * maximum load factor of at least ~0.86, so 0.75 is a conservative load
383 * factor that will typically allow mincells items to fit without ever
384 * growing the table.
385 */
386 assert(LG_CKH_BUCKET_CELLS > 0);
387 mincells = ((minitems + (3 - (minitems % 3))) / 3) << 2;
388 for (lg_mincells = LG_CKH_BUCKET_CELLS;
389 (ZU(1) << lg_mincells) < mincells;
390 lg_mincells++) {
391 /* Do nothing. */
392 }
393 ckh->lg_minbuckets = lg_mincells - LG_CKH_BUCKET_CELLS;
394 ckh->lg_curbuckets = lg_mincells - LG_CKH_BUCKET_CELLS;
395 ckh->hash = ckh_hash;
396 ckh->keycomp = keycomp;
397
398 usize = sz_sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE);
399 if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
400 ret = true;
401 goto label_return;
402 }
403 ckh->tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, true,
404 NULL, true, arena_ichoose(tsd, NULL));
405 if (ckh->tab == NULL) {
406 ret = true;
407 goto label_return;
408 }
409
410 ret = false;
411label_return:
412 return ret;
413}
414
415void
416ckh_delete(tsd_t *tsd, ckh_t *ckh) {
417 assert(ckh != NULL);
418
419#ifdef CKH_VERBOSE
420 malloc_printf(
421 "%s(%p): ngrows: %"FMTu64", nshrinks: %"FMTu64","
422 " nshrinkfails: %"FMTu64", ninserts: %"FMTu64","
423 " nrelocs: %"FMTu64"\n", __func__, ckh,
424 (unsigned long long)ckh->ngrows,
425 (unsigned long long)ckh->nshrinks,
426 (unsigned long long)ckh->nshrinkfails,
427 (unsigned long long)ckh->ninserts,
428 (unsigned long long)ckh->nrelocs);
429#endif
430
431 idalloctm(tsd_tsdn(tsd), ckh->tab, NULL, NULL, true, true);
432 if (config_debug) {
433 memset(ckh, JEMALLOC_FREE_JUNK, sizeof(ckh_t));
434 }
435}
436
437size_t
438ckh_count(ckh_t *ckh) {
439 assert(ckh != NULL);
440
441 return ckh->count;
442}
443
444bool
445ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data) {
446 size_t i, ncells;
447
448 for (i = *tabind, ncells = (ZU(1) << (ckh->lg_curbuckets +
449 LG_CKH_BUCKET_CELLS)); i < ncells; i++) {
450 if (ckh->tab[i].key != NULL) {
451 if (key != NULL) {
452 *key = (void *)ckh->tab[i].key;
453 }
454 if (data != NULL) {
455 *data = (void *)ckh->tab[i].data;
456 }
457 *tabind = i + 1;
458 return false;
459 }
460 }
461
462 return true;
463}
464
465bool
466ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data) {
467 bool ret;
468
469 assert(ckh != NULL);
470 assert(ckh_search(ckh, key, NULL, NULL));
471
472#ifdef CKH_COUNT
473 ckh->ninserts++;
474#endif
475
476 while (ckh_try_insert(ckh, &key, &data)) {
477 if (ckh_grow(tsd, ckh)) {
478 ret = true;
479 goto label_return;
480 }
481 }
482
483 ret = false;
484label_return:
485 return ret;
486}
487
488bool
489ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key,
490 void **data) {
491 size_t cell;
492
493 assert(ckh != NULL);
494
495 cell = ckh_isearch(ckh, searchkey);
496 if (cell != SIZE_T_MAX) {
497 if (key != NULL) {
498 *key = (void *)ckh->tab[cell].key;
499 }
500 if (data != NULL) {
501 *data = (void *)ckh->tab[cell].data;
502 }
503 ckh->tab[cell].key = NULL;
504 ckh->tab[cell].data = NULL; /* Not necessary. */
505
506 ckh->count--;
507 /* Try to halve the table if it is less than 1/4 full. */
508 if (ckh->count < (ZU(1) << (ckh->lg_curbuckets
509 + LG_CKH_BUCKET_CELLS - 2)) && ckh->lg_curbuckets
510 > ckh->lg_minbuckets) {
511 /* Ignore error due to OOM. */
512 ckh_shrink(tsd, ckh);
513 }
514
515 return false;
516 }
517
518 return true;
519}
520
521bool
522ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data) {
523 size_t cell;
524
525 assert(ckh != NULL);
526
527 cell = ckh_isearch(ckh, searchkey);
528 if (cell != SIZE_T_MAX) {
529 if (key != NULL) {
530 *key = (void *)ckh->tab[cell].key;
531 }
532 if (data != NULL) {
533 *data = (void *)ckh->tab[cell].data;
534 }
535 return false;
536 }
537
538 return true;
539}
540
541void
542ckh_string_hash(const void *key, size_t r_hash[2]) {
543 hash(key, strlen((const char *)key), 0x94122f33U, r_hash);
544}
545
546bool
547ckh_string_keycomp(const void *k1, const void *k2) {
548 assert(k1 != NULL);
549 assert(k2 != NULL);
550
551 return !strcmp((char *)k1, (char *)k2);
552}
553
554void
555ckh_pointer_hash(const void *key, size_t r_hash[2]) {
556 union {
557 const void *v;
558 size_t i;
559 } u;
560
561 assert(sizeof(u.v) == sizeof(u.i));
562 u.v = key;
563 hash(&u.i, sizeof(u.i), 0xd983396eU, r_hash);
564}
565
566bool
567ckh_pointer_keycomp(const void *k1, const void *k2) {
568 return (k1 == k2);
569}
diff --git a/examples/redis-unstable/deps/jemalloc/src/counter.c b/examples/redis-unstable/deps/jemalloc/src/counter.c
deleted file mode 100644
index 8f1ae3a..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/counter.c
+++ /dev/null
@@ -1,30 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/counter.h"
5
6bool
7counter_accum_init(counter_accum_t *counter, uint64_t interval) {
8 if (LOCKEDINT_MTX_INIT(counter->mtx, "counter_accum",
9 WITNESS_RANK_COUNTER_ACCUM, malloc_mutex_rank_exclusive)) {
10 return true;
11 }
12 locked_init_u64_unsynchronized(&counter->accumbytes, 0);
13 counter->interval = interval;
14 return false;
15}
16
17void
18counter_prefork(tsdn_t *tsdn, counter_accum_t *counter) {
19 LOCKEDINT_MTX_PREFORK(tsdn, counter->mtx);
20}
21
22void
23counter_postfork_parent(tsdn_t *tsdn, counter_accum_t *counter) {
24 LOCKEDINT_MTX_POSTFORK_PARENT(tsdn, counter->mtx);
25}
26
27void
28counter_postfork_child(tsdn_t *tsdn, counter_accum_t *counter) {
29 LOCKEDINT_MTX_POSTFORK_CHILD(tsdn, counter->mtx);
30}
diff --git a/examples/redis-unstable/deps/jemalloc/src/ctl.c b/examples/redis-unstable/deps/jemalloc/src/ctl.c
deleted file mode 100644
index 135271b..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/ctl.c
+++ /dev/null
@@ -1,4414 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/ctl.h"
6#include "jemalloc/internal/extent_dss.h"
7#include "jemalloc/internal/extent_mmap.h"
8#include "jemalloc/internal/inspect.h"
9#include "jemalloc/internal/mutex.h"
10#include "jemalloc/internal/nstime.h"
11#include "jemalloc/internal/peak_event.h"
12#include "jemalloc/internal/prof_data.h"
13#include "jemalloc/internal/prof_log.h"
14#include "jemalloc/internal/prof_recent.h"
15#include "jemalloc/internal/prof_stats.h"
16#include "jemalloc/internal/prof_sys.h"
17#include "jemalloc/internal/safety_check.h"
18#include "jemalloc/internal/sc.h"
19#include "jemalloc/internal/util.h"
20
21/******************************************************************************/
22/* Data. */
23
24/*
25 * ctl_mtx protects the following:
26 * - ctl_stats->*
27 */
28static malloc_mutex_t ctl_mtx;
29static bool ctl_initialized;
30static ctl_stats_t *ctl_stats;
31static ctl_arenas_t *ctl_arenas;
32
33/******************************************************************************/
34/* Helpers for named and indexed nodes. */
35
36static const ctl_named_node_t *
37ctl_named_node(const ctl_node_t *node) {
38 return ((node->named) ? (const ctl_named_node_t *)node : NULL);
39}
40
41static const ctl_named_node_t *
42ctl_named_children(const ctl_named_node_t *node, size_t index) {
43 const ctl_named_node_t *children = ctl_named_node(node->children);
44
45 return (children ? &children[index] : NULL);
46}
47
48static const ctl_indexed_node_t *
49ctl_indexed_node(const ctl_node_t *node) {
50 return (!node->named ? (const ctl_indexed_node_t *)node : NULL);
51}
52
53/******************************************************************************/
54/* Function prototypes for non-inline static functions. */
55
56#define CTL_PROTO(n) \
57static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
58 void *oldp, size_t *oldlenp, void *newp, size_t newlen);
59
60#define INDEX_PROTO(n) \
61static const ctl_named_node_t *n##_index(tsdn_t *tsdn, \
62 const size_t *mib, size_t miblen, size_t i);
63
64CTL_PROTO(version)
65CTL_PROTO(epoch)
66CTL_PROTO(background_thread)
67CTL_PROTO(max_background_threads)
68CTL_PROTO(thread_tcache_enabled)
69CTL_PROTO(thread_tcache_flush)
70CTL_PROTO(thread_peak_read)
71CTL_PROTO(thread_peak_reset)
72CTL_PROTO(thread_prof_name)
73CTL_PROTO(thread_prof_active)
74CTL_PROTO(thread_arena)
75CTL_PROTO(thread_allocated)
76CTL_PROTO(thread_allocatedp)
77CTL_PROTO(thread_deallocated)
78CTL_PROTO(thread_deallocatedp)
79CTL_PROTO(thread_idle)
80CTL_PROTO(config_cache_oblivious)
81CTL_PROTO(config_debug)
82CTL_PROTO(config_fill)
83CTL_PROTO(config_lazy_lock)
84CTL_PROTO(config_malloc_conf)
85CTL_PROTO(config_opt_safety_checks)
86CTL_PROTO(config_prof)
87CTL_PROTO(config_prof_libgcc)
88CTL_PROTO(config_prof_libunwind)
89CTL_PROTO(config_stats)
90CTL_PROTO(config_utrace)
91CTL_PROTO(config_xmalloc)
92CTL_PROTO(opt_abort)
93CTL_PROTO(opt_abort_conf)
94CTL_PROTO(opt_cache_oblivious)
95CTL_PROTO(opt_trust_madvise)
96CTL_PROTO(opt_confirm_conf)
97CTL_PROTO(opt_hpa)
98CTL_PROTO(opt_hpa_slab_max_alloc)
99CTL_PROTO(opt_hpa_hugification_threshold)
100CTL_PROTO(opt_hpa_hugify_delay_ms)
101CTL_PROTO(opt_hpa_min_purge_interval_ms)
102CTL_PROTO(opt_hpa_dirty_mult)
103CTL_PROTO(opt_hpa_sec_nshards)
104CTL_PROTO(opt_hpa_sec_max_alloc)
105CTL_PROTO(opt_hpa_sec_max_bytes)
106CTL_PROTO(opt_hpa_sec_bytes_after_flush)
107CTL_PROTO(opt_hpa_sec_batch_fill_extra)
108CTL_PROTO(opt_metadata_thp)
109CTL_PROTO(opt_retain)
110CTL_PROTO(opt_dss)
111CTL_PROTO(opt_narenas)
112CTL_PROTO(opt_percpu_arena)
113CTL_PROTO(opt_oversize_threshold)
114CTL_PROTO(opt_background_thread)
115CTL_PROTO(opt_mutex_max_spin)
116CTL_PROTO(opt_max_background_threads)
117CTL_PROTO(opt_dirty_decay_ms)
118CTL_PROTO(opt_muzzy_decay_ms)
119CTL_PROTO(opt_stats_print)
120CTL_PROTO(opt_stats_print_opts)
121CTL_PROTO(opt_stats_interval)
122CTL_PROTO(opt_stats_interval_opts)
123CTL_PROTO(opt_junk)
124CTL_PROTO(opt_zero)
125CTL_PROTO(opt_utrace)
126CTL_PROTO(opt_xmalloc)
127CTL_PROTO(opt_experimental_infallible_new)
128CTL_PROTO(opt_tcache)
129CTL_PROTO(opt_tcache_max)
130CTL_PROTO(opt_tcache_nslots_small_min)
131CTL_PROTO(opt_tcache_nslots_small_max)
132CTL_PROTO(opt_tcache_nslots_large)
133CTL_PROTO(opt_lg_tcache_nslots_mul)
134CTL_PROTO(opt_tcache_gc_incr_bytes)
135CTL_PROTO(opt_tcache_gc_delay_bytes)
136CTL_PROTO(opt_lg_tcache_flush_small_div)
137CTL_PROTO(opt_lg_tcache_flush_large_div)
138CTL_PROTO(opt_thp)
139CTL_PROTO(opt_lg_extent_max_active_fit)
140CTL_PROTO(opt_prof)
141CTL_PROTO(opt_prof_prefix)
142CTL_PROTO(opt_prof_active)
143CTL_PROTO(opt_prof_thread_active_init)
144CTL_PROTO(opt_lg_prof_sample)
145CTL_PROTO(opt_lg_prof_interval)
146CTL_PROTO(opt_prof_gdump)
147CTL_PROTO(opt_prof_final)
148CTL_PROTO(opt_prof_leak)
149CTL_PROTO(opt_prof_leak_error)
150CTL_PROTO(opt_prof_accum)
151CTL_PROTO(opt_prof_recent_alloc_max)
152CTL_PROTO(opt_prof_stats)
153CTL_PROTO(opt_prof_sys_thread_name)
154CTL_PROTO(opt_prof_time_res)
155CTL_PROTO(opt_lg_san_uaf_align)
156CTL_PROTO(opt_zero_realloc)
157CTL_PROTO(tcache_create)
158CTL_PROTO(tcache_flush)
159CTL_PROTO(tcache_destroy)
160CTL_PROTO(arena_i_initialized)
161CTL_PROTO(arena_i_decay)
162CTL_PROTO(arena_i_purge)
163CTL_PROTO(arena_i_reset)
164CTL_PROTO(arena_i_destroy)
165CTL_PROTO(arena_i_dss)
166CTL_PROTO(arena_i_oversize_threshold)
167CTL_PROTO(arena_i_dirty_decay_ms)
168CTL_PROTO(arena_i_muzzy_decay_ms)
169CTL_PROTO(arena_i_extent_hooks)
170CTL_PROTO(arena_i_retain_grow_limit)
171INDEX_PROTO(arena_i)
172CTL_PROTO(arenas_bin_i_size)
173CTL_PROTO(arenas_bin_i_nregs)
174CTL_PROTO(arenas_bin_i_slab_size)
175CTL_PROTO(arenas_bin_i_nshards)
176INDEX_PROTO(arenas_bin_i)
177CTL_PROTO(arenas_lextent_i_size)
178INDEX_PROTO(arenas_lextent_i)
179CTL_PROTO(arenas_narenas)
180CTL_PROTO(arenas_dirty_decay_ms)
181CTL_PROTO(arenas_muzzy_decay_ms)
182CTL_PROTO(arenas_quantum)
183CTL_PROTO(arenas_page)
184CTL_PROTO(arenas_tcache_max)
185CTL_PROTO(arenas_nbins)
186CTL_PROTO(arenas_nhbins)
187CTL_PROTO(arenas_nlextents)
188CTL_PROTO(arenas_create)
189CTL_PROTO(arenas_lookup)
190CTL_PROTO(prof_thread_active_init)
191CTL_PROTO(prof_active)
192CTL_PROTO(prof_dump)
193CTL_PROTO(prof_gdump)
194CTL_PROTO(prof_prefix)
195CTL_PROTO(prof_reset)
196CTL_PROTO(prof_interval)
197CTL_PROTO(lg_prof_sample)
198CTL_PROTO(prof_log_start)
199CTL_PROTO(prof_log_stop)
200CTL_PROTO(prof_stats_bins_i_live)
201CTL_PROTO(prof_stats_bins_i_accum)
202INDEX_PROTO(prof_stats_bins_i)
203CTL_PROTO(prof_stats_lextents_i_live)
204CTL_PROTO(prof_stats_lextents_i_accum)
205INDEX_PROTO(prof_stats_lextents_i)
206CTL_PROTO(stats_arenas_i_small_allocated)
207CTL_PROTO(stats_arenas_i_small_nmalloc)
208CTL_PROTO(stats_arenas_i_small_ndalloc)
209CTL_PROTO(stats_arenas_i_small_nrequests)
210CTL_PROTO(stats_arenas_i_small_nfills)
211CTL_PROTO(stats_arenas_i_small_nflushes)
212CTL_PROTO(stats_arenas_i_large_allocated)
213CTL_PROTO(stats_arenas_i_large_nmalloc)
214CTL_PROTO(stats_arenas_i_large_ndalloc)
215CTL_PROTO(stats_arenas_i_large_nrequests)
216CTL_PROTO(stats_arenas_i_large_nfills)
217CTL_PROTO(stats_arenas_i_large_nflushes)
218CTL_PROTO(stats_arenas_i_bins_j_nmalloc)
219CTL_PROTO(stats_arenas_i_bins_j_ndalloc)
220CTL_PROTO(stats_arenas_i_bins_j_nrequests)
221CTL_PROTO(stats_arenas_i_bins_j_curregs)
222CTL_PROTO(stats_arenas_i_bins_j_nfills)
223CTL_PROTO(stats_arenas_i_bins_j_nflushes)
224CTL_PROTO(stats_arenas_i_bins_j_nslabs)
225CTL_PROTO(stats_arenas_i_bins_j_nreslabs)
226CTL_PROTO(stats_arenas_i_bins_j_curslabs)
227CTL_PROTO(stats_arenas_i_bins_j_nonfull_slabs)
228INDEX_PROTO(stats_arenas_i_bins_j)
229CTL_PROTO(stats_arenas_i_lextents_j_nmalloc)
230CTL_PROTO(stats_arenas_i_lextents_j_ndalloc)
231CTL_PROTO(stats_arenas_i_lextents_j_nrequests)
232CTL_PROTO(stats_arenas_i_lextents_j_curlextents)
233INDEX_PROTO(stats_arenas_i_lextents_j)
234CTL_PROTO(stats_arenas_i_extents_j_ndirty)
235CTL_PROTO(stats_arenas_i_extents_j_nmuzzy)
236CTL_PROTO(stats_arenas_i_extents_j_nretained)
237CTL_PROTO(stats_arenas_i_extents_j_dirty_bytes)
238CTL_PROTO(stats_arenas_i_extents_j_muzzy_bytes)
239CTL_PROTO(stats_arenas_i_extents_j_retained_bytes)
240INDEX_PROTO(stats_arenas_i_extents_j)
241CTL_PROTO(stats_arenas_i_hpa_shard_npurge_passes)
242CTL_PROTO(stats_arenas_i_hpa_shard_npurges)
243CTL_PROTO(stats_arenas_i_hpa_shard_nhugifies)
244CTL_PROTO(stats_arenas_i_hpa_shard_ndehugifies)
245
246/* We have a set of stats for full slabs. */
247CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge)
248CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge)
249CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge)
250CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_nactive_huge)
251CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_ndirty_nonhuge)
252CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_ndirty_huge)
253
254/* A parallel set for the empty slabs. */
255CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_nonhuge)
256CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_huge)
257CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_nactive_nonhuge)
258CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_nactive_huge)
259CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_ndirty_nonhuge)
260CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_ndirty_huge)
261
262/*
263 * And one for the slabs that are neither empty nor full, but indexed by how
264 * full they are.
265 */
266CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge)
267CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge)
268CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge)
269CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge)
270CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge)
271CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge)
272
273INDEX_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j)
274CTL_PROTO(stats_arenas_i_nthreads)
275CTL_PROTO(stats_arenas_i_uptime)
276CTL_PROTO(stats_arenas_i_dss)
277CTL_PROTO(stats_arenas_i_dirty_decay_ms)
278CTL_PROTO(stats_arenas_i_muzzy_decay_ms)
279CTL_PROTO(stats_arenas_i_pactive)
280CTL_PROTO(stats_arenas_i_pdirty)
281CTL_PROTO(stats_arenas_i_pmuzzy)
282CTL_PROTO(stats_arenas_i_mapped)
283CTL_PROTO(stats_arenas_i_retained)
284CTL_PROTO(stats_arenas_i_extent_avail)
285CTL_PROTO(stats_arenas_i_dirty_npurge)
286CTL_PROTO(stats_arenas_i_dirty_nmadvise)
287CTL_PROTO(stats_arenas_i_dirty_purged)
288CTL_PROTO(stats_arenas_i_muzzy_npurge)
289CTL_PROTO(stats_arenas_i_muzzy_nmadvise)
290CTL_PROTO(stats_arenas_i_muzzy_purged)
291CTL_PROTO(stats_arenas_i_base)
292CTL_PROTO(stats_arenas_i_internal)
293CTL_PROTO(stats_arenas_i_metadata_thp)
294CTL_PROTO(stats_arenas_i_tcache_bytes)
295CTL_PROTO(stats_arenas_i_tcache_stashed_bytes)
296CTL_PROTO(stats_arenas_i_resident)
297CTL_PROTO(stats_arenas_i_abandoned_vm)
298CTL_PROTO(stats_arenas_i_hpa_sec_bytes)
299INDEX_PROTO(stats_arenas_i)
300CTL_PROTO(stats_allocated)
301CTL_PROTO(stats_active)
302CTL_PROTO(stats_background_thread_num_threads)
303CTL_PROTO(stats_background_thread_num_runs)
304CTL_PROTO(stats_background_thread_run_interval)
305CTL_PROTO(stats_metadata)
306CTL_PROTO(stats_metadata_thp)
307CTL_PROTO(stats_resident)
308CTL_PROTO(stats_mapped)
309CTL_PROTO(stats_retained)
310CTL_PROTO(stats_zero_reallocs)
311CTL_PROTO(experimental_hooks_install)
312CTL_PROTO(experimental_hooks_remove)
313CTL_PROTO(experimental_hooks_prof_backtrace)
314CTL_PROTO(experimental_hooks_prof_dump)
315CTL_PROTO(experimental_hooks_safety_check_abort)
316CTL_PROTO(experimental_thread_activity_callback)
317CTL_PROTO(experimental_utilization_query)
318CTL_PROTO(experimental_utilization_batch_query)
319CTL_PROTO(experimental_arenas_i_pactivep)
320INDEX_PROTO(experimental_arenas_i)
321CTL_PROTO(experimental_prof_recent_alloc_max)
322CTL_PROTO(experimental_prof_recent_alloc_dump)
323CTL_PROTO(experimental_batch_alloc)
324CTL_PROTO(experimental_arenas_create_ext)
325
326#define MUTEX_STATS_CTL_PROTO_GEN(n) \
327CTL_PROTO(stats_##n##_num_ops) \
328CTL_PROTO(stats_##n##_num_wait) \
329CTL_PROTO(stats_##n##_num_spin_acq) \
330CTL_PROTO(stats_##n##_num_owner_switch) \
331CTL_PROTO(stats_##n##_total_wait_time) \
332CTL_PROTO(stats_##n##_max_wait_time) \
333CTL_PROTO(stats_##n##_max_num_thds)
334
335/* Global mutexes. */
336#define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(mutexes_##mtx)
337MUTEX_PROF_GLOBAL_MUTEXES
338#undef OP
339
340/* Per arena mutexes. */
341#define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(arenas_i_mutexes_##mtx)
342MUTEX_PROF_ARENA_MUTEXES
343#undef OP
344
345/* Arena bin mutexes. */
346MUTEX_STATS_CTL_PROTO_GEN(arenas_i_bins_j_mutex)
347#undef MUTEX_STATS_CTL_PROTO_GEN
348
349CTL_PROTO(stats_mutexes_reset)
350
351/******************************************************************************/
352/* mallctl tree. */
353
354#define NAME(n) {true}, n
355#define CHILD(t, c) \
356 sizeof(c##_node) / sizeof(ctl_##t##_node_t), \
357 (ctl_node_t *)c##_node, \
358 NULL
359#define CTL(c) 0, NULL, c##_ctl
360
361/*
362 * Only handles internal indexed nodes, since there are currently no external
363 * ones.
364 */
365#define INDEX(i) {false}, i##_index
366
367static const ctl_named_node_t thread_tcache_node[] = {
368 {NAME("enabled"), CTL(thread_tcache_enabled)},
369 {NAME("flush"), CTL(thread_tcache_flush)}
370};
371
372static const ctl_named_node_t thread_peak_node[] = {
373 {NAME("read"), CTL(thread_peak_read)},
374 {NAME("reset"), CTL(thread_peak_reset)},
375};
376
377static const ctl_named_node_t thread_prof_node[] = {
378 {NAME("name"), CTL(thread_prof_name)},
379 {NAME("active"), CTL(thread_prof_active)}
380};
381
382static const ctl_named_node_t thread_node[] = {
383 {NAME("arena"), CTL(thread_arena)},
384 {NAME("allocated"), CTL(thread_allocated)},
385 {NAME("allocatedp"), CTL(thread_allocatedp)},
386 {NAME("deallocated"), CTL(thread_deallocated)},
387 {NAME("deallocatedp"), CTL(thread_deallocatedp)},
388 {NAME("tcache"), CHILD(named, thread_tcache)},
389 {NAME("peak"), CHILD(named, thread_peak)},
390 {NAME("prof"), CHILD(named, thread_prof)},
391 {NAME("idle"), CTL(thread_idle)}
392};
393
394static const ctl_named_node_t config_node[] = {
395 {NAME("cache_oblivious"), CTL(config_cache_oblivious)},
396 {NAME("debug"), CTL(config_debug)},
397 {NAME("fill"), CTL(config_fill)},
398 {NAME("lazy_lock"), CTL(config_lazy_lock)},
399 {NAME("malloc_conf"), CTL(config_malloc_conf)},
400 {NAME("opt_safety_checks"), CTL(config_opt_safety_checks)},
401 {NAME("prof"), CTL(config_prof)},
402 {NAME("prof_libgcc"), CTL(config_prof_libgcc)},
403 {NAME("prof_libunwind"), CTL(config_prof_libunwind)},
404 {NAME("stats"), CTL(config_stats)},
405 {NAME("utrace"), CTL(config_utrace)},
406 {NAME("xmalloc"), CTL(config_xmalloc)}
407};
408
409static const ctl_named_node_t opt_node[] = {
410 {NAME("abort"), CTL(opt_abort)},
411 {NAME("abort_conf"), CTL(opt_abort_conf)},
412 {NAME("cache_oblivious"), CTL(opt_cache_oblivious)},
413 {NAME("trust_madvise"), CTL(opt_trust_madvise)},
414 {NAME("confirm_conf"), CTL(opt_confirm_conf)},
415 {NAME("hpa"), CTL(opt_hpa)},
416 {NAME("hpa_slab_max_alloc"), CTL(opt_hpa_slab_max_alloc)},
417 {NAME("hpa_hugification_threshold"),
418 CTL(opt_hpa_hugification_threshold)},
419 {NAME("hpa_hugify_delay_ms"), CTL(opt_hpa_hugify_delay_ms)},
420 {NAME("hpa_min_purge_interval_ms"), CTL(opt_hpa_min_purge_interval_ms)},
421 {NAME("hpa_dirty_mult"), CTL(opt_hpa_dirty_mult)},
422 {NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)},
423 {NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)},
424 {NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)},
425 {NAME("hpa_sec_bytes_after_flush"),
426 CTL(opt_hpa_sec_bytes_after_flush)},
427 {NAME("hpa_sec_batch_fill_extra"),
428 CTL(opt_hpa_sec_batch_fill_extra)},
429 {NAME("metadata_thp"), CTL(opt_metadata_thp)},
430 {NAME("retain"), CTL(opt_retain)},
431 {NAME("dss"), CTL(opt_dss)},
432 {NAME("narenas"), CTL(opt_narenas)},
433 {NAME("percpu_arena"), CTL(opt_percpu_arena)},
434 {NAME("oversize_threshold"), CTL(opt_oversize_threshold)},
435 {NAME("mutex_max_spin"), CTL(opt_mutex_max_spin)},
436 {NAME("background_thread"), CTL(opt_background_thread)},
437 {NAME("max_background_threads"), CTL(opt_max_background_threads)},
438 {NAME("dirty_decay_ms"), CTL(opt_dirty_decay_ms)},
439 {NAME("muzzy_decay_ms"), CTL(opt_muzzy_decay_ms)},
440 {NAME("stats_print"), CTL(opt_stats_print)},
441 {NAME("stats_print_opts"), CTL(opt_stats_print_opts)},
442 {NAME("stats_interval"), CTL(opt_stats_interval)},
443 {NAME("stats_interval_opts"), CTL(opt_stats_interval_opts)},
444 {NAME("junk"), CTL(opt_junk)},
445 {NAME("zero"), CTL(opt_zero)},
446 {NAME("utrace"), CTL(opt_utrace)},
447 {NAME("xmalloc"), CTL(opt_xmalloc)},
448 {NAME("experimental_infallible_new"),
449 CTL(opt_experimental_infallible_new)},
450 {NAME("tcache"), CTL(opt_tcache)},
451 {NAME("tcache_max"), CTL(opt_tcache_max)},
452 {NAME("tcache_nslots_small_min"),
453 CTL(opt_tcache_nslots_small_min)},
454 {NAME("tcache_nslots_small_max"),
455 CTL(opt_tcache_nslots_small_max)},
456 {NAME("tcache_nslots_large"), CTL(opt_tcache_nslots_large)},
457 {NAME("lg_tcache_nslots_mul"), CTL(opt_lg_tcache_nslots_mul)},
458 {NAME("tcache_gc_incr_bytes"), CTL(opt_tcache_gc_incr_bytes)},
459 {NAME("tcache_gc_delay_bytes"), CTL(opt_tcache_gc_delay_bytes)},
460 {NAME("lg_tcache_flush_small_div"),
461 CTL(opt_lg_tcache_flush_small_div)},
462 {NAME("lg_tcache_flush_large_div"),
463 CTL(opt_lg_tcache_flush_large_div)},
464 {NAME("thp"), CTL(opt_thp)},
465 {NAME("lg_extent_max_active_fit"), CTL(opt_lg_extent_max_active_fit)},
466 {NAME("prof"), CTL(opt_prof)},
467 {NAME("prof_prefix"), CTL(opt_prof_prefix)},
468 {NAME("prof_active"), CTL(opt_prof_active)},
469 {NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)},
470 {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)},
471 {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)},
472 {NAME("prof_gdump"), CTL(opt_prof_gdump)},
473 {NAME("prof_final"), CTL(opt_prof_final)},
474 {NAME("prof_leak"), CTL(opt_prof_leak)},
475 {NAME("prof_leak_error"), CTL(opt_prof_leak_error)},
476 {NAME("prof_accum"), CTL(opt_prof_accum)},
477 {NAME("prof_recent_alloc_max"), CTL(opt_prof_recent_alloc_max)},
478 {NAME("prof_stats"), CTL(opt_prof_stats)},
479 {NAME("prof_sys_thread_name"), CTL(opt_prof_sys_thread_name)},
480 {NAME("prof_time_resolution"), CTL(opt_prof_time_res)},
481 {NAME("lg_san_uaf_align"), CTL(opt_lg_san_uaf_align)},
482 {NAME("zero_realloc"), CTL(opt_zero_realloc)}
483};
484
485static const ctl_named_node_t tcache_node[] = {
486 {NAME("create"), CTL(tcache_create)},
487 {NAME("flush"), CTL(tcache_flush)},
488 {NAME("destroy"), CTL(tcache_destroy)}
489};
490
491static const ctl_named_node_t arena_i_node[] = {
492 {NAME("initialized"), CTL(arena_i_initialized)},
493 {NAME("decay"), CTL(arena_i_decay)},
494 {NAME("purge"), CTL(arena_i_purge)},
495 {NAME("reset"), CTL(arena_i_reset)},
496 {NAME("destroy"), CTL(arena_i_destroy)},
497 {NAME("dss"), CTL(arena_i_dss)},
498 /*
499 * Undocumented for now, since we anticipate an arena API in flux after
500 * we cut the last 5-series release.
501 */
502 {NAME("oversize_threshold"), CTL(arena_i_oversize_threshold)},
503 {NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)},
504 {NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)},
505 {NAME("extent_hooks"), CTL(arena_i_extent_hooks)},
506 {NAME("retain_grow_limit"), CTL(arena_i_retain_grow_limit)}
507};
508static const ctl_named_node_t super_arena_i_node[] = {
509 {NAME(""), CHILD(named, arena_i)}
510};
511
512static const ctl_indexed_node_t arena_node[] = {
513 {INDEX(arena_i)}
514};
515
516static const ctl_named_node_t arenas_bin_i_node[] = {
517 {NAME("size"), CTL(arenas_bin_i_size)},
518 {NAME("nregs"), CTL(arenas_bin_i_nregs)},
519 {NAME("slab_size"), CTL(arenas_bin_i_slab_size)},
520 {NAME("nshards"), CTL(arenas_bin_i_nshards)}
521};
522static const ctl_named_node_t super_arenas_bin_i_node[] = {
523 {NAME(""), CHILD(named, arenas_bin_i)}
524};
525
526static const ctl_indexed_node_t arenas_bin_node[] = {
527 {INDEX(arenas_bin_i)}
528};
529
530static const ctl_named_node_t arenas_lextent_i_node[] = {
531 {NAME("size"), CTL(arenas_lextent_i_size)}
532};
533static const ctl_named_node_t super_arenas_lextent_i_node[] = {
534 {NAME(""), CHILD(named, arenas_lextent_i)}
535};
536
537static const ctl_indexed_node_t arenas_lextent_node[] = {
538 {INDEX(arenas_lextent_i)}
539};
540
541static const ctl_named_node_t arenas_node[] = {
542 {NAME("narenas"), CTL(arenas_narenas)},
543 {NAME("dirty_decay_ms"), CTL(arenas_dirty_decay_ms)},
544 {NAME("muzzy_decay_ms"), CTL(arenas_muzzy_decay_ms)},
545 {NAME("quantum"), CTL(arenas_quantum)},
546 {NAME("page"), CTL(arenas_page)},
547 {NAME("tcache_max"), CTL(arenas_tcache_max)},
548 {NAME("nbins"), CTL(arenas_nbins)},
549 {NAME("nhbins"), CTL(arenas_nhbins)},
550 {NAME("bin"), CHILD(indexed, arenas_bin)},
551 {NAME("nlextents"), CTL(arenas_nlextents)},
552 {NAME("lextent"), CHILD(indexed, arenas_lextent)},
553 {NAME("create"), CTL(arenas_create)},
554 {NAME("lookup"), CTL(arenas_lookup)}
555};
556
557static const ctl_named_node_t prof_stats_bins_i_node[] = {
558 {NAME("live"), CTL(prof_stats_bins_i_live)},
559 {NAME("accum"), CTL(prof_stats_bins_i_accum)}
560};
561
562static const ctl_named_node_t super_prof_stats_bins_i_node[] = {
563 {NAME(""), CHILD(named, prof_stats_bins_i)}
564};
565
566static const ctl_indexed_node_t prof_stats_bins_node[] = {
567 {INDEX(prof_stats_bins_i)}
568};
569
570static const ctl_named_node_t prof_stats_lextents_i_node[] = {
571 {NAME("live"), CTL(prof_stats_lextents_i_live)},
572 {NAME("accum"), CTL(prof_stats_lextents_i_accum)}
573};
574
575static const ctl_named_node_t super_prof_stats_lextents_i_node[] = {
576 {NAME(""), CHILD(named, prof_stats_lextents_i)}
577};
578
579static const ctl_indexed_node_t prof_stats_lextents_node[] = {
580 {INDEX(prof_stats_lextents_i)}
581};
582
583static const ctl_named_node_t prof_stats_node[] = {
584 {NAME("bins"), CHILD(indexed, prof_stats_bins)},
585 {NAME("lextents"), CHILD(indexed, prof_stats_lextents)},
586};
587
588static const ctl_named_node_t prof_node[] = {
589 {NAME("thread_active_init"), CTL(prof_thread_active_init)},
590 {NAME("active"), CTL(prof_active)},
591 {NAME("dump"), CTL(prof_dump)},
592 {NAME("gdump"), CTL(prof_gdump)},
593 {NAME("prefix"), CTL(prof_prefix)},
594 {NAME("reset"), CTL(prof_reset)},
595 {NAME("interval"), CTL(prof_interval)},
596 {NAME("lg_sample"), CTL(lg_prof_sample)},
597 {NAME("log_start"), CTL(prof_log_start)},
598 {NAME("log_stop"), CTL(prof_log_stop)},
599 {NAME("stats"), CHILD(named, prof_stats)}
600};
601
602static const ctl_named_node_t stats_arenas_i_small_node[] = {
603 {NAME("allocated"), CTL(stats_arenas_i_small_allocated)},
604 {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)},
605 {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)},
606 {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)},
607 {NAME("nfills"), CTL(stats_arenas_i_small_nfills)},
608 {NAME("nflushes"), CTL(stats_arenas_i_small_nflushes)}
609};
610
611static const ctl_named_node_t stats_arenas_i_large_node[] = {
612 {NAME("allocated"), CTL(stats_arenas_i_large_allocated)},
613 {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)},
614 {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)},
615 {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)},
616 {NAME("nfills"), CTL(stats_arenas_i_large_nfills)},
617 {NAME("nflushes"), CTL(stats_arenas_i_large_nflushes)}
618};
619
620#define MUTEX_PROF_DATA_NODE(prefix) \
621static const ctl_named_node_t stats_##prefix##_node[] = { \
622 {NAME("num_ops"), \
623 CTL(stats_##prefix##_num_ops)}, \
624 {NAME("num_wait"), \
625 CTL(stats_##prefix##_num_wait)}, \
626 {NAME("num_spin_acq"), \
627 CTL(stats_##prefix##_num_spin_acq)}, \
628 {NAME("num_owner_switch"), \
629 CTL(stats_##prefix##_num_owner_switch)}, \
630 {NAME("total_wait_time"), \
631 CTL(stats_##prefix##_total_wait_time)}, \
632 {NAME("max_wait_time"), \
633 CTL(stats_##prefix##_max_wait_time)}, \
634 {NAME("max_num_thds"), \
635 CTL(stats_##prefix##_max_num_thds)} \
636 /* Note that # of current waiting thread not provided. */ \
637};
638
639MUTEX_PROF_DATA_NODE(arenas_i_bins_j_mutex)
640
641static const ctl_named_node_t stats_arenas_i_bins_j_node[] = {
642 {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)},
643 {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)},
644 {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)},
645 {NAME("curregs"), CTL(stats_arenas_i_bins_j_curregs)},
646 {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)},
647 {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)},
648 {NAME("nslabs"), CTL(stats_arenas_i_bins_j_nslabs)},
649 {NAME("nreslabs"), CTL(stats_arenas_i_bins_j_nreslabs)},
650 {NAME("curslabs"), CTL(stats_arenas_i_bins_j_curslabs)},
651 {NAME("nonfull_slabs"), CTL(stats_arenas_i_bins_j_nonfull_slabs)},
652 {NAME("mutex"), CHILD(named, stats_arenas_i_bins_j_mutex)}
653};
654
655static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = {
656 {NAME(""), CHILD(named, stats_arenas_i_bins_j)}
657};
658
659static const ctl_indexed_node_t stats_arenas_i_bins_node[] = {
660 {INDEX(stats_arenas_i_bins_j)}
661};
662
663static const ctl_named_node_t stats_arenas_i_lextents_j_node[] = {
664 {NAME("nmalloc"), CTL(stats_arenas_i_lextents_j_nmalloc)},
665 {NAME("ndalloc"), CTL(stats_arenas_i_lextents_j_ndalloc)},
666 {NAME("nrequests"), CTL(stats_arenas_i_lextents_j_nrequests)},
667 {NAME("curlextents"), CTL(stats_arenas_i_lextents_j_curlextents)}
668};
669static const ctl_named_node_t super_stats_arenas_i_lextents_j_node[] = {
670 {NAME(""), CHILD(named, stats_arenas_i_lextents_j)}
671};
672
673static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = {
674 {INDEX(stats_arenas_i_lextents_j)}
675};
676
677static const ctl_named_node_t stats_arenas_i_extents_j_node[] = {
678 {NAME("ndirty"), CTL(stats_arenas_i_extents_j_ndirty)},
679 {NAME("nmuzzy"), CTL(stats_arenas_i_extents_j_nmuzzy)},
680 {NAME("nretained"), CTL(stats_arenas_i_extents_j_nretained)},
681 {NAME("dirty_bytes"), CTL(stats_arenas_i_extents_j_dirty_bytes)},
682 {NAME("muzzy_bytes"), CTL(stats_arenas_i_extents_j_muzzy_bytes)},
683 {NAME("retained_bytes"), CTL(stats_arenas_i_extents_j_retained_bytes)}
684};
685
686static const ctl_named_node_t super_stats_arenas_i_extents_j_node[] = {
687 {NAME(""), CHILD(named, stats_arenas_i_extents_j)}
688};
689
690static const ctl_indexed_node_t stats_arenas_i_extents_node[] = {
691 {INDEX(stats_arenas_i_extents_j)}
692};
693
694#define OP(mtx) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx)
695MUTEX_PROF_ARENA_MUTEXES
696#undef OP
697
698static const ctl_named_node_t stats_arenas_i_mutexes_node[] = {
699#define OP(mtx) {NAME(#mtx), CHILD(named, stats_arenas_i_mutexes_##mtx)},
700MUTEX_PROF_ARENA_MUTEXES
701#undef OP
702};
703
704static const ctl_named_node_t stats_arenas_i_hpa_shard_full_slabs_node[] = {
705 {NAME("npageslabs_nonhuge"),
706 CTL(stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge)},
707 {NAME("npageslabs_huge"),
708 CTL(stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge)},
709 {NAME("nactive_nonhuge"),
710 CTL(stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge)},
711 {NAME("nactive_huge"),
712 CTL(stats_arenas_i_hpa_shard_full_slabs_nactive_huge)},
713 {NAME("ndirty_nonhuge"),
714 CTL(stats_arenas_i_hpa_shard_full_slabs_ndirty_nonhuge)},
715 {NAME("ndirty_huge"),
716 CTL(stats_arenas_i_hpa_shard_full_slabs_ndirty_huge)}
717};
718
719static const ctl_named_node_t stats_arenas_i_hpa_shard_empty_slabs_node[] = {
720 {NAME("npageslabs_nonhuge"),
721 CTL(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_nonhuge)},
722 {NAME("npageslabs_huge"),
723 CTL(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_huge)},
724 {NAME("nactive_nonhuge"),
725 CTL(stats_arenas_i_hpa_shard_empty_slabs_nactive_nonhuge)},
726 {NAME("nactive_huge"),
727 CTL(stats_arenas_i_hpa_shard_empty_slabs_nactive_huge)},
728 {NAME("ndirty_nonhuge"),
729 CTL(stats_arenas_i_hpa_shard_empty_slabs_ndirty_nonhuge)},
730 {NAME("ndirty_huge"),
731 CTL(stats_arenas_i_hpa_shard_empty_slabs_ndirty_huge)}
732};
733
734static const ctl_named_node_t stats_arenas_i_hpa_shard_nonfull_slabs_j_node[] = {
735 {NAME("npageslabs_nonhuge"),
736 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge)},
737 {NAME("npageslabs_huge"),
738 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge)},
739 {NAME("nactive_nonhuge"),
740 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge)},
741 {NAME("nactive_huge"),
742 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge)},
743 {NAME("ndirty_nonhuge"),
744 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge)},
745 {NAME("ndirty_huge"),
746 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge)}
747};
748
749static const ctl_named_node_t super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node[] = {
750 {NAME(""),
751 CHILD(named, stats_arenas_i_hpa_shard_nonfull_slabs_j)}
752};
753
754static const ctl_indexed_node_t stats_arenas_i_hpa_shard_nonfull_slabs_node[] =
755{
756 {INDEX(stats_arenas_i_hpa_shard_nonfull_slabs_j)}
757};
758
759static const ctl_named_node_t stats_arenas_i_hpa_shard_node[] = {
760 {NAME("full_slabs"), CHILD(named,
761 stats_arenas_i_hpa_shard_full_slabs)},
762 {NAME("empty_slabs"), CHILD(named,
763 stats_arenas_i_hpa_shard_empty_slabs)},
764 {NAME("nonfull_slabs"), CHILD(indexed,
765 stats_arenas_i_hpa_shard_nonfull_slabs)},
766
767 {NAME("npurge_passes"), CTL(stats_arenas_i_hpa_shard_npurge_passes)},
768 {NAME("npurges"), CTL(stats_arenas_i_hpa_shard_npurges)},
769 {NAME("nhugifies"), CTL(stats_arenas_i_hpa_shard_nhugifies)},
770 {NAME("ndehugifies"), CTL(stats_arenas_i_hpa_shard_ndehugifies)}
771};
772
773static const ctl_named_node_t stats_arenas_i_node[] = {
774 {NAME("nthreads"), CTL(stats_arenas_i_nthreads)},
775 {NAME("uptime"), CTL(stats_arenas_i_uptime)},
776 {NAME("dss"), CTL(stats_arenas_i_dss)},
777 {NAME("dirty_decay_ms"), CTL(stats_arenas_i_dirty_decay_ms)},
778 {NAME("muzzy_decay_ms"), CTL(stats_arenas_i_muzzy_decay_ms)},
779 {NAME("pactive"), CTL(stats_arenas_i_pactive)},
780 {NAME("pdirty"), CTL(stats_arenas_i_pdirty)},
781 {NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)},
782 {NAME("mapped"), CTL(stats_arenas_i_mapped)},
783 {NAME("retained"), CTL(stats_arenas_i_retained)},
784 {NAME("extent_avail"), CTL(stats_arenas_i_extent_avail)},
785 {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)},
786 {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)},
787 {NAME("dirty_purged"), CTL(stats_arenas_i_dirty_purged)},
788 {NAME("muzzy_npurge"), CTL(stats_arenas_i_muzzy_npurge)},
789 {NAME("muzzy_nmadvise"), CTL(stats_arenas_i_muzzy_nmadvise)},
790 {NAME("muzzy_purged"), CTL(stats_arenas_i_muzzy_purged)},
791 {NAME("base"), CTL(stats_arenas_i_base)},
792 {NAME("internal"), CTL(stats_arenas_i_internal)},
793 {NAME("metadata_thp"), CTL(stats_arenas_i_metadata_thp)},
794 {NAME("tcache_bytes"), CTL(stats_arenas_i_tcache_bytes)},
795 {NAME("tcache_stashed_bytes"),
796 CTL(stats_arenas_i_tcache_stashed_bytes)},
797 {NAME("resident"), CTL(stats_arenas_i_resident)},
798 {NAME("abandoned_vm"), CTL(stats_arenas_i_abandoned_vm)},
799 {NAME("hpa_sec_bytes"), CTL(stats_arenas_i_hpa_sec_bytes)},
800 {NAME("small"), CHILD(named, stats_arenas_i_small)},
801 {NAME("large"), CHILD(named, stats_arenas_i_large)},
802 {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)},
803 {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)},
804 {NAME("extents"), CHILD(indexed, stats_arenas_i_extents)},
805 {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)},
806 {NAME("hpa_shard"), CHILD(named, stats_arenas_i_hpa_shard)}
807};
808static const ctl_named_node_t super_stats_arenas_i_node[] = {
809 {NAME(""), CHILD(named, stats_arenas_i)}
810};
811
812static const ctl_indexed_node_t stats_arenas_node[] = {
813 {INDEX(stats_arenas_i)}
814};
815
816static const ctl_named_node_t stats_background_thread_node[] = {
817 {NAME("num_threads"), CTL(stats_background_thread_num_threads)},
818 {NAME("num_runs"), CTL(stats_background_thread_num_runs)},
819 {NAME("run_interval"), CTL(stats_background_thread_run_interval)}
820};
821
822#define OP(mtx) MUTEX_PROF_DATA_NODE(mutexes_##mtx)
823MUTEX_PROF_GLOBAL_MUTEXES
824#undef OP
825
826static const ctl_named_node_t stats_mutexes_node[] = {
827#define OP(mtx) {NAME(#mtx), CHILD(named, stats_mutexes_##mtx)},
828MUTEX_PROF_GLOBAL_MUTEXES
829#undef OP
830 {NAME("reset"), CTL(stats_mutexes_reset)}
831};
832#undef MUTEX_PROF_DATA_NODE
833
834static const ctl_named_node_t stats_node[] = {
835 {NAME("allocated"), CTL(stats_allocated)},
836 {NAME("active"), CTL(stats_active)},
837 {NAME("metadata"), CTL(stats_metadata)},
838 {NAME("metadata_thp"), CTL(stats_metadata_thp)},
839 {NAME("resident"), CTL(stats_resident)},
840 {NAME("mapped"), CTL(stats_mapped)},
841 {NAME("retained"), CTL(stats_retained)},
842 {NAME("background_thread"),
843 CHILD(named, stats_background_thread)},
844 {NAME("mutexes"), CHILD(named, stats_mutexes)},
845 {NAME("arenas"), CHILD(indexed, stats_arenas)},
846 {NAME("zero_reallocs"), CTL(stats_zero_reallocs)},
847};
848
849static const ctl_named_node_t experimental_hooks_node[] = {
850 {NAME("install"), CTL(experimental_hooks_install)},
851 {NAME("remove"), CTL(experimental_hooks_remove)},
852 {NAME("prof_backtrace"), CTL(experimental_hooks_prof_backtrace)},
853 {NAME("prof_dump"), CTL(experimental_hooks_prof_dump)},
854 {NAME("safety_check_abort"), CTL(experimental_hooks_safety_check_abort)},
855};
856
857static const ctl_named_node_t experimental_thread_node[] = {
858 {NAME("activity_callback"),
859 CTL(experimental_thread_activity_callback)}
860};
861
862static const ctl_named_node_t experimental_utilization_node[] = {
863 {NAME("query"), CTL(experimental_utilization_query)},
864 {NAME("batch_query"), CTL(experimental_utilization_batch_query)}
865};
866
867static const ctl_named_node_t experimental_arenas_i_node[] = {
868 {NAME("pactivep"), CTL(experimental_arenas_i_pactivep)}
869};
870static const ctl_named_node_t super_experimental_arenas_i_node[] = {
871 {NAME(""), CHILD(named, experimental_arenas_i)}
872};
873
874static const ctl_indexed_node_t experimental_arenas_node[] = {
875 {INDEX(experimental_arenas_i)}
876};
877
878static const ctl_named_node_t experimental_prof_recent_node[] = {
879 {NAME("alloc_max"), CTL(experimental_prof_recent_alloc_max)},
880 {NAME("alloc_dump"), CTL(experimental_prof_recent_alloc_dump)},
881};
882
883static const ctl_named_node_t experimental_node[] = {
884 {NAME("hooks"), CHILD(named, experimental_hooks)},
885 {NAME("utilization"), CHILD(named, experimental_utilization)},
886 {NAME("arenas"), CHILD(indexed, experimental_arenas)},
887 {NAME("arenas_create_ext"), CTL(experimental_arenas_create_ext)},
888 {NAME("prof_recent"), CHILD(named, experimental_prof_recent)},
889 {NAME("batch_alloc"), CTL(experimental_batch_alloc)},
890 {NAME("thread"), CHILD(named, experimental_thread)}
891};
892
893static const ctl_named_node_t root_node[] = {
894 {NAME("version"), CTL(version)},
895 {NAME("epoch"), CTL(epoch)},
896 {NAME("background_thread"), CTL(background_thread)},
897 {NAME("max_background_threads"), CTL(max_background_threads)},
898 {NAME("thread"), CHILD(named, thread)},
899 {NAME("config"), CHILD(named, config)},
900 {NAME("opt"), CHILD(named, opt)},
901 {NAME("tcache"), CHILD(named, tcache)},
902 {NAME("arena"), CHILD(indexed, arena)},
903 {NAME("arenas"), CHILD(named, arenas)},
904 {NAME("prof"), CHILD(named, prof)},
905 {NAME("stats"), CHILD(named, stats)},
906 {NAME("experimental"), CHILD(named, experimental)}
907};
908static const ctl_named_node_t super_root_node[] = {
909 {NAME(""), CHILD(named, root)}
910};
911
912#undef NAME
913#undef CHILD
914#undef CTL
915#undef INDEX
916
917/******************************************************************************/
918
919/*
920 * Sets *dst + *src non-atomically. This is safe, since everything is
921 * synchronized by the ctl mutex.
922 */
923static void
924ctl_accum_locked_u64(locked_u64_t *dst, locked_u64_t *src) {
925 locked_inc_u64_unsynchronized(dst,
926 locked_read_u64_unsynchronized(src));
927}
928
929static void
930ctl_accum_atomic_zu(atomic_zu_t *dst, atomic_zu_t *src) {
931 size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED);
932 size_t cur_src = atomic_load_zu(src, ATOMIC_RELAXED);
933 atomic_store_zu(dst, cur_dst + cur_src, ATOMIC_RELAXED);
934}
935
936/******************************************************************************/
937
938static unsigned
939arenas_i2a_impl(size_t i, bool compat, bool validate) {
940 unsigned a;
941
942 switch (i) {
943 case MALLCTL_ARENAS_ALL:
944 a = 0;
945 break;
946 case MALLCTL_ARENAS_DESTROYED:
947 a = 1;
948 break;
949 default:
950 if (compat && i == ctl_arenas->narenas) {
951 /*
952 * Provide deprecated backward compatibility for
953 * accessing the merged stats at index narenas rather
954 * than via MALLCTL_ARENAS_ALL. This is scheduled for
955 * removal in 6.0.0.
956 */
957 a = 0;
958 } else if (validate && i >= ctl_arenas->narenas) {
959 a = UINT_MAX;
960 } else {
961 /*
962 * This function should never be called for an index
963 * more than one past the range of indices that have
964 * initialized ctl data.
965 */
966 assert(i < ctl_arenas->narenas || (!validate && i ==
967 ctl_arenas->narenas));
968 a = (unsigned)i + 2;
969 }
970 break;
971 }
972
973 return a;
974}
975
976static unsigned
977arenas_i2a(size_t i) {
978 return arenas_i2a_impl(i, true, false);
979}
980
981static ctl_arena_t *
982arenas_i_impl(tsd_t *tsd, size_t i, bool compat, bool init) {
983 ctl_arena_t *ret;
984
985 assert(!compat || !init);
986
987 ret = ctl_arenas->arenas[arenas_i2a_impl(i, compat, false)];
988 if (init && ret == NULL) {
989 if (config_stats) {
990 struct container_s {
991 ctl_arena_t ctl_arena;
992 ctl_arena_stats_t astats;
993 };
994 struct container_s *cont =
995 (struct container_s *)base_alloc(tsd_tsdn(tsd),
996 b0get(), sizeof(struct container_s), QUANTUM);
997 if (cont == NULL) {
998 return NULL;
999 }
1000 ret = &cont->ctl_arena;
1001 ret->astats = &cont->astats;
1002 } else {
1003 ret = (ctl_arena_t *)base_alloc(tsd_tsdn(tsd), b0get(),
1004 sizeof(ctl_arena_t), QUANTUM);
1005 if (ret == NULL) {
1006 return NULL;
1007 }
1008 }
1009 ret->arena_ind = (unsigned)i;
1010 ctl_arenas->arenas[arenas_i2a_impl(i, compat, false)] = ret;
1011 }
1012
1013 assert(ret == NULL || arenas_i2a(ret->arena_ind) == arenas_i2a(i));
1014 return ret;
1015}
1016
1017static ctl_arena_t *
1018arenas_i(size_t i) {
1019 ctl_arena_t *ret = arenas_i_impl(tsd_fetch(), i, true, false);
1020 assert(ret != NULL);
1021 return ret;
1022}
1023
1024static void
1025ctl_arena_clear(ctl_arena_t *ctl_arena) {
1026 ctl_arena->nthreads = 0;
1027 ctl_arena->dss = dss_prec_names[dss_prec_limit];
1028 ctl_arena->dirty_decay_ms = -1;
1029 ctl_arena->muzzy_decay_ms = -1;
1030 ctl_arena->pactive = 0;
1031 ctl_arena->pdirty = 0;
1032 ctl_arena->pmuzzy = 0;
1033 if (config_stats) {
1034 memset(&ctl_arena->astats->astats, 0, sizeof(arena_stats_t));
1035 ctl_arena->astats->allocated_small = 0;
1036 ctl_arena->astats->nmalloc_small = 0;
1037 ctl_arena->astats->ndalloc_small = 0;
1038 ctl_arena->astats->nrequests_small = 0;
1039 ctl_arena->astats->nfills_small = 0;
1040 ctl_arena->astats->nflushes_small = 0;
1041 memset(ctl_arena->astats->bstats, 0, SC_NBINS *
1042 sizeof(bin_stats_data_t));
1043 memset(ctl_arena->astats->lstats, 0, (SC_NSIZES - SC_NBINS) *
1044 sizeof(arena_stats_large_t));
1045 memset(ctl_arena->astats->estats, 0, SC_NPSIZES *
1046 sizeof(pac_estats_t));
1047 memset(&ctl_arena->astats->hpastats, 0,
1048 sizeof(hpa_shard_stats_t));
1049 memset(&ctl_arena->astats->secstats, 0,
1050 sizeof(sec_stats_t));
1051 }
1052}
1053
1054static void
1055ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) {
1056 unsigned i;
1057
1058 if (config_stats) {
1059 arena_stats_merge(tsdn, arena, &ctl_arena->nthreads,
1060 &ctl_arena->dss, &ctl_arena->dirty_decay_ms,
1061 &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive,
1062 &ctl_arena->pdirty, &ctl_arena->pmuzzy,
1063 &ctl_arena->astats->astats, ctl_arena->astats->bstats,
1064 ctl_arena->astats->lstats, ctl_arena->astats->estats,
1065 &ctl_arena->astats->hpastats, &ctl_arena->astats->secstats);
1066
1067 for (i = 0; i < SC_NBINS; i++) {
1068 bin_stats_t *bstats =
1069 &ctl_arena->astats->bstats[i].stats_data;
1070 ctl_arena->astats->allocated_small += bstats->curregs *
1071 sz_index2size(i);
1072 ctl_arena->astats->nmalloc_small += bstats->nmalloc;
1073 ctl_arena->astats->ndalloc_small += bstats->ndalloc;
1074 ctl_arena->astats->nrequests_small += bstats->nrequests;
1075 ctl_arena->astats->nfills_small += bstats->nfills;
1076 ctl_arena->astats->nflushes_small += bstats->nflushes;
1077 }
1078 } else {
1079 arena_basic_stats_merge(tsdn, arena, &ctl_arena->nthreads,
1080 &ctl_arena->dss, &ctl_arena->dirty_decay_ms,
1081 &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive,
1082 &ctl_arena->pdirty, &ctl_arena->pmuzzy);
1083 }
1084}
1085
1086static void
1087ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena,
1088 bool destroyed) {
1089 unsigned i;
1090
1091 if (!destroyed) {
1092 ctl_sdarena->nthreads += ctl_arena->nthreads;
1093 ctl_sdarena->pactive += ctl_arena->pactive;
1094 ctl_sdarena->pdirty += ctl_arena->pdirty;
1095 ctl_sdarena->pmuzzy += ctl_arena->pmuzzy;
1096 } else {
1097 assert(ctl_arena->nthreads == 0);
1098 assert(ctl_arena->pactive == 0);
1099 assert(ctl_arena->pdirty == 0);
1100 assert(ctl_arena->pmuzzy == 0);
1101 }
1102
1103 if (config_stats) {
1104 ctl_arena_stats_t *sdstats = ctl_sdarena->astats;
1105 ctl_arena_stats_t *astats = ctl_arena->astats;
1106
1107 if (!destroyed) {
1108 sdstats->astats.mapped += astats->astats.mapped;
1109 sdstats->astats.pa_shard_stats.pac_stats.retained
1110 += astats->astats.pa_shard_stats.pac_stats.retained;
1111 sdstats->astats.pa_shard_stats.edata_avail
1112 += astats->astats.pa_shard_stats.edata_avail;
1113 }
1114
1115 ctl_accum_locked_u64(
1116 &sdstats->astats.pa_shard_stats.pac_stats.decay_dirty.npurge,
1117 &astats->astats.pa_shard_stats.pac_stats.decay_dirty.npurge);
1118 ctl_accum_locked_u64(
1119 &sdstats->astats.pa_shard_stats.pac_stats.decay_dirty.nmadvise,
1120 &astats->astats.pa_shard_stats.pac_stats.decay_dirty.nmadvise);
1121 ctl_accum_locked_u64(
1122 &sdstats->astats.pa_shard_stats.pac_stats.decay_dirty.purged,
1123 &astats->astats.pa_shard_stats.pac_stats.decay_dirty.purged);
1124
1125 ctl_accum_locked_u64(
1126 &sdstats->astats.pa_shard_stats.pac_stats.decay_muzzy.npurge,
1127 &astats->astats.pa_shard_stats.pac_stats.decay_muzzy.npurge);
1128 ctl_accum_locked_u64(
1129 &sdstats->astats.pa_shard_stats.pac_stats.decay_muzzy.nmadvise,
1130 &astats->astats.pa_shard_stats.pac_stats.decay_muzzy.nmadvise);
1131 ctl_accum_locked_u64(
1132 &sdstats->astats.pa_shard_stats.pac_stats.decay_muzzy.purged,
1133 &astats->astats.pa_shard_stats.pac_stats.decay_muzzy.purged);
1134
1135#define OP(mtx) malloc_mutex_prof_merge( \
1136 &(sdstats->astats.mutex_prof_data[ \
1137 arena_prof_mutex_##mtx]), \
1138 &(astats->astats.mutex_prof_data[ \
1139 arena_prof_mutex_##mtx]));
1140MUTEX_PROF_ARENA_MUTEXES
1141#undef OP
1142 if (!destroyed) {
1143 sdstats->astats.base += astats->astats.base;
1144 sdstats->astats.resident += astats->astats.resident;
1145 sdstats->astats.metadata_thp += astats->astats.metadata_thp;
1146 ctl_accum_atomic_zu(&sdstats->astats.internal,
1147 &astats->astats.internal);
1148 } else {
1149 assert(atomic_load_zu(
1150 &astats->astats.internal, ATOMIC_RELAXED) == 0);
1151 }
1152
1153 if (!destroyed) {
1154 sdstats->allocated_small += astats->allocated_small;
1155 } else {
1156 assert(astats->allocated_small == 0);
1157 }
1158 sdstats->nmalloc_small += astats->nmalloc_small;
1159 sdstats->ndalloc_small += astats->ndalloc_small;
1160 sdstats->nrequests_small += astats->nrequests_small;
1161 sdstats->nfills_small += astats->nfills_small;
1162 sdstats->nflushes_small += astats->nflushes_small;
1163
1164 if (!destroyed) {
1165 sdstats->astats.allocated_large +=
1166 astats->astats.allocated_large;
1167 } else {
1168 assert(astats->astats.allocated_large == 0);
1169 }
1170 sdstats->astats.nmalloc_large += astats->astats.nmalloc_large;
1171 sdstats->astats.ndalloc_large += astats->astats.ndalloc_large;
1172 sdstats->astats.nrequests_large
1173 += astats->astats.nrequests_large;
1174 sdstats->astats.nflushes_large += astats->astats.nflushes_large;
1175 ctl_accum_atomic_zu(
1176 &sdstats->astats.pa_shard_stats.pac_stats.abandoned_vm,
1177 &astats->astats.pa_shard_stats.pac_stats.abandoned_vm);
1178
1179 sdstats->astats.tcache_bytes += astats->astats.tcache_bytes;
1180 sdstats->astats.tcache_stashed_bytes +=
1181 astats->astats.tcache_stashed_bytes;
1182
1183 if (ctl_arena->arena_ind == 0) {
1184 sdstats->astats.uptime = astats->astats.uptime;
1185 }
1186
1187 /* Merge bin stats. */
1188 for (i = 0; i < SC_NBINS; i++) {
1189 bin_stats_t *bstats = &astats->bstats[i].stats_data;
1190 bin_stats_t *merged = &sdstats->bstats[i].stats_data;
1191 merged->nmalloc += bstats->nmalloc;
1192 merged->ndalloc += bstats->ndalloc;
1193 merged->nrequests += bstats->nrequests;
1194 if (!destroyed) {
1195 merged->curregs += bstats->curregs;
1196 } else {
1197 assert(bstats->curregs == 0);
1198 }
1199 merged->nfills += bstats->nfills;
1200 merged->nflushes += bstats->nflushes;
1201 merged->nslabs += bstats->nslabs;
1202 merged->reslabs += bstats->reslabs;
1203 if (!destroyed) {
1204 merged->curslabs += bstats->curslabs;
1205 merged->nonfull_slabs += bstats->nonfull_slabs;
1206 } else {
1207 assert(bstats->curslabs == 0);
1208 assert(bstats->nonfull_slabs == 0);
1209 }
1210 malloc_mutex_prof_merge(&sdstats->bstats[i].mutex_data,
1211 &astats->bstats[i].mutex_data);
1212 }
1213
1214 /* Merge stats for large allocations. */
1215 for (i = 0; i < SC_NSIZES - SC_NBINS; i++) {
1216 ctl_accum_locked_u64(&sdstats->lstats[i].nmalloc,
1217 &astats->lstats[i].nmalloc);
1218 ctl_accum_locked_u64(&sdstats->lstats[i].ndalloc,
1219 &astats->lstats[i].ndalloc);
1220 ctl_accum_locked_u64(&sdstats->lstats[i].nrequests,
1221 &astats->lstats[i].nrequests);
1222 if (!destroyed) {
1223 sdstats->lstats[i].curlextents +=
1224 astats->lstats[i].curlextents;
1225 } else {
1226 assert(astats->lstats[i].curlextents == 0);
1227 }
1228 }
1229
1230 /* Merge extents stats. */
1231 for (i = 0; i < SC_NPSIZES; i++) {
1232 sdstats->estats[i].ndirty += astats->estats[i].ndirty;
1233 sdstats->estats[i].nmuzzy += astats->estats[i].nmuzzy;
1234 sdstats->estats[i].nretained
1235 += astats->estats[i].nretained;
1236 sdstats->estats[i].dirty_bytes
1237 += astats->estats[i].dirty_bytes;
1238 sdstats->estats[i].muzzy_bytes
1239 += astats->estats[i].muzzy_bytes;
1240 sdstats->estats[i].retained_bytes
1241 += astats->estats[i].retained_bytes;
1242 }
1243
1244 /* Merge HPA stats. */
1245 hpa_shard_stats_accum(&sdstats->hpastats, &astats->hpastats);
1246 sec_stats_accum(&sdstats->secstats, &astats->secstats);
1247 }
1248}
1249
1250static void
1251ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, ctl_arena_t *ctl_sdarena,
1252 unsigned i, bool destroyed) {
1253 ctl_arena_t *ctl_arena = arenas_i(i);
1254
1255 ctl_arena_clear(ctl_arena);
1256 ctl_arena_stats_amerge(tsdn, ctl_arena, arena);
1257 /* Merge into sum stats as well. */
1258 ctl_arena_stats_sdmerge(ctl_sdarena, ctl_arena, destroyed);
1259}
1260
1261static unsigned
1262ctl_arena_init(tsd_t *tsd, const arena_config_t *config) {
1263 unsigned arena_ind;
1264 ctl_arena_t *ctl_arena;
1265
1266 if ((ctl_arena = ql_last(&ctl_arenas->destroyed, destroyed_link)) !=
1267 NULL) {
1268 ql_remove(&ctl_arenas->destroyed, ctl_arena, destroyed_link);
1269 arena_ind = ctl_arena->arena_ind;
1270 } else {
1271 arena_ind = ctl_arenas->narenas;
1272 }
1273
1274 /* Trigger stats allocation. */
1275 if (arenas_i_impl(tsd, arena_ind, false, true) == NULL) {
1276 return UINT_MAX;
1277 }
1278
1279 /* Initialize new arena. */
1280 if (arena_init(tsd_tsdn(tsd), arena_ind, config) == NULL) {
1281 return UINT_MAX;
1282 }
1283
1284 if (arena_ind == ctl_arenas->narenas) {
1285 ctl_arenas->narenas++;
1286 }
1287
1288 return arena_ind;
1289}
1290
1291static void
1292ctl_background_thread_stats_read(tsdn_t *tsdn) {
1293 background_thread_stats_t *stats = &ctl_stats->background_thread;
1294 if (!have_background_thread ||
1295 background_thread_stats_read(tsdn, stats)) {
1296 memset(stats, 0, sizeof(background_thread_stats_t));
1297 nstime_init_zero(&stats->run_interval);
1298 }
1299 malloc_mutex_prof_copy(
1300 &ctl_stats->mutex_prof_data[global_prof_mutex_max_per_bg_thd],
1301 &stats->max_counter_per_bg_thd);
1302}
1303
1304static void
1305ctl_refresh(tsdn_t *tsdn) {
1306 unsigned i;
1307 ctl_arena_t *ctl_sarena = arenas_i(MALLCTL_ARENAS_ALL);
1308 VARIABLE_ARRAY(arena_t *, tarenas, ctl_arenas->narenas);
1309
1310 /*
1311 * Clear sum stats, since they will be merged into by
1312 * ctl_arena_refresh().
1313 */
1314 ctl_arena_clear(ctl_sarena);
1315
1316 for (i = 0; i < ctl_arenas->narenas; i++) {
1317 tarenas[i] = arena_get(tsdn, i, false);
1318 }
1319
1320 for (i = 0; i < ctl_arenas->narenas; i++) {
1321 ctl_arena_t *ctl_arena = arenas_i(i);
1322 bool initialized = (tarenas[i] != NULL);
1323
1324 ctl_arena->initialized = initialized;
1325 if (initialized) {
1326 ctl_arena_refresh(tsdn, tarenas[i], ctl_sarena, i,
1327 false);
1328 }
1329 }
1330
1331 if (config_stats) {
1332 ctl_stats->allocated = ctl_sarena->astats->allocated_small +
1333 ctl_sarena->astats->astats.allocated_large;
1334 ctl_stats->active = (ctl_sarena->pactive << LG_PAGE);
1335 ctl_stats->metadata = ctl_sarena->astats->astats.base +
1336 atomic_load_zu(&ctl_sarena->astats->astats.internal,
1337 ATOMIC_RELAXED);
1338 ctl_stats->resident = ctl_sarena->astats->astats.resident;
1339 ctl_stats->metadata_thp =
1340 ctl_sarena->astats->astats.metadata_thp;
1341 ctl_stats->mapped = ctl_sarena->astats->astats.mapped;
1342 ctl_stats->retained = ctl_sarena->astats->astats
1343 .pa_shard_stats.pac_stats.retained;
1344
1345 ctl_background_thread_stats_read(tsdn);
1346
1347#define READ_GLOBAL_MUTEX_PROF_DATA(i, mtx) \
1348 malloc_mutex_lock(tsdn, &mtx); \
1349 malloc_mutex_prof_read(tsdn, &ctl_stats->mutex_prof_data[i], &mtx); \
1350 malloc_mutex_unlock(tsdn, &mtx);
1351
1352 if (config_prof && opt_prof) {
1353 READ_GLOBAL_MUTEX_PROF_DATA(
1354 global_prof_mutex_prof, bt2gctx_mtx);
1355 READ_GLOBAL_MUTEX_PROF_DATA(
1356 global_prof_mutex_prof_thds_data, tdatas_mtx);
1357 READ_GLOBAL_MUTEX_PROF_DATA(
1358 global_prof_mutex_prof_dump, prof_dump_mtx);
1359 READ_GLOBAL_MUTEX_PROF_DATA(
1360 global_prof_mutex_prof_recent_alloc,
1361 prof_recent_alloc_mtx);
1362 READ_GLOBAL_MUTEX_PROF_DATA(
1363 global_prof_mutex_prof_recent_dump,
1364 prof_recent_dump_mtx);
1365 READ_GLOBAL_MUTEX_PROF_DATA(
1366 global_prof_mutex_prof_stats, prof_stats_mtx);
1367 }
1368 if (have_background_thread) {
1369 READ_GLOBAL_MUTEX_PROF_DATA(
1370 global_prof_mutex_background_thread,
1371 background_thread_lock);
1372 } else {
1373 memset(&ctl_stats->mutex_prof_data[
1374 global_prof_mutex_background_thread], 0,
1375 sizeof(mutex_prof_data_t));
1376 }
1377 /* We own ctl mutex already. */
1378 malloc_mutex_prof_read(tsdn,
1379 &ctl_stats->mutex_prof_data[global_prof_mutex_ctl],
1380 &ctl_mtx);
1381#undef READ_GLOBAL_MUTEX_PROF_DATA
1382 }
1383 ctl_arenas->epoch++;
1384}
1385
1386static bool
1387ctl_init(tsd_t *tsd) {
1388 bool ret;
1389 tsdn_t *tsdn = tsd_tsdn(tsd);
1390
1391 malloc_mutex_lock(tsdn, &ctl_mtx);
1392 if (!ctl_initialized) {
1393 ctl_arena_t *ctl_sarena, *ctl_darena;
1394 unsigned i;
1395
1396 /*
1397 * Allocate demand-zeroed space for pointers to the full
1398 * range of supported arena indices.
1399 */
1400 if (ctl_arenas == NULL) {
1401 ctl_arenas = (ctl_arenas_t *)base_alloc(tsdn,
1402 b0get(), sizeof(ctl_arenas_t), QUANTUM);
1403 if (ctl_arenas == NULL) {
1404 ret = true;
1405 goto label_return;
1406 }
1407 }
1408
1409 if (config_stats && ctl_stats == NULL) {
1410 ctl_stats = (ctl_stats_t *)base_alloc(tsdn, b0get(),
1411 sizeof(ctl_stats_t), QUANTUM);
1412 if (ctl_stats == NULL) {
1413 ret = true;
1414 goto label_return;
1415 }
1416 }
1417
1418 /*
1419 * Allocate space for the current full range of arenas
1420 * here rather than doing it lazily elsewhere, in order
1421 * to limit when OOM-caused errors can occur.
1422 */
1423 if ((ctl_sarena = arenas_i_impl(tsd, MALLCTL_ARENAS_ALL, false,
1424 true)) == NULL) {
1425 ret = true;
1426 goto label_return;
1427 }
1428 ctl_sarena->initialized = true;
1429
1430 if ((ctl_darena = arenas_i_impl(tsd, MALLCTL_ARENAS_DESTROYED,
1431 false, true)) == NULL) {
1432 ret = true;
1433 goto label_return;
1434 }
1435 ctl_arena_clear(ctl_darena);
1436 /*
1437 * Don't toggle ctl_darena to initialized until an arena is
1438 * actually destroyed, so that arena.<i>.initialized can be used
1439 * to query whether the stats are relevant.
1440 */
1441
1442 ctl_arenas->narenas = narenas_total_get();
1443 for (i = 0; i < ctl_arenas->narenas; i++) {
1444 if (arenas_i_impl(tsd, i, false, true) == NULL) {
1445 ret = true;
1446 goto label_return;
1447 }
1448 }
1449
1450 ql_new(&ctl_arenas->destroyed);
1451 ctl_refresh(tsdn);
1452
1453 ctl_initialized = true;
1454 }
1455
1456 ret = false;
1457label_return:
1458 malloc_mutex_unlock(tsdn, &ctl_mtx);
1459 return ret;
1460}
1461
1462static int
1463ctl_lookup(tsdn_t *tsdn, const ctl_named_node_t *starting_node,
1464 const char *name, const ctl_named_node_t **ending_nodep, size_t *mibp,
1465 size_t *depthp) {
1466 int ret;
1467 const char *elm, *tdot, *dot;
1468 size_t elen, i, j;
1469 const ctl_named_node_t *node;
1470
1471 elm = name;
1472 /* Equivalent to strchrnul(). */
1473 dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : strchr(elm, '\0');
1474 elen = (size_t)((uintptr_t)dot - (uintptr_t)elm);
1475 if (elen == 0) {
1476 ret = ENOENT;
1477 goto label_return;
1478 }
1479 node = starting_node;
1480 for (i = 0; i < *depthp; i++) {
1481 assert(node);
1482 assert(node->nchildren > 0);
1483 if (ctl_named_node(node->children) != NULL) {
1484 const ctl_named_node_t *pnode = node;
1485
1486 /* Children are named. */
1487 for (j = 0; j < node->nchildren; j++) {
1488 const ctl_named_node_t *child =
1489 ctl_named_children(node, j);
1490 if (strlen(child->name) == elen &&
1491 strncmp(elm, child->name, elen) == 0) {
1492 node = child;
1493 mibp[i] = j;
1494 break;
1495 }
1496 }
1497 if (node == pnode) {
1498 ret = ENOENT;
1499 goto label_return;
1500 }
1501 } else {
1502 uintmax_t index;
1503 const ctl_indexed_node_t *inode;
1504
1505 /* Children are indexed. */
1506 index = malloc_strtoumax(elm, NULL, 10);
1507 if (index == UINTMAX_MAX || index > SIZE_T_MAX) {
1508 ret = ENOENT;
1509 goto label_return;
1510 }
1511
1512 inode = ctl_indexed_node(node->children);
1513 node = inode->index(tsdn, mibp, *depthp, (size_t)index);
1514 if (node == NULL) {
1515 ret = ENOENT;
1516 goto label_return;
1517 }
1518
1519 mibp[i] = (size_t)index;
1520 }
1521
1522 /* Reached the end? */
1523 if (node->ctl != NULL || *dot == '\0') {
1524 /* Terminal node. */
1525 if (*dot != '\0') {
1526 /*
1527 * The name contains more elements than are
1528 * in this path through the tree.
1529 */
1530 ret = ENOENT;
1531 goto label_return;
1532 }
1533 /* Complete lookup successful. */
1534 *depthp = i + 1;
1535 break;
1536 }
1537
1538 /* Update elm. */
1539 elm = &dot[1];
1540 dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot :
1541 strchr(elm, '\0');
1542 elen = (size_t)((uintptr_t)dot - (uintptr_t)elm);
1543 }
1544 if (ending_nodep != NULL) {
1545 *ending_nodep = node;
1546 }
1547
1548 ret = 0;
1549label_return:
1550 return ret;
1551}
1552
1553int
1554ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp,
1555 void *newp, size_t newlen) {
1556 int ret;
1557 size_t depth;
1558 size_t mib[CTL_MAX_DEPTH];
1559 const ctl_named_node_t *node;
1560
1561 if (!ctl_initialized && ctl_init(tsd)) {
1562 ret = EAGAIN;
1563 goto label_return;
1564 }
1565
1566 depth = CTL_MAX_DEPTH;
1567 ret = ctl_lookup(tsd_tsdn(tsd), super_root_node, name, &node, mib,
1568 &depth);
1569 if (ret != 0) {
1570 goto label_return;
1571 }
1572
1573 if (node != NULL && node->ctl) {
1574 ret = node->ctl(tsd, mib, depth, oldp, oldlenp, newp, newlen);
1575 } else {
1576 /* The name refers to a partial path through the ctl tree. */
1577 ret = ENOENT;
1578 }
1579
1580label_return:
1581 return(ret);
1582}
1583
1584int
1585ctl_nametomib(tsd_t *tsd, const char *name, size_t *mibp, size_t *miblenp) {
1586 int ret;
1587
1588 if (!ctl_initialized && ctl_init(tsd)) {
1589 ret = EAGAIN;
1590 goto label_return;
1591 }
1592
1593 ret = ctl_lookup(tsd_tsdn(tsd), super_root_node, name, NULL, mibp,
1594 miblenp);
1595label_return:
1596 return(ret);
1597}
1598
1599static int
1600ctl_lookupbymib(tsdn_t *tsdn, const ctl_named_node_t **ending_nodep,
1601 const size_t *mib, size_t miblen) {
1602 int ret;
1603
1604 const ctl_named_node_t *node = super_root_node;
1605 for (size_t i = 0; i < miblen; i++) {
1606 assert(node);
1607 assert(node->nchildren > 0);
1608 if (ctl_named_node(node->children) != NULL) {
1609 /* Children are named. */
1610 if (node->nchildren <= mib[i]) {
1611 ret = ENOENT;
1612 goto label_return;
1613 }
1614 node = ctl_named_children(node, mib[i]);
1615 } else {
1616 const ctl_indexed_node_t *inode;
1617
1618 /* Indexed element. */
1619 inode = ctl_indexed_node(node->children);
1620 node = inode->index(tsdn, mib, miblen, mib[i]);
1621 if (node == NULL) {
1622 ret = ENOENT;
1623 goto label_return;
1624 }
1625 }
1626 }
1627 assert(ending_nodep != NULL);
1628 *ending_nodep = node;
1629 ret = 0;
1630
1631label_return:
1632 return(ret);
1633}
1634
1635int
1636ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
1637 size_t *oldlenp, void *newp, size_t newlen) {
1638 int ret;
1639 const ctl_named_node_t *node;
1640
1641 if (!ctl_initialized && ctl_init(tsd)) {
1642 ret = EAGAIN;
1643 goto label_return;
1644 }
1645
1646 ret = ctl_lookupbymib(tsd_tsdn(tsd), &node, mib, miblen);
1647 if (ret != 0) {
1648 goto label_return;
1649 }
1650
1651 /* Call the ctl function. */
1652 if (node && node->ctl) {
1653 ret = node->ctl(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
1654 } else {
1655 /* Partial MIB. */
1656 ret = ENOENT;
1657 }
1658
1659label_return:
1660 return(ret);
1661}
1662
1663int
1664ctl_mibnametomib(tsd_t *tsd, size_t *mib, size_t miblen, const char *name,
1665 size_t *miblenp) {
1666 int ret;
1667 const ctl_named_node_t *node;
1668
1669 if (!ctl_initialized && ctl_init(tsd)) {
1670 ret = EAGAIN;
1671 goto label_return;
1672 }
1673
1674 ret = ctl_lookupbymib(tsd_tsdn(tsd), &node, mib, miblen);
1675 if (ret != 0) {
1676 goto label_return;
1677 }
1678 if (node == NULL || node->ctl != NULL) {
1679 ret = ENOENT;
1680 goto label_return;
1681 }
1682
1683 assert(miblenp != NULL);
1684 assert(*miblenp >= miblen);
1685 *miblenp -= miblen;
1686 ret = ctl_lookup(tsd_tsdn(tsd), node, name, NULL, mib + miblen,
1687 miblenp);
1688 *miblenp += miblen;
1689label_return:
1690 return(ret);
1691}
1692
1693int
1694ctl_bymibname(tsd_t *tsd, size_t *mib, size_t miblen, const char *name,
1695 size_t *miblenp, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
1696 int ret;
1697 const ctl_named_node_t *node;
1698
1699 if (!ctl_initialized && ctl_init(tsd)) {
1700 ret = EAGAIN;
1701 goto label_return;
1702 }
1703
1704 ret = ctl_lookupbymib(tsd_tsdn(tsd), &node, mib, miblen);
1705 if (ret != 0) {
1706 goto label_return;
1707 }
1708 if (node == NULL || node->ctl != NULL) {
1709 ret = ENOENT;
1710 goto label_return;
1711 }
1712
1713 assert(miblenp != NULL);
1714 assert(*miblenp >= miblen);
1715 *miblenp -= miblen;
1716 /*
1717 * The same node supplies the starting node and stores the ending node.
1718 */
1719 ret = ctl_lookup(tsd_tsdn(tsd), node, name, &node, mib + miblen,
1720 miblenp);
1721 *miblenp += miblen;
1722 if (ret != 0) {
1723 goto label_return;
1724 }
1725
1726 if (node != NULL && node->ctl) {
1727 ret = node->ctl(tsd, mib, *miblenp, oldp, oldlenp, newp,
1728 newlen);
1729 } else {
1730 /* The name refers to a partial path through the ctl tree. */
1731 ret = ENOENT;
1732 }
1733
1734label_return:
1735 return(ret);
1736}
1737
1738bool
1739ctl_boot(void) {
1740 if (malloc_mutex_init(&ctl_mtx, "ctl", WITNESS_RANK_CTL,
1741 malloc_mutex_rank_exclusive)) {
1742 return true;
1743 }
1744
1745 ctl_initialized = false;
1746
1747 return false;
1748}
1749
1750void
1751ctl_prefork(tsdn_t *tsdn) {
1752 malloc_mutex_prefork(tsdn, &ctl_mtx);
1753}
1754
1755void
1756ctl_postfork_parent(tsdn_t *tsdn) {
1757 malloc_mutex_postfork_parent(tsdn, &ctl_mtx);
1758}
1759
1760void
1761ctl_postfork_child(tsdn_t *tsdn) {
1762 malloc_mutex_postfork_child(tsdn, &ctl_mtx);
1763}
1764
1765void
1766ctl_mtx_assert_held(tsdn_t *tsdn) {
1767 malloc_mutex_assert_owner(tsdn, &ctl_mtx);
1768}
1769
1770/******************************************************************************/
1771/* *_ctl() functions. */
1772
1773#define READONLY() do { \
1774 if (newp != NULL || newlen != 0) { \
1775 ret = EPERM; \
1776 goto label_return; \
1777 } \
1778} while (0)
1779
1780#define WRITEONLY() do { \
1781 if (oldp != NULL || oldlenp != NULL) { \
1782 ret = EPERM; \
1783 goto label_return; \
1784 } \
1785} while (0)
1786
1787/* Can read or write, but not both. */
1788#define READ_XOR_WRITE() do { \
1789 if ((oldp != NULL && oldlenp != NULL) && (newp != NULL || \
1790 newlen != 0)) { \
1791 ret = EPERM; \
1792 goto label_return; \
1793 } \
1794} while (0)
1795
1796/* Can neither read nor write. */
1797#define NEITHER_READ_NOR_WRITE() do { \
1798 if (oldp != NULL || oldlenp != NULL || newp != NULL || \
1799 newlen != 0) { \
1800 ret = EPERM; \
1801 goto label_return; \
1802 } \
1803} while (0)
1804
1805/* Verify that the space provided is enough. */
1806#define VERIFY_READ(t) do { \
1807 if (oldp == NULL || oldlenp == NULL || *oldlenp != sizeof(t)) { \
1808 *oldlenp = 0; \
1809 ret = EINVAL; \
1810 goto label_return; \
1811 } \
1812} while (0)
1813
1814#define READ(v, t) do { \
1815 if (oldp != NULL && oldlenp != NULL) { \
1816 if (*oldlenp != sizeof(t)) { \
1817 size_t copylen = (sizeof(t) <= *oldlenp) \
1818 ? sizeof(t) : *oldlenp; \
1819 memcpy(oldp, (void *)&(v), copylen); \
1820 *oldlenp = copylen; \
1821 ret = EINVAL; \
1822 goto label_return; \
1823 } \
1824 *(t *)oldp = (v); \
1825 } \
1826} while (0)
1827
1828#define WRITE(v, t) do { \
1829 if (newp != NULL) { \
1830 if (newlen != sizeof(t)) { \
1831 ret = EINVAL; \
1832 goto label_return; \
1833 } \
1834 (v) = *(t *)newp; \
1835 } \
1836} while (0)
1837
1838#define ASSURED_WRITE(v, t) do { \
1839 if (newp == NULL || newlen != sizeof(t)) { \
1840 ret = EINVAL; \
1841 goto label_return; \
1842 } \
1843 (v) = *(t *)newp; \
1844} while (0)
1845
1846#define MIB_UNSIGNED(v, i) do { \
1847 if (mib[i] > UINT_MAX) { \
1848 ret = EFAULT; \
1849 goto label_return; \
1850 } \
1851 v = (unsigned)mib[i]; \
1852} while (0)
1853
1854/*
1855 * There's a lot of code duplication in the following macros due to limitations
1856 * in how nested cpp macros are expanded.
1857 */
1858#define CTL_RO_CLGEN(c, l, n, v, t) \
1859static int \
1860n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
1861 size_t *oldlenp, void *newp, size_t newlen) { \
1862 int ret; \
1863 t oldval; \
1864 \
1865 if (!(c)) { \
1866 return ENOENT; \
1867 } \
1868 if (l) { \
1869 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \
1870 } \
1871 READONLY(); \
1872 oldval = (v); \
1873 READ(oldval, t); \
1874 \
1875 ret = 0; \
1876label_return: \
1877 if (l) { \
1878 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \
1879 } \
1880 return ret; \
1881}
1882
1883#define CTL_RO_CGEN(c, n, v, t) \
1884static int \
1885n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
1886 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \
1887 int ret; \
1888 t oldval; \
1889 \
1890 if (!(c)) { \
1891 return ENOENT; \
1892 } \
1893 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \
1894 READONLY(); \
1895 oldval = (v); \
1896 READ(oldval, t); \
1897 \
1898 ret = 0; \
1899label_return: \
1900 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \
1901 return ret; \
1902}
1903
1904#define CTL_RO_GEN(n, v, t) \
1905static int \
1906n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
1907 size_t *oldlenp, void *newp, size_t newlen) { \
1908 int ret; \
1909 t oldval; \
1910 \
1911 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \
1912 READONLY(); \
1913 oldval = (v); \
1914 READ(oldval, t); \
1915 \
1916 ret = 0; \
1917label_return: \
1918 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \
1919 return ret; \
1920}
1921
1922/*
1923 * ctl_mtx is not acquired, under the assumption that no pertinent data will
1924 * mutate during the call.
1925 */
1926#define CTL_RO_NL_CGEN(c, n, v, t) \
1927static int \
1928n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
1929 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \
1930 int ret; \
1931 t oldval; \
1932 \
1933 if (!(c)) { \
1934 return ENOENT; \
1935 } \
1936 READONLY(); \
1937 oldval = (v); \
1938 READ(oldval, t); \
1939 \
1940 ret = 0; \
1941label_return: \
1942 return ret; \
1943}
1944
1945#define CTL_RO_NL_GEN(n, v, t) \
1946static int \
1947n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
1948 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \
1949 int ret; \
1950 t oldval; \
1951 \
1952 READONLY(); \
1953 oldval = (v); \
1954 READ(oldval, t); \
1955 \
1956 ret = 0; \
1957label_return: \
1958 return ret; \
1959}
1960
1961#define CTL_RO_CONFIG_GEN(n, t) \
1962static int \
1963n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
1964 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \
1965 int ret; \
1966 t oldval; \
1967 \
1968 READONLY(); \
1969 oldval = n; \
1970 READ(oldval, t); \
1971 \
1972 ret = 0; \
1973label_return: \
1974 return ret; \
1975}
1976
1977/******************************************************************************/
1978
1979CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *)
1980
1981static int
1982epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
1983 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
1984 int ret;
1985 UNUSED uint64_t newval;
1986
1987 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
1988 WRITE(newval, uint64_t);
1989 if (newp != NULL) {
1990 ctl_refresh(tsd_tsdn(tsd));
1991 }
1992 READ(ctl_arenas->epoch, uint64_t);
1993
1994 ret = 0;
1995label_return:
1996 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
1997 return ret;
1998}
1999
2000static int
2001background_thread_ctl(tsd_t *tsd, const size_t *mib,
2002 size_t miblen, void *oldp, size_t *oldlenp,
2003 void *newp, size_t newlen) {
2004 int ret;
2005 bool oldval;
2006
2007 if (!have_background_thread) {
2008 return ENOENT;
2009 }
2010 background_thread_ctl_init(tsd_tsdn(tsd));
2011
2012 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2013 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
2014 if (newp == NULL) {
2015 oldval = background_thread_enabled();
2016 READ(oldval, bool);
2017 } else {
2018 if (newlen != sizeof(bool)) {
2019 ret = EINVAL;
2020 goto label_return;
2021 }
2022 oldval = background_thread_enabled();
2023 READ(oldval, bool);
2024
2025 bool newval = *(bool *)newp;
2026 if (newval == oldval) {
2027 ret = 0;
2028 goto label_return;
2029 }
2030
2031 background_thread_enabled_set(tsd_tsdn(tsd), newval);
2032 if (newval) {
2033 if (background_threads_enable(tsd)) {
2034 ret = EFAULT;
2035 goto label_return;
2036 }
2037 } else {
2038 if (background_threads_disable(tsd)) {
2039 ret = EFAULT;
2040 goto label_return;
2041 }
2042 }
2043 }
2044 ret = 0;
2045label_return:
2046 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
2047 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2048
2049 return ret;
2050}
2051
2052static int
2053max_background_threads_ctl(tsd_t *tsd, const size_t *mib,
2054 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2055 size_t newlen) {
2056 int ret;
2057 size_t oldval;
2058
2059 if (!have_background_thread) {
2060 return ENOENT;
2061 }
2062 background_thread_ctl_init(tsd_tsdn(tsd));
2063
2064 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2065 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
2066 if (newp == NULL) {
2067 oldval = max_background_threads;
2068 READ(oldval, size_t);
2069 } else {
2070 if (newlen != sizeof(size_t)) {
2071 ret = EINVAL;
2072 goto label_return;
2073 }
2074 oldval = max_background_threads;
2075 READ(oldval, size_t);
2076
2077 size_t newval = *(size_t *)newp;
2078 if (newval == oldval) {
2079 ret = 0;
2080 goto label_return;
2081 }
2082 if (newval > opt_max_background_threads) {
2083 ret = EINVAL;
2084 goto label_return;
2085 }
2086
2087 if (background_thread_enabled()) {
2088 background_thread_enabled_set(tsd_tsdn(tsd), false);
2089 if (background_threads_disable(tsd)) {
2090 ret = EFAULT;
2091 goto label_return;
2092 }
2093 max_background_threads = newval;
2094 background_thread_enabled_set(tsd_tsdn(tsd), true);
2095 if (background_threads_enable(tsd)) {
2096 ret = EFAULT;
2097 goto label_return;
2098 }
2099 } else {
2100 max_background_threads = newval;
2101 }
2102 }
2103 ret = 0;
2104label_return:
2105 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
2106 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2107
2108 return ret;
2109}
2110
2111/******************************************************************************/
2112
2113CTL_RO_CONFIG_GEN(config_cache_oblivious, bool)
2114CTL_RO_CONFIG_GEN(config_debug, bool)
2115CTL_RO_CONFIG_GEN(config_fill, bool)
2116CTL_RO_CONFIG_GEN(config_lazy_lock, bool)
2117CTL_RO_CONFIG_GEN(config_malloc_conf, const char *)
2118CTL_RO_CONFIG_GEN(config_opt_safety_checks, bool)
2119CTL_RO_CONFIG_GEN(config_prof, bool)
2120CTL_RO_CONFIG_GEN(config_prof_libgcc, bool)
2121CTL_RO_CONFIG_GEN(config_prof_libunwind, bool)
2122CTL_RO_CONFIG_GEN(config_stats, bool)
2123CTL_RO_CONFIG_GEN(config_utrace, bool)
2124CTL_RO_CONFIG_GEN(config_xmalloc, bool)
2125
2126/******************************************************************************/
2127
2128CTL_RO_NL_GEN(opt_abort, opt_abort, bool)
2129CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool)
2130CTL_RO_NL_GEN(opt_cache_oblivious, opt_cache_oblivious, bool)
2131CTL_RO_NL_GEN(opt_trust_madvise, opt_trust_madvise, bool)
2132CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool)
2133
2134/* HPA options. */
2135CTL_RO_NL_GEN(opt_hpa, opt_hpa, bool)
2136CTL_RO_NL_GEN(opt_hpa_hugification_threshold,
2137 opt_hpa_opts.hugification_threshold, size_t)
2138CTL_RO_NL_GEN(opt_hpa_hugify_delay_ms, opt_hpa_opts.hugify_delay_ms, uint64_t)
2139CTL_RO_NL_GEN(opt_hpa_min_purge_interval_ms, opt_hpa_opts.min_purge_interval_ms,
2140 uint64_t)
2141
2142/*
2143 * This will have to change before we publicly document this option; fxp_t and
2144 * its representation are internal implementation details.
2145 */
2146CTL_RO_NL_GEN(opt_hpa_dirty_mult, opt_hpa_opts.dirty_mult, fxp_t)
2147CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t)
2148
2149/* HPA SEC options */
2150CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_opts.nshards, size_t)
2151CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_opts.max_alloc, size_t)
2152CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_opts.max_bytes, size_t)
2153CTL_RO_NL_GEN(opt_hpa_sec_bytes_after_flush, opt_hpa_sec_opts.bytes_after_flush,
2154 size_t)
2155CTL_RO_NL_GEN(opt_hpa_sec_batch_fill_extra, opt_hpa_sec_opts.batch_fill_extra,
2156 size_t)
2157
2158CTL_RO_NL_GEN(opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp],
2159 const char *)
2160CTL_RO_NL_GEN(opt_retain, opt_retain, bool)
2161CTL_RO_NL_GEN(opt_dss, opt_dss, const char *)
2162CTL_RO_NL_GEN(opt_narenas, opt_narenas, unsigned)
2163CTL_RO_NL_GEN(opt_percpu_arena, percpu_arena_mode_names[opt_percpu_arena],
2164 const char *)
2165CTL_RO_NL_GEN(opt_mutex_max_spin, opt_mutex_max_spin, int64_t)
2166CTL_RO_NL_GEN(opt_oversize_threshold, opt_oversize_threshold, size_t)
2167CTL_RO_NL_GEN(opt_background_thread, opt_background_thread, bool)
2168CTL_RO_NL_GEN(opt_max_background_threads, opt_max_background_threads, size_t)
2169CTL_RO_NL_GEN(opt_dirty_decay_ms, opt_dirty_decay_ms, ssize_t)
2170CTL_RO_NL_GEN(opt_muzzy_decay_ms, opt_muzzy_decay_ms, ssize_t)
2171CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool)
2172CTL_RO_NL_GEN(opt_stats_print_opts, opt_stats_print_opts, const char *)
2173CTL_RO_NL_GEN(opt_stats_interval, opt_stats_interval, int64_t)
2174CTL_RO_NL_GEN(opt_stats_interval_opts, opt_stats_interval_opts, const char *)
2175CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, const char *)
2176CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool)
2177CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool)
2178CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool)
2179CTL_RO_NL_CGEN(config_enable_cxx, opt_experimental_infallible_new,
2180 opt_experimental_infallible_new, bool)
2181CTL_RO_NL_GEN(opt_tcache, opt_tcache, bool)
2182CTL_RO_NL_GEN(opt_tcache_max, opt_tcache_max, size_t)
2183CTL_RO_NL_GEN(opt_tcache_nslots_small_min, opt_tcache_nslots_small_min,
2184 unsigned)
2185CTL_RO_NL_GEN(opt_tcache_nslots_small_max, opt_tcache_nslots_small_max,
2186 unsigned)
2187CTL_RO_NL_GEN(opt_tcache_nslots_large, opt_tcache_nslots_large, unsigned)
2188CTL_RO_NL_GEN(opt_lg_tcache_nslots_mul, opt_lg_tcache_nslots_mul, ssize_t)
2189CTL_RO_NL_GEN(opt_tcache_gc_incr_bytes, opt_tcache_gc_incr_bytes, size_t)
2190CTL_RO_NL_GEN(opt_tcache_gc_delay_bytes, opt_tcache_gc_delay_bytes, size_t)
2191CTL_RO_NL_GEN(opt_lg_tcache_flush_small_div, opt_lg_tcache_flush_small_div,
2192 unsigned)
2193CTL_RO_NL_GEN(opt_lg_tcache_flush_large_div, opt_lg_tcache_flush_large_div,
2194 unsigned)
2195CTL_RO_NL_GEN(opt_thp, thp_mode_names[opt_thp], const char *)
2196CTL_RO_NL_GEN(opt_lg_extent_max_active_fit, opt_lg_extent_max_active_fit,
2197 size_t)
2198CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool)
2199CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *)
2200CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool)
2201CTL_RO_NL_CGEN(config_prof, opt_prof_thread_active_init,
2202 opt_prof_thread_active_init, bool)
2203CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t)
2204CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool)
2205CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t)
2206CTL_RO_NL_CGEN(config_prof, opt_prof_gdump, opt_prof_gdump, bool)
2207CTL_RO_NL_CGEN(config_prof, opt_prof_final, opt_prof_final, bool)
2208CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool)
2209CTL_RO_NL_CGEN(config_prof, opt_prof_leak_error, opt_prof_leak_error, bool)
2210CTL_RO_NL_CGEN(config_prof, opt_prof_recent_alloc_max,
2211 opt_prof_recent_alloc_max, ssize_t)
2212CTL_RO_NL_CGEN(config_prof, opt_prof_stats, opt_prof_stats, bool)
2213CTL_RO_NL_CGEN(config_prof, opt_prof_sys_thread_name, opt_prof_sys_thread_name,
2214 bool)
2215CTL_RO_NL_CGEN(config_prof, opt_prof_time_res,
2216 prof_time_res_mode_names[opt_prof_time_res], const char *)
2217CTL_RO_NL_CGEN(config_uaf_detection, opt_lg_san_uaf_align,
2218 opt_lg_san_uaf_align, ssize_t)
2219CTL_RO_NL_GEN(opt_zero_realloc,
2220 zero_realloc_mode_names[opt_zero_realloc_action], const char *)
2221
2222/******************************************************************************/
2223
2224static int
2225thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2226 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2227 int ret;
2228 arena_t *oldarena;
2229 unsigned newind, oldind;
2230
2231 oldarena = arena_choose(tsd, NULL);
2232 if (oldarena == NULL) {
2233 return EAGAIN;
2234 }
2235 newind = oldind = arena_ind_get(oldarena);
2236 WRITE(newind, unsigned);
2237 READ(oldind, unsigned);
2238
2239 if (newind != oldind) {
2240 arena_t *newarena;
2241
2242 if (newind >= narenas_total_get()) {
2243 /* New arena index is out of range. */
2244 ret = EFAULT;
2245 goto label_return;
2246 }
2247
2248 if (have_percpu_arena &&
2249 PERCPU_ARENA_ENABLED(opt_percpu_arena)) {
2250 if (newind < percpu_arena_ind_limit(opt_percpu_arena)) {
2251 /*
2252 * If perCPU arena is enabled, thread_arena
2253 * control is not allowed for the auto arena
2254 * range.
2255 */
2256 ret = EPERM;
2257 goto label_return;
2258 }
2259 }
2260
2261 /* Initialize arena if necessary. */
2262 newarena = arena_get(tsd_tsdn(tsd), newind, true);
2263 if (newarena == NULL) {
2264 ret = EAGAIN;
2265 goto label_return;
2266 }
2267 /* Set new arena/tcache associations. */
2268 arena_migrate(tsd, oldarena, newarena);
2269 if (tcache_available(tsd)) {
2270 tcache_arena_reassociate(tsd_tsdn(tsd),
2271 tsd_tcache_slowp_get(tsd), tsd_tcachep_get(tsd),
2272 newarena);
2273 }
2274 }
2275
2276 ret = 0;
2277label_return:
2278 return ret;
2279}
2280
2281CTL_RO_NL_GEN(thread_allocated, tsd_thread_allocated_get(tsd), uint64_t)
2282CTL_RO_NL_GEN(thread_allocatedp, tsd_thread_allocatedp_get(tsd), uint64_t *)
2283CTL_RO_NL_GEN(thread_deallocated, tsd_thread_deallocated_get(tsd), uint64_t)
2284CTL_RO_NL_GEN(thread_deallocatedp, tsd_thread_deallocatedp_get(tsd), uint64_t *)
2285
2286static int
2287thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib,
2288 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2289 size_t newlen) {
2290 int ret;
2291 bool oldval;
2292
2293 oldval = tcache_enabled_get(tsd);
2294 if (newp != NULL) {
2295 if (newlen != sizeof(bool)) {
2296 ret = EINVAL;
2297 goto label_return;
2298 }
2299 tcache_enabled_set(tsd, *(bool *)newp);
2300 }
2301 READ(oldval, bool);
2302
2303 ret = 0;
2304label_return:
2305 return ret;
2306}
2307
2308static int
2309thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib,
2310 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2311 size_t newlen) {
2312 int ret;
2313
2314 if (!tcache_available(tsd)) {
2315 ret = EFAULT;
2316 goto label_return;
2317 }
2318
2319 NEITHER_READ_NOR_WRITE();
2320
2321 tcache_flush(tsd);
2322
2323 ret = 0;
2324label_return:
2325 return ret;
2326}
2327
2328static int
2329thread_peak_read_ctl(tsd_t *tsd, const size_t *mib,
2330 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2331 size_t newlen) {
2332 int ret;
2333 if (!config_stats) {
2334 return ENOENT;
2335 }
2336 READONLY();
2337 peak_event_update(tsd);
2338 uint64_t result = peak_event_max(tsd);
2339 READ(result, uint64_t);
2340 ret = 0;
2341label_return:
2342 return ret;
2343}
2344
2345static int
2346thread_peak_reset_ctl(tsd_t *tsd, const size_t *mib,
2347 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2348 size_t newlen) {
2349 int ret;
2350 if (!config_stats) {
2351 return ENOENT;
2352 }
2353 NEITHER_READ_NOR_WRITE();
2354 peak_event_zero(tsd);
2355 ret = 0;
2356label_return:
2357 return ret;
2358}
2359
2360static int
2361thread_prof_name_ctl(tsd_t *tsd, const size_t *mib,
2362 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2363 size_t newlen) {
2364 int ret;
2365
2366 if (!config_prof || !opt_prof) {
2367 return ENOENT;
2368 }
2369
2370 READ_XOR_WRITE();
2371
2372 if (newp != NULL) {
2373 if (newlen != sizeof(const char *)) {
2374 ret = EINVAL;
2375 goto label_return;
2376 }
2377
2378 if ((ret = prof_thread_name_set(tsd, *(const char **)newp)) !=
2379 0) {
2380 goto label_return;
2381 }
2382 } else {
2383 const char *oldname = prof_thread_name_get(tsd);
2384 READ(oldname, const char *);
2385 }
2386
2387 ret = 0;
2388label_return:
2389 return ret;
2390}
2391
2392static int
2393thread_prof_active_ctl(tsd_t *tsd, const size_t *mib,
2394 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2395 size_t newlen) {
2396 int ret;
2397 bool oldval;
2398
2399 if (!config_prof) {
2400 return ENOENT;
2401 }
2402
2403 oldval = opt_prof ? prof_thread_active_get(tsd) : false;
2404 if (newp != NULL) {
2405 if (!opt_prof) {
2406 ret = ENOENT;
2407 goto label_return;
2408 }
2409 if (newlen != sizeof(bool)) {
2410 ret = EINVAL;
2411 goto label_return;
2412 }
2413 if (prof_thread_active_set(tsd, *(bool *)newp)) {
2414 ret = EAGAIN;
2415 goto label_return;
2416 }
2417 }
2418 READ(oldval, bool);
2419
2420 ret = 0;
2421label_return:
2422 return ret;
2423}
2424
2425static int
2426thread_idle_ctl(tsd_t *tsd, const size_t *mib,
2427 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2428 size_t newlen) {
2429 int ret;
2430
2431 NEITHER_READ_NOR_WRITE();
2432
2433 if (tcache_available(tsd)) {
2434 tcache_flush(tsd);
2435 }
2436 /*
2437 * This heuristic is perhaps not the most well-considered. But it
2438 * matches the only idling policy we have experience with in the status
2439 * quo. Over time we should investigate more principled approaches.
2440 */
2441 if (opt_narenas > ncpus * 2) {
2442 arena_t *arena = arena_choose(tsd, NULL);
2443 if (arena != NULL) {
2444 arena_decay(tsd_tsdn(tsd), arena, false, true);
2445 }
2446 /*
2447 * The missing arena case is not actually an error; a thread
2448 * might be idle before it associates itself to one. This is
2449 * unusual, but not wrong.
2450 */
2451 }
2452
2453 ret = 0;
2454label_return:
2455 return ret;
2456}
2457
2458/******************************************************************************/
2459
2460static int
2461tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2462 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2463 int ret;
2464 unsigned tcache_ind;
2465
2466 READONLY();
2467 VERIFY_READ(unsigned);
2468 if (tcaches_create(tsd, b0get(), &tcache_ind)) {
2469 ret = EFAULT;
2470 goto label_return;
2471 }
2472 READ(tcache_ind, unsigned);
2473
2474 ret = 0;
2475label_return:
2476 return ret;
2477}
2478
2479static int
2480tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2481 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2482 int ret;
2483 unsigned tcache_ind;
2484
2485 WRITEONLY();
2486 ASSURED_WRITE(tcache_ind, unsigned);
2487 tcaches_flush(tsd, tcache_ind);
2488
2489 ret = 0;
2490label_return:
2491 return ret;
2492}
2493
2494static int
2495tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2496 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2497 int ret;
2498 unsigned tcache_ind;
2499
2500 WRITEONLY();
2501 ASSURED_WRITE(tcache_ind, unsigned);
2502 tcaches_destroy(tsd, tcache_ind);
2503
2504 ret = 0;
2505label_return:
2506 return ret;
2507}
2508
2509/******************************************************************************/
2510
2511static int
2512arena_i_initialized_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2513 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2514 int ret;
2515 tsdn_t *tsdn = tsd_tsdn(tsd);
2516 unsigned arena_ind;
2517 bool initialized;
2518
2519 READONLY();
2520 MIB_UNSIGNED(arena_ind, 1);
2521
2522 malloc_mutex_lock(tsdn, &ctl_mtx);
2523 initialized = arenas_i(arena_ind)->initialized;
2524 malloc_mutex_unlock(tsdn, &ctl_mtx);
2525
2526 READ(initialized, bool);
2527
2528 ret = 0;
2529label_return:
2530 return ret;
2531}
2532
2533static void
2534arena_i_decay(tsdn_t *tsdn, unsigned arena_ind, bool all) {
2535 malloc_mutex_lock(tsdn, &ctl_mtx);
2536 {
2537 unsigned narenas = ctl_arenas->narenas;
2538
2539 /*
2540 * Access via index narenas is deprecated, and scheduled for
2541 * removal in 6.0.0.
2542 */
2543 if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind == narenas) {
2544 unsigned i;
2545 VARIABLE_ARRAY(arena_t *, tarenas, narenas);
2546
2547 for (i = 0; i < narenas; i++) {
2548 tarenas[i] = arena_get(tsdn, i, false);
2549 }
2550
2551 /*
2552 * No further need to hold ctl_mtx, since narenas and
2553 * tarenas contain everything needed below.
2554 */
2555 malloc_mutex_unlock(tsdn, &ctl_mtx);
2556
2557 for (i = 0; i < narenas; i++) {
2558 if (tarenas[i] != NULL) {
2559 arena_decay(tsdn, tarenas[i], false,
2560 all);
2561 }
2562 }
2563 } else {
2564 arena_t *tarena;
2565
2566 assert(arena_ind < narenas);
2567
2568 tarena = arena_get(tsdn, arena_ind, false);
2569
2570 /* No further need to hold ctl_mtx. */
2571 malloc_mutex_unlock(tsdn, &ctl_mtx);
2572
2573 if (tarena != NULL) {
2574 arena_decay(tsdn, tarena, false, all);
2575 }
2576 }
2577 }
2578}
2579
2580static int
2581arena_i_decay_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2582 size_t *oldlenp, void *newp, size_t newlen) {
2583 int ret;
2584 unsigned arena_ind;
2585
2586 NEITHER_READ_NOR_WRITE();
2587 MIB_UNSIGNED(arena_ind, 1);
2588 arena_i_decay(tsd_tsdn(tsd), arena_ind, false);
2589
2590 ret = 0;
2591label_return:
2592 return ret;
2593}
2594
2595static int
2596arena_i_purge_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2597 size_t *oldlenp, void *newp, size_t newlen) {
2598 int ret;
2599 unsigned arena_ind;
2600
2601 NEITHER_READ_NOR_WRITE();
2602 MIB_UNSIGNED(arena_ind, 1);
2603 arena_i_decay(tsd_tsdn(tsd), arena_ind, true);
2604
2605 ret = 0;
2606label_return:
2607 return ret;
2608}
2609
2610static int
2611arena_i_reset_destroy_helper(tsd_t *tsd, const size_t *mib, size_t miblen,
2612 void *oldp, size_t *oldlenp, void *newp, size_t newlen, unsigned *arena_ind,
2613 arena_t **arena) {
2614 int ret;
2615
2616 NEITHER_READ_NOR_WRITE();
2617 MIB_UNSIGNED(*arena_ind, 1);
2618
2619 *arena = arena_get(tsd_tsdn(tsd), *arena_ind, false);
2620 if (*arena == NULL || arena_is_auto(*arena)) {
2621 ret = EFAULT;
2622 goto label_return;
2623 }
2624
2625 ret = 0;
2626label_return:
2627 return ret;
2628}
2629
2630static void
2631arena_reset_prepare_background_thread(tsd_t *tsd, unsigned arena_ind) {
2632 /* Temporarily disable the background thread during arena reset. */
2633 if (have_background_thread) {
2634 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
2635 if (background_thread_enabled()) {
2636 background_thread_info_t *info =
2637 background_thread_info_get(arena_ind);
2638 assert(info->state == background_thread_started);
2639 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
2640 info->state = background_thread_paused;
2641 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
2642 }
2643 }
2644}
2645
2646static void
2647arena_reset_finish_background_thread(tsd_t *tsd, unsigned arena_ind) {
2648 if (have_background_thread) {
2649 if (background_thread_enabled()) {
2650 background_thread_info_t *info =
2651 background_thread_info_get(arena_ind);
2652 assert(info->state == background_thread_paused);
2653 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
2654 info->state = background_thread_started;
2655 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
2656 }
2657 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
2658 }
2659}
2660
2661static int
2662arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2663 size_t *oldlenp, void *newp, size_t newlen) {
2664 int ret;
2665 unsigned arena_ind;
2666 arena_t *arena;
2667
2668 ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp,
2669 newp, newlen, &arena_ind, &arena);
2670 if (ret != 0) {
2671 return ret;
2672 }
2673
2674 arena_reset_prepare_background_thread(tsd, arena_ind);
2675 arena_reset(tsd, arena);
2676 arena_reset_finish_background_thread(tsd, arena_ind);
2677
2678 return ret;
2679}
2680
2681static int
2682arena_i_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2683 size_t *oldlenp, void *newp, size_t newlen) {
2684 int ret;
2685 unsigned arena_ind;
2686 arena_t *arena;
2687 ctl_arena_t *ctl_darena, *ctl_arena;
2688
2689 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2690
2691 ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp,
2692 newp, newlen, &arena_ind, &arena);
2693 if (ret != 0) {
2694 goto label_return;
2695 }
2696
2697 if (arena_nthreads_get(arena, false) != 0 || arena_nthreads_get(arena,
2698 true) != 0) {
2699 ret = EFAULT;
2700 goto label_return;
2701 }
2702
2703 arena_reset_prepare_background_thread(tsd, arena_ind);
2704 /* Merge stats after resetting and purging arena. */
2705 arena_reset(tsd, arena);
2706 arena_decay(tsd_tsdn(tsd), arena, false, true);
2707 ctl_darena = arenas_i(MALLCTL_ARENAS_DESTROYED);
2708 ctl_darena->initialized = true;
2709 ctl_arena_refresh(tsd_tsdn(tsd), arena, ctl_darena, arena_ind, true);
2710 /* Destroy arena. */
2711 arena_destroy(tsd, arena);
2712 ctl_arena = arenas_i(arena_ind);
2713 ctl_arena->initialized = false;
2714 /* Record arena index for later recycling via arenas.create. */
2715 ql_elm_new(ctl_arena, destroyed_link);
2716 ql_tail_insert(&ctl_arenas->destroyed, ctl_arena, destroyed_link);
2717 arena_reset_finish_background_thread(tsd, arena_ind);
2718
2719 assert(ret == 0);
2720label_return:
2721 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2722
2723 return ret;
2724}
2725
2726static int
2727arena_i_dss_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2728 size_t *oldlenp, void *newp, size_t newlen) {
2729 int ret;
2730 const char *dss = NULL;
2731 unsigned arena_ind;
2732 dss_prec_t dss_prec_old = dss_prec_limit;
2733 dss_prec_t dss_prec = dss_prec_limit;
2734
2735 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2736 WRITE(dss, const char *);
2737 MIB_UNSIGNED(arena_ind, 1);
2738 if (dss != NULL) {
2739 int i;
2740 bool match = false;
2741
2742 for (i = 0; i < dss_prec_limit; i++) {
2743 if (strcmp(dss_prec_names[i], dss) == 0) {
2744 dss_prec = i;
2745 match = true;
2746 break;
2747 }
2748 }
2749
2750 if (!match) {
2751 ret = EINVAL;
2752 goto label_return;
2753 }
2754 }
2755
2756 /*
2757 * Access via index narenas is deprecated, and scheduled for removal in
2758 * 6.0.0.
2759 */
2760 if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind ==
2761 ctl_arenas->narenas) {
2762 if (dss_prec != dss_prec_limit &&
2763 extent_dss_prec_set(dss_prec)) {
2764 ret = EFAULT;
2765 goto label_return;
2766 }
2767 dss_prec_old = extent_dss_prec_get();
2768 } else {
2769 arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
2770 if (arena == NULL || (dss_prec != dss_prec_limit &&
2771 arena_dss_prec_set(arena, dss_prec))) {
2772 ret = EFAULT;
2773 goto label_return;
2774 }
2775 dss_prec_old = arena_dss_prec_get(arena);
2776 }
2777
2778 dss = dss_prec_names[dss_prec_old];
2779 READ(dss, const char *);
2780
2781 ret = 0;
2782label_return:
2783 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2784 return ret;
2785}
2786
2787static int
2788arena_i_oversize_threshold_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2789 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2790 int ret;
2791
2792 unsigned arena_ind;
2793 MIB_UNSIGNED(arena_ind, 1);
2794
2795 arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
2796 if (arena == NULL) {
2797 ret = EFAULT;
2798 goto label_return;
2799 }
2800
2801 if (oldp != NULL && oldlenp != NULL) {
2802 size_t oldval = atomic_load_zu(
2803 &arena->pa_shard.pac.oversize_threshold, ATOMIC_RELAXED);
2804 READ(oldval, size_t);
2805 }
2806 if (newp != NULL) {
2807 if (newlen != sizeof(size_t)) {
2808 ret = EINVAL;
2809 goto label_return;
2810 }
2811 atomic_store_zu(&arena->pa_shard.pac.oversize_threshold,
2812 *(size_t *)newp, ATOMIC_RELAXED);
2813 }
2814 ret = 0;
2815label_return:
2816 return ret;
2817}
2818
2819static int
2820arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen,
2821 void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) {
2822 int ret;
2823 unsigned arena_ind;
2824 arena_t *arena;
2825
2826 MIB_UNSIGNED(arena_ind, 1);
2827 arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
2828 if (arena == NULL) {
2829 ret = EFAULT;
2830 goto label_return;
2831 }
2832 extent_state_t state = dirty ? extent_state_dirty : extent_state_muzzy;
2833
2834 if (oldp != NULL && oldlenp != NULL) {
2835 size_t oldval = arena_decay_ms_get(arena, state);
2836 READ(oldval, ssize_t);
2837 }
2838 if (newp != NULL) {
2839 if (newlen != sizeof(ssize_t)) {
2840 ret = EINVAL;
2841 goto label_return;
2842 }
2843 if (arena_is_huge(arena_ind) && *(ssize_t *)newp > 0) {
2844 /*
2845 * By default the huge arena purges eagerly. If it is
2846 * set to non-zero decay time afterwards, background
2847 * thread might be needed.
2848 */
2849 if (background_thread_create(tsd, arena_ind)) {
2850 ret = EFAULT;
2851 goto label_return;
2852 }
2853 }
2854
2855 if (arena_decay_ms_set(tsd_tsdn(tsd), arena, state,
2856 *(ssize_t *)newp)) {
2857 ret = EFAULT;
2858 goto label_return;
2859 }
2860 }
2861
2862 ret = 0;
2863label_return:
2864 return ret;
2865}
2866
2867static int
2868arena_i_dirty_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2869 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2870 return arena_i_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp,
2871 newlen, true);
2872}
2873
2874static int
2875arena_i_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2876 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2877 return arena_i_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp,
2878 newlen, false);
2879}
2880
2881static int
2882arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2883 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2884 int ret;
2885 unsigned arena_ind;
2886 arena_t *arena;
2887
2888 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2889 MIB_UNSIGNED(arena_ind, 1);
2890 if (arena_ind < narenas_total_get()) {
2891 extent_hooks_t *old_extent_hooks;
2892 arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
2893 if (arena == NULL) {
2894 if (arena_ind >= narenas_auto) {
2895 ret = EFAULT;
2896 goto label_return;
2897 }
2898 old_extent_hooks =
2899 (extent_hooks_t *)&ehooks_default_extent_hooks;
2900 READ(old_extent_hooks, extent_hooks_t *);
2901 if (newp != NULL) {
2902 /* Initialize a new arena as a side effect. */
2903 extent_hooks_t *new_extent_hooks
2904 JEMALLOC_CC_SILENCE_INIT(NULL);
2905 WRITE(new_extent_hooks, extent_hooks_t *);
2906 arena_config_t config = arena_config_default;
2907 config.extent_hooks = new_extent_hooks;
2908
2909 arena = arena_init(tsd_tsdn(tsd), arena_ind,
2910 &config);
2911 if (arena == NULL) {
2912 ret = EFAULT;
2913 goto label_return;
2914 }
2915 }
2916 } else {
2917 if (newp != NULL) {
2918 extent_hooks_t *new_extent_hooks
2919 JEMALLOC_CC_SILENCE_INIT(NULL);
2920 WRITE(new_extent_hooks, extent_hooks_t *);
2921 old_extent_hooks = arena_set_extent_hooks(tsd,
2922 arena, new_extent_hooks);
2923 READ(old_extent_hooks, extent_hooks_t *);
2924 } else {
2925 old_extent_hooks =
2926 ehooks_get_extent_hooks_ptr(
2927 arena_get_ehooks(arena));
2928 READ(old_extent_hooks, extent_hooks_t *);
2929 }
2930 }
2931 } else {
2932 ret = EFAULT;
2933 goto label_return;
2934 }
2935 ret = 0;
2936label_return:
2937 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2938 return ret;
2939}
2940
2941static int
2942arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib,
2943 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2944 size_t newlen) {
2945 int ret;
2946 unsigned arena_ind;
2947 arena_t *arena;
2948
2949 if (!opt_retain) {
2950 /* Only relevant when retain is enabled. */
2951 return ENOENT;
2952 }
2953
2954 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2955 MIB_UNSIGNED(arena_ind, 1);
2956 if (arena_ind < narenas_total_get() && (arena =
2957 arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) {
2958 size_t old_limit, new_limit;
2959 if (newp != NULL) {
2960 WRITE(new_limit, size_t);
2961 }
2962 bool err = arena_retain_grow_limit_get_set(tsd, arena,
2963 &old_limit, newp != NULL ? &new_limit : NULL);
2964 if (!err) {
2965 READ(old_limit, size_t);
2966 ret = 0;
2967 } else {
2968 ret = EFAULT;
2969 }
2970 } else {
2971 ret = EFAULT;
2972 }
2973label_return:
2974 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2975 return ret;
2976}
2977
2978static const ctl_named_node_t *
2979arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen,
2980 size_t i) {
2981 const ctl_named_node_t *ret;
2982
2983 malloc_mutex_lock(tsdn, &ctl_mtx);
2984 switch (i) {
2985 case MALLCTL_ARENAS_ALL:
2986 case MALLCTL_ARENAS_DESTROYED:
2987 break;
2988 default:
2989 if (i > ctl_arenas->narenas) {
2990 ret = NULL;
2991 goto label_return;
2992 }
2993 break;
2994 }
2995
2996 ret = super_arena_i_node;
2997label_return:
2998 malloc_mutex_unlock(tsdn, &ctl_mtx);
2999 return ret;
3000}
3001
3002/******************************************************************************/
3003
3004static int
3005arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3006 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3007 int ret;
3008 unsigned narenas;
3009
3010 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
3011 READONLY();
3012 narenas = ctl_arenas->narenas;
3013 READ(narenas, unsigned);
3014
3015 ret = 0;
3016label_return:
3017 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
3018 return ret;
3019}
3020
3021static int
3022arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib,
3023 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
3024 size_t newlen, bool dirty) {
3025 int ret;
3026
3027 if (oldp != NULL && oldlenp != NULL) {
3028 size_t oldval = (dirty ? arena_dirty_decay_ms_default_get() :
3029 arena_muzzy_decay_ms_default_get());
3030 READ(oldval, ssize_t);
3031 }
3032 if (newp != NULL) {
3033 if (newlen != sizeof(ssize_t)) {
3034 ret = EINVAL;
3035 goto label_return;
3036 }
3037 if (dirty ? arena_dirty_decay_ms_default_set(*(ssize_t *)newp)
3038 : arena_muzzy_decay_ms_default_set(*(ssize_t *)newp)) {
3039 ret = EFAULT;
3040 goto label_return;
3041 }
3042 }
3043
3044 ret = 0;
3045label_return:
3046 return ret;
3047}
3048
3049static int
3050arenas_dirty_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3051 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3052 return arenas_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp,
3053 newlen, true);
3054}
3055
3056static int
3057arenas_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3058 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3059 return arenas_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp,
3060 newlen, false);
3061}
3062
3063CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t)
3064CTL_RO_NL_GEN(arenas_page, PAGE, size_t)
3065CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t)
3066CTL_RO_NL_GEN(arenas_nbins, SC_NBINS, unsigned)
3067CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned)
3068CTL_RO_NL_GEN(arenas_bin_i_size, bin_infos[mib[2]].reg_size, size_t)
3069CTL_RO_NL_GEN(arenas_bin_i_nregs, bin_infos[mib[2]].nregs, uint32_t)
3070CTL_RO_NL_GEN(arenas_bin_i_slab_size, bin_infos[mib[2]].slab_size, size_t)
3071CTL_RO_NL_GEN(arenas_bin_i_nshards, bin_infos[mib[2]].n_shards, uint32_t)
3072static const ctl_named_node_t *
3073arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib,
3074 size_t miblen, size_t i) {
3075 if (i > SC_NBINS) {
3076 return NULL;
3077 }
3078 return super_arenas_bin_i_node;
3079}
3080
3081CTL_RO_NL_GEN(arenas_nlextents, SC_NSIZES - SC_NBINS, unsigned)
3082CTL_RO_NL_GEN(arenas_lextent_i_size, sz_index2size(SC_NBINS+(szind_t)mib[2]),
3083 size_t)
3084static const ctl_named_node_t *
3085arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib,
3086 size_t miblen, size_t i) {
3087 if (i > SC_NSIZES - SC_NBINS) {
3088 return NULL;
3089 }
3090 return super_arenas_lextent_i_node;
3091}
3092
3093static int
3094arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3095 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3096 int ret;
3097 unsigned arena_ind;
3098
3099 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
3100
3101 VERIFY_READ(unsigned);
3102 arena_config_t config = arena_config_default;
3103 WRITE(config.extent_hooks, extent_hooks_t *);
3104 if ((arena_ind = ctl_arena_init(tsd, &config)) == UINT_MAX) {
3105 ret = EAGAIN;
3106 goto label_return;
3107 }
3108 READ(arena_ind, unsigned);
3109
3110 ret = 0;
3111label_return:
3112 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
3113 return ret;
3114}
3115
3116static int
3117experimental_arenas_create_ext_ctl(tsd_t *tsd,
3118 const size_t *mib, size_t miblen,
3119 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3120 int ret;
3121 unsigned arena_ind;
3122
3123 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
3124
3125 arena_config_t config = arena_config_default;
3126 VERIFY_READ(unsigned);
3127 WRITE(config, arena_config_t);
3128
3129 if ((arena_ind = ctl_arena_init(tsd, &config)) == UINT_MAX) {
3130 ret = EAGAIN;
3131 goto label_return;
3132 }
3133 READ(arena_ind, unsigned);
3134 ret = 0;
3135label_return:
3136 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
3137 return ret;
3138}
3139
3140static int
3141arenas_lookup_ctl(tsd_t *tsd, const size_t *mib,
3142 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
3143 size_t newlen) {
3144 int ret;
3145 unsigned arena_ind;
3146 void *ptr;
3147 edata_t *edata;
3148 arena_t *arena;
3149
3150 ptr = NULL;
3151 ret = EINVAL;
3152 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
3153 WRITE(ptr, void *);
3154 edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr);
3155 if (edata == NULL) {
3156 goto label_return;
3157 }
3158
3159 arena = arena_get_from_edata(edata);
3160 if (arena == NULL) {
3161 goto label_return;
3162 }
3163
3164 arena_ind = arena_ind_get(arena);
3165 READ(arena_ind, unsigned);
3166
3167 ret = 0;
3168label_return:
3169 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
3170 return ret;
3171}
3172
3173/******************************************************************************/
3174
3175static int
3176prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib,
3177 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
3178 size_t newlen) {
3179 int ret;
3180 bool oldval;
3181
3182 if (!config_prof) {
3183 return ENOENT;
3184 }
3185
3186 if (newp != NULL) {
3187 if (!opt_prof) {
3188 ret = ENOENT;
3189 goto label_return;
3190 }
3191 if (newlen != sizeof(bool)) {
3192 ret = EINVAL;
3193 goto label_return;
3194 }
3195 oldval = prof_thread_active_init_set(tsd_tsdn(tsd),
3196 *(bool *)newp);
3197 } else {
3198 oldval = opt_prof ? prof_thread_active_init_get(tsd_tsdn(tsd)) :
3199 false;
3200 }
3201 READ(oldval, bool);
3202
3203 ret = 0;
3204label_return:
3205 return ret;
3206}
3207
3208static int
3209prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3210 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3211 int ret;
3212 bool oldval;
3213
3214 if (!config_prof) {
3215 ret = ENOENT;
3216 goto label_return;
3217 }
3218
3219 if (newp != NULL) {
3220 if (newlen != sizeof(bool)) {
3221 ret = EINVAL;
3222 goto label_return;
3223 }
3224 bool val = *(bool *)newp;
3225 if (!opt_prof) {
3226 if (val) {
3227 ret = ENOENT;
3228 goto label_return;
3229 } else {
3230 /* No change needed (already off). */
3231 oldval = false;
3232 }
3233 } else {
3234 oldval = prof_active_set(tsd_tsdn(tsd), val);
3235 }
3236 } else {
3237 oldval = opt_prof ? prof_active_get(tsd_tsdn(tsd)) : false;
3238 }
3239 READ(oldval, bool);
3240
3241 ret = 0;
3242label_return:
3243 return ret;
3244}
3245
3246static int
3247prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3248 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3249 int ret;
3250 const char *filename = NULL;
3251
3252 if (!config_prof || !opt_prof) {
3253 return ENOENT;
3254 }
3255
3256 WRITEONLY();
3257 WRITE(filename, const char *);
3258
3259 if (prof_mdump(tsd, filename)) {
3260 ret = EFAULT;
3261 goto label_return;
3262 }
3263
3264 ret = 0;
3265label_return:
3266 return ret;
3267}
3268
3269static int
3270prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3271 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3272 int ret;
3273 bool oldval;
3274
3275 if (!config_prof) {
3276 return ENOENT;
3277 }
3278
3279 if (newp != NULL) {
3280 if (!opt_prof) {
3281 ret = ENOENT;
3282 goto label_return;
3283 }
3284 if (newlen != sizeof(bool)) {
3285 ret = EINVAL;
3286 goto label_return;
3287 }
3288 oldval = prof_gdump_set(tsd_tsdn(tsd), *(bool *)newp);
3289 } else {
3290 oldval = opt_prof ? prof_gdump_get(tsd_tsdn(tsd)) : false;
3291 }
3292 READ(oldval, bool);
3293
3294 ret = 0;
3295label_return:
3296 return ret;
3297}
3298
3299static int
3300prof_prefix_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3301 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3302 int ret;
3303 const char *prefix = NULL;
3304
3305 if (!config_prof || !opt_prof) {
3306 return ENOENT;
3307 }
3308
3309 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
3310 WRITEONLY();
3311 WRITE(prefix, const char *);
3312
3313 ret = prof_prefix_set(tsd_tsdn(tsd), prefix) ? EFAULT : 0;
3314label_return:
3315 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
3316 return ret;
3317}
3318
3319static int
3320prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3321 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3322 int ret;
3323 size_t lg_sample = lg_prof_sample;
3324
3325 if (!config_prof || !opt_prof) {
3326 return ENOENT;
3327 }
3328
3329 WRITEONLY();
3330 WRITE(lg_sample, size_t);
3331 if (lg_sample >= (sizeof(uint64_t) << 3)) {
3332 lg_sample = (sizeof(uint64_t) << 3) - 1;
3333 }
3334
3335 prof_reset(tsd, lg_sample);
3336
3337 ret = 0;
3338label_return:
3339 return ret;
3340}
3341
3342CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t)
3343CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t)
3344
3345static int
3346prof_log_start_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
3347 size_t *oldlenp, void *newp, size_t newlen) {
3348 int ret;
3349
3350 const char *filename = NULL;
3351
3352 if (!config_prof || !opt_prof) {
3353 return ENOENT;
3354 }
3355
3356 WRITEONLY();
3357 WRITE(filename, const char *);
3358
3359 if (prof_log_start(tsd_tsdn(tsd), filename)) {
3360 ret = EFAULT;
3361 goto label_return;
3362 }
3363
3364 ret = 0;
3365label_return:
3366 return ret;
3367}
3368
3369static int
3370prof_log_stop_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
3371 size_t *oldlenp, void *newp, size_t newlen) {
3372 if (!config_prof || !opt_prof) {
3373 return ENOENT;
3374 }
3375
3376 if (prof_log_stop(tsd_tsdn(tsd))) {
3377 return EFAULT;
3378 }
3379
3380 return 0;
3381}
3382
3383static int
3384experimental_hooks_prof_backtrace_ctl(tsd_t *tsd, const size_t *mib,
3385 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3386 int ret;
3387
3388 if (oldp == NULL && newp == NULL) {
3389 ret = EINVAL;
3390 goto label_return;
3391 }
3392 if (oldp != NULL) {
3393 prof_backtrace_hook_t old_hook =
3394 prof_backtrace_hook_get();
3395 READ(old_hook, prof_backtrace_hook_t);
3396 }
3397 if (newp != NULL) {
3398 if (!opt_prof) {
3399 ret = ENOENT;
3400 goto label_return;
3401 }
3402 prof_backtrace_hook_t new_hook JEMALLOC_CC_SILENCE_INIT(NULL);
3403 WRITE(new_hook, prof_backtrace_hook_t);
3404 if (new_hook == NULL) {
3405 ret = EINVAL;
3406 goto label_return;
3407 }
3408 prof_backtrace_hook_set(new_hook);
3409 }
3410 ret = 0;
3411label_return:
3412 return ret;
3413}
3414
3415static int
3416experimental_hooks_prof_dump_ctl(tsd_t *tsd, const size_t *mib,
3417 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3418 int ret;
3419
3420 if (oldp == NULL && newp == NULL) {
3421 ret = EINVAL;
3422 goto label_return;
3423 }
3424 if (oldp != NULL) {
3425 prof_dump_hook_t old_hook =
3426 prof_dump_hook_get();
3427 READ(old_hook, prof_dump_hook_t);
3428 }
3429 if (newp != NULL) {
3430 if (!opt_prof) {
3431 ret = ENOENT;
3432 goto label_return;
3433 }
3434 prof_dump_hook_t new_hook JEMALLOC_CC_SILENCE_INIT(NULL);
3435 WRITE(new_hook, prof_dump_hook_t);
3436 prof_dump_hook_set(new_hook);
3437 }
3438 ret = 0;
3439label_return:
3440 return ret;
3441}
3442
3443/* For integration test purpose only. No plan to move out of experimental. */
3444static int
3445experimental_hooks_safety_check_abort_ctl(tsd_t *tsd, const size_t *mib,
3446 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3447 int ret;
3448
3449 WRITEONLY();
3450 if (newp != NULL) {
3451 if (newlen != sizeof(safety_check_abort_hook_t)) {
3452 ret = EINVAL;
3453 goto label_return;
3454 }
3455 safety_check_abort_hook_t hook JEMALLOC_CC_SILENCE_INIT(NULL);
3456 WRITE(hook, safety_check_abort_hook_t);
3457 safety_check_set_abort(hook);
3458 }
3459 ret = 0;
3460label_return:
3461 return ret;
3462}
3463
3464/******************************************************************************/
3465
3466CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t)
3467CTL_RO_CGEN(config_stats, stats_active, ctl_stats->active, size_t)
3468CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats->metadata, size_t)
3469CTL_RO_CGEN(config_stats, stats_metadata_thp, ctl_stats->metadata_thp, size_t)
3470CTL_RO_CGEN(config_stats, stats_resident, ctl_stats->resident, size_t)
3471CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats->mapped, size_t)
3472CTL_RO_CGEN(config_stats, stats_retained, ctl_stats->retained, size_t)
3473
3474CTL_RO_CGEN(config_stats, stats_background_thread_num_threads,
3475 ctl_stats->background_thread.num_threads, size_t)
3476CTL_RO_CGEN(config_stats, stats_background_thread_num_runs,
3477 ctl_stats->background_thread.num_runs, uint64_t)
3478CTL_RO_CGEN(config_stats, stats_background_thread_run_interval,
3479 nstime_ns(&ctl_stats->background_thread.run_interval), uint64_t)
3480
3481CTL_RO_CGEN(config_stats, stats_zero_reallocs,
3482 atomic_load_zu(&zero_realloc_count, ATOMIC_RELAXED), size_t)
3483
3484CTL_RO_GEN(stats_arenas_i_dss, arenas_i(mib[2])->dss, const char *)
3485CTL_RO_GEN(stats_arenas_i_dirty_decay_ms, arenas_i(mib[2])->dirty_decay_ms,
3486 ssize_t)
3487CTL_RO_GEN(stats_arenas_i_muzzy_decay_ms, arenas_i(mib[2])->muzzy_decay_ms,
3488 ssize_t)
3489CTL_RO_GEN(stats_arenas_i_nthreads, arenas_i(mib[2])->nthreads, unsigned)
3490CTL_RO_GEN(stats_arenas_i_uptime,
3491 nstime_ns(&arenas_i(mib[2])->astats->astats.uptime), uint64_t)
3492CTL_RO_GEN(stats_arenas_i_pactive, arenas_i(mib[2])->pactive, size_t)
3493CTL_RO_GEN(stats_arenas_i_pdirty, arenas_i(mib[2])->pdirty, size_t)
3494CTL_RO_GEN(stats_arenas_i_pmuzzy, arenas_i(mib[2])->pmuzzy, size_t)
3495CTL_RO_CGEN(config_stats, stats_arenas_i_mapped,
3496 arenas_i(mib[2])->astats->astats.mapped, size_t)
3497CTL_RO_CGEN(config_stats, stats_arenas_i_retained,
3498 arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.retained, size_t)
3499CTL_RO_CGEN(config_stats, stats_arenas_i_extent_avail,
3500 arenas_i(mib[2])->astats->astats.pa_shard_stats.edata_avail, size_t)
3501
3502CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_npurge,
3503 locked_read_u64_unsynchronized(
3504 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_dirty.npurge),
3505 uint64_t)
3506CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_nmadvise,
3507 locked_read_u64_unsynchronized(
3508 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_dirty.nmadvise),
3509 uint64_t)
3510CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_purged,
3511 locked_read_u64_unsynchronized(
3512 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_dirty.purged),
3513 uint64_t)
3514
3515CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_npurge,
3516 locked_read_u64_unsynchronized(
3517 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.npurge),
3518 uint64_t)
3519CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_nmadvise,
3520 locked_read_u64_unsynchronized(
3521 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.nmadvise),
3522 uint64_t)
3523CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_purged,
3524 locked_read_u64_unsynchronized(
3525 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.purged),
3526 uint64_t)
3527
3528CTL_RO_CGEN(config_stats, stats_arenas_i_base,
3529 arenas_i(mib[2])->astats->astats.base,
3530 size_t)
3531CTL_RO_CGEN(config_stats, stats_arenas_i_internal,
3532 atomic_load_zu(&arenas_i(mib[2])->astats->astats.internal, ATOMIC_RELAXED),
3533 size_t)
3534CTL_RO_CGEN(config_stats, stats_arenas_i_metadata_thp,
3535 arenas_i(mib[2])->astats->astats.metadata_thp, size_t)
3536CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_bytes,
3537 arenas_i(mib[2])->astats->astats.tcache_bytes, size_t)
3538CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_stashed_bytes,
3539 arenas_i(mib[2])->astats->astats.tcache_stashed_bytes, size_t)
3540CTL_RO_CGEN(config_stats, stats_arenas_i_resident,
3541 arenas_i(mib[2])->astats->astats.resident,
3542 size_t)
3543CTL_RO_CGEN(config_stats, stats_arenas_i_abandoned_vm,
3544 atomic_load_zu(
3545 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.abandoned_vm,
3546 ATOMIC_RELAXED), size_t)
3547
3548CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_bytes,
3549 arenas_i(mib[2])->astats->secstats.bytes, size_t)
3550
3551CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated,
3552 arenas_i(mib[2])->astats->allocated_small, size_t)
3553CTL_RO_CGEN(config_stats, stats_arenas_i_small_nmalloc,
3554 arenas_i(mib[2])->astats->nmalloc_small, uint64_t)
3555CTL_RO_CGEN(config_stats, stats_arenas_i_small_ndalloc,
3556 arenas_i(mib[2])->astats->ndalloc_small, uint64_t)
3557CTL_RO_CGEN(config_stats, stats_arenas_i_small_nrequests,
3558 arenas_i(mib[2])->astats->nrequests_small, uint64_t)
3559CTL_RO_CGEN(config_stats, stats_arenas_i_small_nfills,
3560 arenas_i(mib[2])->astats->nfills_small, uint64_t)
3561CTL_RO_CGEN(config_stats, stats_arenas_i_small_nflushes,
3562 arenas_i(mib[2])->astats->nflushes_small, uint64_t)
3563CTL_RO_CGEN(config_stats, stats_arenas_i_large_allocated,
3564 arenas_i(mib[2])->astats->astats.allocated_large, size_t)
3565CTL_RO_CGEN(config_stats, stats_arenas_i_large_nmalloc,
3566 arenas_i(mib[2])->astats->astats.nmalloc_large, uint64_t)
3567CTL_RO_CGEN(config_stats, stats_arenas_i_large_ndalloc,
3568 arenas_i(mib[2])->astats->astats.ndalloc_large, uint64_t)
3569CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests,
3570 arenas_i(mib[2])->astats->astats.nrequests_large, uint64_t)
3571/*
3572 * Note: "nmalloc_large" here instead of "nfills" in the read. This is
3573 * intentional (large has no batch fill).
3574 */
3575CTL_RO_CGEN(config_stats, stats_arenas_i_large_nfills,
3576 arenas_i(mib[2])->astats->astats.nmalloc_large, uint64_t)
3577CTL_RO_CGEN(config_stats, stats_arenas_i_large_nflushes,
3578 arenas_i(mib[2])->astats->astats.nflushes_large, uint64_t)
3579
3580/* Lock profiling related APIs below. */
3581#define RO_MUTEX_CTL_GEN(n, l) \
3582CTL_RO_CGEN(config_stats, stats_##n##_num_ops, \
3583 l.n_lock_ops, uint64_t) \
3584CTL_RO_CGEN(config_stats, stats_##n##_num_wait, \
3585 l.n_wait_times, uint64_t) \
3586CTL_RO_CGEN(config_stats, stats_##n##_num_spin_acq, \
3587 l.n_spin_acquired, uint64_t) \
3588CTL_RO_CGEN(config_stats, stats_##n##_num_owner_switch, \
3589 l.n_owner_switches, uint64_t) \
3590CTL_RO_CGEN(config_stats, stats_##n##_total_wait_time, \
3591 nstime_ns(&l.tot_wait_time), uint64_t) \
3592CTL_RO_CGEN(config_stats, stats_##n##_max_wait_time, \
3593 nstime_ns(&l.max_wait_time), uint64_t) \
3594CTL_RO_CGEN(config_stats, stats_##n##_max_num_thds, \
3595 l.max_n_thds, uint32_t)
3596
3597/* Global mutexes. */
3598#define OP(mtx) \
3599 RO_MUTEX_CTL_GEN(mutexes_##mtx, \
3600 ctl_stats->mutex_prof_data[global_prof_mutex_##mtx])
3601MUTEX_PROF_GLOBAL_MUTEXES
3602#undef OP
3603
3604/* Per arena mutexes */
3605#define OP(mtx) RO_MUTEX_CTL_GEN(arenas_i_mutexes_##mtx, \
3606 arenas_i(mib[2])->astats->astats.mutex_prof_data[arena_prof_mutex_##mtx])
3607MUTEX_PROF_ARENA_MUTEXES
3608#undef OP
3609
3610/* tcache bin mutex */
3611RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex,
3612 arenas_i(mib[2])->astats->bstats[mib[4]].mutex_data)
3613#undef RO_MUTEX_CTL_GEN
3614
3615/* Resets all mutex stats, including global, arena and bin mutexes. */
3616static int
3617stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib,
3618 size_t miblen, void *oldp, size_t *oldlenp,
3619 void *newp, size_t newlen) {
3620 if (!config_stats) {
3621 return ENOENT;
3622 }
3623
3624 tsdn_t *tsdn = tsd_tsdn(tsd);
3625
3626#define MUTEX_PROF_RESET(mtx) \
3627 malloc_mutex_lock(tsdn, &mtx); \
3628 malloc_mutex_prof_data_reset(tsdn, &mtx); \
3629 malloc_mutex_unlock(tsdn, &mtx);
3630
3631 /* Global mutexes: ctl and prof. */
3632 MUTEX_PROF_RESET(ctl_mtx);
3633 if (have_background_thread) {
3634 MUTEX_PROF_RESET(background_thread_lock);
3635 }
3636 if (config_prof && opt_prof) {
3637 MUTEX_PROF_RESET(bt2gctx_mtx);
3638 MUTEX_PROF_RESET(tdatas_mtx);
3639 MUTEX_PROF_RESET(prof_dump_mtx);
3640 MUTEX_PROF_RESET(prof_recent_alloc_mtx);
3641 MUTEX_PROF_RESET(prof_recent_dump_mtx);
3642 MUTEX_PROF_RESET(prof_stats_mtx);
3643 }
3644
3645 /* Per arena mutexes. */
3646 unsigned n = narenas_total_get();
3647
3648 for (unsigned i = 0; i < n; i++) {
3649 arena_t *arena = arena_get(tsdn, i, false);
3650 if (!arena) {
3651 continue;
3652 }
3653 MUTEX_PROF_RESET(arena->large_mtx);
3654 MUTEX_PROF_RESET(arena->pa_shard.edata_cache.mtx);
3655 MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_dirty.mtx);
3656 MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_muzzy.mtx);
3657 MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_retained.mtx);
3658 MUTEX_PROF_RESET(arena->pa_shard.pac.decay_dirty.mtx);
3659 MUTEX_PROF_RESET(arena->pa_shard.pac.decay_muzzy.mtx);
3660 MUTEX_PROF_RESET(arena->tcache_ql_mtx);
3661 MUTEX_PROF_RESET(arena->base->mtx);
3662
3663 for (szind_t j = 0; j < SC_NBINS; j++) {
3664 for (unsigned k = 0; k < bin_infos[j].n_shards; k++) {
3665 bin_t *bin = arena_get_bin(arena, j, k);
3666 MUTEX_PROF_RESET(bin->lock);
3667 }
3668 }
3669 }
3670#undef MUTEX_PROF_RESET
3671 return 0;
3672}
3673
3674CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc,
3675 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nmalloc, uint64_t)
3676CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_ndalloc,
3677 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.ndalloc, uint64_t)
3678CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nrequests,
3679 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nrequests, uint64_t)
3680CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curregs,
3681 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.curregs, size_t)
3682CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nfills,
3683 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nfills, uint64_t)
3684CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nflushes,
3685 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nflushes, uint64_t)
3686CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nslabs,
3687 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nslabs, uint64_t)
3688CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreslabs,
3689 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.reslabs, uint64_t)
3690CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curslabs,
3691 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.curslabs, size_t)
3692CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nonfull_slabs,
3693 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nonfull_slabs, size_t)
3694
3695static const ctl_named_node_t *
3696stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib,
3697 size_t miblen, size_t j) {
3698 if (j > SC_NBINS) {
3699 return NULL;
3700 }
3701 return super_stats_arenas_i_bins_j_node;
3702}
3703
3704CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nmalloc,
3705 locked_read_u64_unsynchronized(
3706 &arenas_i(mib[2])->astats->lstats[mib[4]].nmalloc), uint64_t)
3707CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_ndalloc,
3708 locked_read_u64_unsynchronized(
3709 &arenas_i(mib[2])->astats->lstats[mib[4]].ndalloc), uint64_t)
3710CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nrequests,
3711 locked_read_u64_unsynchronized(
3712 &arenas_i(mib[2])->astats->lstats[mib[4]].nrequests), uint64_t)
3713CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents,
3714 arenas_i(mib[2])->astats->lstats[mib[4]].curlextents, size_t)
3715
3716static const ctl_named_node_t *
3717stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib,
3718 size_t miblen, size_t j) {
3719 if (j > SC_NSIZES - SC_NBINS) {
3720 return NULL;
3721 }
3722 return super_stats_arenas_i_lextents_j_node;
3723}
3724
3725CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_ndirty,
3726 arenas_i(mib[2])->astats->estats[mib[4]].ndirty, size_t);
3727CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nmuzzy,
3728 arenas_i(mib[2])->astats->estats[mib[4]].nmuzzy, size_t);
3729CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nretained,
3730 arenas_i(mib[2])->astats->estats[mib[4]].nretained, size_t);
3731CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_dirty_bytes,
3732 arenas_i(mib[2])->astats->estats[mib[4]].dirty_bytes, size_t);
3733CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_muzzy_bytes,
3734 arenas_i(mib[2])->astats->estats[mib[4]].muzzy_bytes, size_t);
3735CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_retained_bytes,
3736 arenas_i(mib[2])->astats->estats[mib[4]].retained_bytes, size_t);
3737
3738static const ctl_named_node_t *
3739stats_arenas_i_extents_j_index(tsdn_t *tsdn, const size_t *mib,
3740 size_t miblen, size_t j) {
3741 if (j >= SC_NPSIZES) {
3742 return NULL;
3743 }
3744 return super_stats_arenas_i_extents_j_node;
3745}
3746
3747CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_npurge_passes,
3748 arenas_i(mib[2])->astats->hpastats.nonderived_stats.npurge_passes, uint64_t);
3749CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_npurges,
3750 arenas_i(mib[2])->astats->hpastats.nonderived_stats.npurges, uint64_t);
3751CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nhugifies,
3752 arenas_i(mib[2])->astats->hpastats.nonderived_stats.nhugifies, uint64_t);
3753CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_ndehugifies,
3754 arenas_i(mib[2])->astats->hpastats.nonderived_stats.ndehugifies, uint64_t);
3755
3756/* Full, nonhuge */
3757CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge,
3758 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].npageslabs,
3759 size_t);
3760CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge,
3761 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].nactive, size_t);
3762CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ndirty_nonhuge,
3763 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].ndirty, size_t);
3764
3765/* Full, huge */
3766CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge,
3767 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].npageslabs,
3768 size_t);
3769CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_huge,
3770 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].nactive, size_t);
3771CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ndirty_huge,
3772 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].ndirty, size_t);
3773
3774/* Empty, nonhuge */
3775CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_npageslabs_nonhuge,
3776 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].npageslabs,
3777 size_t);
3778CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_nactive_nonhuge,
3779 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].nactive, size_t);
3780CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_ndirty_nonhuge,
3781 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].ndirty, size_t);
3782
3783/* Empty, huge */
3784CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_npageslabs_huge,
3785 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].npageslabs,
3786 size_t);
3787CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_nactive_huge,
3788 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].nactive, size_t);
3789CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_ndirty_huge,
3790 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].ndirty, size_t);
3791
3792/* Nonfull, nonhuge */
3793CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge,
3794 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].npageslabs,
3795 size_t);
3796CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge,
3797 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].nactive,
3798 size_t);
3799CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge,
3800 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].ndirty,
3801 size_t);
3802
3803/* Nonfull, huge */
3804CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge,
3805 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].npageslabs,
3806 size_t);
3807CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge,
3808 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].nactive,
3809 size_t);
3810CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge,
3811 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].ndirty,
3812 size_t);
3813
3814static const ctl_named_node_t *
3815stats_arenas_i_hpa_shard_nonfull_slabs_j_index(tsdn_t *tsdn, const size_t *mib,
3816 size_t miblen, size_t j) {
3817 if (j >= PSSET_NPSIZES) {
3818 return NULL;
3819 }
3820 return super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node;
3821}
3822
3823static bool
3824ctl_arenas_i_verify(size_t i) {
3825 size_t a = arenas_i2a_impl(i, true, true);
3826 if (a == UINT_MAX || !ctl_arenas->arenas[a]->initialized) {
3827 return true;
3828 }
3829
3830 return false;
3831}
3832
3833static const ctl_named_node_t *
3834stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib,
3835 size_t miblen, size_t i) {
3836 const ctl_named_node_t *ret;
3837
3838 malloc_mutex_lock(tsdn, &ctl_mtx);
3839 if (ctl_arenas_i_verify(i)) {
3840 ret = NULL;
3841 goto label_return;
3842 }
3843
3844 ret = super_stats_arenas_i_node;
3845label_return:
3846 malloc_mutex_unlock(tsdn, &ctl_mtx);
3847 return ret;
3848}
3849
3850static int
3851experimental_hooks_install_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3852 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3853 int ret;
3854 if (oldp == NULL || oldlenp == NULL|| newp == NULL) {
3855 ret = EINVAL;
3856 goto label_return;
3857 }
3858 /*
3859 * Note: this is a *private* struct. This is an experimental interface;
3860 * forcing the user to know the jemalloc internals well enough to
3861 * extract the ABI hopefully ensures nobody gets too comfortable with
3862 * this API, which can change at a moment's notice.
3863 */
3864 hooks_t hooks;
3865 WRITE(hooks, hooks_t);
3866 void *handle = hook_install(tsd_tsdn(tsd), &hooks);
3867 if (handle == NULL) {
3868 ret = EAGAIN;
3869 goto label_return;
3870 }
3871 READ(handle, void *);
3872
3873 ret = 0;
3874label_return:
3875 return ret;
3876}
3877
3878static int
3879experimental_hooks_remove_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3880 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3881 int ret;
3882 WRITEONLY();
3883 void *handle = NULL;
3884 WRITE(handle, void *);
3885 if (handle == NULL) {
3886 ret = EINVAL;
3887 goto label_return;
3888 }
3889 hook_remove(tsd_tsdn(tsd), handle);
3890 ret = 0;
3891label_return:
3892 return ret;
3893}
3894
3895static int
3896experimental_thread_activity_callback_ctl(tsd_t *tsd, const size_t *mib,
3897 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3898 int ret;
3899
3900 if (!config_stats) {
3901 return ENOENT;
3902 }
3903
3904 activity_callback_thunk_t t_old = tsd_activity_callback_thunk_get(tsd);
3905 READ(t_old, activity_callback_thunk_t);
3906
3907 if (newp != NULL) {
3908 /*
3909 * This initialization is unnecessary. If it's omitted, though,
3910 * clang gets confused and warns on the subsequent use of t_new.
3911 */
3912 activity_callback_thunk_t t_new = {NULL, NULL};
3913 WRITE(t_new, activity_callback_thunk_t);
3914 tsd_activity_callback_thunk_set(tsd, t_new);
3915 }
3916 ret = 0;
3917label_return:
3918 return ret;
3919}
3920
3921/*
3922 * Output six memory utilization entries for an input pointer, the first one of
3923 * type (void *) and the remaining five of type size_t, describing the following
3924 * (in the same order):
3925 *
3926 * (a) memory address of the extent a potential reallocation would go into,
3927 * == the five fields below describe about the extent the pointer resides in ==
3928 * (b) number of free regions in the extent,
3929 * (c) number of regions in the extent,
3930 * (d) size of the extent in terms of bytes,
3931 * (e) total number of free regions in the bin the extent belongs to, and
3932 * (f) total number of regions in the bin the extent belongs to.
3933 *
3934 * Note that "(e)" and "(f)" are only available when stats are enabled;
3935 * otherwise their values are undefined.
3936 *
3937 * This API is mainly intended for small class allocations, where extents are
3938 * used as slab. Note that if the bin the extent belongs to is completely
3939 * full, "(a)" will be NULL.
3940 *
3941 * In case of large class allocations, "(a)" will be NULL, and "(e)" and "(f)"
3942 * will be zero (if stats are enabled; otherwise undefined). The other three
3943 * fields will be properly set though the values are trivial: "(b)" will be 0,
3944 * "(c)" will be 1, and "(d)" will be the usable size.
3945 *
3946 * The input pointer and size are respectively passed in by newp and newlen,
3947 * and the output fields and size are respectively oldp and *oldlenp.
3948 *
3949 * It can be beneficial to define the following macros to make it easier to
3950 * access the output:
3951 *
3952 * #define SLABCUR_READ(out) (*(void **)out)
3953 * #define COUNTS(out) ((size_t *)((void **)out + 1))
3954 * #define NFREE_READ(out) COUNTS(out)[0]
3955 * #define NREGS_READ(out) COUNTS(out)[1]
3956 * #define SIZE_READ(out) COUNTS(out)[2]
3957 * #define BIN_NFREE_READ(out) COUNTS(out)[3]
3958 * #define BIN_NREGS_READ(out) COUNTS(out)[4]
3959 *
3960 * and then write e.g. NFREE_READ(oldp) to fetch the output. See the unit test
3961 * test_query in test/unit/extent_util.c for an example.
3962 *
3963 * For a typical defragmentation workflow making use of this API for
3964 * understanding the fragmentation level, please refer to the comment for
3965 * experimental_utilization_batch_query_ctl.
3966 *
3967 * It's up to the application how to determine the significance of
3968 * fragmentation relying on the outputs returned. Possible choices are:
3969 *
3970 * (a) if extent utilization ratio is below certain threshold,
3971 * (b) if extent memory consumption is above certain threshold,
3972 * (c) if extent utilization ratio is significantly below bin utilization ratio,
3973 * (d) if input pointer deviates a lot from potential reallocation address, or
3974 * (e) some selection/combination of the above.
3975 *
3976 * The caller needs to make sure that the input/output arguments are valid,
3977 * in particular, that the size of the output is correct, i.e.:
3978 *
3979 * *oldlenp = sizeof(void *) + sizeof(size_t) * 5
3980 *
3981 * Otherwise, the function immediately returns EINVAL without touching anything.
3982 *
3983 * In the rare case where there's no associated extent found for the input
3984 * pointer, the function zeros out all output fields and return. Please refer
3985 * to the comment for experimental_utilization_batch_query_ctl to understand the
3986 * motivation from C++.
3987 */
3988static int
3989experimental_utilization_query_ctl(tsd_t *tsd, const size_t *mib,
3990 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3991 int ret;
3992
3993 assert(sizeof(inspect_extent_util_stats_verbose_t)
3994 == sizeof(void *) + sizeof(size_t) * 5);
3995
3996 if (oldp == NULL || oldlenp == NULL
3997 || *oldlenp != sizeof(inspect_extent_util_stats_verbose_t)
3998 || newp == NULL) {
3999 ret = EINVAL;
4000 goto label_return;
4001 }
4002
4003 void *ptr = NULL;
4004 WRITE(ptr, void *);
4005 inspect_extent_util_stats_verbose_t *util_stats
4006 = (inspect_extent_util_stats_verbose_t *)oldp;
4007 inspect_extent_util_stats_verbose_get(tsd_tsdn(tsd), ptr,
4008 &util_stats->nfree, &util_stats->nregs, &util_stats->size,
4009 &util_stats->bin_nfree, &util_stats->bin_nregs,
4010 &util_stats->slabcur_addr);
4011 ret = 0;
4012
4013label_return:
4014 return ret;
4015}
4016
4017/*
4018 * Given an input array of pointers, output three memory utilization entries of
4019 * type size_t for each input pointer about the extent it resides in:
4020 *
4021 * (a) number of free regions in the extent,
4022 * (b) number of regions in the extent, and
4023 * (c) size of the extent in terms of bytes.
4024 *
4025 * This API is mainly intended for small class allocations, where extents are
4026 * used as slab. In case of large class allocations, the outputs are trivial:
4027 * "(a)" will be 0, "(b)" will be 1, and "(c)" will be the usable size.
4028 *
4029 * Note that multiple input pointers may reside on a same extent so the output
4030 * fields may contain duplicates.
4031 *
4032 * The format of the input/output looks like:
4033 *
4034 * input[0]: 1st_pointer_to_query | output[0]: 1st_extent_n_free_regions
4035 * | output[1]: 1st_extent_n_regions
4036 * | output[2]: 1st_extent_size
4037 * input[1]: 2nd_pointer_to_query | output[3]: 2nd_extent_n_free_regions
4038 * | output[4]: 2nd_extent_n_regions
4039 * | output[5]: 2nd_extent_size
4040 * ... | ...
4041 *
4042 * The input array and size are respectively passed in by newp and newlen, and
4043 * the output array and size are respectively oldp and *oldlenp.
4044 *
4045 * It can be beneficial to define the following macros to make it easier to
4046 * access the output:
4047 *
4048 * #define NFREE_READ(out, i) out[(i) * 3]
4049 * #define NREGS_READ(out, i) out[(i) * 3 + 1]
4050 * #define SIZE_READ(out, i) out[(i) * 3 + 2]
4051 *
4052 * and then write e.g. NFREE_READ(oldp, i) to fetch the output. See the unit
4053 * test test_batch in test/unit/extent_util.c for a concrete example.
4054 *
4055 * A typical workflow would be composed of the following steps:
4056 *
4057 * (1) flush tcache: mallctl("thread.tcache.flush", ...)
4058 * (2) initialize input array of pointers to query fragmentation
4059 * (3) allocate output array to hold utilization statistics
4060 * (4) query utilization: mallctl("experimental.utilization.batch_query", ...)
4061 * (5) (optional) decide if it's worthwhile to defragment; otherwise stop here
4062 * (6) disable tcache: mallctl("thread.tcache.enabled", ...)
4063 * (7) defragment allocations with significant fragmentation, e.g.:
4064 * for each allocation {
4065 * if it's fragmented {
4066 * malloc(...);
4067 * memcpy(...);
4068 * free(...);
4069 * }
4070 * }
4071 * (8) enable tcache: mallctl("thread.tcache.enabled", ...)
4072 *
4073 * The application can determine the significance of fragmentation themselves
4074 * relying on the statistics returned, both at the overall level i.e. step "(5)"
4075 * and at individual allocation level i.e. within step "(7)". Possible choices
4076 * are:
4077 *
4078 * (a) whether memory utilization ratio is below certain threshold,
4079 * (b) whether memory consumption is above certain threshold, or
4080 * (c) some combination of the two.
4081 *
4082 * The caller needs to make sure that the input/output arrays are valid and
4083 * their sizes are proper as well as matched, meaning:
4084 *
4085 * (a) newlen = n_pointers * sizeof(const void *)
4086 * (b) *oldlenp = n_pointers * sizeof(size_t) * 3
4087 * (c) n_pointers > 0
4088 *
4089 * Otherwise, the function immediately returns EINVAL without touching anything.
4090 *
4091 * In the rare case where there's no associated extent found for some pointers,
4092 * rather than immediately terminating the computation and raising an error,
4093 * the function simply zeros out the corresponding output fields and continues
4094 * the computation until all input pointers are handled. The motivations of
4095 * such a design are as follows:
4096 *
4097 * (a) The function always either processes nothing or processes everything, and
4098 * never leaves the output half touched and half untouched.
4099 *
4100 * (b) It facilitates usage needs especially common in C++. A vast variety of
4101 * C++ objects are instantiated with multiple dynamic memory allocations. For
4102 * example, std::string and std::vector typically use at least two allocations,
4103 * one for the metadata and one for the actual content. Other types may use
4104 * even more allocations. When inquiring about utilization statistics, the
4105 * caller often wants to examine into all such allocations, especially internal
4106 * one(s), rather than just the topmost one. The issue comes when some
4107 * implementations do certain optimizations to reduce/aggregate some internal
4108 * allocations, e.g. putting short strings directly into the metadata, and such
4109 * decisions are not known to the caller. Therefore, we permit pointers to
4110 * memory usages that may not be returned by previous malloc calls, and we
4111 * provide the caller a convenient way to identify such cases.
4112 */
4113static int
4114experimental_utilization_batch_query_ctl(tsd_t *tsd, const size_t *mib,
4115 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4116 int ret;
4117
4118 assert(sizeof(inspect_extent_util_stats_t) == sizeof(size_t) * 3);
4119
4120 const size_t len = newlen / sizeof(const void *);
4121 if (oldp == NULL || oldlenp == NULL || newp == NULL || newlen == 0
4122 || newlen != len * sizeof(const void *)
4123 || *oldlenp != len * sizeof(inspect_extent_util_stats_t)) {
4124 ret = EINVAL;
4125 goto label_return;
4126 }
4127
4128 void **ptrs = (void **)newp;
4129 inspect_extent_util_stats_t *util_stats =
4130 (inspect_extent_util_stats_t *)oldp;
4131 size_t i;
4132 for (i = 0; i < len; ++i) {
4133 inspect_extent_util_stats_get(tsd_tsdn(tsd), ptrs[i],
4134 &util_stats[i].nfree, &util_stats[i].nregs,
4135 &util_stats[i].size);
4136 }
4137 ret = 0;
4138
4139label_return:
4140 return ret;
4141}
4142
4143static const ctl_named_node_t *
4144experimental_arenas_i_index(tsdn_t *tsdn, const size_t *mib,
4145 size_t miblen, size_t i) {
4146 const ctl_named_node_t *ret;
4147
4148 malloc_mutex_lock(tsdn, &ctl_mtx);
4149 if (ctl_arenas_i_verify(i)) {
4150 ret = NULL;
4151 goto label_return;
4152 }
4153 ret = super_experimental_arenas_i_node;
4154label_return:
4155 malloc_mutex_unlock(tsdn, &ctl_mtx);
4156 return ret;
4157}
4158
4159static int
4160experimental_arenas_i_pactivep_ctl(tsd_t *tsd, const size_t *mib,
4161 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4162 if (!config_stats) {
4163 return ENOENT;
4164 }
4165 if (oldp == NULL || oldlenp == NULL || *oldlenp != sizeof(size_t *)) {
4166 return EINVAL;
4167 }
4168
4169 unsigned arena_ind;
4170 arena_t *arena;
4171 int ret;
4172 size_t *pactivep;
4173
4174 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
4175 READONLY();
4176 MIB_UNSIGNED(arena_ind, 2);
4177 if (arena_ind < narenas_total_get() && (arena =
4178 arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) {
4179#if defined(JEMALLOC_GCC_ATOMIC_ATOMICS) || \
4180 defined(JEMALLOC_GCC_SYNC_ATOMICS) || defined(_MSC_VER)
4181 /* Expose the underlying counter for fast read. */
4182 pactivep = (size_t *)&(arena->pa_shard.nactive.repr);
4183 READ(pactivep, size_t *);
4184 ret = 0;
4185#else
4186 ret = EFAULT;
4187#endif
4188 } else {
4189 ret = EFAULT;
4190 }
4191label_return:
4192 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
4193 return ret;
4194}
4195
4196static int
4197experimental_prof_recent_alloc_max_ctl(tsd_t *tsd, const size_t *mib,
4198 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4199 int ret;
4200
4201 if (!(config_prof && opt_prof)) {
4202 ret = ENOENT;
4203 goto label_return;
4204 }
4205
4206 ssize_t old_max;
4207 if (newp != NULL) {
4208 if (newlen != sizeof(ssize_t)) {
4209 ret = EINVAL;
4210 goto label_return;
4211 }
4212 ssize_t max = *(ssize_t *)newp;
4213 if (max < -1) {
4214 ret = EINVAL;
4215 goto label_return;
4216 }
4217 old_max = prof_recent_alloc_max_ctl_write(tsd, max);
4218 } else {
4219 old_max = prof_recent_alloc_max_ctl_read();
4220 }
4221 READ(old_max, ssize_t);
4222
4223 ret = 0;
4224
4225label_return:
4226 return ret;
4227}
4228
4229typedef struct write_cb_packet_s write_cb_packet_t;
4230struct write_cb_packet_s {
4231 write_cb_t *write_cb;
4232 void *cbopaque;
4233};
4234
4235static int
4236experimental_prof_recent_alloc_dump_ctl(tsd_t *tsd, const size_t *mib,
4237 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4238 int ret;
4239
4240 if (!(config_prof && opt_prof)) {
4241 ret = ENOENT;
4242 goto label_return;
4243 }
4244
4245 assert(sizeof(write_cb_packet_t) == sizeof(void *) * 2);
4246
4247 WRITEONLY();
4248 write_cb_packet_t write_cb_packet;
4249 ASSURED_WRITE(write_cb_packet, write_cb_packet_t);
4250
4251 prof_recent_alloc_dump(tsd, write_cb_packet.write_cb,
4252 write_cb_packet.cbopaque);
4253
4254 ret = 0;
4255
4256label_return:
4257 return ret;
4258}
4259
4260typedef struct batch_alloc_packet_s batch_alloc_packet_t;
4261struct batch_alloc_packet_s {
4262 void **ptrs;
4263 size_t num;
4264 size_t size;
4265 int flags;
4266};
4267
4268static int
4269experimental_batch_alloc_ctl(tsd_t *tsd, const size_t *mib,
4270 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4271 int ret;
4272
4273 VERIFY_READ(size_t);
4274
4275 batch_alloc_packet_t batch_alloc_packet;
4276 ASSURED_WRITE(batch_alloc_packet, batch_alloc_packet_t);
4277 size_t filled = batch_alloc(batch_alloc_packet.ptrs,
4278 batch_alloc_packet.num, batch_alloc_packet.size,
4279 batch_alloc_packet.flags);
4280 READ(filled, size_t);
4281
4282 ret = 0;
4283
4284label_return:
4285 return ret;
4286}
4287
4288static int
4289prof_stats_bins_i_live_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
4290 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4291 int ret;
4292 unsigned binind;
4293 prof_stats_t stats;
4294
4295 if (!(config_prof && opt_prof && opt_prof_stats)) {
4296 ret = ENOENT;
4297 goto label_return;
4298 }
4299
4300 READONLY();
4301 MIB_UNSIGNED(binind, 3);
4302 if (binind >= SC_NBINS) {
4303 ret = EINVAL;
4304 goto label_return;
4305 }
4306 prof_stats_get_live(tsd, (szind_t)binind, &stats);
4307 READ(stats, prof_stats_t);
4308
4309 ret = 0;
4310label_return:
4311 return ret;
4312}
4313
4314static int
4315prof_stats_bins_i_accum_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
4316 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4317 int ret;
4318 unsigned binind;
4319 prof_stats_t stats;
4320
4321 if (!(config_prof && opt_prof && opt_prof_stats)) {
4322 ret = ENOENT;
4323 goto label_return;
4324 }
4325
4326 READONLY();
4327 MIB_UNSIGNED(binind, 3);
4328 if (binind >= SC_NBINS) {
4329 ret = EINVAL;
4330 goto label_return;
4331 }
4332 prof_stats_get_accum(tsd, (szind_t)binind, &stats);
4333 READ(stats, prof_stats_t);
4334
4335 ret = 0;
4336label_return:
4337 return ret;
4338}
4339
4340static const ctl_named_node_t *
4341prof_stats_bins_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen,
4342 size_t i) {
4343 if (!(config_prof && opt_prof && opt_prof_stats)) {
4344 return NULL;
4345 }
4346 if (i >= SC_NBINS) {
4347 return NULL;
4348 }
4349 return super_prof_stats_bins_i_node;
4350}
4351
4352static int
4353prof_stats_lextents_i_live_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
4354 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4355 int ret;
4356 unsigned lextent_ind;
4357 prof_stats_t stats;
4358
4359 if (!(config_prof && opt_prof && opt_prof_stats)) {
4360 ret = ENOENT;
4361 goto label_return;
4362 }
4363
4364 READONLY();
4365 MIB_UNSIGNED(lextent_ind, 3);
4366 if (lextent_ind >= SC_NSIZES - SC_NBINS) {
4367 ret = EINVAL;
4368 goto label_return;
4369 }
4370 prof_stats_get_live(tsd, (szind_t)(lextent_ind + SC_NBINS), &stats);
4371 READ(stats, prof_stats_t);
4372
4373 ret = 0;
4374label_return:
4375 return ret;
4376}
4377
4378static int
4379prof_stats_lextents_i_accum_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
4380 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4381 int ret;
4382 unsigned lextent_ind;
4383 prof_stats_t stats;
4384
4385 if (!(config_prof && opt_prof && opt_prof_stats)) {
4386 ret = ENOENT;
4387 goto label_return;
4388 }
4389
4390 READONLY();
4391 MIB_UNSIGNED(lextent_ind, 3);
4392 if (lextent_ind >= SC_NSIZES - SC_NBINS) {
4393 ret = EINVAL;
4394 goto label_return;
4395 }
4396 prof_stats_get_accum(tsd, (szind_t)(lextent_ind + SC_NBINS), &stats);
4397 READ(stats, prof_stats_t);
4398
4399 ret = 0;
4400label_return:
4401 return ret;
4402}
4403
4404static const ctl_named_node_t *
4405prof_stats_lextents_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen,
4406 size_t i) {
4407 if (!(config_prof && opt_prof && opt_prof_stats)) {
4408 return NULL;
4409 }
4410 if (i >= SC_NSIZES - SC_NBINS) {
4411 return NULL;
4412 }
4413 return super_prof_stats_lextents_i_node;
4414}
diff --git a/examples/redis-unstable/deps/jemalloc/src/decay.c b/examples/redis-unstable/deps/jemalloc/src/decay.c
deleted file mode 100644
index d801b2b..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/decay.c
+++ /dev/null
@@ -1,295 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/decay.h"
5
6static const uint64_t h_steps[SMOOTHSTEP_NSTEPS] = {
7#define STEP(step, h, x, y) \
8 h,
9 SMOOTHSTEP
10#undef STEP
11};
12
13/*
14 * Generate a new deadline that is uniformly random within the next epoch after
15 * the current one.
16 */
17void
18decay_deadline_init(decay_t *decay) {
19 nstime_copy(&decay->deadline, &decay->epoch);
20 nstime_add(&decay->deadline, &decay->interval);
21 if (decay_ms_read(decay) > 0) {
22 nstime_t jitter;
23
24 nstime_init(&jitter, prng_range_u64(&decay->jitter_state,
25 nstime_ns(&decay->interval)));
26 nstime_add(&decay->deadline, &jitter);
27 }
28}
29
30void
31decay_reinit(decay_t *decay, nstime_t *cur_time, ssize_t decay_ms) {
32 atomic_store_zd(&decay->time_ms, decay_ms, ATOMIC_RELAXED);
33 if (decay_ms > 0) {
34 nstime_init(&decay->interval, (uint64_t)decay_ms *
35 KQU(1000000));
36 nstime_idivide(&decay->interval, SMOOTHSTEP_NSTEPS);
37 }
38
39 nstime_copy(&decay->epoch, cur_time);
40 decay->jitter_state = (uint64_t)(uintptr_t)decay;
41 decay_deadline_init(decay);
42 decay->nunpurged = 0;
43 memset(decay->backlog, 0, SMOOTHSTEP_NSTEPS * sizeof(size_t));
44}
45
46bool
47decay_init(decay_t *decay, nstime_t *cur_time, ssize_t decay_ms) {
48 if (config_debug) {
49 for (size_t i = 0; i < sizeof(decay_t); i++) {
50 assert(((char *)decay)[i] == 0);
51 }
52 decay->ceil_npages = 0;
53 }
54 if (malloc_mutex_init(&decay->mtx, "decay", WITNESS_RANK_DECAY,
55 malloc_mutex_rank_exclusive)) {
56 return true;
57 }
58 decay->purging = false;
59 decay_reinit(decay, cur_time, decay_ms);
60 return false;
61}
62
63bool
64decay_ms_valid(ssize_t decay_ms) {
65 if (decay_ms < -1) {
66 return false;
67 }
68 if (decay_ms == -1 || (uint64_t)decay_ms <= NSTIME_SEC_MAX *
69 KQU(1000)) {
70 return true;
71 }
72 return false;
73}
74
75static void
76decay_maybe_update_time(decay_t *decay, nstime_t *new_time) {
77 if (unlikely(!nstime_monotonic() && nstime_compare(&decay->epoch,
78 new_time) > 0)) {
79 /*
80 * Time went backwards. Move the epoch back in time and
81 * generate a new deadline, with the expectation that time
82 * typically flows forward for long enough periods of time that
83 * epochs complete. Unfortunately, this strategy is susceptible
84 * to clock jitter triggering premature epoch advances, but
85 * clock jitter estimation and compensation isn't feasible here
86 * because calls into this code are event-driven.
87 */
88 nstime_copy(&decay->epoch, new_time);
89 decay_deadline_init(decay);
90 } else {
91 /* Verify that time does not go backwards. */
92 assert(nstime_compare(&decay->epoch, new_time) <= 0);
93 }
94}
95
96static size_t
97decay_backlog_npages_limit(const decay_t *decay) {
98 /*
99 * For each element of decay_backlog, multiply by the corresponding
100 * fixed-point smoothstep decay factor. Sum the products, then divide
101 * to round down to the nearest whole number of pages.
102 */
103 uint64_t sum = 0;
104 for (unsigned i = 0; i < SMOOTHSTEP_NSTEPS; i++) {
105 sum += decay->backlog[i] * h_steps[i];
106 }
107 size_t npages_limit_backlog = (size_t)(sum >> SMOOTHSTEP_BFP);
108
109 return npages_limit_backlog;
110}
111
112/*
113 * Update backlog, assuming that 'nadvance_u64' time intervals have passed.
114 * Trailing 'nadvance_u64' records should be erased and 'current_npages' is
115 * placed as the newest record.
116 */
117static void
118decay_backlog_update(decay_t *decay, uint64_t nadvance_u64,
119 size_t current_npages) {
120 if (nadvance_u64 >= SMOOTHSTEP_NSTEPS) {
121 memset(decay->backlog, 0, (SMOOTHSTEP_NSTEPS-1) *
122 sizeof(size_t));
123 } else {
124 size_t nadvance_z = (size_t)nadvance_u64;
125
126 assert((uint64_t)nadvance_z == nadvance_u64);
127
128 memmove(decay->backlog, &decay->backlog[nadvance_z],
129 (SMOOTHSTEP_NSTEPS - nadvance_z) * sizeof(size_t));
130 if (nadvance_z > 1) {
131 memset(&decay->backlog[SMOOTHSTEP_NSTEPS -
132 nadvance_z], 0, (nadvance_z-1) * sizeof(size_t));
133 }
134 }
135
136 size_t npages_delta = (current_npages > decay->nunpurged) ?
137 current_npages - decay->nunpurged : 0;
138 decay->backlog[SMOOTHSTEP_NSTEPS-1] = npages_delta;
139
140 if (config_debug) {
141 if (current_npages > decay->ceil_npages) {
142 decay->ceil_npages = current_npages;
143 }
144 size_t npages_limit = decay_backlog_npages_limit(decay);
145 assert(decay->ceil_npages >= npages_limit);
146 if (decay->ceil_npages > npages_limit) {
147 decay->ceil_npages = npages_limit;
148 }
149 }
150}
151
152static inline bool
153decay_deadline_reached(const decay_t *decay, const nstime_t *time) {
154 return (nstime_compare(&decay->deadline, time) <= 0);
155}
156
157uint64_t
158decay_npages_purge_in(decay_t *decay, nstime_t *time, size_t npages_new) {
159 uint64_t decay_interval_ns = decay_epoch_duration_ns(decay);
160 size_t n_epoch = (size_t)(nstime_ns(time) / decay_interval_ns);
161
162 uint64_t npages_purge;
163 if (n_epoch >= SMOOTHSTEP_NSTEPS) {
164 npages_purge = npages_new;
165 } else {
166 uint64_t h_steps_max = h_steps[SMOOTHSTEP_NSTEPS - 1];
167 assert(h_steps_max >=
168 h_steps[SMOOTHSTEP_NSTEPS - 1 - n_epoch]);
169 npages_purge = npages_new * (h_steps_max -
170 h_steps[SMOOTHSTEP_NSTEPS - 1 - n_epoch]);
171 npages_purge >>= SMOOTHSTEP_BFP;
172 }
173 return npages_purge;
174}
175
176bool
177decay_maybe_advance_epoch(decay_t *decay, nstime_t *new_time,
178 size_t npages_current) {
179 /* Handle possible non-monotonicity of time. */
180 decay_maybe_update_time(decay, new_time);
181
182 if (!decay_deadline_reached(decay, new_time)) {
183 return false;
184 }
185 nstime_t delta;
186 nstime_copy(&delta, new_time);
187 nstime_subtract(&delta, &decay->epoch);
188
189 uint64_t nadvance_u64 = nstime_divide(&delta, &decay->interval);
190 assert(nadvance_u64 > 0);
191
192 /* Add nadvance_u64 decay intervals to epoch. */
193 nstime_copy(&delta, &decay->interval);
194 nstime_imultiply(&delta, nadvance_u64);
195 nstime_add(&decay->epoch, &delta);
196
197 /* Set a new deadline. */
198 decay_deadline_init(decay);
199
200 /* Update the backlog. */
201 decay_backlog_update(decay, nadvance_u64, npages_current);
202
203 decay->npages_limit = decay_backlog_npages_limit(decay);
204 decay->nunpurged = (decay->npages_limit > npages_current) ?
205 decay->npages_limit : npages_current;
206
207 return true;
208}
209
210/*
211 * Calculate how many pages should be purged after 'interval'.
212 *
213 * First, calculate how many pages should remain at the moment, then subtract
214 * the number of pages that should remain after 'interval'. The difference is
215 * how many pages should be purged until then.
216 *
217 * The number of pages that should remain at a specific moment is calculated
218 * like this: pages(now) = sum(backlog[i] * h_steps[i]). After 'interval'
219 * passes, backlog would shift 'interval' positions to the left and sigmoid
220 * curve would be applied starting with backlog[interval].
221 *
222 * The implementation doesn't directly map to the description, but it's
223 * essentially the same calculation, optimized to avoid iterating over
224 * [interval..SMOOTHSTEP_NSTEPS) twice.
225 */
226static inline size_t
227decay_npurge_after_interval(decay_t *decay, size_t interval) {
228 size_t i;
229 uint64_t sum = 0;
230 for (i = 0; i < interval; i++) {
231 sum += decay->backlog[i] * h_steps[i];
232 }
233 for (; i < SMOOTHSTEP_NSTEPS; i++) {
234 sum += decay->backlog[i] *
235 (h_steps[i] - h_steps[i - interval]);
236 }
237
238 return (size_t)(sum >> SMOOTHSTEP_BFP);
239}
240
241uint64_t decay_ns_until_purge(decay_t *decay, size_t npages_current,
242 uint64_t npages_threshold) {
243 if (!decay_gradually(decay)) {
244 return DECAY_UNBOUNDED_TIME_TO_PURGE;
245 }
246 uint64_t decay_interval_ns = decay_epoch_duration_ns(decay);
247 assert(decay_interval_ns > 0);
248 if (npages_current == 0) {
249 unsigned i;
250 for (i = 0; i < SMOOTHSTEP_NSTEPS; i++) {
251 if (decay->backlog[i] > 0) {
252 break;
253 }
254 }
255 if (i == SMOOTHSTEP_NSTEPS) {
256 /* No dirty pages recorded. Sleep indefinitely. */
257 return DECAY_UNBOUNDED_TIME_TO_PURGE;
258 }
259 }
260 if (npages_current <= npages_threshold) {
261 /* Use max interval. */
262 return decay_interval_ns * SMOOTHSTEP_NSTEPS;
263 }
264
265 /* Minimal 2 intervals to ensure reaching next epoch deadline. */
266 size_t lb = 2;
267 size_t ub = SMOOTHSTEP_NSTEPS;
268
269 size_t npurge_lb, npurge_ub;
270 npurge_lb = decay_npurge_after_interval(decay, lb);
271 if (npurge_lb > npages_threshold) {
272 return decay_interval_ns * lb;
273 }
274 npurge_ub = decay_npurge_after_interval(decay, ub);
275 if (npurge_ub < npages_threshold) {
276 return decay_interval_ns * ub;
277 }
278
279 unsigned n_search = 0;
280 size_t target, npurge;
281 while ((npurge_lb + npages_threshold < npurge_ub) && (lb + 2 < ub)) {
282 target = (lb + ub) / 2;
283 npurge = decay_npurge_after_interval(decay, target);
284 if (npurge > npages_threshold) {
285 ub = target;
286 npurge_ub = npurge;
287 } else {
288 lb = target;
289 npurge_lb = npurge;
290 }
291 assert(n_search < lg_floor(SMOOTHSTEP_NSTEPS) + 1);
292 ++n_search;
293 }
294 return decay_interval_ns * (ub + lb) / 2;
295}
diff --git a/examples/redis-unstable/deps/jemalloc/src/div.c b/examples/redis-unstable/deps/jemalloc/src/div.c
deleted file mode 100644
index 808892a..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/div.c
+++ /dev/null
@@ -1,55 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2
3#include "jemalloc/internal/div.h"
4
5#include "jemalloc/internal/assert.h"
6
7/*
8 * Suppose we have n = q * d, all integers. We know n and d, and want q = n / d.
9 *
10 * For any k, we have (here, all division is exact; not C-style rounding):
11 * floor(ceil(2^k / d) * n / 2^k) = floor((2^k + r) / d * n / 2^k), where
12 * r = (-2^k) mod d.
13 *
14 * Expanding this out:
15 * ... = floor(2^k / d * n / 2^k + r / d * n / 2^k)
16 * = floor(n / d + (r / d) * (n / 2^k)).
17 *
18 * The fractional part of n / d is 0 (because of the assumption that d divides n
19 * exactly), so we have:
20 * ... = n / d + floor((r / d) * (n / 2^k))
21 *
22 * So that our initial expression is equal to the quantity we seek, so long as
23 * (r / d) * (n / 2^k) < 1.
24 *
25 * r is a remainder mod d, so r < d and r / d < 1 always. We can make
26 * n / 2 ^ k < 1 by setting k = 32. This gets us a value of magic that works.
27 */
28
29void
30div_init(div_info_t *div_info, size_t d) {
31 /* Nonsensical. */
32 assert(d != 0);
33 /*
34 * This would make the value of magic too high to fit into a uint32_t
35 * (we would want magic = 2^32 exactly). This would mess with code gen
36 * on 32-bit machines.
37 */
38 assert(d != 1);
39
40 uint64_t two_to_k = ((uint64_t)1 << 32);
41 uint32_t magic = (uint32_t)(two_to_k / d);
42
43 /*
44 * We want magic = ceil(2^k / d), but C gives us floor. We have to
45 * increment it unless the result was exact (i.e. unless d is a power of
46 * two).
47 */
48 if (two_to_k % d != 0) {
49 magic++;
50 }
51 div_info->magic = magic;
52#ifdef JEMALLOC_DEBUG
53 div_info->d = d;
54#endif
55}
diff --git a/examples/redis-unstable/deps/jemalloc/src/ecache.c b/examples/redis-unstable/deps/jemalloc/src/ecache.c
deleted file mode 100644
index a242227..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/ecache.c
+++ /dev/null
@@ -1,35 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/san.h"
5
6bool
7ecache_init(tsdn_t *tsdn, ecache_t *ecache, extent_state_t state, unsigned ind,
8 bool delay_coalesce) {
9 if (malloc_mutex_init(&ecache->mtx, "extents", WITNESS_RANK_EXTENTS,
10 malloc_mutex_rank_exclusive)) {
11 return true;
12 }
13 ecache->state = state;
14 ecache->ind = ind;
15 ecache->delay_coalesce = delay_coalesce;
16 eset_init(&ecache->eset, state);
17 eset_init(&ecache->guarded_eset, state);
18
19 return false;
20}
21
22void
23ecache_prefork(tsdn_t *tsdn, ecache_t *ecache) {
24 malloc_mutex_prefork(tsdn, &ecache->mtx);
25}
26
27void
28ecache_postfork_parent(tsdn_t *tsdn, ecache_t *ecache) {
29 malloc_mutex_postfork_parent(tsdn, &ecache->mtx);
30}
31
32void
33ecache_postfork_child(tsdn_t *tsdn, ecache_t *ecache) {
34 malloc_mutex_postfork_child(tsdn, &ecache->mtx);
35}
diff --git a/examples/redis-unstable/deps/jemalloc/src/edata.c b/examples/redis-unstable/deps/jemalloc/src/edata.c
deleted file mode 100644
index 82b6f56..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/edata.c
+++ /dev/null
@@ -1,6 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4ph_gen(, edata_avail, edata_t, avail_link,
5 edata_esnead_comp)
6ph_gen(, edata_heap, edata_t, heap_link, edata_snad_comp)
diff --git a/examples/redis-unstable/deps/jemalloc/src/edata_cache.c b/examples/redis-unstable/deps/jemalloc/src/edata_cache.c
deleted file mode 100644
index 6bc1848..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/edata_cache.c
+++ /dev/null
@@ -1,154 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4bool
5edata_cache_init(edata_cache_t *edata_cache, base_t *base) {
6 edata_avail_new(&edata_cache->avail);
7 /*
8 * This is not strictly necessary, since the edata_cache_t is only
9 * created inside an arena, which is zeroed on creation. But this is
10 * handy as a safety measure.
11 */
12 atomic_store_zu(&edata_cache->count, 0, ATOMIC_RELAXED);
13 if (malloc_mutex_init(&edata_cache->mtx, "edata_cache",
14 WITNESS_RANK_EDATA_CACHE, malloc_mutex_rank_exclusive)) {
15 return true;
16 }
17 edata_cache->base = base;
18 return false;
19}
20
21edata_t *
22edata_cache_get(tsdn_t *tsdn, edata_cache_t *edata_cache) {
23 malloc_mutex_lock(tsdn, &edata_cache->mtx);
24 edata_t *edata = edata_avail_first(&edata_cache->avail);
25 if (edata == NULL) {
26 malloc_mutex_unlock(tsdn, &edata_cache->mtx);
27 return base_alloc_edata(tsdn, edata_cache->base);
28 }
29 edata_avail_remove(&edata_cache->avail, edata);
30 atomic_load_sub_store_zu(&edata_cache->count, 1);
31 malloc_mutex_unlock(tsdn, &edata_cache->mtx);
32 return edata;
33}
34
35void
36edata_cache_put(tsdn_t *tsdn, edata_cache_t *edata_cache, edata_t *edata) {
37 malloc_mutex_lock(tsdn, &edata_cache->mtx);
38 edata_avail_insert(&edata_cache->avail, edata);
39 atomic_load_add_store_zu(&edata_cache->count, 1);
40 malloc_mutex_unlock(tsdn, &edata_cache->mtx);
41}
42
43void
44edata_cache_prefork(tsdn_t *tsdn, edata_cache_t *edata_cache) {
45 malloc_mutex_prefork(tsdn, &edata_cache->mtx);
46}
47
48void
49edata_cache_postfork_parent(tsdn_t *tsdn, edata_cache_t *edata_cache) {
50 malloc_mutex_postfork_parent(tsdn, &edata_cache->mtx);
51}
52
53void
54edata_cache_postfork_child(tsdn_t *tsdn, edata_cache_t *edata_cache) {
55 malloc_mutex_postfork_child(tsdn, &edata_cache->mtx);
56}
57
58void
59edata_cache_fast_init(edata_cache_fast_t *ecs, edata_cache_t *fallback) {
60 edata_list_inactive_init(&ecs->list);
61 ecs->fallback = fallback;
62 ecs->disabled = false;
63}
64
65static void
66edata_cache_fast_try_fill_from_fallback(tsdn_t *tsdn,
67 edata_cache_fast_t *ecs) {
68 edata_t *edata;
69 malloc_mutex_lock(tsdn, &ecs->fallback->mtx);
70 for (int i = 0; i < EDATA_CACHE_FAST_FILL; i++) {
71 edata = edata_avail_remove_first(&ecs->fallback->avail);
72 if (edata == NULL) {
73 break;
74 }
75 edata_list_inactive_append(&ecs->list, edata);
76 atomic_load_sub_store_zu(&ecs->fallback->count, 1);
77 }
78 malloc_mutex_unlock(tsdn, &ecs->fallback->mtx);
79}
80
81edata_t *
82edata_cache_fast_get(tsdn_t *tsdn, edata_cache_fast_t *ecs) {
83 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
84 WITNESS_RANK_EDATA_CACHE, 0);
85
86 if (ecs->disabled) {
87 assert(edata_list_inactive_first(&ecs->list) == NULL);
88 return edata_cache_get(tsdn, ecs->fallback);
89 }
90
91 edata_t *edata = edata_list_inactive_first(&ecs->list);
92 if (edata != NULL) {
93 edata_list_inactive_remove(&ecs->list, edata);
94 return edata;
95 }
96 /* Slow path; requires synchronization. */
97 edata_cache_fast_try_fill_from_fallback(tsdn, ecs);
98 edata = edata_list_inactive_first(&ecs->list);
99 if (edata != NULL) {
100 edata_list_inactive_remove(&ecs->list, edata);
101 } else {
102 /*
103 * Slowest path (fallback was also empty); allocate something
104 * new.
105 */
106 edata = base_alloc_edata(tsdn, ecs->fallback->base);
107 }
108 return edata;
109}
110
111static void
112edata_cache_fast_flush_all(tsdn_t *tsdn, edata_cache_fast_t *ecs) {
113 /*
114 * You could imagine smarter cache management policies (like
115 * only flushing down to some threshold in anticipation of
116 * future get requests). But just flushing everything provides
117 * a good opportunity to defrag too, and lets us share code between the
118 * flush and disable pathways.
119 */
120 edata_t *edata;
121 size_t nflushed = 0;
122 malloc_mutex_lock(tsdn, &ecs->fallback->mtx);
123 while ((edata = edata_list_inactive_first(&ecs->list)) != NULL) {
124 edata_list_inactive_remove(&ecs->list, edata);
125 edata_avail_insert(&ecs->fallback->avail, edata);
126 nflushed++;
127 }
128 atomic_load_add_store_zu(&ecs->fallback->count, nflushed);
129 malloc_mutex_unlock(tsdn, &ecs->fallback->mtx);
130}
131
132void
133edata_cache_fast_put(tsdn_t *tsdn, edata_cache_fast_t *ecs, edata_t *edata) {
134 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
135 WITNESS_RANK_EDATA_CACHE, 0);
136
137 if (ecs->disabled) {
138 assert(edata_list_inactive_first(&ecs->list) == NULL);
139 edata_cache_put(tsdn, ecs->fallback, edata);
140 return;
141 }
142
143 /*
144 * Prepend rather than append, to do LIFO ordering in the hopes of some
145 * cache locality.
146 */
147 edata_list_inactive_prepend(&ecs->list, edata);
148}
149
150void
151edata_cache_fast_disable(tsdn_t *tsdn, edata_cache_fast_t *ecs) {
152 edata_cache_fast_flush_all(tsdn, ecs);
153 ecs->disabled = true;
154}
diff --git a/examples/redis-unstable/deps/jemalloc/src/ehooks.c b/examples/redis-unstable/deps/jemalloc/src/ehooks.c
deleted file mode 100644
index 383e9de..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/ehooks.c
+++ /dev/null
@@ -1,275 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/ehooks.h"
5#include "jemalloc/internal/extent_mmap.h"
6
7void
8ehooks_init(ehooks_t *ehooks, extent_hooks_t *extent_hooks, unsigned ind) {
9 /* All other hooks are optional; this one is not. */
10 assert(extent_hooks->alloc != NULL);
11 ehooks->ind = ind;
12 ehooks_set_extent_hooks_ptr(ehooks, extent_hooks);
13}
14
15/*
16 * If the caller specifies (!*zero), it is still possible to receive zeroed
17 * memory, in which case *zero is toggled to true. arena_extent_alloc() takes
18 * advantage of this to avoid demanding zeroed extents, but taking advantage of
19 * them if they are returned.
20 */
21static void *
22extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
23 size_t alignment, bool *zero, bool *commit, dss_prec_t dss_prec) {
24 void *ret;
25
26 assert(size != 0);
27 assert(alignment != 0);
28
29 /* "primary" dss. */
30 if (have_dss && dss_prec == dss_prec_primary && (ret =
31 extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,
32 commit)) != NULL) {
33 return ret;
34 }
35 /* mmap. */
36 if ((ret = extent_alloc_mmap(new_addr, size, alignment, zero, commit))
37 != NULL) {
38 return ret;
39 }
40 /* "secondary" dss. */
41 if (have_dss && dss_prec == dss_prec_secondary && (ret =
42 extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,
43 commit)) != NULL) {
44 return ret;
45 }
46
47 /* All strategies for allocation failed. */
48 return NULL;
49}
50
51void *
52ehooks_default_alloc_impl(tsdn_t *tsdn, void *new_addr, size_t size,
53 size_t alignment, bool *zero, bool *commit, unsigned arena_ind) {
54 arena_t *arena = arena_get(tsdn, arena_ind, false);
55 /* NULL arena indicates arena_create. */
56 assert(arena != NULL || alignment == HUGEPAGE);
57 dss_prec_t dss = (arena == NULL) ? dss_prec_disabled :
58 (dss_prec_t)atomic_load_u(&arena->dss_prec, ATOMIC_RELAXED);
59 void *ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment,
60 zero, commit, dss);
61 if (have_madvise_huge && ret) {
62 pages_set_thp_state(ret, size);
63 }
64 return ret;
65}
66
67static void *
68ehooks_default_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
69 size_t alignment, bool *zero, bool *commit, unsigned arena_ind) {
70 return ehooks_default_alloc_impl(tsdn_fetch(), new_addr, size,
71 ALIGNMENT_CEILING(alignment, PAGE), zero, commit, arena_ind);
72}
73
74bool
75ehooks_default_dalloc_impl(void *addr, size_t size) {
76 if (!have_dss || !extent_in_dss(addr)) {
77 return extent_dalloc_mmap(addr, size);
78 }
79 return true;
80}
81
82static bool
83ehooks_default_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size,
84 bool committed, unsigned arena_ind) {
85 return ehooks_default_dalloc_impl(addr, size);
86}
87
88void
89ehooks_default_destroy_impl(void *addr, size_t size) {
90 if (!have_dss || !extent_in_dss(addr)) {
91 pages_unmap(addr, size);
92 }
93}
94
95static void
96ehooks_default_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size,
97 bool committed, unsigned arena_ind) {
98 ehooks_default_destroy_impl(addr, size);
99}
100
101bool
102ehooks_default_commit_impl(void *addr, size_t offset, size_t length) {
103 return pages_commit((void *)((uintptr_t)addr + (uintptr_t)offset),
104 length);
105}
106
107static bool
108ehooks_default_commit(extent_hooks_t *extent_hooks, void *addr, size_t size,
109 size_t offset, size_t length, unsigned arena_ind) {
110 return ehooks_default_commit_impl(addr, offset, length);
111}
112
113bool
114ehooks_default_decommit_impl(void *addr, size_t offset, size_t length) {
115 return pages_decommit((void *)((uintptr_t)addr + (uintptr_t)offset),
116 length);
117}
118
119static bool
120ehooks_default_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size,
121 size_t offset, size_t length, unsigned arena_ind) {
122 return ehooks_default_decommit_impl(addr, offset, length);
123}
124
125#ifdef PAGES_CAN_PURGE_LAZY
126bool
127ehooks_default_purge_lazy_impl(void *addr, size_t offset, size_t length) {
128 return pages_purge_lazy((void *)((uintptr_t)addr + (uintptr_t)offset),
129 length);
130}
131
132static bool
133ehooks_default_purge_lazy(extent_hooks_t *extent_hooks, void *addr, size_t size,
134 size_t offset, size_t length, unsigned arena_ind) {
135 assert(addr != NULL);
136 assert((offset & PAGE_MASK) == 0);
137 assert(length != 0);
138 assert((length & PAGE_MASK) == 0);
139 return ehooks_default_purge_lazy_impl(addr, offset, length);
140}
141#endif
142
143#ifdef PAGES_CAN_PURGE_FORCED
144bool
145ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length) {
146 return pages_purge_forced((void *)((uintptr_t)addr +
147 (uintptr_t)offset), length);
148}
149
150static bool
151ehooks_default_purge_forced(extent_hooks_t *extent_hooks, void *addr,
152 size_t size, size_t offset, size_t length, unsigned arena_ind) {
153 assert(addr != NULL);
154 assert((offset & PAGE_MASK) == 0);
155 assert(length != 0);
156 assert((length & PAGE_MASK) == 0);
157 return ehooks_default_purge_forced_impl(addr, offset, length);
158}
159#endif
160
161bool
162ehooks_default_split_impl() {
163 if (!maps_coalesce) {
164 /*
165 * Without retain, only whole regions can be purged (required by
166 * MEM_RELEASE on Windows) -- therefore disallow splitting. See
167 * comments in extent_head_no_merge().
168 */
169 return !opt_retain;
170 }
171
172 return false;
173}
174
175static bool
176ehooks_default_split(extent_hooks_t *extent_hooks, void *addr, size_t size,
177 size_t size_a, size_t size_b, bool committed, unsigned arena_ind) {
178 return ehooks_default_split_impl();
179}
180
181bool
182ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, void *addr_b) {
183 assert(addr_a < addr_b);
184 /*
185 * For non-DSS cases --
186 * a) W/o maps_coalesce, merge is not always allowed (Windows):
187 * 1) w/o retain, never merge (first branch below).
188 * 2) with retain, only merge extents from the same VirtualAlloc
189 * region (in which case MEM_DECOMMIT is utilized for purging).
190 *
191 * b) With maps_coalesce, it's always possible to merge.
192 * 1) w/o retain, always allow merge (only about dirty / muzzy).
193 * 2) with retain, to preserve the SN / first-fit, merge is still
194 * disallowed if b is a head extent, i.e. no merging across
195 * different mmap regions.
196 *
197 * a2) and b2) are implemented in emap_try_acquire_edata_neighbor, and
198 * sanity checked in the second branch below.
199 */
200 if (!maps_coalesce && !opt_retain) {
201 return true;
202 }
203 if (config_debug) {
204 edata_t *a = emap_edata_lookup(tsdn, &arena_emap_global,
205 addr_a);
206 bool head_a = edata_is_head_get(a);
207 edata_t *b = emap_edata_lookup(tsdn, &arena_emap_global,
208 addr_b);
209 bool head_b = edata_is_head_get(b);
210 emap_assert_mapped(tsdn, &arena_emap_global, a);
211 emap_assert_mapped(tsdn, &arena_emap_global, b);
212 assert(extent_neighbor_head_state_mergeable(head_a, head_b,
213 /* forward */ true));
214 }
215 if (have_dss && !extent_dss_mergeable(addr_a, addr_b)) {
216 return true;
217 }
218
219 return false;
220}
221
222bool
223ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
224 void *addr_b, size_t size_b, bool committed, unsigned arena_ind) {
225 tsdn_t *tsdn = tsdn_fetch();
226
227 return ehooks_default_merge_impl(tsdn, addr_a, addr_b);
228}
229
230void
231ehooks_default_zero_impl(void *addr, size_t size) {
232 /*
233 * By default, we try to zero out memory using OS-provided demand-zeroed
234 * pages. If the user has specifically requested hugepages, though, we
235 * don't want to purge in the middle of a hugepage (which would break it
236 * up), so we act conservatively and use memset.
237 */
238 bool needs_memset = true;
239 if (opt_thp != thp_mode_always) {
240 needs_memset = pages_purge_forced(addr, size);
241 }
242 if (needs_memset) {
243 memset(addr, 0, size);
244 }
245}
246
247void
248ehooks_default_guard_impl(void *guard1, void *guard2) {
249 pages_mark_guards(guard1, guard2);
250}
251
252void
253ehooks_default_unguard_impl(void *guard1, void *guard2) {
254 pages_unmark_guards(guard1, guard2);
255}
256
257const extent_hooks_t ehooks_default_extent_hooks = {
258 ehooks_default_alloc,
259 ehooks_default_dalloc,
260 ehooks_default_destroy,
261 ehooks_default_commit,
262 ehooks_default_decommit,
263#ifdef PAGES_CAN_PURGE_LAZY
264 ehooks_default_purge_lazy,
265#else
266 NULL,
267#endif
268#ifdef PAGES_CAN_PURGE_FORCED
269 ehooks_default_purge_forced,
270#else
271 NULL,
272#endif
273 ehooks_default_split,
274 ehooks_default_merge
275};
diff --git a/examples/redis-unstable/deps/jemalloc/src/emap.c b/examples/redis-unstable/deps/jemalloc/src/emap.c
deleted file mode 100644
index 9cc95a7..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/emap.c
+++ /dev/null
@@ -1,386 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/emap.h"
5
6enum emap_lock_result_e {
7 emap_lock_result_success,
8 emap_lock_result_failure,
9 emap_lock_result_no_extent
10};
11typedef enum emap_lock_result_e emap_lock_result_t;
12
13bool
14emap_init(emap_t *emap, base_t *base, bool zeroed) {
15 return rtree_new(&emap->rtree, base, zeroed);
16}
17
18void
19emap_update_edata_state(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
20 extent_state_t state) {
21 witness_assert_positive_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
22 WITNESS_RANK_CORE);
23
24 edata_state_set(edata, state);
25
26 EMAP_DECLARE_RTREE_CTX;
27 rtree_leaf_elm_t *elm1 = rtree_leaf_elm_lookup(tsdn, &emap->rtree,
28 rtree_ctx, (uintptr_t)edata_base_get(edata), /* dependent */ true,
29 /* init_missing */ false);
30 assert(elm1 != NULL);
31 rtree_leaf_elm_t *elm2 = edata_size_get(edata) == PAGE ? NULL :
32 rtree_leaf_elm_lookup(tsdn, &emap->rtree, rtree_ctx,
33 (uintptr_t)edata_last_get(edata), /* dependent */ true,
34 /* init_missing */ false);
35
36 rtree_leaf_elm_state_update(tsdn, &emap->rtree, elm1, elm2, state);
37
38 emap_assert_mapped(tsdn, emap, edata);
39}
40
41static inline edata_t *
42emap_try_acquire_edata_neighbor_impl(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
43 extent_pai_t pai, extent_state_t expected_state, bool forward,
44 bool expanding) {
45 witness_assert_positive_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
46 WITNESS_RANK_CORE);
47 assert(!edata_guarded_get(edata));
48 assert(!expanding || forward);
49 assert(!edata_state_in_transition(expected_state));
50 assert(expected_state == extent_state_dirty ||
51 expected_state == extent_state_muzzy ||
52 expected_state == extent_state_retained);
53
54 void *neighbor_addr = forward ? edata_past_get(edata) :
55 edata_before_get(edata);
56 /*
57 * This is subtle; the rtree code asserts that its input pointer is
58 * non-NULL, and this is a useful thing to check. But it's possible
59 * that edata corresponds to an address of (void *)PAGE (in practice,
60 * this has only been observed on FreeBSD when address-space
61 * randomization is on, but it could in principle happen anywhere). In
62 * this case, edata_before_get(edata) is NULL, triggering the assert.
63 */
64 if (neighbor_addr == NULL) {
65 return NULL;
66 }
67
68 EMAP_DECLARE_RTREE_CTX;
69 rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &emap->rtree,
70 rtree_ctx, (uintptr_t)neighbor_addr, /* dependent*/ false,
71 /* init_missing */ false);
72 if (elm == NULL) {
73 return NULL;
74 }
75
76 rtree_contents_t neighbor_contents = rtree_leaf_elm_read(tsdn,
77 &emap->rtree, elm, /* dependent */ true);
78 if (!extent_can_acquire_neighbor(edata, neighbor_contents, pai,
79 expected_state, forward, expanding)) {
80 return NULL;
81 }
82
83 /* From this point, the neighbor edata can be safely acquired. */
84 edata_t *neighbor = neighbor_contents.edata;
85 assert(edata_state_get(neighbor) == expected_state);
86 emap_update_edata_state(tsdn, emap, neighbor, extent_state_merging);
87 if (expanding) {
88 extent_assert_can_expand(edata, neighbor);
89 } else {
90 extent_assert_can_coalesce(edata, neighbor);
91 }
92
93 return neighbor;
94}
95
96edata_t *
97emap_try_acquire_edata_neighbor(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
98 extent_pai_t pai, extent_state_t expected_state, bool forward) {
99 return emap_try_acquire_edata_neighbor_impl(tsdn, emap, edata, pai,
100 expected_state, forward, /* expand */ false);
101}
102
103edata_t *
104emap_try_acquire_edata_neighbor_expand(tsdn_t *tsdn, emap_t *emap,
105 edata_t *edata, extent_pai_t pai, extent_state_t expected_state) {
106 /* Try expanding forward. */
107 return emap_try_acquire_edata_neighbor_impl(tsdn, emap, edata, pai,
108 expected_state, /* forward */ true, /* expand */ true);
109}
110
111void
112emap_release_edata(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
113 extent_state_t new_state) {
114 assert(emap_edata_in_transition(tsdn, emap, edata));
115 assert(emap_edata_is_acquired(tsdn, emap, edata));
116
117 emap_update_edata_state(tsdn, emap, edata, new_state);
118}
119
120static bool
121emap_rtree_leaf_elms_lookup(tsdn_t *tsdn, emap_t *emap, rtree_ctx_t *rtree_ctx,
122 const edata_t *edata, bool dependent, bool init_missing,
123 rtree_leaf_elm_t **r_elm_a, rtree_leaf_elm_t **r_elm_b) {
124 *r_elm_a = rtree_leaf_elm_lookup(tsdn, &emap->rtree, rtree_ctx,
125 (uintptr_t)edata_base_get(edata), dependent, init_missing);
126 if (!dependent && *r_elm_a == NULL) {
127 return true;
128 }
129 assert(*r_elm_a != NULL);
130
131 *r_elm_b = rtree_leaf_elm_lookup(tsdn, &emap->rtree, rtree_ctx,
132 (uintptr_t)edata_last_get(edata), dependent, init_missing);
133 if (!dependent && *r_elm_b == NULL) {
134 return true;
135 }
136 assert(*r_elm_b != NULL);
137
138 return false;
139}
140
141static void
142emap_rtree_write_acquired(tsdn_t *tsdn, emap_t *emap, rtree_leaf_elm_t *elm_a,
143 rtree_leaf_elm_t *elm_b, edata_t *edata, szind_t szind, bool slab) {
144 rtree_contents_t contents;
145 contents.edata = edata;
146 contents.metadata.szind = szind;
147 contents.metadata.slab = slab;
148 contents.metadata.is_head = (edata == NULL) ? false :
149 edata_is_head_get(edata);
150 contents.metadata.state = (edata == NULL) ? 0 : edata_state_get(edata);
151 rtree_leaf_elm_write(tsdn, &emap->rtree, elm_a, contents);
152 if (elm_b != NULL) {
153 rtree_leaf_elm_write(tsdn, &emap->rtree, elm_b, contents);
154 }
155}
156
157bool
158emap_register_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
159 szind_t szind, bool slab) {
160 assert(edata_state_get(edata) == extent_state_active);
161 EMAP_DECLARE_RTREE_CTX;
162
163 rtree_leaf_elm_t *elm_a, *elm_b;
164 bool err = emap_rtree_leaf_elms_lookup(tsdn, emap, rtree_ctx, edata,
165 false, true, &elm_a, &elm_b);
166 if (err) {
167 return true;
168 }
169 assert(rtree_leaf_elm_read(tsdn, &emap->rtree, elm_a,
170 /* dependent */ false).edata == NULL);
171 assert(rtree_leaf_elm_read(tsdn, &emap->rtree, elm_b,
172 /* dependent */ false).edata == NULL);
173 emap_rtree_write_acquired(tsdn, emap, elm_a, elm_b, edata, szind, slab);
174 return false;
175}
176
177/* Invoked *after* emap_register_boundary. */
178void
179emap_register_interior(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
180 szind_t szind) {
181 EMAP_DECLARE_RTREE_CTX;
182
183 assert(edata_slab_get(edata));
184 assert(edata_state_get(edata) == extent_state_active);
185
186 if (config_debug) {
187 /* Making sure the boundary is registered already. */
188 rtree_leaf_elm_t *elm_a, *elm_b;
189 bool err = emap_rtree_leaf_elms_lookup(tsdn, emap, rtree_ctx,
190 edata, /* dependent */ true, /* init_missing */ false,
191 &elm_a, &elm_b);
192 assert(!err);
193 rtree_contents_t contents_a, contents_b;
194 contents_a = rtree_leaf_elm_read(tsdn, &emap->rtree, elm_a,
195 /* dependent */ true);
196 contents_b = rtree_leaf_elm_read(tsdn, &emap->rtree, elm_b,
197 /* dependent */ true);
198 assert(contents_a.edata == edata && contents_b.edata == edata);
199 assert(contents_a.metadata.slab && contents_b.metadata.slab);
200 }
201
202 rtree_contents_t contents;
203 contents.edata = edata;
204 contents.metadata.szind = szind;
205 contents.metadata.slab = true;
206 contents.metadata.state = extent_state_active;
207 contents.metadata.is_head = false; /* Not allowed to access. */
208
209 assert(edata_size_get(edata) > (2 << LG_PAGE));
210 rtree_write_range(tsdn, &emap->rtree, rtree_ctx,
211 (uintptr_t)edata_base_get(edata) + PAGE,
212 (uintptr_t)edata_last_get(edata) - PAGE, contents);
213}
214
215void
216emap_deregister_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
217 /*
218 * The edata must be either in an acquired state, or protected by state
219 * based locks.
220 */
221 if (!emap_edata_is_acquired(tsdn, emap, edata)) {
222 witness_assert_positive_depth_to_rank(
223 tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE);
224 }
225
226 EMAP_DECLARE_RTREE_CTX;
227 rtree_leaf_elm_t *elm_a, *elm_b;
228
229 emap_rtree_leaf_elms_lookup(tsdn, emap, rtree_ctx, edata,
230 true, false, &elm_a, &elm_b);
231 emap_rtree_write_acquired(tsdn, emap, elm_a, elm_b, NULL, SC_NSIZES,
232 false);
233}
234
235void
236emap_deregister_interior(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
237 EMAP_DECLARE_RTREE_CTX;
238
239 assert(edata_slab_get(edata));
240 if (edata_size_get(edata) > (2 << LG_PAGE)) {
241 rtree_clear_range(tsdn, &emap->rtree, rtree_ctx,
242 (uintptr_t)edata_base_get(edata) + PAGE,
243 (uintptr_t)edata_last_get(edata) - PAGE);
244 }
245}
246
247void
248emap_remap(tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind,
249 bool slab) {
250 EMAP_DECLARE_RTREE_CTX;
251
252 if (szind != SC_NSIZES) {
253 rtree_contents_t contents;
254 contents.edata = edata;
255 contents.metadata.szind = szind;
256 contents.metadata.slab = slab;
257 contents.metadata.is_head = edata_is_head_get(edata);
258 contents.metadata.state = edata_state_get(edata);
259
260 rtree_write(tsdn, &emap->rtree, rtree_ctx,
261 (uintptr_t)edata_addr_get(edata), contents);
262 /*
263 * Recall that this is called only for active->inactive and
264 * inactive->active transitions (since only active extents have
265 * meaningful values for szind and slab). Active, non-slab
266 * extents only need to handle lookups at their head (on
267 * deallocation), so we don't bother filling in the end
268 * boundary.
269 *
270 * For slab extents, we do the end-mapping change. This still
271 * leaves the interior unmodified; an emap_register_interior
272 * call is coming in those cases, though.
273 */
274 if (slab && edata_size_get(edata) > PAGE) {
275 uintptr_t key = (uintptr_t)edata_past_get(edata)
276 - (uintptr_t)PAGE;
277 rtree_write(tsdn, &emap->rtree, rtree_ctx, key,
278 contents);
279 }
280 }
281}
282
283bool
284emap_split_prepare(tsdn_t *tsdn, emap_t *emap, emap_prepare_t *prepare,
285 edata_t *edata, size_t size_a, edata_t *trail, size_t size_b) {
286 EMAP_DECLARE_RTREE_CTX;
287
288 /*
289 * We use incorrect constants for things like arena ind, zero, ranged,
290 * and commit state, and head status. This is a fake edata_t, used to
291 * facilitate a lookup.
292 */
293 edata_t lead = {0};
294 edata_init(&lead, 0U, edata_addr_get(edata), size_a, false, 0, 0,
295 extent_state_active, false, false, EXTENT_PAI_PAC, EXTENT_NOT_HEAD);
296
297 emap_rtree_leaf_elms_lookup(tsdn, emap, rtree_ctx, &lead, false, true,
298 &prepare->lead_elm_a, &prepare->lead_elm_b);
299 emap_rtree_leaf_elms_lookup(tsdn, emap, rtree_ctx, trail, false, true,
300 &prepare->trail_elm_a, &prepare->trail_elm_b);
301
302 if (prepare->lead_elm_a == NULL || prepare->lead_elm_b == NULL
303 || prepare->trail_elm_a == NULL || prepare->trail_elm_b == NULL) {
304 return true;
305 }
306 return false;
307}
308
309void
310emap_split_commit(tsdn_t *tsdn, emap_t *emap, emap_prepare_t *prepare,
311 edata_t *lead, size_t size_a, edata_t *trail, size_t size_b) {
312 /*
313 * We should think about not writing to the lead leaf element. We can
314 * get into situations where a racing realloc-like call can disagree
315 * with a size lookup request. I think it's fine to declare that these
316 * situations are race bugs, but there's an argument to be made that for
317 * things like xallocx, a size lookup call should return either the old
318 * size or the new size, but not anything else.
319 */
320 emap_rtree_write_acquired(tsdn, emap, prepare->lead_elm_a,
321 prepare->lead_elm_b, lead, SC_NSIZES, /* slab */ false);
322 emap_rtree_write_acquired(tsdn, emap, prepare->trail_elm_a,
323 prepare->trail_elm_b, trail, SC_NSIZES, /* slab */ false);
324}
325
326void
327emap_merge_prepare(tsdn_t *tsdn, emap_t *emap, emap_prepare_t *prepare,
328 edata_t *lead, edata_t *trail) {
329 EMAP_DECLARE_RTREE_CTX;
330 emap_rtree_leaf_elms_lookup(tsdn, emap, rtree_ctx, lead, true, false,
331 &prepare->lead_elm_a, &prepare->lead_elm_b);
332 emap_rtree_leaf_elms_lookup(tsdn, emap, rtree_ctx, trail, true, false,
333 &prepare->trail_elm_a, &prepare->trail_elm_b);
334}
335
336void
337emap_merge_commit(tsdn_t *tsdn, emap_t *emap, emap_prepare_t *prepare,
338 edata_t *lead, edata_t *trail) {
339 rtree_contents_t clear_contents;
340 clear_contents.edata = NULL;
341 clear_contents.metadata.szind = SC_NSIZES;
342 clear_contents.metadata.slab = false;
343 clear_contents.metadata.is_head = false;
344 clear_contents.metadata.state = (extent_state_t)0;
345
346 if (prepare->lead_elm_b != NULL) {
347 rtree_leaf_elm_write(tsdn, &emap->rtree,
348 prepare->lead_elm_b, clear_contents);
349 }
350
351 rtree_leaf_elm_t *merged_b;
352 if (prepare->trail_elm_b != NULL) {
353 rtree_leaf_elm_write(tsdn, &emap->rtree,
354 prepare->trail_elm_a, clear_contents);
355 merged_b = prepare->trail_elm_b;
356 } else {
357 merged_b = prepare->trail_elm_a;
358 }
359
360 emap_rtree_write_acquired(tsdn, emap, prepare->lead_elm_a, merged_b,
361 lead, SC_NSIZES, false);
362}
363
364void
365emap_do_assert_mapped(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
366 EMAP_DECLARE_RTREE_CTX;
367
368 rtree_contents_t contents = rtree_read(tsdn, &emap->rtree, rtree_ctx,
369 (uintptr_t)edata_base_get(edata));
370 assert(contents.edata == edata);
371 assert(contents.metadata.is_head == edata_is_head_get(edata));
372 assert(contents.metadata.state == edata_state_get(edata));
373}
374
375void
376emap_do_assert_not_mapped(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
377 emap_full_alloc_ctx_t context1 = {0};
378 emap_full_alloc_ctx_try_lookup(tsdn, emap, edata_base_get(edata),
379 &context1);
380 assert(context1.edata == NULL);
381
382 emap_full_alloc_ctx_t context2 = {0};
383 emap_full_alloc_ctx_try_lookup(tsdn, emap, edata_last_get(edata),
384 &context2);
385 assert(context2.edata == NULL);
386}
diff --git a/examples/redis-unstable/deps/jemalloc/src/eset.c b/examples/redis-unstable/deps/jemalloc/src/eset.c
deleted file mode 100644
index 6f8f335..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/eset.c
+++ /dev/null
@@ -1,282 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/eset.h"
5
6#define ESET_NPSIZES (SC_NPSIZES + 1)
7
8static void
9eset_bin_init(eset_bin_t *bin) {
10 edata_heap_new(&bin->heap);
11 /*
12 * heap_min doesn't need initialization; it gets filled in when the bin
13 * goes from non-empty to empty.
14 */
15}
16
17static void
18eset_bin_stats_init(eset_bin_stats_t *bin_stats) {
19 atomic_store_zu(&bin_stats->nextents, 0, ATOMIC_RELAXED);
20 atomic_store_zu(&bin_stats->nbytes, 0, ATOMIC_RELAXED);
21}
22
23void
24eset_init(eset_t *eset, extent_state_t state) {
25 for (unsigned i = 0; i < ESET_NPSIZES; i++) {
26 eset_bin_init(&eset->bins[i]);
27 eset_bin_stats_init(&eset->bin_stats[i]);
28 }
29 fb_init(eset->bitmap, ESET_NPSIZES);
30 edata_list_inactive_init(&eset->lru);
31 eset->state = state;
32}
33
34size_t
35eset_npages_get(eset_t *eset) {
36 return atomic_load_zu(&eset->npages, ATOMIC_RELAXED);
37}
38
39size_t
40eset_nextents_get(eset_t *eset, pszind_t pind) {
41 return atomic_load_zu(&eset->bin_stats[pind].nextents, ATOMIC_RELAXED);
42}
43
44size_t
45eset_nbytes_get(eset_t *eset, pszind_t pind) {
46 return atomic_load_zu(&eset->bin_stats[pind].nbytes, ATOMIC_RELAXED);
47}
48
49static void
50eset_stats_add(eset_t *eset, pszind_t pind, size_t sz) {
51 size_t cur = atomic_load_zu(&eset->bin_stats[pind].nextents,
52 ATOMIC_RELAXED);
53 atomic_store_zu(&eset->bin_stats[pind].nextents, cur + 1,
54 ATOMIC_RELAXED);
55 cur = atomic_load_zu(&eset->bin_stats[pind].nbytes, ATOMIC_RELAXED);
56 atomic_store_zu(&eset->bin_stats[pind].nbytes, cur + sz,
57 ATOMIC_RELAXED);
58}
59
60static void
61eset_stats_sub(eset_t *eset, pszind_t pind, size_t sz) {
62 size_t cur = atomic_load_zu(&eset->bin_stats[pind].nextents,
63 ATOMIC_RELAXED);
64 atomic_store_zu(&eset->bin_stats[pind].nextents, cur - 1,
65 ATOMIC_RELAXED);
66 cur = atomic_load_zu(&eset->bin_stats[pind].nbytes, ATOMIC_RELAXED);
67 atomic_store_zu(&eset->bin_stats[pind].nbytes, cur - sz,
68 ATOMIC_RELAXED);
69}
70
71void
72eset_insert(eset_t *eset, edata_t *edata) {
73 assert(edata_state_get(edata) == eset->state);
74
75 size_t size = edata_size_get(edata);
76 size_t psz = sz_psz_quantize_floor(size);
77 pszind_t pind = sz_psz2ind(psz);
78
79 edata_cmp_summary_t edata_cmp_summary = edata_cmp_summary_get(edata);
80 if (edata_heap_empty(&eset->bins[pind].heap)) {
81 fb_set(eset->bitmap, ESET_NPSIZES, (size_t)pind);
82 /* Only element is automatically the min element. */
83 eset->bins[pind].heap_min = edata_cmp_summary;
84 } else {
85 /*
86 * There's already a min element; update the summary if we're
87 * about to insert a lower one.
88 */
89 if (edata_cmp_summary_comp(edata_cmp_summary,
90 eset->bins[pind].heap_min) < 0) {
91 eset->bins[pind].heap_min = edata_cmp_summary;
92 }
93 }
94 edata_heap_insert(&eset->bins[pind].heap, edata);
95
96 if (config_stats) {
97 eset_stats_add(eset, pind, size);
98 }
99
100 edata_list_inactive_append(&eset->lru, edata);
101 size_t npages = size >> LG_PAGE;
102 /*
103 * All modifications to npages hold the mutex (as asserted above), so we
104 * don't need an atomic fetch-add; we can get by with a load followed by
105 * a store.
106 */
107 size_t cur_eset_npages =
108 atomic_load_zu(&eset->npages, ATOMIC_RELAXED);
109 atomic_store_zu(&eset->npages, cur_eset_npages + npages,
110 ATOMIC_RELAXED);
111}
112
113void
114eset_remove(eset_t *eset, edata_t *edata) {
115 assert(edata_state_get(edata) == eset->state ||
116 edata_state_in_transition(edata_state_get(edata)));
117
118 size_t size = edata_size_get(edata);
119 size_t psz = sz_psz_quantize_floor(size);
120 pszind_t pind = sz_psz2ind(psz);
121 if (config_stats) {
122 eset_stats_sub(eset, pind, size);
123 }
124
125 edata_cmp_summary_t edata_cmp_summary = edata_cmp_summary_get(edata);
126 edata_heap_remove(&eset->bins[pind].heap, edata);
127 if (edata_heap_empty(&eset->bins[pind].heap)) {
128 fb_unset(eset->bitmap, ESET_NPSIZES, (size_t)pind);
129 } else {
130 /*
131 * This is a little weird; we compare if the summaries are
132 * equal, rather than if the edata we removed was the heap
133 * minimum. The reason why is that getting the heap minimum
134 * can cause a pairing heap merge operation. We can avoid this
135 * if we only update the min if it's changed, in which case the
136 * summaries of the removed element and the min element should
137 * compare equal.
138 */
139 if (edata_cmp_summary_comp(edata_cmp_summary,
140 eset->bins[pind].heap_min) == 0) {
141 eset->bins[pind].heap_min = edata_cmp_summary_get(
142 edata_heap_first(&eset->bins[pind].heap));
143 }
144 }
145 edata_list_inactive_remove(&eset->lru, edata);
146 size_t npages = size >> LG_PAGE;
147 /*
148 * As in eset_insert, we hold eset->mtx and so don't need atomic
149 * operations for updating eset->npages.
150 */
151 size_t cur_extents_npages =
152 atomic_load_zu(&eset->npages, ATOMIC_RELAXED);
153 assert(cur_extents_npages >= npages);
154 atomic_store_zu(&eset->npages,
155 cur_extents_npages - (size >> LG_PAGE), ATOMIC_RELAXED);
156}
157
158/*
159 * Find an extent with size [min_size, max_size) to satisfy the alignment
160 * requirement. For each size, try only the first extent in the heap.
161 */
162static edata_t *
163eset_fit_alignment(eset_t *eset, size_t min_size, size_t max_size,
164 size_t alignment) {
165 pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(min_size));
166 pszind_t pind_max = sz_psz2ind(sz_psz_quantize_ceil(max_size));
167
168 for (pszind_t i =
169 (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)pind);
170 i < pind_max;
171 i = (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)i + 1)) {
172 assert(i < SC_NPSIZES);
173 assert(!edata_heap_empty(&eset->bins[i].heap));
174 edata_t *edata = edata_heap_first(&eset->bins[i].heap);
175 uintptr_t base = (uintptr_t)edata_base_get(edata);
176 size_t candidate_size = edata_size_get(edata);
177 assert(candidate_size >= min_size);
178
179 uintptr_t next_align = ALIGNMENT_CEILING((uintptr_t)base,
180 PAGE_CEILING(alignment));
181 if (base > next_align || base + candidate_size <= next_align) {
182 /* Overflow or not crossing the next alignment. */
183 continue;
184 }
185
186 size_t leadsize = next_align - base;
187 if (candidate_size - leadsize >= min_size) {
188 return edata;
189 }
190 }
191
192 return NULL;
193}
194
195/*
196 * Do first-fit extent selection, i.e. select the oldest/lowest extent that is
197 * large enough.
198 *
199 * lg_max_fit is the (log of the) maximum ratio between the requested size and
200 * the returned size that we'll allow. This can reduce fragmentation by
201 * avoiding reusing and splitting large extents for smaller sizes. In practice,
202 * it's set to opt_lg_extent_max_active_fit for the dirty eset and SC_PTR_BITS
203 * for others.
204 */
205static edata_t *
206eset_first_fit(eset_t *eset, size_t size, bool exact_only,
207 unsigned lg_max_fit) {
208 edata_t *ret = NULL;
209 edata_cmp_summary_t ret_summ JEMALLOC_CC_SILENCE_INIT({0});
210
211 pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(size));
212
213 if (exact_only) {
214 return edata_heap_empty(&eset->bins[pind].heap) ? NULL :
215 edata_heap_first(&eset->bins[pind].heap);
216 }
217
218 for (pszind_t i =
219 (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)pind);
220 i < ESET_NPSIZES;
221 i = (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)i + 1)) {
222 assert(!edata_heap_empty(&eset->bins[i].heap));
223 if (lg_max_fit == SC_PTR_BITS) {
224 /*
225 * We'll shift by this below, and shifting out all the
226 * bits is undefined. Decreasing is safe, since the
227 * page size is larger than 1 byte.
228 */
229 lg_max_fit = SC_PTR_BITS - 1;
230 }
231 if ((sz_pind2sz(i) >> lg_max_fit) > size) {
232 break;
233 }
234 if (ret == NULL || edata_cmp_summary_comp(
235 eset->bins[i].heap_min, ret_summ) < 0) {
236 /*
237 * We grab the edata as early as possible, even though
238 * we might change it later. Practically, a large
239 * portion of eset_fit calls succeed at the first valid
240 * index, so this doesn't cost much, and we get the
241 * effect of prefetching the edata as early as possible.
242 */
243 edata_t *edata = edata_heap_first(&eset->bins[i].heap);
244 assert(edata_size_get(edata) >= size);
245 assert(ret == NULL || edata_snad_comp(edata, ret) < 0);
246 assert(ret == NULL || edata_cmp_summary_comp(
247 eset->bins[i].heap_min,
248 edata_cmp_summary_get(edata)) == 0);
249 ret = edata;
250 ret_summ = eset->bins[i].heap_min;
251 }
252 if (i == SC_NPSIZES) {
253 break;
254 }
255 assert(i < SC_NPSIZES);
256 }
257
258 return ret;
259}
260
261edata_t *
262eset_fit(eset_t *eset, size_t esize, size_t alignment, bool exact_only,
263 unsigned lg_max_fit) {
264 size_t max_size = esize + PAGE_CEILING(alignment) - PAGE;
265 /* Beware size_t wrap-around. */
266 if (max_size < esize) {
267 return NULL;
268 }
269
270 edata_t *edata = eset_first_fit(eset, max_size, exact_only, lg_max_fit);
271
272 if (alignment > PAGE && edata == NULL) {
273 /*
274 * max_size guarantees the alignment requirement but is rather
275 * pessimistic. Next we try to satisfy the aligned allocation
276 * with sizes in [esize, max_size).
277 */
278 edata = eset_fit_alignment(eset, esize, max_size, alignment);
279 }
280
281 return edata;
282}
diff --git a/examples/redis-unstable/deps/jemalloc/src/exp_grow.c b/examples/redis-unstable/deps/jemalloc/src/exp_grow.c
deleted file mode 100644
index 386471f..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/exp_grow.c
+++ /dev/null
@@ -1,8 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4void
5exp_grow_init(exp_grow_t *exp_grow) {
6 exp_grow->next = sz_psz2ind(HUGEPAGE);
7 exp_grow->limit = sz_psz2ind(SC_LARGE_MAXCLASS);
8}
diff --git a/examples/redis-unstable/deps/jemalloc/src/extent.c b/examples/redis-unstable/deps/jemalloc/src/extent.c
deleted file mode 100644
index cf3d1f3..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/extent.c
+++ /dev/null
@@ -1,1326 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/emap.h"
6#include "jemalloc/internal/extent_dss.h"
7#include "jemalloc/internal/extent_mmap.h"
8#include "jemalloc/internal/ph.h"
9#include "jemalloc/internal/mutex.h"
10
11/******************************************************************************/
12/* Data. */
13
14size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT;
15
16static bool extent_commit_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
17 size_t offset, size_t length, bool growing_retained);
18static bool extent_purge_lazy_impl(tsdn_t *tsdn, ehooks_t *ehooks,
19 edata_t *edata, size_t offset, size_t length, bool growing_retained);
20static bool extent_purge_forced_impl(tsdn_t *tsdn, ehooks_t *ehooks,
21 edata_t *edata, size_t offset, size_t length, bool growing_retained);
22static edata_t *extent_split_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
23 edata_t *edata, size_t size_a, size_t size_b, bool holding_core_locks);
24static bool extent_merge_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
25 edata_t *a, edata_t *b, bool holding_core_locks);
26
27/* Used exclusively for gdump triggering. */
28static atomic_zu_t curpages;
29static atomic_zu_t highpages;
30
31/******************************************************************************/
32/*
33 * Function prototypes for static functions that are referenced prior to
34 * definition.
35 */
36
37static void extent_deregister(tsdn_t *tsdn, pac_t *pac, edata_t *edata);
38static edata_t *extent_recycle(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
39 ecache_t *ecache, edata_t *expand_edata, size_t usize, size_t alignment,
40 bool zero, bool *commit, bool growing_retained, bool guarded);
41static edata_t *extent_try_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
42 ecache_t *ecache, edata_t *edata, bool *coalesced);
43static edata_t *extent_alloc_retained(tsdn_t *tsdn, pac_t *pac,
44 ehooks_t *ehooks, edata_t *expand_edata, size_t size, size_t alignment,
45 bool zero, bool *commit, bool guarded);
46
47/******************************************************************************/
48
49size_t
50extent_sn_next(pac_t *pac) {
51 return atomic_fetch_add_zu(&pac->extent_sn_next, 1, ATOMIC_RELAXED);
52}
53
54static inline bool
55extent_may_force_decay(pac_t *pac) {
56 return !(pac_decay_ms_get(pac, extent_state_dirty) == -1
57 || pac_decay_ms_get(pac, extent_state_muzzy) == -1);
58}
59
60static bool
61extent_try_delayed_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
62 ecache_t *ecache, edata_t *edata) {
63 emap_update_edata_state(tsdn, pac->emap, edata, extent_state_active);
64
65 bool coalesced;
66 edata = extent_try_coalesce(tsdn, pac, ehooks, ecache,
67 edata, &coalesced);
68 emap_update_edata_state(tsdn, pac->emap, edata, ecache->state);
69
70 if (!coalesced) {
71 return true;
72 }
73 eset_insert(&ecache->eset, edata);
74 return false;
75}
76
77edata_t *
78ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
79 edata_t *expand_edata, size_t size, size_t alignment, bool zero,
80 bool guarded) {
81 assert(size != 0);
82 assert(alignment != 0);
83 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
84 WITNESS_RANK_CORE, 0);
85
86 bool commit = true;
87 edata_t *edata = extent_recycle(tsdn, pac, ehooks, ecache, expand_edata,
88 size, alignment, zero, &commit, false, guarded);
89 assert(edata == NULL || edata_pai_get(edata) == EXTENT_PAI_PAC);
90 assert(edata == NULL || edata_guarded_get(edata) == guarded);
91 return edata;
92}
93
94edata_t *
95ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
96 edata_t *expand_edata, size_t size, size_t alignment, bool zero,
97 bool guarded) {
98 assert(size != 0);
99 assert(alignment != 0);
100 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
101 WITNESS_RANK_CORE, 0);
102
103 bool commit = true;
104 edata_t *edata = extent_alloc_retained(tsdn, pac, ehooks, expand_edata,
105 size, alignment, zero, &commit, guarded);
106 if (edata == NULL) {
107 if (opt_retain && expand_edata != NULL) {
108 /*
109 * When retain is enabled and trying to expand, we do
110 * not attempt extent_alloc_wrapper which does mmap that
111 * is very unlikely to succeed (unless it happens to be
112 * at the end).
113 */
114 return NULL;
115 }
116 if (guarded) {
117 /*
118 * Means no cached guarded extents available (and no
119 * grow_retained was attempted). The pac_alloc flow
120 * will alloc regular extents to make new guarded ones.
121 */
122 return NULL;
123 }
124 void *new_addr = (expand_edata == NULL) ? NULL :
125 edata_past_get(expand_edata);
126 edata = extent_alloc_wrapper(tsdn, pac, ehooks, new_addr,
127 size, alignment, zero, &commit,
128 /* growing_retained */ false);
129 }
130
131 assert(edata == NULL || edata_pai_get(edata) == EXTENT_PAI_PAC);
132 return edata;
133}
134
135void
136ecache_dalloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
137 edata_t *edata) {
138 assert(edata_base_get(edata) != NULL);
139 assert(edata_size_get(edata) != 0);
140 assert(edata_pai_get(edata) == EXTENT_PAI_PAC);
141 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
142 WITNESS_RANK_CORE, 0);
143
144 edata_addr_set(edata, edata_base_get(edata));
145 edata_zeroed_set(edata, false);
146
147 extent_record(tsdn, pac, ehooks, ecache, edata);
148}
149
150edata_t *
151ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
152 ecache_t *ecache, size_t npages_min) {
153 malloc_mutex_lock(tsdn, &ecache->mtx);
154
155 /*
156 * Get the LRU coalesced extent, if any. If coalescing was delayed,
157 * the loop will iterate until the LRU extent is fully coalesced.
158 */
159 edata_t *edata;
160 while (true) {
161 /* Get the LRU extent, if any. */
162 eset_t *eset = &ecache->eset;
163 edata = edata_list_inactive_first(&eset->lru);
164 if (edata == NULL) {
165 /*
166 * Next check if there are guarded extents. They are
167 * more expensive to purge (since they are not
168 * mergeable), thus in favor of caching them longer.
169 */
170 eset = &ecache->guarded_eset;
171 edata = edata_list_inactive_first(&eset->lru);
172 if (edata == NULL) {
173 goto label_return;
174 }
175 }
176 /* Check the eviction limit. */
177 size_t extents_npages = ecache_npages_get(ecache);
178 if (extents_npages <= npages_min) {
179 edata = NULL;
180 goto label_return;
181 }
182 eset_remove(eset, edata);
183 if (!ecache->delay_coalesce || edata_guarded_get(edata)) {
184 break;
185 }
186 /* Try to coalesce. */
187 if (extent_try_delayed_coalesce(tsdn, pac, ehooks, ecache,
188 edata)) {
189 break;
190 }
191 /*
192 * The LRU extent was just coalesced and the result placed in
193 * the LRU at its neighbor's position. Start over.
194 */
195 }
196
197 /*
198 * Either mark the extent active or deregister it to protect against
199 * concurrent operations.
200 */
201 switch (ecache->state) {
202 case extent_state_active:
203 not_reached();
204 case extent_state_dirty:
205 case extent_state_muzzy:
206 emap_update_edata_state(tsdn, pac->emap, edata,
207 extent_state_active);
208 break;
209 case extent_state_retained:
210 extent_deregister(tsdn, pac, edata);
211 break;
212 default:
213 not_reached();
214 }
215
216label_return:
217 malloc_mutex_unlock(tsdn, &ecache->mtx);
218 return edata;
219}
220
221/*
222 * This can only happen when we fail to allocate a new extent struct (which
223 * indicates OOM), e.g. when trying to split an existing extent.
224 */
225static void
226extents_abandon_vm(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
227 edata_t *edata, bool growing_retained) {
228 size_t sz = edata_size_get(edata);
229 if (config_stats) {
230 atomic_fetch_add_zu(&pac->stats->abandoned_vm, sz,
231 ATOMIC_RELAXED);
232 }
233 /*
234 * Leak extent after making sure its pages have already been purged, so
235 * that this is only a virtual memory leak.
236 */
237 if (ecache->state == extent_state_dirty) {
238 if (extent_purge_lazy_impl(tsdn, ehooks, edata, 0, sz,
239 growing_retained)) {
240 extent_purge_forced_impl(tsdn, ehooks, edata, 0,
241 edata_size_get(edata), growing_retained);
242 }
243 }
244 edata_cache_put(tsdn, pac->edata_cache, edata);
245}
246
247static void
248extent_deactivate_locked_impl(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache,
249 edata_t *edata) {
250 malloc_mutex_assert_owner(tsdn, &ecache->mtx);
251 assert(edata_arena_ind_get(edata) == ecache_ind_get(ecache));
252
253 emap_update_edata_state(tsdn, pac->emap, edata, ecache->state);
254 eset_t *eset = edata_guarded_get(edata) ? &ecache->guarded_eset :
255 &ecache->eset;
256 eset_insert(eset, edata);
257}
258
259static void
260extent_deactivate_locked(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache,
261 edata_t *edata) {
262 assert(edata_state_get(edata) == extent_state_active);
263 extent_deactivate_locked_impl(tsdn, pac, ecache, edata);
264}
265
266static void
267extent_deactivate_check_state_locked(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache,
268 edata_t *edata, extent_state_t expected_state) {
269 assert(edata_state_get(edata) == expected_state);
270 extent_deactivate_locked_impl(tsdn, pac, ecache, edata);
271}
272
273static void
274extent_activate_locked(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, eset_t *eset,
275 edata_t *edata) {
276 assert(edata_arena_ind_get(edata) == ecache_ind_get(ecache));
277 assert(edata_state_get(edata) == ecache->state ||
278 edata_state_get(edata) == extent_state_merging);
279
280 eset_remove(eset, edata);
281 emap_update_edata_state(tsdn, pac->emap, edata, extent_state_active);
282}
283
284void
285extent_gdump_add(tsdn_t *tsdn, const edata_t *edata) {
286 cassert(config_prof);
287 /* prof_gdump() requirement. */
288 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
289 WITNESS_RANK_CORE, 0);
290
291 if (opt_prof && edata_state_get(edata) == extent_state_active) {
292 size_t nadd = edata_size_get(edata) >> LG_PAGE;
293 size_t cur = atomic_fetch_add_zu(&curpages, nadd,
294 ATOMIC_RELAXED) + nadd;
295 size_t high = atomic_load_zu(&highpages, ATOMIC_RELAXED);
296 while (cur > high && !atomic_compare_exchange_weak_zu(
297 &highpages, &high, cur, ATOMIC_RELAXED, ATOMIC_RELAXED)) {
298 /*
299 * Don't refresh cur, because it may have decreased
300 * since this thread lost the highpages update race.
301 * Note that high is updated in case of CAS failure.
302 */
303 }
304 if (cur > high && prof_gdump_get_unlocked()) {
305 prof_gdump(tsdn);
306 }
307 }
308}
309
310static void
311extent_gdump_sub(tsdn_t *tsdn, const edata_t *edata) {
312 cassert(config_prof);
313
314 if (opt_prof && edata_state_get(edata) == extent_state_active) {
315 size_t nsub = edata_size_get(edata) >> LG_PAGE;
316 assert(atomic_load_zu(&curpages, ATOMIC_RELAXED) >= nsub);
317 atomic_fetch_sub_zu(&curpages, nsub, ATOMIC_RELAXED);
318 }
319}
320
321static bool
322extent_register_impl(tsdn_t *tsdn, pac_t *pac, edata_t *edata, bool gdump_add) {
323 assert(edata_state_get(edata) == extent_state_active);
324 /*
325 * No locking needed, as the edata must be in active state, which
326 * prevents other threads from accessing the edata.
327 */
328 if (emap_register_boundary(tsdn, pac->emap, edata, SC_NSIZES,
329 /* slab */ false)) {
330 return true;
331 }
332
333 if (config_prof && gdump_add) {
334 extent_gdump_add(tsdn, edata);
335 }
336
337 return false;
338}
339
340static bool
341extent_register(tsdn_t *tsdn, pac_t *pac, edata_t *edata) {
342 return extent_register_impl(tsdn, pac, edata, true);
343}
344
345static bool
346extent_register_no_gdump_add(tsdn_t *tsdn, pac_t *pac, edata_t *edata) {
347 return extent_register_impl(tsdn, pac, edata, false);
348}
349
350static void
351extent_reregister(tsdn_t *tsdn, pac_t *pac, edata_t *edata) {
352 bool err = extent_register(tsdn, pac, edata);
353 assert(!err);
354}
355
356/*
357 * Removes all pointers to the given extent from the global rtree.
358 */
359static void
360extent_deregister_impl(tsdn_t *tsdn, pac_t *pac, edata_t *edata,
361 bool gdump) {
362 emap_deregister_boundary(tsdn, pac->emap, edata);
363
364 if (config_prof && gdump) {
365 extent_gdump_sub(tsdn, edata);
366 }
367}
368
369static void
370extent_deregister(tsdn_t *tsdn, pac_t *pac, edata_t *edata) {
371 extent_deregister_impl(tsdn, pac, edata, true);
372}
373
374static void
375extent_deregister_no_gdump_sub(tsdn_t *tsdn, pac_t *pac,
376 edata_t *edata) {
377 extent_deregister_impl(tsdn, pac, edata, false);
378}
379
380/*
381 * Tries to find and remove an extent from ecache that can be used for the
382 * given allocation request.
383 */
384static edata_t *
385extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
386 ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment,
387 bool guarded) {
388 malloc_mutex_assert_owner(tsdn, &ecache->mtx);
389 assert(alignment > 0);
390 if (config_debug && expand_edata != NULL) {
391 /*
392 * Non-NULL expand_edata indicates in-place expanding realloc.
393 * new_addr must either refer to a non-existing extent, or to
394 * the base of an extant extent, since only active slabs support
395 * interior lookups (which of course cannot be recycled).
396 */
397 void *new_addr = edata_past_get(expand_edata);
398 assert(PAGE_ADDR2BASE(new_addr) == new_addr);
399 assert(alignment <= PAGE);
400 }
401
402 edata_t *edata;
403 eset_t *eset = guarded ? &ecache->guarded_eset : &ecache->eset;
404 if (expand_edata != NULL) {
405 edata = emap_try_acquire_edata_neighbor_expand(tsdn, pac->emap,
406 expand_edata, EXTENT_PAI_PAC, ecache->state);
407 if (edata != NULL) {
408 extent_assert_can_expand(expand_edata, edata);
409 if (edata_size_get(edata) < size) {
410 emap_release_edata(tsdn, pac->emap, edata,
411 ecache->state);
412 edata = NULL;
413 }
414 }
415 } else {
416 /*
417 * A large extent might be broken up from its original size to
418 * some small size to satisfy a small request. When that small
419 * request is freed, though, it won't merge back with the larger
420 * extent if delayed coalescing is on. The large extent can
421 * then no longer satify a request for its original size. To
422 * limit this effect, when delayed coalescing is enabled, we
423 * put a cap on how big an extent we can split for a request.
424 */
425 unsigned lg_max_fit = ecache->delay_coalesce
426 ? (unsigned)opt_lg_extent_max_active_fit : SC_PTR_BITS;
427
428 /*
429 * If split and merge are not allowed (Windows w/o retain), try
430 * exact fit only.
431 *
432 * For simplicity purposes, splitting guarded extents is not
433 * supported. Hence, we do only exact fit for guarded
434 * allocations.
435 */
436 bool exact_only = (!maps_coalesce && !opt_retain) || guarded;
437 edata = eset_fit(eset, size, alignment, exact_only,
438 lg_max_fit);
439 }
440 if (edata == NULL) {
441 return NULL;
442 }
443 assert(!guarded || edata_guarded_get(edata));
444 extent_activate_locked(tsdn, pac, ecache, eset, edata);
445
446 return edata;
447}
448
449/*
450 * Given an allocation request and an extent guaranteed to be able to satisfy
451 * it, this splits off lead and trail extents, leaving edata pointing to an
452 * extent satisfying the allocation.
453 * This function doesn't put lead or trail into any ecache; it's the caller's
454 * job to ensure that they can be reused.
455 */
456typedef enum {
457 /*
458 * Split successfully. lead, edata, and trail, are modified to extents
459 * describing the ranges before, in, and after the given allocation.
460 */
461 extent_split_interior_ok,
462 /*
463 * The extent can't satisfy the given allocation request. None of the
464 * input edata_t *s are touched.
465 */
466 extent_split_interior_cant_alloc,
467 /*
468 * In a potentially invalid state. Must leak (if *to_leak is non-NULL),
469 * and salvage what's still salvageable (if *to_salvage is non-NULL).
470 * None of lead, edata, or trail are valid.
471 */
472 extent_split_interior_error
473} extent_split_interior_result_t;
474
475static extent_split_interior_result_t
476extent_split_interior(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
477 /* The result of splitting, in case of success. */
478 edata_t **edata, edata_t **lead, edata_t **trail,
479 /* The mess to clean up, in case of error. */
480 edata_t **to_leak, edata_t **to_salvage,
481 edata_t *expand_edata, size_t size, size_t alignment) {
482 size_t leadsize = ALIGNMENT_CEILING((uintptr_t)edata_base_get(*edata),
483 PAGE_CEILING(alignment)) - (uintptr_t)edata_base_get(*edata);
484 assert(expand_edata == NULL || leadsize == 0);
485 if (edata_size_get(*edata) < leadsize + size) {
486 return extent_split_interior_cant_alloc;
487 }
488 size_t trailsize = edata_size_get(*edata) - leadsize - size;
489
490 *lead = NULL;
491 *trail = NULL;
492 *to_leak = NULL;
493 *to_salvage = NULL;
494
495 /* Split the lead. */
496 if (leadsize != 0) {
497 assert(!edata_guarded_get(*edata));
498 *lead = *edata;
499 *edata = extent_split_impl(tsdn, pac, ehooks, *lead, leadsize,
500 size + trailsize, /* holding_core_locks*/ true);
501 if (*edata == NULL) {
502 *to_leak = *lead;
503 *lead = NULL;
504 return extent_split_interior_error;
505 }
506 }
507
508 /* Split the trail. */
509 if (trailsize != 0) {
510 assert(!edata_guarded_get(*edata));
511 *trail = extent_split_impl(tsdn, pac, ehooks, *edata, size,
512 trailsize, /* holding_core_locks */ true);
513 if (*trail == NULL) {
514 *to_leak = *edata;
515 *to_salvage = *lead;
516 *lead = NULL;
517 *edata = NULL;
518 return extent_split_interior_error;
519 }
520 }
521
522 return extent_split_interior_ok;
523}
524
525/*
526 * This fulfills the indicated allocation request out of the given extent (which
527 * the caller should have ensured was big enough). If there's any unused space
528 * before or after the resulting allocation, that space is given its own extent
529 * and put back into ecache.
530 */
531static edata_t *
532extent_recycle_split(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
533 ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment,
534 edata_t *edata, bool growing_retained) {
535 assert(!edata_guarded_get(edata) || size == edata_size_get(edata));
536 malloc_mutex_assert_owner(tsdn, &ecache->mtx);
537
538 edata_t *lead;
539 edata_t *trail;
540 edata_t *to_leak JEMALLOC_CC_SILENCE_INIT(NULL);
541 edata_t *to_salvage JEMALLOC_CC_SILENCE_INIT(NULL);
542
543 extent_split_interior_result_t result = extent_split_interior(
544 tsdn, pac, ehooks, &edata, &lead, &trail, &to_leak, &to_salvage,
545 expand_edata, size, alignment);
546
547 if (!maps_coalesce && result != extent_split_interior_ok
548 && !opt_retain) {
549 /*
550 * Split isn't supported (implies Windows w/o retain). Avoid
551 * leaking the extent.
552 */
553 assert(to_leak != NULL && lead == NULL && trail == NULL);
554 extent_deactivate_locked(tsdn, pac, ecache, to_leak);
555 return NULL;
556 }
557
558 if (result == extent_split_interior_ok) {
559 if (lead != NULL) {
560 extent_deactivate_locked(tsdn, pac, ecache, lead);
561 }
562 if (trail != NULL) {
563 extent_deactivate_locked(tsdn, pac, ecache, trail);
564 }
565 return edata;
566 } else {
567 /*
568 * We should have picked an extent that was large enough to
569 * fulfill our allocation request.
570 */
571 assert(result == extent_split_interior_error);
572 if (to_salvage != NULL) {
573 extent_deregister(tsdn, pac, to_salvage);
574 }
575 if (to_leak != NULL) {
576 extent_deregister_no_gdump_sub(tsdn, pac, to_leak);
577 /*
578 * May go down the purge path (which assume no ecache
579 * locks). Only happens with OOM caused split failures.
580 */
581 malloc_mutex_unlock(tsdn, &ecache->mtx);
582 extents_abandon_vm(tsdn, pac, ehooks, ecache, to_leak,
583 growing_retained);
584 malloc_mutex_lock(tsdn, &ecache->mtx);
585 }
586 return NULL;
587 }
588 unreachable();
589}
590
591/*
592 * Tries to satisfy the given allocation request by reusing one of the extents
593 * in the given ecache_t.
594 */
595static edata_t *
596extent_recycle(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
597 edata_t *expand_edata, size_t size, size_t alignment, bool zero,
598 bool *commit, bool growing_retained, bool guarded) {
599 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
600 WITNESS_RANK_CORE, growing_retained ? 1 : 0);
601 assert(!guarded || expand_edata == NULL);
602 assert(!guarded || alignment <= PAGE);
603
604 malloc_mutex_lock(tsdn, &ecache->mtx);
605
606 edata_t *edata = extent_recycle_extract(tsdn, pac, ehooks, ecache,
607 expand_edata, size, alignment, guarded);
608 if (edata == NULL) {
609 malloc_mutex_unlock(tsdn, &ecache->mtx);
610 return NULL;
611 }
612
613 edata = extent_recycle_split(tsdn, pac, ehooks, ecache, expand_edata,
614 size, alignment, edata, growing_retained);
615 malloc_mutex_unlock(tsdn, &ecache->mtx);
616 if (edata == NULL) {
617 return NULL;
618 }
619
620 assert(edata_state_get(edata) == extent_state_active);
621 if (extent_commit_zero(tsdn, ehooks, edata, *commit, zero,
622 growing_retained)) {
623 extent_record(tsdn, pac, ehooks, ecache, edata);
624 return NULL;
625 }
626 if (edata_committed_get(edata)) {
627 /*
628 * This reverses the purpose of this variable - previously it
629 * was treated as an input parameter, now it turns into an
630 * output parameter, reporting if the edata has actually been
631 * committed.
632 */
633 *commit = true;
634 }
635 return edata;
636}
637
638/*
639 * If virtual memory is retained, create increasingly larger extents from which
640 * to split requested extents in order to limit the total number of disjoint
641 * virtual memory ranges retained by each shard.
642 */
643static edata_t *
644extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
645 size_t size, size_t alignment, bool zero, bool *commit) {
646 malloc_mutex_assert_owner(tsdn, &pac->grow_mtx);
647
648 size_t alloc_size_min = size + PAGE_CEILING(alignment) - PAGE;
649 /* Beware size_t wrap-around. */
650 if (alloc_size_min < size) {
651 goto label_err;
652 }
653 /*
654 * Find the next extent size in the series that would be large enough to
655 * satisfy this request.
656 */
657 size_t alloc_size;
658 pszind_t exp_grow_skip;
659 bool err = exp_grow_size_prepare(&pac->exp_grow, alloc_size_min,
660 &alloc_size, &exp_grow_skip);
661 if (err) {
662 goto label_err;
663 }
664
665 edata_t *edata = edata_cache_get(tsdn, pac->edata_cache);
666 if (edata == NULL) {
667 goto label_err;
668 }
669 bool zeroed = false;
670 bool committed = false;
671
672 void *ptr = ehooks_alloc(tsdn, ehooks, NULL, alloc_size, PAGE, &zeroed,
673 &committed);
674
675 if (ptr == NULL) {
676 edata_cache_put(tsdn, pac->edata_cache, edata);
677 goto label_err;
678 }
679
680 edata_init(edata, ecache_ind_get(&pac->ecache_retained), ptr,
681 alloc_size, false, SC_NSIZES, extent_sn_next(pac),
682 extent_state_active, zeroed, committed, EXTENT_PAI_PAC,
683 EXTENT_IS_HEAD);
684
685 if (extent_register_no_gdump_add(tsdn, pac, edata)) {
686 edata_cache_put(tsdn, pac->edata_cache, edata);
687 goto label_err;
688 }
689
690 if (edata_committed_get(edata)) {
691 *commit = true;
692 }
693
694 edata_t *lead;
695 edata_t *trail;
696 edata_t *to_leak JEMALLOC_CC_SILENCE_INIT(NULL);
697 edata_t *to_salvage JEMALLOC_CC_SILENCE_INIT(NULL);
698
699 extent_split_interior_result_t result = extent_split_interior(tsdn,
700 pac, ehooks, &edata, &lead, &trail, &to_leak, &to_salvage, NULL,
701 size, alignment);
702
703 if (result == extent_split_interior_ok) {
704 if (lead != NULL) {
705 extent_record(tsdn, pac, ehooks, &pac->ecache_retained,
706 lead);
707 }
708 if (trail != NULL) {
709 extent_record(tsdn, pac, ehooks, &pac->ecache_retained,
710 trail);
711 }
712 } else {
713 /*
714 * We should have allocated a sufficiently large extent; the
715 * cant_alloc case should not occur.
716 */
717 assert(result == extent_split_interior_error);
718 if (to_salvage != NULL) {
719 if (config_prof) {
720 extent_gdump_add(tsdn, to_salvage);
721 }
722 extent_record(tsdn, pac, ehooks, &pac->ecache_retained,
723 to_salvage);
724 }
725 if (to_leak != NULL) {
726 extent_deregister_no_gdump_sub(tsdn, pac, to_leak);
727 extents_abandon_vm(tsdn, pac, ehooks,
728 &pac->ecache_retained, to_leak, true);
729 }
730 goto label_err;
731 }
732
733 if (*commit && !edata_committed_get(edata)) {
734 if (extent_commit_impl(tsdn, ehooks, edata, 0,
735 edata_size_get(edata), true)) {
736 extent_record(tsdn, pac, ehooks,
737 &pac->ecache_retained, edata);
738 goto label_err;
739 }
740 /* A successful commit should return zeroed memory. */
741 if (config_debug) {
742 void *addr = edata_addr_get(edata);
743 size_t *p = (size_t *)(uintptr_t)addr;
744 /* Check the first page only. */
745 for (size_t i = 0; i < PAGE / sizeof(size_t); i++) {
746 assert(p[i] == 0);
747 }
748 }
749 }
750
751 /*
752 * Increment extent_grow_next if doing so wouldn't exceed the allowed
753 * range.
754 */
755 /* All opportunities for failure are past. */
756 exp_grow_size_commit(&pac->exp_grow, exp_grow_skip);
757 malloc_mutex_unlock(tsdn, &pac->grow_mtx);
758
759 if (config_prof) {
760 /* Adjust gdump stats now that extent is final size. */
761 extent_gdump_add(tsdn, edata);
762 }
763 if (zero && !edata_zeroed_get(edata)) {
764 ehooks_zero(tsdn, ehooks, edata_base_get(edata),
765 edata_size_get(edata));
766 }
767 return edata;
768label_err:
769 malloc_mutex_unlock(tsdn, &pac->grow_mtx);
770 return NULL;
771}
772
773static edata_t *
774extent_alloc_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
775 edata_t *expand_edata, size_t size, size_t alignment, bool zero,
776 bool *commit, bool guarded) {
777 assert(size != 0);
778 assert(alignment != 0);
779
780 malloc_mutex_lock(tsdn, &pac->grow_mtx);
781
782 edata_t *edata = extent_recycle(tsdn, pac, ehooks,
783 &pac->ecache_retained, expand_edata, size, alignment, zero, commit,
784 /* growing_retained */ true, guarded);
785 if (edata != NULL) {
786 malloc_mutex_unlock(tsdn, &pac->grow_mtx);
787 if (config_prof) {
788 extent_gdump_add(tsdn, edata);
789 }
790 } else if (opt_retain && expand_edata == NULL && !guarded) {
791 edata = extent_grow_retained(tsdn, pac, ehooks, size,
792 alignment, zero, commit);
793 /* extent_grow_retained() always releases pac->grow_mtx. */
794 } else {
795 malloc_mutex_unlock(tsdn, &pac->grow_mtx);
796 }
797 malloc_mutex_assert_not_owner(tsdn, &pac->grow_mtx);
798
799 return edata;
800}
801
802static bool
803extent_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
804 edata_t *inner, edata_t *outer, bool forward) {
805 extent_assert_can_coalesce(inner, outer);
806 eset_remove(&ecache->eset, outer);
807
808 bool err = extent_merge_impl(tsdn, pac, ehooks,
809 forward ? inner : outer, forward ? outer : inner,
810 /* holding_core_locks */ true);
811 if (err) {
812 extent_deactivate_check_state_locked(tsdn, pac, ecache, outer,
813 extent_state_merging);
814 }
815
816 return err;
817}
818
819static edata_t *
820extent_try_coalesce_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
821 ecache_t *ecache, edata_t *edata, bool *coalesced) {
822 assert(!edata_guarded_get(edata));
823 /*
824 * We avoid checking / locking inactive neighbors for large size
825 * classes, since they are eagerly coalesced on deallocation which can
826 * cause lock contention.
827 */
828 /*
829 * Continue attempting to coalesce until failure, to protect against
830 * races with other threads that are thwarted by this one.
831 */
832 bool again;
833 do {
834 again = false;
835
836 /* Try to coalesce forward. */
837 edata_t *next = emap_try_acquire_edata_neighbor(tsdn, pac->emap,
838 edata, EXTENT_PAI_PAC, ecache->state, /* forward */ true);
839 if (next != NULL) {
840 if (!extent_coalesce(tsdn, pac, ehooks, ecache, edata,
841 next, true)) {
842 if (ecache->delay_coalesce) {
843 /* Do minimal coalescing. */
844 *coalesced = true;
845 return edata;
846 }
847 again = true;
848 }
849 }
850
851 /* Try to coalesce backward. */
852 edata_t *prev = emap_try_acquire_edata_neighbor(tsdn, pac->emap,
853 edata, EXTENT_PAI_PAC, ecache->state, /* forward */ false);
854 if (prev != NULL) {
855 if (!extent_coalesce(tsdn, pac, ehooks, ecache, edata,
856 prev, false)) {
857 edata = prev;
858 if (ecache->delay_coalesce) {
859 /* Do minimal coalescing. */
860 *coalesced = true;
861 return edata;
862 }
863 again = true;
864 }
865 }
866 } while (again);
867
868 if (ecache->delay_coalesce) {
869 *coalesced = false;
870 }
871 return edata;
872}
873
874static edata_t *
875extent_try_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
876 ecache_t *ecache, edata_t *edata, bool *coalesced) {
877 return extent_try_coalesce_impl(tsdn, pac, ehooks, ecache, edata,
878 coalesced);
879}
880
881static edata_t *
882extent_try_coalesce_large(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
883 ecache_t *ecache, edata_t *edata, bool *coalesced) {
884 return extent_try_coalesce_impl(tsdn, pac, ehooks, ecache, edata,
885 coalesced);
886}
887
888/* Purge a single extent to retained / unmapped directly. */
889static void
890extent_maximally_purge(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
891 edata_t *edata) {
892 size_t extent_size = edata_size_get(edata);
893 extent_dalloc_wrapper(tsdn, pac, ehooks, edata);
894 if (config_stats) {
895 /* Update stats accordingly. */
896 LOCKEDINT_MTX_LOCK(tsdn, *pac->stats_mtx);
897 locked_inc_u64(tsdn,
898 LOCKEDINT_MTX(*pac->stats_mtx),
899 &pac->stats->decay_dirty.nmadvise, 1);
900 locked_inc_u64(tsdn,
901 LOCKEDINT_MTX(*pac->stats_mtx),
902 &pac->stats->decay_dirty.purged,
903 extent_size >> LG_PAGE);
904 LOCKEDINT_MTX_UNLOCK(tsdn, *pac->stats_mtx);
905 atomic_fetch_sub_zu(&pac->stats->pac_mapped, extent_size,
906 ATOMIC_RELAXED);
907 }
908}
909
910/*
911 * Does the metadata management portions of putting an unused extent into the
912 * given ecache_t (coalesces and inserts into the eset).
913 */
914void
915extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
916 edata_t *edata) {
917 assert((ecache->state != extent_state_dirty &&
918 ecache->state != extent_state_muzzy) ||
919 !edata_zeroed_get(edata));
920
921 malloc_mutex_lock(tsdn, &ecache->mtx);
922
923 emap_assert_mapped(tsdn, pac->emap, edata);
924
925 if (edata_guarded_get(edata)) {
926 goto label_skip_coalesce;
927 }
928 if (!ecache->delay_coalesce) {
929 edata = extent_try_coalesce(tsdn, pac, ehooks, ecache, edata,
930 NULL);
931 } else if (edata_size_get(edata) >= SC_LARGE_MINCLASS) {
932 assert(ecache == &pac->ecache_dirty);
933 /* Always coalesce large extents eagerly. */
934 bool coalesced;
935 do {
936 assert(edata_state_get(edata) == extent_state_active);
937 edata = extent_try_coalesce_large(tsdn, pac, ehooks,
938 ecache, edata, &coalesced);
939 } while (coalesced);
940 if (edata_size_get(edata) >=
941 atomic_load_zu(&pac->oversize_threshold, ATOMIC_RELAXED)
942 && extent_may_force_decay(pac)) {
943 /* Shortcut to purge the oversize extent eagerly. */
944 malloc_mutex_unlock(tsdn, &ecache->mtx);
945 extent_maximally_purge(tsdn, pac, ehooks, edata);
946 return;
947 }
948 }
949label_skip_coalesce:
950 extent_deactivate_locked(tsdn, pac, ecache, edata);
951
952 malloc_mutex_unlock(tsdn, &ecache->mtx);
953}
954
955void
956extent_dalloc_gap(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
957 edata_t *edata) {
958 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
959 WITNESS_RANK_CORE, 0);
960
961 if (extent_register(tsdn, pac, edata)) {
962 edata_cache_put(tsdn, pac->edata_cache, edata);
963 return;
964 }
965 extent_dalloc_wrapper(tsdn, pac, ehooks, edata);
966}
967
968static bool
969extent_dalloc_wrapper_try(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
970 edata_t *edata) {
971 bool err;
972
973 assert(edata_base_get(edata) != NULL);
974 assert(edata_size_get(edata) != 0);
975 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
976 WITNESS_RANK_CORE, 0);
977
978 edata_addr_set(edata, edata_base_get(edata));
979
980 /* Try to deallocate. */
981 err = ehooks_dalloc(tsdn, ehooks, edata_base_get(edata),
982 edata_size_get(edata), edata_committed_get(edata));
983
984 if (!err) {
985 edata_cache_put(tsdn, pac->edata_cache, edata);
986 }
987
988 return err;
989}
990
991edata_t *
992extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
993 void *new_addr, size_t size, size_t alignment, bool zero, bool *commit,
994 bool growing_retained) {
995 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
996 WITNESS_RANK_CORE, growing_retained ? 1 : 0);
997
998 edata_t *edata = edata_cache_get(tsdn, pac->edata_cache);
999 if (edata == NULL) {
1000 return NULL;
1001 }
1002 size_t palignment = ALIGNMENT_CEILING(alignment, PAGE);
1003 void *addr = ehooks_alloc(tsdn, ehooks, new_addr, size, palignment,
1004 &zero, commit);
1005 if (addr == NULL) {
1006 edata_cache_put(tsdn, pac->edata_cache, edata);
1007 return NULL;
1008 }
1009 edata_init(edata, ecache_ind_get(&pac->ecache_dirty), addr,
1010 size, /* slab */ false, SC_NSIZES, extent_sn_next(pac),
1011 extent_state_active, zero, *commit, EXTENT_PAI_PAC,
1012 opt_retain ? EXTENT_IS_HEAD : EXTENT_NOT_HEAD);
1013 /*
1014 * Retained memory is not counted towards gdump. Only if an extent is
1015 * allocated as a separate mapping, i.e. growing_retained is false, then
1016 * gdump should be updated.
1017 */
1018 bool gdump_add = !growing_retained;
1019 if (extent_register_impl(tsdn, pac, edata, gdump_add)) {
1020 edata_cache_put(tsdn, pac->edata_cache, edata);
1021 return NULL;
1022 }
1023
1024 return edata;
1025}
1026
1027void
1028extent_dalloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
1029 edata_t *edata) {
1030 assert(edata_pai_get(edata) == EXTENT_PAI_PAC);
1031 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
1032 WITNESS_RANK_CORE, 0);
1033
1034 /* Avoid calling the default extent_dalloc unless have to. */
1035 if (!ehooks_dalloc_will_fail(ehooks)) {
1036 /* Remove guard pages for dalloc / unmap. */
1037 if (edata_guarded_get(edata)) {
1038 assert(ehooks_are_default(ehooks));
1039 san_unguard_pages_two_sided(tsdn, ehooks, edata,
1040 pac->emap);
1041 }
1042 /*
1043 * Deregister first to avoid a race with other allocating
1044 * threads, and reregister if deallocation fails.
1045 */
1046 extent_deregister(tsdn, pac, edata);
1047 if (!extent_dalloc_wrapper_try(tsdn, pac, ehooks, edata)) {
1048 return;
1049 }
1050 extent_reregister(tsdn, pac, edata);
1051 }
1052
1053 /* Try to decommit; purge if that fails. */
1054 bool zeroed;
1055 if (!edata_committed_get(edata)) {
1056 zeroed = true;
1057 } else if (!extent_decommit_wrapper(tsdn, ehooks, edata, 0,
1058 edata_size_get(edata))) {
1059 zeroed = true;
1060 } else if (!ehooks_purge_forced(tsdn, ehooks, edata_base_get(edata),
1061 edata_size_get(edata), 0, edata_size_get(edata))) {
1062 zeroed = true;
1063 } else if (edata_state_get(edata) == extent_state_muzzy ||
1064 !ehooks_purge_lazy(tsdn, ehooks, edata_base_get(edata),
1065 edata_size_get(edata), 0, edata_size_get(edata))) {
1066 zeroed = false;
1067 } else {
1068 zeroed = false;
1069 }
1070 edata_zeroed_set(edata, zeroed);
1071
1072 if (config_prof) {
1073 extent_gdump_sub(tsdn, edata);
1074 }
1075
1076 extent_record(tsdn, pac, ehooks, &pac->ecache_retained, edata);
1077}
1078
1079void
1080extent_destroy_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
1081 edata_t *edata) {
1082 assert(edata_base_get(edata) != NULL);
1083 assert(edata_size_get(edata) != 0);
1084 extent_state_t state = edata_state_get(edata);
1085 assert(state == extent_state_retained || state == extent_state_active);
1086 assert(emap_edata_is_acquired(tsdn, pac->emap, edata));
1087 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
1088 WITNESS_RANK_CORE, 0);
1089
1090 if (edata_guarded_get(edata)) {
1091 assert(opt_retain);
1092 san_unguard_pages_pre_destroy(tsdn, ehooks, edata, pac->emap);
1093 }
1094 edata_addr_set(edata, edata_base_get(edata));
1095
1096 /* Try to destroy; silently fail otherwise. */
1097 ehooks_destroy(tsdn, ehooks, edata_base_get(edata),
1098 edata_size_get(edata), edata_committed_get(edata));
1099
1100 edata_cache_put(tsdn, pac->edata_cache, edata);
1101}
1102
1103static bool
1104extent_commit_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
1105 size_t offset, size_t length, bool growing_retained) {
1106 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
1107 WITNESS_RANK_CORE, growing_retained ? 1 : 0);
1108 bool err = ehooks_commit(tsdn, ehooks, edata_base_get(edata),
1109 edata_size_get(edata), offset, length);
1110 edata_committed_set(edata, edata_committed_get(edata) || !err);
1111 return err;
1112}
1113
1114bool
1115extent_commit_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
1116 size_t offset, size_t length) {
1117 return extent_commit_impl(tsdn, ehooks, edata, offset, length,
1118 /* growing_retained */ false);
1119}
1120
1121bool
1122extent_decommit_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
1123 size_t offset, size_t length) {
1124 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
1125 WITNESS_RANK_CORE, 0);
1126 bool err = ehooks_decommit(tsdn, ehooks, edata_base_get(edata),
1127 edata_size_get(edata), offset, length);
1128 edata_committed_set(edata, edata_committed_get(edata) && err);
1129 return err;
1130}
1131
1132static bool
1133extent_purge_lazy_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
1134 size_t offset, size_t length, bool growing_retained) {
1135 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
1136 WITNESS_RANK_CORE, growing_retained ? 1 : 0);
1137 bool err = ehooks_purge_lazy(tsdn, ehooks, edata_base_get(edata),
1138 edata_size_get(edata), offset, length);
1139 return err;
1140}
1141
1142bool
1143extent_purge_lazy_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
1144 size_t offset, size_t length) {
1145 return extent_purge_lazy_impl(tsdn, ehooks, edata, offset,
1146 length, false);
1147}
1148
1149static bool
1150extent_purge_forced_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
1151 size_t offset, size_t length, bool growing_retained) {
1152 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
1153 WITNESS_RANK_CORE, growing_retained ? 1 : 0);
1154 bool err = ehooks_purge_forced(tsdn, ehooks, edata_base_get(edata),
1155 edata_size_get(edata), offset, length);
1156 return err;
1157}
1158
1159bool
1160extent_purge_forced_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
1161 size_t offset, size_t length) {
1162 return extent_purge_forced_impl(tsdn, ehooks, edata, offset, length,
1163 false);
1164}
1165
1166/*
1167 * Accepts the extent to split, and the characteristics of each side of the
1168 * split. The 'a' parameters go with the 'lead' of the resulting pair of
1169 * extents (the lower addressed portion of the split), and the 'b' parameters go
1170 * with the trail (the higher addressed portion). This makes 'extent' the lead,
1171 * and returns the trail (except in case of error).
1172 */
1173static edata_t *
1174extent_split_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
1175 edata_t *edata, size_t size_a, size_t size_b, bool holding_core_locks) {
1176 assert(edata_size_get(edata) == size_a + size_b);
1177 /* Only the shrink path may split w/o holding core locks. */
1178 if (holding_core_locks) {
1179 witness_assert_positive_depth_to_rank(
1180 tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE);
1181 } else {
1182 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
1183 WITNESS_RANK_CORE, 0);
1184 }
1185
1186 if (ehooks_split_will_fail(ehooks)) {
1187 return NULL;
1188 }
1189
1190 edata_t *trail = edata_cache_get(tsdn, pac->edata_cache);
1191 if (trail == NULL) {
1192 goto label_error_a;
1193 }
1194
1195 edata_init(trail, edata_arena_ind_get(edata),
1196 (void *)((uintptr_t)edata_base_get(edata) + size_a), size_b,
1197 /* slab */ false, SC_NSIZES, edata_sn_get(edata),
1198 edata_state_get(edata), edata_zeroed_get(edata),
1199 edata_committed_get(edata), EXTENT_PAI_PAC, EXTENT_NOT_HEAD);
1200 emap_prepare_t prepare;
1201 bool err = emap_split_prepare(tsdn, pac->emap, &prepare, edata,
1202 size_a, trail, size_b);
1203 if (err) {
1204 goto label_error_b;
1205 }
1206
1207 /*
1208 * No need to acquire trail or edata, because: 1) trail was new (just
1209 * allocated); and 2) edata is either an active allocation (the shrink
1210 * path), or in an acquired state (extracted from the ecache on the
1211 * extent_recycle_split path).
1212 */
1213 assert(emap_edata_is_acquired(tsdn, pac->emap, edata));
1214 assert(emap_edata_is_acquired(tsdn, pac->emap, trail));
1215
1216 err = ehooks_split(tsdn, ehooks, edata_base_get(edata), size_a + size_b,
1217 size_a, size_b, edata_committed_get(edata));
1218
1219 if (err) {
1220 goto label_error_b;
1221 }
1222
1223 edata_size_set(edata, size_a);
1224 emap_split_commit(tsdn, pac->emap, &prepare, edata, size_a, trail,
1225 size_b);
1226
1227 return trail;
1228label_error_b:
1229 edata_cache_put(tsdn, pac->edata_cache, trail);
1230label_error_a:
1231 return NULL;
1232}
1233
1234edata_t *
1235extent_split_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata,
1236 size_t size_a, size_t size_b, bool holding_core_locks) {
1237 return extent_split_impl(tsdn, pac, ehooks, edata, size_a, size_b,
1238 holding_core_locks);
1239}
1240
1241static bool
1242extent_merge_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *a,
1243 edata_t *b, bool holding_core_locks) {
1244 /* Only the expanding path may merge w/o holding ecache locks. */
1245 if (holding_core_locks) {
1246 witness_assert_positive_depth_to_rank(
1247 tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE);
1248 } else {
1249 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
1250 WITNESS_RANK_CORE, 0);
1251 }
1252
1253 assert(edata_base_get(a) < edata_base_get(b));
1254 assert(edata_arena_ind_get(a) == edata_arena_ind_get(b));
1255 assert(edata_arena_ind_get(a) == ehooks_ind_get(ehooks));
1256 emap_assert_mapped(tsdn, pac->emap, a);
1257 emap_assert_mapped(tsdn, pac->emap, b);
1258
1259 bool err = ehooks_merge(tsdn, ehooks, edata_base_get(a),
1260 edata_size_get(a), edata_base_get(b), edata_size_get(b),
1261 edata_committed_get(a));
1262
1263 if (err) {
1264 return true;
1265 }
1266
1267 /*
1268 * The rtree writes must happen while all the relevant elements are
1269 * owned, so the following code uses decomposed helper functions rather
1270 * than extent_{,de}register() to do things in the right order.
1271 */
1272 emap_prepare_t prepare;
1273 emap_merge_prepare(tsdn, pac->emap, &prepare, a, b);
1274
1275 assert(edata_state_get(a) == extent_state_active ||
1276 edata_state_get(a) == extent_state_merging);
1277 edata_state_set(a, extent_state_active);
1278 edata_size_set(a, edata_size_get(a) + edata_size_get(b));
1279 edata_sn_set(a, (edata_sn_get(a) < edata_sn_get(b)) ?
1280 edata_sn_get(a) : edata_sn_get(b));
1281 edata_zeroed_set(a, edata_zeroed_get(a) && edata_zeroed_get(b));
1282
1283 emap_merge_commit(tsdn, pac->emap, &prepare, a, b);
1284
1285 edata_cache_put(tsdn, pac->edata_cache, b);
1286
1287 return false;
1288}
1289
1290bool
1291extent_merge_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
1292 edata_t *a, edata_t *b) {
1293 return extent_merge_impl(tsdn, pac, ehooks, a, b,
1294 /* holding_core_locks */ false);
1295}
1296
1297bool
1298extent_commit_zero(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
1299 bool commit, bool zero, bool growing_retained) {
1300 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
1301 WITNESS_RANK_CORE, growing_retained ? 1 : 0);
1302
1303 if (commit && !edata_committed_get(edata)) {
1304 if (extent_commit_impl(tsdn, ehooks, edata, 0,
1305 edata_size_get(edata), growing_retained)) {
1306 return true;
1307 }
1308 }
1309 if (zero && !edata_zeroed_get(edata)) {
1310 void *addr = edata_base_get(edata);
1311 size_t size = edata_size_get(edata);
1312 ehooks_zero(tsdn, ehooks, addr, size);
1313 }
1314 return false;
1315}
1316
1317bool
1318extent_boot(void) {
1319 assert(sizeof(slab_data_t) >= sizeof(e_prof_info_t));
1320
1321 if (have_dss) {
1322 extent_dss_boot();
1323 }
1324
1325 return false;
1326}
diff --git a/examples/redis-unstable/deps/jemalloc/src/extent_dss.c b/examples/redis-unstable/deps/jemalloc/src/extent_dss.c
deleted file mode 100644
index 9a35bac..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/extent_dss.c
+++ /dev/null
@@ -1,277 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/extent_dss.h"
6#include "jemalloc/internal/spin.h"
7
8/******************************************************************************/
9/* Data. */
10
11const char *opt_dss = DSS_DEFAULT;
12
13const char *dss_prec_names[] = {
14 "disabled",
15 "primary",
16 "secondary",
17 "N/A"
18};
19
20/*
21 * Current dss precedence default, used when creating new arenas. NB: This is
22 * stored as unsigned rather than dss_prec_t because in principle there's no
23 * guarantee that sizeof(dss_prec_t) is the same as sizeof(unsigned), and we use
24 * atomic operations to synchronize the setting.
25 */
26static atomic_u_t dss_prec_default = ATOMIC_INIT(
27 (unsigned)DSS_PREC_DEFAULT);
28
29/* Base address of the DSS. */
30static void *dss_base;
31/* Atomic boolean indicating whether a thread is currently extending DSS. */
32static atomic_b_t dss_extending;
33/* Atomic boolean indicating whether the DSS is exhausted. */
34static atomic_b_t dss_exhausted;
35/* Atomic current upper limit on DSS addresses. */
36static atomic_p_t dss_max;
37
38/******************************************************************************/
39
40static void *
41extent_dss_sbrk(intptr_t increment) {
42#ifdef JEMALLOC_DSS
43 return sbrk(increment);
44#else
45 not_implemented();
46 return NULL;
47#endif
48}
49
50dss_prec_t
51extent_dss_prec_get(void) {
52 dss_prec_t ret;
53
54 if (!have_dss) {
55 return dss_prec_disabled;
56 }
57 ret = (dss_prec_t)atomic_load_u(&dss_prec_default, ATOMIC_ACQUIRE);
58 return ret;
59}
60
61bool
62extent_dss_prec_set(dss_prec_t dss_prec) {
63 if (!have_dss) {
64 return (dss_prec != dss_prec_disabled);
65 }
66 atomic_store_u(&dss_prec_default, (unsigned)dss_prec, ATOMIC_RELEASE);
67 return false;
68}
69
70static void
71extent_dss_extending_start(void) {
72 spin_t spinner = SPIN_INITIALIZER;
73 while (true) {
74 bool expected = false;
75 if (atomic_compare_exchange_weak_b(&dss_extending, &expected,
76 true, ATOMIC_ACQ_REL, ATOMIC_RELAXED)) {
77 break;
78 }
79 spin_adaptive(&spinner);
80 }
81}
82
83static void
84extent_dss_extending_finish(void) {
85 assert(atomic_load_b(&dss_extending, ATOMIC_RELAXED));
86
87 atomic_store_b(&dss_extending, false, ATOMIC_RELEASE);
88}
89
90static void *
91extent_dss_max_update(void *new_addr) {
92 /*
93 * Get the current end of the DSS as max_cur and assure that dss_max is
94 * up to date.
95 */
96 void *max_cur = extent_dss_sbrk(0);
97 if (max_cur == (void *)-1) {
98 return NULL;
99 }
100 atomic_store_p(&dss_max, max_cur, ATOMIC_RELEASE);
101 /* Fixed new_addr can only be supported if it is at the edge of DSS. */
102 if (new_addr != NULL && max_cur != new_addr) {
103 return NULL;
104 }
105 return max_cur;
106}
107
108void *
109extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
110 size_t alignment, bool *zero, bool *commit) {
111 edata_t *gap;
112
113 cassert(have_dss);
114 assert(size > 0);
115 assert(alignment == ALIGNMENT_CEILING(alignment, PAGE));
116
117 /*
118 * sbrk() uses a signed increment argument, so take care not to
119 * interpret a large allocation request as a negative increment.
120 */
121 if ((intptr_t)size < 0) {
122 return NULL;
123 }
124
125 gap = edata_cache_get(tsdn, &arena->pa_shard.edata_cache);
126 if (gap == NULL) {
127 return NULL;
128 }
129
130 extent_dss_extending_start();
131 if (!atomic_load_b(&dss_exhausted, ATOMIC_ACQUIRE)) {
132 /*
133 * The loop is necessary to recover from races with other
134 * threads that are using the DSS for something other than
135 * malloc.
136 */
137 while (true) {
138 void *max_cur = extent_dss_max_update(new_addr);
139 if (max_cur == NULL) {
140 goto label_oom;
141 }
142
143 bool head_state = opt_retain ? EXTENT_IS_HEAD :
144 EXTENT_NOT_HEAD;
145 /*
146 * Compute how much page-aligned gap space (if any) is
147 * necessary to satisfy alignment. This space can be
148 * recycled for later use.
149 */
150 void *gap_addr_page = (void *)(PAGE_CEILING(
151 (uintptr_t)max_cur));
152 void *ret = (void *)ALIGNMENT_CEILING(
153 (uintptr_t)gap_addr_page, alignment);
154 size_t gap_size_page = (uintptr_t)ret -
155 (uintptr_t)gap_addr_page;
156 if (gap_size_page != 0) {
157 edata_init(gap, arena_ind_get(arena),
158 gap_addr_page, gap_size_page, false,
159 SC_NSIZES, extent_sn_next(
160 &arena->pa_shard.pac),
161 extent_state_active, false, true,
162 EXTENT_PAI_PAC, head_state);
163 }
164 /*
165 * Compute the address just past the end of the desired
166 * allocation space.
167 */
168 void *dss_next = (void *)((uintptr_t)ret + size);
169 if ((uintptr_t)ret < (uintptr_t)max_cur ||
170 (uintptr_t)dss_next < (uintptr_t)max_cur) {
171 goto label_oom; /* Wrap-around. */
172 }
173 /* Compute the increment, including subpage bytes. */
174 void *gap_addr_subpage = max_cur;
175 size_t gap_size_subpage = (uintptr_t)ret -
176 (uintptr_t)gap_addr_subpage;
177 intptr_t incr = gap_size_subpage + size;
178
179 assert((uintptr_t)max_cur + incr == (uintptr_t)ret +
180 size);
181
182 /* Try to allocate. */
183 void *dss_prev = extent_dss_sbrk(incr);
184 if (dss_prev == max_cur) {
185 /* Success. */
186 atomic_store_p(&dss_max, dss_next,
187 ATOMIC_RELEASE);
188 extent_dss_extending_finish();
189
190 if (gap_size_page != 0) {
191 ehooks_t *ehooks = arena_get_ehooks(
192 arena);
193 extent_dalloc_gap(tsdn,
194 &arena->pa_shard.pac, ehooks, gap);
195 } else {
196 edata_cache_put(tsdn,
197 &arena->pa_shard.edata_cache, gap);
198 }
199 if (!*commit) {
200 *commit = pages_decommit(ret, size);
201 }
202 if (*zero && *commit) {
203 edata_t edata = {0};
204 ehooks_t *ehooks = arena_get_ehooks(
205 arena);
206
207 edata_init(&edata,
208 arena_ind_get(arena), ret, size,
209 size, false, SC_NSIZES,
210 extent_state_active, false, true,
211 EXTENT_PAI_PAC, head_state);
212 if (extent_purge_forced_wrapper(tsdn,
213 ehooks, &edata, 0, size)) {
214 memset(ret, 0, size);
215 }
216 }
217 return ret;
218 }
219 /*
220 * Failure, whether due to OOM or a race with a raw
221 * sbrk() call from outside the allocator.
222 */
223 if (dss_prev == (void *)-1) {
224 /* OOM. */
225 atomic_store_b(&dss_exhausted, true,
226 ATOMIC_RELEASE);
227 goto label_oom;
228 }
229 }
230 }
231label_oom:
232 extent_dss_extending_finish();
233 edata_cache_put(tsdn, &arena->pa_shard.edata_cache, gap);
234 return NULL;
235}
236
237static bool
238extent_in_dss_helper(void *addr, void *max) {
239 return ((uintptr_t)addr >= (uintptr_t)dss_base && (uintptr_t)addr <
240 (uintptr_t)max);
241}
242
243bool
244extent_in_dss(void *addr) {
245 cassert(have_dss);
246
247 return extent_in_dss_helper(addr, atomic_load_p(&dss_max,
248 ATOMIC_ACQUIRE));
249}
250
251bool
252extent_dss_mergeable(void *addr_a, void *addr_b) {
253 void *max;
254
255 cassert(have_dss);
256
257 if ((uintptr_t)addr_a < (uintptr_t)dss_base && (uintptr_t)addr_b <
258 (uintptr_t)dss_base) {
259 return true;
260 }
261
262 max = atomic_load_p(&dss_max, ATOMIC_ACQUIRE);
263 return (extent_in_dss_helper(addr_a, max) ==
264 extent_in_dss_helper(addr_b, max));
265}
266
267void
268extent_dss_boot(void) {
269 cassert(have_dss);
270
271 dss_base = extent_dss_sbrk(0);
272 atomic_store_b(&dss_extending, false, ATOMIC_RELAXED);
273 atomic_store_b(&dss_exhausted, dss_base == (void *)-1, ATOMIC_RELAXED);
274 atomic_store_p(&dss_max, dss_base, ATOMIC_RELAXED);
275}
276
277/******************************************************************************/
diff --git a/examples/redis-unstable/deps/jemalloc/src/extent_mmap.c b/examples/redis-unstable/deps/jemalloc/src/extent_mmap.c
deleted file mode 100644
index 5f0ee2d..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/extent_mmap.c
+++ /dev/null
@@ -1,41 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/extent_mmap.h"
6
7/******************************************************************************/
8/* Data. */
9
10bool opt_retain =
11#ifdef JEMALLOC_RETAIN
12 true
13#else
14 false
15#endif
16 ;
17
18/******************************************************************************/
19
20void *
21extent_alloc_mmap(void *new_addr, size_t size, size_t alignment, bool *zero,
22 bool *commit) {
23 assert(alignment == ALIGNMENT_CEILING(alignment, PAGE));
24 void *ret = pages_map(new_addr, size, alignment, commit);
25 if (ret == NULL) {
26 return NULL;
27 }
28 assert(ret != NULL);
29 if (*commit) {
30 *zero = true;
31 }
32 return ret;
33}
34
35bool
36extent_dalloc_mmap(void *addr, size_t size) {
37 if (!opt_retain) {
38 pages_unmap(addr, size);
39 }
40 return opt_retain;
41}
diff --git a/examples/redis-unstable/deps/jemalloc/src/fxp.c b/examples/redis-unstable/deps/jemalloc/src/fxp.c
deleted file mode 100644
index 96585f0..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/fxp.c
+++ /dev/null
@@ -1,124 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/fxp.h"
5
6static bool
7fxp_isdigit(char c) {
8 return '0' <= c && c <= '9';
9}
10
11bool
12fxp_parse(fxp_t *result, const char *str, char **end) {
13 /*
14 * Using malloc_strtoumax in this method isn't as handy as you might
15 * expect (I tried). In the fractional part, significant leading zeros
16 * mean that you still need to do your own parsing, now with trickier
17 * math. In the integer part, the casting (uintmax_t to uint32_t)
18 * forces more reasoning about bounds than just checking for overflow as
19 * we parse.
20 */
21 uint32_t integer_part = 0;
22
23 const char *cur = str;
24
25 /* The string must start with a digit or a decimal point. */
26 if (*cur != '.' && !fxp_isdigit(*cur)) {
27 return true;
28 }
29
30 while ('0' <= *cur && *cur <= '9') {
31 integer_part *= 10;
32 integer_part += *cur - '0';
33 if (integer_part >= (1U << 16)) {
34 return true;
35 }
36 cur++;
37 }
38
39 /*
40 * We've parsed all digits at the beginning of the string, without
41 * overflow. Either we're done, or there's a fractional part.
42 */
43 if (*cur != '.') {
44 *result = (integer_part << 16);
45 if (end != NULL) {
46 *end = (char *)cur;
47 }
48 return false;
49 }
50
51 /* There's a fractional part. */
52 cur++;
53 if (!fxp_isdigit(*cur)) {
54 /* Shouldn't end on the decimal point. */
55 return true;
56 }
57
58 /*
59 * We use a lot of precision for the fractional part, even though we'll
60 * discard most of it; this lets us get exact values for the important
61 * special case where the denominator is a small power of 2 (for
62 * instance, 1/512 == 0.001953125 is exactly representable even with
63 * only 16 bits of fractional precision). We need to left-shift by 16
64 * before dividing so we pick the number of digits to be
65 * floor(log(2**48)) = 14.
66 */
67 uint64_t fractional_part = 0;
68 uint64_t frac_div = 1;
69 for (int i = 0; i < FXP_FRACTIONAL_PART_DIGITS; i++) {
70 fractional_part *= 10;
71 frac_div *= 10;
72 if (fxp_isdigit(*cur)) {
73 fractional_part += *cur - '0';
74 cur++;
75 }
76 }
77 /*
78 * We only parse the first maxdigits characters, but we can still ignore
79 * any digits after that.
80 */
81 while (fxp_isdigit(*cur)) {
82 cur++;
83 }
84
85 assert(fractional_part < frac_div);
86 uint32_t fractional_repr = (uint32_t)(
87 (fractional_part << 16) / frac_div);
88
89 /* Success! */
90 *result = (integer_part << 16) + fractional_repr;
91 if (end != NULL) {
92 *end = (char *)cur;
93 }
94 return false;
95}
96
97void
98fxp_print(fxp_t a, char buf[FXP_BUF_SIZE]) {
99 uint32_t integer_part = fxp_round_down(a);
100 uint32_t fractional_part = (a & ((1U << 16) - 1));
101
102 int leading_fraction_zeros = 0;
103 uint64_t fraction_digits = fractional_part;
104 for (int i = 0; i < FXP_FRACTIONAL_PART_DIGITS; i++) {
105 if (fraction_digits < (1U << 16)
106 && fraction_digits * 10 >= (1U << 16)) {
107 leading_fraction_zeros = i;
108 }
109 fraction_digits *= 10;
110 }
111 fraction_digits >>= 16;
112 while (fraction_digits > 0 && fraction_digits % 10 == 0) {
113 fraction_digits /= 10;
114 }
115
116 size_t printed = malloc_snprintf(buf, FXP_BUF_SIZE, "%"FMTu32".",
117 integer_part);
118 for (int i = 0; i < leading_fraction_zeros; i++) {
119 buf[printed] = '0';
120 printed++;
121 }
122 malloc_snprintf(&buf[printed], FXP_BUF_SIZE - printed, "%"FMTu64,
123 fraction_digits);
124}
diff --git a/examples/redis-unstable/deps/jemalloc/src/hook.c b/examples/redis-unstable/deps/jemalloc/src/hook.c
deleted file mode 100644
index 493edbb..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/hook.c
+++ /dev/null
@@ -1,195 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2
3#include "jemalloc/internal/hook.h"
4
5#include "jemalloc/internal/atomic.h"
6#include "jemalloc/internal/mutex.h"
7#include "jemalloc/internal/seq.h"
8
9typedef struct hooks_internal_s hooks_internal_t;
10struct hooks_internal_s {
11 hooks_t hooks;
12 bool in_use;
13};
14
15seq_define(hooks_internal_t, hooks)
16
17static atomic_u_t nhooks = ATOMIC_INIT(0);
18static seq_hooks_t hooks[HOOK_MAX];
19static malloc_mutex_t hooks_mu;
20
21bool
22hook_boot() {
23 return malloc_mutex_init(&hooks_mu, "hooks", WITNESS_RANK_HOOK,
24 malloc_mutex_rank_exclusive);
25}
26
27static void *
28hook_install_locked(hooks_t *to_install) {
29 hooks_internal_t hooks_internal;
30 for (int i = 0; i < HOOK_MAX; i++) {
31 bool success = seq_try_load_hooks(&hooks_internal, &hooks[i]);
32 /* We hold mu; no concurrent access. */
33 assert(success);
34 if (!hooks_internal.in_use) {
35 hooks_internal.hooks = *to_install;
36 hooks_internal.in_use = true;
37 seq_store_hooks(&hooks[i], &hooks_internal);
38 atomic_store_u(&nhooks,
39 atomic_load_u(&nhooks, ATOMIC_RELAXED) + 1,
40 ATOMIC_RELAXED);
41 return &hooks[i];
42 }
43 }
44 return NULL;
45}
46
47void *
48hook_install(tsdn_t *tsdn, hooks_t *to_install) {
49 malloc_mutex_lock(tsdn, &hooks_mu);
50 void *ret = hook_install_locked(to_install);
51 if (ret != NULL) {
52 tsd_global_slow_inc(tsdn);
53 }
54 malloc_mutex_unlock(tsdn, &hooks_mu);
55 return ret;
56}
57
58static void
59hook_remove_locked(seq_hooks_t *to_remove) {
60 hooks_internal_t hooks_internal;
61 bool success = seq_try_load_hooks(&hooks_internal, to_remove);
62 /* We hold mu; no concurrent access. */
63 assert(success);
64 /* Should only remove hooks that were added. */
65 assert(hooks_internal.in_use);
66 hooks_internal.in_use = false;
67 seq_store_hooks(to_remove, &hooks_internal);
68 atomic_store_u(&nhooks, atomic_load_u(&nhooks, ATOMIC_RELAXED) - 1,
69 ATOMIC_RELAXED);
70}
71
72void
73hook_remove(tsdn_t *tsdn, void *opaque) {
74 if (config_debug) {
75 char *hooks_begin = (char *)&hooks[0];
76 char *hooks_end = (char *)&hooks[HOOK_MAX];
77 char *hook = (char *)opaque;
78 assert(hooks_begin <= hook && hook < hooks_end
79 && (hook - hooks_begin) % sizeof(seq_hooks_t) == 0);
80 }
81 malloc_mutex_lock(tsdn, &hooks_mu);
82 hook_remove_locked((seq_hooks_t *)opaque);
83 tsd_global_slow_dec(tsdn);
84 malloc_mutex_unlock(tsdn, &hooks_mu);
85}
86
87#define FOR_EACH_HOOK_BEGIN(hooks_internal_ptr) \
88for (int for_each_hook_counter = 0; \
89 for_each_hook_counter < HOOK_MAX; \
90 for_each_hook_counter++) { \
91 bool for_each_hook_success = seq_try_load_hooks( \
92 (hooks_internal_ptr), &hooks[for_each_hook_counter]); \
93 if (!for_each_hook_success) { \
94 continue; \
95 } \
96 if (!(hooks_internal_ptr)->in_use) { \
97 continue; \
98 }
99#define FOR_EACH_HOOK_END \
100}
101
102static bool *
103hook_reentrantp() {
104 /*
105 * We prevent user reentrancy within hooks. This is basically just a
106 * thread-local bool that triggers an early-exit.
107 *
108 * We don't fold in_hook into reentrancy. There are two reasons for
109 * this:
110 * - Right now, we turn on reentrancy during things like extent hook
111 * execution. Allocating during extent hooks is not officially
112 * supported, but we don't want to break it for the time being. These
113 * sorts of allocations should probably still be hooked, though.
114 * - If a hook allocates, we may want it to be relatively fast (after
115 * all, it executes on every allocator operation). Turning on
116 * reentrancy is a fairly heavyweight mode (disabling tcache,
117 * redirecting to arena 0, etc.). It's possible we may one day want
118 * to turn on reentrant mode here, if it proves too difficult to keep
119 * this working. But that's fairly easy for us to see; OTOH, people
120 * not using hooks because they're too slow is easy for us to miss.
121 *
122 * The tricky part is
123 * that this code might get invoked even if we don't have access to tsd.
124 * This function mimics getting a pointer to thread-local data, except
125 * that it might secretly return a pointer to some global data if we
126 * know that the caller will take the early-exit path.
127 * If we return a bool that indicates that we are reentrant, then the
128 * caller will go down the early exit path, leaving the global
129 * untouched.
130 */
131 static bool in_hook_global = true;
132 tsdn_t *tsdn = tsdn_fetch();
133 bool *in_hook = tsdn_in_hookp_get(tsdn);
134 if (in_hook!= NULL) {
135 return in_hook;
136 }
137 return &in_hook_global;
138}
139
140#define HOOK_PROLOGUE \
141 if (likely(atomic_load_u(&nhooks, ATOMIC_RELAXED) == 0)) { \
142 return; \
143 } \
144 bool *in_hook = hook_reentrantp(); \
145 if (*in_hook) { \
146 return; \
147 } \
148 *in_hook = true;
149
150#define HOOK_EPILOGUE \
151 *in_hook = false;
152
153void
154hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw,
155 uintptr_t args_raw[3]) {
156 HOOK_PROLOGUE
157
158 hooks_internal_t hook;
159 FOR_EACH_HOOK_BEGIN(&hook)
160 hook_alloc h = hook.hooks.alloc_hook;
161 if (h != NULL) {
162 h(hook.hooks.extra, type, result, result_raw, args_raw);
163 }
164 FOR_EACH_HOOK_END
165
166 HOOK_EPILOGUE
167}
168
169void
170hook_invoke_dalloc(hook_dalloc_t type, void *address, uintptr_t args_raw[3]) {
171 HOOK_PROLOGUE
172 hooks_internal_t hook;
173 FOR_EACH_HOOK_BEGIN(&hook)
174 hook_dalloc h = hook.hooks.dalloc_hook;
175 if (h != NULL) {
176 h(hook.hooks.extra, type, address, args_raw);
177 }
178 FOR_EACH_HOOK_END
179 HOOK_EPILOGUE
180}
181
182void
183hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize,
184 size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]) {
185 HOOK_PROLOGUE
186 hooks_internal_t hook;
187 FOR_EACH_HOOK_BEGIN(&hook)
188 hook_expand h = hook.hooks.expand_hook;
189 if (h != NULL) {
190 h(hook.hooks.extra, type, address, old_usize, new_usize,
191 result_raw, args_raw);
192 }
193 FOR_EACH_HOOK_END
194 HOOK_EPILOGUE
195}
diff --git a/examples/redis-unstable/deps/jemalloc/src/hpa.c b/examples/redis-unstable/deps/jemalloc/src/hpa.c
deleted file mode 100644
index 7e2aeba..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/hpa.c
+++ /dev/null
@@ -1,1044 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/hpa.h"
5
6#include "jemalloc/internal/fb.h"
7#include "jemalloc/internal/witness.h"
8
9#define HPA_EDEN_SIZE (128 * HUGEPAGE)
10
11static edata_t *hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
12 size_t alignment, bool zero, bool guarded, bool frequent_reuse,
13 bool *deferred_work_generated);
14static size_t hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size,
15 size_t nallocs, edata_list_active_t *results, bool *deferred_work_generated);
16static bool hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
17 size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
18static bool hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
19 size_t old_size, size_t new_size, bool *deferred_work_generated);
20static void hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata,
21 bool *deferred_work_generated);
22static void hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self,
23 edata_list_active_t *list, bool *deferred_work_generated);
24static uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self);
25
26bool
27hpa_supported() {
28#ifdef _WIN32
29 /*
30 * At least until the API and implementation is somewhat settled, we
31 * don't want to try to debug the VM subsystem on the hardest-to-test
32 * platform.
33 */
34 return false;
35#endif
36 if (!pages_can_hugify) {
37 return false;
38 }
39 /*
40 * We fundamentally rely on a address-space-hungry growth strategy for
41 * hugepages.
42 */
43 if (LG_SIZEOF_PTR != 3) {
44 return false;
45 }
46 /*
47 * If we couldn't detect the value of HUGEPAGE, HUGEPAGE_PAGES becomes
48 * this sentinel value -- see the comment in pages.h.
49 */
50 if (HUGEPAGE_PAGES == 1) {
51 return false;
52 }
53 return true;
54}
55
56static void
57hpa_do_consistency_checks(hpa_shard_t *shard) {
58 assert(shard->base != NULL);
59}
60
61bool
62hpa_central_init(hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks) {
63 /* malloc_conf processing should have filtered out these cases. */
64 assert(hpa_supported());
65 bool err;
66 err = malloc_mutex_init(&central->grow_mtx, "hpa_central_grow",
67 WITNESS_RANK_HPA_CENTRAL_GROW, malloc_mutex_rank_exclusive);
68 if (err) {
69 return true;
70 }
71 err = malloc_mutex_init(&central->mtx, "hpa_central",
72 WITNESS_RANK_HPA_CENTRAL, malloc_mutex_rank_exclusive);
73 if (err) {
74 return true;
75 }
76 central->base = base;
77 central->eden = NULL;
78 central->eden_len = 0;
79 central->age_counter = 0;
80 central->hooks = *hooks;
81 return false;
82}
83
84static hpdata_t *
85hpa_alloc_ps(tsdn_t *tsdn, hpa_central_t *central) {
86 return (hpdata_t *)base_alloc(tsdn, central->base, sizeof(hpdata_t),
87 CACHELINE);
88}
89
90hpdata_t *
91hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size,
92 bool *oom) {
93 /* Don't yet support big allocations; these should get filtered out. */
94 assert(size <= HUGEPAGE);
95 /*
96 * Should only try to extract from the central allocator if the local
97 * shard is exhausted. We should hold the grow_mtx on that shard.
98 */
99 witness_assert_positive_depth_to_rank(
100 tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_HPA_SHARD_GROW);
101
102 malloc_mutex_lock(tsdn, &central->grow_mtx);
103 *oom = false;
104
105 hpdata_t *ps = NULL;
106
107 /* Is eden a perfect fit? */
108 if (central->eden != NULL && central->eden_len == HUGEPAGE) {
109 ps = hpa_alloc_ps(tsdn, central);
110 if (ps == NULL) {
111 *oom = true;
112 malloc_mutex_unlock(tsdn, &central->grow_mtx);
113 return NULL;
114 }
115 hpdata_init(ps, central->eden, central->age_counter++);
116 central->eden = NULL;
117 central->eden_len = 0;
118 malloc_mutex_unlock(tsdn, &central->grow_mtx);
119 return ps;
120 }
121
122 /*
123 * We're about to try to allocate from eden by splitting. If eden is
124 * NULL, we have to allocate it too. Otherwise, we just have to
125 * allocate an edata_t for the new psset.
126 */
127 if (central->eden == NULL) {
128 /*
129 * During development, we're primarily concerned with systems
130 * with overcommit. Eventually, we should be more careful here.
131 */
132 bool commit = true;
133 /* Allocate address space, bailing if we fail. */
134 void *new_eden = pages_map(NULL, HPA_EDEN_SIZE, HUGEPAGE,
135 &commit);
136 if (new_eden == NULL) {
137 *oom = true;
138 malloc_mutex_unlock(tsdn, &central->grow_mtx);
139 return NULL;
140 }
141 ps = hpa_alloc_ps(tsdn, central);
142 if (ps == NULL) {
143 pages_unmap(new_eden, HPA_EDEN_SIZE);
144 *oom = true;
145 malloc_mutex_unlock(tsdn, &central->grow_mtx);
146 return NULL;
147 }
148 central->eden = new_eden;
149 central->eden_len = HPA_EDEN_SIZE;
150 } else {
151 /* Eden is already nonempty; only need an edata for ps. */
152 ps = hpa_alloc_ps(tsdn, central);
153 if (ps == NULL) {
154 *oom = true;
155 malloc_mutex_unlock(tsdn, &central->grow_mtx);
156 return NULL;
157 }
158 }
159 assert(ps != NULL);
160 assert(central->eden != NULL);
161 assert(central->eden_len > HUGEPAGE);
162 assert(central->eden_len % HUGEPAGE == 0);
163 assert(HUGEPAGE_ADDR2BASE(central->eden) == central->eden);
164
165 hpdata_init(ps, central->eden, central->age_counter++);
166
167 char *eden_char = (char *)central->eden;
168 eden_char += HUGEPAGE;
169 central->eden = (void *)eden_char;
170 central->eden_len -= HUGEPAGE;
171
172 malloc_mutex_unlock(tsdn, &central->grow_mtx);
173
174 return ps;
175}
176
177bool
178hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
179 base_t *base, edata_cache_t *edata_cache, unsigned ind,
180 const hpa_shard_opts_t *opts) {
181 /* malloc_conf processing should have filtered out these cases. */
182 assert(hpa_supported());
183 bool err;
184 err = malloc_mutex_init(&shard->grow_mtx, "hpa_shard_grow",
185 WITNESS_RANK_HPA_SHARD_GROW, malloc_mutex_rank_exclusive);
186 if (err) {
187 return true;
188 }
189 err = malloc_mutex_init(&shard->mtx, "hpa_shard",
190 WITNESS_RANK_HPA_SHARD, malloc_mutex_rank_exclusive);
191 if (err) {
192 return true;
193 }
194
195 assert(edata_cache != NULL);
196 shard->central = central;
197 shard->base = base;
198 edata_cache_fast_init(&shard->ecf, edata_cache);
199 psset_init(&shard->psset);
200 shard->age_counter = 0;
201 shard->ind = ind;
202 shard->emap = emap;
203
204 shard->opts = *opts;
205
206 shard->npending_purge = 0;
207 nstime_init_zero(&shard->last_purge);
208
209 shard->stats.npurge_passes = 0;
210 shard->stats.npurges = 0;
211 shard->stats.nhugifies = 0;
212 shard->stats.ndehugifies = 0;
213
214 /*
215 * Fill these in last, so that if an hpa_shard gets used despite
216 * initialization failing, we'll at least crash instead of just
217 * operating on corrupted data.
218 */
219 shard->pai.alloc = &hpa_alloc;
220 shard->pai.alloc_batch = &hpa_alloc_batch;
221 shard->pai.expand = &hpa_expand;
222 shard->pai.shrink = &hpa_shrink;
223 shard->pai.dalloc = &hpa_dalloc;
224 shard->pai.dalloc_batch = &hpa_dalloc_batch;
225 shard->pai.time_until_deferred_work = &hpa_time_until_deferred_work;
226
227 hpa_do_consistency_checks(shard);
228
229 return false;
230}
231
232/*
233 * Note that the stats functions here follow the usual stats naming conventions;
234 * "merge" obtains the stats from some live object of instance, while "accum"
235 * only combines the stats from one stats objet to another. Hence the lack of
236 * locking here.
237 */
238static void
239hpa_shard_nonderived_stats_accum(hpa_shard_nonderived_stats_t *dst,
240 hpa_shard_nonderived_stats_t *src) {
241 dst->npurge_passes += src->npurge_passes;
242 dst->npurges += src->npurges;
243 dst->nhugifies += src->nhugifies;
244 dst->ndehugifies += src->ndehugifies;
245}
246
247void
248hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src) {
249 psset_stats_accum(&dst->psset_stats, &src->psset_stats);
250 hpa_shard_nonderived_stats_accum(&dst->nonderived_stats,
251 &src->nonderived_stats);
252}
253
254void
255hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard,
256 hpa_shard_stats_t *dst) {
257 hpa_do_consistency_checks(shard);
258
259 malloc_mutex_lock(tsdn, &shard->grow_mtx);
260 malloc_mutex_lock(tsdn, &shard->mtx);
261 psset_stats_accum(&dst->psset_stats, &shard->psset.stats);
262 hpa_shard_nonderived_stats_accum(&dst->nonderived_stats, &shard->stats);
263 malloc_mutex_unlock(tsdn, &shard->mtx);
264 malloc_mutex_unlock(tsdn, &shard->grow_mtx);
265}
266
267static bool
268hpa_good_hugification_candidate(hpa_shard_t *shard, hpdata_t *ps) {
269 /*
270 * Note that this needs to be >= rather than just >, because of the
271 * important special case in which the hugification threshold is exactly
272 * HUGEPAGE.
273 */
274 return hpdata_nactive_get(ps) * PAGE
275 >= shard->opts.hugification_threshold;
276}
277
278static size_t
279hpa_adjusted_ndirty(tsdn_t *tsdn, hpa_shard_t *shard) {
280 malloc_mutex_assert_owner(tsdn, &shard->mtx);
281 return psset_ndirty(&shard->psset) - shard->npending_purge;
282}
283
284static size_t
285hpa_ndirty_max(tsdn_t *tsdn, hpa_shard_t *shard) {
286 malloc_mutex_assert_owner(tsdn, &shard->mtx);
287 if (shard->opts.dirty_mult == (fxp_t)-1) {
288 return (size_t)-1;
289 }
290 return fxp_mul_frac(psset_nactive(&shard->psset),
291 shard->opts.dirty_mult);
292}
293
294static bool
295hpa_hugify_blocked_by_ndirty(tsdn_t *tsdn, hpa_shard_t *shard) {
296 malloc_mutex_assert_owner(tsdn, &shard->mtx);
297 hpdata_t *to_hugify = psset_pick_hugify(&shard->psset);
298 if (to_hugify == NULL) {
299 return false;
300 }
301 return hpa_adjusted_ndirty(tsdn, shard)
302 + hpdata_nretained_get(to_hugify) > hpa_ndirty_max(tsdn, shard);
303}
304
305static bool
306hpa_should_purge(tsdn_t *tsdn, hpa_shard_t *shard) {
307 malloc_mutex_assert_owner(tsdn, &shard->mtx);
308 if (hpa_adjusted_ndirty(tsdn, shard) > hpa_ndirty_max(tsdn, shard)) {
309 return true;
310 }
311 if (hpa_hugify_blocked_by_ndirty(tsdn, shard)) {
312 return true;
313 }
314 return false;
315}
316
317static void
318hpa_update_purge_hugify_eligibility(tsdn_t *tsdn, hpa_shard_t *shard,
319 hpdata_t *ps) {
320 malloc_mutex_assert_owner(tsdn, &shard->mtx);
321 if (hpdata_changing_state_get(ps)) {
322 hpdata_purge_allowed_set(ps, false);
323 hpdata_disallow_hugify(ps);
324 return;
325 }
326 /*
327 * Hugepages are distinctly costly to purge, so try to avoid it unless
328 * they're *particularly* full of dirty pages. Eventually, we should
329 * use a smarter / more dynamic heuristic for situations where we have
330 * to manually hugify.
331 *
332 * In situations where we don't manually hugify, this problem is
333 * reduced. The "bad" situation we're trying to avoid is one's that's
334 * common in some Linux configurations (where both enabled and defrag
335 * are set to madvise) that can lead to long latency spikes on the first
336 * access after a hugification. The ideal policy in such configurations
337 * is probably time-based for both purging and hugifying; only hugify a
338 * hugepage if it's met the criteria for some extended period of time,
339 * and only dehugify it if it's failed to meet the criteria for an
340 * extended period of time. When background threads are on, we should
341 * try to take this hit on one of them, as well.
342 *
343 * I think the ideal setting is THP always enabled, and defrag set to
344 * deferred; in that case we don't need any explicit calls on the
345 * allocator's end at all; we just try to pack allocations in a
346 * hugepage-friendly manner and let the OS hugify in the background.
347 */
348 hpdata_purge_allowed_set(ps, hpdata_ndirty_get(ps) > 0);
349 if (hpa_good_hugification_candidate(shard, ps)
350 && !hpdata_huge_get(ps)) {
351 nstime_t now;
352 shard->central->hooks.curtime(&now, /* first_reading */ true);
353 hpdata_allow_hugify(ps, now);
354 }
355 /*
356 * Once a hugepage has become eligible for hugification, we don't mark
357 * it as ineligible just because it stops meeting the criteria (this
358 * could lead to situations where a hugepage that spends most of its
359 * time meeting the criteria never quite getting hugified if there are
360 * intervening deallocations). The idea is that the hugification delay
361 * will allow them to get purged, reseting their "hugify-allowed" bit.
362 * If they don't get purged, then the hugification isn't hurting and
363 * might help. As an exception, we don't hugify hugepages that are now
364 * empty; it definitely doesn't help there until the hugepage gets
365 * reused, which is likely not for a while.
366 */
367 if (hpdata_nactive_get(ps) == 0) {
368 hpdata_disallow_hugify(ps);
369 }
370}
371
372static bool
373hpa_shard_has_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
374 malloc_mutex_assert_owner(tsdn, &shard->mtx);
375 hpdata_t *to_hugify = psset_pick_hugify(&shard->psset);
376 return to_hugify != NULL || hpa_should_purge(tsdn, shard);
377}
378
379/* Returns whether or not we purged anything. */
380static bool
381hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) {
382 malloc_mutex_assert_owner(tsdn, &shard->mtx);
383
384 hpdata_t *to_purge = psset_pick_purge(&shard->psset);
385 if (to_purge == NULL) {
386 return false;
387 }
388 assert(hpdata_purge_allowed_get(to_purge));
389 assert(!hpdata_changing_state_get(to_purge));
390
391 /*
392 * Don't let anyone else purge or hugify this page while
393 * we're purging it (allocations and deallocations are
394 * OK).
395 */
396 psset_update_begin(&shard->psset, to_purge);
397 assert(hpdata_alloc_allowed_get(to_purge));
398 hpdata_mid_purge_set(to_purge, true);
399 hpdata_purge_allowed_set(to_purge, false);
400 hpdata_disallow_hugify(to_purge);
401 /*
402 * Unlike with hugification (where concurrent
403 * allocations are allowed), concurrent allocation out
404 * of a hugepage being purged is unsafe; we might hand
405 * out an extent for an allocation and then purge it
406 * (clearing out user data).
407 */
408 hpdata_alloc_allowed_set(to_purge, false);
409 psset_update_end(&shard->psset, to_purge);
410
411 /* Gather all the metadata we'll need during the purge. */
412 bool dehugify = hpdata_huge_get(to_purge);
413 hpdata_purge_state_t purge_state;
414 size_t num_to_purge = hpdata_purge_begin(to_purge, &purge_state);
415
416 shard->npending_purge += num_to_purge;
417
418 malloc_mutex_unlock(tsdn, &shard->mtx);
419
420 /* Actually do the purging, now that the lock is dropped. */
421 if (dehugify) {
422 shard->central->hooks.dehugify(hpdata_addr_get(to_purge),
423 HUGEPAGE);
424 }
425 size_t total_purged = 0;
426 uint64_t purges_this_pass = 0;
427 void *purge_addr;
428 size_t purge_size;
429 while (hpdata_purge_next(to_purge, &purge_state, &purge_addr,
430 &purge_size)) {
431 total_purged += purge_size;
432 assert(total_purged <= HUGEPAGE);
433 purges_this_pass++;
434 shard->central->hooks.purge(purge_addr, purge_size);
435 }
436
437 malloc_mutex_lock(tsdn, &shard->mtx);
438 /* The shard updates */
439 shard->npending_purge -= num_to_purge;
440 shard->stats.npurge_passes++;
441 shard->stats.npurges += purges_this_pass;
442 shard->central->hooks.curtime(&shard->last_purge,
443 /* first_reading */ false);
444 if (dehugify) {
445 shard->stats.ndehugifies++;
446 }
447
448 /* The hpdata updates. */
449 psset_update_begin(&shard->psset, to_purge);
450 if (dehugify) {
451 hpdata_dehugify(to_purge);
452 }
453 hpdata_purge_end(to_purge, &purge_state);
454 hpdata_mid_purge_set(to_purge, false);
455
456 hpdata_alloc_allowed_set(to_purge, true);
457 hpa_update_purge_hugify_eligibility(tsdn, shard, to_purge);
458
459 psset_update_end(&shard->psset, to_purge);
460
461 return true;
462}
463
464/* Returns whether or not we hugified anything. */
465static bool
466hpa_try_hugify(tsdn_t *tsdn, hpa_shard_t *shard) {
467 malloc_mutex_assert_owner(tsdn, &shard->mtx);
468
469 if (hpa_hugify_blocked_by_ndirty(tsdn, shard)) {
470 return false;
471 }
472
473 hpdata_t *to_hugify = psset_pick_hugify(&shard->psset);
474 if (to_hugify == NULL) {
475 return false;
476 }
477 assert(hpdata_hugify_allowed_get(to_hugify));
478 assert(!hpdata_changing_state_get(to_hugify));
479
480 /* Make sure that it's been hugifiable for long enough. */
481 nstime_t time_hugify_allowed = hpdata_time_hugify_allowed(to_hugify);
482 uint64_t millis = shard->central->hooks.ms_since(&time_hugify_allowed);
483 if (millis < shard->opts.hugify_delay_ms) {
484 return false;
485 }
486
487 /*
488 * Don't let anyone else purge or hugify this page while
489 * we're hugifying it (allocations and deallocations are
490 * OK).
491 */
492 psset_update_begin(&shard->psset, to_hugify);
493 hpdata_mid_hugify_set(to_hugify, true);
494 hpdata_purge_allowed_set(to_hugify, false);
495 hpdata_disallow_hugify(to_hugify);
496 assert(hpdata_alloc_allowed_get(to_hugify));
497 psset_update_end(&shard->psset, to_hugify);
498
499 malloc_mutex_unlock(tsdn, &shard->mtx);
500
501 shard->central->hooks.hugify(hpdata_addr_get(to_hugify), HUGEPAGE);
502
503 malloc_mutex_lock(tsdn, &shard->mtx);
504 shard->stats.nhugifies++;
505
506 psset_update_begin(&shard->psset, to_hugify);
507 hpdata_hugify(to_hugify);
508 hpdata_mid_hugify_set(to_hugify, false);
509 hpa_update_purge_hugify_eligibility(tsdn, shard, to_hugify);
510 psset_update_end(&shard->psset, to_hugify);
511
512 return true;
513}
514
515/*
516 * Execution of deferred work is forced if it's triggered by an explicit
517 * hpa_shard_do_deferred_work() call.
518 */
519static void
520hpa_shard_maybe_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard,
521 bool forced) {
522 malloc_mutex_assert_owner(tsdn, &shard->mtx);
523 if (!forced && shard->opts.deferral_allowed) {
524 return;
525 }
526 /*
527 * If we're on a background thread, do work so long as there's work to
528 * be done. Otherwise, bound latency to not be *too* bad by doing at
529 * most a small fixed number of operations.
530 */
531 bool hugified = false;
532 bool purged = false;
533 size_t max_ops = (forced ? (size_t)-1 : 16);
534 size_t nops = 0;
535 do {
536 /*
537 * Always purge before hugifying, to make sure we get some
538 * ability to hit our quiescence targets.
539 */
540 purged = false;
541 while (hpa_should_purge(tsdn, shard) && nops < max_ops) {
542 purged = hpa_try_purge(tsdn, shard);
543 if (purged) {
544 nops++;
545 }
546 }
547 hugified = hpa_try_hugify(tsdn, shard);
548 if (hugified) {
549 nops++;
550 }
551 malloc_mutex_assert_owner(tsdn, &shard->mtx);
552 malloc_mutex_assert_owner(tsdn, &shard->mtx);
553 } while ((hugified || purged) && nops < max_ops);
554}
555
556static edata_t *
557hpa_try_alloc_one_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
558 bool *oom) {
559 bool err;
560 edata_t *edata = edata_cache_fast_get(tsdn, &shard->ecf);
561 if (edata == NULL) {
562 *oom = true;
563 return NULL;
564 }
565
566 hpdata_t *ps = psset_pick_alloc(&shard->psset, size);
567 if (ps == NULL) {
568 edata_cache_fast_put(tsdn, &shard->ecf, edata);
569 return NULL;
570 }
571
572 psset_update_begin(&shard->psset, ps);
573
574 if (hpdata_empty(ps)) {
575 /*
576 * If the pageslab used to be empty, treat it as though it's
577 * brand new for fragmentation-avoidance purposes; what we're
578 * trying to approximate is the age of the allocations *in* that
579 * pageslab, and the allocations in the new pageslab are
580 * definitionally the youngest in this hpa shard.
581 */
582 hpdata_age_set(ps, shard->age_counter++);
583 }
584
585 void *addr = hpdata_reserve_alloc(ps, size);
586 edata_init(edata, shard->ind, addr, size, /* slab */ false,
587 SC_NSIZES, /* sn */ hpdata_age_get(ps), extent_state_active,
588 /* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA,
589 EXTENT_NOT_HEAD);
590 edata_ps_set(edata, ps);
591
592 /*
593 * This could theoretically be moved outside of the critical section,
594 * but that introduces the potential for a race. Without the lock, the
595 * (initially nonempty, since this is the reuse pathway) pageslab we
596 * allocated out of could become otherwise empty while the lock is
597 * dropped. This would force us to deal with a pageslab eviction down
598 * the error pathway, which is a pain.
599 */
600 err = emap_register_boundary(tsdn, shard->emap, edata,
601 SC_NSIZES, /* slab */ false);
602 if (err) {
603 hpdata_unreserve(ps, edata_addr_get(edata),
604 edata_size_get(edata));
605 /*
606 * We should arguably reset dirty state here, but this would
607 * require some sort of prepare + commit functionality that's a
608 * little much to deal with for now.
609 *
610 * We don't have a do_deferred_work down this pathway, on the
611 * principle that we didn't *really* affect shard state (we
612 * tweaked the stats, but our tweaks weren't really accurate).
613 */
614 psset_update_end(&shard->psset, ps);
615 edata_cache_fast_put(tsdn, &shard->ecf, edata);
616 *oom = true;
617 return NULL;
618 }
619
620 hpa_update_purge_hugify_eligibility(tsdn, shard, ps);
621 psset_update_end(&shard->psset, ps);
622 return edata;
623}
624
625static size_t
626hpa_try_alloc_batch_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
627 bool *oom, size_t nallocs, edata_list_active_t *results,
628 bool *deferred_work_generated) {
629 malloc_mutex_lock(tsdn, &shard->mtx);
630 size_t nsuccess = 0;
631 for (; nsuccess < nallocs; nsuccess++) {
632 edata_t *edata = hpa_try_alloc_one_no_grow(tsdn, shard, size,
633 oom);
634 if (edata == NULL) {
635 break;
636 }
637 edata_list_active_append(results, edata);
638 }
639
640 hpa_shard_maybe_do_deferred_work(tsdn, shard, /* forced */ false);
641 *deferred_work_generated = hpa_shard_has_deferred_work(tsdn, shard);
642 malloc_mutex_unlock(tsdn, &shard->mtx);
643 return nsuccess;
644}
645
646static size_t
647hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
648 size_t nallocs, edata_list_active_t *results,
649 bool *deferred_work_generated) {
650 assert(size <= shard->opts.slab_max_alloc);
651 bool oom = false;
652
653 size_t nsuccess = hpa_try_alloc_batch_no_grow(tsdn, shard, size, &oom,
654 nallocs, results, deferred_work_generated);
655
656 if (nsuccess == nallocs || oom) {
657 return nsuccess;
658 }
659
660 /*
661 * We didn't OOM, but weren't able to fill everything requested of us;
662 * try to grow.
663 */
664 malloc_mutex_lock(tsdn, &shard->grow_mtx);
665 /*
666 * Check for grow races; maybe some earlier thread expanded the psset
667 * in between when we dropped the main mutex and grabbed the grow mutex.
668 */
669 nsuccess += hpa_try_alloc_batch_no_grow(tsdn, shard, size, &oom,
670 nallocs - nsuccess, results, deferred_work_generated);
671 if (nsuccess == nallocs || oom) {
672 malloc_mutex_unlock(tsdn, &shard->grow_mtx);
673 return nsuccess;
674 }
675
676 /*
677 * Note that we don't hold shard->mtx here (while growing);
678 * deallocations (and allocations of smaller sizes) may still succeed
679 * while we're doing this potentially expensive system call.
680 */
681 hpdata_t *ps = hpa_central_extract(tsdn, shard->central, size, &oom);
682 if (ps == NULL) {
683 malloc_mutex_unlock(tsdn, &shard->grow_mtx);
684 return nsuccess;
685 }
686
687 /*
688 * We got the pageslab; allocate from it. This does an unlock followed
689 * by a lock on the same mutex, and holds the grow mutex while doing
690 * deferred work, but this is an uncommon path; the simplicity is worth
691 * it.
692 */
693 malloc_mutex_lock(tsdn, &shard->mtx);
694 psset_insert(&shard->psset, ps);
695 malloc_mutex_unlock(tsdn, &shard->mtx);
696
697 nsuccess += hpa_try_alloc_batch_no_grow(tsdn, shard, size, &oom,
698 nallocs - nsuccess, results, deferred_work_generated);
699 /*
700 * Drop grow_mtx before doing deferred work; other threads blocked on it
701 * should be allowed to proceed while we're working.
702 */
703 malloc_mutex_unlock(tsdn, &shard->grow_mtx);
704
705 return nsuccess;
706}
707
708static hpa_shard_t *
709hpa_from_pai(pai_t *self) {
710 assert(self->alloc = &hpa_alloc);
711 assert(self->expand = &hpa_expand);
712 assert(self->shrink = &hpa_shrink);
713 assert(self->dalloc = &hpa_dalloc);
714 return (hpa_shard_t *)self;
715}
716
717static size_t
718hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
719 edata_list_active_t *results, bool *deferred_work_generated) {
720 assert(nallocs > 0);
721 assert((size & PAGE_MASK) == 0);
722 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
723 WITNESS_RANK_CORE, 0);
724 hpa_shard_t *shard = hpa_from_pai(self);
725
726 if (size > shard->opts.slab_max_alloc) {
727 return 0;
728 }
729
730 size_t nsuccess = hpa_alloc_batch_psset(tsdn, shard, size, nallocs,
731 results, deferred_work_generated);
732
733 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
734 WITNESS_RANK_CORE, 0);
735
736 /*
737 * Guard the sanity checks with config_debug because the loop cannot be
738 * proven non-circular by the compiler, even if everything within the
739 * loop is optimized away.
740 */
741 if (config_debug) {
742 edata_t *edata;
743 ql_foreach(edata, &results->head, ql_link_active) {
744 emap_assert_mapped(tsdn, shard->emap, edata);
745 assert(edata_pai_get(edata) == EXTENT_PAI_HPA);
746 assert(edata_state_get(edata) == extent_state_active);
747 assert(edata_arena_ind_get(edata) == shard->ind);
748 assert(edata_szind_get_maybe_invalid(edata) ==
749 SC_NSIZES);
750 assert(!edata_slab_get(edata));
751 assert(edata_committed_get(edata));
752 assert(edata_base_get(edata) == edata_addr_get(edata));
753 assert(edata_base_get(edata) != NULL);
754 }
755 }
756 return nsuccess;
757}
758
759static edata_t *
760hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
761 bool guarded, bool frequent_reuse, bool *deferred_work_generated) {
762 assert((size & PAGE_MASK) == 0);
763 assert(!guarded);
764 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
765 WITNESS_RANK_CORE, 0);
766
767 /* We don't handle alignment or zeroing for now. */
768 if (alignment > PAGE || zero) {
769 return NULL;
770 }
771 /*
772 * An alloc with alignment == PAGE and zero == false is equivalent to a
773 * batch alloc of 1. Just do that, so we can share code.
774 */
775 edata_list_active_t results;
776 edata_list_active_init(&results);
777 size_t nallocs = hpa_alloc_batch(tsdn, self, size, /* nallocs */ 1,
778 &results, deferred_work_generated);
779 assert(nallocs == 0 || nallocs == 1);
780 edata_t *edata = edata_list_active_first(&results);
781 return edata;
782}
783
784static bool
785hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
786 size_t new_size, bool zero, bool *deferred_work_generated) {
787 /* Expand not yet supported. */
788 return true;
789}
790
791static bool
792hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
793 size_t old_size, size_t new_size, bool *deferred_work_generated) {
794 /* Shrink not yet supported. */
795 return true;
796}
797
798static void
799hpa_dalloc_prepare_unlocked(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata) {
800 malloc_mutex_assert_not_owner(tsdn, &shard->mtx);
801
802 assert(edata_pai_get(edata) == EXTENT_PAI_HPA);
803 assert(edata_state_get(edata) == extent_state_active);
804 assert(edata_arena_ind_get(edata) == shard->ind);
805 assert(edata_szind_get_maybe_invalid(edata) == SC_NSIZES);
806 assert(edata_committed_get(edata));
807 assert(edata_base_get(edata) != NULL);
808
809 /*
810 * Another thread shouldn't be trying to touch the metadata of an
811 * allocation being freed. The one exception is a merge attempt from a
812 * lower-addressed PAC extent; in this case we have a nominal race on
813 * the edata metadata bits, but in practice the fact that the PAI bits
814 * are different will prevent any further access. The race is bad, but
815 * benign in practice, and the long term plan is to track enough state
816 * in the rtree to prevent these merge attempts in the first place.
817 */
818 edata_addr_set(edata, edata_base_get(edata));
819 edata_zeroed_set(edata, false);
820 emap_deregister_boundary(tsdn, shard->emap, edata);
821}
822
823static void
824hpa_dalloc_locked(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata) {
825 malloc_mutex_assert_owner(tsdn, &shard->mtx);
826
827 /*
828 * Release the metadata early, to avoid having to remember to do it
829 * while we're also doing tricky purging logic. First, we need to grab
830 * a few bits of metadata from it.
831 *
832 * Note that the shard mutex protects ps's metadata too; it wouldn't be
833 * correct to try to read most information out of it without the lock.
834 */
835 hpdata_t *ps = edata_ps_get(edata);
836 /* Currently, all edatas come from pageslabs. */
837 assert(ps != NULL);
838 void *unreserve_addr = edata_addr_get(edata);
839 size_t unreserve_size = edata_size_get(edata);
840 edata_cache_fast_put(tsdn, &shard->ecf, edata);
841
842 psset_update_begin(&shard->psset, ps);
843 hpdata_unreserve(ps, unreserve_addr, unreserve_size);
844 hpa_update_purge_hugify_eligibility(tsdn, shard, ps);
845 psset_update_end(&shard->psset, ps);
846}
847
848static void
849hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list,
850 bool *deferred_work_generated) {
851 hpa_shard_t *shard = hpa_from_pai(self);
852
853 edata_t *edata;
854 ql_foreach(edata, &list->head, ql_link_active) {
855 hpa_dalloc_prepare_unlocked(tsdn, shard, edata);
856 }
857
858 malloc_mutex_lock(tsdn, &shard->mtx);
859 /* Now, remove from the list. */
860 while ((edata = edata_list_active_first(list)) != NULL) {
861 edata_list_active_remove(list, edata);
862 hpa_dalloc_locked(tsdn, shard, edata);
863 }
864 hpa_shard_maybe_do_deferred_work(tsdn, shard, /* forced */ false);
865 *deferred_work_generated =
866 hpa_shard_has_deferred_work(tsdn, shard);
867
868 malloc_mutex_unlock(tsdn, &shard->mtx);
869}
870
871static void
872hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata,
873 bool *deferred_work_generated) {
874 assert(!edata_guarded_get(edata));
875 /* Just a dalloc_batch of size 1; this lets us share logic. */
876 edata_list_active_t dalloc_list;
877 edata_list_active_init(&dalloc_list);
878 edata_list_active_append(&dalloc_list, edata);
879 hpa_dalloc_batch(tsdn, self, &dalloc_list, deferred_work_generated);
880}
881
882/*
883 * Calculate time until either purging or hugification ought to happen.
884 * Called by background threads.
885 */
886static uint64_t
887hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
888 hpa_shard_t *shard = hpa_from_pai(self);
889 uint64_t time_ns = BACKGROUND_THREAD_DEFERRED_MAX;
890
891 malloc_mutex_lock(tsdn, &shard->mtx);
892
893 hpdata_t *to_hugify = psset_pick_hugify(&shard->psset);
894 if (to_hugify != NULL) {
895 nstime_t time_hugify_allowed =
896 hpdata_time_hugify_allowed(to_hugify);
897 uint64_t since_hugify_allowed_ms =
898 shard->central->hooks.ms_since(&time_hugify_allowed);
899 /*
900 * If not enough time has passed since hugification was allowed,
901 * sleep for the rest.
902 */
903 if (since_hugify_allowed_ms < shard->opts.hugify_delay_ms) {
904 time_ns = shard->opts.hugify_delay_ms -
905 since_hugify_allowed_ms;
906 time_ns *= 1000 * 1000;
907 } else {
908 malloc_mutex_unlock(tsdn, &shard->mtx);
909 return BACKGROUND_THREAD_DEFERRED_MIN;
910 }
911 }
912
913 if (hpa_should_purge(tsdn, shard)) {
914 /*
915 * If we haven't purged before, no need to check interval
916 * between purges. Simply purge as soon as possible.
917 */
918 if (shard->stats.npurge_passes == 0) {
919 malloc_mutex_unlock(tsdn, &shard->mtx);
920 return BACKGROUND_THREAD_DEFERRED_MIN;
921 }
922 uint64_t since_last_purge_ms = shard->central->hooks.ms_since(
923 &shard->last_purge);
924
925 if (since_last_purge_ms < shard->opts.min_purge_interval_ms) {
926 uint64_t until_purge_ns;
927 until_purge_ns = shard->opts.min_purge_interval_ms -
928 since_last_purge_ms;
929 until_purge_ns *= 1000 * 1000;
930
931 if (until_purge_ns < time_ns) {
932 time_ns = until_purge_ns;
933 }
934 } else {
935 time_ns = BACKGROUND_THREAD_DEFERRED_MIN;
936 }
937 }
938 malloc_mutex_unlock(tsdn, &shard->mtx);
939 return time_ns;
940}
941
942void
943hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard) {
944 hpa_do_consistency_checks(shard);
945
946 malloc_mutex_lock(tsdn, &shard->mtx);
947 edata_cache_fast_disable(tsdn, &shard->ecf);
948 malloc_mutex_unlock(tsdn, &shard->mtx);
949}
950
951static void
952hpa_shard_assert_stats_empty(psset_bin_stats_t *bin_stats) {
953 assert(bin_stats->npageslabs == 0);
954 assert(bin_stats->nactive == 0);
955}
956
957static void
958hpa_assert_empty(tsdn_t *tsdn, hpa_shard_t *shard, psset_t *psset) {
959 malloc_mutex_assert_owner(tsdn, &shard->mtx);
960 for (int huge = 0; huge <= 1; huge++) {
961 hpa_shard_assert_stats_empty(&psset->stats.full_slabs[huge]);
962 for (pszind_t i = 0; i < PSSET_NPSIZES; i++) {
963 hpa_shard_assert_stats_empty(
964 &psset->stats.nonfull_slabs[i][huge]);
965 }
966 }
967}
968
969void
970hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard) {
971 hpa_do_consistency_checks(shard);
972 /*
973 * By the time we're here, the arena code should have dalloc'd all the
974 * active extents, which means we should have eventually evicted
975 * everything from the psset, so it shouldn't be able to serve even a
976 * 1-page allocation.
977 */
978 if (config_debug) {
979 malloc_mutex_lock(tsdn, &shard->mtx);
980 hpa_assert_empty(tsdn, shard, &shard->psset);
981 malloc_mutex_unlock(tsdn, &shard->mtx);
982 }
983 hpdata_t *ps;
984 while ((ps = psset_pick_alloc(&shard->psset, PAGE)) != NULL) {
985 /* There should be no allocations anywhere. */
986 assert(hpdata_empty(ps));
987 psset_remove(&shard->psset, ps);
988 shard->central->hooks.unmap(hpdata_addr_get(ps), HUGEPAGE);
989 }
990}
991
992void
993hpa_shard_set_deferral_allowed(tsdn_t *tsdn, hpa_shard_t *shard,
994 bool deferral_allowed) {
995 hpa_do_consistency_checks(shard);
996
997 malloc_mutex_lock(tsdn, &shard->mtx);
998 bool deferral_previously_allowed = shard->opts.deferral_allowed;
999 shard->opts.deferral_allowed = deferral_allowed;
1000 if (deferral_previously_allowed && !deferral_allowed) {
1001 hpa_shard_maybe_do_deferred_work(tsdn, shard,
1002 /* forced */ true);
1003 }
1004 malloc_mutex_unlock(tsdn, &shard->mtx);
1005}
1006
1007void
1008hpa_shard_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
1009 hpa_do_consistency_checks(shard);
1010
1011 malloc_mutex_lock(tsdn, &shard->mtx);
1012 hpa_shard_maybe_do_deferred_work(tsdn, shard, /* forced */ true);
1013 malloc_mutex_unlock(tsdn, &shard->mtx);
1014}
1015
1016void
1017hpa_shard_prefork3(tsdn_t *tsdn, hpa_shard_t *shard) {
1018 hpa_do_consistency_checks(shard);
1019
1020 malloc_mutex_prefork(tsdn, &shard->grow_mtx);
1021}
1022
1023void
1024hpa_shard_prefork4(tsdn_t *tsdn, hpa_shard_t *shard) {
1025 hpa_do_consistency_checks(shard);
1026
1027 malloc_mutex_prefork(tsdn, &shard->mtx);
1028}
1029
1030void
1031hpa_shard_postfork_parent(tsdn_t *tsdn, hpa_shard_t *shard) {
1032 hpa_do_consistency_checks(shard);
1033
1034 malloc_mutex_postfork_parent(tsdn, &shard->grow_mtx);
1035 malloc_mutex_postfork_parent(tsdn, &shard->mtx);
1036}
1037
1038void
1039hpa_shard_postfork_child(tsdn_t *tsdn, hpa_shard_t *shard) {
1040 hpa_do_consistency_checks(shard);
1041
1042 malloc_mutex_postfork_child(tsdn, &shard->grow_mtx);
1043 malloc_mutex_postfork_child(tsdn, &shard->mtx);
1044}
diff --git a/examples/redis-unstable/deps/jemalloc/src/hpa_hooks.c b/examples/redis-unstable/deps/jemalloc/src/hpa_hooks.c
deleted file mode 100644
index ade581e..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/hpa_hooks.c
+++ /dev/null
@@ -1,63 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/hpa_hooks.h"
5
6static void *hpa_hooks_map(size_t size);
7static void hpa_hooks_unmap(void *ptr, size_t size);
8static void hpa_hooks_purge(void *ptr, size_t size);
9static void hpa_hooks_hugify(void *ptr, size_t size);
10static void hpa_hooks_dehugify(void *ptr, size_t size);
11static void hpa_hooks_curtime(nstime_t *r_nstime, bool first_reading);
12static uint64_t hpa_hooks_ms_since(nstime_t *past_nstime);
13
14hpa_hooks_t hpa_hooks_default = {
15 &hpa_hooks_map,
16 &hpa_hooks_unmap,
17 &hpa_hooks_purge,
18 &hpa_hooks_hugify,
19 &hpa_hooks_dehugify,
20 &hpa_hooks_curtime,
21 &hpa_hooks_ms_since
22};
23
24static void *
25hpa_hooks_map(size_t size) {
26 bool commit = true;
27 return pages_map(NULL, size, HUGEPAGE, &commit);
28}
29
30static void
31hpa_hooks_unmap(void *ptr, size_t size) {
32 pages_unmap(ptr, size);
33}
34
35static void
36hpa_hooks_purge(void *ptr, size_t size) {
37 pages_purge_forced(ptr, size);
38}
39
40static void
41hpa_hooks_hugify(void *ptr, size_t size) {
42 bool err = pages_huge(ptr, size);
43 (void)err;
44}
45
46static void
47hpa_hooks_dehugify(void *ptr, size_t size) {
48 bool err = pages_nohuge(ptr, size);
49 (void)err;
50}
51
52static void
53hpa_hooks_curtime(nstime_t *r_nstime, bool first_reading) {
54 if (first_reading) {
55 nstime_init_zero(r_nstime);
56 }
57 nstime_update(r_nstime);
58}
59
60static uint64_t
61hpa_hooks_ms_since(nstime_t *past_nstime) {
62 return nstime_ns_since(past_nstime) / 1000 / 1000;
63}
diff --git a/examples/redis-unstable/deps/jemalloc/src/hpdata.c b/examples/redis-unstable/deps/jemalloc/src/hpdata.c
deleted file mode 100644
index e7d7294..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/hpdata.c
+++ /dev/null
@@ -1,325 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/hpdata.h"
5
6static int
7hpdata_age_comp(const hpdata_t *a, const hpdata_t *b) {
8 uint64_t a_age = hpdata_age_get(a);
9 uint64_t b_age = hpdata_age_get(b);
10 /*
11 * hpdata ages are operation counts in the psset; no two should be the
12 * same.
13 */
14 assert(a_age != b_age);
15 return (a_age > b_age) - (a_age < b_age);
16}
17
18ph_gen(, hpdata_age_heap, hpdata_t, age_link, hpdata_age_comp)
19
20void
21hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) {
22 hpdata_addr_set(hpdata, addr);
23 hpdata_age_set(hpdata, age);
24 hpdata->h_huge = false;
25 hpdata->h_alloc_allowed = true;
26 hpdata->h_in_psset_alloc_container = false;
27 hpdata->h_purge_allowed = false;
28 hpdata->h_hugify_allowed = false;
29 hpdata->h_in_psset_hugify_container = false;
30 hpdata->h_mid_purge = false;
31 hpdata->h_mid_hugify = false;
32 hpdata->h_updating = false;
33 hpdata->h_in_psset = false;
34 hpdata_longest_free_range_set(hpdata, HUGEPAGE_PAGES);
35 hpdata->h_nactive = 0;
36 fb_init(hpdata->active_pages, HUGEPAGE_PAGES);
37 hpdata->h_ntouched = 0;
38 fb_init(hpdata->touched_pages, HUGEPAGE_PAGES);
39
40 hpdata_assert_consistent(hpdata);
41}
42
43void *
44hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz) {
45 hpdata_assert_consistent(hpdata);
46 /*
47 * This is a metadata change; the hpdata should therefore either not be
48 * in the psset, or should have explicitly marked itself as being
49 * mid-update.
50 */
51 assert(!hpdata->h_in_psset || hpdata->h_updating);
52 assert(hpdata->h_alloc_allowed);
53 assert((sz & PAGE_MASK) == 0);
54 size_t npages = sz >> LG_PAGE;
55 assert(npages <= hpdata_longest_free_range_get(hpdata));
56
57 size_t result;
58
59 size_t start = 0;
60 /*
61 * These are dead stores, but the compiler will issue warnings on them
62 * since it can't tell statically that found is always true below.
63 */
64 size_t begin = 0;
65 size_t len = 0;
66
67 size_t largest_unchosen_range = 0;
68 while (true) {
69 bool found = fb_urange_iter(hpdata->active_pages,
70 HUGEPAGE_PAGES, start, &begin, &len);
71 /*
72 * A precondition to this function is that hpdata must be able
73 * to serve the allocation.
74 */
75 assert(found);
76 assert(len <= hpdata_longest_free_range_get(hpdata));
77 if (len >= npages) {
78 /*
79 * We use first-fit within the page slabs; this gives
80 * bounded worst-case fragmentation within a slab. It's
81 * not necessarily right; we could experiment with
82 * various other options.
83 */
84 break;
85 }
86 if (len > largest_unchosen_range) {
87 largest_unchosen_range = len;
88 }
89 start = begin + len;
90 }
91 /* We found a range; remember it. */
92 result = begin;
93 fb_set_range(hpdata->active_pages, HUGEPAGE_PAGES, begin, npages);
94 hpdata->h_nactive += npages;
95
96 /*
97 * We might be about to dirty some memory for the first time; update our
98 * count if so.
99 */
100 size_t new_dirty = fb_ucount(hpdata->touched_pages, HUGEPAGE_PAGES,
101 result, npages);
102 fb_set_range(hpdata->touched_pages, HUGEPAGE_PAGES, result, npages);
103 hpdata->h_ntouched += new_dirty;
104
105 /*
106 * If we allocated out of a range that was the longest in the hpdata, it
107 * might be the only one of that size and we'll have to adjust the
108 * metadata.
109 */
110 if (len == hpdata_longest_free_range_get(hpdata)) {
111 start = begin + npages;
112 while (start < HUGEPAGE_PAGES) {
113 bool found = fb_urange_iter(hpdata->active_pages,
114 HUGEPAGE_PAGES, start, &begin, &len);
115 if (!found) {
116 break;
117 }
118 assert(len <= hpdata_longest_free_range_get(hpdata));
119 if (len == hpdata_longest_free_range_get(hpdata)) {
120 largest_unchosen_range = len;
121 break;
122 }
123 if (len > largest_unchosen_range) {
124 largest_unchosen_range = len;
125 }
126 start = begin + len;
127 }
128 hpdata_longest_free_range_set(hpdata, largest_unchosen_range);
129 }
130
131 hpdata_assert_consistent(hpdata);
132 return (void *)(
133 (uintptr_t)hpdata_addr_get(hpdata) + (result << LG_PAGE));
134}
135
136void
137hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) {
138 hpdata_assert_consistent(hpdata);
139 /* See the comment in reserve. */
140 assert(!hpdata->h_in_psset || hpdata->h_updating);
141 assert(((uintptr_t)addr & PAGE_MASK) == 0);
142 assert((sz & PAGE_MASK) == 0);
143 size_t begin = ((uintptr_t)addr - (uintptr_t)hpdata_addr_get(hpdata))
144 >> LG_PAGE;
145 assert(begin < HUGEPAGE_PAGES);
146 size_t npages = sz >> LG_PAGE;
147 size_t old_longest_range = hpdata_longest_free_range_get(hpdata);
148
149 fb_unset_range(hpdata->active_pages, HUGEPAGE_PAGES, begin, npages);
150 /* We might have just created a new, larger range. */
151 size_t new_begin = (fb_fls(hpdata->active_pages, HUGEPAGE_PAGES,
152 begin) + 1);
153 size_t new_end = fb_ffs(hpdata->active_pages, HUGEPAGE_PAGES,
154 begin + npages - 1);
155 size_t new_range_len = new_end - new_begin;
156
157 if (new_range_len > old_longest_range) {
158 hpdata_longest_free_range_set(hpdata, new_range_len);
159 }
160
161 hpdata->h_nactive -= npages;
162
163 hpdata_assert_consistent(hpdata);
164}
165
166size_t
167hpdata_purge_begin(hpdata_t *hpdata, hpdata_purge_state_t *purge_state) {
168 hpdata_assert_consistent(hpdata);
169 /*
170 * See the comment below; we might purge any inactive extent, so it's
171 * unsafe for any other thread to turn any inactive extent active while
172 * we're operating on it.
173 */
174 assert(!hpdata_alloc_allowed_get(hpdata));
175
176 purge_state->npurged = 0;
177 purge_state->next_purge_search_begin = 0;
178
179 /*
180 * Initialize to_purge.
181 *
182 * It's possible to end up in situations where two dirty extents are
183 * separated by a retained extent:
184 * - 1 page allocated.
185 * - 1 page allocated.
186 * - 1 pages allocated.
187 *
188 * If the middle page is freed and purged, and then the first and third
189 * pages are freed, and then another purge pass happens, the hpdata
190 * looks like this:
191 * - 1 page dirty.
192 * - 1 page retained.
193 * - 1 page dirty.
194 *
195 * But it's safe to do a single 3-page purge.
196 *
197 * We do this by first computing the dirty pages, and then filling in
198 * any gaps by extending each range in the dirty bitmap to extend until
199 * the next active page. This purges more pages, but the expensive part
200 * of purging is the TLB shootdowns, rather than the kernel state
201 * tracking; doing a little bit more of the latter is fine if it saves
202 * us from doing some of the former.
203 */
204
205 /*
206 * The dirty pages are those that are touched but not active. Note that
207 * in a normal-ish case, HUGEPAGE_PAGES is something like 512 and the
208 * fb_group_t is 64 bits, so this is 64 bytes, spread across 8
209 * fb_group_ts.
210 */
211 fb_group_t dirty_pages[FB_NGROUPS(HUGEPAGE_PAGES)];
212 fb_init(dirty_pages, HUGEPAGE_PAGES);
213 fb_bit_not(dirty_pages, hpdata->active_pages, HUGEPAGE_PAGES);
214 fb_bit_and(dirty_pages, dirty_pages, hpdata->touched_pages,
215 HUGEPAGE_PAGES);
216
217 fb_init(purge_state->to_purge, HUGEPAGE_PAGES);
218 size_t next_bit = 0;
219 while (next_bit < HUGEPAGE_PAGES) {
220 size_t next_dirty = fb_ffs(dirty_pages, HUGEPAGE_PAGES,
221 next_bit);
222 /* Recall that fb_ffs returns nbits if no set bit is found. */
223 if (next_dirty == HUGEPAGE_PAGES) {
224 break;
225 }
226 size_t next_active = fb_ffs(hpdata->active_pages,
227 HUGEPAGE_PAGES, next_dirty);
228 /*
229 * Don't purge past the end of the dirty extent, into retained
230 * pages. This helps the kernel a tiny bit, but honestly it's
231 * mostly helpful for testing (where we tend to write test cases
232 * that think in terms of the dirty ranges).
233 */
234 ssize_t last_dirty = fb_fls(dirty_pages, HUGEPAGE_PAGES,
235 next_active - 1);
236 assert(last_dirty >= 0);
237 assert((size_t)last_dirty >= next_dirty);
238 assert((size_t)last_dirty - next_dirty + 1 <= HUGEPAGE_PAGES);
239
240 fb_set_range(purge_state->to_purge, HUGEPAGE_PAGES, next_dirty,
241 last_dirty - next_dirty + 1);
242 next_bit = next_active + 1;
243 }
244
245 /* We should purge, at least, everything dirty. */
246 size_t ndirty = hpdata->h_ntouched - hpdata->h_nactive;
247 purge_state->ndirty_to_purge = ndirty;
248 assert(ndirty <= fb_scount(
249 purge_state->to_purge, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES));
250 assert(ndirty == fb_scount(dirty_pages, HUGEPAGE_PAGES, 0,
251 HUGEPAGE_PAGES));
252
253 hpdata_assert_consistent(hpdata);
254
255 return ndirty;
256}
257
258bool
259hpdata_purge_next(hpdata_t *hpdata, hpdata_purge_state_t *purge_state,
260 void **r_purge_addr, size_t *r_purge_size) {
261 /*
262 * Note that we don't have a consistency check here; we're accessing
263 * hpdata without synchronization, and therefore have no right to expect
264 * a consistent state.
265 */
266 assert(!hpdata_alloc_allowed_get(hpdata));
267
268 if (purge_state->next_purge_search_begin == HUGEPAGE_PAGES) {
269 return false;
270 }
271 size_t purge_begin;
272 size_t purge_len;
273 bool found_range = fb_srange_iter(purge_state->to_purge, HUGEPAGE_PAGES,
274 purge_state->next_purge_search_begin, &purge_begin, &purge_len);
275 if (!found_range) {
276 return false;
277 }
278
279 *r_purge_addr = (void *)(
280 (uintptr_t)hpdata_addr_get(hpdata) + purge_begin * PAGE);
281 *r_purge_size = purge_len * PAGE;
282
283 purge_state->next_purge_search_begin = purge_begin + purge_len;
284 purge_state->npurged += purge_len;
285 assert(purge_state->npurged <= HUGEPAGE_PAGES);
286
287 return true;
288}
289
290void
291hpdata_purge_end(hpdata_t *hpdata, hpdata_purge_state_t *purge_state) {
292 assert(!hpdata_alloc_allowed_get(hpdata));
293 hpdata_assert_consistent(hpdata);
294 /* See the comment in reserve. */
295 assert(!hpdata->h_in_psset || hpdata->h_updating);
296
297 assert(purge_state->npurged == fb_scount(purge_state->to_purge,
298 HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES));
299 assert(purge_state->npurged >= purge_state->ndirty_to_purge);
300
301 fb_bit_not(purge_state->to_purge, purge_state->to_purge,
302 HUGEPAGE_PAGES);
303 fb_bit_and(hpdata->touched_pages, hpdata->touched_pages,
304 purge_state->to_purge, HUGEPAGE_PAGES);
305 assert(hpdata->h_ntouched >= purge_state->ndirty_to_purge);
306 hpdata->h_ntouched -= purge_state->ndirty_to_purge;
307
308 hpdata_assert_consistent(hpdata);
309}
310
311void
312hpdata_hugify(hpdata_t *hpdata) {
313 hpdata_assert_consistent(hpdata);
314 hpdata->h_huge = true;
315 fb_set_range(hpdata->touched_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES);
316 hpdata->h_ntouched = HUGEPAGE_PAGES;
317 hpdata_assert_consistent(hpdata);
318}
319
320void
321hpdata_dehugify(hpdata_t *hpdata) {
322 hpdata_assert_consistent(hpdata);
323 hpdata->h_huge = false;
324 hpdata_assert_consistent(hpdata);
325}
diff --git a/examples/redis-unstable/deps/jemalloc/src/inspect.c b/examples/redis-unstable/deps/jemalloc/src/inspect.c
deleted file mode 100644
index 911b5d5..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/inspect.c
+++ /dev/null
@@ -1,77 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4void
5inspect_extent_util_stats_get(tsdn_t *tsdn, const void *ptr, size_t *nfree,
6 size_t *nregs, size_t *size) {
7 assert(ptr != NULL && nfree != NULL && nregs != NULL && size != NULL);
8
9 const edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
10 if (unlikely(edata == NULL)) {
11 *nfree = *nregs = *size = 0;
12 return;
13 }
14
15 *size = edata_size_get(edata);
16 if (!edata_slab_get(edata)) {
17 *nfree = 0;
18 *nregs = 1;
19 } else {
20 *nfree = edata_nfree_get(edata);
21 *nregs = bin_infos[edata_szind_get(edata)].nregs;
22 assert(*nfree <= *nregs);
23 assert(*nfree * edata_usize_get(edata) <= *size);
24 }
25}
26
27void
28inspect_extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr,
29 size_t *nfree, size_t *nregs, size_t *size, size_t *bin_nfree,
30 size_t *bin_nregs, void **slabcur_addr) {
31 assert(ptr != NULL && nfree != NULL && nregs != NULL && size != NULL
32 && bin_nfree != NULL && bin_nregs != NULL && slabcur_addr != NULL);
33
34 const edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
35 if (unlikely(edata == NULL)) {
36 *nfree = *nregs = *size = *bin_nfree = *bin_nregs = 0;
37 *slabcur_addr = NULL;
38 return;
39 }
40
41 *size = edata_size_get(edata);
42 if (!edata_slab_get(edata)) {
43 *nfree = *bin_nfree = *bin_nregs = 0;
44 *nregs = 1;
45 *slabcur_addr = NULL;
46 return;
47 }
48
49 *nfree = edata_nfree_get(edata);
50 const szind_t szind = edata_szind_get(edata);
51 *nregs = bin_infos[szind].nregs;
52 assert(*nfree <= *nregs);
53 assert(*nfree * edata_usize_get(edata) <= *size);
54
55 arena_t *arena = (arena_t *)atomic_load_p(
56 &arenas[edata_arena_ind_get(edata)], ATOMIC_RELAXED);
57 assert(arena != NULL);
58 const unsigned binshard = edata_binshard_get(edata);
59 bin_t *bin = arena_get_bin(arena, szind, binshard);
60
61 malloc_mutex_lock(tsdn, &bin->lock);
62 if (config_stats) {
63 *bin_nregs = *nregs * bin->stats.curslabs;
64 assert(*bin_nregs >= bin->stats.curregs);
65 *bin_nfree = *bin_nregs - bin->stats.curregs;
66 } else {
67 *bin_nfree = *bin_nregs = 0;
68 }
69 edata_t *slab;
70 if (bin->slabcur != NULL) {
71 slab = bin->slabcur;
72 } else {
73 slab = edata_heap_first(&bin->slabs_nonfull);
74 }
75 *slabcur_addr = slab != NULL ? edata_addr_get(slab) : NULL;
76 malloc_mutex_unlock(tsdn, &bin->lock);
77}
diff --git a/examples/redis-unstable/deps/jemalloc/src/jemalloc.c b/examples/redis-unstable/deps/jemalloc/src/jemalloc.c
deleted file mode 100644
index 9a115f8..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/jemalloc.c
+++ /dev/null
@@ -1,4539 +0,0 @@
1#define JEMALLOC_C_
2#include "jemalloc/internal/jemalloc_preamble.h"
3#include "jemalloc/internal/jemalloc_internal_includes.h"
4
5#include "jemalloc/internal/assert.h"
6#include "jemalloc/internal/atomic.h"
7#include "jemalloc/internal/buf_writer.h"
8#include "jemalloc/internal/ctl.h"
9#include "jemalloc/internal/emap.h"
10#include "jemalloc/internal/extent_dss.h"
11#include "jemalloc/internal/extent_mmap.h"
12#include "jemalloc/internal/fxp.h"
13#include "jemalloc/internal/san.h"
14#include "jemalloc/internal/hook.h"
15#include "jemalloc/internal/jemalloc_internal_types.h"
16#include "jemalloc/internal/log.h"
17#include "jemalloc/internal/malloc_io.h"
18#include "jemalloc/internal/mutex.h"
19#include "jemalloc/internal/nstime.h"
20#include "jemalloc/internal/rtree.h"
21#include "jemalloc/internal/safety_check.h"
22#include "jemalloc/internal/sc.h"
23#include "jemalloc/internal/spin.h"
24#include "jemalloc/internal/sz.h"
25#include "jemalloc/internal/ticker.h"
26#include "jemalloc/internal/thread_event.h"
27#include "jemalloc/internal/util.h"
28
29/******************************************************************************/
30/* Data. */
31
32/* Runtime configuration options. */
33const char *je_malloc_conf
34#ifndef _WIN32
35 JEMALLOC_ATTR(weak)
36#endif
37 ;
38/*
39 * The usual rule is that the closer to runtime you are, the higher priority
40 * your configuration settings are (so the jemalloc config options get lower
41 * priority than the per-binary setting, which gets lower priority than the /etc
42 * setting, which gets lower priority than the environment settings).
43 *
44 * But it's a fairly common use case in some testing environments for a user to
45 * be able to control the binary, but nothing else (e.g. a performancy canary
46 * uses the production OS and environment variables, but can run any binary in
47 * those circumstances). For these use cases, it's handy to have an in-binary
48 * mechanism for overriding environment variable settings, with the idea that if
49 * the results are positive they get promoted to the official settings, and
50 * moved from the binary to the environment variable.
51 *
52 * We don't actually want this to be widespread, so we'll give it a silly name
53 * and not mention it in headers or documentation.
54 */
55const char *je_malloc_conf_2_conf_harder
56#ifndef _WIN32
57 JEMALLOC_ATTR(weak)
58#endif
59 ;
60
61bool opt_abort =
62#ifdef JEMALLOC_DEBUG
63 true
64#else
65 false
66#endif
67 ;
68bool opt_abort_conf =
69#ifdef JEMALLOC_DEBUG
70 true
71#else
72 false
73#endif
74 ;
75/* Intentionally default off, even with debug builds. */
76bool opt_confirm_conf = false;
77const char *opt_junk =
78#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
79 "true"
80#else
81 "false"
82#endif
83 ;
84bool opt_junk_alloc =
85#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
86 true
87#else
88 false
89#endif
90 ;
91bool opt_junk_free =
92#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
93 true
94#else
95 false
96#endif
97 ;
98bool opt_trust_madvise =
99#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS
100 false
101#else
102 true
103#endif
104 ;
105
106bool opt_cache_oblivious =
107#ifdef JEMALLOC_CACHE_OBLIVIOUS
108 true
109#else
110 false
111#endif
112 ;
113
114zero_realloc_action_t opt_zero_realloc_action =
115#ifdef JEMALLOC_ZERO_REALLOC_DEFAULT_FREE
116 zero_realloc_action_free
117#else
118 zero_realloc_action_alloc
119#endif
120 ;
121
122atomic_zu_t zero_realloc_count = ATOMIC_INIT(0);
123
124const char *zero_realloc_mode_names[] = {
125 "alloc",
126 "free",
127 "abort",
128};
129
130/*
131 * These are the documented values for junk fill debugging facilities -- see the
132 * man page.
133 */
134static const uint8_t junk_alloc_byte = 0xa5;
135static const uint8_t junk_free_byte = 0x5a;
136
137static void default_junk_alloc(void *ptr, size_t usize) {
138 memset(ptr, junk_alloc_byte, usize);
139}
140
141static void default_junk_free(void *ptr, size_t usize) {
142 memset(ptr, junk_free_byte, usize);
143}
144
145void (*junk_alloc_callback)(void *ptr, size_t size) = &default_junk_alloc;
146void (*junk_free_callback)(void *ptr, size_t size) = &default_junk_free;
147
148bool opt_utrace = false;
149bool opt_xmalloc = false;
150bool opt_experimental_infallible_new = false;
151bool opt_zero = false;
152unsigned opt_narenas = 0;
153fxp_t opt_narenas_ratio = FXP_INIT_INT(4);
154
155unsigned ncpus;
156
157/* Protects arenas initialization. */
158malloc_mutex_t arenas_lock;
159
160/* The global hpa, and whether it's on. */
161bool opt_hpa = false;
162hpa_shard_opts_t opt_hpa_opts = HPA_SHARD_OPTS_DEFAULT;
163sec_opts_t opt_hpa_sec_opts = SEC_OPTS_DEFAULT;
164
165/*
166 * Arenas that are used to service external requests. Not all elements of the
167 * arenas array are necessarily used; arenas are created lazily as needed.
168 *
169 * arenas[0..narenas_auto) are used for automatic multiplexing of threads and
170 * arenas. arenas[narenas_auto..narenas_total) are only used if the application
171 * takes some action to create them and allocate from them.
172 *
173 * Points to an arena_t.
174 */
175JEMALLOC_ALIGNED(CACHELINE)
176atomic_p_t arenas[MALLOCX_ARENA_LIMIT];
177static atomic_u_t narenas_total; /* Use narenas_total_*(). */
178/* Below three are read-only after initialization. */
179static arena_t *a0; /* arenas[0]. */
180unsigned narenas_auto;
181unsigned manual_arena_base;
182
183malloc_init_t malloc_init_state = malloc_init_uninitialized;
184
185/* False should be the common case. Set to true to trigger initialization. */
186bool malloc_slow = true;
187
188/* When malloc_slow is true, set the corresponding bits for sanity check. */
189enum {
190 flag_opt_junk_alloc = (1U),
191 flag_opt_junk_free = (1U << 1),
192 flag_opt_zero = (1U << 2),
193 flag_opt_utrace = (1U << 3),
194 flag_opt_xmalloc = (1U << 4)
195};
196static uint8_t malloc_slow_flags;
197
198#ifdef JEMALLOC_THREADED_INIT
199/* Used to let the initializing thread recursively allocate. */
200# define NO_INITIALIZER ((unsigned long)0)
201# define INITIALIZER pthread_self()
202# define IS_INITIALIZER (malloc_initializer == pthread_self())
203static pthread_t malloc_initializer = NO_INITIALIZER;
204#else
205# define NO_INITIALIZER false
206# define INITIALIZER true
207# define IS_INITIALIZER malloc_initializer
208static bool malloc_initializer = NO_INITIALIZER;
209#endif
210
211/* Used to avoid initialization races. */
212#ifdef _WIN32
213#if _WIN32_WINNT >= 0x0600
214static malloc_mutex_t init_lock = SRWLOCK_INIT;
215#else
216static malloc_mutex_t init_lock;
217static bool init_lock_initialized = false;
218
219JEMALLOC_ATTR(constructor)
220static void WINAPI
221_init_init_lock(void) {
222 /*
223 * If another constructor in the same binary is using mallctl to e.g.
224 * set up extent hooks, it may end up running before this one, and
225 * malloc_init_hard will crash trying to lock the uninitialized lock. So
226 * we force an initialization of the lock in malloc_init_hard as well.
227 * We don't try to care about atomicity of the accessed to the
228 * init_lock_initialized boolean, since it really only matters early in
229 * the process creation, before any separate thread normally starts
230 * doing anything.
231 */
232 if (!init_lock_initialized) {
233 malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT,
234 malloc_mutex_rank_exclusive);
235 }
236 init_lock_initialized = true;
237}
238
239#ifdef _MSC_VER
240# pragma section(".CRT$XCU", read)
241JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used)
242static const void (WINAPI *init_init_lock)(void) = _init_init_lock;
243#endif
244#endif
245#else
246static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER;
247#endif
248
249typedef struct {
250 void *p; /* Input pointer (as in realloc(p, s)). */
251 size_t s; /* Request size. */
252 void *r; /* Result pointer. */
253} malloc_utrace_t;
254
255#ifdef JEMALLOC_UTRACE
256# define UTRACE(a, b, c) do { \
257 if (unlikely(opt_utrace)) { \
258 int utrace_serrno = errno; \
259 malloc_utrace_t ut; \
260 ut.p = (a); \
261 ut.s = (b); \
262 ut.r = (c); \
263 UTRACE_CALL(&ut, sizeof(ut)); \
264 errno = utrace_serrno; \
265 } \
266} while (0)
267#else
268# define UTRACE(a, b, c)
269#endif
270
271/* Whether encountered any invalid config options. */
272static bool had_conf_error = false;
273
274/******************************************************************************/
275/*
276 * Function prototypes for static functions that are referenced prior to
277 * definition.
278 */
279
280static bool malloc_init_hard_a0(void);
281static bool malloc_init_hard(void);
282
283/******************************************************************************/
284/*
285 * Begin miscellaneous support functions.
286 */
287
288JEMALLOC_ALWAYS_INLINE bool
289malloc_init_a0(void) {
290 if (unlikely(malloc_init_state == malloc_init_uninitialized)) {
291 return malloc_init_hard_a0();
292 }
293 return false;
294}
295
296JEMALLOC_ALWAYS_INLINE bool
297malloc_init(void) {
298 if (unlikely(!malloc_initialized()) && malloc_init_hard()) {
299 return true;
300 }
301 return false;
302}
303
304/*
305 * The a0*() functions are used instead of i{d,}alloc() in situations that
306 * cannot tolerate TLS variable access.
307 */
308
309static void *
310a0ialloc(size_t size, bool zero, bool is_internal) {
311 if (unlikely(malloc_init_a0())) {
312 return NULL;
313 }
314
315 return iallocztm(TSDN_NULL, size, sz_size2index(size), zero, NULL,
316 is_internal, arena_get(TSDN_NULL, 0, true), true);
317}
318
319static void
320a0idalloc(void *ptr, bool is_internal) {
321 idalloctm(TSDN_NULL, ptr, NULL, NULL, is_internal, true);
322}
323
324void *
325a0malloc(size_t size) {
326 return a0ialloc(size, false, true);
327}
328
329void
330a0dalloc(void *ptr) {
331 a0idalloc(ptr, true);
332}
333
334/*
335 * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-sensitive
336 * situations that cannot tolerate TLS variable access (TLS allocation and very
337 * early internal data structure initialization).
338 */
339
340void *
341bootstrap_malloc(size_t size) {
342 if (unlikely(size == 0)) {
343 size = 1;
344 }
345
346 return a0ialloc(size, false, false);
347}
348
349void *
350bootstrap_calloc(size_t num, size_t size) {
351 size_t num_size;
352
353 num_size = num * size;
354 if (unlikely(num_size == 0)) {
355 assert(num == 0 || size == 0);
356 num_size = 1;
357 }
358
359 return a0ialloc(num_size, true, false);
360}
361
362void
363bootstrap_free(void *ptr) {
364 if (unlikely(ptr == NULL)) {
365 return;
366 }
367
368 a0idalloc(ptr, false);
369}
370
371void
372arena_set(unsigned ind, arena_t *arena) {
373 atomic_store_p(&arenas[ind], arena, ATOMIC_RELEASE);
374}
375
376static void
377narenas_total_set(unsigned narenas) {
378 atomic_store_u(&narenas_total, narenas, ATOMIC_RELEASE);
379}
380
381static void
382narenas_total_inc(void) {
383 atomic_fetch_add_u(&narenas_total, 1, ATOMIC_RELEASE);
384}
385
386unsigned
387narenas_total_get(void) {
388 return atomic_load_u(&narenas_total, ATOMIC_ACQUIRE);
389}
390
391/* Create a new arena and insert it into the arenas array at index ind. */
392static arena_t *
393arena_init_locked(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
394 arena_t *arena;
395
396 assert(ind <= narenas_total_get());
397 if (ind >= MALLOCX_ARENA_LIMIT) {
398 return NULL;
399 }
400 if (ind == narenas_total_get()) {
401 narenas_total_inc();
402 }
403
404 /*
405 * Another thread may have already initialized arenas[ind] if it's an
406 * auto arena.
407 */
408 arena = arena_get(tsdn, ind, false);
409 if (arena != NULL) {
410 assert(arena_is_auto(arena));
411 return arena;
412 }
413
414 /* Actually initialize the arena. */
415 arena = arena_new(tsdn, ind, config);
416
417 return arena;
418}
419
420static void
421arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) {
422 if (ind == 0) {
423 return;
424 }
425 /*
426 * Avoid creating a new background thread just for the huge arena, which
427 * purges eagerly by default.
428 */
429 if (have_background_thread && !arena_is_huge(ind)) {
430 if (background_thread_create(tsdn_tsd(tsdn), ind)) {
431 malloc_printf("<jemalloc>: error in background thread "
432 "creation for arena %u. Abort.\n", ind);
433 abort();
434 }
435 }
436}
437
438arena_t *
439arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
440 arena_t *arena;
441
442 malloc_mutex_lock(tsdn, &arenas_lock);
443 arena = arena_init_locked(tsdn, ind, config);
444 malloc_mutex_unlock(tsdn, &arenas_lock);
445
446 arena_new_create_background_thread(tsdn, ind);
447
448 return arena;
449}
450
451static void
452arena_bind(tsd_t *tsd, unsigned ind, bool internal) {
453 arena_t *arena = arena_get(tsd_tsdn(tsd), ind, false);
454 arena_nthreads_inc(arena, internal);
455
456 if (internal) {
457 tsd_iarena_set(tsd, arena);
458 } else {
459 tsd_arena_set(tsd, arena);
460 unsigned shard = atomic_fetch_add_u(&arena->binshard_next, 1,
461 ATOMIC_RELAXED);
462 tsd_binshards_t *bins = tsd_binshardsp_get(tsd);
463 for (unsigned i = 0; i < SC_NBINS; i++) {
464 assert(bin_infos[i].n_shards > 0 &&
465 bin_infos[i].n_shards <= BIN_SHARDS_MAX);
466 bins->binshard[i] = shard % bin_infos[i].n_shards;
467 }
468 }
469}
470
471void
472arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) {
473 assert(oldarena != NULL);
474 assert(newarena != NULL);
475
476 arena_nthreads_dec(oldarena, false);
477 arena_nthreads_inc(newarena, false);
478 tsd_arena_set(tsd, newarena);
479
480 if (arena_nthreads_get(oldarena, false) == 0) {
481 /* Purge if the old arena has no associated threads anymore. */
482 arena_decay(tsd_tsdn(tsd), oldarena,
483 /* is_background_thread */ false, /* all */ true);
484 }
485}
486
487static void
488arena_unbind(tsd_t *tsd, unsigned ind, bool internal) {
489 arena_t *arena;
490
491 arena = arena_get(tsd_tsdn(tsd), ind, false);
492 arena_nthreads_dec(arena, internal);
493
494 if (internal) {
495 tsd_iarena_set(tsd, NULL);
496 } else {
497 tsd_arena_set(tsd, NULL);
498 }
499}
500
501/* Slow path, called only by arena_choose(). */
502arena_t *
503arena_choose_hard(tsd_t *tsd, bool internal) {
504 arena_t *ret JEMALLOC_CC_SILENCE_INIT(NULL);
505
506 if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)) {
507 unsigned choose = percpu_arena_choose();
508 ret = arena_get(tsd_tsdn(tsd), choose, true);
509 assert(ret != NULL);
510 arena_bind(tsd, arena_ind_get(ret), false);
511 arena_bind(tsd, arena_ind_get(ret), true);
512
513 return ret;
514 }
515
516 if (narenas_auto > 1) {
517 unsigned i, j, choose[2], first_null;
518 bool is_new_arena[2];
519
520 /*
521 * Determine binding for both non-internal and internal
522 * allocation.
523 *
524 * choose[0]: For application allocation.
525 * choose[1]: For internal metadata allocation.
526 */
527
528 for (j = 0; j < 2; j++) {
529 choose[j] = 0;
530 is_new_arena[j] = false;
531 }
532
533 first_null = narenas_auto;
534 malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock);
535 assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL);
536 for (i = 1; i < narenas_auto; i++) {
537 if (arena_get(tsd_tsdn(tsd), i, false) != NULL) {
538 /*
539 * Choose the first arena that has the lowest
540 * number of threads assigned to it.
541 */
542 for (j = 0; j < 2; j++) {
543 if (arena_nthreads_get(arena_get(
544 tsd_tsdn(tsd), i, false), !!j) <
545 arena_nthreads_get(arena_get(
546 tsd_tsdn(tsd), choose[j], false),
547 !!j)) {
548 choose[j] = i;
549 }
550 }
551 } else if (first_null == narenas_auto) {
552 /*
553 * Record the index of the first uninitialized
554 * arena, in case all extant arenas are in use.
555 *
556 * NB: It is possible for there to be
557 * discontinuities in terms of initialized
558 * versus uninitialized arenas, due to the
559 * "thread.arena" mallctl.
560 */
561 first_null = i;
562 }
563 }
564
565 for (j = 0; j < 2; j++) {
566 if (arena_nthreads_get(arena_get(tsd_tsdn(tsd),
567 choose[j], false), !!j) == 0 || first_null ==
568 narenas_auto) {
569 /*
570 * Use an unloaded arena, or the least loaded
571 * arena if all arenas are already initialized.
572 */
573 if (!!j == internal) {
574 ret = arena_get(tsd_tsdn(tsd),
575 choose[j], false);
576 }
577 } else {
578 arena_t *arena;
579
580 /* Initialize a new arena. */
581 choose[j] = first_null;
582 arena = arena_init_locked(tsd_tsdn(tsd),
583 choose[j], &arena_config_default);
584 if (arena == NULL) {
585 malloc_mutex_unlock(tsd_tsdn(tsd),
586 &arenas_lock);
587 return NULL;
588 }
589 is_new_arena[j] = true;
590 if (!!j == internal) {
591 ret = arena;
592 }
593 }
594 arena_bind(tsd, choose[j], !!j);
595 }
596 malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock);
597
598 for (j = 0; j < 2; j++) {
599 if (is_new_arena[j]) {
600 assert(choose[j] > 0);
601 arena_new_create_background_thread(
602 tsd_tsdn(tsd), choose[j]);
603 }
604 }
605
606 } else {
607 ret = arena_get(tsd_tsdn(tsd), 0, false);
608 arena_bind(tsd, 0, false);
609 arena_bind(tsd, 0, true);
610 }
611
612 return ret;
613}
614
615void
616iarena_cleanup(tsd_t *tsd) {
617 arena_t *iarena;
618
619 iarena = tsd_iarena_get(tsd);
620 if (iarena != NULL) {
621 arena_unbind(tsd, arena_ind_get(iarena), true);
622 }
623}
624
625void
626arena_cleanup(tsd_t *tsd) {
627 arena_t *arena;
628
629 arena = tsd_arena_get(tsd);
630 if (arena != NULL) {
631 arena_unbind(tsd, arena_ind_get(arena), false);
632 }
633}
634
635static void
636stats_print_atexit(void) {
637 if (config_stats) {
638 tsdn_t *tsdn;
639 unsigned narenas, i;
640
641 tsdn = tsdn_fetch();
642
643 /*
644 * Merge stats from extant threads. This is racy, since
645 * individual threads do not lock when recording tcache stats
646 * events. As a consequence, the final stats may be slightly
647 * out of date by the time they are reported, if other threads
648 * continue to allocate.
649 */
650 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
651 arena_t *arena = arena_get(tsdn, i, false);
652 if (arena != NULL) {
653 tcache_slow_t *tcache_slow;
654
655 malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
656 ql_foreach(tcache_slow, &arena->tcache_ql,
657 link) {
658 tcache_stats_merge(tsdn,
659 tcache_slow->tcache, arena);
660 }
661 malloc_mutex_unlock(tsdn,
662 &arena->tcache_ql_mtx);
663 }
664 }
665 }
666 je_malloc_stats_print(NULL, NULL, opt_stats_print_opts);
667}
668
669/*
670 * Ensure that we don't hold any locks upon entry to or exit from allocator
671 * code (in a "broad" sense that doesn't count a reentrant allocation as an
672 * entrance or exit).
673 */
674JEMALLOC_ALWAYS_INLINE void
675check_entry_exit_locking(tsdn_t *tsdn) {
676 if (!config_debug) {
677 return;
678 }
679 if (tsdn_null(tsdn)) {
680 return;
681 }
682 tsd_t *tsd = tsdn_tsd(tsdn);
683 /*
684 * It's possible we hold locks at entry/exit if we're in a nested
685 * allocation.
686 */
687 int8_t reentrancy_level = tsd_reentrancy_level_get(tsd);
688 if (reentrancy_level != 0) {
689 return;
690 }
691 witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
692}
693
694/*
695 * End miscellaneous support functions.
696 */
697/******************************************************************************/
698/*
699 * Begin initialization functions.
700 */
701
702static char *
703jemalloc_secure_getenv(const char *name) {
704#ifdef JEMALLOC_HAVE_SECURE_GETENV
705 return secure_getenv(name);
706#else
707# ifdef JEMALLOC_HAVE_ISSETUGID
708 if (issetugid() != 0) {
709 return NULL;
710 }
711# endif
712 return getenv(name);
713#endif
714}
715
716static unsigned
717malloc_ncpus(void) {
718 long result;
719
720#ifdef _WIN32
721 SYSTEM_INFO si;
722 GetSystemInfo(&si);
723 result = si.dwNumberOfProcessors;
724#elif defined(CPU_COUNT)
725 /*
726 * glibc >= 2.6 has the CPU_COUNT macro.
727 *
728 * glibc's sysconf() uses isspace(). glibc allocates for the first time
729 * *before* setting up the isspace tables. Therefore we need a
730 * different method to get the number of CPUs.
731 *
732 * The getaffinity approach is also preferred when only a subset of CPUs
733 * is available, to avoid using more arenas than necessary.
734 */
735 {
736# if defined(__FreeBSD__) || defined(__DragonFly__)
737 cpuset_t set;
738# else
739 cpu_set_t set;
740# endif
741# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
742 sched_getaffinity(0, sizeof(set), &set);
743# else
744 pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
745# endif
746 result = CPU_COUNT(&set);
747 }
748#else
749 result = sysconf(_SC_NPROCESSORS_ONLN);
750#endif
751 return ((result == -1) ? 1 : (unsigned)result);
752}
753
754/*
755 * Ensure that number of CPUs is determistinc, i.e. it is the same based on:
756 * - sched_getaffinity()
757 * - _SC_NPROCESSORS_ONLN
758 * - _SC_NPROCESSORS_CONF
759 * Since otherwise tricky things is possible with percpu arenas in use.
760 */
761static bool
762malloc_cpu_count_is_deterministic()
763{
764#ifdef _WIN32
765 return true;
766#else
767 long cpu_onln = sysconf(_SC_NPROCESSORS_ONLN);
768 long cpu_conf = sysconf(_SC_NPROCESSORS_CONF);
769 if (cpu_onln != cpu_conf) {
770 return false;
771 }
772# if defined(CPU_COUNT)
773# if defined(__FreeBSD__) || defined(__DragonFly__)
774 cpuset_t set;
775# else
776 cpu_set_t set;
777# endif /* __FreeBSD__ */
778# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
779 sched_getaffinity(0, sizeof(set), &set);
780# else /* !JEMALLOC_HAVE_SCHED_SETAFFINITY */
781 pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
782# endif /* JEMALLOC_HAVE_SCHED_SETAFFINITY */
783 long cpu_affinity = CPU_COUNT(&set);
784 if (cpu_affinity != cpu_conf) {
785 return false;
786 }
787# endif /* CPU_COUNT */
788 return true;
789#endif
790}
791
792static void
793init_opt_stats_opts(const char *v, size_t vlen, char *dest) {
794 size_t opts_len = strlen(dest);
795 assert(opts_len <= stats_print_tot_num_options);
796
797 for (size_t i = 0; i < vlen; i++) {
798 switch (v[i]) {
799#define OPTION(o, v, d, s) case o: break;
800 STATS_PRINT_OPTIONS
801#undef OPTION
802 default: continue;
803 }
804
805 if (strchr(dest, v[i]) != NULL) {
806 /* Ignore repeated. */
807 continue;
808 }
809
810 dest[opts_len++] = v[i];
811 dest[opts_len] = '\0';
812 assert(opts_len <= stats_print_tot_num_options);
813 }
814 assert(opts_len == strlen(dest));
815}
816
817/* Reads the next size pair in a multi-sized option. */
818static bool
819malloc_conf_multi_sizes_next(const char **slab_size_segment_cur,
820 size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *new_size) {
821 const char *cur = *slab_size_segment_cur;
822 char *end;
823 uintmax_t um;
824
825 set_errno(0);
826
827 /* First number, then '-' */
828 um = malloc_strtoumax(cur, &end, 0);
829 if (get_errno() != 0 || *end != '-') {
830 return true;
831 }
832 *slab_start = (size_t)um;
833 cur = end + 1;
834
835 /* Second number, then ':' */
836 um = malloc_strtoumax(cur, &end, 0);
837 if (get_errno() != 0 || *end != ':') {
838 return true;
839 }
840 *slab_end = (size_t)um;
841 cur = end + 1;
842
843 /* Last number */
844 um = malloc_strtoumax(cur, &end, 0);
845 if (get_errno() != 0) {
846 return true;
847 }
848 *new_size = (size_t)um;
849
850 /* Consume the separator if there is one. */
851 if (*end == '|') {
852 end++;
853 }
854
855 *vlen_left -= end - *slab_size_segment_cur;
856 *slab_size_segment_cur = end;
857
858 return false;
859}
860
861static bool
862malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
863 char const **v_p, size_t *vlen_p) {
864 bool accept;
865 const char *opts = *opts_p;
866
867 *k_p = opts;
868
869 for (accept = false; !accept;) {
870 switch (*opts) {
871 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
872 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
873 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
874 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
875 case 'Y': case 'Z':
876 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
877 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
878 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
879 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
880 case 'y': case 'z':
881 case '0': case '1': case '2': case '3': case '4': case '5':
882 case '6': case '7': case '8': case '9':
883 case '_':
884 opts++;
885 break;
886 case ':':
887 opts++;
888 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p;
889 *v_p = opts;
890 accept = true;
891 break;
892 case '\0':
893 if (opts != *opts_p) {
894 malloc_write("<jemalloc>: Conf string ends "
895 "with key\n");
896 had_conf_error = true;
897 }
898 return true;
899 default:
900 malloc_write("<jemalloc>: Malformed conf string\n");
901 had_conf_error = true;
902 return true;
903 }
904 }
905
906 for (accept = false; !accept;) {
907 switch (*opts) {
908 case ',':
909 opts++;
910 /*
911 * Look ahead one character here, because the next time
912 * this function is called, it will assume that end of
913 * input has been cleanly reached if no input remains,
914 * but we have optimistically already consumed the
915 * comma if one exists.
916 */
917 if (*opts == '\0') {
918 malloc_write("<jemalloc>: Conf string ends "
919 "with comma\n");
920 had_conf_error = true;
921 }
922 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p;
923 accept = true;
924 break;
925 case '\0':
926 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p;
927 accept = true;
928 break;
929 default:
930 opts++;
931 break;
932 }
933 }
934
935 *opts_p = opts;
936 return false;
937}
938
939static void
940malloc_abort_invalid_conf(void) {
941 assert(opt_abort_conf);
942 malloc_printf("<jemalloc>: Abort (abort_conf:true) on invalid conf "
943 "value (see above).\n");
944 abort();
945}
946
947static void
948malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v,
949 size_t vlen) {
950 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k,
951 (int)vlen, v);
952 /* If abort_conf is set, error out after processing all options. */
953 const char *experimental = "experimental_";
954 if (strncmp(k, experimental, strlen(experimental)) == 0) {
955 /* However, tolerate experimental features. */
956 return;
957 }
958 had_conf_error = true;
959}
960
961static void
962malloc_slow_flag_init(void) {
963 /*
964 * Combine the runtime options into malloc_slow for fast path. Called
965 * after processing all the options.
966 */
967 malloc_slow_flags |= (opt_junk_alloc ? flag_opt_junk_alloc : 0)
968 | (opt_junk_free ? flag_opt_junk_free : 0)
969 | (opt_zero ? flag_opt_zero : 0)
970 | (opt_utrace ? flag_opt_utrace : 0)
971 | (opt_xmalloc ? flag_opt_xmalloc : 0);
972
973 malloc_slow = (malloc_slow_flags != 0);
974}
975
976/* Number of sources for initializing malloc_conf */
977#define MALLOC_CONF_NSOURCES 5
978
979static const char *
980obtain_malloc_conf(unsigned which_source, char buf[PATH_MAX + 1]) {
981 if (config_debug) {
982 static unsigned read_source = 0;
983 /*
984 * Each source should only be read once, to minimize # of
985 * syscalls on init.
986 */
987 assert(read_source++ == which_source);
988 }
989 assert(which_source < MALLOC_CONF_NSOURCES);
990
991 const char *ret;
992 switch (which_source) {
993 case 0:
994 ret = config_malloc_conf;
995 break;
996 case 1:
997 if (je_malloc_conf != NULL) {
998 /* Use options that were compiled into the program. */
999 ret = je_malloc_conf;
1000 } else {
1001 /* No configuration specified. */
1002 ret = NULL;
1003 }
1004 break;
1005 case 2: {
1006 ssize_t linklen = 0;
1007#ifndef _WIN32
1008 int saved_errno = errno;
1009 const char *linkname =
1010# ifdef JEMALLOC_PREFIX
1011 "/etc/"JEMALLOC_PREFIX"malloc.conf"
1012# else
1013 "/etc/malloc.conf"
1014# endif
1015 ;
1016
1017 /*
1018 * Try to use the contents of the "/etc/malloc.conf" symbolic
1019 * link's name.
1020 */
1021#ifndef JEMALLOC_READLINKAT
1022 linklen = readlink(linkname, buf, PATH_MAX);
1023#else
1024 linklen = readlinkat(AT_FDCWD, linkname, buf, PATH_MAX);
1025#endif
1026 if (linklen == -1) {
1027 /* No configuration specified. */
1028 linklen = 0;
1029 /* Restore errno. */
1030 set_errno(saved_errno);
1031 }
1032#endif
1033 buf[linklen] = '\0';
1034 ret = buf;
1035 break;
1036 } case 3: {
1037 const char *envname =
1038#ifdef JEMALLOC_PREFIX
1039 JEMALLOC_CPREFIX"MALLOC_CONF"
1040#else
1041 "MALLOC_CONF"
1042#endif
1043 ;
1044
1045 if ((ret = jemalloc_secure_getenv(envname)) != NULL) {
1046 /*
1047 * Do nothing; opts is already initialized to the value
1048 * of the MALLOC_CONF environment variable.
1049 */
1050 } else {
1051 /* No configuration specified. */
1052 ret = NULL;
1053 }
1054 break;
1055 } case 4: {
1056 ret = je_malloc_conf_2_conf_harder;
1057 break;
1058 } default:
1059 not_reached();
1060 ret = NULL;
1061 }
1062 return ret;
1063}
1064
1065static void
1066malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
1067 bool initial_call, const char *opts_cache[MALLOC_CONF_NSOURCES],
1068 char buf[PATH_MAX + 1]) {
1069 static const char *opts_explain[MALLOC_CONF_NSOURCES] = {
1070 "string specified via --with-malloc-conf",
1071 "string pointed to by the global variable malloc_conf",
1072 ("\"name\" of the file referenced by the symbolic link named "
1073 "/etc/malloc.conf"),
1074 "value of the environment variable MALLOC_CONF",
1075 ("string pointed to by the global variable "
1076 "malloc_conf_2_conf_harder"),
1077 };
1078 unsigned i;
1079 const char *opts, *k, *v;
1080 size_t klen, vlen;
1081
1082 for (i = 0; i < MALLOC_CONF_NSOURCES; i++) {
1083 /* Get runtime configuration. */
1084 if (initial_call) {
1085 opts_cache[i] = obtain_malloc_conf(i, buf);
1086 }
1087 opts = opts_cache[i];
1088 if (!initial_call && opt_confirm_conf) {
1089 malloc_printf(
1090 "<jemalloc>: malloc_conf #%u (%s): \"%s\"\n",
1091 i + 1, opts_explain[i], opts != NULL ? opts : "");
1092 }
1093 if (opts == NULL) {
1094 continue;
1095 }
1096
1097 while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v,
1098 &vlen)) {
1099
1100#define CONF_ERROR(msg, k, klen, v, vlen) \
1101 if (!initial_call) { \
1102 malloc_conf_error( \
1103 msg, k, klen, v, vlen); \
1104 cur_opt_valid = false; \
1105 }
1106#define CONF_CONTINUE { \
1107 if (!initial_call && opt_confirm_conf \
1108 && cur_opt_valid) { \
1109 malloc_printf("<jemalloc>: -- " \
1110 "Set conf value: %.*s:%.*s" \
1111 "\n", (int)klen, k, \
1112 (int)vlen, v); \
1113 } \
1114 continue; \
1115 }
1116#define CONF_MATCH(n) \
1117 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0)
1118#define CONF_MATCH_VALUE(n) \
1119 (sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0)
1120#define CONF_HANDLE_BOOL(o, n) \
1121 if (CONF_MATCH(n)) { \
1122 if (CONF_MATCH_VALUE("true")) { \
1123 o = true; \
1124 } else if (CONF_MATCH_VALUE("false")) { \
1125 o = false; \
1126 } else { \
1127 CONF_ERROR("Invalid conf value",\
1128 k, klen, v, vlen); \
1129 } \
1130 CONF_CONTINUE; \
1131 }
1132 /*
1133 * One of the CONF_MIN macros below expands, in one of the use points,
1134 * to "unsigned integer < 0", which is always false, triggering the
1135 * GCC -Wtype-limits warning, which we disable here and re-enable below.
1136 */
1137 JEMALLOC_DIAGNOSTIC_PUSH
1138 JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS
1139
1140#define CONF_DONT_CHECK_MIN(um, min) false
1141#define CONF_CHECK_MIN(um, min) ((um) < (min))
1142#define CONF_DONT_CHECK_MAX(um, max) false
1143#define CONF_CHECK_MAX(um, max) ((um) > (max))
1144
1145#define CONF_VALUE_READ(max_t, result) \
1146 char *end; \
1147 set_errno(0); \
1148 result = (max_t)malloc_strtoumax(v, &end, 0);
1149#define CONF_VALUE_READ_FAIL() \
1150 (get_errno() != 0 || (uintptr_t)end - (uintptr_t)v != vlen)
1151
1152#define CONF_HANDLE_T(t, max_t, o, n, min, max, check_min, check_max, clip) \
1153 if (CONF_MATCH(n)) { \
1154 max_t mv; \
1155 CONF_VALUE_READ(max_t, mv) \
1156 if (CONF_VALUE_READ_FAIL()) { \
1157 CONF_ERROR("Invalid conf value",\
1158 k, klen, v, vlen); \
1159 } else if (clip) { \
1160 if (check_min(mv, (t)(min))) { \
1161 o = (t)(min); \
1162 } else if ( \
1163 check_max(mv, (t)(max))) { \
1164 o = (t)(max); \
1165 } else { \
1166 o = (t)mv; \
1167 } \
1168 } else { \
1169 if (check_min(mv, (t)(min)) || \
1170 check_max(mv, (t)(max))) { \
1171 CONF_ERROR( \
1172 "Out-of-range " \
1173 "conf value", \
1174 k, klen, v, vlen); \
1175 } else { \
1176 o = (t)mv; \
1177 } \
1178 } \
1179 CONF_CONTINUE; \
1180 }
1181#define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \
1182 CONF_HANDLE_T(t, uintmax_t, o, n, min, max, check_min, \
1183 check_max, clip)
1184#define CONF_HANDLE_T_SIGNED(t, o, n, min, max, check_min, check_max, clip)\
1185 CONF_HANDLE_T(t, intmax_t, o, n, min, max, check_min, \
1186 check_max, clip)
1187
1188#define CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max, \
1189 clip) \
1190 CONF_HANDLE_T_U(unsigned, o, n, min, max, \
1191 check_min, check_max, clip)
1192#define CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip) \
1193 CONF_HANDLE_T_U(size_t, o, n, min, max, \
1194 check_min, check_max, clip)
1195#define CONF_HANDLE_INT64_T(o, n, min, max, check_min, check_max, clip) \
1196 CONF_HANDLE_T_SIGNED(int64_t, o, n, min, max, \
1197 check_min, check_max, clip)
1198#define CONF_HANDLE_UINT64_T(o, n, min, max, check_min, check_max, clip)\
1199 CONF_HANDLE_T_U(uint64_t, o, n, min, max, \
1200 check_min, check_max, clip)
1201#define CONF_HANDLE_SSIZE_T(o, n, min, max) \
1202 CONF_HANDLE_T_SIGNED(ssize_t, o, n, min, max, \
1203 CONF_CHECK_MIN, CONF_CHECK_MAX, false)
1204#define CONF_HANDLE_CHAR_P(o, n, d) \
1205 if (CONF_MATCH(n)) { \
1206 size_t cpylen = (vlen <= \
1207 sizeof(o)-1) ? vlen : \
1208 sizeof(o)-1; \
1209 strncpy(o, v, cpylen); \
1210 o[cpylen] = '\0'; \
1211 CONF_CONTINUE; \
1212 }
1213
1214 bool cur_opt_valid = true;
1215
1216 CONF_HANDLE_BOOL(opt_confirm_conf, "confirm_conf")
1217 if (initial_call) {
1218 continue;
1219 }
1220
1221 CONF_HANDLE_BOOL(opt_abort, "abort")
1222 CONF_HANDLE_BOOL(opt_abort_conf, "abort_conf")
1223 CONF_HANDLE_BOOL(opt_trust_madvise, "trust_madvise")
1224 if (strncmp("metadata_thp", k, klen) == 0) {
1225 int m;
1226 bool match = false;
1227 for (m = 0; m < metadata_thp_mode_limit; m++) {
1228 if (strncmp(metadata_thp_mode_names[m],
1229 v, vlen) == 0) {
1230 opt_metadata_thp = m;
1231 match = true;
1232 break;
1233 }
1234 }
1235 if (!match) {
1236 CONF_ERROR("Invalid conf value",
1237 k, klen, v, vlen);
1238 }
1239 CONF_CONTINUE;
1240 }
1241 CONF_HANDLE_BOOL(opt_retain, "retain")
1242 if (strncmp("dss", k, klen) == 0) {
1243 int m;
1244 bool match = false;
1245 for (m = 0; m < dss_prec_limit; m++) {
1246 if (strncmp(dss_prec_names[m], v, vlen)
1247 == 0) {
1248 if (extent_dss_prec_set(m)) {
1249 CONF_ERROR(
1250 "Error setting dss",
1251 k, klen, v, vlen);
1252 } else {
1253 opt_dss =
1254 dss_prec_names[m];
1255 match = true;
1256 break;
1257 }
1258 }
1259 }
1260 if (!match) {
1261 CONF_ERROR("Invalid conf value",
1262 k, klen, v, vlen);
1263 }
1264 CONF_CONTINUE;
1265 }
1266 if (CONF_MATCH("narenas")) {
1267 if (CONF_MATCH_VALUE("default")) {
1268 opt_narenas = 0;
1269 CONF_CONTINUE;
1270 } else {
1271 CONF_HANDLE_UNSIGNED(opt_narenas,
1272 "narenas", 1, UINT_MAX,
1273 CONF_CHECK_MIN, CONF_DONT_CHECK_MAX,
1274 /* clip */ false)
1275 }
1276 }
1277 if (CONF_MATCH("narenas_ratio")) {
1278 char *end;
1279 bool err = fxp_parse(&opt_narenas_ratio, v,
1280 &end);
1281 if (err || (size_t)(end - v) != vlen) {
1282 CONF_ERROR("Invalid conf value",
1283 k, klen, v, vlen);
1284 }
1285 CONF_CONTINUE;
1286 }
1287 if (CONF_MATCH("bin_shards")) {
1288 const char *bin_shards_segment_cur = v;
1289 size_t vlen_left = vlen;
1290 do {
1291 size_t size_start;
1292 size_t size_end;
1293 size_t nshards;
1294 bool err = malloc_conf_multi_sizes_next(
1295 &bin_shards_segment_cur, &vlen_left,
1296 &size_start, &size_end, &nshards);
1297 if (err || bin_update_shard_size(
1298 bin_shard_sizes, size_start,
1299 size_end, nshards)) {
1300 CONF_ERROR(
1301 "Invalid settings for "
1302 "bin_shards", k, klen, v,
1303 vlen);
1304 break;
1305 }
1306 } while (vlen_left > 0);
1307 CONF_CONTINUE;
1308 }
1309 CONF_HANDLE_INT64_T(opt_mutex_max_spin,
1310 "mutex_max_spin", -1, INT64_MAX, CONF_CHECK_MIN,
1311 CONF_DONT_CHECK_MAX, false);
1312 CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms,
1313 "dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) <
1314 QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) :
1315 SSIZE_MAX);
1316 CONF_HANDLE_SSIZE_T(opt_muzzy_decay_ms,
1317 "muzzy_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) <
1318 QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) :
1319 SSIZE_MAX);
1320 CONF_HANDLE_BOOL(opt_stats_print, "stats_print")
1321 if (CONF_MATCH("stats_print_opts")) {
1322 init_opt_stats_opts(v, vlen,
1323 opt_stats_print_opts);
1324 CONF_CONTINUE;
1325 }
1326 CONF_HANDLE_INT64_T(opt_stats_interval,
1327 "stats_interval", -1, INT64_MAX,
1328 CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, false)
1329 if (CONF_MATCH("stats_interval_opts")) {
1330 init_opt_stats_opts(v, vlen,
1331 opt_stats_interval_opts);
1332 CONF_CONTINUE;
1333 }
1334 if (config_fill) {
1335 if (CONF_MATCH("junk")) {
1336 if (CONF_MATCH_VALUE("true")) {
1337 opt_junk = "true";
1338 opt_junk_alloc = opt_junk_free =
1339 true;
1340 } else if (CONF_MATCH_VALUE("false")) {
1341 opt_junk = "false";
1342 opt_junk_alloc = opt_junk_free =
1343 false;
1344 } else if (CONF_MATCH_VALUE("alloc")) {
1345 opt_junk = "alloc";
1346 opt_junk_alloc = true;
1347 opt_junk_free = false;
1348 } else if (CONF_MATCH_VALUE("free")) {
1349 opt_junk = "free";
1350 opt_junk_alloc = false;
1351 opt_junk_free = true;
1352 } else {
1353 CONF_ERROR(
1354 "Invalid conf value",
1355 k, klen, v, vlen);
1356 }
1357 CONF_CONTINUE;
1358 }
1359 CONF_HANDLE_BOOL(opt_zero, "zero")
1360 }
1361 if (config_utrace) {
1362 CONF_HANDLE_BOOL(opt_utrace, "utrace")
1363 }
1364 if (config_xmalloc) {
1365 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc")
1366 }
1367 if (config_enable_cxx) {
1368 CONF_HANDLE_BOOL(
1369 opt_experimental_infallible_new,
1370 "experimental_infallible_new")
1371 }
1372
1373 CONF_HANDLE_BOOL(opt_tcache, "tcache")
1374 CONF_HANDLE_SIZE_T(opt_tcache_max, "tcache_max",
1375 0, TCACHE_MAXCLASS_LIMIT, CONF_DONT_CHECK_MIN,
1376 CONF_CHECK_MAX, /* clip */ true)
1377 if (CONF_MATCH("lg_tcache_max")) {
1378 size_t m;
1379 CONF_VALUE_READ(size_t, m)
1380 if (CONF_VALUE_READ_FAIL()) {
1381 CONF_ERROR("Invalid conf value",
1382 k, klen, v, vlen);
1383 } else {
1384 /* clip if necessary */
1385 if (m > TCACHE_LG_MAXCLASS_LIMIT) {
1386 m = TCACHE_LG_MAXCLASS_LIMIT;
1387 }
1388 opt_tcache_max = (size_t)1 << m;
1389 }
1390 CONF_CONTINUE;
1391 }
1392 /*
1393 * Anyone trying to set a value outside -16 to 16 is
1394 * deeply confused.
1395 */
1396 CONF_HANDLE_SSIZE_T(opt_lg_tcache_nslots_mul,
1397 "lg_tcache_nslots_mul", -16, 16)
1398 /* Ditto with values past 2048. */
1399 CONF_HANDLE_UNSIGNED(opt_tcache_nslots_small_min,
1400 "tcache_nslots_small_min", 1, 2048,
1401 CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
1402 CONF_HANDLE_UNSIGNED(opt_tcache_nslots_small_max,
1403 "tcache_nslots_small_max", 1, 2048,
1404 CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
1405 CONF_HANDLE_UNSIGNED(opt_tcache_nslots_large,
1406 "tcache_nslots_large", 1, 2048,
1407 CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
1408 CONF_HANDLE_SIZE_T(opt_tcache_gc_incr_bytes,
1409 "tcache_gc_incr_bytes", 1024, SIZE_T_MAX,
1410 CONF_CHECK_MIN, CONF_DONT_CHECK_MAX,
1411 /* clip */ true)
1412 CONF_HANDLE_SIZE_T(opt_tcache_gc_delay_bytes,
1413 "tcache_gc_delay_bytes", 0, SIZE_T_MAX,
1414 CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX,
1415 /* clip */ false)
1416 CONF_HANDLE_UNSIGNED(opt_lg_tcache_flush_small_div,
1417 "lg_tcache_flush_small_div", 1, 16,
1418 CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
1419 CONF_HANDLE_UNSIGNED(opt_lg_tcache_flush_large_div,
1420 "lg_tcache_flush_large_div", 1, 16,
1421 CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
1422
1423 /*
1424 * The runtime option of oversize_threshold remains
1425 * undocumented. It may be tweaked in the next major
1426 * release (6.0). The default value 8M is rather
1427 * conservative / safe. Tuning it further down may
1428 * improve fragmentation a bit more, but may also cause
1429 * contention on the huge arena.
1430 */
1431 CONF_HANDLE_SIZE_T(opt_oversize_threshold,
1432 "oversize_threshold", 0, SC_LARGE_MAXCLASS,
1433 CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, false)
1434 CONF_HANDLE_SIZE_T(opt_lg_extent_max_active_fit,
1435 "lg_extent_max_active_fit", 0,
1436 (sizeof(size_t) << 3), CONF_DONT_CHECK_MIN,
1437 CONF_CHECK_MAX, false)
1438
1439 if (strncmp("percpu_arena", k, klen) == 0) {
1440 bool match = false;
1441 for (int m = percpu_arena_mode_names_base; m <
1442 percpu_arena_mode_names_limit; m++) {
1443 if (strncmp(percpu_arena_mode_names[m],
1444 v, vlen) == 0) {
1445 if (!have_percpu_arena) {
1446 CONF_ERROR(
1447 "No getcpu support",
1448 k, klen, v, vlen);
1449 }
1450 opt_percpu_arena = m;
1451 match = true;
1452 break;
1453 }
1454 }
1455 if (!match) {
1456 CONF_ERROR("Invalid conf value",
1457 k, klen, v, vlen);
1458 }
1459 CONF_CONTINUE;
1460 }
1461 CONF_HANDLE_BOOL(opt_background_thread,
1462 "background_thread");
1463 CONF_HANDLE_SIZE_T(opt_max_background_threads,
1464 "max_background_threads", 1,
1465 opt_max_background_threads,
1466 CONF_CHECK_MIN, CONF_CHECK_MAX,
1467 true);
1468 CONF_HANDLE_BOOL(opt_hpa, "hpa")
1469 CONF_HANDLE_SIZE_T(opt_hpa_opts.slab_max_alloc,
1470 "hpa_slab_max_alloc", PAGE, HUGEPAGE,
1471 CONF_CHECK_MIN, CONF_CHECK_MAX, true);
1472
1473 /*
1474 * Accept either a ratio-based or an exact hugification
1475 * threshold.
1476 */
1477 CONF_HANDLE_SIZE_T(opt_hpa_opts.hugification_threshold,
1478 "hpa_hugification_threshold", PAGE, HUGEPAGE,
1479 CONF_CHECK_MIN, CONF_CHECK_MAX, true);
1480 if (CONF_MATCH("hpa_hugification_threshold_ratio")) {
1481 fxp_t ratio;
1482 char *end;
1483 bool err = fxp_parse(&ratio, v,
1484 &end);
1485 if (err || (size_t)(end - v) != vlen
1486 || ratio > FXP_INIT_INT(1)) {
1487 CONF_ERROR("Invalid conf value",
1488 k, klen, v, vlen);
1489 } else {
1490 opt_hpa_opts.hugification_threshold =
1491 fxp_mul_frac(HUGEPAGE, ratio);
1492 }
1493 CONF_CONTINUE;
1494 }
1495
1496 CONF_HANDLE_UINT64_T(
1497 opt_hpa_opts.hugify_delay_ms, "hpa_hugify_delay_ms",
1498 0, 0, CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX,
1499 false);
1500
1501 CONF_HANDLE_UINT64_T(
1502 opt_hpa_opts.min_purge_interval_ms,
1503 "hpa_min_purge_interval_ms", 0, 0,
1504 CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false);
1505
1506 if (CONF_MATCH("hpa_dirty_mult")) {
1507 if (CONF_MATCH_VALUE("-1")) {
1508 opt_hpa_opts.dirty_mult = (fxp_t)-1;
1509 CONF_CONTINUE;
1510 }
1511 fxp_t ratio;
1512 char *end;
1513 bool err = fxp_parse(&ratio, v,
1514 &end);
1515 if (err || (size_t)(end - v) != vlen) {
1516 CONF_ERROR("Invalid conf value",
1517 k, klen, v, vlen);
1518 } else {
1519 opt_hpa_opts.dirty_mult = ratio;
1520 }
1521 CONF_CONTINUE;
1522 }
1523
1524 CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.nshards,
1525 "hpa_sec_nshards", 0, 0, CONF_CHECK_MIN,
1526 CONF_DONT_CHECK_MAX, true);
1527 CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_alloc,
1528 "hpa_sec_max_alloc", PAGE, 0, CONF_CHECK_MIN,
1529 CONF_DONT_CHECK_MAX, true);
1530 CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes,
1531 "hpa_sec_max_bytes", PAGE, 0, CONF_CHECK_MIN,
1532 CONF_DONT_CHECK_MAX, true);
1533 CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.bytes_after_flush,
1534 "hpa_sec_bytes_after_flush", PAGE, 0,
1535 CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
1536 CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.batch_fill_extra,
1537 "hpa_sec_batch_fill_extra", 0, HUGEPAGE_PAGES,
1538 CONF_CHECK_MIN, CONF_CHECK_MAX, true);
1539
1540 if (CONF_MATCH("slab_sizes")) {
1541 if (CONF_MATCH_VALUE("default")) {
1542 sc_data_init(sc_data);
1543 CONF_CONTINUE;
1544 }
1545 bool err;
1546 const char *slab_size_segment_cur = v;
1547 size_t vlen_left = vlen;
1548 do {
1549 size_t slab_start;
1550 size_t slab_end;
1551 size_t pgs;
1552 err = malloc_conf_multi_sizes_next(
1553 &slab_size_segment_cur,
1554 &vlen_left, &slab_start, &slab_end,
1555 &pgs);
1556 if (!err) {
1557 sc_data_update_slab_size(
1558 sc_data, slab_start,
1559 slab_end, (int)pgs);
1560 } else {
1561 CONF_ERROR("Invalid settings "
1562 "for slab_sizes",
1563 k, klen, v, vlen);
1564 }
1565 } while (!err && vlen_left > 0);
1566 CONF_CONTINUE;
1567 }
1568 if (config_prof) {
1569 CONF_HANDLE_BOOL(opt_prof, "prof")
1570 CONF_HANDLE_CHAR_P(opt_prof_prefix,
1571 "prof_prefix", "jeprof")
1572 CONF_HANDLE_BOOL(opt_prof_active, "prof_active")
1573 CONF_HANDLE_BOOL(opt_prof_thread_active_init,
1574 "prof_thread_active_init")
1575 CONF_HANDLE_SIZE_T(opt_lg_prof_sample,
1576 "lg_prof_sample", 0, (sizeof(uint64_t) << 3)
1577 - 1, CONF_DONT_CHECK_MIN, CONF_CHECK_MAX,
1578 true)
1579 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum")
1580 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval,
1581 "lg_prof_interval", -1,
1582 (sizeof(uint64_t) << 3) - 1)
1583 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump")
1584 CONF_HANDLE_BOOL(opt_prof_final, "prof_final")
1585 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak")
1586 CONF_HANDLE_BOOL(opt_prof_leak_error,
1587 "prof_leak_error")
1588 CONF_HANDLE_BOOL(opt_prof_log, "prof_log")
1589 CONF_HANDLE_SSIZE_T(opt_prof_recent_alloc_max,
1590 "prof_recent_alloc_max", -1, SSIZE_MAX)
1591 CONF_HANDLE_BOOL(opt_prof_stats, "prof_stats")
1592 CONF_HANDLE_BOOL(opt_prof_sys_thread_name,
1593 "prof_sys_thread_name")
1594 if (CONF_MATCH("prof_time_resolution")) {
1595 if (CONF_MATCH_VALUE("default")) {
1596 opt_prof_time_res =
1597 prof_time_res_default;
1598 } else if (CONF_MATCH_VALUE("high")) {
1599 if (!config_high_res_timer) {
1600 CONF_ERROR(
1601 "No high resolution"
1602 " timer support",
1603 k, klen, v, vlen);
1604 } else {
1605 opt_prof_time_res =
1606 prof_time_res_high;
1607 }
1608 } else {
1609 CONF_ERROR("Invalid conf value",
1610 k, klen, v, vlen);
1611 }
1612 CONF_CONTINUE;
1613 }
1614 /*
1615 * Undocumented. When set to false, don't
1616 * correct for an unbiasing bug in jeprof
1617 * attribution. This can be handy if you want
1618 * to get consistent numbers from your binary
1619 * across different jemalloc versions, even if
1620 * those numbers are incorrect. The default is
1621 * true.
1622 */
1623 CONF_HANDLE_BOOL(opt_prof_unbias, "prof_unbias")
1624 }
1625 if (config_log) {
1626 if (CONF_MATCH("log")) {
1627 size_t cpylen = (
1628 vlen <= sizeof(log_var_names) ?
1629 vlen : sizeof(log_var_names) - 1);
1630 strncpy(log_var_names, v, cpylen);
1631 log_var_names[cpylen] = '\0';
1632 CONF_CONTINUE;
1633 }
1634 }
1635 if (CONF_MATCH("thp")) {
1636 bool match = false;
1637 for (int m = 0; m < thp_mode_names_limit; m++) {
1638 if (strncmp(thp_mode_names[m],v, vlen)
1639 == 0) {
1640 if (!have_madvise_huge && !have_memcntl) {
1641 CONF_ERROR(
1642 "No THP support",
1643 k, klen, v, vlen);
1644 }
1645 opt_thp = m;
1646 match = true;
1647 break;
1648 }
1649 }
1650 if (!match) {
1651 CONF_ERROR("Invalid conf value",
1652 k, klen, v, vlen);
1653 }
1654 CONF_CONTINUE;
1655 }
1656 if (CONF_MATCH("zero_realloc")) {
1657 if (CONF_MATCH_VALUE("alloc")) {
1658 opt_zero_realloc_action
1659 = zero_realloc_action_alloc;
1660 } else if (CONF_MATCH_VALUE("free")) {
1661 opt_zero_realloc_action
1662 = zero_realloc_action_free;
1663 } else if (CONF_MATCH_VALUE("abort")) {
1664 opt_zero_realloc_action
1665 = zero_realloc_action_abort;
1666 } else {
1667 CONF_ERROR("Invalid conf value",
1668 k, klen, v, vlen);
1669 }
1670 CONF_CONTINUE;
1671 }
1672 if (config_uaf_detection &&
1673 CONF_MATCH("lg_san_uaf_align")) {
1674 ssize_t a;
1675 CONF_VALUE_READ(ssize_t, a)
1676 if (CONF_VALUE_READ_FAIL() || a < -1) {
1677 CONF_ERROR("Invalid conf value",
1678 k, klen, v, vlen);
1679 }
1680 if (a == -1) {
1681 opt_lg_san_uaf_align = -1;
1682 CONF_CONTINUE;
1683 }
1684
1685 /* clip if necessary */
1686 ssize_t max_allowed = (sizeof(size_t) << 3) - 1;
1687 ssize_t min_allowed = LG_PAGE;
1688 if (a > max_allowed) {
1689 a = max_allowed;
1690 } else if (a < min_allowed) {
1691 a = min_allowed;
1692 }
1693
1694 opt_lg_san_uaf_align = a;
1695 CONF_CONTINUE;
1696 }
1697
1698 CONF_HANDLE_SIZE_T(opt_san_guard_small,
1699 "san_guard_small", 0, SIZE_T_MAX,
1700 CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false)
1701 CONF_HANDLE_SIZE_T(opt_san_guard_large,
1702 "san_guard_large", 0, SIZE_T_MAX,
1703 CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false)
1704
1705 CONF_ERROR("Invalid conf pair", k, klen, v, vlen);
1706#undef CONF_ERROR
1707#undef CONF_CONTINUE
1708#undef CONF_MATCH
1709#undef CONF_MATCH_VALUE
1710#undef CONF_HANDLE_BOOL
1711#undef CONF_DONT_CHECK_MIN
1712#undef CONF_CHECK_MIN
1713#undef CONF_DONT_CHECK_MAX
1714#undef CONF_CHECK_MAX
1715#undef CONF_HANDLE_T
1716#undef CONF_HANDLE_T_U
1717#undef CONF_HANDLE_T_SIGNED
1718#undef CONF_HANDLE_UNSIGNED
1719#undef CONF_HANDLE_SIZE_T
1720#undef CONF_HANDLE_SSIZE_T
1721#undef CONF_HANDLE_CHAR_P
1722 /* Re-enable diagnostic "-Wtype-limits" */
1723 JEMALLOC_DIAGNOSTIC_POP
1724 }
1725 if (opt_abort_conf && had_conf_error) {
1726 malloc_abort_invalid_conf();
1727 }
1728 }
1729 atomic_store_b(&log_init_done, true, ATOMIC_RELEASE);
1730}
1731
1732static bool
1733malloc_conf_init_check_deps(void) {
1734 if (opt_prof_leak_error && !opt_prof_final) {
1735 malloc_printf("<jemalloc>: prof_leak_error is set w/o "
1736 "prof_final.\n");
1737 return true;
1738 }
1739
1740 return false;
1741}
1742
1743static void
1744malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) {
1745 const char *opts_cache[MALLOC_CONF_NSOURCES] = {NULL, NULL, NULL, NULL,
1746 NULL};
1747 char buf[PATH_MAX + 1];
1748
1749 /* The first call only set the confirm_conf option and opts_cache */
1750 malloc_conf_init_helper(NULL, NULL, true, opts_cache, buf);
1751 malloc_conf_init_helper(sc_data, bin_shard_sizes, false, opts_cache,
1752 NULL);
1753 if (malloc_conf_init_check_deps()) {
1754 /* check_deps does warning msg only; abort below if needed. */
1755 if (opt_abort_conf) {
1756 malloc_abort_invalid_conf();
1757 }
1758 }
1759}
1760
1761#undef MALLOC_CONF_NSOURCES
1762
1763static bool
1764malloc_init_hard_needed(void) {
1765 if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state ==
1766 malloc_init_recursible)) {
1767 /*
1768 * Another thread initialized the allocator before this one
1769 * acquired init_lock, or this thread is the initializing
1770 * thread, and it is recursively allocating.
1771 */
1772 return false;
1773 }
1774#ifdef JEMALLOC_THREADED_INIT
1775 if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) {
1776 /* Busy-wait until the initializing thread completes. */
1777 spin_t spinner = SPIN_INITIALIZER;
1778 do {
1779 malloc_mutex_unlock(TSDN_NULL, &init_lock);
1780 spin_adaptive(&spinner);
1781 malloc_mutex_lock(TSDN_NULL, &init_lock);
1782 } while (!malloc_initialized());
1783 return false;
1784 }
1785#endif
1786 return true;
1787}
1788
1789static bool
1790malloc_init_hard_a0_locked() {
1791 malloc_initializer = INITIALIZER;
1792
1793 JEMALLOC_DIAGNOSTIC_PUSH
1794 JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
1795 sc_data_t sc_data = {0};
1796 JEMALLOC_DIAGNOSTIC_POP
1797
1798 /*
1799 * Ordering here is somewhat tricky; we need sc_boot() first, since that
1800 * determines what the size classes will be, and then
1801 * malloc_conf_init(), since any slab size tweaking will need to be done
1802 * before sz_boot and bin_info_boot, which assume that the values they
1803 * read out of sc_data_global are final.
1804 */
1805 sc_boot(&sc_data);
1806 unsigned bin_shard_sizes[SC_NBINS];
1807 bin_shard_sizes_boot(bin_shard_sizes);
1808 /*
1809 * prof_boot0 only initializes opt_prof_prefix. We need to do it before
1810 * we parse malloc_conf options, in case malloc_conf parsing overwrites
1811 * it.
1812 */
1813 if (config_prof) {
1814 prof_boot0();
1815 }
1816 malloc_conf_init(&sc_data, bin_shard_sizes);
1817 san_init(opt_lg_san_uaf_align);
1818 sz_boot(&sc_data, opt_cache_oblivious);
1819 bin_info_boot(&sc_data, bin_shard_sizes);
1820
1821 if (opt_stats_print) {
1822 /* Print statistics at exit. */
1823 if (atexit(stats_print_atexit) != 0) {
1824 malloc_write("<jemalloc>: Error in atexit()\n");
1825 if (opt_abort) {
1826 abort();
1827 }
1828 }
1829 }
1830
1831 if (stats_boot()) {
1832 return true;
1833 }
1834 if (pages_boot()) {
1835 return true;
1836 }
1837 if (base_boot(TSDN_NULL)) {
1838 return true;
1839 }
1840 /* emap_global is static, hence zeroed. */
1841 if (emap_init(&arena_emap_global, b0get(), /* zeroed */ true)) {
1842 return true;
1843 }
1844 if (extent_boot()) {
1845 return true;
1846 }
1847 if (ctl_boot()) {
1848 return true;
1849 }
1850 if (config_prof) {
1851 prof_boot1();
1852 }
1853 if (opt_hpa && !hpa_supported()) {
1854 malloc_printf("<jemalloc>: HPA not supported in the current "
1855 "configuration; %s.",
1856 opt_abort_conf ? "aborting" : "disabling");
1857 if (opt_abort_conf) {
1858 malloc_abort_invalid_conf();
1859 } else {
1860 opt_hpa = false;
1861 }
1862 }
1863 if (arena_boot(&sc_data, b0get(), opt_hpa)) {
1864 return true;
1865 }
1866 if (tcache_boot(TSDN_NULL, b0get())) {
1867 return true;
1868 }
1869 if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS,
1870 malloc_mutex_rank_exclusive)) {
1871 return true;
1872 }
1873 hook_boot();
1874 /*
1875 * Create enough scaffolding to allow recursive allocation in
1876 * malloc_ncpus().
1877 */
1878 narenas_auto = 1;
1879 manual_arena_base = narenas_auto + 1;
1880 memset(arenas, 0, sizeof(arena_t *) * narenas_auto);
1881 /*
1882 * Initialize one arena here. The rest are lazily created in
1883 * arena_choose_hard().
1884 */
1885 if (arena_init(TSDN_NULL, 0, &arena_config_default) == NULL) {
1886 return true;
1887 }
1888 a0 = arena_get(TSDN_NULL, 0, false);
1889
1890 if (opt_hpa && !hpa_supported()) {
1891 malloc_printf("<jemalloc>: HPA not supported in the current "
1892 "configuration; %s.",
1893 opt_abort_conf ? "aborting" : "disabling");
1894 if (opt_abort_conf) {
1895 malloc_abort_invalid_conf();
1896 } else {
1897 opt_hpa = false;
1898 }
1899 } else if (opt_hpa) {
1900 hpa_shard_opts_t hpa_shard_opts = opt_hpa_opts;
1901 hpa_shard_opts.deferral_allowed = background_thread_enabled();
1902 if (pa_shard_enable_hpa(TSDN_NULL, &a0->pa_shard,
1903 &hpa_shard_opts, &opt_hpa_sec_opts)) {
1904 return true;
1905 }
1906 }
1907
1908 malloc_init_state = malloc_init_a0_initialized;
1909
1910 return false;
1911}
1912
1913static bool
1914malloc_init_hard_a0(void) {
1915 bool ret;
1916
1917 malloc_mutex_lock(TSDN_NULL, &init_lock);
1918 ret = malloc_init_hard_a0_locked();
1919 malloc_mutex_unlock(TSDN_NULL, &init_lock);
1920 return ret;
1921}
1922
1923/* Initialize data structures which may trigger recursive allocation. */
1924static bool
1925malloc_init_hard_recursible(void) {
1926 malloc_init_state = malloc_init_recursible;
1927
1928 ncpus = malloc_ncpus();
1929 if (opt_percpu_arena != percpu_arena_disabled) {
1930 bool cpu_count_is_deterministic =
1931 malloc_cpu_count_is_deterministic();
1932 if (!cpu_count_is_deterministic) {
1933 /*
1934 * If # of CPU is not deterministic, and narenas not
1935 * specified, disables per cpu arena since it may not
1936 * detect CPU IDs properly.
1937 */
1938 if (opt_narenas == 0) {
1939 opt_percpu_arena = percpu_arena_disabled;
1940 malloc_write("<jemalloc>: Number of CPUs "
1941 "detected is not deterministic. Per-CPU "
1942 "arena disabled.\n");
1943 if (opt_abort_conf) {
1944 malloc_abort_invalid_conf();
1945 }
1946 if (opt_abort) {
1947 abort();
1948 }
1949 }
1950 }
1951 }
1952
1953#if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \
1954 && !defined(JEMALLOC_ZONE) && !defined(_WIN32) && \
1955 !defined(__native_client__))
1956 /* LinuxThreads' pthread_atfork() allocates. */
1957 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
1958 jemalloc_postfork_child) != 0) {
1959 malloc_write("<jemalloc>: Error in pthread_atfork()\n");
1960 if (opt_abort) {
1961 abort();
1962 }
1963 return true;
1964 }
1965#endif
1966
1967 if (background_thread_boot0()) {
1968 return true;
1969 }
1970
1971 return false;
1972}
1973
1974static unsigned
1975malloc_narenas_default(void) {
1976 assert(ncpus > 0);
1977 /*
1978 * For SMP systems, create more than one arena per CPU by
1979 * default.
1980 */
1981 if (ncpus > 1) {
1982 fxp_t fxp_ncpus = FXP_INIT_INT(ncpus);
1983 fxp_t goal = fxp_mul(fxp_ncpus, opt_narenas_ratio);
1984 uint32_t int_goal = fxp_round_nearest(goal);
1985 if (int_goal == 0) {
1986 return 1;
1987 }
1988 return int_goal;
1989 } else {
1990 return 1;
1991 }
1992}
1993
1994static percpu_arena_mode_t
1995percpu_arena_as_initialized(percpu_arena_mode_t mode) {
1996 assert(!malloc_initialized());
1997 assert(mode <= percpu_arena_disabled);
1998
1999 if (mode != percpu_arena_disabled) {
2000 mode += percpu_arena_mode_enabled_base;
2001 }
2002
2003 return mode;
2004}
2005
2006static bool
2007malloc_init_narenas(void) {
2008 assert(ncpus > 0);
2009
2010 if (opt_percpu_arena != percpu_arena_disabled) {
2011 if (!have_percpu_arena || malloc_getcpu() < 0) {
2012 opt_percpu_arena = percpu_arena_disabled;
2013 malloc_printf("<jemalloc>: perCPU arena getcpu() not "
2014 "available. Setting narenas to %u.\n", opt_narenas ?
2015 opt_narenas : malloc_narenas_default());
2016 if (opt_abort) {
2017 abort();
2018 }
2019 } else {
2020 if (ncpus >= MALLOCX_ARENA_LIMIT) {
2021 malloc_printf("<jemalloc>: narenas w/ percpu"
2022 "arena beyond limit (%d)\n", ncpus);
2023 if (opt_abort) {
2024 abort();
2025 }
2026 return true;
2027 }
2028 /* NB: opt_percpu_arena isn't fully initialized yet. */
2029 if (percpu_arena_as_initialized(opt_percpu_arena) ==
2030 per_phycpu_arena && ncpus % 2 != 0) {
2031 malloc_printf("<jemalloc>: invalid "
2032 "configuration -- per physical CPU arena "
2033 "with odd number (%u) of CPUs (no hyper "
2034 "threading?).\n", ncpus);
2035 if (opt_abort)
2036 abort();
2037 }
2038 unsigned n = percpu_arena_ind_limit(
2039 percpu_arena_as_initialized(opt_percpu_arena));
2040 if (opt_narenas < n) {
2041 /*
2042 * If narenas is specified with percpu_arena
2043 * enabled, actual narenas is set as the greater
2044 * of the two. percpu_arena_choose will be free
2045 * to use any of the arenas based on CPU
2046 * id. This is conservative (at a small cost)
2047 * but ensures correctness.
2048 *
2049 * If for some reason the ncpus determined at
2050 * boot is not the actual number (e.g. because
2051 * of affinity setting from numactl), reserving
2052 * narenas this way provides a workaround for
2053 * percpu_arena.
2054 */
2055 opt_narenas = n;
2056 }
2057 }
2058 }
2059 if (opt_narenas == 0) {
2060 opt_narenas = malloc_narenas_default();
2061 }
2062 assert(opt_narenas > 0);
2063
2064 narenas_auto = opt_narenas;
2065 /*
2066 * Limit the number of arenas to the indexing range of MALLOCX_ARENA().
2067 */
2068 if (narenas_auto >= MALLOCX_ARENA_LIMIT) {
2069 narenas_auto = MALLOCX_ARENA_LIMIT - 1;
2070 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n",
2071 narenas_auto);
2072 }
2073 narenas_total_set(narenas_auto);
2074 if (arena_init_huge()) {
2075 narenas_total_inc();
2076 }
2077 manual_arena_base = narenas_total_get();
2078
2079 return false;
2080}
2081
2082static void
2083malloc_init_percpu(void) {
2084 opt_percpu_arena = percpu_arena_as_initialized(opt_percpu_arena);
2085}
2086
2087static bool
2088malloc_init_hard_finish(void) {
2089 if (malloc_mutex_boot()) {
2090 return true;
2091 }
2092
2093 malloc_init_state = malloc_init_initialized;
2094 malloc_slow_flag_init();
2095
2096 return false;
2097}
2098
2099static void
2100malloc_init_hard_cleanup(tsdn_t *tsdn, bool reentrancy_set) {
2101 malloc_mutex_assert_owner(tsdn, &init_lock);
2102 malloc_mutex_unlock(tsdn, &init_lock);
2103 if (reentrancy_set) {
2104 assert(!tsdn_null(tsdn));
2105 tsd_t *tsd = tsdn_tsd(tsdn);
2106 assert(tsd_reentrancy_level_get(tsd) > 0);
2107 post_reentrancy(tsd);
2108 }
2109}
2110
2111static bool
2112malloc_init_hard(void) {
2113 tsd_t *tsd;
2114
2115#if defined(_WIN32) && _WIN32_WINNT < 0x0600
2116 _init_init_lock();
2117#endif
2118 malloc_mutex_lock(TSDN_NULL, &init_lock);
2119
2120#define UNLOCK_RETURN(tsdn, ret, reentrancy) \
2121 malloc_init_hard_cleanup(tsdn, reentrancy); \
2122 return ret;
2123
2124 if (!malloc_init_hard_needed()) {
2125 UNLOCK_RETURN(TSDN_NULL, false, false)
2126 }
2127
2128 if (malloc_init_state != malloc_init_a0_initialized &&
2129 malloc_init_hard_a0_locked()) {
2130 UNLOCK_RETURN(TSDN_NULL, true, false)
2131 }
2132
2133 malloc_mutex_unlock(TSDN_NULL, &init_lock);
2134 /* Recursive allocation relies on functional tsd. */
2135 tsd = malloc_tsd_boot0();
2136 if (tsd == NULL) {
2137 return true;
2138 }
2139 if (malloc_init_hard_recursible()) {
2140 return true;
2141 }
2142
2143 malloc_mutex_lock(tsd_tsdn(tsd), &init_lock);
2144 /* Set reentrancy level to 1 during init. */
2145 pre_reentrancy(tsd, NULL);
2146 /* Initialize narenas before prof_boot2 (for allocation). */
2147 if (malloc_init_narenas()
2148 || background_thread_boot1(tsd_tsdn(tsd), b0get())) {
2149 UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
2150 }
2151 if (config_prof && prof_boot2(tsd, b0get())) {
2152 UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
2153 }
2154
2155 malloc_init_percpu();
2156
2157 if (malloc_init_hard_finish()) {
2158 UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
2159 }
2160 post_reentrancy(tsd);
2161 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
2162
2163 witness_assert_lockless(witness_tsd_tsdn(
2164 tsd_witness_tsdp_get_unsafe(tsd)));
2165 malloc_tsd_boot1();
2166 /* Update TSD after tsd_boot1. */
2167 tsd = tsd_fetch();
2168 if (opt_background_thread) {
2169 assert(have_background_thread);
2170 /*
2171 * Need to finish init & unlock first before creating background
2172 * threads (pthread_create depends on malloc). ctl_init (which
2173 * sets isthreaded) needs to be called without holding any lock.
2174 */
2175 background_thread_ctl_init(tsd_tsdn(tsd));
2176 if (background_thread_create(tsd, 0)) {
2177 return true;
2178 }
2179 }
2180#undef UNLOCK_RETURN
2181 return false;
2182}
2183
2184/*
2185 * End initialization functions.
2186 */
2187/******************************************************************************/
2188/*
2189 * Begin allocation-path internal functions and data structures.
2190 */
2191
2192/*
2193 * Settings determined by the documented behavior of the allocation functions.
2194 */
2195typedef struct static_opts_s static_opts_t;
2196struct static_opts_s {
2197 /* Whether or not allocation size may overflow. */
2198 bool may_overflow;
2199
2200 /*
2201 * Whether or not allocations (with alignment) of size 0 should be
2202 * treated as size 1.
2203 */
2204 bool bump_empty_aligned_alloc;
2205 /*
2206 * Whether to assert that allocations are not of size 0 (after any
2207 * bumping).
2208 */
2209 bool assert_nonempty_alloc;
2210
2211 /*
2212 * Whether or not to modify the 'result' argument to malloc in case of
2213 * error.
2214 */
2215 bool null_out_result_on_error;
2216 /* Whether to set errno when we encounter an error condition. */
2217 bool set_errno_on_error;
2218
2219 /*
2220 * The minimum valid alignment for functions requesting aligned storage.
2221 */
2222 size_t min_alignment;
2223
2224 /* The error string to use if we oom. */
2225 const char *oom_string;
2226 /* The error string to use if the passed-in alignment is invalid. */
2227 const char *invalid_alignment_string;
2228
2229 /*
2230 * False if we're configured to skip some time-consuming operations.
2231 *
2232 * This isn't really a malloc "behavior", but it acts as a useful
2233 * summary of several other static (or at least, static after program
2234 * initialization) options.
2235 */
2236 bool slow;
2237 /*
2238 * Return size.
2239 */
2240 bool usize;
2241};
2242
2243JEMALLOC_ALWAYS_INLINE void
2244static_opts_init(static_opts_t *static_opts) {
2245 static_opts->may_overflow = false;
2246 static_opts->bump_empty_aligned_alloc = false;
2247 static_opts->assert_nonempty_alloc = false;
2248 static_opts->null_out_result_on_error = false;
2249 static_opts->set_errno_on_error = false;
2250 static_opts->min_alignment = 0;
2251 static_opts->oom_string = "";
2252 static_opts->invalid_alignment_string = "";
2253 static_opts->slow = false;
2254 static_opts->usize = false;
2255}
2256
2257/*
2258 * These correspond to the macros in jemalloc/jemalloc_macros.h. Broadly, we
2259 * should have one constant here per magic value there. Note however that the
2260 * representations need not be related.
2261 */
2262#define TCACHE_IND_NONE ((unsigned)-1)
2263#define TCACHE_IND_AUTOMATIC ((unsigned)-2)
2264#define ARENA_IND_AUTOMATIC ((unsigned)-1)
2265
2266typedef struct dynamic_opts_s dynamic_opts_t;
2267struct dynamic_opts_s {
2268 void **result;
2269 size_t usize;
2270 size_t num_items;
2271 size_t item_size;
2272 size_t alignment;
2273 bool zero;
2274 unsigned tcache_ind;
2275 unsigned arena_ind;
2276};
2277
2278JEMALLOC_ALWAYS_INLINE void
2279dynamic_opts_init(dynamic_opts_t *dynamic_opts) {
2280 dynamic_opts->result = NULL;
2281 dynamic_opts->usize = 0;
2282 dynamic_opts->num_items = 0;
2283 dynamic_opts->item_size = 0;
2284 dynamic_opts->alignment = 0;
2285 dynamic_opts->zero = false;
2286 dynamic_opts->tcache_ind = TCACHE_IND_AUTOMATIC;
2287 dynamic_opts->arena_ind = ARENA_IND_AUTOMATIC;
2288}
2289
2290/*
2291 * ind parameter is optional and is only checked and filled if alignment == 0;
2292 * return true if result is out of range.
2293 */
2294JEMALLOC_ALWAYS_INLINE bool
2295aligned_usize_get(size_t size, size_t alignment, size_t *usize, szind_t *ind,
2296 bool bump_empty_aligned_alloc) {
2297 assert(usize != NULL);
2298 if (alignment == 0) {
2299 if (ind != NULL) {
2300 *ind = sz_size2index(size);
2301 if (unlikely(*ind >= SC_NSIZES)) {
2302 return true;
2303 }
2304 *usize = sz_index2size(*ind);
2305 assert(*usize > 0 && *usize <= SC_LARGE_MAXCLASS);
2306 return false;
2307 }
2308 *usize = sz_s2u(size);
2309 } else {
2310 if (bump_empty_aligned_alloc && unlikely(size == 0)) {
2311 size = 1;
2312 }
2313 *usize = sz_sa2u(size, alignment);
2314 }
2315 if (unlikely(*usize == 0 || *usize > SC_LARGE_MAXCLASS)) {
2316 return true;
2317 }
2318 return false;
2319}
2320
2321JEMALLOC_ALWAYS_INLINE bool
2322zero_get(bool guarantee, bool slow) {
2323 if (config_fill && slow && unlikely(opt_zero)) {
2324 return true;
2325 } else {
2326 return guarantee;
2327 }
2328}
2329
2330JEMALLOC_ALWAYS_INLINE tcache_t *
2331tcache_get_from_ind(tsd_t *tsd, unsigned tcache_ind, bool slow, bool is_alloc) {
2332 tcache_t *tcache;
2333 if (tcache_ind == TCACHE_IND_AUTOMATIC) {
2334 if (likely(!slow)) {
2335 /* Getting tcache ptr unconditionally. */
2336 tcache = tsd_tcachep_get(tsd);
2337 assert(tcache == tcache_get(tsd));
2338 } else if (is_alloc ||
2339 likely(tsd_reentrancy_level_get(tsd) == 0)) {
2340 tcache = tcache_get(tsd);
2341 } else {
2342 tcache = NULL;
2343 }
2344 } else {
2345 /*
2346 * Should not specify tcache on deallocation path when being
2347 * reentrant.
2348 */
2349 assert(is_alloc || tsd_reentrancy_level_get(tsd) == 0 ||
2350 tsd_state_nocleanup(tsd));
2351 if (tcache_ind == TCACHE_IND_NONE) {
2352 tcache = NULL;
2353 } else {
2354 tcache = tcaches_get(tsd, tcache_ind);
2355 }
2356 }
2357 return tcache;
2358}
2359
2360/* Return true if a manual arena is specified and arena_get() OOMs. */
2361JEMALLOC_ALWAYS_INLINE bool
2362arena_get_from_ind(tsd_t *tsd, unsigned arena_ind, arena_t **arena_p) {
2363 if (arena_ind == ARENA_IND_AUTOMATIC) {
2364 /*
2365 * In case of automatic arena management, we defer arena
2366 * computation until as late as we can, hoping to fill the
2367 * allocation out of the tcache.
2368 */
2369 *arena_p = NULL;
2370 } else {
2371 *arena_p = arena_get(tsd_tsdn(tsd), arena_ind, true);
2372 if (unlikely(*arena_p == NULL) && arena_ind >= narenas_auto) {
2373 return true;
2374 }
2375 }
2376 return false;
2377}
2378
2379/* ind is ignored if dopts->alignment > 0. */
2380JEMALLOC_ALWAYS_INLINE void *
2381imalloc_no_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd,
2382 size_t size, size_t usize, szind_t ind) {
2383 /* Fill in the tcache. */
2384 tcache_t *tcache = tcache_get_from_ind(tsd, dopts->tcache_ind,
2385 sopts->slow, /* is_alloc */ true);
2386
2387 /* Fill in the arena. */
2388 arena_t *arena;
2389 if (arena_get_from_ind(tsd, dopts->arena_ind, &arena)) {
2390 return NULL;
2391 }
2392
2393 if (unlikely(dopts->alignment != 0)) {
2394 return ipalloct(tsd_tsdn(tsd), usize, dopts->alignment,
2395 dopts->zero, tcache, arena);
2396 }
2397
2398 return iallocztm(tsd_tsdn(tsd), size, ind, dopts->zero, tcache, false,
2399 arena, sopts->slow);
2400}
2401
2402JEMALLOC_ALWAYS_INLINE void *
2403imalloc_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd,
2404 size_t usize, szind_t ind) {
2405 void *ret;
2406
2407 /*
2408 * For small allocations, sampling bumps the usize. If so, we allocate
2409 * from the ind_large bucket.
2410 */
2411 szind_t ind_large;
2412 size_t bumped_usize = usize;
2413
2414 dopts->alignment = prof_sample_align(dopts->alignment);
2415 if (usize <= SC_SMALL_MAXCLASS) {
2416 assert(((dopts->alignment == 0) ?
2417 sz_s2u(SC_LARGE_MINCLASS) :
2418 sz_sa2u(SC_LARGE_MINCLASS, dopts->alignment))
2419 == SC_LARGE_MINCLASS);
2420 ind_large = sz_size2index(SC_LARGE_MINCLASS);
2421 bumped_usize = sz_s2u(SC_LARGE_MINCLASS);
2422 ret = imalloc_no_sample(sopts, dopts, tsd, bumped_usize,
2423 bumped_usize, ind_large);
2424 if (unlikely(ret == NULL)) {
2425 return NULL;
2426 }
2427 arena_prof_promote(tsd_tsdn(tsd), ret, usize);
2428 } else {
2429 ret = imalloc_no_sample(sopts, dopts, tsd, usize, usize, ind);
2430 }
2431 assert(prof_sample_aligned(ret));
2432
2433 return ret;
2434}
2435
2436/*
2437 * Returns true if the allocation will overflow, and false otherwise. Sets
2438 * *size to the product either way.
2439 */
2440JEMALLOC_ALWAYS_INLINE bool
2441compute_size_with_overflow(bool may_overflow, dynamic_opts_t *dopts,
2442 size_t *size) {
2443 /*
2444 * This function is just num_items * item_size, except that we may have
2445 * to check for overflow.
2446 */
2447
2448 if (!may_overflow) {
2449 assert(dopts->num_items == 1);
2450 *size = dopts->item_size;
2451 return false;
2452 }
2453
2454 /* A size_t with its high-half bits all set to 1. */
2455 static const size_t high_bits = SIZE_T_MAX << (sizeof(size_t) * 8 / 2);
2456
2457 *size = dopts->item_size * dopts->num_items;
2458
2459 if (unlikely(*size == 0)) {
2460 return (dopts->num_items != 0 && dopts->item_size != 0);
2461 }
2462
2463 /*
2464 * We got a non-zero size, but we don't know if we overflowed to get
2465 * there. To avoid having to do a divide, we'll be clever and note that
2466 * if both A and B can be represented in N/2 bits, then their product
2467 * can be represented in N bits (without the possibility of overflow).
2468 */
2469 if (likely((high_bits & (dopts->num_items | dopts->item_size)) == 0)) {
2470 return false;
2471 }
2472 if (likely(*size / dopts->item_size == dopts->num_items)) {
2473 return false;
2474 }
2475 return true;
2476}
2477
2478JEMALLOC_ALWAYS_INLINE int
2479imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
2480 /* Where the actual allocated memory will live. */
2481 void *allocation = NULL;
2482 /* Filled in by compute_size_with_overflow below. */
2483 size_t size = 0;
2484 /*
2485 * The zero initialization for ind is actually dead store, in that its
2486 * value is reset before any branch on its value is taken. Sometimes
2487 * though, it's convenient to pass it as arguments before this point.
2488 * To avoid undefined behavior then, we initialize it with dummy stores.
2489 */
2490 szind_t ind = 0;
2491 /* usize will always be properly initialized. */
2492 size_t usize;
2493
2494 /* Reentrancy is only checked on slow path. */
2495 int8_t reentrancy_level;
2496
2497 /* Compute the amount of memory the user wants. */
2498 if (unlikely(compute_size_with_overflow(sopts->may_overflow, dopts,
2499 &size))) {
2500 goto label_oom;
2501 }
2502
2503 if (unlikely(dopts->alignment < sopts->min_alignment
2504 || (dopts->alignment & (dopts->alignment - 1)) != 0)) {
2505 goto label_invalid_alignment;
2506 }
2507
2508 /* This is the beginning of the "core" algorithm. */
2509 dopts->zero = zero_get(dopts->zero, sopts->slow);
2510 if (aligned_usize_get(size, dopts->alignment, &usize, &ind,
2511 sopts->bump_empty_aligned_alloc)) {
2512 goto label_oom;
2513 }
2514 dopts->usize = usize;
2515 /* Validate the user input. */
2516 if (sopts->assert_nonempty_alloc) {
2517 assert (size != 0);
2518 }
2519
2520 check_entry_exit_locking(tsd_tsdn(tsd));
2521
2522 /*
2523 * If we need to handle reentrancy, we can do it out of a
2524 * known-initialized arena (i.e. arena 0).
2525 */
2526 reentrancy_level = tsd_reentrancy_level_get(tsd);
2527 if (sopts->slow && unlikely(reentrancy_level > 0)) {
2528 /*
2529 * We should never specify particular arenas or tcaches from
2530 * within our internal allocations.
2531 */
2532 assert(dopts->tcache_ind == TCACHE_IND_AUTOMATIC ||
2533 dopts->tcache_ind == TCACHE_IND_NONE);
2534 assert(dopts->arena_ind == ARENA_IND_AUTOMATIC);
2535 dopts->tcache_ind = TCACHE_IND_NONE;
2536 /* We know that arena 0 has already been initialized. */
2537 dopts->arena_ind = 0;
2538 }
2539
2540 /*
2541 * If dopts->alignment > 0, then ind is still 0, but usize was computed
2542 * in the previous if statement. Down the positive alignment path,
2543 * imalloc_no_sample and imalloc_sample will ignore ind.
2544 */
2545
2546 /* If profiling is on, get our profiling context. */
2547 if (config_prof && opt_prof) {
2548 bool prof_active = prof_active_get_unlocked();
2549 bool sample_event = te_prof_sample_event_lookahead(tsd, usize);
2550 prof_tctx_t *tctx = prof_alloc_prep(tsd, prof_active,
2551 sample_event);
2552
2553 emap_alloc_ctx_t alloc_ctx;
2554 if (likely((uintptr_t)tctx == (uintptr_t)1U)) {
2555 alloc_ctx.slab = (usize <= SC_SMALL_MAXCLASS);
2556 allocation = imalloc_no_sample(
2557 sopts, dopts, tsd, usize, usize, ind);
2558 } else if ((uintptr_t)tctx > (uintptr_t)1U) {
2559 allocation = imalloc_sample(
2560 sopts, dopts, tsd, usize, ind);
2561 alloc_ctx.slab = false;
2562 } else {
2563 allocation = NULL;
2564 }
2565
2566 if (unlikely(allocation == NULL)) {
2567 prof_alloc_rollback(tsd, tctx);
2568 goto label_oom;
2569 }
2570 prof_malloc(tsd, allocation, size, usize, &alloc_ctx, tctx);
2571 } else {
2572 assert(!opt_prof);
2573 allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize,
2574 ind);
2575 if (unlikely(allocation == NULL)) {
2576 goto label_oom;
2577 }
2578 }
2579
2580 /*
2581 * Allocation has been done at this point. We still have some
2582 * post-allocation work to do though.
2583 */
2584
2585 thread_alloc_event(tsd, usize);
2586
2587 assert(dopts->alignment == 0
2588 || ((uintptr_t)allocation & (dopts->alignment - 1)) == ZU(0));
2589
2590 assert(usize == isalloc(tsd_tsdn(tsd), allocation));
2591
2592 if (config_fill && sopts->slow && !dopts->zero
2593 && unlikely(opt_junk_alloc)) {
2594 junk_alloc_callback(allocation, usize);
2595 }
2596
2597 if (sopts->slow) {
2598 UTRACE(0, size, allocation);
2599 }
2600
2601 /* Success! */
2602 check_entry_exit_locking(tsd_tsdn(tsd));
2603 *dopts->result = allocation;
2604 return 0;
2605
2606label_oom:
2607 if (unlikely(sopts->slow) && config_xmalloc && unlikely(opt_xmalloc)) {
2608 malloc_write(sopts->oom_string);
2609 abort();
2610 }
2611
2612 if (sopts->slow) {
2613 UTRACE(NULL, size, NULL);
2614 }
2615
2616 check_entry_exit_locking(tsd_tsdn(tsd));
2617
2618 if (sopts->set_errno_on_error) {
2619 set_errno(ENOMEM);
2620 }
2621
2622 if (sopts->null_out_result_on_error) {
2623 *dopts->result = NULL;
2624 }
2625
2626 return ENOMEM;
2627
2628 /*
2629 * This label is only jumped to by one goto; we move it out of line
2630 * anyways to avoid obscuring the non-error paths, and for symmetry with
2631 * the oom case.
2632 */
2633label_invalid_alignment:
2634 if (config_xmalloc && unlikely(opt_xmalloc)) {
2635 malloc_write(sopts->invalid_alignment_string);
2636 abort();
2637 }
2638
2639 if (sopts->set_errno_on_error) {
2640 set_errno(EINVAL);
2641 }
2642
2643 if (sopts->slow) {
2644 UTRACE(NULL, size, NULL);
2645 }
2646
2647 check_entry_exit_locking(tsd_tsdn(tsd));
2648
2649 if (sopts->null_out_result_on_error) {
2650 *dopts->result = NULL;
2651 }
2652
2653 return EINVAL;
2654}
2655
2656JEMALLOC_ALWAYS_INLINE bool
2657imalloc_init_check(static_opts_t *sopts, dynamic_opts_t *dopts) {
2658 if (unlikely(!malloc_initialized()) && unlikely(malloc_init())) {
2659 if (config_xmalloc && unlikely(opt_xmalloc)) {
2660 malloc_write(sopts->oom_string);
2661 abort();
2662 }
2663 UTRACE(NULL, dopts->num_items * dopts->item_size, NULL);
2664 set_errno(ENOMEM);
2665 *dopts->result = NULL;
2666
2667 return false;
2668 }
2669
2670 return true;
2671}
2672
2673/* Returns the errno-style error code of the allocation. */
2674JEMALLOC_ALWAYS_INLINE int
2675imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) {
2676 if (tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) {
2677 return ENOMEM;
2678 }
2679
2680 /* We always need the tsd. Let's grab it right away. */
2681 tsd_t *tsd = tsd_fetch();
2682 assert(tsd);
2683 if (likely(tsd_fast(tsd))) {
2684 /* Fast and common path. */
2685 tsd_assert_fast(tsd);
2686 sopts->slow = false;
2687 return imalloc_body(sopts, dopts, tsd);
2688 } else {
2689 if (!tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) {
2690 return ENOMEM;
2691 }
2692
2693 sopts->slow = true;
2694 return imalloc_body(sopts, dopts, tsd);
2695 }
2696}
2697
2698JEMALLOC_NOINLINE
2699void *
2700malloc_default(size_t size, size_t *usize) {
2701 void *ret;
2702 static_opts_t sopts;
2703 dynamic_opts_t dopts;
2704
2705 /*
2706 * This variant has logging hook on exit but not on entry. It's callled
2707 * only by je_malloc, below, which emits the entry one for us (and, if
2708 * it calls us, does so only via tail call).
2709 */
2710
2711 static_opts_init(&sopts);
2712 dynamic_opts_init(&dopts);
2713
2714 sopts.null_out_result_on_error = true;
2715 sopts.set_errno_on_error = true;
2716 sopts.oom_string = "<jemalloc>: Error in malloc(): out of memory\n";
2717
2718 dopts.result = &ret;
2719 dopts.num_items = 1;
2720 dopts.item_size = size;
2721
2722 imalloc(&sopts, &dopts);
2723 /*
2724 * Note that this branch gets optimized away -- it immediately follows
2725 * the check on tsd_fast that sets sopts.slow.
2726 */
2727 if (sopts.slow) {
2728 uintptr_t args[3] = {size};
2729 hook_invoke_alloc(hook_alloc_malloc, ret, (uintptr_t)ret, args);
2730 }
2731
2732 LOG("core.malloc.exit", "result: %p", ret);
2733
2734 if (usize) *usize = dopts.usize;
2735 return ret;
2736}
2737
2738/******************************************************************************/
2739/*
2740 * Begin malloc(3)-compatible functions.
2741 */
2742
2743static inline void *je_malloc_internal(size_t size, size_t *usize) {
2744 return imalloc_fastpath(size, &malloc_default, usize);
2745}
2746
2747JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2748void JEMALLOC_NOTHROW *
2749JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
2750je_malloc(size_t size) {
2751 return je_malloc_internal(size, NULL);
2752}
2753
2754JEMALLOC_EXPORT int JEMALLOC_NOTHROW
2755JEMALLOC_ATTR(nonnull(1))
2756je_posix_memalign(void **memptr, size_t alignment, size_t size) {
2757 int ret;
2758 static_opts_t sopts;
2759 dynamic_opts_t dopts;
2760
2761 LOG("core.posix_memalign.entry", "mem ptr: %p, alignment: %zu, "
2762 "size: %zu", memptr, alignment, size);
2763
2764 static_opts_init(&sopts);
2765 dynamic_opts_init(&dopts);
2766
2767 sopts.bump_empty_aligned_alloc = true;
2768 sopts.min_alignment = sizeof(void *);
2769 sopts.oom_string =
2770 "<jemalloc>: Error allocating aligned memory: out of memory\n";
2771 sopts.invalid_alignment_string =
2772 "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
2773
2774 dopts.result = memptr;
2775 dopts.num_items = 1;
2776 dopts.item_size = size;
2777 dopts.alignment = alignment;
2778
2779 ret = imalloc(&sopts, &dopts);
2780 if (sopts.slow) {
2781 uintptr_t args[3] = {(uintptr_t)memptr, (uintptr_t)alignment,
2782 (uintptr_t)size};
2783 hook_invoke_alloc(hook_alloc_posix_memalign, *memptr,
2784 (uintptr_t)ret, args);
2785 }
2786
2787 LOG("core.posix_memalign.exit", "result: %d, alloc ptr: %p", ret,
2788 *memptr);
2789
2790 return ret;
2791}
2792
2793JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2794void JEMALLOC_NOTHROW *
2795JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2)
2796je_aligned_alloc(size_t alignment, size_t size) {
2797 void *ret;
2798
2799 static_opts_t sopts;
2800 dynamic_opts_t dopts;
2801
2802 LOG("core.aligned_alloc.entry", "alignment: %zu, size: %zu\n",
2803 alignment, size);
2804
2805 static_opts_init(&sopts);
2806 dynamic_opts_init(&dopts);
2807
2808 sopts.bump_empty_aligned_alloc = true;
2809 sopts.null_out_result_on_error = true;
2810 sopts.set_errno_on_error = true;
2811 sopts.min_alignment = 1;
2812 sopts.oom_string =
2813 "<jemalloc>: Error allocating aligned memory: out of memory\n";
2814 sopts.invalid_alignment_string =
2815 "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
2816
2817 dopts.result = &ret;
2818 dopts.num_items = 1;
2819 dopts.item_size = size;
2820 dopts.alignment = alignment;
2821
2822 imalloc(&sopts, &dopts);
2823 if (sopts.slow) {
2824 uintptr_t args[3] = {(uintptr_t)alignment, (uintptr_t)size};
2825 hook_invoke_alloc(hook_alloc_aligned_alloc, ret,
2826 (uintptr_t)ret, args);
2827 }
2828
2829 LOG("core.aligned_alloc.exit", "result: %p", ret);
2830
2831 return ret;
2832}
2833
2834static void *je_calloc_internal(size_t num, size_t size, size_t *usize) {
2835 void *ret;
2836 static_opts_t sopts;
2837 dynamic_opts_t dopts;
2838
2839 LOG("core.calloc.entry", "num: %zu, size: %zu\n", num, size);
2840
2841 static_opts_init(&sopts);
2842 dynamic_opts_init(&dopts);
2843
2844 sopts.may_overflow = true;
2845 sopts.null_out_result_on_error = true;
2846 sopts.set_errno_on_error = true;
2847 sopts.oom_string = "<jemalloc>: Error in calloc(): out of memory\n";
2848
2849 dopts.result = &ret;
2850 dopts.num_items = num;
2851 dopts.item_size = size;
2852 dopts.zero = true;
2853
2854 imalloc(&sopts, &dopts);
2855 if (sopts.slow) {
2856 uintptr_t args[3] = {(uintptr_t)num, (uintptr_t)size};
2857 hook_invoke_alloc(hook_alloc_calloc, ret, (uintptr_t)ret, args);
2858 }
2859
2860 LOG("core.calloc.exit", "result: %p", ret);
2861
2862 if (usize) *usize = dopts.usize;
2863 return ret;
2864}
2865
2866JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2867void JEMALLOC_NOTHROW *
2868JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2)
2869je_calloc(size_t num, size_t size) {
2870 return je_calloc_internal(num, size, NULL);
2871}
2872
2873JEMALLOC_ALWAYS_INLINE void
2874ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path, size_t *usable) {
2875 if (!slow_path) {
2876 tsd_assert_fast(tsd);
2877 }
2878 check_entry_exit_locking(tsd_tsdn(tsd));
2879 if (tsd_reentrancy_level_get(tsd) != 0) {
2880 assert(slow_path);
2881 }
2882
2883 assert(ptr != NULL);
2884 assert(malloc_initialized() || IS_INITIALIZER);
2885
2886 emap_alloc_ctx_t alloc_ctx;
2887 emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr,
2888 &alloc_ctx);
2889 assert(alloc_ctx.szind != SC_NSIZES);
2890
2891 size_t usize = sz_index2size(alloc_ctx.szind);
2892 if (config_prof && opt_prof) {
2893 prof_free(tsd, ptr, usize, &alloc_ctx);
2894 }
2895
2896 if (likely(!slow_path)) {
2897 idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
2898 false);
2899 } else {
2900 if (config_fill && slow_path && opt_junk_free) {
2901 junk_free_callback(ptr, usize);
2902 }
2903 idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
2904 true);
2905 }
2906 thread_dalloc_event(tsd, usize);
2907 if (usable) *usable = usize;
2908}
2909
2910JEMALLOC_ALWAYS_INLINE bool
2911maybe_check_alloc_ctx(tsd_t *tsd, void *ptr, emap_alloc_ctx_t *alloc_ctx) {
2912 if (config_opt_size_checks) {
2913 emap_alloc_ctx_t dbg_ctx;
2914 emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr,
2915 &dbg_ctx);
2916 if (alloc_ctx->szind != dbg_ctx.szind) {
2917 safety_check_fail_sized_dealloc(
2918 /* current_dealloc */ true, ptr,
2919 /* true_size */ sz_size2index(dbg_ctx.szind),
2920 /* input_size */ sz_size2index(alloc_ctx->szind));
2921 return true;
2922 }
2923 if (alloc_ctx->slab != dbg_ctx.slab) {
2924 safety_check_fail(
2925 "Internal heap corruption detected: "
2926 "mismatch in slab bit");
2927 return true;
2928 }
2929 }
2930 return false;
2931}
2932
2933JEMALLOC_ALWAYS_INLINE void
2934isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
2935 if (!slow_path) {
2936 tsd_assert_fast(tsd);
2937 }
2938 check_entry_exit_locking(tsd_tsdn(tsd));
2939 if (tsd_reentrancy_level_get(tsd) != 0) {
2940 assert(slow_path);
2941 }
2942
2943 assert(ptr != NULL);
2944 assert(malloc_initialized() || IS_INITIALIZER);
2945
2946 emap_alloc_ctx_t alloc_ctx;
2947 if (!config_prof) {
2948 alloc_ctx.szind = sz_size2index(usize);
2949 alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
2950 } else {
2951 if (likely(!prof_sample_aligned(ptr))) {
2952 /*
2953 * When the ptr is not page aligned, it was not sampled.
2954 * usize can be trusted to determine szind and slab.
2955 */
2956 alloc_ctx.szind = sz_size2index(usize);
2957 alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
2958 } else if (opt_prof) {
2959 emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global,
2960 ptr, &alloc_ctx);
2961
2962 if (config_opt_safety_checks) {
2963 /* Small alloc may have !slab (sampled). */
2964 if (unlikely(alloc_ctx.szind !=
2965 sz_size2index(usize))) {
2966 safety_check_fail_sized_dealloc(
2967 /* current_dealloc */ true, ptr,
2968 /* true_size */ sz_index2size(
2969 alloc_ctx.szind),
2970 /* input_size */ usize);
2971 }
2972 }
2973 } else {
2974 alloc_ctx.szind = sz_size2index(usize);
2975 alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
2976 }
2977 }
2978 bool fail = maybe_check_alloc_ctx(tsd, ptr, &alloc_ctx);
2979 if (fail) {
2980 /*
2981 * This is a heap corruption bug. In real life we'll crash; for
2982 * the unit test we just want to avoid breaking anything too
2983 * badly to get a test result out. Let's leak instead of trying
2984 * to free.
2985 */
2986 return;
2987 }
2988
2989 if (config_prof && opt_prof) {
2990 prof_free(tsd, ptr, usize, &alloc_ctx);
2991 }
2992 if (likely(!slow_path)) {
2993 isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, &alloc_ctx,
2994 false);
2995 } else {
2996 if (config_fill && slow_path && opt_junk_free) {
2997 junk_free_callback(ptr, usize);
2998 }
2999 isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, &alloc_ctx,
3000 true);
3001 }
3002 thread_dalloc_event(tsd, usize);
3003}
3004
3005JEMALLOC_NOINLINE
3006void
3007free_default(void *ptr, size_t *usize) {
3008 UTRACE(ptr, 0, 0);
3009 if (likely(ptr != NULL)) {
3010 /*
3011 * We avoid setting up tsd fully (e.g. tcache, arena binding)
3012 * based on only free() calls -- other activities trigger the
3013 * minimal to full transition. This is because free() may
3014 * happen during thread shutdown after tls deallocation: if a
3015 * thread never had any malloc activities until then, a
3016 * fully-setup tsd won't be destructed properly.
3017 */
3018 tsd_t *tsd = tsd_fetch_min();
3019 check_entry_exit_locking(tsd_tsdn(tsd));
3020
3021 if (likely(tsd_fast(tsd))) {
3022 tcache_t *tcache = tcache_get_from_ind(tsd,
3023 TCACHE_IND_AUTOMATIC, /* slow */ false,
3024 /* is_alloc */ false);
3025 ifree(tsd, ptr, tcache, /* slow */ false, usize);
3026 } else {
3027 tcache_t *tcache = tcache_get_from_ind(tsd,
3028 TCACHE_IND_AUTOMATIC, /* slow */ true,
3029 /* is_alloc */ false);
3030 uintptr_t args_raw[3] = {(uintptr_t)ptr};
3031 hook_invoke_dalloc(hook_dalloc_free, ptr, args_raw);
3032 ifree(tsd, ptr, tcache, /* slow */ true, usize);
3033 }
3034
3035 check_entry_exit_locking(tsd_tsdn(tsd));
3036 }
3037}
3038
3039JEMALLOC_ALWAYS_INLINE bool
3040free_fastpath_nonfast_aligned(void *ptr, bool check_prof) {
3041 /*
3042 * free_fastpath do not handle two uncommon cases: 1) sampled profiled
3043 * objects and 2) sampled junk & stash for use-after-free detection.
3044 * Both have special alignments which are used to escape the fastpath.
3045 *
3046 * prof_sample is page-aligned, which covers the UAF check when both
3047 * are enabled (the assertion below). Avoiding redundant checks since
3048 * this is on the fastpath -- at most one runtime branch from this.
3049 */
3050 if (config_debug && cache_bin_nonfast_aligned(ptr)) {
3051 assert(prof_sample_aligned(ptr));
3052 }
3053
3054 if (config_prof && check_prof) {
3055 /* When prof is enabled, the prof_sample alignment is enough. */
3056 if (prof_sample_aligned(ptr)) {
3057 return true;
3058 } else {
3059 return false;
3060 }
3061 }
3062
3063 if (config_uaf_detection) {
3064 if (cache_bin_nonfast_aligned(ptr)) {
3065 return true;
3066 } else {
3067 return false;
3068 }
3069 }
3070
3071 return false;
3072}
3073
3074/* Returns whether or not the free attempt was successful. */
3075JEMALLOC_ALWAYS_INLINE
3076bool free_fastpath(void *ptr, size_t size, bool size_hint, size_t *usable_size) {
3077 tsd_t *tsd = tsd_get(false);
3078 /* The branch gets optimized away unless tsd_get_allocates(). */
3079 if (unlikely(tsd == NULL)) {
3080 return false;
3081 }
3082 /*
3083 * The tsd_fast() / initialized checks are folded into the branch
3084 * testing (deallocated_after >= threshold) later in this function.
3085 * The threshold will be set to 0 when !tsd_fast.
3086 */
3087 assert(tsd_fast(tsd) ||
3088 *tsd_thread_deallocated_next_event_fastp_get_unsafe(tsd) == 0);
3089
3090 emap_alloc_ctx_t alloc_ctx;
3091 if (!size_hint) {
3092 bool err = emap_alloc_ctx_try_lookup_fast(tsd,
3093 &arena_emap_global, ptr, &alloc_ctx);
3094
3095 /* Note: profiled objects will have alloc_ctx.slab set */
3096 if (unlikely(err || !alloc_ctx.slab ||
3097 free_fastpath_nonfast_aligned(ptr,
3098 /* check_prof */ false))) {
3099 return false;
3100 }
3101 assert(alloc_ctx.szind != SC_NSIZES);
3102 } else {
3103 /*
3104 * Check for both sizes that are too large, and for sampled /
3105 * special aligned objects. The alignment check will also check
3106 * for null ptr.
3107 */
3108 if (unlikely(size > SC_LOOKUP_MAXCLASS ||
3109 free_fastpath_nonfast_aligned(ptr,
3110 /* check_prof */ true))) {
3111 return false;
3112 }
3113 alloc_ctx.szind = sz_size2index_lookup(size);
3114 /* Max lookup class must be small. */
3115 assert(alloc_ctx.szind < SC_NBINS);
3116 /* This is a dead store, except when opt size checking is on. */
3117 alloc_ctx.slab = true;
3118 }
3119 /*
3120 * Currently the fastpath only handles small sizes. The branch on
3121 * SC_LOOKUP_MAXCLASS makes sure of it. This lets us avoid checking
3122 * tcache szind upper limit (i.e. tcache_maxclass) as well.
3123 */
3124 assert(alloc_ctx.slab);
3125
3126 uint64_t deallocated, threshold;
3127 te_free_fastpath_ctx(tsd, &deallocated, &threshold);
3128
3129 size_t usize = sz_index2size(alloc_ctx.szind);
3130 uint64_t deallocated_after = deallocated + usize;
3131 /*
3132 * Check for events and tsd non-nominal (fast_threshold will be set to
3133 * 0) in a single branch. Note that this handles the uninitialized case
3134 * as well (TSD init will be triggered on the non-fastpath). Therefore
3135 * anything depends on a functional TSD (e.g. the alloc_ctx sanity check
3136 * below) needs to be after this branch.
3137 */
3138 if (unlikely(deallocated_after >= threshold)) {
3139 return false;
3140 }
3141 assert(tsd_fast(tsd));
3142 bool fail = maybe_check_alloc_ctx(tsd, ptr, &alloc_ctx);
3143 if (fail) {
3144 /* See the comment in isfree. */
3145 if (usable_size) *usable_size = usize;
3146 return true;
3147 }
3148
3149 tcache_t *tcache = tcache_get_from_ind(tsd, TCACHE_IND_AUTOMATIC,
3150 /* slow */ false, /* is_alloc */ false);
3151 cache_bin_t *bin = &tcache->bins[alloc_ctx.szind];
3152
3153 /*
3154 * If junking were enabled, this is where we would do it. It's not
3155 * though, since we ensured above that we're on the fast path. Assert
3156 * that to double-check.
3157 */
3158 assert(!opt_junk_free);
3159
3160 if (!cache_bin_dalloc_easy(bin, ptr)) {
3161 return false;
3162 }
3163
3164 *tsd_thread_deallocatedp_get(tsd) = deallocated_after;
3165
3166 if (usable_size) *usable_size = usize;
3167 return true;
3168}
3169
3170static inline void je_free_internal(void *ptr, size_t *usize) {
3171 LOG("core.free.entry", "ptr: %p", ptr);
3172
3173 if (!free_fastpath(ptr, 0, false, usize)) {
3174 free_default(ptr, usize);
3175 }
3176
3177 LOG("core.free.exit", "");
3178}
3179
3180JEMALLOC_EXPORT void JEMALLOC_NOTHROW
3181je_free(void *ptr) {
3182 je_free_internal(ptr, NULL);
3183}
3184
3185/*
3186 * End malloc(3)-compatible functions.
3187 */
3188/******************************************************************************/
3189/*
3190 * Begin non-standard override functions.
3191 */
3192
3193#ifdef JEMALLOC_OVERRIDE_MEMALIGN
3194JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
3195void JEMALLOC_NOTHROW *
3196JEMALLOC_ATTR(malloc)
3197je_memalign(size_t alignment, size_t size) {
3198 void *ret;
3199 static_opts_t sopts;
3200 dynamic_opts_t dopts;
3201
3202 LOG("core.memalign.entry", "alignment: %zu, size: %zu\n", alignment,
3203 size);
3204
3205 static_opts_init(&sopts);
3206 dynamic_opts_init(&dopts);
3207
3208 sopts.min_alignment = 1;
3209 sopts.oom_string =
3210 "<jemalloc>: Error allocating aligned memory: out of memory\n";
3211 sopts.invalid_alignment_string =
3212 "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
3213 sopts.null_out_result_on_error = true;
3214
3215 dopts.result = &ret;
3216 dopts.num_items = 1;
3217 dopts.item_size = size;
3218 dopts.alignment = alignment;
3219
3220 imalloc(&sopts, &dopts);
3221 if (sopts.slow) {
3222 uintptr_t args[3] = {alignment, size};
3223 hook_invoke_alloc(hook_alloc_memalign, ret, (uintptr_t)ret,
3224 args);
3225 }
3226
3227 LOG("core.memalign.exit", "result: %p", ret);
3228 return ret;
3229}
3230#endif
3231
3232#ifdef JEMALLOC_OVERRIDE_VALLOC
3233JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
3234void JEMALLOC_NOTHROW *
3235JEMALLOC_ATTR(malloc)
3236je_valloc(size_t size) {
3237 void *ret;
3238
3239 static_opts_t sopts;
3240 dynamic_opts_t dopts;
3241
3242 LOG("core.valloc.entry", "size: %zu\n", size);
3243
3244 static_opts_init(&sopts);
3245 dynamic_opts_init(&dopts);
3246
3247 sopts.null_out_result_on_error = true;
3248 sopts.min_alignment = PAGE;
3249 sopts.oom_string =
3250 "<jemalloc>: Error allocating aligned memory: out of memory\n";
3251 sopts.invalid_alignment_string =
3252 "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
3253
3254 dopts.result = &ret;
3255 dopts.num_items = 1;
3256 dopts.item_size = size;
3257 dopts.alignment = PAGE;
3258
3259 imalloc(&sopts, &dopts);
3260 if (sopts.slow) {
3261 uintptr_t args[3] = {size};
3262 hook_invoke_alloc(hook_alloc_valloc, ret, (uintptr_t)ret, args);
3263 }
3264
3265 LOG("core.valloc.exit", "result: %p\n", ret);
3266 return ret;
3267}
3268#endif
3269
3270#if defined(JEMALLOC_IS_MALLOC) && defined(JEMALLOC_GLIBC_MALLOC_HOOK)
3271/*
3272 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
3273 * to inconsistently reference libc's malloc(3)-compatible functions
3274 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
3275 *
3276 * These definitions interpose hooks in glibc. The functions are actually
3277 * passed an extra argument for the caller return address, which will be
3278 * ignored.
3279 */
3280#include <features.h> // defines __GLIBC__ if we are compiling against glibc
3281
3282JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free;
3283JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc;
3284JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc;
3285# ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK
3286JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) =
3287 je_memalign;
3288# endif
3289
3290# ifdef __GLIBC__
3291/*
3292 * To enable static linking with glibc, the libc specific malloc interface must
3293 * be implemented also, so none of glibc's malloc.o functions are added to the
3294 * link.
3295 */
3296# define ALIAS(je_fn) __attribute__((alias (#je_fn), used))
3297/* To force macro expansion of je_ prefix before stringification. */
3298# define PREALIAS(je_fn) ALIAS(je_fn)
3299# ifdef JEMALLOC_OVERRIDE___LIBC_CALLOC
3300void *__libc_calloc(size_t n, size_t size) PREALIAS(je_calloc);
3301# endif
3302# ifdef JEMALLOC_OVERRIDE___LIBC_FREE
3303void __libc_free(void* ptr) PREALIAS(je_free);
3304# endif
3305# ifdef JEMALLOC_OVERRIDE___LIBC_MALLOC
3306void *__libc_malloc(size_t size) PREALIAS(je_malloc);
3307# endif
3308# ifdef JEMALLOC_OVERRIDE___LIBC_MEMALIGN
3309void *__libc_memalign(size_t align, size_t s) PREALIAS(je_memalign);
3310# endif
3311# ifdef JEMALLOC_OVERRIDE___LIBC_REALLOC
3312void *__libc_realloc(void* ptr, size_t size) PREALIAS(je_realloc);
3313# endif
3314# ifdef JEMALLOC_OVERRIDE___LIBC_VALLOC
3315void *__libc_valloc(size_t size) PREALIAS(je_valloc);
3316# endif
3317# ifdef JEMALLOC_OVERRIDE___POSIX_MEMALIGN
3318int __posix_memalign(void** r, size_t a, size_t s) PREALIAS(je_posix_memalign);
3319# endif
3320# undef PREALIAS
3321# undef ALIAS
3322# endif
3323#endif
3324
3325/*
3326 * End non-standard override functions.
3327 */
3328/******************************************************************************/
3329/*
3330 * Begin non-standard functions.
3331 */
3332
3333JEMALLOC_ALWAYS_INLINE unsigned
3334mallocx_tcache_get(int flags) {
3335 if (likely((flags & MALLOCX_TCACHE_MASK) == 0)) {
3336 return TCACHE_IND_AUTOMATIC;
3337 } else if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
3338 return TCACHE_IND_NONE;
3339 } else {
3340 return MALLOCX_TCACHE_GET(flags);
3341 }
3342}
3343
3344JEMALLOC_ALWAYS_INLINE unsigned
3345mallocx_arena_get(int flags) {
3346 if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
3347 return MALLOCX_ARENA_GET(flags);
3348 } else {
3349 return ARENA_IND_AUTOMATIC;
3350 }
3351}
3352
3353#ifdef JEMALLOC_EXPERIMENTAL_SMALLOCX_API
3354
3355#define JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) x ## y
3356#define JEMALLOC_SMALLOCX_CONCAT_HELPER2(x, y) \
3357 JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y)
3358
3359typedef struct {
3360 void *ptr;
3361 size_t size;
3362} smallocx_return_t;
3363
3364JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
3365smallocx_return_t JEMALLOC_NOTHROW
3366/*
3367 * The attribute JEMALLOC_ATTR(malloc) cannot be used due to:
3368 * - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86488
3369 */
3370JEMALLOC_SMALLOCX_CONCAT_HELPER2(je_smallocx_, JEMALLOC_VERSION_GID_IDENT)
3371 (size_t size, int flags) {
3372 /*
3373 * Note: the attribute JEMALLOC_ALLOC_SIZE(1) cannot be
3374 * used here because it makes writing beyond the `size`
3375 * of the `ptr` undefined behavior, but the objective
3376 * of this function is to allow writing beyond `size`
3377 * up to `smallocx_return_t::size`.
3378 */
3379 smallocx_return_t ret;
3380 static_opts_t sopts;
3381 dynamic_opts_t dopts;
3382
3383 LOG("core.smallocx.entry", "size: %zu, flags: %d", size, flags);
3384
3385 static_opts_init(&sopts);
3386 dynamic_opts_init(&dopts);
3387
3388 sopts.assert_nonempty_alloc = true;
3389 sopts.null_out_result_on_error = true;
3390 sopts.oom_string = "<jemalloc>: Error in mallocx(): out of memory\n";
3391 sopts.usize = true;
3392
3393 dopts.result = &ret.ptr;
3394 dopts.num_items = 1;
3395 dopts.item_size = size;
3396 if (unlikely(flags != 0)) {
3397 dopts.alignment = MALLOCX_ALIGN_GET(flags);
3398 dopts.zero = MALLOCX_ZERO_GET(flags);
3399 dopts.tcache_ind = mallocx_tcache_get(flags);
3400 dopts.arena_ind = mallocx_arena_get(flags);
3401 }
3402
3403 imalloc(&sopts, &dopts);
3404 assert(dopts.usize == je_nallocx(size, flags));
3405 ret.size = dopts.usize;
3406
3407 LOG("core.smallocx.exit", "result: %p, size: %zu", ret.ptr, ret.size);
3408 return ret;
3409}
3410#undef JEMALLOC_SMALLOCX_CONCAT_HELPER
3411#undef JEMALLOC_SMALLOCX_CONCAT_HELPER2
3412#endif
3413
3414JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
3415void JEMALLOC_NOTHROW *
3416JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
3417je_mallocx(size_t size, int flags) {
3418 void *ret;
3419 static_opts_t sopts;
3420 dynamic_opts_t dopts;
3421
3422 LOG("core.mallocx.entry", "size: %zu, flags: %d", size, flags);
3423
3424 static_opts_init(&sopts);
3425 dynamic_opts_init(&dopts);
3426
3427 sopts.assert_nonempty_alloc = true;
3428 sopts.null_out_result_on_error = true;
3429 sopts.oom_string = "<jemalloc>: Error in mallocx(): out of memory\n";
3430
3431 dopts.result = &ret;
3432 dopts.num_items = 1;
3433 dopts.item_size = size;
3434 if (unlikely(flags != 0)) {
3435 dopts.alignment = MALLOCX_ALIGN_GET(flags);
3436 dopts.zero = MALLOCX_ZERO_GET(flags);
3437 dopts.tcache_ind = mallocx_tcache_get(flags);
3438 dopts.arena_ind = mallocx_arena_get(flags);
3439 }
3440
3441 imalloc(&sopts, &dopts);
3442 if (sopts.slow) {
3443 uintptr_t args[3] = {size, flags};
3444 hook_invoke_alloc(hook_alloc_mallocx, ret, (uintptr_t)ret,
3445 args);
3446 }
3447
3448 LOG("core.mallocx.exit", "result: %p", ret);
3449 return ret;
3450}
3451
3452static void *
3453irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize,
3454 size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena,
3455 prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) {
3456 void *p;
3457
3458 if (tctx == NULL) {
3459 return NULL;
3460 }
3461
3462 alignment = prof_sample_align(alignment);
3463 if (usize <= SC_SMALL_MAXCLASS) {
3464 p = iralloct(tsdn, old_ptr, old_usize,
3465 SC_LARGE_MINCLASS, alignment, zero, tcache,
3466 arena, hook_args);
3467 if (p == NULL) {
3468 return NULL;
3469 }
3470 arena_prof_promote(tsdn, p, usize);
3471 } else {
3472 p = iralloct(tsdn, old_ptr, old_usize, usize, alignment, zero,
3473 tcache, arena, hook_args);
3474 }
3475 assert(prof_sample_aligned(p));
3476
3477 return p;
3478}
3479
3480JEMALLOC_ALWAYS_INLINE void *
3481irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
3482 size_t alignment, size_t usize, bool zero, tcache_t *tcache,
3483 arena_t *arena, emap_alloc_ctx_t *alloc_ctx,
3484 hook_ralloc_args_t *hook_args) {
3485 prof_info_t old_prof_info;
3486 prof_info_get_and_reset_recent(tsd, old_ptr, alloc_ctx, &old_prof_info);
3487 bool prof_active = prof_active_get_unlocked();
3488 bool sample_event = te_prof_sample_event_lookahead(tsd, usize);
3489 prof_tctx_t *tctx = prof_alloc_prep(tsd, prof_active, sample_event);
3490 void *p;
3491 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
3492 p = irallocx_prof_sample(tsd_tsdn(tsd), old_ptr, old_usize,
3493 usize, alignment, zero, tcache, arena, tctx, hook_args);
3494 } else {
3495 p = iralloct(tsd_tsdn(tsd), old_ptr, old_usize, size, alignment,
3496 zero, tcache, arena, hook_args);
3497 }
3498 if (unlikely(p == NULL)) {
3499 prof_alloc_rollback(tsd, tctx);
3500 return NULL;
3501 }
3502 assert(usize == isalloc(tsd_tsdn(tsd), p));
3503 prof_realloc(tsd, p, size, usize, tctx, prof_active, old_ptr,
3504 old_usize, &old_prof_info, sample_event);
3505
3506 return p;
3507}
3508
3509static void *
3510do_rallocx(void *ptr, size_t size, int flags, bool is_realloc, size_t *old_usable_size, size_t *new_usable_size) {
3511 void *p;
3512 tsd_t *tsd;
3513 size_t usize;
3514 size_t old_usize;
3515 size_t alignment = MALLOCX_ALIGN_GET(flags);
3516 arena_t *arena;
3517
3518 assert(ptr != NULL);
3519 assert(size != 0);
3520 assert(malloc_initialized() || IS_INITIALIZER);
3521 tsd = tsd_fetch();
3522 check_entry_exit_locking(tsd_tsdn(tsd));
3523
3524 bool zero = zero_get(MALLOCX_ZERO_GET(flags), /* slow */ true);
3525
3526 unsigned arena_ind = mallocx_arena_get(flags);
3527 if (arena_get_from_ind(tsd, arena_ind, &arena)) {
3528 goto label_oom;
3529 }
3530
3531 unsigned tcache_ind = mallocx_tcache_get(flags);
3532 tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind,
3533 /* slow */ true, /* is_alloc */ true);
3534
3535 emap_alloc_ctx_t alloc_ctx;
3536 emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr,
3537 &alloc_ctx);
3538 assert(alloc_ctx.szind != SC_NSIZES);
3539 old_usize = sz_index2size(alloc_ctx.szind);
3540 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
3541 if (aligned_usize_get(size, alignment, &usize, NULL, false)) {
3542 goto label_oom;
3543 }
3544
3545 hook_ralloc_args_t hook_args = {is_realloc, {(uintptr_t)ptr, size,
3546 flags, 0}};
3547 if (config_prof && opt_prof) {
3548 p = irallocx_prof(tsd, ptr, old_usize, size, alignment, usize,
3549 zero, tcache, arena, &alloc_ctx, &hook_args);
3550 if (unlikely(p == NULL)) {
3551 goto label_oom;
3552 }
3553 } else {
3554 p = iralloct(tsd_tsdn(tsd), ptr, old_usize, size, alignment,
3555 zero, tcache, arena, &hook_args);
3556 if (unlikely(p == NULL)) {
3557 goto label_oom;
3558 }
3559 assert(usize == isalloc(tsd_tsdn(tsd), p));
3560 }
3561 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
3562 thread_alloc_event(tsd, usize);
3563 thread_dalloc_event(tsd, old_usize);
3564
3565 UTRACE(ptr, size, p);
3566 check_entry_exit_locking(tsd_tsdn(tsd));
3567
3568 if (config_fill && unlikely(opt_junk_alloc) && usize > old_usize
3569 && !zero) {
3570 size_t excess_len = usize - old_usize;
3571 void *excess_start = (void *)((uintptr_t)p + old_usize);
3572 junk_alloc_callback(excess_start, excess_len);
3573 }
3574
3575 if (old_usable_size) *old_usable_size = old_usize;
3576 if (new_usable_size) *new_usable_size = usize;
3577 return p;
3578label_oom:
3579 if (config_xmalloc && unlikely(opt_xmalloc)) {
3580 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n");
3581 abort();
3582 }
3583 UTRACE(ptr, size, 0);
3584 check_entry_exit_locking(tsd_tsdn(tsd));
3585
3586 return NULL;
3587}
3588
3589JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
3590void JEMALLOC_NOTHROW *
3591JEMALLOC_ALLOC_SIZE(2)
3592je_rallocx(void *ptr, size_t size, int flags) {
3593 LOG("core.rallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr,
3594 size, flags);
3595 void *ret = do_rallocx(ptr, size, flags, false, NULL, NULL);
3596 LOG("core.rallocx.exit", "result: %p", ret);
3597 return ret;
3598}
3599
3600static void *
3601do_realloc_nonnull_zero(void *ptr, size_t *old_usize, size_t *new_usize) {
3602 if (config_stats) {
3603 atomic_fetch_add_zu(&zero_realloc_count, 1, ATOMIC_RELAXED);
3604 }
3605 if (opt_zero_realloc_action == zero_realloc_action_alloc) {
3606 /*
3607 * The user might have gotten an alloc setting while expecting a
3608 * free setting. If that's the case, we at least try to
3609 * reduce the harm, and turn off the tcache while allocating, so
3610 * that we'll get a true first fit.
3611 */
3612 return do_rallocx(ptr, 1, MALLOCX_TCACHE_NONE, true, old_usize, new_usize);
3613 } else if (opt_zero_realloc_action == zero_realloc_action_free) {
3614 UTRACE(ptr, 0, 0);
3615 tsd_t *tsd = tsd_fetch();
3616 check_entry_exit_locking(tsd_tsdn(tsd));
3617
3618 tcache_t *tcache = tcache_get_from_ind(tsd,
3619 TCACHE_IND_AUTOMATIC, /* slow */ true,
3620 /* is_alloc */ false);
3621 uintptr_t args[3] = {(uintptr_t)ptr, 0};
3622 hook_invoke_dalloc(hook_dalloc_realloc, ptr, args);
3623 size_t usize;
3624 ifree(tsd, ptr, tcache, true, &usize);
3625 if (old_usize) *old_usize = usize;
3626 if (new_usize) *new_usize = 0;
3627
3628 check_entry_exit_locking(tsd_tsdn(tsd));
3629 return NULL;
3630 } else {
3631 safety_check_fail("Called realloc(non-null-ptr, 0) with "
3632 "zero_realloc:abort set\n");
3633 /* In real code, this will never run; the safety check failure
3634 * will call abort. In the unit test, we just want to bail out
3635 * without corrupting internal state that the test needs to
3636 * finish.
3637 */
3638 return NULL;
3639 }
3640}
3641
3642static inline void *je_realloc_internal(void *ptr, size_t size, size_t *old_usize, size_t *new_usize) {
3643 LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size);
3644
3645 if (likely(ptr != NULL && size != 0)) {
3646 void *ret = do_rallocx(ptr, size, 0, true, old_usize, new_usize);
3647 LOG("core.realloc.exit", "result: %p", ret);
3648 return ret;
3649 } else if (ptr != NULL && size == 0) {
3650 void *ret = do_realloc_nonnull_zero(ptr, old_usize, new_usize);
3651 LOG("core.realloc.exit", "result: %p", ret);
3652 return ret;
3653 } else {
3654 /* realloc(NULL, size) is equivalent to malloc(size). */
3655 void *ret;
3656
3657 static_opts_t sopts;
3658 dynamic_opts_t dopts;
3659
3660 static_opts_init(&sopts);
3661 dynamic_opts_init(&dopts);
3662
3663 sopts.null_out_result_on_error = true;
3664 sopts.set_errno_on_error = true;
3665 sopts.oom_string =
3666 "<jemalloc>: Error in realloc(): out of memory\n";
3667
3668 dopts.result = &ret;
3669 dopts.num_items = 1;
3670 dopts.item_size = size;
3671
3672 imalloc(&sopts, &dopts);
3673 if (sopts.slow) {
3674 uintptr_t args[3] = {(uintptr_t)ptr, size};
3675 hook_invoke_alloc(hook_alloc_realloc, ret,
3676 (uintptr_t)ret, args);
3677 }
3678 LOG("core.realloc.exit", "result: %p", ret);
3679 if (old_usize) *old_usize = 0;
3680 if (new_usize) *new_usize = dopts.usize;
3681 return ret;
3682 }
3683}
3684
3685JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
3686void JEMALLOC_NOTHROW *
3687JEMALLOC_ALLOC_SIZE(2)
3688je_realloc(void *ptr, size_t size) {
3689 return je_realloc_internal(ptr, size, NULL, NULL);
3690}
3691
3692JEMALLOC_ALWAYS_INLINE size_t
3693ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
3694 size_t extra, size_t alignment, bool zero) {
3695 size_t newsize;
3696
3697 if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero,
3698 &newsize)) {
3699 return old_usize;
3700 }
3701
3702 return newsize;
3703}
3704
3705static size_t
3706ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
3707 size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx) {
3708 /* Sampled allocation needs to be page aligned. */
3709 if (tctx == NULL || !prof_sample_aligned(ptr)) {
3710 return old_usize;
3711 }
3712
3713 return ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment,
3714 zero);
3715}
3716
3717JEMALLOC_ALWAYS_INLINE size_t
3718ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
3719 size_t extra, size_t alignment, bool zero, emap_alloc_ctx_t *alloc_ctx) {
3720 /*
3721 * old_prof_info is only used for asserting that the profiling info
3722 * isn't changed by the ixalloc() call.
3723 */
3724 prof_info_t old_prof_info;
3725 prof_info_get(tsd, ptr, alloc_ctx, &old_prof_info);
3726
3727 /*
3728 * usize isn't knowable before ixalloc() returns when extra is non-zero.
3729 * Therefore, compute its maximum possible value and use that in
3730 * prof_alloc_prep() to decide whether to capture a backtrace.
3731 * prof_realloc() will use the actual usize to decide whether to sample.
3732 */
3733 size_t usize_max;
3734 if (aligned_usize_get(size + extra, alignment, &usize_max, NULL,
3735 false)) {
3736 /*
3737 * usize_max is out of range, and chances are that allocation
3738 * will fail, but use the maximum possible value and carry on
3739 * with prof_alloc_prep(), just in case allocation succeeds.
3740 */
3741 usize_max = SC_LARGE_MAXCLASS;
3742 }
3743 bool prof_active = prof_active_get_unlocked();
3744 bool sample_event = te_prof_sample_event_lookahead(tsd, usize_max);
3745 prof_tctx_t *tctx = prof_alloc_prep(tsd, prof_active, sample_event);
3746
3747 size_t usize;
3748 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
3749 usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize,
3750 size, extra, alignment, zero, tctx);
3751 } else {
3752 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
3753 extra, alignment, zero);
3754 }
3755
3756 /*
3757 * At this point we can still safely get the original profiling
3758 * information associated with the ptr, because (a) the edata_t object
3759 * associated with the ptr still lives and (b) the profiling info
3760 * fields are not touched. "(a)" is asserted in the outer je_xallocx()
3761 * function, and "(b)" is indirectly verified below by checking that
3762 * the alloc_tctx field is unchanged.
3763 */
3764 prof_info_t prof_info;
3765 if (usize == old_usize) {
3766 prof_info_get(tsd, ptr, alloc_ctx, &prof_info);
3767 prof_alloc_rollback(tsd, tctx);
3768 } else {
3769 prof_info_get_and_reset_recent(tsd, ptr, alloc_ctx, &prof_info);
3770 assert(usize <= usize_max);
3771 sample_event = te_prof_sample_event_lookahead(tsd, usize);
3772 prof_realloc(tsd, ptr, size, usize, tctx, prof_active, ptr,
3773 old_usize, &prof_info, sample_event);
3774 }
3775
3776 assert(old_prof_info.alloc_tctx == prof_info.alloc_tctx);
3777 return usize;
3778}
3779
3780JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
3781je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
3782 tsd_t *tsd;
3783 size_t usize, old_usize;
3784 size_t alignment = MALLOCX_ALIGN_GET(flags);
3785 bool zero = zero_get(MALLOCX_ZERO_GET(flags), /* slow */ true);
3786
3787 LOG("core.xallocx.entry", "ptr: %p, size: %zu, extra: %zu, "
3788 "flags: %d", ptr, size, extra, flags);
3789
3790 assert(ptr != NULL);
3791 assert(size != 0);
3792 assert(SIZE_T_MAX - size >= extra);
3793 assert(malloc_initialized() || IS_INITIALIZER);
3794 tsd = tsd_fetch();
3795 check_entry_exit_locking(tsd_tsdn(tsd));
3796
3797 /*
3798 * old_edata is only for verifying that xallocx() keeps the edata_t
3799 * object associated with the ptr (though the content of the edata_t
3800 * object can be changed).
3801 */
3802 edata_t *old_edata = emap_edata_lookup(tsd_tsdn(tsd),
3803 &arena_emap_global, ptr);
3804
3805 emap_alloc_ctx_t alloc_ctx;
3806 emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr,
3807 &alloc_ctx);
3808 assert(alloc_ctx.szind != SC_NSIZES);
3809 old_usize = sz_index2size(alloc_ctx.szind);
3810 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
3811 /*
3812 * The API explicitly absolves itself of protecting against (size +
3813 * extra) numerical overflow, but we may need to clamp extra to avoid
3814 * exceeding SC_LARGE_MAXCLASS.
3815 *
3816 * Ordinarily, size limit checking is handled deeper down, but here we
3817 * have to check as part of (size + extra) clamping, since we need the
3818 * clamped value in the above helper functions.
3819 */
3820 if (unlikely(size > SC_LARGE_MAXCLASS)) {
3821 usize = old_usize;
3822 goto label_not_resized;
3823 }
3824 if (unlikely(SC_LARGE_MAXCLASS - size < extra)) {
3825 extra = SC_LARGE_MAXCLASS - size;
3826 }
3827
3828 if (config_prof && opt_prof) {
3829 usize = ixallocx_prof(tsd, ptr, old_usize, size, extra,
3830 alignment, zero, &alloc_ctx);
3831 } else {
3832 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
3833 extra, alignment, zero);
3834 }
3835
3836 /*
3837 * xallocx() should keep using the same edata_t object (though its
3838 * content can be changed).
3839 */
3840 assert(emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr)
3841 == old_edata);
3842
3843 if (unlikely(usize == old_usize)) {
3844 goto label_not_resized;
3845 }
3846 thread_alloc_event(tsd, usize);
3847 thread_dalloc_event(tsd, old_usize);
3848
3849 if (config_fill && unlikely(opt_junk_alloc) && usize > old_usize &&
3850 !zero) {
3851 size_t excess_len = usize - old_usize;
3852 void *excess_start = (void *)((uintptr_t)ptr + old_usize);
3853 junk_alloc_callback(excess_start, excess_len);
3854 }
3855label_not_resized:
3856 if (unlikely(!tsd_fast(tsd))) {
3857 uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags};
3858 hook_invoke_expand(hook_expand_xallocx, ptr, old_usize,
3859 usize, (uintptr_t)usize, args);
3860 }
3861
3862 UTRACE(ptr, size, ptr);
3863 check_entry_exit_locking(tsd_tsdn(tsd));
3864
3865 LOG("core.xallocx.exit", "result: %zu", usize);
3866 return usize;
3867}
3868
3869JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
3870JEMALLOC_ATTR(pure)
3871je_sallocx(const void *ptr, int flags) {
3872 size_t usize;
3873 tsdn_t *tsdn;
3874
3875 LOG("core.sallocx.entry", "ptr: %p, flags: %d", ptr, flags);
3876
3877 assert(malloc_initialized() || IS_INITIALIZER);
3878 assert(ptr != NULL);
3879
3880 tsdn = tsdn_fetch();
3881 check_entry_exit_locking(tsdn);
3882
3883 if (config_debug || force_ivsalloc) {
3884 usize = ivsalloc(tsdn, ptr);
3885 assert(force_ivsalloc || usize != 0);
3886 } else {
3887 usize = isalloc(tsdn, ptr);
3888 }
3889
3890 check_entry_exit_locking(tsdn);
3891
3892 LOG("core.sallocx.exit", "result: %zu", usize);
3893 return usize;
3894}
3895
3896JEMALLOC_EXPORT void JEMALLOC_NOTHROW
3897je_dallocx(void *ptr, int flags) {
3898 LOG("core.dallocx.entry", "ptr: %p, flags: %d", ptr, flags);
3899
3900 assert(ptr != NULL);
3901 assert(malloc_initialized() || IS_INITIALIZER);
3902
3903 tsd_t *tsd = tsd_fetch_min();
3904 bool fast = tsd_fast(tsd);
3905 check_entry_exit_locking(tsd_tsdn(tsd));
3906
3907 unsigned tcache_ind = mallocx_tcache_get(flags);
3908 tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind, !fast,
3909 /* is_alloc */ false);
3910
3911 UTRACE(ptr, 0, 0);
3912 if (likely(fast)) {
3913 tsd_assert_fast(tsd);
3914 ifree(tsd, ptr, tcache, false, NULL);
3915 } else {
3916 uintptr_t args_raw[3] = {(uintptr_t)ptr, flags};
3917 hook_invoke_dalloc(hook_dalloc_dallocx, ptr, args_raw);
3918 ifree(tsd, ptr, tcache, true, NULL);
3919 }
3920 check_entry_exit_locking(tsd_tsdn(tsd));
3921
3922 LOG("core.dallocx.exit", "");
3923}
3924
3925JEMALLOC_ALWAYS_INLINE size_t
3926inallocx(tsdn_t *tsdn, size_t size, int flags) {
3927 check_entry_exit_locking(tsdn);
3928 size_t usize;
3929 /* In case of out of range, let the user see it rather than fail. */
3930 aligned_usize_get(size, MALLOCX_ALIGN_GET(flags), &usize, NULL, false);
3931 check_entry_exit_locking(tsdn);
3932 return usize;
3933}
3934
3935JEMALLOC_NOINLINE void
3936sdallocx_default(void *ptr, size_t size, int flags) {
3937 assert(ptr != NULL);
3938 assert(malloc_initialized() || IS_INITIALIZER);
3939
3940 tsd_t *tsd = tsd_fetch_min();
3941 bool fast = tsd_fast(tsd);
3942 size_t usize = inallocx(tsd_tsdn(tsd), size, flags);
3943 check_entry_exit_locking(tsd_tsdn(tsd));
3944
3945 unsigned tcache_ind = mallocx_tcache_get(flags);
3946 tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind, !fast,
3947 /* is_alloc */ false);
3948
3949 UTRACE(ptr, 0, 0);
3950 if (likely(fast)) {
3951 tsd_assert_fast(tsd);
3952 isfree(tsd, ptr, usize, tcache, false);
3953 } else {
3954 uintptr_t args_raw[3] = {(uintptr_t)ptr, size, flags};
3955 hook_invoke_dalloc(hook_dalloc_sdallocx, ptr, args_raw);
3956 isfree(tsd, ptr, usize, tcache, true);
3957 }
3958 check_entry_exit_locking(tsd_tsdn(tsd));
3959}
3960
3961JEMALLOC_EXPORT void JEMALLOC_NOTHROW
3962je_sdallocx(void *ptr, size_t size, int flags) {
3963 LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr,
3964 size, flags);
3965
3966 if (flags != 0 || !free_fastpath(ptr, size, true, NULL)) {
3967 sdallocx_default(ptr, size, flags);
3968 }
3969
3970 LOG("core.sdallocx.exit", "");
3971}
3972
3973void JEMALLOC_NOTHROW
3974je_sdallocx_noflags(void *ptr, size_t size) {
3975 LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: 0", ptr,
3976 size);
3977
3978 if (!free_fastpath(ptr, size, true, NULL)) {
3979 sdallocx_default(ptr, size, 0);
3980 }
3981
3982 LOG("core.sdallocx.exit", "");
3983}
3984
3985JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
3986JEMALLOC_ATTR(pure)
3987je_nallocx(size_t size, int flags) {
3988 size_t usize;
3989 tsdn_t *tsdn;
3990
3991 assert(size != 0);
3992
3993 if (unlikely(malloc_init())) {
3994 LOG("core.nallocx.exit", "result: %zu", ZU(0));
3995 return 0;
3996 }
3997
3998 tsdn = tsdn_fetch();
3999 check_entry_exit_locking(tsdn);
4000
4001 usize = inallocx(tsdn, size, flags);
4002 if (unlikely(usize > SC_LARGE_MAXCLASS)) {
4003 LOG("core.nallocx.exit", "result: %zu", ZU(0));
4004 return 0;
4005 }
4006
4007 check_entry_exit_locking(tsdn);
4008 LOG("core.nallocx.exit", "result: %zu", usize);
4009 return usize;
4010}
4011
4012JEMALLOC_EXPORT int JEMALLOC_NOTHROW
4013je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
4014 size_t newlen) {
4015 int ret;
4016 tsd_t *tsd;
4017
4018 LOG("core.mallctl.entry", "name: %s", name);
4019
4020 if (unlikely(malloc_init())) {
4021 LOG("core.mallctl.exit", "result: %d", EAGAIN);
4022 return EAGAIN;
4023 }
4024
4025 tsd = tsd_fetch();
4026 check_entry_exit_locking(tsd_tsdn(tsd));
4027 ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen);
4028 check_entry_exit_locking(tsd_tsdn(tsd));
4029
4030 LOG("core.mallctl.exit", "result: %d", ret);
4031 return ret;
4032}
4033
4034JEMALLOC_EXPORT int JEMALLOC_NOTHROW
4035je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) {
4036 int ret;
4037
4038 LOG("core.mallctlnametomib.entry", "name: %s", name);
4039
4040 if (unlikely(malloc_init())) {
4041 LOG("core.mallctlnametomib.exit", "result: %d", EAGAIN);
4042 return EAGAIN;
4043 }
4044
4045 tsd_t *tsd = tsd_fetch();
4046 check_entry_exit_locking(tsd_tsdn(tsd));
4047 ret = ctl_nametomib(tsd, name, mibp, miblenp);
4048 check_entry_exit_locking(tsd_tsdn(tsd));
4049
4050 LOG("core.mallctlnametomib.exit", "result: %d", ret);
4051 return ret;
4052}
4053
4054JEMALLOC_EXPORT int JEMALLOC_NOTHROW
4055je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
4056 void *newp, size_t newlen) {
4057 int ret;
4058 tsd_t *tsd;
4059
4060 LOG("core.mallctlbymib.entry", "");
4061
4062 if (unlikely(malloc_init())) {
4063 LOG("core.mallctlbymib.exit", "result: %d", EAGAIN);
4064 return EAGAIN;
4065 }
4066
4067 tsd = tsd_fetch();
4068 check_entry_exit_locking(tsd_tsdn(tsd));
4069 ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
4070 check_entry_exit_locking(tsd_tsdn(tsd));
4071 LOG("core.mallctlbymib.exit", "result: %d", ret);
4072 return ret;
4073}
4074
4075#define STATS_PRINT_BUFSIZE 65536
4076JEMALLOC_EXPORT void JEMALLOC_NOTHROW
4077je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
4078 const char *opts) {
4079 tsdn_t *tsdn;
4080
4081 LOG("core.malloc_stats_print.entry", "");
4082
4083 tsdn = tsdn_fetch();
4084 check_entry_exit_locking(tsdn);
4085
4086 if (config_debug) {
4087 stats_print(write_cb, cbopaque, opts);
4088 } else {
4089 buf_writer_t buf_writer;
4090 buf_writer_init(tsdn, &buf_writer, write_cb, cbopaque, NULL,
4091 STATS_PRINT_BUFSIZE);
4092 stats_print(buf_writer_cb, &buf_writer, opts);
4093 buf_writer_terminate(tsdn, &buf_writer);
4094 }
4095
4096 check_entry_exit_locking(tsdn);
4097 LOG("core.malloc_stats_print.exit", "");
4098}
4099#undef STATS_PRINT_BUFSIZE
4100
4101JEMALLOC_ALWAYS_INLINE size_t
4102je_malloc_usable_size_impl(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
4103 assert(malloc_initialized() || IS_INITIALIZER);
4104
4105 tsdn_t *tsdn = tsdn_fetch();
4106 check_entry_exit_locking(tsdn);
4107
4108 size_t ret;
4109 if (unlikely(ptr == NULL)) {
4110 ret = 0;
4111 } else {
4112 if (config_debug || force_ivsalloc) {
4113 ret = ivsalloc(tsdn, ptr);
4114 assert(force_ivsalloc || ret != 0);
4115 } else {
4116 ret = isalloc(tsdn, ptr);
4117 }
4118 }
4119 check_entry_exit_locking(tsdn);
4120
4121 return ret;
4122}
4123
4124JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
4125je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
4126 LOG("core.malloc_usable_size.entry", "ptr: %p", ptr);
4127
4128 size_t ret = je_malloc_usable_size_impl(ptr);
4129
4130 LOG("core.malloc_usable_size.exit", "result: %zu", ret);
4131 return ret;
4132}
4133
4134#ifdef JEMALLOC_HAVE_MALLOC_SIZE
4135JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
4136je_malloc_size(const void *ptr) {
4137 LOG("core.malloc_size.entry", "ptr: %p", ptr);
4138
4139 size_t ret = je_malloc_usable_size_impl(ptr);
4140
4141 LOG("core.malloc_size.exit", "result: %zu", ret);
4142 return ret;
4143}
4144#endif
4145
4146static void
4147batch_alloc_prof_sample_assert(tsd_t *tsd, size_t batch, size_t usize) {
4148 assert(config_prof && opt_prof);
4149 bool prof_sample_event = te_prof_sample_event_lookahead(tsd,
4150 batch * usize);
4151 assert(!prof_sample_event);
4152 size_t surplus;
4153 prof_sample_event = te_prof_sample_event_lookahead_surplus(tsd,
4154 (batch + 1) * usize, &surplus);
4155 assert(prof_sample_event);
4156 assert(surplus < usize);
4157}
4158
4159size_t
4160batch_alloc(void **ptrs, size_t num, size_t size, int flags) {
4161 LOG("core.batch_alloc.entry",
4162 "ptrs: %p, num: %zu, size: %zu, flags: %d", ptrs, num, size, flags);
4163
4164 tsd_t *tsd = tsd_fetch();
4165 check_entry_exit_locking(tsd_tsdn(tsd));
4166
4167 size_t filled = 0;
4168
4169 if (unlikely(tsd == NULL || tsd_reentrancy_level_get(tsd) > 0)) {
4170 goto label_done;
4171 }
4172
4173 size_t alignment = MALLOCX_ALIGN_GET(flags);
4174 size_t usize;
4175 if (aligned_usize_get(size, alignment, &usize, NULL, false)) {
4176 goto label_done;
4177 }
4178 szind_t ind = sz_size2index(usize);
4179 bool zero = zero_get(MALLOCX_ZERO_GET(flags), /* slow */ true);
4180
4181 /*
4182 * The cache bin and arena will be lazily initialized; it's hard to
4183 * know in advance whether each of them needs to be initialized.
4184 */
4185 cache_bin_t *bin = NULL;
4186 arena_t *arena = NULL;
4187
4188 size_t nregs = 0;
4189 if (likely(ind < SC_NBINS)) {
4190 nregs = bin_infos[ind].nregs;
4191 assert(nregs > 0);
4192 }
4193
4194 while (filled < num) {
4195 size_t batch = num - filled;
4196 size_t surplus = SIZE_MAX; /* Dead store. */
4197 bool prof_sample_event = config_prof && opt_prof
4198 && prof_active_get_unlocked()
4199 && te_prof_sample_event_lookahead_surplus(tsd,
4200 batch * usize, &surplus);
4201
4202 if (prof_sample_event) {
4203 /*
4204 * Adjust so that the batch does not trigger prof
4205 * sampling.
4206 */
4207 batch -= surplus / usize + 1;
4208 batch_alloc_prof_sample_assert(tsd, batch, usize);
4209 }
4210
4211 size_t progress = 0;
4212
4213 if (likely(ind < SC_NBINS) && batch >= nregs) {
4214 if (arena == NULL) {
4215 unsigned arena_ind = mallocx_arena_get(flags);
4216 if (arena_get_from_ind(tsd, arena_ind,
4217 &arena)) {
4218 goto label_done;
4219 }
4220 if (arena == NULL) {
4221 arena = arena_choose(tsd, NULL);
4222 }
4223 if (unlikely(arena == NULL)) {
4224 goto label_done;
4225 }
4226 }
4227 size_t arena_batch = batch - batch % nregs;
4228 size_t n = arena_fill_small_fresh(tsd_tsdn(tsd), arena,
4229 ind, ptrs + filled, arena_batch, zero);
4230 progress += n;
4231 filled += n;
4232 }
4233
4234 if (likely(ind < nhbins) && progress < batch) {
4235 if (bin == NULL) {
4236 unsigned tcache_ind = mallocx_tcache_get(flags);
4237 tcache_t *tcache = tcache_get_from_ind(tsd,
4238 tcache_ind, /* slow */ true,
4239 /* is_alloc */ true);
4240 if (tcache != NULL) {
4241 bin = &tcache->bins[ind];
4242 }
4243 }
4244 /*
4245 * If we don't have a tcache bin, we don't want to
4246 * immediately give up, because there's the possibility
4247 * that the user explicitly requested to bypass the
4248 * tcache, or that the user explicitly turned off the
4249 * tcache; in such cases, we go through the slow path,
4250 * i.e. the mallocx() call at the end of the while loop.
4251 */
4252 if (bin != NULL) {
4253 size_t bin_batch = batch - progress;
4254 /*
4255 * n can be less than bin_batch, meaning that
4256 * the cache bin does not have enough memory.
4257 * In such cases, we rely on the slow path,
4258 * i.e. the mallocx() call at the end of the
4259 * while loop, to fill in the cache, and in the
4260 * next iteration of the while loop, the tcache
4261 * will contain a lot of memory, and we can
4262 * harvest them here. Compared to the
4263 * alternative approach where we directly go to
4264 * the arena bins here, the overhead of our
4265 * current approach should usually be minimal,
4266 * since we never try to fetch more memory than
4267 * what a slab contains via the tcache. An
4268 * additional benefit is that the tcache will
4269 * not be empty for the next allocation request.
4270 */
4271 size_t n = cache_bin_alloc_batch(bin, bin_batch,
4272 ptrs + filled);
4273 if (config_stats) {
4274 bin->tstats.nrequests += n;
4275 }
4276 if (zero) {
4277 for (size_t i = 0; i < n; ++i) {
4278 memset(ptrs[filled + i], 0,
4279 usize);
4280 }
4281 }
4282 if (config_prof && opt_prof
4283 && unlikely(ind >= SC_NBINS)) {
4284 for (size_t i = 0; i < n; ++i) {
4285 prof_tctx_reset_sampled(tsd,
4286 ptrs[filled + i]);
4287 }
4288 }
4289 progress += n;
4290 filled += n;
4291 }
4292 }
4293
4294 /*
4295 * For thread events other than prof sampling, trigger them as
4296 * if there's a single allocation of size (n * usize). This is
4297 * fine because:
4298 * (a) these events do not alter the allocation itself, and
4299 * (b) it's possible that some event would have been triggered
4300 * multiple times, instead of only once, if the allocations
4301 * were handled individually, but it would do no harm (or
4302 * even be beneficial) to coalesce the triggerings.
4303 */
4304 thread_alloc_event(tsd, progress * usize);
4305
4306 if (progress < batch || prof_sample_event) {
4307 void *p = je_mallocx(size, flags);
4308 if (p == NULL) { /* OOM */
4309 break;
4310 }
4311 if (progress == batch) {
4312 assert(prof_sampled(tsd, p));
4313 }
4314 ptrs[filled++] = p;
4315 }
4316 }
4317
4318label_done:
4319 check_entry_exit_locking(tsd_tsdn(tsd));
4320 LOG("core.batch_alloc.exit", "result: %zu", filled);
4321 return filled;
4322}
4323
4324/*
4325 * End non-standard functions.
4326 */
4327/******************************************************************************/
4328/*
4329 * The following functions are used by threading libraries for protection of
4330 * malloc during fork().
4331 */
4332
4333/*
4334 * If an application creates a thread before doing any allocation in the main
4335 * thread, then calls fork(2) in the main thread followed by memory allocation
4336 * in the child process, a race can occur that results in deadlock within the
4337 * child: the main thread may have forked while the created thread had
4338 * partially initialized the allocator. Ordinarily jemalloc prevents
4339 * fork/malloc races via the following functions it registers during
4340 * initialization using pthread_atfork(), but of course that does no good if
4341 * the allocator isn't fully initialized at fork time. The following library
4342 * constructor is a partial solution to this problem. It may still be possible
4343 * to trigger the deadlock described above, but doing so would involve forking
4344 * via a library constructor that runs before jemalloc's runs.
4345 */
4346#ifndef JEMALLOC_JET
4347JEMALLOC_ATTR(constructor)
4348static void
4349jemalloc_constructor(void) {
4350 malloc_init();
4351}
4352#endif
4353
4354#ifndef JEMALLOC_MUTEX_INIT_CB
4355void
4356jemalloc_prefork(void)
4357#else
4358JEMALLOC_EXPORT void
4359_malloc_prefork(void)
4360#endif
4361{
4362 tsd_t *tsd;
4363 unsigned i, j, narenas;
4364 arena_t *arena;
4365
4366#ifdef JEMALLOC_MUTEX_INIT_CB
4367 if (!malloc_initialized()) {
4368 return;
4369 }
4370#endif
4371 assert(malloc_initialized());
4372
4373 tsd = tsd_fetch();
4374
4375 narenas = narenas_total_get();
4376
4377 witness_prefork(tsd_witness_tsdp_get(tsd));
4378 /* Acquire all mutexes in a safe order. */
4379 ctl_prefork(tsd_tsdn(tsd));
4380 tcache_prefork(tsd_tsdn(tsd));
4381 malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock);
4382 if (have_background_thread) {
4383 background_thread_prefork0(tsd_tsdn(tsd));
4384 }
4385 prof_prefork0(tsd_tsdn(tsd));
4386 if (have_background_thread) {
4387 background_thread_prefork1(tsd_tsdn(tsd));
4388 }
4389 /* Break arena prefork into stages to preserve lock order. */
4390 for (i = 0; i < 9; i++) {
4391 for (j = 0; j < narenas; j++) {
4392 if ((arena = arena_get(tsd_tsdn(tsd), j, false)) !=
4393 NULL) {
4394 switch (i) {
4395 case 0:
4396 arena_prefork0(tsd_tsdn(tsd), arena);
4397 break;
4398 case 1:
4399 arena_prefork1(tsd_tsdn(tsd), arena);
4400 break;
4401 case 2:
4402 arena_prefork2(tsd_tsdn(tsd), arena);
4403 break;
4404 case 3:
4405 arena_prefork3(tsd_tsdn(tsd), arena);
4406 break;
4407 case 4:
4408 arena_prefork4(tsd_tsdn(tsd), arena);
4409 break;
4410 case 5:
4411 arena_prefork5(tsd_tsdn(tsd), arena);
4412 break;
4413 case 6:
4414 arena_prefork6(tsd_tsdn(tsd), arena);
4415 break;
4416 case 7:
4417 arena_prefork7(tsd_tsdn(tsd), arena);
4418 break;
4419 case 8:
4420 arena_prefork8(tsd_tsdn(tsd), arena);
4421 break;
4422 default: not_reached();
4423 }
4424 }
4425 }
4426
4427 }
4428 prof_prefork1(tsd_tsdn(tsd));
4429 stats_prefork(tsd_tsdn(tsd));
4430 tsd_prefork(tsd);
4431}
4432
4433#ifndef JEMALLOC_MUTEX_INIT_CB
4434void
4435jemalloc_postfork_parent(void)
4436#else
4437JEMALLOC_EXPORT void
4438_malloc_postfork(void)
4439#endif
4440{
4441 tsd_t *tsd;
4442 unsigned i, narenas;
4443
4444#ifdef JEMALLOC_MUTEX_INIT_CB
4445 if (!malloc_initialized()) {
4446 return;
4447 }
4448#endif
4449 assert(malloc_initialized());
4450
4451 tsd = tsd_fetch();
4452
4453 tsd_postfork_parent(tsd);
4454
4455 witness_postfork_parent(tsd_witness_tsdp_get(tsd));
4456 /* Release all mutexes, now that fork() has completed. */
4457 stats_postfork_parent(tsd_tsdn(tsd));
4458 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
4459 arena_t *arena;
4460
4461 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
4462 arena_postfork_parent(tsd_tsdn(tsd), arena);
4463 }
4464 }
4465 prof_postfork_parent(tsd_tsdn(tsd));
4466 if (have_background_thread) {
4467 background_thread_postfork_parent(tsd_tsdn(tsd));
4468 }
4469 malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock);
4470 tcache_postfork_parent(tsd_tsdn(tsd));
4471 ctl_postfork_parent(tsd_tsdn(tsd));
4472}
4473
4474void
4475jemalloc_postfork_child(void) {
4476 tsd_t *tsd;
4477 unsigned i, narenas;
4478
4479 assert(malloc_initialized());
4480
4481 tsd = tsd_fetch();
4482
4483 tsd_postfork_child(tsd);
4484
4485 witness_postfork_child(tsd_witness_tsdp_get(tsd));
4486 /* Release all mutexes, now that fork() has completed. */
4487 stats_postfork_child(tsd_tsdn(tsd));
4488 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
4489 arena_t *arena;
4490
4491 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
4492 arena_postfork_child(tsd_tsdn(tsd), arena);
4493 }
4494 }
4495 prof_postfork_child(tsd_tsdn(tsd));
4496 if (have_background_thread) {
4497 background_thread_postfork_child(tsd_tsdn(tsd));
4498 }
4499 malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock);
4500 tcache_postfork_child(tsd_tsdn(tsd));
4501 ctl_postfork_child(tsd_tsdn(tsd));
4502}
4503
4504/******************************************************************************/
4505
4506/* Helps the application decide if a pointer is worth re-allocating in order to reduce fragmentation.
4507 * returns 1 if the allocation should be moved, and 0 if the allocation be kept.
4508 * If the application decides to re-allocate it should use MALLOCX_TCACHE_NONE when doing so. */
4509JEMALLOC_EXPORT int JEMALLOC_NOTHROW
4510get_defrag_hint(void* ptr) {
4511 assert(ptr != NULL);
4512 return iget_defrag_hint(TSDN_NULL, ptr);
4513}
4514
4515JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
4516void JEMALLOC_NOTHROW *
4517JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
4518malloc_with_usize(size_t size, size_t *usize) {
4519 return je_malloc_internal(size, usize);
4520}
4521
4522JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
4523void JEMALLOC_NOTHROW *
4524JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2)
4525calloc_with_usize(size_t num, size_t size, size_t *usize) {
4526 return je_calloc_internal(num, size, usize);
4527}
4528
4529JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
4530void JEMALLOC_NOTHROW *
4531JEMALLOC_ALLOC_SIZE(2)
4532realloc_with_usize(void *ptr, size_t size, size_t *old_usize, size_t *new_usize) {
4533 return je_realloc_internal(ptr, size, old_usize, new_usize);
4534}
4535
4536JEMALLOC_EXPORT void JEMALLOC_NOTHROW
4537free_with_usize(void *ptr, size_t *usize) {
4538 je_free_internal(ptr, usize);
4539}
diff --git a/examples/redis-unstable/deps/jemalloc/src/jemalloc_cpp.cpp b/examples/redis-unstable/deps/jemalloc/src/jemalloc_cpp.cpp
deleted file mode 100644
index aeff8c6..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/jemalloc_cpp.cpp
+++ /dev/null
@@ -1,254 +0,0 @@
1#include <mutex>
2#include <new>
3
4#define JEMALLOC_CPP_CPP_
5#ifdef __cplusplus
6extern "C" {
7#endif
8
9#include "jemalloc/internal/jemalloc_preamble.h"
10#include "jemalloc/internal/jemalloc_internal_includes.h"
11
12#ifdef __cplusplus
13}
14#endif
15
16// All operators in this file are exported.
17
18// Possibly alias hidden versions of malloc and sdallocx to avoid an extra plt
19// thunk?
20//
21// extern __typeof (sdallocx) sdallocx_int
22// __attribute ((alias ("sdallocx"),
23// visibility ("hidden")));
24//
25// ... but it needs to work with jemalloc namespaces.
26
27void *operator new(std::size_t size);
28void *operator new[](std::size_t size);
29void *operator new(std::size_t size, const std::nothrow_t &) noexcept;
30void *operator new[](std::size_t size, const std::nothrow_t &) noexcept;
31void operator delete(void *ptr) noexcept;
32void operator delete[](void *ptr) noexcept;
33void operator delete(void *ptr, const std::nothrow_t &) noexcept;
34void operator delete[](void *ptr, const std::nothrow_t &) noexcept;
35
36#if __cpp_sized_deallocation >= 201309
37/* C++14's sized-delete operators. */
38void operator delete(void *ptr, std::size_t size) noexcept;
39void operator delete[](void *ptr, std::size_t size) noexcept;
40#endif
41
42#if __cpp_aligned_new >= 201606
43/* C++17's over-aligned operators. */
44void *operator new(std::size_t size, std::align_val_t);
45void *operator new(std::size_t size, std::align_val_t, const std::nothrow_t &) noexcept;
46void *operator new[](std::size_t size, std::align_val_t);
47void *operator new[](std::size_t size, std::align_val_t, const std::nothrow_t &) noexcept;
48void operator delete(void* ptr, std::align_val_t) noexcept;
49void operator delete(void* ptr, std::align_val_t, const std::nothrow_t &) noexcept;
50void operator delete(void* ptr, std::size_t size, std::align_val_t al) noexcept;
51void operator delete[](void* ptr, std::align_val_t) noexcept;
52void operator delete[](void* ptr, std::align_val_t, const std::nothrow_t &) noexcept;
53void operator delete[](void* ptr, std::size_t size, std::align_val_t al) noexcept;
54#endif
55
56JEMALLOC_NOINLINE
57static void *
58handleOOM(std::size_t size, bool nothrow) {
59 if (opt_experimental_infallible_new) {
60 safety_check_fail("<jemalloc>: Allocation failed and "
61 "opt.experimental_infallible_new is true. Aborting.\n");
62 return nullptr;
63 }
64
65 void *ptr = nullptr;
66
67 while (ptr == nullptr) {
68 std::new_handler handler;
69 // GCC-4.8 and clang 4.0 do not have std::get_new_handler.
70 {
71 static std::mutex mtx;
72 std::lock_guard<std::mutex> lock(mtx);
73
74 handler = std::set_new_handler(nullptr);
75 std::set_new_handler(handler);
76 }
77 if (handler == nullptr)
78 break;
79
80 try {
81 handler();
82 } catch (const std::bad_alloc &) {
83 break;
84 }
85
86 ptr = je_malloc(size);
87 }
88
89 if (ptr == nullptr && !nothrow)
90 std::__throw_bad_alloc();
91 return ptr;
92}
93
94template <bool IsNoExcept>
95JEMALLOC_NOINLINE
96static void *
97fallback_impl(std::size_t size, std::size_t *usize) noexcept(IsNoExcept) {
98 void *ptr = malloc_default(size, NULL);
99 if (likely(ptr != nullptr)) {
100 return ptr;
101 }
102 return handleOOM(size, IsNoExcept);
103}
104
105template <bool IsNoExcept>
106JEMALLOC_ALWAYS_INLINE
107void *
108newImpl(std::size_t size) noexcept(IsNoExcept) {
109 return imalloc_fastpath(size, &fallback_impl<IsNoExcept>, NULL);
110}
111
112void *
113operator new(std::size_t size) {
114 return newImpl<false>(size);
115}
116
117void *
118operator new[](std::size_t size) {
119 return newImpl<false>(size);
120}
121
122void *
123operator new(std::size_t size, const std::nothrow_t &) noexcept {
124 return newImpl<true>(size);
125}
126
127void *
128operator new[](std::size_t size, const std::nothrow_t &) noexcept {
129 return newImpl<true>(size);
130}
131
132#if __cpp_aligned_new >= 201606
133
134template <bool IsNoExcept>
135JEMALLOC_ALWAYS_INLINE
136void *
137alignedNewImpl(std::size_t size, std::align_val_t alignment) noexcept(IsNoExcept) {
138 void *ptr = je_aligned_alloc(static_cast<std::size_t>(alignment), size);
139 if (likely(ptr != nullptr)) {
140 return ptr;
141 }
142
143 return handleOOM(size, IsNoExcept);
144}
145
146void *
147operator new(std::size_t size, std::align_val_t alignment) {
148 return alignedNewImpl<false>(size, alignment);
149}
150
151void *
152operator new[](std::size_t size, std::align_val_t alignment) {
153 return alignedNewImpl<false>(size, alignment);
154}
155
156void *
157operator new(std::size_t size, std::align_val_t alignment, const std::nothrow_t &) noexcept {
158 return alignedNewImpl<true>(size, alignment);
159}
160
161void *
162operator new[](std::size_t size, std::align_val_t alignment, const std::nothrow_t &) noexcept {
163 return alignedNewImpl<true>(size, alignment);
164}
165
166#endif // __cpp_aligned_new
167
168void
169operator delete(void *ptr) noexcept {
170 je_free(ptr);
171}
172
173void
174operator delete[](void *ptr) noexcept {
175 je_free(ptr);
176}
177
178void
179operator delete(void *ptr, const std::nothrow_t &) noexcept {
180 je_free(ptr);
181}
182
183void operator delete[](void *ptr, const std::nothrow_t &) noexcept {
184 je_free(ptr);
185}
186
187#if __cpp_sized_deallocation >= 201309
188
189JEMALLOC_ALWAYS_INLINE
190void
191sizedDeleteImpl(void* ptr, std::size_t size) noexcept {
192 if (unlikely(ptr == nullptr)) {
193 return;
194 }
195 je_sdallocx_noflags(ptr, size);
196}
197
198void
199operator delete(void *ptr, std::size_t size) noexcept {
200 sizedDeleteImpl(ptr, size);
201}
202
203void
204operator delete[](void *ptr, std::size_t size) noexcept {
205 sizedDeleteImpl(ptr, size);
206}
207
208#endif // __cpp_sized_deallocation
209
210#if __cpp_aligned_new >= 201606
211
212JEMALLOC_ALWAYS_INLINE
213void
214alignedSizedDeleteImpl(void* ptr, std::size_t size, std::align_val_t alignment) noexcept {
215 if (config_debug) {
216 assert(((size_t)alignment & ((size_t)alignment - 1)) == 0);
217 }
218 if (unlikely(ptr == nullptr)) {
219 return;
220 }
221 je_sdallocx(ptr, size, MALLOCX_ALIGN(alignment));
222}
223
224void
225operator delete(void* ptr, std::align_val_t) noexcept {
226 je_free(ptr);
227}
228
229void
230operator delete[](void* ptr, std::align_val_t) noexcept {
231 je_free(ptr);
232}
233
234void
235operator delete(void* ptr, std::align_val_t, const std::nothrow_t&) noexcept {
236 je_free(ptr);
237}
238
239void
240operator delete[](void* ptr, std::align_val_t, const std::nothrow_t&) noexcept {
241 je_free(ptr);
242}
243
244void
245operator delete(void* ptr, std::size_t size, std::align_val_t alignment) noexcept {
246 alignedSizedDeleteImpl(ptr, size, alignment);
247}
248
249void
250operator delete[](void* ptr, std::size_t size, std::align_val_t alignment) noexcept {
251 alignedSizedDeleteImpl(ptr, size, alignment);
252}
253
254#endif // __cpp_aligned_new
diff --git a/examples/redis-unstable/deps/jemalloc/src/large.c b/examples/redis-unstable/deps/jemalloc/src/large.c
deleted file mode 100644
index 5fc4bf5..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/large.c
+++ /dev/null
@@ -1,322 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/emap.h"
6#include "jemalloc/internal/extent_mmap.h"
7#include "jemalloc/internal/mutex.h"
8#include "jemalloc/internal/prof_recent.h"
9#include "jemalloc/internal/util.h"
10
11/******************************************************************************/
12
13void *
14large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero) {
15 assert(usize == sz_s2u(usize));
16
17 return large_palloc(tsdn, arena, usize, CACHELINE, zero);
18}
19
20void *
21large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
22 bool zero) {
23 size_t ausize;
24 edata_t *edata;
25 UNUSED bool idump JEMALLOC_CC_SILENCE_INIT(false);
26
27 assert(!tsdn_null(tsdn) || arena != NULL);
28
29 ausize = sz_sa2u(usize, alignment);
30 if (unlikely(ausize == 0 || ausize > SC_LARGE_MAXCLASS)) {
31 return NULL;
32 }
33
34 if (likely(!tsdn_null(tsdn))) {
35 arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, usize);
36 }
37 if (unlikely(arena == NULL) || (edata = arena_extent_alloc_large(tsdn,
38 arena, usize, alignment, zero)) == NULL) {
39 return NULL;
40 }
41
42 /* See comments in arena_bin_slabs_full_insert(). */
43 if (!arena_is_auto(arena)) {
44 /* Insert edata into large. */
45 malloc_mutex_lock(tsdn, &arena->large_mtx);
46 edata_list_active_append(&arena->large, edata);
47 malloc_mutex_unlock(tsdn, &arena->large_mtx);
48 }
49
50 arena_decay_tick(tsdn, arena);
51 return edata_addr_get(edata);
52}
53
54static bool
55large_ralloc_no_move_shrink(tsdn_t *tsdn, edata_t *edata, size_t usize) {
56 arena_t *arena = arena_get_from_edata(edata);
57 ehooks_t *ehooks = arena_get_ehooks(arena);
58 size_t old_size = edata_size_get(edata);
59 size_t old_usize = edata_usize_get(edata);
60
61 assert(old_usize > usize);
62
63 if (ehooks_split_will_fail(ehooks)) {
64 return true;
65 }
66
67 bool deferred_work_generated = false;
68 bool err = pa_shrink(tsdn, &arena->pa_shard, edata, old_size,
69 usize + sz_large_pad, sz_size2index(usize),
70 &deferred_work_generated);
71 if (err) {
72 return true;
73 }
74 if (deferred_work_generated) {
75 arena_handle_deferred_work(tsdn, arena);
76 }
77 arena_extent_ralloc_large_shrink(tsdn, arena, edata, old_usize);
78
79 return false;
80}
81
82static bool
83large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize,
84 bool zero) {
85 arena_t *arena = arena_get_from_edata(edata);
86
87 size_t old_size = edata_size_get(edata);
88 size_t old_usize = edata_usize_get(edata);
89 size_t new_size = usize + sz_large_pad;
90
91 szind_t szind = sz_size2index(usize);
92
93 bool deferred_work_generated = false;
94 bool err = pa_expand(tsdn, &arena->pa_shard, edata, old_size, new_size,
95 szind, zero, &deferred_work_generated);
96
97 if (deferred_work_generated) {
98 arena_handle_deferred_work(tsdn, arena);
99 }
100
101 if (err) {
102 return true;
103 }
104
105 if (zero) {
106 if (opt_cache_oblivious) {
107 assert(sz_large_pad == PAGE);
108 /*
109 * Zero the trailing bytes of the original allocation's
110 * last page, since they are in an indeterminate state.
111 * There will always be trailing bytes, because ptr's
112 * offset from the beginning of the extent is a multiple
113 * of CACHELINE in [0 .. PAGE).
114 */
115 void *zbase = (void *)
116 ((uintptr_t)edata_addr_get(edata) + old_usize);
117 void *zpast = PAGE_ADDR2BASE((void *)((uintptr_t)zbase +
118 PAGE));
119 size_t nzero = (uintptr_t)zpast - (uintptr_t)zbase;
120 assert(nzero > 0);
121 memset(zbase, 0, nzero);
122 }
123 }
124 arena_extent_ralloc_large_expand(tsdn, arena, edata, old_usize);
125
126 return false;
127}
128
129bool
130large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min,
131 size_t usize_max, bool zero) {
132 size_t oldusize = edata_usize_get(edata);
133
134 /* The following should have been caught by callers. */
135 assert(usize_min > 0 && usize_max <= SC_LARGE_MAXCLASS);
136 /* Both allocation sizes must be large to avoid a move. */
137 assert(oldusize >= SC_LARGE_MINCLASS
138 && usize_max >= SC_LARGE_MINCLASS);
139
140 if (usize_max > oldusize) {
141 /* Attempt to expand the allocation in-place. */
142 if (!large_ralloc_no_move_expand(tsdn, edata, usize_max,
143 zero)) {
144 arena_decay_tick(tsdn, arena_get_from_edata(edata));
145 return false;
146 }
147 /* Try again, this time with usize_min. */
148 if (usize_min < usize_max && usize_min > oldusize &&
149 large_ralloc_no_move_expand(tsdn, edata, usize_min, zero)) {
150 arena_decay_tick(tsdn, arena_get_from_edata(edata));
151 return false;
152 }
153 }
154
155 /*
156 * Avoid moving the allocation if the existing extent size accommodates
157 * the new size.
158 */
159 if (oldusize >= usize_min && oldusize <= usize_max) {
160 arena_decay_tick(tsdn, arena_get_from_edata(edata));
161 return false;
162 }
163
164 /* Attempt to shrink the allocation in-place. */
165 if (oldusize > usize_max) {
166 if (!large_ralloc_no_move_shrink(tsdn, edata, usize_max)) {
167 arena_decay_tick(tsdn, arena_get_from_edata(edata));
168 return false;
169 }
170 }
171 return true;
172}
173
174static void *
175large_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
176 size_t alignment, bool zero) {
177 if (alignment <= CACHELINE) {
178 return large_malloc(tsdn, arena, usize, zero);
179 }
180 return large_palloc(tsdn, arena, usize, alignment, zero);
181}
182
183void *
184large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
185 size_t alignment, bool zero, tcache_t *tcache,
186 hook_ralloc_args_t *hook_args) {
187 edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
188
189 size_t oldusize = edata_usize_get(edata);
190 /* The following should have been caught by callers. */
191 assert(usize > 0 && usize <= SC_LARGE_MAXCLASS);
192 /* Both allocation sizes must be large to avoid a move. */
193 assert(oldusize >= SC_LARGE_MINCLASS
194 && usize >= SC_LARGE_MINCLASS);
195
196 /* Try to avoid moving the allocation. */
197 if (!large_ralloc_no_move(tsdn, edata, usize, usize, zero)) {
198 hook_invoke_expand(hook_args->is_realloc
199 ? hook_expand_realloc : hook_expand_rallocx, ptr, oldusize,
200 usize, (uintptr_t)ptr, hook_args->args);
201 return edata_addr_get(edata);
202 }
203
204 /*
205 * usize and old size are different enough that we need to use a
206 * different size class. In that case, fall back to allocating new
207 * space and copying.
208 */
209 void *ret = large_ralloc_move_helper(tsdn, arena, usize, alignment,
210 zero);
211 if (ret == NULL) {
212 return NULL;
213 }
214
215 hook_invoke_alloc(hook_args->is_realloc
216 ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret,
217 hook_args->args);
218 hook_invoke_dalloc(hook_args->is_realloc
219 ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args);
220
221 size_t copysize = (usize < oldusize) ? usize : oldusize;
222 memcpy(ret, edata_addr_get(edata), copysize);
223 isdalloct(tsdn, edata_addr_get(edata), oldusize, tcache, NULL, true);
224 return ret;
225}
226
227/*
228 * locked indicates whether the arena's large_mtx is currently held.
229 */
230static void
231large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
232 bool locked) {
233 if (!locked) {
234 /* See comments in arena_bin_slabs_full_insert(). */
235 if (!arena_is_auto(arena)) {
236 malloc_mutex_lock(tsdn, &arena->large_mtx);
237 edata_list_active_remove(&arena->large, edata);
238 malloc_mutex_unlock(tsdn, &arena->large_mtx);
239 }
240 } else {
241 /* Only hold the large_mtx if necessary. */
242 if (!arena_is_auto(arena)) {
243 malloc_mutex_assert_owner(tsdn, &arena->large_mtx);
244 edata_list_active_remove(&arena->large, edata);
245 }
246 }
247 arena_extent_dalloc_large_prep(tsdn, arena, edata);
248}
249
250static void
251large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
252 bool deferred_work_generated = false;
253 pa_dalloc(tsdn, &arena->pa_shard, edata, &deferred_work_generated);
254 if (deferred_work_generated) {
255 arena_handle_deferred_work(tsdn, arena);
256 }
257}
258
259void
260large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata) {
261 large_dalloc_prep_impl(tsdn, arena_get_from_edata(edata), edata, true);
262}
263
264void
265large_dalloc_finish(tsdn_t *tsdn, edata_t *edata) {
266 large_dalloc_finish_impl(tsdn, arena_get_from_edata(edata), edata);
267}
268
269void
270large_dalloc(tsdn_t *tsdn, edata_t *edata) {
271 arena_t *arena = arena_get_from_edata(edata);
272 large_dalloc_prep_impl(tsdn, arena, edata, false);
273 large_dalloc_finish_impl(tsdn, arena, edata);
274 arena_decay_tick(tsdn, arena);
275}
276
277size_t
278large_salloc(tsdn_t *tsdn, const edata_t *edata) {
279 return edata_usize_get(edata);
280}
281
282void
283large_prof_info_get(tsd_t *tsd, edata_t *edata, prof_info_t *prof_info,
284 bool reset_recent) {
285 assert(prof_info != NULL);
286
287 prof_tctx_t *alloc_tctx = edata_prof_tctx_get(edata);
288 prof_info->alloc_tctx = alloc_tctx;
289
290 if ((uintptr_t)alloc_tctx > (uintptr_t)1U) {
291 nstime_copy(&prof_info->alloc_time,
292 edata_prof_alloc_time_get(edata));
293 prof_info->alloc_size = edata_prof_alloc_size_get(edata);
294 if (reset_recent) {
295 /*
296 * Reset the pointer on the recent allocation record,
297 * so that this allocation is recorded as released.
298 */
299 prof_recent_alloc_reset(tsd, edata);
300 }
301 }
302}
303
304static void
305large_prof_tctx_set(edata_t *edata, prof_tctx_t *tctx) {
306 edata_prof_tctx_set(edata, tctx);
307}
308
309void
310large_prof_tctx_reset(edata_t *edata) {
311 large_prof_tctx_set(edata, (prof_tctx_t *)(uintptr_t)1U);
312}
313
314void
315large_prof_info_set(edata_t *edata, prof_tctx_t *tctx, size_t size) {
316 nstime_t t;
317 nstime_prof_init_update(&t);
318 edata_prof_alloc_time_set(edata, &t);
319 edata_prof_alloc_size_set(edata, size);
320 edata_prof_recent_alloc_init(edata);
321 large_prof_tctx_set(edata, tctx);
322}
diff --git a/examples/redis-unstable/deps/jemalloc/src/log.c b/examples/redis-unstable/deps/jemalloc/src/log.c
deleted file mode 100644
index 778902f..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/log.c
+++ /dev/null
@@ -1,78 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/log.h"
5
6char log_var_names[JEMALLOC_LOG_VAR_BUFSIZE];
7atomic_b_t log_init_done = ATOMIC_INIT(false);
8
9/*
10 * Returns true if we were able to pick out a segment. Fills in r_segment_end
11 * with a pointer to the first character after the end of the string.
12 */
13static const char *
14log_var_extract_segment(const char* segment_begin) {
15 const char *end;
16 for (end = segment_begin; *end != '\0' && *end != '|'; end++) {
17 }
18 return end;
19}
20
21static bool
22log_var_matches_segment(const char *segment_begin, const char *segment_end,
23 const char *log_var_begin, const char *log_var_end) {
24 assert(segment_begin <= segment_end);
25 assert(log_var_begin < log_var_end);
26
27 ptrdiff_t segment_len = segment_end - segment_begin;
28 ptrdiff_t log_var_len = log_var_end - log_var_begin;
29 /* The special '.' segment matches everything. */
30 if (segment_len == 1 && *segment_begin == '.') {
31 return true;
32 }
33 if (segment_len == log_var_len) {
34 return strncmp(segment_begin, log_var_begin, segment_len) == 0;
35 } else if (segment_len < log_var_len) {
36 return strncmp(segment_begin, log_var_begin, segment_len) == 0
37 && log_var_begin[segment_len] == '.';
38 } else {
39 return false;
40 }
41}
42
43unsigned
44log_var_update_state(log_var_t *log_var) {
45 const char *log_var_begin = log_var->name;
46 const char *log_var_end = log_var->name + strlen(log_var->name);
47
48 /* Pointer to one before the beginning of the current segment. */
49 const char *segment_begin = log_var_names;
50
51 /*
52 * If log_init done is false, we haven't parsed the malloc conf yet. To
53 * avoid log-spew, we default to not displaying anything.
54 */
55 if (!atomic_load_b(&log_init_done, ATOMIC_ACQUIRE)) {
56 return LOG_INITIALIZED_NOT_ENABLED;
57 }
58
59 while (true) {
60 const char *segment_end = log_var_extract_segment(
61 segment_begin);
62 assert(segment_end < log_var_names + JEMALLOC_LOG_VAR_BUFSIZE);
63 if (log_var_matches_segment(segment_begin, segment_end,
64 log_var_begin, log_var_end)) {
65 atomic_store_u(&log_var->state, LOG_ENABLED,
66 ATOMIC_RELAXED);
67 return LOG_ENABLED;
68 }
69 if (*segment_end == '\0') {
70 /* Hit the end of the segment string with no match. */
71 atomic_store_u(&log_var->state,
72 LOG_INITIALIZED_NOT_ENABLED, ATOMIC_RELAXED);
73 return LOG_INITIALIZED_NOT_ENABLED;
74 }
75 /* Otherwise, skip the delimiter and continue. */
76 segment_begin = segment_end + 1;
77 }
78}
diff --git a/examples/redis-unstable/deps/jemalloc/src/malloc_io.c b/examples/redis-unstable/deps/jemalloc/src/malloc_io.c
deleted file mode 100644
index b76885c..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/malloc_io.c
+++ /dev/null
@@ -1,697 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/malloc_io.h"
5#include "jemalloc/internal/util.h"
6
7#ifdef assert
8# undef assert
9#endif
10#ifdef not_reached
11# undef not_reached
12#endif
13#ifdef not_implemented
14# undef not_implemented
15#endif
16#ifdef assert_not_implemented
17# undef assert_not_implemented
18#endif
19
20/*
21 * Define simple versions of assertion macros that won't recurse in case
22 * of assertion failures in malloc_*printf().
23 */
24#define assert(e) do { \
25 if (config_debug && !(e)) { \
26 malloc_write("<jemalloc>: Failed assertion\n"); \
27 abort(); \
28 } \
29} while (0)
30
31#define not_reached() do { \
32 if (config_debug) { \
33 malloc_write("<jemalloc>: Unreachable code reached\n"); \
34 abort(); \
35 } \
36 unreachable(); \
37} while (0)
38
39#define not_implemented() do { \
40 if (config_debug) { \
41 malloc_write("<jemalloc>: Not implemented\n"); \
42 abort(); \
43 } \
44} while (0)
45
46#define assert_not_implemented(e) do { \
47 if (unlikely(config_debug && !(e))) { \
48 not_implemented(); \
49 } \
50} while (0)
51
52/******************************************************************************/
53/* Function prototypes for non-inline static functions. */
54
55#define U2S_BUFSIZE ((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1)
56static char *u2s(uintmax_t x, unsigned base, bool uppercase, char *s,
57 size_t *slen_p);
58#define D2S_BUFSIZE (1 + U2S_BUFSIZE)
59static char *d2s(intmax_t x, char sign, char *s, size_t *slen_p);
60#define O2S_BUFSIZE (1 + U2S_BUFSIZE)
61static char *o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p);
62#define X2S_BUFSIZE (2 + U2S_BUFSIZE)
63static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s,
64 size_t *slen_p);
65
66/******************************************************************************/
67
68/* malloc_message() setup. */
69void
70wrtmessage(void *cbopaque, const char *s) {
71 malloc_write_fd(STDERR_FILENO, s, strlen(s));
72}
73
74JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s);
75
76/*
77 * Wrapper around malloc_message() that avoids the need for
78 * je_malloc_message(...) throughout the code.
79 */
80void
81malloc_write(const char *s) {
82 if (je_malloc_message != NULL) {
83 je_malloc_message(NULL, s);
84 } else {
85 wrtmessage(NULL, s);
86 }
87}
88
89/*
90 * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so
91 * provide a wrapper.
92 */
93int
94buferror(int err, char *buf, size_t buflen) {
95#ifdef _WIN32
96 FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0,
97 (LPSTR)buf, (DWORD)buflen, NULL);
98 return 0;
99#elif defined(JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE) && defined(_GNU_SOURCE)
100 char *b = strerror_r(err, buf, buflen);
101 if (b != buf) {
102 strncpy(buf, b, buflen);
103 buf[buflen-1] = '\0';
104 }
105 return 0;
106#else
107 return strerror_r(err, buf, buflen);
108#endif
109}
110
111uintmax_t
112malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) {
113 uintmax_t ret, digit;
114 unsigned b;
115 bool neg;
116 const char *p, *ns;
117
118 p = nptr;
119 if (base < 0 || base == 1 || base > 36) {
120 ns = p;
121 set_errno(EINVAL);
122 ret = UINTMAX_MAX;
123 goto label_return;
124 }
125 b = base;
126
127 /* Swallow leading whitespace and get sign, if any. */
128 neg = false;
129 while (true) {
130 switch (*p) {
131 case '\t': case '\n': case '\v': case '\f': case '\r': case ' ':
132 p++;
133 break;
134 case '-':
135 neg = true;
136 JEMALLOC_FALLTHROUGH;
137 case '+':
138 p++;
139 JEMALLOC_FALLTHROUGH;
140 default:
141 goto label_prefix;
142 }
143 }
144
145 /* Get prefix, if any. */
146 label_prefix:
147 /*
148 * Note where the first non-whitespace/sign character is so that it is
149 * possible to tell whether any digits are consumed (e.g., " 0" vs.
150 * " -x").
151 */
152 ns = p;
153 if (*p == '0') {
154 switch (p[1]) {
155 case '0': case '1': case '2': case '3': case '4': case '5':
156 case '6': case '7':
157 if (b == 0) {
158 b = 8;
159 }
160 if (b == 8) {
161 p++;
162 }
163 break;
164 case 'X': case 'x':
165 switch (p[2]) {
166 case '0': case '1': case '2': case '3': case '4':
167 case '5': case '6': case '7': case '8': case '9':
168 case 'A': case 'B': case 'C': case 'D': case 'E':
169 case 'F':
170 case 'a': case 'b': case 'c': case 'd': case 'e':
171 case 'f':
172 if (b == 0) {
173 b = 16;
174 }
175 if (b == 16) {
176 p += 2;
177 }
178 break;
179 default:
180 break;
181 }
182 break;
183 default:
184 p++;
185 ret = 0;
186 goto label_return;
187 }
188 }
189 if (b == 0) {
190 b = 10;
191 }
192
193 /* Convert. */
194 ret = 0;
195 while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b)
196 || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b)
197 || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) {
198 uintmax_t pret = ret;
199 ret *= b;
200 ret += digit;
201 if (ret < pret) {
202 /* Overflow. */
203 set_errno(ERANGE);
204 ret = UINTMAX_MAX;
205 goto label_return;
206 }
207 p++;
208 }
209 if (neg) {
210 ret = (uintmax_t)(-((intmax_t)ret));
211 }
212
213 if (p == ns) {
214 /* No conversion performed. */
215 set_errno(EINVAL);
216 ret = UINTMAX_MAX;
217 goto label_return;
218 }
219
220label_return:
221 if (endptr != NULL) {
222 if (p == ns) {
223 /* No characters were converted. */
224 *endptr = (char *)nptr;
225 } else {
226 *endptr = (char *)p;
227 }
228 }
229 return ret;
230}
231
232static char *
233u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) {
234 unsigned i;
235
236 i = U2S_BUFSIZE - 1;
237 s[i] = '\0';
238 switch (base) {
239 case 10:
240 do {
241 i--;
242 s[i] = "0123456789"[x % (uint64_t)10];
243 x /= (uint64_t)10;
244 } while (x > 0);
245 break;
246 case 16: {
247 const char *digits = (uppercase)
248 ? "0123456789ABCDEF"
249 : "0123456789abcdef";
250
251 do {
252 i--;
253 s[i] = digits[x & 0xf];
254 x >>= 4;
255 } while (x > 0);
256 break;
257 } default: {
258 const char *digits = (uppercase)
259 ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
260 : "0123456789abcdefghijklmnopqrstuvwxyz";
261
262 assert(base >= 2 && base <= 36);
263 do {
264 i--;
265 s[i] = digits[x % (uint64_t)base];
266 x /= (uint64_t)base;
267 } while (x > 0);
268 }}
269
270 *slen_p = U2S_BUFSIZE - 1 - i;
271 return &s[i];
272}
273
274static char *
275d2s(intmax_t x, char sign, char *s, size_t *slen_p) {
276 bool neg;
277
278 if ((neg = (x < 0))) {
279 x = -x;
280 }
281 s = u2s(x, 10, false, s, slen_p);
282 if (neg) {
283 sign = '-';
284 }
285 switch (sign) {
286 case '-':
287 if (!neg) {
288 break;
289 }
290 JEMALLOC_FALLTHROUGH;
291 case ' ':
292 case '+':
293 s--;
294 (*slen_p)++;
295 *s = sign;
296 break;
297 default: not_reached();
298 }
299 return s;
300}
301
302static char *
303o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p) {
304 s = u2s(x, 8, false, s, slen_p);
305 if (alt_form && *s != '0') {
306 s--;
307 (*slen_p)++;
308 *s = '0';
309 }
310 return s;
311}
312
313static char *
314x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) {
315 s = u2s(x, 16, uppercase, s, slen_p);
316 if (alt_form) {
317 s -= 2;
318 (*slen_p) += 2;
319 memcpy(s, uppercase ? "0X" : "0x", 2);
320 }
321 return s;
322}
323
324JEMALLOC_COLD
325size_t
326malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
327 size_t i;
328 const char *f;
329
330#define APPEND_C(c) do { \
331 if (i < size) { \
332 str[i] = (c); \
333 } \
334 i++; \
335} while (0)
336#define APPEND_S(s, slen) do { \
337 if (i < size) { \
338 size_t cpylen = (slen <= size - i) ? slen : size - i; \
339 memcpy(&str[i], s, cpylen); \
340 } \
341 i += slen; \
342} while (0)
343#define APPEND_PADDED_S(s, slen, width, left_justify) do { \
344 /* Left padding. */ \
345 size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ? \
346 (size_t)width - slen : 0); \
347 if (!left_justify && pad_len != 0) { \
348 size_t j; \
349 for (j = 0; j < pad_len; j++) { \
350 if (pad_zero) { \
351 APPEND_C('0'); \
352 } else { \
353 APPEND_C(' '); \
354 } \
355 } \
356 } \
357 /* Value. */ \
358 APPEND_S(s, slen); \
359 /* Right padding. */ \
360 if (left_justify && pad_len != 0) { \
361 size_t j; \
362 for (j = 0; j < pad_len; j++) { \
363 APPEND_C(' '); \
364 } \
365 } \
366} while (0)
367#define GET_ARG_NUMERIC(val, len) do { \
368 switch ((unsigned char)len) { \
369 case '?': \
370 val = va_arg(ap, int); \
371 break; \
372 case '?' | 0x80: \
373 val = va_arg(ap, unsigned int); \
374 break; \
375 case 'l': \
376 val = va_arg(ap, long); \
377 break; \
378 case 'l' | 0x80: \
379 val = va_arg(ap, unsigned long); \
380 break; \
381 case 'q': \
382 val = va_arg(ap, long long); \
383 break; \
384 case 'q' | 0x80: \
385 val = va_arg(ap, unsigned long long); \
386 break; \
387 case 'j': \
388 val = va_arg(ap, intmax_t); \
389 break; \
390 case 'j' | 0x80: \
391 val = va_arg(ap, uintmax_t); \
392 break; \
393 case 't': \
394 val = va_arg(ap, ptrdiff_t); \
395 break; \
396 case 'z': \
397 val = va_arg(ap, ssize_t); \
398 break; \
399 case 'z' | 0x80: \
400 val = va_arg(ap, size_t); \
401 break; \
402 case 'p': /* Synthetic; used for %p. */ \
403 val = va_arg(ap, uintptr_t); \
404 break; \
405 default: \
406 not_reached(); \
407 val = 0; \
408 } \
409} while (0)
410
411 i = 0;
412 f = format;
413 while (true) {
414 switch (*f) {
415 case '\0': goto label_out;
416 case '%': {
417 bool alt_form = false;
418 bool left_justify = false;
419 bool plus_space = false;
420 bool plus_plus = false;
421 int prec = -1;
422 int width = -1;
423 unsigned char len = '?';
424 char *s;
425 size_t slen;
426 bool first_width_digit = true;
427 bool pad_zero = false;
428
429 f++;
430 /* Flags. */
431 while (true) {
432 switch (*f) {
433 case '#':
434 assert(!alt_form);
435 alt_form = true;
436 break;
437 case '-':
438 assert(!left_justify);
439 left_justify = true;
440 break;
441 case ' ':
442 assert(!plus_space);
443 plus_space = true;
444 break;
445 case '+':
446 assert(!plus_plus);
447 plus_plus = true;
448 break;
449 default: goto label_width;
450 }
451 f++;
452 }
453 /* Width. */
454 label_width:
455 switch (*f) {
456 case '*':
457 width = va_arg(ap, int);
458 f++;
459 if (width < 0) {
460 left_justify = true;
461 width = -width;
462 }
463 break;
464 case '0':
465 if (first_width_digit) {
466 pad_zero = true;
467 }
468 JEMALLOC_FALLTHROUGH;
469 case '1': case '2': case '3': case '4':
470 case '5': case '6': case '7': case '8': case '9': {
471 uintmax_t uwidth;
472 set_errno(0);
473 uwidth = malloc_strtoumax(f, (char **)&f, 10);
474 assert(uwidth != UINTMAX_MAX || get_errno() !=
475 ERANGE);
476 width = (int)uwidth;
477 first_width_digit = false;
478 break;
479 } default:
480 break;
481 }
482 /* Width/precision separator. */
483 if (*f == '.') {
484 f++;
485 } else {
486 goto label_length;
487 }
488 /* Precision. */
489 switch (*f) {
490 case '*':
491 prec = va_arg(ap, int);
492 f++;
493 break;
494 case '0': case '1': case '2': case '3': case '4':
495 case '5': case '6': case '7': case '8': case '9': {
496 uintmax_t uprec;
497 set_errno(0);
498 uprec = malloc_strtoumax(f, (char **)&f, 10);
499 assert(uprec != UINTMAX_MAX || get_errno() !=
500 ERANGE);
501 prec = (int)uprec;
502 break;
503 }
504 default: break;
505 }
506 /* Length. */
507 label_length:
508 switch (*f) {
509 case 'l':
510 f++;
511 if (*f == 'l') {
512 len = 'q';
513 f++;
514 } else {
515 len = 'l';
516 }
517 break;
518 case 'q': case 'j': case 't': case 'z':
519 len = *f;
520 f++;
521 break;
522 default: break;
523 }
524 /* Conversion specifier. */
525 switch (*f) {
526 case '%':
527 /* %% */
528 APPEND_C(*f);
529 f++;
530 break;
531 case 'd': case 'i': {
532 intmax_t val JEMALLOC_CC_SILENCE_INIT(0);
533 char buf[D2S_BUFSIZE];
534
535 /*
536 * Outputting negative, zero-padded numbers
537 * would require a nontrivial rework of the
538 * interaction between the width and padding
539 * (since 0 padding goes between the '-' and the
540 * number, while ' ' padding goes either before
541 * the - or after the number. Since we
542 * currently don't ever need 0-padded negative
543 * numbers, just don't bother supporting it.
544 */
545 assert(!pad_zero);
546
547 GET_ARG_NUMERIC(val, len);
548 s = d2s(val, (plus_plus ? '+' : (plus_space ?
549 ' ' : '-')), buf, &slen);
550 APPEND_PADDED_S(s, slen, width, left_justify);
551 f++;
552 break;
553 } case 'o': {
554 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
555 char buf[O2S_BUFSIZE];
556
557 GET_ARG_NUMERIC(val, len | 0x80);
558 s = o2s(val, alt_form, buf, &slen);
559 APPEND_PADDED_S(s, slen, width, left_justify);
560 f++;
561 break;
562 } case 'u': {
563 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
564 char buf[U2S_BUFSIZE];
565
566 GET_ARG_NUMERIC(val, len | 0x80);
567 s = u2s(val, 10, false, buf, &slen);
568 APPEND_PADDED_S(s, slen, width, left_justify);
569 f++;
570 break;
571 } case 'x': case 'X': {
572 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
573 char buf[X2S_BUFSIZE];
574
575 GET_ARG_NUMERIC(val, len | 0x80);
576 s = x2s(val, alt_form, *f == 'X', buf, &slen);
577 APPEND_PADDED_S(s, slen, width, left_justify);
578 f++;
579 break;
580 } case 'c': {
581 unsigned char val;
582 char buf[2];
583
584 assert(len == '?' || len == 'l');
585 assert_not_implemented(len != 'l');
586 val = va_arg(ap, int);
587 buf[0] = val;
588 buf[1] = '\0';
589 APPEND_PADDED_S(buf, 1, width, left_justify);
590 f++;
591 break;
592 } case 's':
593 assert(len == '?' || len == 'l');
594 assert_not_implemented(len != 'l');
595 s = va_arg(ap, char *);
596 slen = (prec < 0) ? strlen(s) : (size_t)prec;
597 APPEND_PADDED_S(s, slen, width, left_justify);
598 f++;
599 break;
600 case 'p': {
601 uintmax_t val;
602 char buf[X2S_BUFSIZE];
603
604 GET_ARG_NUMERIC(val, 'p');
605 s = x2s(val, true, false, buf, &slen);
606 APPEND_PADDED_S(s, slen, width, left_justify);
607 f++;
608 break;
609 } default: not_reached();
610 }
611 break;
612 } default: {
613 APPEND_C(*f);
614 f++;
615 break;
616 }}
617 }
618 label_out:
619 if (i < size) {
620 str[i] = '\0';
621 } else {
622 str[size - 1] = '\0';
623 }
624
625#undef APPEND_C
626#undef APPEND_S
627#undef APPEND_PADDED_S
628#undef GET_ARG_NUMERIC
629 return i;
630}
631
632JEMALLOC_FORMAT_PRINTF(3, 4)
633size_t
634malloc_snprintf(char *str, size_t size, const char *format, ...) {
635 size_t ret;
636 va_list ap;
637
638 va_start(ap, format);
639 ret = malloc_vsnprintf(str, size, format, ap);
640 va_end(ap);
641
642 return ret;
643}
644
645void
646malloc_vcprintf(write_cb_t *write_cb, void *cbopaque, const char *format,
647 va_list ap) {
648 char buf[MALLOC_PRINTF_BUFSIZE];
649
650 if (write_cb == NULL) {
651 /*
652 * The caller did not provide an alternate write_cb callback
653 * function, so use the default one. malloc_write() is an
654 * inline function, so use malloc_message() directly here.
655 */
656 write_cb = (je_malloc_message != NULL) ? je_malloc_message :
657 wrtmessage;
658 }
659
660 malloc_vsnprintf(buf, sizeof(buf), format, ap);
661 write_cb(cbopaque, buf);
662}
663
664/*
665 * Print to a callback function in such a way as to (hopefully) avoid memory
666 * allocation.
667 */
668JEMALLOC_FORMAT_PRINTF(3, 4)
669void
670malloc_cprintf(write_cb_t *write_cb, void *cbopaque, const char *format, ...) {
671 va_list ap;
672
673 va_start(ap, format);
674 malloc_vcprintf(write_cb, cbopaque, format, ap);
675 va_end(ap);
676}
677
678/* Print to stderr in such a way as to avoid memory allocation. */
679JEMALLOC_FORMAT_PRINTF(1, 2)
680void
681malloc_printf(const char *format, ...) {
682 va_list ap;
683
684 va_start(ap, format);
685 malloc_vcprintf(NULL, NULL, format, ap);
686 va_end(ap);
687}
688
689/*
690 * Restore normal assertion macros, in order to make it possible to compile all
691 * C files as a single concatenation.
692 */
693#undef assert
694#undef not_reached
695#undef not_implemented
696#undef assert_not_implemented
697#include "jemalloc/internal/assert.h"
diff --git a/examples/redis-unstable/deps/jemalloc/src/mutex.c b/examples/redis-unstable/deps/jemalloc/src/mutex.c
deleted file mode 100644
index 0b3547a..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/mutex.c
+++ /dev/null
@@ -1,228 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/malloc_io.h"
6#include "jemalloc/internal/spin.h"
7
8#ifndef _CRT_SPINCOUNT
9#define _CRT_SPINCOUNT 4000
10#endif
11
12/*
13 * Based on benchmark results, a fixed spin with this amount of retries works
14 * well for our critical sections.
15 */
16int64_t opt_mutex_max_spin = 600;
17
18/******************************************************************************/
19/* Data. */
20
21#ifdef JEMALLOC_LAZY_LOCK
22bool isthreaded = false;
23#endif
24#ifdef JEMALLOC_MUTEX_INIT_CB
25static bool postpone_init = true;
26static malloc_mutex_t *postponed_mutexes = NULL;
27#endif
28
29/******************************************************************************/
30/*
31 * We intercept pthread_create() calls in order to toggle isthreaded if the
32 * process goes multi-threaded.
33 */
34
35#if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32)
36JEMALLOC_EXPORT int
37pthread_create(pthread_t *__restrict thread,
38 const pthread_attr_t *__restrict attr, void *(*start_routine)(void *),
39 void *__restrict arg) {
40 return pthread_create_wrapper(thread, attr, start_routine, arg);
41}
42#endif
43
44/******************************************************************************/
45
46#ifdef JEMALLOC_MUTEX_INIT_CB
47JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
48 void *(calloc_cb)(size_t, size_t));
49#endif
50
51void
52malloc_mutex_lock_slow(malloc_mutex_t *mutex) {
53 mutex_prof_data_t *data = &mutex->prof_data;
54 nstime_t before;
55
56 if (ncpus == 1) {
57 goto label_spin_done;
58 }
59
60 int cnt = 0;
61 do {
62 spin_cpu_spinwait();
63 if (!atomic_load_b(&mutex->locked, ATOMIC_RELAXED)
64 && !malloc_mutex_trylock_final(mutex)) {
65 data->n_spin_acquired++;
66 return;
67 }
68 } while (cnt++ < opt_mutex_max_spin || opt_mutex_max_spin == -1);
69
70 if (!config_stats) {
71 /* Only spin is useful when stats is off. */
72 malloc_mutex_lock_final(mutex);
73 return;
74 }
75label_spin_done:
76 nstime_init_update(&before);
77 /* Copy before to after to avoid clock skews. */
78 nstime_t after;
79 nstime_copy(&after, &before);
80 uint32_t n_thds = atomic_fetch_add_u32(&data->n_waiting_thds, 1,
81 ATOMIC_RELAXED) + 1;
82 /* One last try as above two calls may take quite some cycles. */
83 if (!malloc_mutex_trylock_final(mutex)) {
84 atomic_fetch_sub_u32(&data->n_waiting_thds, 1, ATOMIC_RELAXED);
85 data->n_spin_acquired++;
86 return;
87 }
88
89 /* True slow path. */
90 malloc_mutex_lock_final(mutex);
91 /* Update more slow-path only counters. */
92 atomic_fetch_sub_u32(&data->n_waiting_thds, 1, ATOMIC_RELAXED);
93 nstime_update(&after);
94
95 nstime_t delta;
96 nstime_copy(&delta, &after);
97 nstime_subtract(&delta, &before);
98
99 data->n_wait_times++;
100 nstime_add(&data->tot_wait_time, &delta);
101 if (nstime_compare(&data->max_wait_time, &delta) < 0) {
102 nstime_copy(&data->max_wait_time, &delta);
103 }
104 if (n_thds > data->max_n_thds) {
105 data->max_n_thds = n_thds;
106 }
107}
108
109static void
110mutex_prof_data_init(mutex_prof_data_t *data) {
111 memset(data, 0, sizeof(mutex_prof_data_t));
112 nstime_init_zero(&data->max_wait_time);
113 nstime_init_zero(&data->tot_wait_time);
114 data->prev_owner = NULL;
115}
116
117void
118malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex) {
119 malloc_mutex_assert_owner(tsdn, mutex);
120 mutex_prof_data_init(&mutex->prof_data);
121}
122
123static int
124mutex_addr_comp(const witness_t *witness1, void *mutex1,
125 const witness_t *witness2, void *mutex2) {
126 assert(mutex1 != NULL);
127 assert(mutex2 != NULL);
128 uintptr_t mu1int = (uintptr_t)mutex1;
129 uintptr_t mu2int = (uintptr_t)mutex2;
130 if (mu1int < mu2int) {
131 return -1;
132 } else if (mu1int == mu2int) {
133 return 0;
134 } else {
135 return 1;
136 }
137}
138
139bool
140malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
141 witness_rank_t rank, malloc_mutex_lock_order_t lock_order) {
142 mutex_prof_data_init(&mutex->prof_data);
143#ifdef _WIN32
144# if _WIN32_WINNT >= 0x0600
145 InitializeSRWLock(&mutex->lock);
146# else
147 if (!InitializeCriticalSectionAndSpinCount(&mutex->lock,
148 _CRT_SPINCOUNT)) {
149 return true;
150 }
151# endif
152#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
153 mutex->lock = OS_UNFAIR_LOCK_INIT;
154#elif (defined(JEMALLOC_MUTEX_INIT_CB))
155 if (postpone_init) {
156 mutex->postponed_next = postponed_mutexes;
157 postponed_mutexes = mutex;
158 } else {
159 if (_pthread_mutex_init_calloc_cb(&mutex->lock,
160 bootstrap_calloc) != 0) {
161 return true;
162 }
163 }
164#else
165 pthread_mutexattr_t attr;
166
167 if (pthread_mutexattr_init(&attr) != 0) {
168 return true;
169 }
170 pthread_mutexattr_settype(&attr, MALLOC_MUTEX_TYPE);
171 if (pthread_mutex_init(&mutex->lock, &attr) != 0) {
172 pthread_mutexattr_destroy(&attr);
173 return true;
174 }
175 pthread_mutexattr_destroy(&attr);
176#endif
177 if (config_debug) {
178 mutex->lock_order = lock_order;
179 if (lock_order == malloc_mutex_address_ordered) {
180 witness_init(&mutex->witness, name, rank,
181 mutex_addr_comp, mutex);
182 } else {
183 witness_init(&mutex->witness, name, rank, NULL, NULL);
184 }
185 }
186 return false;
187}
188
189void
190malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex) {
191 malloc_mutex_lock(tsdn, mutex);
192}
193
194void
195malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex) {
196 malloc_mutex_unlock(tsdn, mutex);
197}
198
199void
200malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex) {
201#ifdef JEMALLOC_MUTEX_INIT_CB
202 malloc_mutex_unlock(tsdn, mutex);
203#else
204 if (malloc_mutex_init(mutex, mutex->witness.name,
205 mutex->witness.rank, mutex->lock_order)) {
206 malloc_printf("<jemalloc>: Error re-initializing mutex in "
207 "child\n");
208 if (opt_abort) {
209 abort();
210 }
211 }
212#endif
213}
214
215bool
216malloc_mutex_boot(void) {
217#ifdef JEMALLOC_MUTEX_INIT_CB
218 postpone_init = false;
219 while (postponed_mutexes != NULL) {
220 if (_pthread_mutex_init_calloc_cb(&postponed_mutexes->lock,
221 bootstrap_calloc) != 0) {
222 return true;
223 }
224 postponed_mutexes = postponed_mutexes->postponed_next;
225 }
226#endif
227 return false;
228}
diff --git a/examples/redis-unstable/deps/jemalloc/src/nstime.c b/examples/redis-unstable/deps/jemalloc/src/nstime.c
deleted file mode 100644
index a1a5377..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/nstime.c
+++ /dev/null
@@ -1,289 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/nstime.h"
5
6#include "jemalloc/internal/assert.h"
7
8#define BILLION UINT64_C(1000000000)
9#define MILLION UINT64_C(1000000)
10
11static void
12nstime_set_initialized(nstime_t *time) {
13#ifdef JEMALLOC_DEBUG
14 time->magic = NSTIME_MAGIC;
15#endif
16}
17
18static void
19nstime_assert_initialized(const nstime_t *time) {
20#ifdef JEMALLOC_DEBUG
21 /*
22 * Some parts (e.g. stats) rely on memset to zero initialize. Treat
23 * these as valid initialization.
24 */
25 assert(time->magic == NSTIME_MAGIC ||
26 (time->magic == 0 && time->ns == 0));
27#endif
28}
29
30static void
31nstime_pair_assert_initialized(const nstime_t *t1, const nstime_t *t2) {
32 nstime_assert_initialized(t1);
33 nstime_assert_initialized(t2);
34}
35
36static void
37nstime_initialize_operand(nstime_t *time) {
38 /*
39 * Operations like nstime_add may have the initial operand being zero
40 * initialized (covered by the assert below). Full-initialize needed
41 * before changing it to non-zero.
42 */
43 nstime_assert_initialized(time);
44 nstime_set_initialized(time);
45}
46
47void
48nstime_init(nstime_t *time, uint64_t ns) {
49 nstime_set_initialized(time);
50 time->ns = ns;
51}
52
53void
54nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec) {
55 nstime_set_initialized(time);
56 time->ns = sec * BILLION + nsec;
57}
58
59uint64_t
60nstime_ns(const nstime_t *time) {
61 nstime_assert_initialized(time);
62 return time->ns;
63}
64
65uint64_t
66nstime_msec(const nstime_t *time) {
67 nstime_assert_initialized(time);
68 return time->ns / MILLION;
69}
70
71uint64_t
72nstime_sec(const nstime_t *time) {
73 nstime_assert_initialized(time);
74 return time->ns / BILLION;
75}
76
77uint64_t
78nstime_nsec(const nstime_t *time) {
79 nstime_assert_initialized(time);
80 return time->ns % BILLION;
81}
82
83void
84nstime_copy(nstime_t *time, const nstime_t *source) {
85 /* Source is required to be initialized. */
86 nstime_assert_initialized(source);
87 *time = *source;
88 nstime_assert_initialized(time);
89}
90
91int
92nstime_compare(const nstime_t *a, const nstime_t *b) {
93 nstime_pair_assert_initialized(a, b);
94 return (a->ns > b->ns) - (a->ns < b->ns);
95}
96
97void
98nstime_add(nstime_t *time, const nstime_t *addend) {
99 nstime_pair_assert_initialized(time, addend);
100 assert(UINT64_MAX - time->ns >= addend->ns);
101
102 nstime_initialize_operand(time);
103 time->ns += addend->ns;
104}
105
106void
107nstime_iadd(nstime_t *time, uint64_t addend) {
108 nstime_assert_initialized(time);
109 assert(UINT64_MAX - time->ns >= addend);
110
111 nstime_initialize_operand(time);
112 time->ns += addend;
113}
114
115void
116nstime_subtract(nstime_t *time, const nstime_t *subtrahend) {
117 nstime_pair_assert_initialized(time, subtrahend);
118 assert(nstime_compare(time, subtrahend) >= 0);
119
120 /* No initialize operand -- subtraction must be initialized. */
121 time->ns -= subtrahend->ns;
122}
123
124void
125nstime_isubtract(nstime_t *time, uint64_t subtrahend) {
126 nstime_assert_initialized(time);
127 assert(time->ns >= subtrahend);
128
129 /* No initialize operand -- subtraction must be initialized. */
130 time->ns -= subtrahend;
131}
132
133void
134nstime_imultiply(nstime_t *time, uint64_t multiplier) {
135 nstime_assert_initialized(time);
136 assert((((time->ns | multiplier) & (UINT64_MAX << (sizeof(uint64_t) <<
137 2))) == 0) || ((time->ns * multiplier) / multiplier == time->ns));
138
139 nstime_initialize_operand(time);
140 time->ns *= multiplier;
141}
142
143void
144nstime_idivide(nstime_t *time, uint64_t divisor) {
145 nstime_assert_initialized(time);
146 assert(divisor != 0);
147
148 nstime_initialize_operand(time);
149 time->ns /= divisor;
150}
151
152uint64_t
153nstime_divide(const nstime_t *time, const nstime_t *divisor) {
154 nstime_pair_assert_initialized(time, divisor);
155 assert(divisor->ns != 0);
156
157 /* No initialize operand -- *time itself remains unchanged. */
158 return time->ns / divisor->ns;
159}
160
161/* Returns time since *past, w/o updating *past. */
162uint64_t
163nstime_ns_since(const nstime_t *past) {
164 nstime_assert_initialized(past);
165
166 nstime_t now;
167 nstime_copy(&now, past);
168 nstime_update(&now);
169
170 assert(nstime_compare(&now, past) >= 0);
171 return now.ns - past->ns;
172}
173
174#ifdef _WIN32
175# define NSTIME_MONOTONIC true
176static void
177nstime_get(nstime_t *time) {
178 FILETIME ft;
179 uint64_t ticks_100ns;
180
181 GetSystemTimeAsFileTime(&ft);
182 ticks_100ns = (((uint64_t)ft.dwHighDateTime) << 32) | ft.dwLowDateTime;
183
184 nstime_init(time, ticks_100ns * 100);
185}
186#elif defined(JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE)
187# define NSTIME_MONOTONIC true
188static void
189nstime_get(nstime_t *time) {
190 struct timespec ts;
191
192 clock_gettime(CLOCK_MONOTONIC_COARSE, &ts);
193 nstime_init2(time, ts.tv_sec, ts.tv_nsec);
194}
195#elif defined(JEMALLOC_HAVE_CLOCK_MONOTONIC)
196# define NSTIME_MONOTONIC true
197static void
198nstime_get(nstime_t *time) {
199 struct timespec ts;
200
201 clock_gettime(CLOCK_MONOTONIC, &ts);
202 nstime_init2(time, ts.tv_sec, ts.tv_nsec);
203}
204#elif defined(JEMALLOC_HAVE_MACH_ABSOLUTE_TIME)
205# define NSTIME_MONOTONIC true
206static void
207nstime_get(nstime_t *time) {
208 nstime_init(time, mach_absolute_time());
209}
210#else
211# define NSTIME_MONOTONIC false
212static void
213nstime_get(nstime_t *time) {
214 struct timeval tv;
215
216 gettimeofday(&tv, NULL);
217 nstime_init2(time, tv.tv_sec, tv.tv_usec * 1000);
218}
219#endif
220
221static bool
222nstime_monotonic_impl(void) {
223 return NSTIME_MONOTONIC;
224#undef NSTIME_MONOTONIC
225}
226nstime_monotonic_t *JET_MUTABLE nstime_monotonic = nstime_monotonic_impl;
227
228prof_time_res_t opt_prof_time_res =
229 prof_time_res_default;
230
231const char *prof_time_res_mode_names[] = {
232 "default",
233 "high",
234};
235
236
237static void
238nstime_get_realtime(nstime_t *time) {
239#if defined(JEMALLOC_HAVE_CLOCK_REALTIME) && !defined(_WIN32)
240 struct timespec ts;
241
242 clock_gettime(CLOCK_REALTIME, &ts);
243 nstime_init2(time, ts.tv_sec, ts.tv_nsec);
244#else
245 unreachable();
246#endif
247}
248
249static void
250nstime_prof_update_impl(nstime_t *time) {
251 nstime_t old_time;
252
253 nstime_copy(&old_time, time);
254
255 if (opt_prof_time_res == prof_time_res_high) {
256 nstime_get_realtime(time);
257 } else {
258 nstime_get(time);
259 }
260}
261nstime_prof_update_t *JET_MUTABLE nstime_prof_update = nstime_prof_update_impl;
262
263static void
264nstime_update_impl(nstime_t *time) {
265 nstime_t old_time;
266
267 nstime_copy(&old_time, time);
268 nstime_get(time);
269
270 /* Handle non-monotonic clocks. */
271 if (unlikely(nstime_compare(&old_time, time) > 0)) {
272 nstime_copy(time, &old_time);
273 }
274}
275nstime_update_t *JET_MUTABLE nstime_update = nstime_update_impl;
276
277void
278nstime_init_update(nstime_t *time) {
279 nstime_init_zero(time);
280 nstime_update(time);
281}
282
283void
284nstime_prof_init_update(nstime_t *time) {
285 nstime_init_zero(time);
286 nstime_prof_update(time);
287}
288
289
diff --git a/examples/redis-unstable/deps/jemalloc/src/pa.c b/examples/redis-unstable/deps/jemalloc/src/pa.c
deleted file mode 100644
index eb7e462..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/pa.c
+++ /dev/null
@@ -1,277 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/san.h"
5#include "jemalloc/internal/hpa.h"
6
7static void
8pa_nactive_add(pa_shard_t *shard, size_t add_pages) {
9 atomic_fetch_add_zu(&shard->nactive, add_pages, ATOMIC_RELAXED);
10}
11
12static void
13pa_nactive_sub(pa_shard_t *shard, size_t sub_pages) {
14 assert(atomic_load_zu(&shard->nactive, ATOMIC_RELAXED) >= sub_pages);
15 atomic_fetch_sub_zu(&shard->nactive, sub_pages, ATOMIC_RELAXED);
16}
17
18bool
19pa_central_init(pa_central_t *central, base_t *base, bool hpa,
20 hpa_hooks_t *hpa_hooks) {
21 bool err;
22 if (hpa) {
23 err = hpa_central_init(&central->hpa, base, hpa_hooks);
24 if (err) {
25 return true;
26 }
27 }
28 return false;
29}
30
31bool
32pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central,
33 emap_t *emap, base_t *base, unsigned ind, pa_shard_stats_t *stats,
34 malloc_mutex_t *stats_mtx, nstime_t *cur_time,
35 size_t pac_oversize_threshold, ssize_t dirty_decay_ms,
36 ssize_t muzzy_decay_ms) {
37 /* This will change eventually, but for now it should hold. */
38 assert(base_ind_get(base) == ind);
39 if (edata_cache_init(&shard->edata_cache, base)) {
40 return true;
41 }
42
43 if (pac_init(tsdn, &shard->pac, base, emap, &shard->edata_cache,
44 cur_time, pac_oversize_threshold, dirty_decay_ms, muzzy_decay_ms,
45 &stats->pac_stats, stats_mtx)) {
46 return true;
47 }
48
49 shard->ind = ind;
50
51 shard->ever_used_hpa = false;
52 atomic_store_b(&shard->use_hpa, false, ATOMIC_RELAXED);
53
54 atomic_store_zu(&shard->nactive, 0, ATOMIC_RELAXED);
55
56 shard->stats_mtx = stats_mtx;
57 shard->stats = stats;
58 memset(shard->stats, 0, sizeof(*shard->stats));
59
60 shard->central = central;
61 shard->emap = emap;
62 shard->base = base;
63
64 return false;
65}
66
67bool
68pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard,
69 const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts) {
70 if (hpa_shard_init(&shard->hpa_shard, &shard->central->hpa, shard->emap,
71 shard->base, &shard->edata_cache, shard->ind, hpa_opts)) {
72 return true;
73 }
74 if (sec_init(tsdn, &shard->hpa_sec, shard->base, &shard->hpa_shard.pai,
75 hpa_sec_opts)) {
76 return true;
77 }
78 shard->ever_used_hpa = true;
79 atomic_store_b(&shard->use_hpa, true, ATOMIC_RELAXED);
80
81 return false;
82}
83
84void
85pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) {
86 atomic_store_b(&shard->use_hpa, false, ATOMIC_RELAXED);
87 if (shard->ever_used_hpa) {
88 sec_disable(tsdn, &shard->hpa_sec);
89 hpa_shard_disable(tsdn, &shard->hpa_shard);
90 }
91}
92
93void
94pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard) {
95 atomic_store_zu(&shard->nactive, 0, ATOMIC_RELAXED);
96 if (shard->ever_used_hpa) {
97 sec_flush(tsdn, &shard->hpa_sec);
98 }
99}
100
101static bool
102pa_shard_uses_hpa(pa_shard_t *shard) {
103 return atomic_load_b(&shard->use_hpa, ATOMIC_RELAXED);
104}
105
106void
107pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) {
108 pac_destroy(tsdn, &shard->pac);
109 if (shard->ever_used_hpa) {
110 sec_flush(tsdn, &shard->hpa_sec);
111 hpa_shard_disable(tsdn, &shard->hpa_shard);
112 }
113}
114
115static pai_t *
116pa_get_pai(pa_shard_t *shard, edata_t *edata) {
117 return (edata_pai_get(edata) == EXTENT_PAI_PAC
118 ? &shard->pac.pai : &shard->hpa_sec.pai);
119}
120
121edata_t *
122pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
123 bool slab, szind_t szind, bool zero, bool guarded,
124 bool *deferred_work_generated) {
125 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
126 WITNESS_RANK_CORE, 0);
127 assert(!guarded || alignment <= PAGE);
128
129 edata_t *edata = NULL;
130 if (!guarded && pa_shard_uses_hpa(shard)) {
131 edata = pai_alloc(tsdn, &shard->hpa_sec.pai, size, alignment,
132 zero, /* guarded */ false, slab, deferred_work_generated);
133 }
134 /*
135 * Fall back to the PAC if the HPA is off or couldn't serve the given
136 * allocation request.
137 */
138 if (edata == NULL) {
139 edata = pai_alloc(tsdn, &shard->pac.pai, size, alignment, zero,
140 guarded, slab, deferred_work_generated);
141 }
142 if (edata != NULL) {
143 assert(edata_size_get(edata) == size);
144 pa_nactive_add(shard, size >> LG_PAGE);
145 emap_remap(tsdn, shard->emap, edata, szind, slab);
146 edata_szind_set(edata, szind);
147 edata_slab_set(edata, slab);
148 if (slab && (size > 2 * PAGE)) {
149 emap_register_interior(tsdn, shard->emap, edata, szind);
150 }
151 assert(edata_arena_ind_get(edata) == shard->ind);
152 }
153 return edata;
154}
155
156bool
157pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
158 size_t new_size, szind_t szind, bool zero, bool *deferred_work_generated) {
159 assert(new_size > old_size);
160 assert(edata_size_get(edata) == old_size);
161 assert((new_size & PAGE_MASK) == 0);
162 if (edata_guarded_get(edata)) {
163 return true;
164 }
165 size_t expand_amount = new_size - old_size;
166
167 pai_t *pai = pa_get_pai(shard, edata);
168
169 bool error = pai_expand(tsdn, pai, edata, old_size, new_size, zero,
170 deferred_work_generated);
171 if (error) {
172 return true;
173 }
174
175 pa_nactive_add(shard, expand_amount >> LG_PAGE);
176 edata_szind_set(edata, szind);
177 emap_remap(tsdn, shard->emap, edata, szind, /* slab */ false);
178 return false;
179}
180
181bool
182pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
183 size_t new_size, szind_t szind, bool *deferred_work_generated) {
184 assert(new_size < old_size);
185 assert(edata_size_get(edata) == old_size);
186 assert((new_size & PAGE_MASK) == 0);
187 if (edata_guarded_get(edata)) {
188 return true;
189 }
190 size_t shrink_amount = old_size - new_size;
191
192 pai_t *pai = pa_get_pai(shard, edata);
193 bool error = pai_shrink(tsdn, pai, edata, old_size, new_size,
194 deferred_work_generated);
195 if (error) {
196 return true;
197 }
198 pa_nactive_sub(shard, shrink_amount >> LG_PAGE);
199
200 edata_szind_set(edata, szind);
201 emap_remap(tsdn, shard->emap, edata, szind, /* slab */ false);
202 return false;
203}
204
205void
206pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
207 bool *deferred_work_generated) {
208 emap_remap(tsdn, shard->emap, edata, SC_NSIZES, /* slab */ false);
209 if (edata_slab_get(edata)) {
210 emap_deregister_interior(tsdn, shard->emap, edata);
211 /*
212 * The slab state of the extent isn't cleared. It may be used
213 * by the pai implementation, e.g. to make caching decisions.
214 */
215 }
216 edata_addr_set(edata, edata_base_get(edata));
217 edata_szind_set(edata, SC_NSIZES);
218 pa_nactive_sub(shard, edata_size_get(edata) >> LG_PAGE);
219 pai_t *pai = pa_get_pai(shard, edata);
220 pai_dalloc(tsdn, pai, edata, deferred_work_generated);
221}
222
223bool
224pa_shard_retain_grow_limit_get_set(tsdn_t *tsdn, pa_shard_t *shard,
225 size_t *old_limit, size_t *new_limit) {
226 return pac_retain_grow_limit_get_set(tsdn, &shard->pac, old_limit,
227 new_limit);
228}
229
230bool
231pa_decay_ms_set(tsdn_t *tsdn, pa_shard_t *shard, extent_state_t state,
232 ssize_t decay_ms, pac_purge_eagerness_t eagerness) {
233 return pac_decay_ms_set(tsdn, &shard->pac, state, decay_ms, eagerness);
234}
235
236ssize_t
237pa_decay_ms_get(pa_shard_t *shard, extent_state_t state) {
238 return pac_decay_ms_get(&shard->pac, state);
239}
240
241void
242pa_shard_set_deferral_allowed(tsdn_t *tsdn, pa_shard_t *shard,
243 bool deferral_allowed) {
244 if (pa_shard_uses_hpa(shard)) {
245 hpa_shard_set_deferral_allowed(tsdn, &shard->hpa_shard,
246 deferral_allowed);
247 }
248}
249
250void
251pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
252 if (pa_shard_uses_hpa(shard)) {
253 hpa_shard_do_deferred_work(tsdn, &shard->hpa_shard);
254 }
255}
256
257/*
258 * Get time until next deferred work ought to happen. If there are multiple
259 * things that have been deferred, this function calculates the time until
260 * the soonest of those things.
261 */
262uint64_t
263pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
264 uint64_t time = pai_time_until_deferred_work(tsdn, &shard->pac.pai);
265 if (time == BACKGROUND_THREAD_DEFERRED_MIN) {
266 return time;
267 }
268
269 if (pa_shard_uses_hpa(shard)) {
270 uint64_t hpa =
271 pai_time_until_deferred_work(tsdn, &shard->hpa_shard.pai);
272 if (hpa < time) {
273 time = hpa;
274 }
275 }
276 return time;
277}
diff --git a/examples/redis-unstable/deps/jemalloc/src/pa_extra.c b/examples/redis-unstable/deps/jemalloc/src/pa_extra.c
deleted file mode 100644
index 0f488be..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/pa_extra.c
+++ /dev/null
@@ -1,191 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4/*
5 * This file is logically part of the PA module. While pa.c contains the core
6 * allocator functionality, this file contains boring integration functionality;
7 * things like the pre- and post- fork handlers, and stats merging for CTL
8 * refreshes.
9 */
10
11void
12pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard) {
13 malloc_mutex_prefork(tsdn, &shard->pac.decay_dirty.mtx);
14 malloc_mutex_prefork(tsdn, &shard->pac.decay_muzzy.mtx);
15}
16
17void
18pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard) {
19 if (shard->ever_used_hpa) {
20 sec_prefork2(tsdn, &shard->hpa_sec);
21 }
22}
23
24void
25pa_shard_prefork3(tsdn_t *tsdn, pa_shard_t *shard) {
26 malloc_mutex_prefork(tsdn, &shard->pac.grow_mtx);
27 if (shard->ever_used_hpa) {
28 hpa_shard_prefork3(tsdn, &shard->hpa_shard);
29 }
30}
31
32void
33pa_shard_prefork4(tsdn_t *tsdn, pa_shard_t *shard) {
34 ecache_prefork(tsdn, &shard->pac.ecache_dirty);
35 ecache_prefork(tsdn, &shard->pac.ecache_muzzy);
36 ecache_prefork(tsdn, &shard->pac.ecache_retained);
37 if (shard->ever_used_hpa) {
38 hpa_shard_prefork4(tsdn, &shard->hpa_shard);
39 }
40}
41
42void
43pa_shard_prefork5(tsdn_t *tsdn, pa_shard_t *shard) {
44 edata_cache_prefork(tsdn, &shard->edata_cache);
45}
46
47void
48pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard) {
49 edata_cache_postfork_parent(tsdn, &shard->edata_cache);
50 ecache_postfork_parent(tsdn, &shard->pac.ecache_dirty);
51 ecache_postfork_parent(tsdn, &shard->pac.ecache_muzzy);
52 ecache_postfork_parent(tsdn, &shard->pac.ecache_retained);
53 malloc_mutex_postfork_parent(tsdn, &shard->pac.grow_mtx);
54 malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_dirty.mtx);
55 malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_muzzy.mtx);
56 if (shard->ever_used_hpa) {
57 sec_postfork_parent(tsdn, &shard->hpa_sec);
58 hpa_shard_postfork_parent(tsdn, &shard->hpa_shard);
59 }
60}
61
62void
63pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard) {
64 edata_cache_postfork_child(tsdn, &shard->edata_cache);
65 ecache_postfork_child(tsdn, &shard->pac.ecache_dirty);
66 ecache_postfork_child(tsdn, &shard->pac.ecache_muzzy);
67 ecache_postfork_child(tsdn, &shard->pac.ecache_retained);
68 malloc_mutex_postfork_child(tsdn, &shard->pac.grow_mtx);
69 malloc_mutex_postfork_child(tsdn, &shard->pac.decay_dirty.mtx);
70 malloc_mutex_postfork_child(tsdn, &shard->pac.decay_muzzy.mtx);
71 if (shard->ever_used_hpa) {
72 sec_postfork_child(tsdn, &shard->hpa_sec);
73 hpa_shard_postfork_child(tsdn, &shard->hpa_shard);
74 }
75}
76
77void
78pa_shard_basic_stats_merge(pa_shard_t *shard, size_t *nactive, size_t *ndirty,
79 size_t *nmuzzy) {
80 *nactive += atomic_load_zu(&shard->nactive, ATOMIC_RELAXED);
81 *ndirty += ecache_npages_get(&shard->pac.ecache_dirty);
82 *nmuzzy += ecache_npages_get(&shard->pac.ecache_muzzy);
83}
84
85void
86pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
87 pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,
88 hpa_shard_stats_t *hpa_stats_out, sec_stats_t *sec_stats_out,
89 size_t *resident) {
90 cassert(config_stats);
91
92 pa_shard_stats_out->pac_stats.retained +=
93 ecache_npages_get(&shard->pac.ecache_retained) << LG_PAGE;
94 pa_shard_stats_out->edata_avail += atomic_load_zu(
95 &shard->edata_cache.count, ATOMIC_RELAXED);
96
97 size_t resident_pgs = 0;
98 resident_pgs += atomic_load_zu(&shard->nactive, ATOMIC_RELAXED);
99 resident_pgs += ecache_npages_get(&shard->pac.ecache_dirty);
100 *resident += (resident_pgs << LG_PAGE);
101
102 /* Dirty decay stats */
103 locked_inc_u64_unsynchronized(
104 &pa_shard_stats_out->pac_stats.decay_dirty.npurge,
105 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
106 &shard->pac.stats->decay_dirty.npurge));
107 locked_inc_u64_unsynchronized(
108 &pa_shard_stats_out->pac_stats.decay_dirty.nmadvise,
109 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
110 &shard->pac.stats->decay_dirty.nmadvise));
111 locked_inc_u64_unsynchronized(
112 &pa_shard_stats_out->pac_stats.decay_dirty.purged,
113 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
114 &shard->pac.stats->decay_dirty.purged));
115
116 /* Muzzy decay stats */
117 locked_inc_u64_unsynchronized(
118 &pa_shard_stats_out->pac_stats.decay_muzzy.npurge,
119 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
120 &shard->pac.stats->decay_muzzy.npurge));
121 locked_inc_u64_unsynchronized(
122 &pa_shard_stats_out->pac_stats.decay_muzzy.nmadvise,
123 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
124 &shard->pac.stats->decay_muzzy.nmadvise));
125 locked_inc_u64_unsynchronized(
126 &pa_shard_stats_out->pac_stats.decay_muzzy.purged,
127 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
128 &shard->pac.stats->decay_muzzy.purged));
129
130 atomic_load_add_store_zu(&pa_shard_stats_out->pac_stats.abandoned_vm,
131 atomic_load_zu(&shard->pac.stats->abandoned_vm, ATOMIC_RELAXED));
132
133 for (pszind_t i = 0; i < SC_NPSIZES; i++) {
134 size_t dirty, muzzy, retained, dirty_bytes, muzzy_bytes,
135 retained_bytes;
136 dirty = ecache_nextents_get(&shard->pac.ecache_dirty, i);
137 muzzy = ecache_nextents_get(&shard->pac.ecache_muzzy, i);
138 retained = ecache_nextents_get(&shard->pac.ecache_retained, i);
139 dirty_bytes = ecache_nbytes_get(&shard->pac.ecache_dirty, i);
140 muzzy_bytes = ecache_nbytes_get(&shard->pac.ecache_muzzy, i);
141 retained_bytes = ecache_nbytes_get(&shard->pac.ecache_retained,
142 i);
143
144 estats_out[i].ndirty = dirty;
145 estats_out[i].nmuzzy = muzzy;
146 estats_out[i].nretained = retained;
147 estats_out[i].dirty_bytes = dirty_bytes;
148 estats_out[i].muzzy_bytes = muzzy_bytes;
149 estats_out[i].retained_bytes = retained_bytes;
150 }
151
152 if (shard->ever_used_hpa) {
153 hpa_shard_stats_merge(tsdn, &shard->hpa_shard, hpa_stats_out);
154 sec_stats_merge(tsdn, &shard->hpa_sec, sec_stats_out);
155 }
156}
157
158static void
159pa_shard_mtx_stats_read_single(tsdn_t *tsdn, mutex_prof_data_t *mutex_prof_data,
160 malloc_mutex_t *mtx, int ind) {
161 malloc_mutex_lock(tsdn, mtx);
162 malloc_mutex_prof_read(tsdn, &mutex_prof_data[ind], mtx);
163 malloc_mutex_unlock(tsdn, mtx);
164}
165
166void
167pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard,
168 mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes]) {
169 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
170 &shard->edata_cache.mtx, arena_prof_mutex_extent_avail);
171 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
172 &shard->pac.ecache_dirty.mtx, arena_prof_mutex_extents_dirty);
173 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
174 &shard->pac.ecache_muzzy.mtx, arena_prof_mutex_extents_muzzy);
175 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
176 &shard->pac.ecache_retained.mtx, arena_prof_mutex_extents_retained);
177 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
178 &shard->pac.decay_dirty.mtx, arena_prof_mutex_decay_dirty);
179 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
180 &shard->pac.decay_muzzy.mtx, arena_prof_mutex_decay_muzzy);
181
182 if (shard->ever_used_hpa) {
183 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
184 &shard->hpa_shard.mtx, arena_prof_mutex_hpa_shard);
185 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
186 &shard->hpa_shard.grow_mtx,
187 arena_prof_mutex_hpa_shard_grow);
188 sec_mutex_stats_read(tsdn, &shard->hpa_sec,
189 &mutex_prof_data[arena_prof_mutex_hpa_sec]);
190 }
191}
diff --git a/examples/redis-unstable/deps/jemalloc/src/pac.c b/examples/redis-unstable/deps/jemalloc/src/pac.c
deleted file mode 100644
index 53e3d82..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/pac.c
+++ /dev/null
@@ -1,587 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/pac.h"
5#include "jemalloc/internal/san.h"
6
7static edata_t *pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size,
8 size_t alignment, bool zero, bool guarded, bool frequent_reuse,
9 bool *deferred_work_generated);
10static bool pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
11 size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
12static bool pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
13 size_t old_size, size_t new_size, bool *deferred_work_generated);
14static void pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
15 bool *deferred_work_generated);
16static uint64_t pac_time_until_deferred_work(tsdn_t *tsdn, pai_t *self);
17
18static inline void
19pac_decay_data_get(pac_t *pac, extent_state_t state,
20 decay_t **r_decay, pac_decay_stats_t **r_decay_stats, ecache_t **r_ecache) {
21 switch(state) {
22 case extent_state_dirty:
23 *r_decay = &pac->decay_dirty;
24 *r_decay_stats = &pac->stats->decay_dirty;
25 *r_ecache = &pac->ecache_dirty;
26 return;
27 case extent_state_muzzy:
28 *r_decay = &pac->decay_muzzy;
29 *r_decay_stats = &pac->stats->decay_muzzy;
30 *r_ecache = &pac->ecache_muzzy;
31 return;
32 default:
33 unreachable();
34 }
35}
36
37bool
38pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
39 edata_cache_t *edata_cache, nstime_t *cur_time,
40 size_t pac_oversize_threshold, ssize_t dirty_decay_ms,
41 ssize_t muzzy_decay_ms, pac_stats_t *pac_stats, malloc_mutex_t *stats_mtx) {
42 unsigned ind = base_ind_get(base);
43 /*
44 * Delay coalescing for dirty extents despite the disruptive effect on
45 * memory layout for best-fit extent allocation, since cached extents
46 * are likely to be reused soon after deallocation, and the cost of
47 * merging/splitting extents is non-trivial.
48 */
49 if (ecache_init(tsdn, &pac->ecache_dirty, extent_state_dirty, ind,
50 /* delay_coalesce */ true)) {
51 return true;
52 }
53 /*
54 * Coalesce muzzy extents immediately, because operations on them are in
55 * the critical path much less often than for dirty extents.
56 */
57 if (ecache_init(tsdn, &pac->ecache_muzzy, extent_state_muzzy, ind,
58 /* delay_coalesce */ false)) {
59 return true;
60 }
61 /*
62 * Coalesce retained extents immediately, in part because they will
63 * never be evicted (and therefore there's no opportunity for delayed
64 * coalescing), but also because operations on retained extents are not
65 * in the critical path.
66 */
67 if (ecache_init(tsdn, &pac->ecache_retained, extent_state_retained,
68 ind, /* delay_coalesce */ false)) {
69 return true;
70 }
71 exp_grow_init(&pac->exp_grow);
72 if (malloc_mutex_init(&pac->grow_mtx, "extent_grow",
73 WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) {
74 return true;
75 }
76 atomic_store_zu(&pac->oversize_threshold, pac_oversize_threshold,
77 ATOMIC_RELAXED);
78 if (decay_init(&pac->decay_dirty, cur_time, dirty_decay_ms)) {
79 return true;
80 }
81 if (decay_init(&pac->decay_muzzy, cur_time, muzzy_decay_ms)) {
82 return true;
83 }
84 if (san_bump_alloc_init(&pac->sba)) {
85 return true;
86 }
87
88 pac->base = base;
89 pac->emap = emap;
90 pac->edata_cache = edata_cache;
91 pac->stats = pac_stats;
92 pac->stats_mtx = stats_mtx;
93 atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED);
94
95 pac->pai.alloc = &pac_alloc_impl;
96 pac->pai.alloc_batch = &pai_alloc_batch_default;
97 pac->pai.expand = &pac_expand_impl;
98 pac->pai.shrink = &pac_shrink_impl;
99 pac->pai.dalloc = &pac_dalloc_impl;
100 pac->pai.dalloc_batch = &pai_dalloc_batch_default;
101 pac->pai.time_until_deferred_work = &pac_time_until_deferred_work;
102
103 return false;
104}
105
106static inline bool
107pac_may_have_muzzy(pac_t *pac) {
108 return pac_decay_ms_get(pac, extent_state_muzzy) != 0;
109}
110
111static edata_t *
112pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
113 size_t alignment, bool zero, bool guarded) {
114 assert(!guarded || alignment <= PAGE);
115
116 edata_t *edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
117 NULL, size, alignment, zero, guarded);
118
119 if (edata == NULL && pac_may_have_muzzy(pac)) {
120 edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy,
121 NULL, size, alignment, zero, guarded);
122 }
123 if (edata == NULL) {
124 edata = ecache_alloc_grow(tsdn, pac, ehooks,
125 &pac->ecache_retained, NULL, size, alignment, zero,
126 guarded);
127 if (config_stats && edata != NULL) {
128 atomic_fetch_add_zu(&pac->stats->pac_mapped, size,
129 ATOMIC_RELAXED);
130 }
131 }
132
133 return edata;
134}
135
136static edata_t *
137pac_alloc_new_guarded(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
138 size_t alignment, bool zero, bool frequent_reuse) {
139 assert(alignment <= PAGE);
140
141 edata_t *edata;
142 if (san_bump_enabled() && frequent_reuse) {
143 edata = san_bump_alloc(tsdn, &pac->sba, pac, ehooks, size,
144 zero);
145 } else {
146 size_t size_with_guards = san_two_side_guarded_sz(size);
147 /* Alloc a non-guarded extent first.*/
148 edata = pac_alloc_real(tsdn, pac, ehooks, size_with_guards,
149 /* alignment */ PAGE, zero, /* guarded */ false);
150 if (edata != NULL) {
151 /* Add guards around it. */
152 assert(edata_size_get(edata) == size_with_guards);
153 san_guard_pages_two_sided(tsdn, ehooks, edata,
154 pac->emap, true);
155 }
156 }
157 assert(edata == NULL || (edata_guarded_get(edata) &&
158 edata_size_get(edata) == size));
159
160 return edata;
161}
162
163static edata_t *
164pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment,
165 bool zero, bool guarded, bool frequent_reuse,
166 bool *deferred_work_generated) {
167 pac_t *pac = (pac_t *)self;
168 ehooks_t *ehooks = pac_ehooks_get(pac);
169
170 edata_t *edata = NULL;
171 /*
172 * The condition is an optimization - not frequently reused guarded
173 * allocations are never put in the ecache. pac_alloc_real also
174 * doesn't grow retained for guarded allocations. So pac_alloc_real
175 * for such allocations would always return NULL.
176 * */
177 if (!guarded || frequent_reuse) {
178 edata = pac_alloc_real(tsdn, pac, ehooks, size, alignment,
179 zero, guarded);
180 }
181 if (edata == NULL && guarded) {
182 /* No cached guarded extents; creating a new one. */
183 edata = pac_alloc_new_guarded(tsdn, pac, ehooks, size,
184 alignment, zero, frequent_reuse);
185 }
186
187 return edata;
188}
189
190static bool
191pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
192 size_t new_size, bool zero, bool *deferred_work_generated) {
193 pac_t *pac = (pac_t *)self;
194 ehooks_t *ehooks = pac_ehooks_get(pac);
195
196 size_t mapped_add = 0;
197 size_t expand_amount = new_size - old_size;
198
199 if (ehooks_merge_will_fail(ehooks)) {
200 return true;
201 }
202 edata_t *trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
203 edata, expand_amount, PAGE, zero, /* guarded*/ false);
204 if (trail == NULL) {
205 trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy,
206 edata, expand_amount, PAGE, zero, /* guarded*/ false);
207 }
208 if (trail == NULL) {
209 trail = ecache_alloc_grow(tsdn, pac, ehooks,
210 &pac->ecache_retained, edata, expand_amount, PAGE, zero,
211 /* guarded */ false);
212 mapped_add = expand_amount;
213 }
214 if (trail == NULL) {
215 return true;
216 }
217 if (extent_merge_wrapper(tsdn, pac, ehooks, edata, trail)) {
218 extent_dalloc_wrapper(tsdn, pac, ehooks, trail);
219 return true;
220 }
221 if (config_stats && mapped_add > 0) {
222 atomic_fetch_add_zu(&pac->stats->pac_mapped, mapped_add,
223 ATOMIC_RELAXED);
224 }
225 return false;
226}
227
228static bool
229pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
230 size_t new_size, bool *deferred_work_generated) {
231 pac_t *pac = (pac_t *)self;
232 ehooks_t *ehooks = pac_ehooks_get(pac);
233
234 size_t shrink_amount = old_size - new_size;
235
236 if (ehooks_split_will_fail(ehooks)) {
237 return true;
238 }
239
240 edata_t *trail = extent_split_wrapper(tsdn, pac, ehooks, edata,
241 new_size, shrink_amount, /* holding_core_locks */ false);
242 if (trail == NULL) {
243 return true;
244 }
245 ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, trail);
246 *deferred_work_generated = true;
247 return false;
248}
249
250static void
251pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
252 bool *deferred_work_generated) {
253 pac_t *pac = (pac_t *)self;
254 ehooks_t *ehooks = pac_ehooks_get(pac);
255
256 if (edata_guarded_get(edata)) {
257 /*
258 * Because cached guarded extents do exact fit only, large
259 * guarded extents are restored on dalloc eagerly (otherwise
260 * they will not be reused efficiently). Slab sizes have a
261 * limited number of size classes, and tend to cycle faster.
262 *
263 * In the case where coalesce is restrained (VirtualFree on
264 * Windows), guarded extents are also not cached -- otherwise
265 * during arena destroy / reset, the retained extents would not
266 * be whole regions (i.e. they are split between regular and
267 * guarded).
268 */
269 if (!edata_slab_get(edata) || !maps_coalesce) {
270 assert(edata_size_get(edata) >= SC_LARGE_MINCLASS ||
271 !maps_coalesce);
272 san_unguard_pages_two_sided(tsdn, ehooks, edata,
273 pac->emap);
274 }
275 }
276
277 ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, edata);
278 /* Purging of deallocated pages is deferred */
279 *deferred_work_generated = true;
280}
281
282static inline uint64_t
283pac_ns_until_purge(tsdn_t *tsdn, decay_t *decay, size_t npages) {
284 if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
285 /* Use minimal interval if decay is contended. */
286 return BACKGROUND_THREAD_DEFERRED_MIN;
287 }
288 uint64_t result = decay_ns_until_purge(decay, npages,
289 ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD);
290
291 malloc_mutex_unlock(tsdn, &decay->mtx);
292 return result;
293}
294
295static uint64_t
296pac_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
297 uint64_t time;
298 pac_t *pac = (pac_t *)self;
299
300 time = pac_ns_until_purge(tsdn,
301 &pac->decay_dirty,
302 ecache_npages_get(&pac->ecache_dirty));
303 if (time == BACKGROUND_THREAD_DEFERRED_MIN) {
304 return time;
305 }
306
307 uint64_t muzzy = pac_ns_until_purge(tsdn,
308 &pac->decay_muzzy,
309 ecache_npages_get(&pac->ecache_muzzy));
310 if (muzzy < time) {
311 time = muzzy;
312 }
313 return time;
314}
315
316bool
317pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit,
318 size_t *new_limit) {
319 pszind_t new_ind JEMALLOC_CC_SILENCE_INIT(0);
320 if (new_limit != NULL) {
321 size_t limit = *new_limit;
322 /* Grow no more than the new limit. */
323 if ((new_ind = sz_psz2ind(limit + 1) - 1) >= SC_NPSIZES) {
324 return true;
325 }
326 }
327
328 malloc_mutex_lock(tsdn, &pac->grow_mtx);
329 if (old_limit != NULL) {
330 *old_limit = sz_pind2sz(pac->exp_grow.limit);
331 }
332 if (new_limit != NULL) {
333 pac->exp_grow.limit = new_ind;
334 }
335 malloc_mutex_unlock(tsdn, &pac->grow_mtx);
336
337 return false;
338}
339
340static size_t
341pac_stash_decayed(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache,
342 size_t npages_limit, size_t npages_decay_max,
343 edata_list_inactive_t *result) {
344 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
345 WITNESS_RANK_CORE, 0);
346 ehooks_t *ehooks = pac_ehooks_get(pac);
347
348 /* Stash extents according to npages_limit. */
349 size_t nstashed = 0;
350 while (nstashed < npages_decay_max) {
351 edata_t *edata = ecache_evict(tsdn, pac, ehooks, ecache,
352 npages_limit);
353 if (edata == NULL) {
354 break;
355 }
356 edata_list_inactive_append(result, edata);
357 nstashed += edata_size_get(edata) >> LG_PAGE;
358 }
359 return nstashed;
360}
361
362static size_t
363pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
364 pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay,
365 edata_list_inactive_t *decay_extents) {
366 bool err;
367
368 size_t nmadvise = 0;
369 size_t nunmapped = 0;
370 size_t npurged = 0;
371
372 ehooks_t *ehooks = pac_ehooks_get(pac);
373
374 bool try_muzzy = !fully_decay
375 && pac_decay_ms_get(pac, extent_state_muzzy) != 0;
376
377 for (edata_t *edata = edata_list_inactive_first(decay_extents); edata !=
378 NULL; edata = edata_list_inactive_first(decay_extents)) {
379 edata_list_inactive_remove(decay_extents, edata);
380
381 size_t size = edata_size_get(edata);
382 size_t npages = size >> LG_PAGE;
383
384 nmadvise++;
385 npurged += npages;
386
387 switch (ecache->state) {
388 case extent_state_active:
389 not_reached();
390 case extent_state_dirty:
391 if (try_muzzy) {
392 err = extent_purge_lazy_wrapper(tsdn, ehooks,
393 edata, /* offset */ 0, size);
394 if (!err) {
395 ecache_dalloc(tsdn, pac, ehooks,
396 &pac->ecache_muzzy, edata);
397 break;
398 }
399 }
400 JEMALLOC_FALLTHROUGH;
401 case extent_state_muzzy:
402 extent_dalloc_wrapper(tsdn, pac, ehooks, edata);
403 nunmapped += npages;
404 break;
405 case extent_state_retained:
406 default:
407 not_reached();
408 }
409 }
410
411 if (config_stats) {
412 LOCKEDINT_MTX_LOCK(tsdn, *pac->stats_mtx);
413 locked_inc_u64(tsdn, LOCKEDINT_MTX(*pac->stats_mtx),
414 &decay_stats->npurge, 1);
415 locked_inc_u64(tsdn, LOCKEDINT_MTX(*pac->stats_mtx),
416 &decay_stats->nmadvise, nmadvise);
417 locked_inc_u64(tsdn, LOCKEDINT_MTX(*pac->stats_mtx),
418 &decay_stats->purged, npurged);
419 LOCKEDINT_MTX_UNLOCK(tsdn, *pac->stats_mtx);
420 atomic_fetch_sub_zu(&pac->stats->pac_mapped,
421 nunmapped << LG_PAGE, ATOMIC_RELAXED);
422 }
423
424 return npurged;
425}
426
427/*
428 * npages_limit: Decay at most npages_decay_max pages without violating the
429 * invariant: (ecache_npages_get(ecache) >= npages_limit). We need an upper
430 * bound on number of pages in order to prevent unbounded growth (namely in
431 * stashed), otherwise unbounded new pages could be added to extents during the
432 * current decay run, so that the purging thread never finishes.
433 */
434static void
435pac_decay_to_limit(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
436 pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay,
437 size_t npages_limit, size_t npages_decay_max) {
438 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
439 WITNESS_RANK_CORE, 1);
440
441 if (decay->purging || npages_decay_max == 0) {
442 return;
443 }
444 decay->purging = true;
445 malloc_mutex_unlock(tsdn, &decay->mtx);
446
447 edata_list_inactive_t decay_extents;
448 edata_list_inactive_init(&decay_extents);
449 size_t npurge = pac_stash_decayed(tsdn, pac, ecache, npages_limit,
450 npages_decay_max, &decay_extents);
451 if (npurge != 0) {
452 size_t npurged = pac_decay_stashed(tsdn, pac, decay,
453 decay_stats, ecache, fully_decay, &decay_extents);
454 assert(npurged == npurge);
455 }
456
457 malloc_mutex_lock(tsdn, &decay->mtx);
458 decay->purging = false;
459}
460
461void
462pac_decay_all(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
463 pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay) {
464 malloc_mutex_assert_owner(tsdn, &decay->mtx);
465 pac_decay_to_limit(tsdn, pac, decay, decay_stats, ecache, fully_decay,
466 /* npages_limit */ 0, ecache_npages_get(ecache));
467}
468
469static void
470pac_decay_try_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
471 pac_decay_stats_t *decay_stats, ecache_t *ecache,
472 size_t current_npages, size_t npages_limit) {
473 if (current_npages > npages_limit) {
474 pac_decay_to_limit(tsdn, pac, decay, decay_stats, ecache,
475 /* fully_decay */ false, npages_limit,
476 current_npages - npages_limit);
477 }
478}
479
480bool
481pac_maybe_decay_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
482 pac_decay_stats_t *decay_stats, ecache_t *ecache,
483 pac_purge_eagerness_t eagerness) {
484 malloc_mutex_assert_owner(tsdn, &decay->mtx);
485
486 /* Purge all or nothing if the option is disabled. */
487 ssize_t decay_ms = decay_ms_read(decay);
488 if (decay_ms <= 0) {
489 if (decay_ms == 0) {
490 pac_decay_to_limit(tsdn, pac, decay, decay_stats,
491 ecache, /* fully_decay */ false,
492 /* npages_limit */ 0, ecache_npages_get(ecache));
493 }
494 return false;
495 }
496
497 /*
498 * If the deadline has been reached, advance to the current epoch and
499 * purge to the new limit if necessary. Note that dirty pages created
500 * during the current epoch are not subject to purge until a future
501 * epoch, so as a result purging only happens during epoch advances, or
502 * being triggered by background threads (scheduled event).
503 */
504 nstime_t time;
505 nstime_init_update(&time);
506 size_t npages_current = ecache_npages_get(ecache);
507 bool epoch_advanced = decay_maybe_advance_epoch(decay, &time,
508 npages_current);
509 if (eagerness == PAC_PURGE_ALWAYS
510 || (epoch_advanced && eagerness == PAC_PURGE_ON_EPOCH_ADVANCE)) {
511 size_t npages_limit = decay_npages_limit_get(decay);
512 pac_decay_try_purge(tsdn, pac, decay, decay_stats, ecache,
513 npages_current, npages_limit);
514 }
515
516 return epoch_advanced;
517}
518
519bool
520pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
521 ssize_t decay_ms, pac_purge_eagerness_t eagerness) {
522 decay_t *decay;
523 pac_decay_stats_t *decay_stats;
524 ecache_t *ecache;
525 pac_decay_data_get(pac, state, &decay, &decay_stats, &ecache);
526
527 if (!decay_ms_valid(decay_ms)) {
528 return true;
529 }
530
531 malloc_mutex_lock(tsdn, &decay->mtx);
532 /*
533 * Restart decay backlog from scratch, which may cause many dirty pages
534 * to be immediately purged. It would conceptually be possible to map
535 * the old backlog onto the new backlog, but there is no justification
536 * for such complexity since decay_ms changes are intended to be
537 * infrequent, either between the {-1, 0, >0} states, or a one-time
538 * arbitrary change during initial arena configuration.
539 */
540 nstime_t cur_time;
541 nstime_init_update(&cur_time);
542 decay_reinit(decay, &cur_time, decay_ms);
543 pac_maybe_decay_purge(tsdn, pac, decay, decay_stats, ecache, eagerness);
544 malloc_mutex_unlock(tsdn, &decay->mtx);
545
546 return false;
547}
548
549ssize_t
550pac_decay_ms_get(pac_t *pac, extent_state_t state) {
551 decay_t *decay;
552 pac_decay_stats_t *decay_stats;
553 ecache_t *ecache;
554 pac_decay_data_get(pac, state, &decay, &decay_stats, &ecache);
555 return decay_ms_read(decay);
556}
557
558void
559pac_reset(tsdn_t *tsdn, pac_t *pac) {
560 /*
561 * No-op for now; purging is still done at the arena-level. It should
562 * get moved in here, though.
563 */
564 (void)tsdn;
565 (void)pac;
566}
567
568void
569pac_destroy(tsdn_t *tsdn, pac_t *pac) {
570 assert(ecache_npages_get(&pac->ecache_dirty) == 0);
571 assert(ecache_npages_get(&pac->ecache_muzzy) == 0);
572 /*
573 * Iterate over the retained extents and destroy them. This gives the
574 * extent allocator underlying the extent hooks an opportunity to unmap
575 * all retained memory without having to keep its own metadata
576 * structures. In practice, virtual memory for dss-allocated extents is
577 * leaked here, so best practice is to avoid dss for arenas to be
578 * destroyed, or provide custom extent hooks that track retained
579 * dss-based extents for later reuse.
580 */
581 ehooks_t *ehooks = pac_ehooks_get(pac);
582 edata_t *edata;
583 while ((edata = ecache_evict(tsdn, pac, ehooks,
584 &pac->ecache_retained, 0)) != NULL) {
585 extent_destroy_wrapper(tsdn, pac, ehooks, edata);
586 }
587}
diff --git a/examples/redis-unstable/deps/jemalloc/src/pages.c b/examples/redis-unstable/deps/jemalloc/src/pages.c
deleted file mode 100644
index 8c83a7d..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/pages.c
+++ /dev/null
@@ -1,824 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2
3#include "jemalloc/internal/pages.h"
4
5#include "jemalloc/internal/jemalloc_internal_includes.h"
6
7#include "jemalloc/internal/assert.h"
8#include "jemalloc/internal/malloc_io.h"
9
10#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
11#include <sys/sysctl.h>
12#ifdef __FreeBSD__
13#include <vm/vm_param.h>
14#endif
15#endif
16#ifdef __NetBSD__
17#include <sys/bitops.h> /* ilog2 */
18#endif
19#ifdef JEMALLOC_HAVE_VM_MAKE_TAG
20#define PAGES_FD_TAG VM_MAKE_TAG(101U)
21#else
22#define PAGES_FD_TAG -1
23#endif
24
25/******************************************************************************/
26/* Data. */
27
28/* Actual operating system page size, detected during bootstrap, <= PAGE. */
29static size_t os_page;
30
31#ifndef _WIN32
32# define PAGES_PROT_COMMIT (PROT_READ | PROT_WRITE)
33# define PAGES_PROT_DECOMMIT (PROT_NONE)
34static int mmap_flags;
35#endif
36static bool os_overcommits;
37
38const char *thp_mode_names[] = {
39 "default",
40 "always",
41 "never",
42 "not supported"
43};
44thp_mode_t opt_thp = THP_MODE_DEFAULT;
45thp_mode_t init_system_thp_mode;
46
47/* Runtime support for lazy purge. Irrelevant when !pages_can_purge_lazy. */
48static bool pages_can_purge_lazy_runtime = true;
49
50#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS
51static int madvise_dont_need_zeros_is_faulty = -1;
52/**
53 * Check that MADV_DONTNEED will actually zero pages on subsequent access.
54 *
55 * Since qemu does not support this, yet [1], and you can get very tricky
56 * assert if you will run program with jemalloc in use under qemu:
57 *
58 * <jemalloc>: ../contrib/jemalloc/src/extent.c:1195: Failed assertion: "p[i] == 0"
59 *
60 * [1]: https://patchwork.kernel.org/patch/10576637/
61 */
62static int madvise_MADV_DONTNEED_zeroes_pages()
63{
64 int works = -1;
65 size_t size = PAGE;
66
67 void * addr = mmap(NULL, size, PROT_READ|PROT_WRITE,
68 MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
69
70 if (addr == MAP_FAILED) {
71 malloc_write("<jemalloc>: Cannot allocate memory for "
72 "MADV_DONTNEED check\n");
73 if (opt_abort) {
74 abort();
75 }
76 }
77
78 memset(addr, 'A', size);
79 if (madvise(addr, size, MADV_DONTNEED) == 0) {
80 works = memchr(addr, 'A', size) == NULL;
81 } else {
82 /*
83 * If madvise() does not support MADV_DONTNEED, then we can
84 * call it anyway, and use it's return code.
85 */
86 works = 1;
87 }
88
89 if (munmap(addr, size) != 0) {
90 malloc_write("<jemalloc>: Cannot deallocate memory for "
91 "MADV_DONTNEED check\n");
92 if (opt_abort) {
93 abort();
94 }
95 }
96
97 return works;
98}
99#endif
100
101/******************************************************************************/
102/*
103 * Function prototypes for static functions that are referenced prior to
104 * definition.
105 */
106
107static void os_pages_unmap(void *addr, size_t size);
108
109/******************************************************************************/
110
111static void *
112os_pages_map(void *addr, size_t size, size_t alignment, bool *commit) {
113 assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr);
114 assert(ALIGNMENT_CEILING(size, os_page) == size);
115 assert(size != 0);
116
117 if (os_overcommits) {
118 *commit = true;
119 }
120
121 void *ret;
122#ifdef _WIN32
123 /*
124 * If VirtualAlloc can't allocate at the given address when one is
125 * given, it fails and returns NULL.
126 */
127 ret = VirtualAlloc(addr, size, MEM_RESERVE | (*commit ? MEM_COMMIT : 0),
128 PAGE_READWRITE);
129#else
130 /*
131 * We don't use MAP_FIXED here, because it can cause the *replacement*
132 * of existing mappings, and we only want to create new mappings.
133 */
134 {
135#ifdef __NetBSD__
136 /*
137 * On NetBSD PAGE for a platform is defined to the
138 * maximum page size of all machine architectures
139 * for that platform, so that we can use the same
140 * binaries across all machine architectures.
141 */
142 if (alignment > os_page || PAGE > os_page) {
143 unsigned int a = ilog2(MAX(alignment, PAGE));
144 mmap_flags |= MAP_ALIGNED(a);
145 }
146#endif
147 int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
148
149 ret = mmap(addr, size, prot, mmap_flags, PAGES_FD_TAG, 0);
150 }
151 assert(ret != NULL);
152
153 if (ret == MAP_FAILED) {
154 ret = NULL;
155 } else if (addr != NULL && ret != addr) {
156 /*
157 * We succeeded in mapping memory, but not in the right place.
158 */
159 os_pages_unmap(ret, size);
160 ret = NULL;
161 }
162#endif
163 assert(ret == NULL || (addr == NULL && ret != addr) || (addr != NULL &&
164 ret == addr));
165 return ret;
166}
167
168static void *
169os_pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size,
170 bool *commit) {
171 void *ret = (void *)((uintptr_t)addr + leadsize);
172
173 assert(alloc_size >= leadsize + size);
174#ifdef _WIN32
175 os_pages_unmap(addr, alloc_size);
176 void *new_addr = os_pages_map(ret, size, PAGE, commit);
177 if (new_addr == ret) {
178 return ret;
179 }
180 if (new_addr != NULL) {
181 os_pages_unmap(new_addr, size);
182 }
183 return NULL;
184#else
185 size_t trailsize = alloc_size - leadsize - size;
186
187 if (leadsize != 0) {
188 os_pages_unmap(addr, leadsize);
189 }
190 if (trailsize != 0) {
191 os_pages_unmap((void *)((uintptr_t)ret + size), trailsize);
192 }
193 return ret;
194#endif
195}
196
197static void
198os_pages_unmap(void *addr, size_t size) {
199 assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr);
200 assert(ALIGNMENT_CEILING(size, os_page) == size);
201
202#ifdef _WIN32
203 if (VirtualFree(addr, 0, MEM_RELEASE) == 0)
204#else
205 if (munmap(addr, size) == -1)
206#endif
207 {
208 char buf[BUFERROR_BUF];
209
210 buferror(get_errno(), buf, sizeof(buf));
211 malloc_printf("<jemalloc>: Error in "
212#ifdef _WIN32
213 "VirtualFree"
214#else
215 "munmap"
216#endif
217 "(): %s\n", buf);
218 if (opt_abort) {
219 abort();
220 }
221 }
222}
223
224static void *
225pages_map_slow(size_t size, size_t alignment, bool *commit) {
226 size_t alloc_size = size + alignment - os_page;
227 /* Beware size_t wrap-around. */
228 if (alloc_size < size) {
229 return NULL;
230 }
231
232 void *ret;
233 do {
234 void *pages = os_pages_map(NULL, alloc_size, alignment, commit);
235 if (pages == NULL) {
236 return NULL;
237 }
238 size_t leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment)
239 - (uintptr_t)pages;
240 ret = os_pages_trim(pages, alloc_size, leadsize, size, commit);
241 } while (ret == NULL);
242
243 assert(ret != NULL);
244 assert(PAGE_ADDR2BASE(ret) == ret);
245 return ret;
246}
247
248void *
249pages_map(void *addr, size_t size, size_t alignment, bool *commit) {
250 assert(alignment >= PAGE);
251 assert(ALIGNMENT_ADDR2BASE(addr, alignment) == addr);
252
253#if defined(__FreeBSD__) && defined(MAP_EXCL)
254 /*
255 * FreeBSD has mechanisms both to mmap at specific address without
256 * touching existing mappings, and to mmap with specific alignment.
257 */
258 {
259 if (os_overcommits) {
260 *commit = true;
261 }
262
263 int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
264 int flags = mmap_flags;
265
266 if (addr != NULL) {
267 flags |= MAP_FIXED | MAP_EXCL;
268 } else {
269 unsigned alignment_bits = ffs_zu(alignment);
270 assert(alignment_bits > 0);
271 flags |= MAP_ALIGNED(alignment_bits);
272 }
273
274 void *ret = mmap(addr, size, prot, flags, -1, 0);
275 if (ret == MAP_FAILED) {
276 ret = NULL;
277 }
278
279 return ret;
280 }
281#endif
282 /*
283 * Ideally, there would be a way to specify alignment to mmap() (like
284 * NetBSD has), but in the absence of such a feature, we have to work
285 * hard to efficiently create aligned mappings. The reliable, but
286 * slow method is to create a mapping that is over-sized, then trim the
287 * excess. However, that always results in one or two calls to
288 * os_pages_unmap(), and it can leave holes in the process's virtual
289 * memory map if memory grows downward.
290 *
291 * Optimistically try mapping precisely the right amount before falling
292 * back to the slow method, with the expectation that the optimistic
293 * approach works most of the time.
294 */
295
296 void *ret = os_pages_map(addr, size, os_page, commit);
297 if (ret == NULL || ret == addr) {
298 return ret;
299 }
300 assert(addr == NULL);
301 if (ALIGNMENT_ADDR2OFFSET(ret, alignment) != 0) {
302 os_pages_unmap(ret, size);
303 return pages_map_slow(size, alignment, commit);
304 }
305
306 assert(PAGE_ADDR2BASE(ret) == ret);
307 return ret;
308}
309
310void
311pages_unmap(void *addr, size_t size) {
312 assert(PAGE_ADDR2BASE(addr) == addr);
313 assert(PAGE_CEILING(size) == size);
314
315 os_pages_unmap(addr, size);
316}
317
318static bool
319os_pages_commit(void *addr, size_t size, bool commit) {
320 assert(PAGE_ADDR2BASE(addr) == addr);
321 assert(PAGE_CEILING(size) == size);
322
323#ifdef _WIN32
324 return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT,
325 PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT)));
326#else
327 {
328 int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
329 void *result = mmap(addr, size, prot, mmap_flags | MAP_FIXED,
330 PAGES_FD_TAG, 0);
331 if (result == MAP_FAILED) {
332 return true;
333 }
334 if (result != addr) {
335 /*
336 * We succeeded in mapping memory, but not in the right
337 * place.
338 */
339 os_pages_unmap(result, size);
340 return true;
341 }
342 return false;
343 }
344#endif
345}
346
347static bool
348pages_commit_impl(void *addr, size_t size, bool commit) {
349 if (os_overcommits) {
350 return true;
351 }
352
353 return os_pages_commit(addr, size, commit);
354}
355
356bool
357pages_commit(void *addr, size_t size) {
358 return pages_commit_impl(addr, size, true);
359}
360
361bool
362pages_decommit(void *addr, size_t size) {
363 return pages_commit_impl(addr, size, false);
364}
365
366void
367pages_mark_guards(void *head, void *tail) {
368 assert(head != NULL || tail != NULL);
369 assert(head == NULL || tail == NULL ||
370 (uintptr_t)head < (uintptr_t)tail);
371#ifdef JEMALLOC_HAVE_MPROTECT
372 if (head != NULL) {
373 mprotect(head, PAGE, PROT_NONE);
374 }
375 if (tail != NULL) {
376 mprotect(tail, PAGE, PROT_NONE);
377 }
378#else
379 /* Decommit sets to PROT_NONE / MEM_DECOMMIT. */
380 if (head != NULL) {
381 os_pages_commit(head, PAGE, false);
382 }
383 if (tail != NULL) {
384 os_pages_commit(tail, PAGE, false);
385 }
386#endif
387}
388
389void
390pages_unmark_guards(void *head, void *tail) {
391 assert(head != NULL || tail != NULL);
392 assert(head == NULL || tail == NULL ||
393 (uintptr_t)head < (uintptr_t)tail);
394#ifdef JEMALLOC_HAVE_MPROTECT
395 bool head_and_tail = (head != NULL) && (tail != NULL);
396 size_t range = head_and_tail ?
397 (uintptr_t)tail - (uintptr_t)head + PAGE :
398 SIZE_T_MAX;
399 /*
400 * The amount of work that the kernel does in mprotect depends on the
401 * range argument. SC_LARGE_MINCLASS is an arbitrary threshold chosen
402 * to prevent kernel from doing too much work that would outweigh the
403 * savings of performing one less system call.
404 */
405 bool ranged_mprotect = head_and_tail && range <= SC_LARGE_MINCLASS;
406 if (ranged_mprotect) {
407 mprotect(head, range, PROT_READ | PROT_WRITE);
408 } else {
409 if (head != NULL) {
410 mprotect(head, PAGE, PROT_READ | PROT_WRITE);
411 }
412 if (tail != NULL) {
413 mprotect(tail, PAGE, PROT_READ | PROT_WRITE);
414 }
415 }
416#else
417 if (head != NULL) {
418 os_pages_commit(head, PAGE, true);
419 }
420 if (tail != NULL) {
421 os_pages_commit(tail, PAGE, true);
422 }
423#endif
424}
425
426bool
427pages_purge_lazy(void *addr, size_t size) {
428 assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr);
429 assert(PAGE_CEILING(size) == size);
430
431 if (!pages_can_purge_lazy) {
432 return true;
433 }
434 if (!pages_can_purge_lazy_runtime) {
435 /*
436 * Built with lazy purge enabled, but detected it was not
437 * supported on the current system.
438 */
439 return true;
440 }
441
442#ifdef _WIN32
443 VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
444 return false;
445#elif defined(JEMALLOC_PURGE_MADVISE_FREE)
446 return (madvise(addr, size,
447# ifdef MADV_FREE
448 MADV_FREE
449# else
450 JEMALLOC_MADV_FREE
451# endif
452 ) != 0);
453#elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED) && \
454 !defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS)
455 return (madvise(addr, size, MADV_DONTNEED) != 0);
456#elif defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED) && \
457 !defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS)
458 return (posix_madvise(addr, size, POSIX_MADV_DONTNEED) != 0);
459#else
460 not_reached();
461#endif
462}
463
464bool
465pages_purge_forced(void *addr, size_t size) {
466 assert(PAGE_ADDR2BASE(addr) == addr);
467 assert(PAGE_CEILING(size) == size);
468
469 if (!pages_can_purge_forced) {
470 return true;
471 }
472
473#if defined(JEMALLOC_PURGE_MADVISE_DONTNEED) && \
474 defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS)
475 return (unlikely(madvise_dont_need_zeros_is_faulty) ||
476 madvise(addr, size, MADV_DONTNEED) != 0);
477#elif defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED) && \
478 defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS)
479 return (unlikely(madvise_dont_need_zeros_is_faulty) ||
480 posix_madvise(addr, size, POSIX_MADV_DONTNEED) != 0);
481#elif defined(JEMALLOC_MAPS_COALESCE)
482 /* Try to overlay a new demand-zeroed mapping. */
483 return pages_commit(addr, size);
484#else
485 not_reached();
486#endif
487}
488
489static bool
490pages_huge_impl(void *addr, size_t size, bool aligned) {
491 if (aligned) {
492 assert(HUGEPAGE_ADDR2BASE(addr) == addr);
493 assert(HUGEPAGE_CEILING(size) == size);
494 }
495#if defined(JEMALLOC_HAVE_MADVISE_HUGE)
496 return (madvise(addr, size, MADV_HUGEPAGE) != 0);
497#elif defined(JEMALLOC_HAVE_MEMCNTL)
498 struct memcntl_mha m = {0};
499 m.mha_cmd = MHA_MAPSIZE_VA;
500 m.mha_pagesize = HUGEPAGE;
501 return (memcntl(addr, size, MC_HAT_ADVISE, (caddr_t)&m, 0, 0) == 0);
502#else
503 return true;
504#endif
505}
506
507bool
508pages_huge(void *addr, size_t size) {
509 return pages_huge_impl(addr, size, true);
510}
511
512static bool
513pages_huge_unaligned(void *addr, size_t size) {
514 return pages_huge_impl(addr, size, false);
515}
516
517static bool
518pages_nohuge_impl(void *addr, size_t size, bool aligned) {
519 if (aligned) {
520 assert(HUGEPAGE_ADDR2BASE(addr) == addr);
521 assert(HUGEPAGE_CEILING(size) == size);
522 }
523
524#ifdef JEMALLOC_HAVE_MADVISE_HUGE
525 return (madvise(addr, size, MADV_NOHUGEPAGE) != 0);
526#else
527 return false;
528#endif
529}
530
531bool
532pages_nohuge(void *addr, size_t size) {
533 return pages_nohuge_impl(addr, size, true);
534}
535
536static bool
537pages_nohuge_unaligned(void *addr, size_t size) {
538 return pages_nohuge_impl(addr, size, false);
539}
540
541bool
542pages_dontdump(void *addr, size_t size) {
543 assert(PAGE_ADDR2BASE(addr) == addr);
544 assert(PAGE_CEILING(size) == size);
545#if defined(JEMALLOC_MADVISE_DONTDUMP)
546 return madvise(addr, size, MADV_DONTDUMP) != 0;
547#elif defined(JEMALLOC_MADVISE_NOCORE)
548 return madvise(addr, size, MADV_NOCORE) != 0;
549#else
550 return false;
551#endif
552}
553
554bool
555pages_dodump(void *addr, size_t size) {
556 assert(PAGE_ADDR2BASE(addr) == addr);
557 assert(PAGE_CEILING(size) == size);
558#if defined(JEMALLOC_MADVISE_DONTDUMP)
559 return madvise(addr, size, MADV_DODUMP) != 0;
560#elif defined(JEMALLOC_MADVISE_NOCORE)
561 return madvise(addr, size, MADV_CORE) != 0;
562#else
563 return false;
564#endif
565}
566
567
568static size_t
569os_page_detect(void) {
570#ifdef _WIN32
571 SYSTEM_INFO si;
572 GetSystemInfo(&si);
573 return si.dwPageSize;
574#elif defined(__FreeBSD__)
575 /*
576 * This returns the value obtained from
577 * the auxv vector, avoiding a syscall.
578 */
579 return getpagesize();
580#else
581 long result = sysconf(_SC_PAGESIZE);
582 if (result == -1) {
583 return LG_PAGE;
584 }
585 return (size_t)result;
586#endif
587}
588
589#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
590static bool
591os_overcommits_sysctl(void) {
592 int vm_overcommit;
593 size_t sz;
594
595 sz = sizeof(vm_overcommit);
596#if defined(__FreeBSD__) && defined(VM_OVERCOMMIT)
597 int mib[2];
598
599 mib[0] = CTL_VM;
600 mib[1] = VM_OVERCOMMIT;
601 if (sysctl(mib, 2, &vm_overcommit, &sz, NULL, 0) != 0) {
602 return false; /* Error. */
603 }
604#else
605 if (sysctlbyname("vm.overcommit", &vm_overcommit, &sz, NULL, 0) != 0) {
606 return false; /* Error. */
607 }
608#endif
609
610 return ((vm_overcommit & 0x3) == 0);
611}
612#endif
613
614#ifdef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY
615/*
616 * Use syscall(2) rather than {open,read,close}(2) when possible to avoid
617 * reentry during bootstrapping if another library has interposed system call
618 * wrappers.
619 */
620static bool
621os_overcommits_proc(void) {
622 int fd;
623 char buf[1];
624
625#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open)
626 #if defined(O_CLOEXEC)
627 fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY |
628 O_CLOEXEC);
629 #else
630 fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY);
631 if (fd != -1) {
632 fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
633 }
634 #endif
635#elif defined(JEMALLOC_USE_SYSCALL) && defined(SYS_openat)
636 #if defined(O_CLOEXEC)
637 fd = (int)syscall(SYS_openat,
638 AT_FDCWD, "/proc/sys/vm/overcommit_memory", O_RDONLY | O_CLOEXEC);
639 #else
640 fd = (int)syscall(SYS_openat,
641 AT_FDCWD, "/proc/sys/vm/overcommit_memory", O_RDONLY);
642 if (fd != -1) {
643 fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
644 }
645 #endif
646#else
647 #if defined(O_CLOEXEC)
648 fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY | O_CLOEXEC);
649 #else
650 fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY);
651 if (fd != -1) {
652 fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
653 }
654 #endif
655#endif
656
657 if (fd == -1) {
658 return false; /* Error. */
659 }
660
661 ssize_t nread = malloc_read_fd(fd, &buf, sizeof(buf));
662#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close)
663 syscall(SYS_close, fd);
664#else
665 close(fd);
666#endif
667
668 if (nread < 1) {
669 return false; /* Error. */
670 }
671 /*
672 * /proc/sys/vm/overcommit_memory meanings:
673 * 0: Heuristic overcommit.
674 * 1: Always overcommit.
675 * 2: Never overcommit.
676 */
677 return (buf[0] == '0' || buf[0] == '1');
678}
679#endif
680
681void
682pages_set_thp_state (void *ptr, size_t size) {
683 if (opt_thp == thp_mode_default || opt_thp == init_system_thp_mode) {
684 return;
685 }
686 assert(opt_thp != thp_mode_not_supported &&
687 init_system_thp_mode != thp_mode_not_supported);
688
689 if (opt_thp == thp_mode_always
690 && init_system_thp_mode != thp_mode_never) {
691 assert(init_system_thp_mode == thp_mode_default);
692 pages_huge_unaligned(ptr, size);
693 } else if (opt_thp == thp_mode_never) {
694 assert(init_system_thp_mode == thp_mode_default ||
695 init_system_thp_mode == thp_mode_always);
696 pages_nohuge_unaligned(ptr, size);
697 }
698}
699
700static void
701init_thp_state(void) {
702 if (!have_madvise_huge && !have_memcntl) {
703 if (metadata_thp_enabled() && opt_abort) {
704 malloc_write("<jemalloc>: no MADV_HUGEPAGE support\n");
705 abort();
706 }
707 goto label_error;
708 }
709#if defined(JEMALLOC_HAVE_MADVISE_HUGE)
710 static const char sys_state_madvise[] = "always [madvise] never\n";
711 static const char sys_state_always[] = "[always] madvise never\n";
712 static const char sys_state_never[] = "always madvise [never]\n";
713 char buf[sizeof(sys_state_madvise)];
714
715#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open)
716 int fd = (int)syscall(SYS_open,
717 "/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
718#elif defined(JEMALLOC_USE_SYSCALL) && defined(SYS_openat)
719 int fd = (int)syscall(SYS_openat,
720 AT_FDCWD, "/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
721#else
722 int fd = open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
723#endif
724 if (fd == -1) {
725 goto label_error;
726 }
727
728 ssize_t nread = malloc_read_fd(fd, &buf, sizeof(buf));
729#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close)
730 syscall(SYS_close, fd);
731#else
732 close(fd);
733#endif
734
735 if (nread < 0) {
736 goto label_error;
737 }
738
739 if (strncmp(buf, sys_state_madvise, (size_t)nread) == 0) {
740 init_system_thp_mode = thp_mode_default;
741 } else if (strncmp(buf, sys_state_always, (size_t)nread) == 0) {
742 init_system_thp_mode = thp_mode_always;
743 } else if (strncmp(buf, sys_state_never, (size_t)nread) == 0) {
744 init_system_thp_mode = thp_mode_never;
745 } else {
746 goto label_error;
747 }
748 return;
749#elif defined(JEMALLOC_HAVE_MEMCNTL)
750 init_system_thp_mode = thp_mode_default;
751 return;
752#endif
753label_error:
754 opt_thp = init_system_thp_mode = thp_mode_not_supported;
755}
756
757bool
758pages_boot(void) {
759 os_page = os_page_detect();
760 if (os_page > PAGE) {
761 malloc_write("<jemalloc>: Unsupported system page size\n");
762 if (opt_abort) {
763 abort();
764 }
765 return true;
766 }
767
768#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS
769 if (!opt_trust_madvise) {
770 madvise_dont_need_zeros_is_faulty = !madvise_MADV_DONTNEED_zeroes_pages();
771 if (madvise_dont_need_zeros_is_faulty) {
772 malloc_write("<jemalloc>: MADV_DONTNEED does not work (memset will be used instead)\n");
773 malloc_write("<jemalloc>: (This is the expected behaviour if you are running under QEMU)\n");
774 }
775 } else {
776 /* In case opt_trust_madvise is disable,
777 * do not do runtime check */
778 madvise_dont_need_zeros_is_faulty = 0;
779 }
780#endif
781
782#ifndef _WIN32
783 mmap_flags = MAP_PRIVATE | MAP_ANON;
784#endif
785
786#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
787 os_overcommits = os_overcommits_sysctl();
788#elif defined(JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY)
789 os_overcommits = os_overcommits_proc();
790# ifdef MAP_NORESERVE
791 if (os_overcommits) {
792 mmap_flags |= MAP_NORESERVE;
793 }
794# endif
795#elif defined(__NetBSD__)
796 os_overcommits = true;
797#else
798 os_overcommits = false;
799#endif
800
801 init_thp_state();
802
803#ifdef __FreeBSD__
804 /*
805 * FreeBSD doesn't need the check; madvise(2) is known to work.
806 */
807#else
808 /* Detect lazy purge runtime support. */
809 if (pages_can_purge_lazy) {
810 bool committed = false;
811 void *madv_free_page = os_pages_map(NULL, PAGE, PAGE, &committed);
812 if (madv_free_page == NULL) {
813 return true;
814 }
815 assert(pages_can_purge_lazy_runtime);
816 if (pages_purge_lazy(madv_free_page, PAGE)) {
817 pages_can_purge_lazy_runtime = false;
818 }
819 os_pages_unmap(madv_free_page, PAGE);
820 }
821#endif
822
823 return false;
824}
diff --git a/examples/redis-unstable/deps/jemalloc/src/pai.c b/examples/redis-unstable/deps/jemalloc/src/pai.c
deleted file mode 100644
index 45c8772..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/pai.c
+++ /dev/null
@@ -1,31 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4size_t
5pai_alloc_batch_default(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
6 edata_list_active_t *results, bool *deferred_work_generated) {
7 for (size_t i = 0; i < nallocs; i++) {
8 bool deferred_by_alloc = false;
9 edata_t *edata = pai_alloc(tsdn, self, size, PAGE,
10 /* zero */ false, /* guarded */ false,
11 /* frequent_reuse */ false, &deferred_by_alloc);
12 *deferred_work_generated |= deferred_by_alloc;
13 if (edata == NULL) {
14 return i;
15 }
16 edata_list_active_append(results, edata);
17 }
18 return nallocs;
19}
20
21void
22pai_dalloc_batch_default(tsdn_t *tsdn, pai_t *self,
23 edata_list_active_t *list, bool *deferred_work_generated) {
24 edata_t *edata;
25 while ((edata = edata_list_active_first(list)) != NULL) {
26 bool deferred_by_dalloc = false;
27 edata_list_active_remove(list, edata);
28 pai_dalloc(tsdn, self, edata, &deferred_by_dalloc);
29 *deferred_work_generated |= deferred_by_dalloc;
30 }
31}
diff --git a/examples/redis-unstable/deps/jemalloc/src/peak_event.c b/examples/redis-unstable/deps/jemalloc/src/peak_event.c
deleted file mode 100644
index 4093fbc..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/peak_event.c
+++ /dev/null
@@ -1,82 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/peak_event.h"
5
6#include "jemalloc/internal/activity_callback.h"
7#include "jemalloc/internal/peak.h"
8
9/*
10 * Update every 64K by default. We're not exposing this as a configuration
11 * option for now; we don't want to bind ourselves too tightly to any particular
12 * performance requirements for small values, or guarantee that we'll even be
13 * able to provide fine-grained accuracy.
14 */
15#define PEAK_EVENT_WAIT (64 * 1024)
16
17/* Update the peak with current tsd state. */
18void
19peak_event_update(tsd_t *tsd) {
20 uint64_t alloc = tsd_thread_allocated_get(tsd);
21 uint64_t dalloc = tsd_thread_deallocated_get(tsd);
22 peak_t *peak = tsd_peakp_get(tsd);
23 peak_update(peak, alloc, dalloc);
24}
25
26static void
27peak_event_activity_callback(tsd_t *tsd) {
28 activity_callback_thunk_t *thunk = tsd_activity_callback_thunkp_get(
29 tsd);
30 uint64_t alloc = tsd_thread_allocated_get(tsd);
31 uint64_t dalloc = tsd_thread_deallocated_get(tsd);
32 if (thunk->callback != NULL) {
33 thunk->callback(thunk->uctx, alloc, dalloc);
34 }
35}
36
37/* Set current state to zero. */
38void
39peak_event_zero(tsd_t *tsd) {
40 uint64_t alloc = tsd_thread_allocated_get(tsd);
41 uint64_t dalloc = tsd_thread_deallocated_get(tsd);
42 peak_t *peak = tsd_peakp_get(tsd);
43 peak_set_zero(peak, alloc, dalloc);
44}
45
46uint64_t
47peak_event_max(tsd_t *tsd) {
48 peak_t *peak = tsd_peakp_get(tsd);
49 return peak_max(peak);
50}
51
52uint64_t
53peak_alloc_new_event_wait(tsd_t *tsd) {
54 return PEAK_EVENT_WAIT;
55}
56
57uint64_t
58peak_alloc_postponed_event_wait(tsd_t *tsd) {
59 return TE_MIN_START_WAIT;
60}
61
62void
63peak_alloc_event_handler(tsd_t *tsd, uint64_t elapsed) {
64 peak_event_update(tsd);
65 peak_event_activity_callback(tsd);
66}
67
68uint64_t
69peak_dalloc_new_event_wait(tsd_t *tsd) {
70 return PEAK_EVENT_WAIT;
71}
72
73uint64_t
74peak_dalloc_postponed_event_wait(tsd_t *tsd) {
75 return TE_MIN_START_WAIT;
76}
77
78void
79peak_dalloc_event_handler(tsd_t *tsd, uint64_t elapsed) {
80 peak_event_update(tsd);
81 peak_event_activity_callback(tsd);
82}
diff --git a/examples/redis-unstable/deps/jemalloc/src/prof.c b/examples/redis-unstable/deps/jemalloc/src/prof.c
deleted file mode 100644
index 7a6d5d5..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/prof.c
+++ /dev/null
@@ -1,789 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/ctl.h"
5#include "jemalloc/internal/assert.h"
6#include "jemalloc/internal/mutex.h"
7#include "jemalloc/internal/counter.h"
8#include "jemalloc/internal/prof_data.h"
9#include "jemalloc/internal/prof_log.h"
10#include "jemalloc/internal/prof_recent.h"
11#include "jemalloc/internal/prof_stats.h"
12#include "jemalloc/internal/prof_sys.h"
13#include "jemalloc/internal/prof_hook.h"
14#include "jemalloc/internal/thread_event.h"
15
16/*
17 * This file implements the profiling "APIs" needed by other parts of jemalloc,
18 * and also manages the relevant "operational" data, mainly options and mutexes;
19 * the core profiling data structures are encapsulated in prof_data.c.
20 */
21
22/******************************************************************************/
23
24/* Data. */
25
26bool opt_prof = false;
27bool opt_prof_active = true;
28bool opt_prof_thread_active_init = true;
29size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT;
30ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT;
31bool opt_prof_gdump = false;
32bool opt_prof_final = false;
33bool opt_prof_leak = false;
34bool opt_prof_leak_error = false;
35bool opt_prof_accum = false;
36char opt_prof_prefix[PROF_DUMP_FILENAME_LEN];
37bool opt_prof_sys_thread_name = false;
38bool opt_prof_unbias = true;
39
40/* Accessed via prof_sample_event_handler(). */
41static counter_accum_t prof_idump_accumulated;
42
43/*
44 * Initialized as opt_prof_active, and accessed via
45 * prof_active_[gs]et{_unlocked,}().
46 */
47bool prof_active_state;
48static malloc_mutex_t prof_active_mtx;
49
50/*
51 * Initialized as opt_prof_thread_active_init, and accessed via
52 * prof_thread_active_init_[gs]et().
53 */
54static bool prof_thread_active_init;
55static malloc_mutex_t prof_thread_active_init_mtx;
56
57/*
58 * Initialized as opt_prof_gdump, and accessed via
59 * prof_gdump_[gs]et{_unlocked,}().
60 */
61bool prof_gdump_val;
62static malloc_mutex_t prof_gdump_mtx;
63
64uint64_t prof_interval = 0;
65
66size_t lg_prof_sample;
67
68static uint64_t next_thr_uid;
69static malloc_mutex_t next_thr_uid_mtx;
70
71/* Do not dump any profiles until bootstrapping is complete. */
72bool prof_booted = false;
73
74/* Logically a prof_backtrace_hook_t. */
75atomic_p_t prof_backtrace_hook;
76
77/* Logically a prof_dump_hook_t. */
78atomic_p_t prof_dump_hook;
79
80/******************************************************************************/
81
82void
83prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx) {
84 cassert(config_prof);
85
86 if (tsd_reentrancy_level_get(tsd) > 0) {
87 assert((uintptr_t)tctx == (uintptr_t)1U);
88 return;
89 }
90
91 if ((uintptr_t)tctx > (uintptr_t)1U) {
92 malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);
93 tctx->prepared = false;
94 prof_tctx_try_destroy(tsd, tctx);
95 }
96}
97
98void
99prof_malloc_sample_object(tsd_t *tsd, const void *ptr, size_t size,
100 size_t usize, prof_tctx_t *tctx) {
101 cassert(config_prof);
102
103 if (opt_prof_sys_thread_name) {
104 prof_sys_thread_name_fetch(tsd);
105 }
106
107 edata_t *edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global,
108 ptr);
109 prof_info_set(tsd, edata, tctx, size);
110
111 szind_t szind = sz_size2index(usize);
112
113 malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);
114 /*
115 * We need to do these map lookups while holding the lock, to avoid the
116 * possibility of races with prof_reset calls, which update the map and
117 * then acquire the lock. This actually still leaves a data race on the
118 * contents of the unbias map, but we have not yet gone through and
119 * atomic-ified the prof module, and compilers are not yet causing us
120 * issues. The key thing is to make sure that, if we read garbage data,
121 * the prof_reset call is about to mark our tctx as expired before any
122 * dumping of our corrupted output is attempted.
123 */
124 size_t shifted_unbiased_cnt = prof_shifted_unbiased_cnt[szind];
125 size_t unbiased_bytes = prof_unbiased_sz[szind];
126 tctx->cnts.curobjs++;
127 tctx->cnts.curobjs_shifted_unbiased += shifted_unbiased_cnt;
128 tctx->cnts.curbytes += usize;
129 tctx->cnts.curbytes_unbiased += unbiased_bytes;
130 if (opt_prof_accum) {
131 tctx->cnts.accumobjs++;
132 tctx->cnts.accumobjs_shifted_unbiased += shifted_unbiased_cnt;
133 tctx->cnts.accumbytes += usize;
134 tctx->cnts.accumbytes_unbiased += unbiased_bytes;
135 }
136 bool record_recent = prof_recent_alloc_prepare(tsd, tctx);
137 tctx->prepared = false;
138 malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);
139 if (record_recent) {
140 assert(tctx == edata_prof_tctx_get(edata));
141 prof_recent_alloc(tsd, edata, size, usize);
142 }
143
144 if (opt_prof_stats) {
145 prof_stats_inc(tsd, szind, size);
146 }
147}
148
149void
150prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_info_t *prof_info) {
151 cassert(config_prof);
152
153 assert(prof_info != NULL);
154 prof_tctx_t *tctx = prof_info->alloc_tctx;
155 assert((uintptr_t)tctx > (uintptr_t)1U);
156
157 szind_t szind = sz_size2index(usize);
158 malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);
159
160 assert(tctx->cnts.curobjs > 0);
161 assert(tctx->cnts.curbytes >= usize);
162 /*
163 * It's not correct to do equivalent asserts for unbiased bytes, because
164 * of the potential for races with prof.reset calls. The map contents
165 * should really be atomic, but we have not atomic-ified the prof module
166 * yet.
167 */
168 tctx->cnts.curobjs--;
169 tctx->cnts.curobjs_shifted_unbiased -= prof_shifted_unbiased_cnt[szind];
170 tctx->cnts.curbytes -= usize;
171 tctx->cnts.curbytes_unbiased -= prof_unbiased_sz[szind];
172
173 prof_try_log(tsd, usize, prof_info);
174
175 prof_tctx_try_destroy(tsd, tctx);
176
177 if (opt_prof_stats) {
178 prof_stats_dec(tsd, szind, prof_info->alloc_size);
179 }
180}
181
182prof_tctx_t *
183prof_tctx_create(tsd_t *tsd) {
184 if (!tsd_nominal(tsd) || tsd_reentrancy_level_get(tsd) > 0) {
185 return NULL;
186 }
187
188 prof_tdata_t *tdata = prof_tdata_get(tsd, true);
189 if (tdata == NULL) {
190 return NULL;
191 }
192
193 prof_bt_t bt;
194 bt_init(&bt, tdata->vec);
195 prof_backtrace(tsd, &bt);
196 return prof_lookup(tsd, &bt);
197}
198
199/*
200 * The bodies of this function and prof_leakcheck() are compiled out unless heap
201 * profiling is enabled, so that it is possible to compile jemalloc with
202 * floating point support completely disabled. Avoiding floating point code is
203 * important on memory-constrained systems, but it also enables a workaround for
204 * versions of glibc that don't properly save/restore floating point registers
205 * during dynamic lazy symbol loading (which internally calls into whatever
206 * malloc implementation happens to be integrated into the application). Note
207 * that some compilers (e.g. gcc 4.8) may use floating point registers for fast
208 * memory moves, so jemalloc must be compiled with such optimizations disabled
209 * (e.g.
210 * -mno-sse) in order for the workaround to be complete.
211 */
212uint64_t
213prof_sample_new_event_wait(tsd_t *tsd) {
214#ifdef JEMALLOC_PROF
215 if (lg_prof_sample == 0) {
216 return TE_MIN_START_WAIT;
217 }
218
219 /*
220 * Compute sample interval as a geometrically distributed random
221 * variable with mean (2^lg_prof_sample).
222 *
223 * __ __
224 * | log(u) | 1
225 * bytes_until_sample = | -------- |, where p = ---------------
226 * | log(1-p) | lg_prof_sample
227 * 2
228 *
229 * For more information on the math, see:
230 *
231 * Non-Uniform Random Variate Generation
232 * Luc Devroye
233 * Springer-Verlag, New York, 1986
234 * pp 500
235 * (http://luc.devroye.org/rnbookindex.html)
236 *
237 * In the actual computation, there's a non-zero probability that our
238 * pseudo random number generator generates an exact 0, and to avoid
239 * log(0), we set u to 1.0 in case r is 0. Therefore u effectively is
240 * uniformly distributed in (0, 1] instead of [0, 1). Further, rather
241 * than taking the ceiling, we take the floor and then add 1, since
242 * otherwise bytes_until_sample would be 0 if u is exactly 1.0.
243 */
244 uint64_t r = prng_lg_range_u64(tsd_prng_statep_get(tsd), 53);
245 double u = (r == 0U) ? 1.0 : (double)r * (1.0/9007199254740992.0L);
246 return (uint64_t)(log(u) /
247 log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample))))
248 + (uint64_t)1U;
249#else
250 not_reached();
251 return TE_MAX_START_WAIT;
252#endif
253}
254
255uint64_t
256prof_sample_postponed_event_wait(tsd_t *tsd) {
257 /*
258 * The postponed wait time for prof sample event is computed as if we
259 * want a new wait time (i.e. as if the event were triggered). If we
260 * instead postpone to the immediate next allocation, like how we're
261 * handling the other events, then we can have sampling bias, if e.g.
262 * the allocation immediately following a reentrancy always comes from
263 * the same stack trace.
264 */
265 return prof_sample_new_event_wait(tsd);
266}
267
268void
269prof_sample_event_handler(tsd_t *tsd, uint64_t elapsed) {
270 cassert(config_prof);
271 assert(elapsed > 0 && elapsed != TE_INVALID_ELAPSED);
272 if (prof_interval == 0 || !prof_active_get_unlocked()) {
273 return;
274 }
275 if (counter_accum(tsd_tsdn(tsd), &prof_idump_accumulated, elapsed)) {
276 prof_idump(tsd_tsdn(tsd));
277 }
278}
279
280static void
281prof_fdump(void) {
282 tsd_t *tsd;
283
284 cassert(config_prof);
285 assert(opt_prof_final);
286
287 if (!prof_booted) {
288 return;
289 }
290 tsd = tsd_fetch();
291 assert(tsd_reentrancy_level_get(tsd) == 0);
292
293 prof_fdump_impl(tsd);
294}
295
296static bool
297prof_idump_accum_init(void) {
298 cassert(config_prof);
299
300 return counter_accum_init(&prof_idump_accumulated, prof_interval);
301}
302
303void
304prof_idump(tsdn_t *tsdn) {
305 tsd_t *tsd;
306 prof_tdata_t *tdata;
307
308 cassert(config_prof);
309
310 if (!prof_booted || tsdn_null(tsdn) || !prof_active_get_unlocked()) {
311 return;
312 }
313 tsd = tsdn_tsd(tsdn);
314 if (tsd_reentrancy_level_get(tsd) > 0) {
315 return;
316 }
317
318 tdata = prof_tdata_get(tsd, true);
319 if (tdata == NULL) {
320 return;
321 }
322 if (tdata->enq) {
323 tdata->enq_idump = true;
324 return;
325 }
326
327 prof_idump_impl(tsd);
328}
329
330bool
331prof_mdump(tsd_t *tsd, const char *filename) {
332 cassert(config_prof);
333 assert(tsd_reentrancy_level_get(tsd) == 0);
334
335 if (!opt_prof || !prof_booted) {
336 return true;
337 }
338
339 return prof_mdump_impl(tsd, filename);
340}
341
342void
343prof_gdump(tsdn_t *tsdn) {
344 tsd_t *tsd;
345 prof_tdata_t *tdata;
346
347 cassert(config_prof);
348
349 if (!prof_booted || tsdn_null(tsdn) || !prof_active_get_unlocked()) {
350 return;
351 }
352 tsd = tsdn_tsd(tsdn);
353 if (tsd_reentrancy_level_get(tsd) > 0) {
354 return;
355 }
356
357 tdata = prof_tdata_get(tsd, false);
358 if (tdata == NULL) {
359 return;
360 }
361 if (tdata->enq) {
362 tdata->enq_gdump = true;
363 return;
364 }
365
366 prof_gdump_impl(tsd);
367}
368
369static uint64_t
370prof_thr_uid_alloc(tsdn_t *tsdn) {
371 uint64_t thr_uid;
372
373 malloc_mutex_lock(tsdn, &next_thr_uid_mtx);
374 thr_uid = next_thr_uid;
375 next_thr_uid++;
376 malloc_mutex_unlock(tsdn, &next_thr_uid_mtx);
377
378 return thr_uid;
379}
380
381prof_tdata_t *
382prof_tdata_init(tsd_t *tsd) {
383 return prof_tdata_init_impl(tsd, prof_thr_uid_alloc(tsd_tsdn(tsd)), 0,
384 NULL, prof_thread_active_init_get(tsd_tsdn(tsd)));
385}
386
387prof_tdata_t *
388prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) {
389 uint64_t thr_uid = tdata->thr_uid;
390 uint64_t thr_discrim = tdata->thr_discrim + 1;
391 char *thread_name = (tdata->thread_name != NULL) ?
392 prof_thread_name_alloc(tsd, tdata->thread_name) : NULL;
393 bool active = tdata->active;
394
395 prof_tdata_detach(tsd, tdata);
396 return prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name,
397 active);
398}
399
400void
401prof_tdata_cleanup(tsd_t *tsd) {
402 prof_tdata_t *tdata;
403
404 if (!config_prof) {
405 return;
406 }
407
408 tdata = tsd_prof_tdata_get(tsd);
409 if (tdata != NULL) {
410 prof_tdata_detach(tsd, tdata);
411 }
412}
413
414bool
415prof_active_get(tsdn_t *tsdn) {
416 bool prof_active_current;
417
418 prof_active_assert();
419 malloc_mutex_lock(tsdn, &prof_active_mtx);
420 prof_active_current = prof_active_state;
421 malloc_mutex_unlock(tsdn, &prof_active_mtx);
422 return prof_active_current;
423}
424
425bool
426prof_active_set(tsdn_t *tsdn, bool active) {
427 bool prof_active_old;
428
429 prof_active_assert();
430 malloc_mutex_lock(tsdn, &prof_active_mtx);
431 prof_active_old = prof_active_state;
432 prof_active_state = active;
433 malloc_mutex_unlock(tsdn, &prof_active_mtx);
434 prof_active_assert();
435 return prof_active_old;
436}
437
438const char *
439prof_thread_name_get(tsd_t *tsd) {
440 assert(tsd_reentrancy_level_get(tsd) == 0);
441
442 prof_tdata_t *tdata;
443
444 tdata = prof_tdata_get(tsd, true);
445 if (tdata == NULL) {
446 return "";
447 }
448 return (tdata->thread_name != NULL ? tdata->thread_name : "");
449}
450
451int
452prof_thread_name_set(tsd_t *tsd, const char *thread_name) {
453 if (opt_prof_sys_thread_name) {
454 return ENOENT;
455 } else {
456 return prof_thread_name_set_impl(tsd, thread_name);
457 }
458}
459
460bool
461prof_thread_active_get(tsd_t *tsd) {
462 assert(tsd_reentrancy_level_get(tsd) == 0);
463
464 prof_tdata_t *tdata;
465
466 tdata = prof_tdata_get(tsd, true);
467 if (tdata == NULL) {
468 return false;
469 }
470 return tdata->active;
471}
472
473bool
474prof_thread_active_set(tsd_t *tsd, bool active) {
475 assert(tsd_reentrancy_level_get(tsd) == 0);
476
477 prof_tdata_t *tdata;
478
479 tdata = prof_tdata_get(tsd, true);
480 if (tdata == NULL) {
481 return true;
482 }
483 tdata->active = active;
484 return false;
485}
486
487bool
488prof_thread_active_init_get(tsdn_t *tsdn) {
489 bool active_init;
490
491 malloc_mutex_lock(tsdn, &prof_thread_active_init_mtx);
492 active_init = prof_thread_active_init;
493 malloc_mutex_unlock(tsdn, &prof_thread_active_init_mtx);
494 return active_init;
495}
496
497bool
498prof_thread_active_init_set(tsdn_t *tsdn, bool active_init) {
499 bool active_init_old;
500
501 malloc_mutex_lock(tsdn, &prof_thread_active_init_mtx);
502 active_init_old = prof_thread_active_init;
503 prof_thread_active_init = active_init;
504 malloc_mutex_unlock(tsdn, &prof_thread_active_init_mtx);
505 return active_init_old;
506}
507
508bool
509prof_gdump_get(tsdn_t *tsdn) {
510 bool prof_gdump_current;
511
512 malloc_mutex_lock(tsdn, &prof_gdump_mtx);
513 prof_gdump_current = prof_gdump_val;
514 malloc_mutex_unlock(tsdn, &prof_gdump_mtx);
515 return prof_gdump_current;
516}
517
518bool
519prof_gdump_set(tsdn_t *tsdn, bool gdump) {
520 bool prof_gdump_old;
521
522 malloc_mutex_lock(tsdn, &prof_gdump_mtx);
523 prof_gdump_old = prof_gdump_val;
524 prof_gdump_val = gdump;
525 malloc_mutex_unlock(tsdn, &prof_gdump_mtx);
526 return prof_gdump_old;
527}
528
529void
530prof_backtrace_hook_set(prof_backtrace_hook_t hook) {
531 atomic_store_p(&prof_backtrace_hook, hook, ATOMIC_RELEASE);
532}
533
534prof_backtrace_hook_t
535prof_backtrace_hook_get() {
536 return (prof_backtrace_hook_t)atomic_load_p(&prof_backtrace_hook,
537 ATOMIC_ACQUIRE);
538}
539
540void
541prof_dump_hook_set(prof_dump_hook_t hook) {
542 atomic_store_p(&prof_dump_hook, hook, ATOMIC_RELEASE);
543}
544
545prof_dump_hook_t
546prof_dump_hook_get() {
547 return (prof_dump_hook_t)atomic_load_p(&prof_dump_hook,
548 ATOMIC_ACQUIRE);
549}
550
551void
552prof_boot0(void) {
553 cassert(config_prof);
554
555 memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT,
556 sizeof(PROF_PREFIX_DEFAULT));
557}
558
559void
560prof_boot1(void) {
561 cassert(config_prof);
562
563 /*
564 * opt_prof must be in its final state before any arenas are
565 * initialized, so this function must be executed early.
566 */
567 if (opt_prof_leak_error && !opt_prof_leak) {
568 opt_prof_leak = true;
569 }
570
571 if (opt_prof_leak && !opt_prof) {
572 /*
573 * Enable opt_prof, but in such a way that profiles are never
574 * automatically dumped.
575 */
576 opt_prof = true;
577 opt_prof_gdump = false;
578 } else if (opt_prof) {
579 if (opt_lg_prof_interval >= 0) {
580 prof_interval = (((uint64_t)1U) <<
581 opt_lg_prof_interval);
582 }
583 }
584}
585
586bool
587prof_boot2(tsd_t *tsd, base_t *base) {
588 cassert(config_prof);
589
590 /*
591 * Initialize the global mutexes unconditionally to maintain correct
592 * stats when opt_prof is false.
593 */
594 if (malloc_mutex_init(&prof_active_mtx, "prof_active",
595 WITNESS_RANK_PROF_ACTIVE, malloc_mutex_rank_exclusive)) {
596 return true;
597 }
598 if (malloc_mutex_init(&prof_gdump_mtx, "prof_gdump",
599 WITNESS_RANK_PROF_GDUMP, malloc_mutex_rank_exclusive)) {
600 return true;
601 }
602 if (malloc_mutex_init(&prof_thread_active_init_mtx,
603 "prof_thread_active_init", WITNESS_RANK_PROF_THREAD_ACTIVE_INIT,
604 malloc_mutex_rank_exclusive)) {
605 return true;
606 }
607 if (malloc_mutex_init(&bt2gctx_mtx, "prof_bt2gctx",
608 WITNESS_RANK_PROF_BT2GCTX, malloc_mutex_rank_exclusive)) {
609 return true;
610 }
611 if (malloc_mutex_init(&tdatas_mtx, "prof_tdatas",
612 WITNESS_RANK_PROF_TDATAS, malloc_mutex_rank_exclusive)) {
613 return true;
614 }
615 if (malloc_mutex_init(&next_thr_uid_mtx, "prof_next_thr_uid",
616 WITNESS_RANK_PROF_NEXT_THR_UID, malloc_mutex_rank_exclusive)) {
617 return true;
618 }
619 if (malloc_mutex_init(&prof_stats_mtx, "prof_stats",
620 WITNESS_RANK_PROF_STATS, malloc_mutex_rank_exclusive)) {
621 return true;
622 }
623 if (malloc_mutex_init(&prof_dump_filename_mtx,
624 "prof_dump_filename", WITNESS_RANK_PROF_DUMP_FILENAME,
625 malloc_mutex_rank_exclusive)) {
626 return true;
627 }
628 if (malloc_mutex_init(&prof_dump_mtx, "prof_dump",
629 WITNESS_RANK_PROF_DUMP, malloc_mutex_rank_exclusive)) {
630 return true;
631 }
632
633 if (opt_prof) {
634 lg_prof_sample = opt_lg_prof_sample;
635 prof_unbias_map_init();
636 prof_active_state = opt_prof_active;
637 prof_gdump_val = opt_prof_gdump;
638 prof_thread_active_init = opt_prof_thread_active_init;
639
640 if (prof_data_init(tsd)) {
641 return true;
642 }
643
644 next_thr_uid = 0;
645 if (prof_idump_accum_init()) {
646 return true;
647 }
648
649 if (opt_prof_final && opt_prof_prefix[0] != '\0' &&
650 atexit(prof_fdump) != 0) {
651 malloc_write("<jemalloc>: Error in atexit()\n");
652 if (opt_abort) {
653 abort();
654 }
655 }
656
657 if (prof_log_init(tsd)) {
658 return true;
659 }
660
661 if (prof_recent_init()) {
662 return true;
663 }
664
665 prof_base = base;
666
667 gctx_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd), base,
668 PROF_NCTX_LOCKS * sizeof(malloc_mutex_t), CACHELINE);
669 if (gctx_locks == NULL) {
670 return true;
671 }
672 for (unsigned i = 0; i < PROF_NCTX_LOCKS; i++) {
673 if (malloc_mutex_init(&gctx_locks[i], "prof_gctx",
674 WITNESS_RANK_PROF_GCTX,
675 malloc_mutex_rank_exclusive)) {
676 return true;
677 }
678 }
679
680 tdata_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd), base,
681 PROF_NTDATA_LOCKS * sizeof(malloc_mutex_t), CACHELINE);
682 if (tdata_locks == NULL) {
683 return true;
684 }
685 for (unsigned i = 0; i < PROF_NTDATA_LOCKS; i++) {
686 if (malloc_mutex_init(&tdata_locks[i], "prof_tdata",
687 WITNESS_RANK_PROF_TDATA,
688 malloc_mutex_rank_exclusive)) {
689 return true;
690 }
691 }
692
693 prof_unwind_init();
694 prof_hooks_init();
695 }
696 prof_booted = true;
697
698 return false;
699}
700
701void
702prof_prefork0(tsdn_t *tsdn) {
703 if (config_prof && opt_prof) {
704 unsigned i;
705
706 malloc_mutex_prefork(tsdn, &prof_dump_mtx);
707 malloc_mutex_prefork(tsdn, &bt2gctx_mtx);
708 malloc_mutex_prefork(tsdn, &tdatas_mtx);
709 for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
710 malloc_mutex_prefork(tsdn, &tdata_locks[i]);
711 }
712 malloc_mutex_prefork(tsdn, &log_mtx);
713 for (i = 0; i < PROF_NCTX_LOCKS; i++) {
714 malloc_mutex_prefork(tsdn, &gctx_locks[i]);
715 }
716 malloc_mutex_prefork(tsdn, &prof_recent_dump_mtx);
717 }
718}
719
720void
721prof_prefork1(tsdn_t *tsdn) {
722 if (config_prof && opt_prof) {
723 counter_prefork(tsdn, &prof_idump_accumulated);
724 malloc_mutex_prefork(tsdn, &prof_active_mtx);
725 malloc_mutex_prefork(tsdn, &prof_dump_filename_mtx);
726 malloc_mutex_prefork(tsdn, &prof_gdump_mtx);
727 malloc_mutex_prefork(tsdn, &prof_recent_alloc_mtx);
728 malloc_mutex_prefork(tsdn, &prof_stats_mtx);
729 malloc_mutex_prefork(tsdn, &next_thr_uid_mtx);
730 malloc_mutex_prefork(tsdn, &prof_thread_active_init_mtx);
731 }
732}
733
734void
735prof_postfork_parent(tsdn_t *tsdn) {
736 if (config_prof && opt_prof) {
737 unsigned i;
738
739 malloc_mutex_postfork_parent(tsdn,
740 &prof_thread_active_init_mtx);
741 malloc_mutex_postfork_parent(tsdn, &next_thr_uid_mtx);
742 malloc_mutex_postfork_parent(tsdn, &prof_stats_mtx);
743 malloc_mutex_postfork_parent(tsdn, &prof_recent_alloc_mtx);
744 malloc_mutex_postfork_parent(tsdn, &prof_gdump_mtx);
745 malloc_mutex_postfork_parent(tsdn, &prof_dump_filename_mtx);
746 malloc_mutex_postfork_parent(tsdn, &prof_active_mtx);
747 counter_postfork_parent(tsdn, &prof_idump_accumulated);
748 malloc_mutex_postfork_parent(tsdn, &prof_recent_dump_mtx);
749 for (i = 0; i < PROF_NCTX_LOCKS; i++) {
750 malloc_mutex_postfork_parent(tsdn, &gctx_locks[i]);
751 }
752 malloc_mutex_postfork_parent(tsdn, &log_mtx);
753 for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
754 malloc_mutex_postfork_parent(tsdn, &tdata_locks[i]);
755 }
756 malloc_mutex_postfork_parent(tsdn, &tdatas_mtx);
757 malloc_mutex_postfork_parent(tsdn, &bt2gctx_mtx);
758 malloc_mutex_postfork_parent(tsdn, &prof_dump_mtx);
759 }
760}
761
762void
763prof_postfork_child(tsdn_t *tsdn) {
764 if (config_prof && opt_prof) {
765 unsigned i;
766
767 malloc_mutex_postfork_child(tsdn, &prof_thread_active_init_mtx);
768 malloc_mutex_postfork_child(tsdn, &next_thr_uid_mtx);
769 malloc_mutex_postfork_child(tsdn, &prof_stats_mtx);
770 malloc_mutex_postfork_child(tsdn, &prof_recent_alloc_mtx);
771 malloc_mutex_postfork_child(tsdn, &prof_gdump_mtx);
772 malloc_mutex_postfork_child(tsdn, &prof_dump_filename_mtx);
773 malloc_mutex_postfork_child(tsdn, &prof_active_mtx);
774 counter_postfork_child(tsdn, &prof_idump_accumulated);
775 malloc_mutex_postfork_child(tsdn, &prof_recent_dump_mtx);
776 for (i = 0; i < PROF_NCTX_LOCKS; i++) {
777 malloc_mutex_postfork_child(tsdn, &gctx_locks[i]);
778 }
779 malloc_mutex_postfork_child(tsdn, &log_mtx);
780 for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
781 malloc_mutex_postfork_child(tsdn, &tdata_locks[i]);
782 }
783 malloc_mutex_postfork_child(tsdn, &tdatas_mtx);
784 malloc_mutex_postfork_child(tsdn, &bt2gctx_mtx);
785 malloc_mutex_postfork_child(tsdn, &prof_dump_mtx);
786 }
787}
788
789/******************************************************************************/
diff --git a/examples/redis-unstable/deps/jemalloc/src/prof_data.c b/examples/redis-unstable/deps/jemalloc/src/prof_data.c
deleted file mode 100644
index bfa55be..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/prof_data.c
+++ /dev/null
@@ -1,1447 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/ckh.h"
6#include "jemalloc/internal/hash.h"
7#include "jemalloc/internal/malloc_io.h"
8#include "jemalloc/internal/prof_data.h"
9
10/*
11 * This file defines and manages the core profiling data structures.
12 *
13 * Conceptually, profiling data can be imagined as a table with three columns:
14 * thread, stack trace, and current allocation size. (When prof_accum is on,
15 * there's one additional column which is the cumulative allocation size.)
16 *
17 * Implementation wise, each thread maintains a hash recording the stack trace
18 * to allocation size correspondences, which are basically the individual rows
19 * in the table. In addition, two global "indices" are built to make data
20 * aggregation efficient (for dumping): bt2gctx and tdatas, which are basically
21 * the "grouped by stack trace" and "grouped by thread" views of the same table,
22 * respectively. Note that the allocation size is only aggregated to the two
23 * indices at dumping time, so as to optimize for performance.
24 */
25
26/******************************************************************************/
27
28malloc_mutex_t bt2gctx_mtx;
29malloc_mutex_t tdatas_mtx;
30malloc_mutex_t prof_dump_mtx;
31
32/*
33 * Table of mutexes that are shared among gctx's. These are leaf locks, so
34 * there is no problem with using them for more than one gctx at the same time.
35 * The primary motivation for this sharing though is that gctx's are ephemeral,
36 * and destroying mutexes causes complications for systems that allocate when
37 * creating/destroying mutexes.
38 */
39malloc_mutex_t *gctx_locks;
40static atomic_u_t cum_gctxs; /* Atomic counter. */
41
42/*
43 * Table of mutexes that are shared among tdata's. No operations require
44 * holding multiple tdata locks, so there is no problem with using them for more
45 * than one tdata at the same time, even though a gctx lock may be acquired
46 * while holding a tdata lock.
47 */
48malloc_mutex_t *tdata_locks;
49
50/*
51 * Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data
52 * structure that knows about all backtraces currently captured.
53 */
54static ckh_t bt2gctx;
55
56/*
57 * Tree of all extant prof_tdata_t structures, regardless of state,
58 * {attached,detached,expired}.
59 */
60static prof_tdata_tree_t tdatas;
61
62size_t prof_unbiased_sz[PROF_SC_NSIZES];
63size_t prof_shifted_unbiased_cnt[PROF_SC_NSIZES];
64
65/******************************************************************************/
66/* Red-black trees. */
67
68static int
69prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) {
70 uint64_t a_thr_uid = a->thr_uid;
71 uint64_t b_thr_uid = b->thr_uid;
72 int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid);
73 if (ret == 0) {
74 uint64_t a_thr_discrim = a->thr_discrim;
75 uint64_t b_thr_discrim = b->thr_discrim;
76 ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim <
77 b_thr_discrim);
78 if (ret == 0) {
79 uint64_t a_tctx_uid = a->tctx_uid;
80 uint64_t b_tctx_uid = b->tctx_uid;
81 ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid <
82 b_tctx_uid);
83 }
84 }
85 return ret;
86}
87
88rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t,
89 tctx_link, prof_tctx_comp)
90
91static int
92prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) {
93 unsigned a_len = a->bt.len;
94 unsigned b_len = b->bt.len;
95 unsigned comp_len = (a_len < b_len) ? a_len : b_len;
96 int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *));
97 if (ret == 0) {
98 ret = (a_len > b_len) - (a_len < b_len);
99 }
100 return ret;
101}
102
103rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link,
104 prof_gctx_comp)
105
106static int
107prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) {
108 int ret;
109 uint64_t a_uid = a->thr_uid;
110 uint64_t b_uid = b->thr_uid;
111
112 ret = ((a_uid > b_uid) - (a_uid < b_uid));
113 if (ret == 0) {
114 uint64_t a_discrim = a->thr_discrim;
115 uint64_t b_discrim = b->thr_discrim;
116
117 ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim));
118 }
119 return ret;
120}
121
122rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link,
123 prof_tdata_comp)
124
125/******************************************************************************/
126
127static malloc_mutex_t *
128prof_gctx_mutex_choose(void) {
129 unsigned ngctxs = atomic_fetch_add_u(&cum_gctxs, 1, ATOMIC_RELAXED);
130
131 return &gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS];
132}
133
134static malloc_mutex_t *
135prof_tdata_mutex_choose(uint64_t thr_uid) {
136 return &tdata_locks[thr_uid % PROF_NTDATA_LOCKS];
137}
138
139bool
140prof_data_init(tsd_t *tsd) {
141 tdata_tree_new(&tdatas);
142 return ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS,
143 prof_bt_hash, prof_bt_keycomp);
144}
145
146static void
147prof_enter(tsd_t *tsd, prof_tdata_t *tdata) {
148 cassert(config_prof);
149 assert(tdata == prof_tdata_get(tsd, false));
150
151 if (tdata != NULL) {
152 assert(!tdata->enq);
153 tdata->enq = true;
154 }
155
156 malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);
157}
158
159static void
160prof_leave(tsd_t *tsd, prof_tdata_t *tdata) {
161 cassert(config_prof);
162 assert(tdata == prof_tdata_get(tsd, false));
163
164 malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);
165
166 if (tdata != NULL) {
167 bool idump, gdump;
168
169 assert(tdata->enq);
170 tdata->enq = false;
171 idump = tdata->enq_idump;
172 tdata->enq_idump = false;
173 gdump = tdata->enq_gdump;
174 tdata->enq_gdump = false;
175
176 if (idump) {
177 prof_idump(tsd_tsdn(tsd));
178 }
179 if (gdump) {
180 prof_gdump(tsd_tsdn(tsd));
181 }
182 }
183}
184
185static prof_gctx_t *
186prof_gctx_create(tsdn_t *tsdn, prof_bt_t *bt) {
187 /*
188 * Create a single allocation that has space for vec of length bt->len.
189 */
190 size_t size = offsetof(prof_gctx_t, vec) + (bt->len * sizeof(void *));
191 prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsdn, size,
192 sz_size2index(size), false, NULL, true, arena_get(TSDN_NULL, 0, true),
193 true);
194 if (gctx == NULL) {
195 return NULL;
196 }
197 gctx->lock = prof_gctx_mutex_choose();
198 /*
199 * Set nlimbo to 1, in order to avoid a race condition with
200 * prof_tctx_destroy()/prof_gctx_try_destroy().
201 */
202 gctx->nlimbo = 1;
203 tctx_tree_new(&gctx->tctxs);
204 /* Duplicate bt. */
205 memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *));
206 gctx->bt.vec = gctx->vec;
207 gctx->bt.len = bt->len;
208 return gctx;
209}
210
211static void
212prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self,
213 prof_gctx_t *gctx) {
214 cassert(config_prof);
215
216 /*
217 * Check that gctx is still unused by any thread cache before destroying
218 * it. prof_lookup() increments gctx->nlimbo in order to avoid a race
219 * condition with this function, as does prof_tctx_destroy() in order to
220 * avoid a race between the main body of prof_tctx_destroy() and entry
221 * into this function.
222 */
223 prof_enter(tsd, tdata_self);
224 malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
225 assert(gctx->nlimbo != 0);
226 if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) {
227 /* Remove gctx from bt2gctx. */
228 if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL)) {
229 not_reached();
230 }
231 prof_leave(tsd, tdata_self);
232 /* Destroy gctx. */
233 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
234 idalloctm(tsd_tsdn(tsd), gctx, NULL, NULL, true, true);
235 } else {
236 /*
237 * Compensate for increment in prof_tctx_destroy() or
238 * prof_lookup().
239 */
240 gctx->nlimbo--;
241 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
242 prof_leave(tsd, tdata_self);
243 }
244}
245
246static bool
247prof_gctx_should_destroy(prof_gctx_t *gctx) {
248 if (opt_prof_accum) {
249 return false;
250 }
251 if (!tctx_tree_empty(&gctx->tctxs)) {
252 return false;
253 }
254 if (gctx->nlimbo != 0) {
255 return false;
256 }
257 return true;
258}
259
260static bool
261prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
262 void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx) {
263 union {
264 prof_gctx_t *p;
265 void *v;
266 } gctx, tgctx;
267 union {
268 prof_bt_t *p;
269 void *v;
270 } btkey;
271 bool new_gctx;
272
273 prof_enter(tsd, tdata);
274 if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
275 /* bt has never been seen before. Insert it. */
276 prof_leave(tsd, tdata);
277 tgctx.p = prof_gctx_create(tsd_tsdn(tsd), bt);
278 if (tgctx.v == NULL) {
279 return true;
280 }
281 prof_enter(tsd, tdata);
282 if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
283 gctx.p = tgctx.p;
284 btkey.p = &gctx.p->bt;
285 if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) {
286 /* OOM. */
287 prof_leave(tsd, tdata);
288 idalloctm(tsd_tsdn(tsd), gctx.v, NULL, NULL,
289 true, true);
290 return true;
291 }
292 new_gctx = true;
293 } else {
294 new_gctx = false;
295 }
296 } else {
297 tgctx.v = NULL;
298 new_gctx = false;
299 }
300
301 if (!new_gctx) {
302 /*
303 * Increment nlimbo, in order to avoid a race condition with
304 * prof_tctx_destroy()/prof_gctx_try_destroy().
305 */
306 malloc_mutex_lock(tsd_tsdn(tsd), gctx.p->lock);
307 gctx.p->nlimbo++;
308 malloc_mutex_unlock(tsd_tsdn(tsd), gctx.p->lock);
309 new_gctx = false;
310
311 if (tgctx.v != NULL) {
312 /* Lost race to insert. */
313 idalloctm(tsd_tsdn(tsd), tgctx.v, NULL, NULL, true,
314 true);
315 }
316 }
317 prof_leave(tsd, tdata);
318
319 *p_btkey = btkey.v;
320 *p_gctx = gctx.p;
321 *p_new_gctx = new_gctx;
322 return false;
323}
324
325prof_tctx_t *
326prof_lookup(tsd_t *tsd, prof_bt_t *bt) {
327 union {
328 prof_tctx_t *p;
329 void *v;
330 } ret;
331 prof_tdata_t *tdata;
332 bool not_found;
333
334 cassert(config_prof);
335
336 tdata = prof_tdata_get(tsd, false);
337 assert(tdata != NULL);
338
339 malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
340 not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v);
341 if (!not_found) { /* Note double negative! */
342 ret.p->prepared = true;
343 }
344 malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
345 if (not_found) {
346 void *btkey;
347 prof_gctx_t *gctx;
348 bool new_gctx, error;
349
350 /*
351 * This thread's cache lacks bt. Look for it in the global
352 * cache.
353 */
354 if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,
355 &new_gctx)) {
356 return NULL;
357 }
358
359 /* Link a prof_tctx_t into gctx for this thread. */
360 ret.v = iallocztm(tsd_tsdn(tsd), sizeof(prof_tctx_t),
361 sz_size2index(sizeof(prof_tctx_t)), false, NULL, true,
362 arena_ichoose(tsd, NULL), true);
363 if (ret.p == NULL) {
364 if (new_gctx) {
365 prof_gctx_try_destroy(tsd, tdata, gctx);
366 }
367 return NULL;
368 }
369 ret.p->tdata = tdata;
370 ret.p->thr_uid = tdata->thr_uid;
371 ret.p->thr_discrim = tdata->thr_discrim;
372 ret.p->recent_count = 0;
373 memset(&ret.p->cnts, 0, sizeof(prof_cnt_t));
374 ret.p->gctx = gctx;
375 ret.p->tctx_uid = tdata->tctx_uid_next++;
376 ret.p->prepared = true;
377 ret.p->state = prof_tctx_state_initializing;
378 malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
379 error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);
380 malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
381 if (error) {
382 if (new_gctx) {
383 prof_gctx_try_destroy(tsd, tdata, gctx);
384 }
385 idalloctm(tsd_tsdn(tsd), ret.v, NULL, NULL, true, true);
386 return NULL;
387 }
388 malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
389 ret.p->state = prof_tctx_state_nominal;
390 tctx_tree_insert(&gctx->tctxs, ret.p);
391 gctx->nlimbo--;
392 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
393 }
394
395 return ret.p;
396}
397
398/* Used in unit tests. */
399static prof_tdata_t *
400prof_tdata_count_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata,
401 void *arg) {
402 size_t *tdata_count = (size_t *)arg;
403
404 (*tdata_count)++;
405
406 return NULL;
407}
408
409/* Used in unit tests. */
410size_t
411prof_tdata_count(void) {
412 size_t tdata_count = 0;
413 tsdn_t *tsdn;
414
415 tsdn = tsdn_fetch();
416 malloc_mutex_lock(tsdn, &tdatas_mtx);
417 tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter,
418 (void *)&tdata_count);
419 malloc_mutex_unlock(tsdn, &tdatas_mtx);
420
421 return tdata_count;
422}
423
424/* Used in unit tests. */
425size_t
426prof_bt_count(void) {
427 size_t bt_count;
428 tsd_t *tsd;
429 prof_tdata_t *tdata;
430
431 tsd = tsd_fetch();
432 tdata = prof_tdata_get(tsd, false);
433 if (tdata == NULL) {
434 return 0;
435 }
436
437 malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);
438 bt_count = ckh_count(&bt2gctx);
439 malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);
440
441 return bt_count;
442}
443
444char *
445prof_thread_name_alloc(tsd_t *tsd, const char *thread_name) {
446 char *ret;
447 size_t size;
448
449 if (thread_name == NULL) {
450 return NULL;
451 }
452
453 size = strlen(thread_name) + 1;
454 if (size == 1) {
455 return "";
456 }
457
458 ret = iallocztm(tsd_tsdn(tsd), size, sz_size2index(size), false, NULL,
459 true, arena_get(TSDN_NULL, 0, true), true);
460 if (ret == NULL) {
461 return NULL;
462 }
463 memcpy(ret, thread_name, size);
464 return ret;
465}
466
467int
468prof_thread_name_set_impl(tsd_t *tsd, const char *thread_name) {
469 assert(tsd_reentrancy_level_get(tsd) == 0);
470
471 prof_tdata_t *tdata;
472 unsigned i;
473 char *s;
474
475 tdata = prof_tdata_get(tsd, true);
476 if (tdata == NULL) {
477 return EAGAIN;
478 }
479
480 /* Validate input. */
481 if (thread_name == NULL) {
482 return EFAULT;
483 }
484 for (i = 0; thread_name[i] != '\0'; i++) {
485 char c = thread_name[i];
486 if (!isgraph(c) && !isblank(c)) {
487 return EFAULT;
488 }
489 }
490
491 s = prof_thread_name_alloc(tsd, thread_name);
492 if (s == NULL) {
493 return EAGAIN;
494 }
495
496 if (tdata->thread_name != NULL) {
497 idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,
498 true);
499 tdata->thread_name = NULL;
500 }
501 if (strlen(s) > 0) {
502 tdata->thread_name = s;
503 }
504 return 0;
505}
506
507JEMALLOC_FORMAT_PRINTF(3, 4)
508static void
509prof_dump_printf(write_cb_t *prof_dump_write, void *cbopaque,
510 const char *format, ...) {
511 va_list ap;
512 char buf[PROF_PRINTF_BUFSIZE];
513
514 va_start(ap, format);
515 malloc_vsnprintf(buf, sizeof(buf), format, ap);
516 va_end(ap);
517 prof_dump_write(cbopaque, buf);
518}
519
520/*
521 * Casting a double to a uint64_t may not necessarily be in range; this can be
522 * UB. I don't think this is practically possible with the cur counters, but
523 * plausibly could be with the accum counters.
524 */
525#ifdef JEMALLOC_PROF
526static uint64_t
527prof_double_uint64_cast(double d) {
528 /*
529 * Note: UINT64_MAX + 1 is exactly representable as a double on all
530 * reasonable platforms (certainly those we'll support). Writing this
531 * as !(a < b) instead of (a >= b) means that we're NaN-safe.
532 */
533 double rounded = round(d);
534 if (!(rounded < (double)UINT64_MAX)) {
535 return UINT64_MAX;
536 }
537 return (uint64_t)rounded;
538}
539#endif
540
541void prof_unbias_map_init() {
542 /* See the comment in prof_sample_new_event_wait */
543#ifdef JEMALLOC_PROF
544 for (szind_t i = 0; i < SC_NSIZES; i++) {
545 double sz = (double)sz_index2size(i);
546 double rate = (double)(ZU(1) << lg_prof_sample);
547 double div_val = 1.0 - exp(-sz / rate);
548 double unbiased_sz = sz / div_val;
549 /*
550 * The "true" right value for the unbiased count is
551 * 1.0/(1 - exp(-sz/rate)). The problem is, we keep the counts
552 * as integers (for a variety of reasons -- rounding errors
553 * could trigger asserts, and not all libcs can properly handle
554 * floating point arithmetic during malloc calls inside libc).
555 * Rounding to an integer, though, can lead to rounding errors
556 * of over 30% for sizes close to the sampling rate. So
557 * instead, we multiply by a constant, dividing the maximum
558 * possible roundoff error by that constant. To avoid overflow
559 * in summing up size_t values, the largest safe constant we can
560 * pick is the size of the smallest allocation.
561 */
562 double cnt_shift = (double)(ZU(1) << SC_LG_TINY_MIN);
563 double shifted_unbiased_cnt = cnt_shift / div_val;
564 prof_unbiased_sz[i] = (size_t)round(unbiased_sz);
565 prof_shifted_unbiased_cnt[i] = (size_t)round(
566 shifted_unbiased_cnt);
567 }
568#else
569 unreachable();
570#endif
571}
572
573/*
574 * The unbiasing story is long. The jeprof unbiasing logic was copied from
575 * pprof. Both shared an issue: they unbiased using the average size of the
576 * allocations at a particular stack trace. This can work out OK if allocations
577 * are mostly of the same size given some stack, but not otherwise. We now
578 * internally track what the unbiased results ought to be. We can't just report
579 * them as they are though; they'll still go through the jeprof unbiasing
580 * process. Instead, we figure out what values we can feed *into* jeprof's
581 * unbiasing mechanism that will lead to getting the right values out.
582 *
583 * It'll unbias count and aggregate size as:
584 *
585 * c_out = c_in * 1/(1-exp(-s_in/c_in/R)
586 * s_out = s_in * 1/(1-exp(-s_in/c_in/R)
587 *
588 * We want to solve for the values of c_in and s_in that will
589 * give the c_out and s_out that we've computed internally.
590 *
591 * Let's do a change of variables (both to make the math easier and to make it
592 * easier to write):
593 * x = s_in / c_in
594 * y = s_in
595 * k = 1/R.
596 *
597 * Then
598 * c_out = y/x * 1/(1-exp(-k*x))
599 * s_out = y * 1/(1-exp(-k*x))
600 *
601 * The first equation gives:
602 * y = x * c_out * (1-exp(-k*x))
603 * The second gives:
604 * y = s_out * (1-exp(-k*x))
605 * So we have
606 * x = s_out / c_out.
607 * And all the other values fall out from that.
608 *
609 * This is all a fair bit of work. The thing we get out of it is that we don't
610 * break backwards compatibility with jeprof (and the various tools that have
611 * copied its unbiasing logic). Eventually, we anticipate a v3 heap profile
612 * dump format based on JSON, at which point I think much of this logic can get
613 * cleaned up (since we'll be taking a compatibility break there anyways).
614 */
615static void
616prof_do_unbias(uint64_t c_out_shifted_i, uint64_t s_out_i, uint64_t *r_c_in,
617 uint64_t *r_s_in) {
618#ifdef JEMALLOC_PROF
619 if (c_out_shifted_i == 0 || s_out_i == 0) {
620 *r_c_in = 0;
621 *r_s_in = 0;
622 return;
623 }
624 /*
625 * See the note in prof_unbias_map_init() to see why we take c_out in a
626 * shifted form.
627 */
628 double c_out = (double)c_out_shifted_i
629 / (double)(ZU(1) << SC_LG_TINY_MIN);
630 double s_out = (double)s_out_i;
631 double R = (double)(ZU(1) << lg_prof_sample);
632
633 double x = s_out / c_out;
634 double y = s_out * (1.0 - exp(-x / R));
635
636 double c_in = y / x;
637 double s_in = y;
638
639 *r_c_in = prof_double_uint64_cast(c_in);
640 *r_s_in = prof_double_uint64_cast(s_in);
641#else
642 unreachable();
643#endif
644}
645
646static void
647prof_dump_print_cnts(write_cb_t *prof_dump_write, void *cbopaque,
648 const prof_cnt_t *cnts) {
649 uint64_t curobjs;
650 uint64_t curbytes;
651 uint64_t accumobjs;
652 uint64_t accumbytes;
653 if (opt_prof_unbias) {
654 prof_do_unbias(cnts->curobjs_shifted_unbiased,
655 cnts->curbytes_unbiased, &curobjs, &curbytes);
656 prof_do_unbias(cnts->accumobjs_shifted_unbiased,
657 cnts->accumbytes_unbiased, &accumobjs, &accumbytes);
658 } else {
659 curobjs = cnts->curobjs;
660 curbytes = cnts->curbytes;
661 accumobjs = cnts->accumobjs;
662 accumbytes = cnts->accumbytes;
663 }
664 prof_dump_printf(prof_dump_write, cbopaque,
665 "%"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]",
666 curobjs, curbytes, accumobjs, accumbytes);
667}
668
669static void
670prof_tctx_merge_tdata(tsdn_t *tsdn, prof_tctx_t *tctx, prof_tdata_t *tdata) {
671 malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);
672
673 malloc_mutex_lock(tsdn, tctx->gctx->lock);
674
675 switch (tctx->state) {
676 case prof_tctx_state_initializing:
677 malloc_mutex_unlock(tsdn, tctx->gctx->lock);
678 return;
679 case prof_tctx_state_nominal:
680 tctx->state = prof_tctx_state_dumping;
681 malloc_mutex_unlock(tsdn, tctx->gctx->lock);
682
683 memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t));
684
685 tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
686 tdata->cnt_summed.curobjs_shifted_unbiased
687 += tctx->dump_cnts.curobjs_shifted_unbiased;
688 tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
689 tdata->cnt_summed.curbytes_unbiased
690 += tctx->dump_cnts.curbytes_unbiased;
691 if (opt_prof_accum) {
692 tdata->cnt_summed.accumobjs +=
693 tctx->dump_cnts.accumobjs;
694 tdata->cnt_summed.accumobjs_shifted_unbiased +=
695 tctx->dump_cnts.accumobjs_shifted_unbiased;
696 tdata->cnt_summed.accumbytes +=
697 tctx->dump_cnts.accumbytes;
698 tdata->cnt_summed.accumbytes_unbiased +=
699 tctx->dump_cnts.accumbytes_unbiased;
700 }
701 break;
702 case prof_tctx_state_dumping:
703 case prof_tctx_state_purgatory:
704 not_reached();
705 }
706}
707
708static void
709prof_tctx_merge_gctx(tsdn_t *tsdn, prof_tctx_t *tctx, prof_gctx_t *gctx) {
710 malloc_mutex_assert_owner(tsdn, gctx->lock);
711
712 gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
713 gctx->cnt_summed.curobjs_shifted_unbiased
714 += tctx->dump_cnts.curobjs_shifted_unbiased;
715 gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
716 gctx->cnt_summed.curbytes_unbiased += tctx->dump_cnts.curbytes_unbiased;
717 if (opt_prof_accum) {
718 gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs;
719 gctx->cnt_summed.accumobjs_shifted_unbiased
720 += tctx->dump_cnts.accumobjs_shifted_unbiased;
721 gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes;
722 gctx->cnt_summed.accumbytes_unbiased
723 += tctx->dump_cnts.accumbytes_unbiased;
724 }
725}
726
727static prof_tctx_t *
728prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) {
729 tsdn_t *tsdn = (tsdn_t *)arg;
730
731 malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);
732
733 switch (tctx->state) {
734 case prof_tctx_state_nominal:
735 /* New since dumping started; ignore. */
736 break;
737 case prof_tctx_state_dumping:
738 case prof_tctx_state_purgatory:
739 prof_tctx_merge_gctx(tsdn, tctx, tctx->gctx);
740 break;
741 default:
742 not_reached();
743 }
744
745 return NULL;
746}
747
748typedef struct prof_dump_iter_arg_s prof_dump_iter_arg_t;
749struct prof_dump_iter_arg_s {
750 tsdn_t *tsdn;
751 write_cb_t *prof_dump_write;
752 void *cbopaque;
753};
754
755static prof_tctx_t *
756prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *opaque) {
757 prof_dump_iter_arg_t *arg = (prof_dump_iter_arg_t *)opaque;
758 malloc_mutex_assert_owner(arg->tsdn, tctx->gctx->lock);
759
760 switch (tctx->state) {
761 case prof_tctx_state_initializing:
762 case prof_tctx_state_nominal:
763 /* Not captured by this dump. */
764 break;
765 case prof_tctx_state_dumping:
766 case prof_tctx_state_purgatory:
767 prof_dump_printf(arg->prof_dump_write, arg->cbopaque,
768 " t%"FMTu64": ", tctx->thr_uid);
769 prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque,
770 &tctx->dump_cnts);
771 arg->prof_dump_write(arg->cbopaque, "\n");
772 break;
773 default:
774 not_reached();
775 }
776 return NULL;
777}
778
779static prof_tctx_t *
780prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) {
781 tsdn_t *tsdn = (tsdn_t *)arg;
782 prof_tctx_t *ret;
783
784 malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);
785
786 switch (tctx->state) {
787 case prof_tctx_state_nominal:
788 /* New since dumping started; ignore. */
789 break;
790 case prof_tctx_state_dumping:
791 tctx->state = prof_tctx_state_nominal;
792 break;
793 case prof_tctx_state_purgatory:
794 ret = tctx;
795 goto label_return;
796 default:
797 not_reached();
798 }
799
800 ret = NULL;
801label_return:
802 return ret;
803}
804
805static void
806prof_dump_gctx_prep(tsdn_t *tsdn, prof_gctx_t *gctx, prof_gctx_tree_t *gctxs) {
807 cassert(config_prof);
808
809 malloc_mutex_lock(tsdn, gctx->lock);
810
811 /*
812 * Increment nlimbo so that gctx won't go away before dump.
813 * Additionally, link gctx into the dump list so that it is included in
814 * prof_dump()'s second pass.
815 */
816 gctx->nlimbo++;
817 gctx_tree_insert(gctxs, gctx);
818
819 memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t));
820
821 malloc_mutex_unlock(tsdn, gctx->lock);
822}
823
824typedef struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg_t;
825struct prof_gctx_merge_iter_arg_s {
826 tsdn_t *tsdn;
827 size_t *leak_ngctx;
828};
829
830static prof_gctx_t *
831prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) {
832 prof_gctx_merge_iter_arg_t *arg = (prof_gctx_merge_iter_arg_t *)opaque;
833
834 malloc_mutex_lock(arg->tsdn, gctx->lock);
835 tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter,
836 (void *)arg->tsdn);
837 if (gctx->cnt_summed.curobjs != 0) {
838 (*arg->leak_ngctx)++;
839 }
840 malloc_mutex_unlock(arg->tsdn, gctx->lock);
841
842 return NULL;
843}
844
845static void
846prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) {
847 prof_tdata_t *tdata = prof_tdata_get(tsd, false);
848 prof_gctx_t *gctx;
849
850 /*
851 * Standard tree iteration won't work here, because as soon as we
852 * decrement gctx->nlimbo and unlock gctx, another thread can
853 * concurrently destroy it, which will corrupt the tree. Therefore,
854 * tear down the tree one node at a time during iteration.
855 */
856 while ((gctx = gctx_tree_first(gctxs)) != NULL) {
857 gctx_tree_remove(gctxs, gctx);
858 malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
859 {
860 prof_tctx_t *next;
861
862 next = NULL;
863 do {
864 prof_tctx_t *to_destroy =
865 tctx_tree_iter(&gctx->tctxs, next,
866 prof_tctx_finish_iter,
867 (void *)tsd_tsdn(tsd));
868 if (to_destroy != NULL) {
869 next = tctx_tree_next(&gctx->tctxs,
870 to_destroy);
871 tctx_tree_remove(&gctx->tctxs,
872 to_destroy);
873 idalloctm(tsd_tsdn(tsd), to_destroy,
874 NULL, NULL, true, true);
875 } else {
876 next = NULL;
877 }
878 } while (next != NULL);
879 }
880 gctx->nlimbo--;
881 if (prof_gctx_should_destroy(gctx)) {
882 gctx->nlimbo++;
883 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
884 prof_gctx_try_destroy(tsd, tdata, gctx);
885 } else {
886 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
887 }
888 }
889}
890
891typedef struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg_t;
892struct prof_tdata_merge_iter_arg_s {
893 tsdn_t *tsdn;
894 prof_cnt_t *cnt_all;
895};
896
897static prof_tdata_t *
898prof_tdata_merge_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata,
899 void *opaque) {
900 prof_tdata_merge_iter_arg_t *arg =
901 (prof_tdata_merge_iter_arg_t *)opaque;
902
903 malloc_mutex_lock(arg->tsdn, tdata->lock);
904 if (!tdata->expired) {
905 size_t tabind;
906 union {
907 prof_tctx_t *p;
908 void *v;
909 } tctx;
910
911 tdata->dumping = true;
912 memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t));
913 for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL,
914 &tctx.v);) {
915 prof_tctx_merge_tdata(arg->tsdn, tctx.p, tdata);
916 }
917
918 arg->cnt_all->curobjs += tdata->cnt_summed.curobjs;
919 arg->cnt_all->curobjs_shifted_unbiased
920 += tdata->cnt_summed.curobjs_shifted_unbiased;
921 arg->cnt_all->curbytes += tdata->cnt_summed.curbytes;
922 arg->cnt_all->curbytes_unbiased
923 += tdata->cnt_summed.curbytes_unbiased;
924 if (opt_prof_accum) {
925 arg->cnt_all->accumobjs += tdata->cnt_summed.accumobjs;
926 arg->cnt_all->accumobjs_shifted_unbiased
927 += tdata->cnt_summed.accumobjs_shifted_unbiased;
928 arg->cnt_all->accumbytes +=
929 tdata->cnt_summed.accumbytes;
930 arg->cnt_all->accumbytes_unbiased +=
931 tdata->cnt_summed.accumbytes_unbiased;
932 }
933 } else {
934 tdata->dumping = false;
935 }
936 malloc_mutex_unlock(arg->tsdn, tdata->lock);
937
938 return NULL;
939}
940
941static prof_tdata_t *
942prof_tdata_dump_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata,
943 void *opaque) {
944 if (!tdata->dumping) {
945 return NULL;
946 }
947
948 prof_dump_iter_arg_t *arg = (prof_dump_iter_arg_t *)opaque;
949 prof_dump_printf(arg->prof_dump_write, arg->cbopaque, " t%"FMTu64": ",
950 tdata->thr_uid);
951 prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque,
952 &tdata->cnt_summed);
953 if (tdata->thread_name != NULL) {
954 arg->prof_dump_write(arg->cbopaque, " ");
955 arg->prof_dump_write(arg->cbopaque, tdata->thread_name);
956 }
957 arg->prof_dump_write(arg->cbopaque, "\n");
958 return NULL;
959}
960
961static void
962prof_dump_header(prof_dump_iter_arg_t *arg, const prof_cnt_t *cnt_all) {
963 prof_dump_printf(arg->prof_dump_write, arg->cbopaque,
964 "heap_v2/%"FMTu64"\n t*: ", ((uint64_t)1U << lg_prof_sample));
965 prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque, cnt_all);
966 arg->prof_dump_write(arg->cbopaque, "\n");
967
968 malloc_mutex_lock(arg->tsdn, &tdatas_mtx);
969 tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter, arg);
970 malloc_mutex_unlock(arg->tsdn, &tdatas_mtx);
971}
972
973static void
974prof_dump_gctx(prof_dump_iter_arg_t *arg, prof_gctx_t *gctx,
975 const prof_bt_t *bt, prof_gctx_tree_t *gctxs) {
976 cassert(config_prof);
977 malloc_mutex_assert_owner(arg->tsdn, gctx->lock);
978
979 /* Avoid dumping such gctx's that have no useful data. */
980 if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) ||
981 (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) {
982 assert(gctx->cnt_summed.curobjs == 0);
983 assert(gctx->cnt_summed.curbytes == 0);
984 /*
985 * These asserts would not be correct -- see the comment on races
986 * in prof.c
987 * assert(gctx->cnt_summed.curobjs_unbiased == 0);
988 * assert(gctx->cnt_summed.curbytes_unbiased == 0);
989 */
990 assert(gctx->cnt_summed.accumobjs == 0);
991 assert(gctx->cnt_summed.accumobjs_shifted_unbiased == 0);
992 assert(gctx->cnt_summed.accumbytes == 0);
993 assert(gctx->cnt_summed.accumbytes_unbiased == 0);
994 return;
995 }
996
997 arg->prof_dump_write(arg->cbopaque, "@");
998 for (unsigned i = 0; i < bt->len; i++) {
999 prof_dump_printf(arg->prof_dump_write, arg->cbopaque,
1000 " %#"FMTxPTR, (uintptr_t)bt->vec[i]);
1001 }
1002
1003 arg->prof_dump_write(arg->cbopaque, "\n t*: ");
1004 prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque,
1005 &gctx->cnt_summed);
1006 arg->prof_dump_write(arg->cbopaque, "\n");
1007
1008 tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter, arg);
1009}
1010
1011/*
1012 * See prof_sample_new_event_wait() comment for why the body of this function
1013 * is conditionally compiled.
1014 */
1015static void
1016prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx) {
1017#ifdef JEMALLOC_PROF
1018 /*
1019 * Scaling is equivalent AdjustSamples() in jeprof, but the result may
1020 * differ slightly from what jeprof reports, because here we scale the
1021 * summary values, whereas jeprof scales each context individually and
1022 * reports the sums of the scaled values.
1023 */
1024 if (cnt_all->curbytes != 0) {
1025 double sample_period = (double)((uint64_t)1 << lg_prof_sample);
1026 double ratio = (((double)cnt_all->curbytes) /
1027 (double)cnt_all->curobjs) / sample_period;
1028 double scale_factor = 1.0 / (1.0 - exp(-ratio));
1029 uint64_t curbytes = (uint64_t)round(((double)cnt_all->curbytes)
1030 * scale_factor);
1031 uint64_t curobjs = (uint64_t)round(((double)cnt_all->curobjs) *
1032 scale_factor);
1033
1034 malloc_printf("<jemalloc>: Leak approximation summary: ~%"FMTu64
1035 " byte%s, ~%"FMTu64" object%s, >= %zu context%s\n",
1036 curbytes, (curbytes != 1) ? "s" : "", curobjs, (curobjs !=
1037 1) ? "s" : "", leak_ngctx, (leak_ngctx != 1) ? "s" : "");
1038 malloc_printf(
1039 "<jemalloc>: Run jeprof on dump output for leak detail\n");
1040 if (opt_prof_leak_error) {
1041 malloc_printf(
1042 "<jemalloc>: Exiting with error code because memory"
1043 " leaks were detected\n");
1044 /*
1045 * Use _exit() with underscore to avoid calling atexit()
1046 * and entering endless cycle.
1047 */
1048 _exit(1);
1049 }
1050 }
1051#endif
1052}
1053
1054static prof_gctx_t *
1055prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) {
1056 prof_dump_iter_arg_t *arg = (prof_dump_iter_arg_t *)opaque;
1057 malloc_mutex_lock(arg->tsdn, gctx->lock);
1058 prof_dump_gctx(arg, gctx, &gctx->bt, gctxs);
1059 malloc_mutex_unlock(arg->tsdn, gctx->lock);
1060 return NULL;
1061}
1062
1063static void
1064prof_dump_prep(tsd_t *tsd, prof_tdata_t *tdata, prof_cnt_t *cnt_all,
1065 size_t *leak_ngctx, prof_gctx_tree_t *gctxs) {
1066 size_t tabind;
1067 union {
1068 prof_gctx_t *p;
1069 void *v;
1070 } gctx;
1071
1072 prof_enter(tsd, tdata);
1073
1074 /*
1075 * Put gctx's in limbo and clear their counters in preparation for
1076 * summing.
1077 */
1078 gctx_tree_new(gctxs);
1079 for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);) {
1080 prof_dump_gctx_prep(tsd_tsdn(tsd), gctx.p, gctxs);
1081 }
1082
1083 /*
1084 * Iterate over tdatas, and for the non-expired ones snapshot their tctx
1085 * stats and merge them into the associated gctx's.
1086 */
1087 memset(cnt_all, 0, sizeof(prof_cnt_t));
1088 prof_tdata_merge_iter_arg_t prof_tdata_merge_iter_arg = {tsd_tsdn(tsd),
1089 cnt_all};
1090 malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
1091 tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter,
1092 &prof_tdata_merge_iter_arg);
1093 malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
1094
1095 /* Merge tctx stats into gctx's. */
1096 *leak_ngctx = 0;
1097 prof_gctx_merge_iter_arg_t prof_gctx_merge_iter_arg = {tsd_tsdn(tsd),
1098 leak_ngctx};
1099 gctx_tree_iter(gctxs, NULL, prof_gctx_merge_iter,
1100 &prof_gctx_merge_iter_arg);
1101
1102 prof_leave(tsd, tdata);
1103}
1104
1105void
1106prof_dump_impl(tsd_t *tsd, write_cb_t *prof_dump_write, void *cbopaque,
1107 prof_tdata_t *tdata, bool leakcheck) {
1108 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_dump_mtx);
1109 prof_cnt_t cnt_all;
1110 size_t leak_ngctx;
1111 prof_gctx_tree_t gctxs;
1112 prof_dump_prep(tsd, tdata, &cnt_all, &leak_ngctx, &gctxs);
1113 prof_dump_iter_arg_t prof_dump_iter_arg = {tsd_tsdn(tsd),
1114 prof_dump_write, cbopaque};
1115 prof_dump_header(&prof_dump_iter_arg, &cnt_all);
1116 gctx_tree_iter(&gctxs, NULL, prof_gctx_dump_iter, &prof_dump_iter_arg);
1117 prof_gctx_finish(tsd, &gctxs);
1118 if (leakcheck) {
1119 prof_leakcheck(&cnt_all, leak_ngctx);
1120 }
1121}
1122
1123/* Used in unit tests. */
1124void
1125prof_cnt_all(prof_cnt_t *cnt_all) {
1126 tsd_t *tsd = tsd_fetch();
1127 prof_tdata_t *tdata = prof_tdata_get(tsd, false);
1128 if (tdata == NULL) {
1129 memset(cnt_all, 0, sizeof(prof_cnt_t));
1130 } else {
1131 size_t leak_ngctx;
1132 prof_gctx_tree_t gctxs;
1133 prof_dump_prep(tsd, tdata, cnt_all, &leak_ngctx, &gctxs);
1134 prof_gctx_finish(tsd, &gctxs);
1135 }
1136}
1137
1138void
1139prof_bt_hash(const void *key, size_t r_hash[2]) {
1140 prof_bt_t *bt = (prof_bt_t *)key;
1141
1142 cassert(config_prof);
1143
1144 hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash);
1145}
1146
1147bool
1148prof_bt_keycomp(const void *k1, const void *k2) {
1149 const prof_bt_t *bt1 = (prof_bt_t *)k1;
1150 const prof_bt_t *bt2 = (prof_bt_t *)k2;
1151
1152 cassert(config_prof);
1153
1154 if (bt1->len != bt2->len) {
1155 return false;
1156 }
1157 return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0);
1158}
1159
1160prof_tdata_t *
1161prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim,
1162 char *thread_name, bool active) {
1163 assert(tsd_reentrancy_level_get(tsd) == 0);
1164
1165 prof_tdata_t *tdata;
1166
1167 cassert(config_prof);
1168
1169 /* Initialize an empty cache for this thread. */
1170 tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), sizeof(prof_tdata_t),
1171 sz_size2index(sizeof(prof_tdata_t)), false, NULL, true,
1172 arena_get(TSDN_NULL, 0, true), true);
1173 if (tdata == NULL) {
1174 return NULL;
1175 }
1176
1177 tdata->lock = prof_tdata_mutex_choose(thr_uid);
1178 tdata->thr_uid = thr_uid;
1179 tdata->thr_discrim = thr_discrim;
1180 tdata->thread_name = thread_name;
1181 tdata->attached = true;
1182 tdata->expired = false;
1183 tdata->tctx_uid_next = 0;
1184
1185 if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash,
1186 prof_bt_keycomp)) {
1187 idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);
1188 return NULL;
1189 }
1190
1191 tdata->enq = false;
1192 tdata->enq_idump = false;
1193 tdata->enq_gdump = false;
1194
1195 tdata->dumping = false;
1196 tdata->active = active;
1197
1198 malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
1199 tdata_tree_insert(&tdatas, tdata);
1200 malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
1201
1202 return tdata;
1203}
1204
1205static bool
1206prof_tdata_should_destroy_unlocked(prof_tdata_t *tdata, bool even_if_attached) {
1207 if (tdata->attached && !even_if_attached) {
1208 return false;
1209 }
1210 if (ckh_count(&tdata->bt2tctx) != 0) {
1211 return false;
1212 }
1213 return true;
1214}
1215
1216static bool
1217prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata,
1218 bool even_if_attached) {
1219 malloc_mutex_assert_owner(tsdn, tdata->lock);
1220
1221 return prof_tdata_should_destroy_unlocked(tdata, even_if_attached);
1222}
1223
1224static void
1225prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata,
1226 bool even_if_attached) {
1227 malloc_mutex_assert_owner(tsd_tsdn(tsd), &tdatas_mtx);
1228 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tdata->lock);
1229
1230 tdata_tree_remove(&tdatas, tdata);
1231
1232 assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached));
1233
1234 if (tdata->thread_name != NULL) {
1235 idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,
1236 true);
1237 }
1238 ckh_delete(tsd, &tdata->bt2tctx);
1239 idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);
1240}
1241
1242static void
1243prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) {
1244 malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
1245 prof_tdata_destroy_locked(tsd, tdata, even_if_attached);
1246 malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
1247}
1248
1249void
1250prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) {
1251 bool destroy_tdata;
1252
1253 malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
1254 if (tdata->attached) {
1255 destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata,
1256 true);
1257 /*
1258 * Only detach if !destroy_tdata, because detaching would allow
1259 * another thread to win the race to destroy tdata.
1260 */
1261 if (!destroy_tdata) {
1262 tdata->attached = false;
1263 }
1264 tsd_prof_tdata_set(tsd, NULL);
1265 } else {
1266 destroy_tdata = false;
1267 }
1268 malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
1269 if (destroy_tdata) {
1270 prof_tdata_destroy(tsd, tdata, true);
1271 }
1272}
1273
1274static bool
1275prof_tdata_expire(tsdn_t *tsdn, prof_tdata_t *tdata) {
1276 bool destroy_tdata;
1277
1278 malloc_mutex_lock(tsdn, tdata->lock);
1279 if (!tdata->expired) {
1280 tdata->expired = true;
1281 destroy_tdata = prof_tdata_should_destroy(tsdn, tdata, false);
1282 } else {
1283 destroy_tdata = false;
1284 }
1285 malloc_mutex_unlock(tsdn, tdata->lock);
1286
1287 return destroy_tdata;
1288}
1289
1290static prof_tdata_t *
1291prof_tdata_reset_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata,
1292 void *arg) {
1293 tsdn_t *tsdn = (tsdn_t *)arg;
1294
1295 return (prof_tdata_expire(tsdn, tdata) ? tdata : NULL);
1296}
1297
1298void
1299prof_reset(tsd_t *tsd, size_t lg_sample) {
1300 prof_tdata_t *next;
1301
1302 assert(lg_sample < (sizeof(uint64_t) << 3));
1303
1304 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);
1305 malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
1306
1307 lg_prof_sample = lg_sample;
1308 prof_unbias_map_init();
1309
1310 next = NULL;
1311 do {
1312 prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next,
1313 prof_tdata_reset_iter, (void *)tsd);
1314 if (to_destroy != NULL) {
1315 next = tdata_tree_next(&tdatas, to_destroy);
1316 prof_tdata_destroy_locked(tsd, to_destroy, false);
1317 } else {
1318 next = NULL;
1319 }
1320 } while (next != NULL);
1321
1322 malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
1323 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);
1324}
1325
1326static bool
1327prof_tctx_should_destroy(tsd_t *tsd, prof_tctx_t *tctx) {
1328 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
1329
1330 if (opt_prof_accum) {
1331 return false;
1332 }
1333 if (tctx->cnts.curobjs != 0) {
1334 return false;
1335 }
1336 if (tctx->prepared) {
1337 return false;
1338 }
1339 if (tctx->recent_count != 0) {
1340 return false;
1341 }
1342 return true;
1343}
1344
1345static void
1346prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) {
1347 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
1348
1349 assert(tctx->cnts.curobjs == 0);
1350 assert(tctx->cnts.curbytes == 0);
1351 /*
1352 * These asserts are not correct -- see the comment about races in
1353 * prof.c
1354 *
1355 * assert(tctx->cnts.curobjs_shifted_unbiased == 0);
1356 * assert(tctx->cnts.curbytes_unbiased == 0);
1357 */
1358 assert(!opt_prof_accum);
1359 assert(tctx->cnts.accumobjs == 0);
1360 assert(tctx->cnts.accumbytes == 0);
1361 /*
1362 * These ones are, since accumbyte counts never go down. Either
1363 * prof_accum is off (in which case these should never have changed from
1364 * their initial value of zero), or it's on (in which case we shouldn't
1365 * be destroying this tctx).
1366 */
1367 assert(tctx->cnts.accumobjs_shifted_unbiased == 0);
1368 assert(tctx->cnts.accumbytes_unbiased == 0);
1369
1370 prof_gctx_t *gctx = tctx->gctx;
1371
1372 {
1373 prof_tdata_t *tdata = tctx->tdata;
1374 tctx->tdata = NULL;
1375 ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);
1376 bool destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd),
1377 tdata, false);
1378 malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
1379 if (destroy_tdata) {
1380 prof_tdata_destroy(tsd, tdata, false);
1381 }
1382 }
1383
1384 bool destroy_tctx, destroy_gctx;
1385
1386 malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
1387 switch (tctx->state) {
1388 case prof_tctx_state_nominal:
1389 tctx_tree_remove(&gctx->tctxs, tctx);
1390 destroy_tctx = true;
1391 if (prof_gctx_should_destroy(gctx)) {
1392 /*
1393 * Increment gctx->nlimbo in order to keep another
1394 * thread from winning the race to destroy gctx while
1395 * this one has gctx->lock dropped. Without this, it
1396 * would be possible for another thread to:
1397 *
1398 * 1) Sample an allocation associated with gctx.
1399 * 2) Deallocate the sampled object.
1400 * 3) Successfully prof_gctx_try_destroy(gctx).
1401 *
1402 * The result would be that gctx no longer exists by the
1403 * time this thread accesses it in
1404 * prof_gctx_try_destroy().
1405 */
1406 gctx->nlimbo++;
1407 destroy_gctx = true;
1408 } else {
1409 destroy_gctx = false;
1410 }
1411 break;
1412 case prof_tctx_state_dumping:
1413 /*
1414 * A dumping thread needs tctx to remain valid until dumping
1415 * has finished. Change state such that the dumping thread will
1416 * complete destruction during a late dump iteration phase.
1417 */
1418 tctx->state = prof_tctx_state_purgatory;
1419 destroy_tctx = false;
1420 destroy_gctx = false;
1421 break;
1422 default:
1423 not_reached();
1424 destroy_tctx = false;
1425 destroy_gctx = false;
1426 }
1427 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
1428 if (destroy_gctx) {
1429 prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx);
1430 }
1431 if (destroy_tctx) {
1432 idalloctm(tsd_tsdn(tsd), tctx, NULL, NULL, true, true);
1433 }
1434}
1435
1436void
1437prof_tctx_try_destroy(tsd_t *tsd, prof_tctx_t *tctx) {
1438 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
1439 if (prof_tctx_should_destroy(tsd, tctx)) {
1440 /* tctx->tdata->lock will be released in prof_tctx_destroy(). */
1441 prof_tctx_destroy(tsd, tctx);
1442 } else {
1443 malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);
1444 }
1445}
1446
1447/******************************************************************************/
diff --git a/examples/redis-unstable/deps/jemalloc/src/prof_log.c b/examples/redis-unstable/deps/jemalloc/src/prof_log.c
deleted file mode 100644
index 0632c3b..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/prof_log.c
+++ /dev/null
@@ -1,717 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/buf_writer.h"
6#include "jemalloc/internal/ckh.h"
7#include "jemalloc/internal/emitter.h"
8#include "jemalloc/internal/hash.h"
9#include "jemalloc/internal/malloc_io.h"
10#include "jemalloc/internal/mutex.h"
11#include "jemalloc/internal/prof_data.h"
12#include "jemalloc/internal/prof_log.h"
13#include "jemalloc/internal/prof_sys.h"
14
15bool opt_prof_log = false;
16typedef enum prof_logging_state_e prof_logging_state_t;
17enum prof_logging_state_e {
18 prof_logging_state_stopped,
19 prof_logging_state_started,
20 prof_logging_state_dumping
21};
22
23/*
24 * - stopped: log_start never called, or previous log_stop has completed.
25 * - started: log_start called, log_stop not called yet. Allocations are logged.
26 * - dumping: log_stop called but not finished; samples are not logged anymore.
27 */
28prof_logging_state_t prof_logging_state = prof_logging_state_stopped;
29
30/* Used in unit tests. */
31static bool prof_log_dummy = false;
32
33/* Incremented for every log file that is output. */
34static uint64_t log_seq = 0;
35static char log_filename[
36 /* Minimize memory bloat for non-prof builds. */
37#ifdef JEMALLOC_PROF
38 PATH_MAX +
39#endif
40 1];
41
42/* Timestamp for most recent call to log_start(). */
43static nstime_t log_start_timestamp;
44
45/* Increment these when adding to the log_bt and log_thr linked lists. */
46static size_t log_bt_index = 0;
47static size_t log_thr_index = 0;
48
49/* Linked list node definitions. These are only used in this file. */
50typedef struct prof_bt_node_s prof_bt_node_t;
51
52struct prof_bt_node_s {
53 prof_bt_node_t *next;
54 size_t index;
55 prof_bt_t bt;
56 /* Variable size backtrace vector pointed to by bt. */
57 void *vec[1];
58};
59
60typedef struct prof_thr_node_s prof_thr_node_t;
61
62struct prof_thr_node_s {
63 prof_thr_node_t *next;
64 size_t index;
65 uint64_t thr_uid;
66 /* Variable size based on thr_name_sz. */
67 char name[1];
68};
69
70typedef struct prof_alloc_node_s prof_alloc_node_t;
71
72/* This is output when logging sampled allocations. */
73struct prof_alloc_node_s {
74 prof_alloc_node_t *next;
75 /* Indices into an array of thread data. */
76 size_t alloc_thr_ind;
77 size_t free_thr_ind;
78
79 /* Indices into an array of backtraces. */
80 size_t alloc_bt_ind;
81 size_t free_bt_ind;
82
83 uint64_t alloc_time_ns;
84 uint64_t free_time_ns;
85
86 size_t usize;
87};
88
89/*
90 * Created on the first call to prof_try_log and deleted on prof_log_stop.
91 * These are the backtraces and threads that have already been logged by an
92 * allocation.
93 */
94static bool log_tables_initialized = false;
95static ckh_t log_bt_node_set;
96static ckh_t log_thr_node_set;
97
98/* Store linked lists for logged data. */
99static prof_bt_node_t *log_bt_first = NULL;
100static prof_bt_node_t *log_bt_last = NULL;
101static prof_thr_node_t *log_thr_first = NULL;
102static prof_thr_node_t *log_thr_last = NULL;
103static prof_alloc_node_t *log_alloc_first = NULL;
104static prof_alloc_node_t *log_alloc_last = NULL;
105
106/* Protects the prof_logging_state and any log_{...} variable. */
107malloc_mutex_t log_mtx;
108
109/******************************************************************************/
110/*
111 * Function prototypes for static functions that are referenced prior to
112 * definition.
113 */
114
115/* Hashtable functions for log_bt_node_set and log_thr_node_set. */
116static void prof_thr_node_hash(const void *key, size_t r_hash[2]);
117static bool prof_thr_node_keycomp(const void *k1, const void *k2);
118static void prof_bt_node_hash(const void *key, size_t r_hash[2]);
119static bool prof_bt_node_keycomp(const void *k1, const void *k2);
120
121/******************************************************************************/
122
123static size_t
124prof_log_bt_index(tsd_t *tsd, prof_bt_t *bt) {
125 assert(prof_logging_state == prof_logging_state_started);
126 malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx);
127
128 prof_bt_node_t dummy_node;
129 dummy_node.bt = *bt;
130 prof_bt_node_t *node;
131
132 /* See if this backtrace is already cached in the table. */
133 if (ckh_search(&log_bt_node_set, (void *)(&dummy_node),
134 (void **)(&node), NULL)) {
135 size_t sz = offsetof(prof_bt_node_t, vec) +
136 (bt->len * sizeof(void *));
137 prof_bt_node_t *new_node = (prof_bt_node_t *)
138 iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL,
139 true, arena_get(TSDN_NULL, 0, true), true);
140 if (log_bt_first == NULL) {
141 log_bt_first = new_node;
142 log_bt_last = new_node;
143 } else {
144 log_bt_last->next = new_node;
145 log_bt_last = new_node;
146 }
147
148 new_node->next = NULL;
149 new_node->index = log_bt_index;
150 /*
151 * Copy the backtrace: bt is inside a tdata or gctx, which
152 * might die before prof_log_stop is called.
153 */
154 new_node->bt.len = bt->len;
155 memcpy(new_node->vec, bt->vec, bt->len * sizeof(void *));
156 new_node->bt.vec = new_node->vec;
157
158 log_bt_index++;
159 ckh_insert(tsd, &log_bt_node_set, (void *)new_node, NULL);
160 return new_node->index;
161 } else {
162 return node->index;
163 }
164}
165
166static size_t
167prof_log_thr_index(tsd_t *tsd, uint64_t thr_uid, const char *name) {
168 assert(prof_logging_state == prof_logging_state_started);
169 malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx);
170
171 prof_thr_node_t dummy_node;
172 dummy_node.thr_uid = thr_uid;
173 prof_thr_node_t *node;
174
175 /* See if this thread is already cached in the table. */
176 if (ckh_search(&log_thr_node_set, (void *)(&dummy_node),
177 (void **)(&node), NULL)) {
178 size_t sz = offsetof(prof_thr_node_t, name) + strlen(name) + 1;
179 prof_thr_node_t *new_node = (prof_thr_node_t *)
180 iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL,
181 true, arena_get(TSDN_NULL, 0, true), true);
182 if (log_thr_first == NULL) {
183 log_thr_first = new_node;
184 log_thr_last = new_node;
185 } else {
186 log_thr_last->next = new_node;
187 log_thr_last = new_node;
188 }
189
190 new_node->next = NULL;
191 new_node->index = log_thr_index;
192 new_node->thr_uid = thr_uid;
193 strcpy(new_node->name, name);
194
195 log_thr_index++;
196 ckh_insert(tsd, &log_thr_node_set, (void *)new_node, NULL);
197 return new_node->index;
198 } else {
199 return node->index;
200 }
201}
202
203JEMALLOC_COLD
204void
205prof_try_log(tsd_t *tsd, size_t usize, prof_info_t *prof_info) {
206 cassert(config_prof);
207 prof_tctx_t *tctx = prof_info->alloc_tctx;
208 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
209
210 prof_tdata_t *cons_tdata = prof_tdata_get(tsd, false);
211 if (cons_tdata == NULL) {
212 /*
213 * We decide not to log these allocations. cons_tdata will be
214 * NULL only when the current thread is in a weird state (e.g.
215 * it's being destroyed).
216 */
217 return;
218 }
219
220 malloc_mutex_lock(tsd_tsdn(tsd), &log_mtx);
221
222 if (prof_logging_state != prof_logging_state_started) {
223 goto label_done;
224 }
225
226 if (!log_tables_initialized) {
227 bool err1 = ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS,
228 prof_bt_node_hash, prof_bt_node_keycomp);
229 bool err2 = ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS,
230 prof_thr_node_hash, prof_thr_node_keycomp);
231 if (err1 || err2) {
232 goto label_done;
233 }
234 log_tables_initialized = true;
235 }
236
237 nstime_t alloc_time = prof_info->alloc_time;
238 nstime_t free_time;
239 nstime_prof_init_update(&free_time);
240
241 size_t sz = sizeof(prof_alloc_node_t);
242 prof_alloc_node_t *new_node = (prof_alloc_node_t *)
243 iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, true,
244 arena_get(TSDN_NULL, 0, true), true);
245
246 const char *prod_thr_name = (tctx->tdata->thread_name == NULL)?
247 "" : tctx->tdata->thread_name;
248 const char *cons_thr_name = prof_thread_name_get(tsd);
249
250 prof_bt_t bt;
251 /* Initialize the backtrace, using the buffer in tdata to store it. */
252 bt_init(&bt, cons_tdata->vec);
253 prof_backtrace(tsd, &bt);
254 prof_bt_t *cons_bt = &bt;
255
256 /* We haven't destroyed tctx yet, so gctx should be good to read. */
257 prof_bt_t *prod_bt = &tctx->gctx->bt;
258
259 new_node->next = NULL;
260 new_node->alloc_thr_ind = prof_log_thr_index(tsd, tctx->tdata->thr_uid,
261 prod_thr_name);
262 new_node->free_thr_ind = prof_log_thr_index(tsd, cons_tdata->thr_uid,
263 cons_thr_name);
264 new_node->alloc_bt_ind = prof_log_bt_index(tsd, prod_bt);
265 new_node->free_bt_ind = prof_log_bt_index(tsd, cons_bt);
266 new_node->alloc_time_ns = nstime_ns(&alloc_time);
267 new_node->free_time_ns = nstime_ns(&free_time);
268 new_node->usize = usize;
269
270 if (log_alloc_first == NULL) {
271 log_alloc_first = new_node;
272 log_alloc_last = new_node;
273 } else {
274 log_alloc_last->next = new_node;
275 log_alloc_last = new_node;
276 }
277
278label_done:
279 malloc_mutex_unlock(tsd_tsdn(tsd), &log_mtx);
280}
281
282static void
283prof_bt_node_hash(const void *key, size_t r_hash[2]) {
284 const prof_bt_node_t *bt_node = (prof_bt_node_t *)key;
285 prof_bt_hash((void *)(&bt_node->bt), r_hash);
286}
287
288static bool
289prof_bt_node_keycomp(const void *k1, const void *k2) {
290 const prof_bt_node_t *bt_node1 = (prof_bt_node_t *)k1;
291 const prof_bt_node_t *bt_node2 = (prof_bt_node_t *)k2;
292 return prof_bt_keycomp((void *)(&bt_node1->bt),
293 (void *)(&bt_node2->bt));
294}
295
296static void
297prof_thr_node_hash(const void *key, size_t r_hash[2]) {
298 const prof_thr_node_t *thr_node = (prof_thr_node_t *)key;
299 hash(&thr_node->thr_uid, sizeof(uint64_t), 0x94122f35U, r_hash);
300}
301
302static bool
303prof_thr_node_keycomp(const void *k1, const void *k2) {
304 const prof_thr_node_t *thr_node1 = (prof_thr_node_t *)k1;
305 const prof_thr_node_t *thr_node2 = (prof_thr_node_t *)k2;
306 return thr_node1->thr_uid == thr_node2->thr_uid;
307}
308
309/* Used in unit tests. */
310size_t
311prof_log_bt_count(void) {
312 cassert(config_prof);
313 size_t cnt = 0;
314 prof_bt_node_t *node = log_bt_first;
315 while (node != NULL) {
316 cnt++;
317 node = node->next;
318 }
319 return cnt;
320}
321
322/* Used in unit tests. */
323size_t
324prof_log_alloc_count(void) {
325 cassert(config_prof);
326 size_t cnt = 0;
327 prof_alloc_node_t *node = log_alloc_first;
328 while (node != NULL) {
329 cnt++;
330 node = node->next;
331 }
332 return cnt;
333}
334
335/* Used in unit tests. */
336size_t
337prof_log_thr_count(void) {
338 cassert(config_prof);
339 size_t cnt = 0;
340 prof_thr_node_t *node = log_thr_first;
341 while (node != NULL) {
342 cnt++;
343 node = node->next;
344 }
345 return cnt;
346}
347
348/* Used in unit tests. */
349bool
350prof_log_is_logging(void) {
351 cassert(config_prof);
352 return prof_logging_state == prof_logging_state_started;
353}
354
355/* Used in unit tests. */
356bool
357prof_log_rep_check(void) {
358 cassert(config_prof);
359 if (prof_logging_state == prof_logging_state_stopped
360 && log_tables_initialized) {
361 return true;
362 }
363
364 if (log_bt_last != NULL && log_bt_last->next != NULL) {
365 return true;
366 }
367 if (log_thr_last != NULL && log_thr_last->next != NULL) {
368 return true;
369 }
370 if (log_alloc_last != NULL && log_alloc_last->next != NULL) {
371 return true;
372 }
373
374 size_t bt_count = prof_log_bt_count();
375 size_t thr_count = prof_log_thr_count();
376 size_t alloc_count = prof_log_alloc_count();
377
378
379 if (prof_logging_state == prof_logging_state_stopped) {
380 if (bt_count != 0 || thr_count != 0 || alloc_count || 0) {
381 return true;
382 }
383 }
384
385 prof_alloc_node_t *node = log_alloc_first;
386 while (node != NULL) {
387 if (node->alloc_bt_ind >= bt_count) {
388 return true;
389 }
390 if (node->free_bt_ind >= bt_count) {
391 return true;
392 }
393 if (node->alloc_thr_ind >= thr_count) {
394 return true;
395 }
396 if (node->free_thr_ind >= thr_count) {
397 return true;
398 }
399 if (node->alloc_time_ns > node->free_time_ns) {
400 return true;
401 }
402 node = node->next;
403 }
404
405 return false;
406}
407
408/* Used in unit tests. */
409void
410prof_log_dummy_set(bool new_value) {
411 cassert(config_prof);
412 prof_log_dummy = new_value;
413}
414
415/* Used as an atexit function to stop logging on exit. */
416static void
417prof_log_stop_final(void) {
418 tsd_t *tsd = tsd_fetch();
419 prof_log_stop(tsd_tsdn(tsd));
420}
421
422JEMALLOC_COLD
423bool
424prof_log_start(tsdn_t *tsdn, const char *filename) {
425 cassert(config_prof);
426
427 if (!opt_prof) {
428 return true;
429 }
430
431 bool ret = false;
432
433 malloc_mutex_lock(tsdn, &log_mtx);
434
435 static bool prof_log_atexit_called = false;
436 if (!prof_log_atexit_called) {
437 prof_log_atexit_called = true;
438 if (atexit(prof_log_stop_final) != 0) {
439 malloc_write("<jemalloc>: Error in atexit() "
440 "for logging\n");
441 if (opt_abort) {
442 abort();
443 }
444 ret = true;
445 goto label_done;
446 }
447 }
448
449 if (prof_logging_state != prof_logging_state_stopped) {
450 ret = true;
451 } else if (filename == NULL) {
452 /* Make default name. */
453 prof_get_default_filename(tsdn, log_filename, log_seq);
454 log_seq++;
455 prof_logging_state = prof_logging_state_started;
456 } else if (strlen(filename) >= PROF_DUMP_FILENAME_LEN) {
457 ret = true;
458 } else {
459 strcpy(log_filename, filename);
460 prof_logging_state = prof_logging_state_started;
461 }
462
463 if (!ret) {
464 nstime_prof_init_update(&log_start_timestamp);
465 }
466label_done:
467 malloc_mutex_unlock(tsdn, &log_mtx);
468
469 return ret;
470}
471
472struct prof_emitter_cb_arg_s {
473 int fd;
474 ssize_t ret;
475};
476
477static void
478prof_emitter_write_cb(void *opaque, const char *to_write) {
479 struct prof_emitter_cb_arg_s *arg =
480 (struct prof_emitter_cb_arg_s *)opaque;
481 size_t bytes = strlen(to_write);
482 if (prof_log_dummy) {
483 return;
484 }
485 arg->ret = malloc_write_fd(arg->fd, to_write, bytes);
486}
487
488/*
489 * prof_log_emit_{...} goes through the appropriate linked list, emitting each
490 * node to the json and deallocating it.
491 */
492static void
493prof_log_emit_threads(tsd_t *tsd, emitter_t *emitter) {
494 emitter_json_array_kv_begin(emitter, "threads");
495 prof_thr_node_t *thr_node = log_thr_first;
496 prof_thr_node_t *thr_old_node;
497 while (thr_node != NULL) {
498 emitter_json_object_begin(emitter);
499
500 emitter_json_kv(emitter, "thr_uid", emitter_type_uint64,
501 &thr_node->thr_uid);
502
503 char *thr_name = thr_node->name;
504
505 emitter_json_kv(emitter, "thr_name", emitter_type_string,
506 &thr_name);
507
508 emitter_json_object_end(emitter);
509 thr_old_node = thr_node;
510 thr_node = thr_node->next;
511 idalloctm(tsd_tsdn(tsd), thr_old_node, NULL, NULL, true, true);
512 }
513 emitter_json_array_end(emitter);
514}
515
516static void
517prof_log_emit_traces(tsd_t *tsd, emitter_t *emitter) {
518 emitter_json_array_kv_begin(emitter, "stack_traces");
519 prof_bt_node_t *bt_node = log_bt_first;
520 prof_bt_node_t *bt_old_node;
521 /*
522 * Calculate how many hex digits we need: twice number of bytes, two for
523 * "0x", and then one more for terminating '\0'.
524 */
525 char buf[2 * sizeof(intptr_t) + 3];
526 size_t buf_sz = sizeof(buf);
527 while (bt_node != NULL) {
528 emitter_json_array_begin(emitter);
529 size_t i;
530 for (i = 0; i < bt_node->bt.len; i++) {
531 malloc_snprintf(buf, buf_sz, "%p", bt_node->bt.vec[i]);
532 char *trace_str = buf;
533 emitter_json_value(emitter, emitter_type_string,
534 &trace_str);
535 }
536 emitter_json_array_end(emitter);
537
538 bt_old_node = bt_node;
539 bt_node = bt_node->next;
540 idalloctm(tsd_tsdn(tsd), bt_old_node, NULL, NULL, true, true);
541 }
542 emitter_json_array_end(emitter);
543}
544
545static void
546prof_log_emit_allocs(tsd_t *tsd, emitter_t *emitter) {
547 emitter_json_array_kv_begin(emitter, "allocations");
548 prof_alloc_node_t *alloc_node = log_alloc_first;
549 prof_alloc_node_t *alloc_old_node;
550 while (alloc_node != NULL) {
551 emitter_json_object_begin(emitter);
552
553 emitter_json_kv(emitter, "alloc_thread", emitter_type_size,
554 &alloc_node->alloc_thr_ind);
555
556 emitter_json_kv(emitter, "free_thread", emitter_type_size,
557 &alloc_node->free_thr_ind);
558
559 emitter_json_kv(emitter, "alloc_trace", emitter_type_size,
560 &alloc_node->alloc_bt_ind);
561
562 emitter_json_kv(emitter, "free_trace", emitter_type_size,
563 &alloc_node->free_bt_ind);
564
565 emitter_json_kv(emitter, "alloc_timestamp",
566 emitter_type_uint64, &alloc_node->alloc_time_ns);
567
568 emitter_json_kv(emitter, "free_timestamp", emitter_type_uint64,
569 &alloc_node->free_time_ns);
570
571 emitter_json_kv(emitter, "usize", emitter_type_uint64,
572 &alloc_node->usize);
573
574 emitter_json_object_end(emitter);
575
576 alloc_old_node = alloc_node;
577 alloc_node = alloc_node->next;
578 idalloctm(tsd_tsdn(tsd), alloc_old_node, NULL, NULL, true,
579 true);
580 }
581 emitter_json_array_end(emitter);
582}
583
584static void
585prof_log_emit_metadata(emitter_t *emitter) {
586 emitter_json_object_kv_begin(emitter, "info");
587
588 nstime_t now;
589
590 nstime_prof_init_update(&now);
591 uint64_t ns = nstime_ns(&now) - nstime_ns(&log_start_timestamp);
592 emitter_json_kv(emitter, "duration", emitter_type_uint64, &ns);
593
594 char *vers = JEMALLOC_VERSION;
595 emitter_json_kv(emitter, "version",
596 emitter_type_string, &vers);
597
598 emitter_json_kv(emitter, "lg_sample_rate",
599 emitter_type_int, &lg_prof_sample);
600
601 const char *res_type = prof_time_res_mode_names[opt_prof_time_res];
602 emitter_json_kv(emitter, "prof_time_resolution", emitter_type_string,
603 &res_type);
604
605 int pid = prof_getpid();
606 emitter_json_kv(emitter, "pid", emitter_type_int, &pid);
607
608 emitter_json_object_end(emitter);
609}
610
611#define PROF_LOG_STOP_BUFSIZE PROF_DUMP_BUFSIZE
612JEMALLOC_COLD
613bool
614prof_log_stop(tsdn_t *tsdn) {
615 cassert(config_prof);
616 if (!opt_prof || !prof_booted) {
617 return true;
618 }
619
620 tsd_t *tsd = tsdn_tsd(tsdn);
621 malloc_mutex_lock(tsdn, &log_mtx);
622
623 if (prof_logging_state != prof_logging_state_started) {
624 malloc_mutex_unlock(tsdn, &log_mtx);
625 return true;
626 }
627
628 /*
629 * Set the state to dumping. We'll set it to stopped when we're done.
630 * Since other threads won't be able to start/stop/log when the state is
631 * dumping, we don't have to hold the lock during the whole method.
632 */
633 prof_logging_state = prof_logging_state_dumping;
634 malloc_mutex_unlock(tsdn, &log_mtx);
635
636
637 emitter_t emitter;
638
639 /* Create a file. */
640
641 int fd;
642 if (prof_log_dummy) {
643 fd = 0;
644 } else {
645 fd = creat(log_filename, 0644);
646 }
647
648 if (fd == -1) {
649 malloc_printf("<jemalloc>: creat() for log file \"%s\" "
650 " failed with %d\n", log_filename, errno);
651 if (opt_abort) {
652 abort();
653 }
654 return true;
655 }
656
657 struct prof_emitter_cb_arg_s arg;
658 arg.fd = fd;
659
660 buf_writer_t buf_writer;
661 buf_writer_init(tsdn, &buf_writer, prof_emitter_write_cb, &arg, NULL,
662 PROF_LOG_STOP_BUFSIZE);
663 emitter_init(&emitter, emitter_output_json_compact, buf_writer_cb,
664 &buf_writer);
665
666 emitter_begin(&emitter);
667 prof_log_emit_metadata(&emitter);
668 prof_log_emit_threads(tsd, &emitter);
669 prof_log_emit_traces(tsd, &emitter);
670 prof_log_emit_allocs(tsd, &emitter);
671 emitter_end(&emitter);
672
673 buf_writer_terminate(tsdn, &buf_writer);
674
675 /* Reset global state. */
676 if (log_tables_initialized) {
677 ckh_delete(tsd, &log_bt_node_set);
678 ckh_delete(tsd, &log_thr_node_set);
679 }
680 log_tables_initialized = false;
681 log_bt_index = 0;
682 log_thr_index = 0;
683 log_bt_first = NULL;
684 log_bt_last = NULL;
685 log_thr_first = NULL;
686 log_thr_last = NULL;
687 log_alloc_first = NULL;
688 log_alloc_last = NULL;
689
690 malloc_mutex_lock(tsdn, &log_mtx);
691 prof_logging_state = prof_logging_state_stopped;
692 malloc_mutex_unlock(tsdn, &log_mtx);
693
694 if (prof_log_dummy) {
695 return false;
696 }
697 return close(fd) || arg.ret == -1;
698}
699#undef PROF_LOG_STOP_BUFSIZE
700
701JEMALLOC_COLD
702bool
703prof_log_init(tsd_t *tsd) {
704 cassert(config_prof);
705 if (malloc_mutex_init(&log_mtx, "prof_log",
706 WITNESS_RANK_PROF_LOG, malloc_mutex_rank_exclusive)) {
707 return true;
708 }
709
710 if (opt_prof_log) {
711 prof_log_start(tsd_tsdn(tsd), NULL);
712 }
713
714 return false;
715}
716
717/******************************************************************************/
diff --git a/examples/redis-unstable/deps/jemalloc/src/prof_recent.c b/examples/redis-unstable/deps/jemalloc/src/prof_recent.c
deleted file mode 100644
index 834a944..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/prof_recent.c
+++ /dev/null
@@ -1,600 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/buf_writer.h"
6#include "jemalloc/internal/emitter.h"
7#include "jemalloc/internal/prof_data.h"
8#include "jemalloc/internal/prof_recent.h"
9
10ssize_t opt_prof_recent_alloc_max = PROF_RECENT_ALLOC_MAX_DEFAULT;
11malloc_mutex_t prof_recent_alloc_mtx; /* Protects the fields below */
12static atomic_zd_t prof_recent_alloc_max;
13static ssize_t prof_recent_alloc_count = 0;
14prof_recent_list_t prof_recent_alloc_list;
15
16malloc_mutex_t prof_recent_dump_mtx; /* Protects dumping. */
17
18static void
19prof_recent_alloc_max_init() {
20 atomic_store_zd(&prof_recent_alloc_max, opt_prof_recent_alloc_max,
21 ATOMIC_RELAXED);
22}
23
24static inline ssize_t
25prof_recent_alloc_max_get_no_lock() {
26 return atomic_load_zd(&prof_recent_alloc_max, ATOMIC_RELAXED);
27}
28
29static inline ssize_t
30prof_recent_alloc_max_get(tsd_t *tsd) {
31 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
32 return prof_recent_alloc_max_get_no_lock();
33}
34
35static inline ssize_t
36prof_recent_alloc_max_update(tsd_t *tsd, ssize_t max) {
37 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
38 ssize_t old_max = prof_recent_alloc_max_get(tsd);
39 atomic_store_zd(&prof_recent_alloc_max, max, ATOMIC_RELAXED);
40 return old_max;
41}
42
43static prof_recent_t *
44prof_recent_allocate_node(tsdn_t *tsdn) {
45 return (prof_recent_t *)iallocztm(tsdn, sizeof(prof_recent_t),
46 sz_size2index(sizeof(prof_recent_t)), false, NULL, true,
47 arena_get(tsdn, 0, false), true);
48}
49
50static void
51prof_recent_free_node(tsdn_t *tsdn, prof_recent_t *node) {
52 assert(node != NULL);
53 assert(isalloc(tsdn, node) == sz_s2u(sizeof(prof_recent_t)));
54 idalloctm(tsdn, node, NULL, NULL, true, true);
55}
56
57static inline void
58increment_recent_count(tsd_t *tsd, prof_tctx_t *tctx) {
59 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
60 ++tctx->recent_count;
61 assert(tctx->recent_count > 0);
62}
63
64bool
65prof_recent_alloc_prepare(tsd_t *tsd, prof_tctx_t *tctx) {
66 cassert(config_prof);
67 assert(opt_prof && prof_booted);
68 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
69 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
70
71 /*
72 * Check whether last-N mode is turned on without trying to acquire the
73 * lock, so as to optimize for the following two scenarios:
74 * (1) Last-N mode is switched off;
75 * (2) Dumping, during which last-N mode is temporarily turned off so
76 * as not to block sampled allocations.
77 */
78 if (prof_recent_alloc_max_get_no_lock() == 0) {
79 return false;
80 }
81
82 /*
83 * Increment recent_count to hold the tctx so that it won't be gone
84 * even after tctx->tdata->lock is released. This acts as a
85 * "placeholder"; the real recording of the allocation requires a lock
86 * on prof_recent_alloc_mtx and is done in prof_recent_alloc (when
87 * tctx->tdata->lock has been released).
88 */
89 increment_recent_count(tsd, tctx);
90 return true;
91}
92
93static void
94decrement_recent_count(tsd_t *tsd, prof_tctx_t *tctx) {
95 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
96 assert(tctx != NULL);
97 malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);
98 assert(tctx->recent_count > 0);
99 --tctx->recent_count;
100 prof_tctx_try_destroy(tsd, tctx);
101}
102
103static inline edata_t *
104prof_recent_alloc_edata_get_no_lock(const prof_recent_t *n) {
105 return (edata_t *)atomic_load_p(&n->alloc_edata, ATOMIC_ACQUIRE);
106}
107
108edata_t *
109prof_recent_alloc_edata_get_no_lock_test(const prof_recent_t *n) {
110 cassert(config_prof);
111 return prof_recent_alloc_edata_get_no_lock(n);
112}
113
114static inline edata_t *
115prof_recent_alloc_edata_get(tsd_t *tsd, const prof_recent_t *n) {
116 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
117 return prof_recent_alloc_edata_get_no_lock(n);
118}
119
120static void
121prof_recent_alloc_edata_set(tsd_t *tsd, prof_recent_t *n, edata_t *edata) {
122 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
123 atomic_store_p(&n->alloc_edata, edata, ATOMIC_RELEASE);
124}
125
126void
127edata_prof_recent_alloc_init(edata_t *edata) {
128 cassert(config_prof);
129 edata_prof_recent_alloc_set_dont_call_directly(edata, NULL);
130}
131
132static inline prof_recent_t *
133edata_prof_recent_alloc_get_no_lock(const edata_t *edata) {
134 cassert(config_prof);
135 return edata_prof_recent_alloc_get_dont_call_directly(edata);
136}
137
138prof_recent_t *
139edata_prof_recent_alloc_get_no_lock_test(const edata_t *edata) {
140 cassert(config_prof);
141 return edata_prof_recent_alloc_get_no_lock(edata);
142}
143
144static inline prof_recent_t *
145edata_prof_recent_alloc_get(tsd_t *tsd, const edata_t *edata) {
146 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
147 prof_recent_t *recent_alloc =
148 edata_prof_recent_alloc_get_no_lock(edata);
149 assert(recent_alloc == NULL ||
150 prof_recent_alloc_edata_get(tsd, recent_alloc) == edata);
151 return recent_alloc;
152}
153
154static prof_recent_t *
155edata_prof_recent_alloc_update_internal(tsd_t *tsd, edata_t *edata,
156 prof_recent_t *recent_alloc) {
157 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
158 prof_recent_t *old_recent_alloc =
159 edata_prof_recent_alloc_get(tsd, edata);
160 edata_prof_recent_alloc_set_dont_call_directly(edata, recent_alloc);
161 return old_recent_alloc;
162}
163
164static void
165edata_prof_recent_alloc_set(tsd_t *tsd, edata_t *edata,
166 prof_recent_t *recent_alloc) {
167 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
168 assert(recent_alloc != NULL);
169 prof_recent_t *old_recent_alloc =
170 edata_prof_recent_alloc_update_internal(tsd, edata, recent_alloc);
171 assert(old_recent_alloc == NULL);
172 prof_recent_alloc_edata_set(tsd, recent_alloc, edata);
173}
174
175static void
176edata_prof_recent_alloc_reset(tsd_t *tsd, edata_t *edata,
177 prof_recent_t *recent_alloc) {
178 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
179 assert(recent_alloc != NULL);
180 prof_recent_t *old_recent_alloc =
181 edata_prof_recent_alloc_update_internal(tsd, edata, NULL);
182 assert(old_recent_alloc == recent_alloc);
183 assert(edata == prof_recent_alloc_edata_get(tsd, recent_alloc));
184 prof_recent_alloc_edata_set(tsd, recent_alloc, NULL);
185}
186
187/*
188 * This function should be called right before an allocation is released, so
189 * that the associated recent allocation record can contain the following
190 * information:
191 * (1) The allocation is released;
192 * (2) The time of the deallocation; and
193 * (3) The prof_tctx associated with the deallocation.
194 */
195void
196prof_recent_alloc_reset(tsd_t *tsd, edata_t *edata) {
197 cassert(config_prof);
198 /*
199 * Check whether the recent allocation record still exists without
200 * trying to acquire the lock.
201 */
202 if (edata_prof_recent_alloc_get_no_lock(edata) == NULL) {
203 return;
204 }
205
206 prof_tctx_t *dalloc_tctx = prof_tctx_create(tsd);
207 /*
208 * In case dalloc_tctx is NULL, e.g. due to OOM, we will not record the
209 * deallocation time / tctx, which is handled later, after we check
210 * again when holding the lock.
211 */
212
213 if (dalloc_tctx != NULL) {
214 malloc_mutex_lock(tsd_tsdn(tsd), dalloc_tctx->tdata->lock);
215 increment_recent_count(tsd, dalloc_tctx);
216 dalloc_tctx->prepared = false;
217 malloc_mutex_unlock(tsd_tsdn(tsd), dalloc_tctx->tdata->lock);
218 }
219
220 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
221 /* Check again after acquiring the lock. */
222 prof_recent_t *recent = edata_prof_recent_alloc_get(tsd, edata);
223 if (recent != NULL) {
224 assert(nstime_equals_zero(&recent->dalloc_time));
225 assert(recent->dalloc_tctx == NULL);
226 if (dalloc_tctx != NULL) {
227 nstime_prof_update(&recent->dalloc_time);
228 recent->dalloc_tctx = dalloc_tctx;
229 dalloc_tctx = NULL;
230 }
231 edata_prof_recent_alloc_reset(tsd, edata, recent);
232 }
233 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
234
235 if (dalloc_tctx != NULL) {
236 /* We lost the rase - the allocation record was just gone. */
237 decrement_recent_count(tsd, dalloc_tctx);
238 }
239}
240
241static void
242prof_recent_alloc_evict_edata(tsd_t *tsd, prof_recent_t *recent_alloc) {
243 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
244 edata_t *edata = prof_recent_alloc_edata_get(tsd, recent_alloc);
245 if (edata != NULL) {
246 edata_prof_recent_alloc_reset(tsd, edata, recent_alloc);
247 }
248}
249
250static bool
251prof_recent_alloc_is_empty(tsd_t *tsd) {
252 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
253 if (ql_empty(&prof_recent_alloc_list)) {
254 assert(prof_recent_alloc_count == 0);
255 return true;
256 } else {
257 assert(prof_recent_alloc_count > 0);
258 return false;
259 }
260}
261
262static void
263prof_recent_alloc_assert_count(tsd_t *tsd) {
264 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
265 if (!config_debug) {
266 return;
267 }
268 ssize_t count = 0;
269 prof_recent_t *n;
270 ql_foreach(n, &prof_recent_alloc_list, link) {
271 ++count;
272 }
273 assert(count == prof_recent_alloc_count);
274 assert(prof_recent_alloc_max_get(tsd) == -1 ||
275 count <= prof_recent_alloc_max_get(tsd));
276}
277
278void
279prof_recent_alloc(tsd_t *tsd, edata_t *edata, size_t size, size_t usize) {
280 cassert(config_prof);
281 assert(edata != NULL);
282 prof_tctx_t *tctx = edata_prof_tctx_get(edata);
283
284 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tctx->tdata->lock);
285 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
286 prof_recent_alloc_assert_count(tsd);
287
288 /*
289 * Reserve a new prof_recent_t node if needed. If needed, we release
290 * the prof_recent_alloc_mtx lock and allocate. Then, rather than
291 * immediately checking for OOM, we regain the lock and try to make use
292 * of the reserve node if needed. There are six scenarios:
293 *
294 * \ now | no need | need but OOMed | need and allocated
295 * later \ | | |
296 * ------------------------------------------------------------
297 * no need | (1) | (2) | (3)
298 * ------------------------------------------------------------
299 * need | (4) | (5) | (6)
300 *
301 * First, "(4)" never happens, because we don't release the lock in the
302 * middle if there's no need for a new node; in such cases "(1)" always
303 * takes place, which is trivial.
304 *
305 * Out of the remaining four scenarios, "(6)" is the common case and is
306 * trivial. "(5)" is also trivial, in which case we'll rollback the
307 * effect of prof_recent_alloc_prepare() as expected.
308 *
309 * "(2)" / "(3)" occurs when the need for a new node is gone after we
310 * regain the lock. If the new node is successfully allocated, i.e. in
311 * the case of "(3)", we'll release it in the end; otherwise, i.e. in
312 * the case of "(2)", we do nothing - we're lucky that the OOM ends up
313 * doing no harm at all.
314 *
315 * Therefore, the only performance cost of the "release lock" ->
316 * "allocate" -> "regain lock" design is the "(3)" case, but it happens
317 * very rarely, so the cost is relatively small compared to the gain of
318 * not having to have the lock order of prof_recent_alloc_mtx above all
319 * the allocation locks.
320 */
321 prof_recent_t *reserve = NULL;
322 if (prof_recent_alloc_max_get(tsd) == -1 ||
323 prof_recent_alloc_count < prof_recent_alloc_max_get(tsd)) {
324 assert(prof_recent_alloc_max_get(tsd) != 0);
325 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
326 reserve = prof_recent_allocate_node(tsd_tsdn(tsd));
327 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
328 prof_recent_alloc_assert_count(tsd);
329 }
330
331 if (prof_recent_alloc_max_get(tsd) == 0) {
332 assert(prof_recent_alloc_is_empty(tsd));
333 goto label_rollback;
334 }
335
336 prof_tctx_t *old_alloc_tctx, *old_dalloc_tctx;
337 if (prof_recent_alloc_count == prof_recent_alloc_max_get(tsd)) {
338 /* If upper limit is reached, rotate the head. */
339 assert(prof_recent_alloc_max_get(tsd) != -1);
340 assert(!prof_recent_alloc_is_empty(tsd));
341 prof_recent_t *head = ql_first(&prof_recent_alloc_list);
342 old_alloc_tctx = head->alloc_tctx;
343 assert(old_alloc_tctx != NULL);
344 old_dalloc_tctx = head->dalloc_tctx;
345 prof_recent_alloc_evict_edata(tsd, head);
346 ql_rotate(&prof_recent_alloc_list, link);
347 } else {
348 /* Otherwise make use of the new node. */
349 assert(prof_recent_alloc_max_get(tsd) == -1 ||
350 prof_recent_alloc_count < prof_recent_alloc_max_get(tsd));
351 if (reserve == NULL) {
352 goto label_rollback;
353 }
354 ql_elm_new(reserve, link);
355 ql_tail_insert(&prof_recent_alloc_list, reserve, link);
356 reserve = NULL;
357 old_alloc_tctx = NULL;
358 old_dalloc_tctx = NULL;
359 ++prof_recent_alloc_count;
360 }
361
362 /* Fill content into the tail node. */
363 prof_recent_t *tail = ql_last(&prof_recent_alloc_list, link);
364 assert(tail != NULL);
365 tail->size = size;
366 tail->usize = usize;
367 nstime_copy(&tail->alloc_time, edata_prof_alloc_time_get(edata));
368 tail->alloc_tctx = tctx;
369 nstime_init_zero(&tail->dalloc_time);
370 tail->dalloc_tctx = NULL;
371 edata_prof_recent_alloc_set(tsd, edata, tail);
372
373 assert(!prof_recent_alloc_is_empty(tsd));
374 prof_recent_alloc_assert_count(tsd);
375 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
376
377 if (reserve != NULL) {
378 prof_recent_free_node(tsd_tsdn(tsd), reserve);
379 }
380
381 /*
382 * Asynchronously handle the tctx of the old node, so that there's no
383 * simultaneous holdings of prof_recent_alloc_mtx and tdata->lock.
384 * In the worst case this may delay the tctx release but it's better
385 * than holding prof_recent_alloc_mtx for longer.
386 */
387 if (old_alloc_tctx != NULL) {
388 decrement_recent_count(tsd, old_alloc_tctx);
389 }
390 if (old_dalloc_tctx != NULL) {
391 decrement_recent_count(tsd, old_dalloc_tctx);
392 }
393 return;
394
395label_rollback:
396 assert(edata_prof_recent_alloc_get(tsd, edata) == NULL);
397 prof_recent_alloc_assert_count(tsd);
398 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
399 if (reserve != NULL) {
400 prof_recent_free_node(tsd_tsdn(tsd), reserve);
401 }
402 decrement_recent_count(tsd, tctx);
403}
404
405ssize_t
406prof_recent_alloc_max_ctl_read() {
407 cassert(config_prof);
408 /* Don't bother to acquire the lock. */
409 return prof_recent_alloc_max_get_no_lock();
410}
411
412static void
413prof_recent_alloc_restore_locked(tsd_t *tsd, prof_recent_list_t *to_delete) {
414 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
415 ssize_t max = prof_recent_alloc_max_get(tsd);
416 if (max == -1 || prof_recent_alloc_count <= max) {
417 /* Easy case - no need to alter the list. */
418 ql_new(to_delete);
419 prof_recent_alloc_assert_count(tsd);
420 return;
421 }
422
423 prof_recent_t *node;
424 ql_foreach(node, &prof_recent_alloc_list, link) {
425 if (prof_recent_alloc_count == max) {
426 break;
427 }
428 prof_recent_alloc_evict_edata(tsd, node);
429 --prof_recent_alloc_count;
430 }
431 assert(prof_recent_alloc_count == max);
432
433 ql_move(to_delete, &prof_recent_alloc_list);
434 if (max == 0) {
435 assert(node == NULL);
436 } else {
437 assert(node != NULL);
438 ql_split(to_delete, node, &prof_recent_alloc_list, link);
439 }
440 assert(!ql_empty(to_delete));
441 prof_recent_alloc_assert_count(tsd);
442}
443
444static void
445prof_recent_alloc_async_cleanup(tsd_t *tsd, prof_recent_list_t *to_delete) {
446 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &prof_recent_dump_mtx);
447 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
448 while (!ql_empty(to_delete)) {
449 prof_recent_t *node = ql_first(to_delete);
450 ql_remove(to_delete, node, link);
451 decrement_recent_count(tsd, node->alloc_tctx);
452 if (node->dalloc_tctx != NULL) {
453 decrement_recent_count(tsd, node->dalloc_tctx);
454 }
455 prof_recent_free_node(tsd_tsdn(tsd), node);
456 }
457}
458
459ssize_t
460prof_recent_alloc_max_ctl_write(tsd_t *tsd, ssize_t max) {
461 cassert(config_prof);
462 assert(max >= -1);
463 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
464 prof_recent_alloc_assert_count(tsd);
465 const ssize_t old_max = prof_recent_alloc_max_update(tsd, max);
466 prof_recent_list_t to_delete;
467 prof_recent_alloc_restore_locked(tsd, &to_delete);
468 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
469 prof_recent_alloc_async_cleanup(tsd, &to_delete);
470 return old_max;
471}
472
473static void
474prof_recent_alloc_dump_bt(emitter_t *emitter, prof_tctx_t *tctx) {
475 char bt_buf[2 * sizeof(intptr_t) + 3];
476 char *s = bt_buf;
477 assert(tctx != NULL);
478 prof_bt_t *bt = &tctx->gctx->bt;
479 for (size_t i = 0; i < bt->len; ++i) {
480 malloc_snprintf(bt_buf, sizeof(bt_buf), "%p", bt->vec[i]);
481 emitter_json_value(emitter, emitter_type_string, &s);
482 }
483}
484
485static void
486prof_recent_alloc_dump_node(emitter_t *emitter, prof_recent_t *node) {
487 emitter_json_object_begin(emitter);
488
489 emitter_json_kv(emitter, "size", emitter_type_size, &node->size);
490 emitter_json_kv(emitter, "usize", emitter_type_size, &node->usize);
491 bool released = prof_recent_alloc_edata_get_no_lock(node) == NULL;
492 emitter_json_kv(emitter, "released", emitter_type_bool, &released);
493
494 emitter_json_kv(emitter, "alloc_thread_uid", emitter_type_uint64,
495 &node->alloc_tctx->thr_uid);
496 prof_tdata_t *alloc_tdata = node->alloc_tctx->tdata;
497 assert(alloc_tdata != NULL);
498 if (alloc_tdata->thread_name != NULL) {
499 emitter_json_kv(emitter, "alloc_thread_name",
500 emitter_type_string, &alloc_tdata->thread_name);
501 }
502 uint64_t alloc_time_ns = nstime_ns(&node->alloc_time);
503 emitter_json_kv(emitter, "alloc_time", emitter_type_uint64,
504 &alloc_time_ns);
505 emitter_json_array_kv_begin(emitter, "alloc_trace");
506 prof_recent_alloc_dump_bt(emitter, node->alloc_tctx);
507 emitter_json_array_end(emitter);
508
509 if (released && node->dalloc_tctx != NULL) {
510 emitter_json_kv(emitter, "dalloc_thread_uid",
511 emitter_type_uint64, &node->dalloc_tctx->thr_uid);
512 prof_tdata_t *dalloc_tdata = node->dalloc_tctx->tdata;
513 assert(dalloc_tdata != NULL);
514 if (dalloc_tdata->thread_name != NULL) {
515 emitter_json_kv(emitter, "dalloc_thread_name",
516 emitter_type_string, &dalloc_tdata->thread_name);
517 }
518 assert(!nstime_equals_zero(&node->dalloc_time));
519 uint64_t dalloc_time_ns = nstime_ns(&node->dalloc_time);
520 emitter_json_kv(emitter, "dalloc_time", emitter_type_uint64,
521 &dalloc_time_ns);
522 emitter_json_array_kv_begin(emitter, "dalloc_trace");
523 prof_recent_alloc_dump_bt(emitter, node->dalloc_tctx);
524 emitter_json_array_end(emitter);
525 }
526
527 emitter_json_object_end(emitter);
528}
529
530#define PROF_RECENT_PRINT_BUFSIZE 65536
531JEMALLOC_COLD
532void
533prof_recent_alloc_dump(tsd_t *tsd, write_cb_t *write_cb, void *cbopaque) {
534 cassert(config_prof);
535 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_dump_mtx);
536 buf_writer_t buf_writer;
537 buf_writer_init(tsd_tsdn(tsd), &buf_writer, write_cb, cbopaque, NULL,
538 PROF_RECENT_PRINT_BUFSIZE);
539 emitter_t emitter;
540 emitter_init(&emitter, emitter_output_json_compact, buf_writer_cb,
541 &buf_writer);
542 prof_recent_list_t temp_list;
543
544 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
545 prof_recent_alloc_assert_count(tsd);
546 ssize_t dump_max = prof_recent_alloc_max_get(tsd);
547 ql_move(&temp_list, &prof_recent_alloc_list);
548 ssize_t dump_count = prof_recent_alloc_count;
549 prof_recent_alloc_count = 0;
550 prof_recent_alloc_assert_count(tsd);
551 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
552
553 emitter_begin(&emitter);
554 uint64_t sample_interval = (uint64_t)1U << lg_prof_sample;
555 emitter_json_kv(&emitter, "sample_interval", emitter_type_uint64,
556 &sample_interval);
557 emitter_json_kv(&emitter, "recent_alloc_max", emitter_type_ssize,
558 &dump_max);
559 emitter_json_array_kv_begin(&emitter, "recent_alloc");
560 prof_recent_t *node;
561 ql_foreach(node, &temp_list, link) {
562 prof_recent_alloc_dump_node(&emitter, node);
563 }
564 emitter_json_array_end(&emitter);
565 emitter_end(&emitter);
566
567 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
568 prof_recent_alloc_assert_count(tsd);
569 ql_concat(&temp_list, &prof_recent_alloc_list, link);
570 ql_move(&prof_recent_alloc_list, &temp_list);
571 prof_recent_alloc_count += dump_count;
572 prof_recent_alloc_restore_locked(tsd, &temp_list);
573 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
574
575 buf_writer_terminate(tsd_tsdn(tsd), &buf_writer);
576 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_dump_mtx);
577
578 prof_recent_alloc_async_cleanup(tsd, &temp_list);
579}
580#undef PROF_RECENT_PRINT_BUFSIZE
581
582bool
583prof_recent_init() {
584 cassert(config_prof);
585 prof_recent_alloc_max_init();
586
587 if (malloc_mutex_init(&prof_recent_alloc_mtx, "prof_recent_alloc",
588 WITNESS_RANK_PROF_RECENT_ALLOC, malloc_mutex_rank_exclusive)) {
589 return true;
590 }
591
592 if (malloc_mutex_init(&prof_recent_dump_mtx, "prof_recent_dump",
593 WITNESS_RANK_PROF_RECENT_DUMP, malloc_mutex_rank_exclusive)) {
594 return true;
595 }
596
597 ql_new(&prof_recent_alloc_list);
598
599 return false;
600}
diff --git a/examples/redis-unstable/deps/jemalloc/src/prof_stats.c b/examples/redis-unstable/deps/jemalloc/src/prof_stats.c
deleted file mode 100644
index 5d1a506..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/prof_stats.c
+++ /dev/null
@@ -1,57 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/prof_stats.h"
5
6bool opt_prof_stats = false;
7malloc_mutex_t prof_stats_mtx;
8static prof_stats_t prof_stats_live[PROF_SC_NSIZES];
9static prof_stats_t prof_stats_accum[PROF_SC_NSIZES];
10
11static void
12prof_stats_enter(tsd_t *tsd, szind_t ind) {
13 assert(opt_prof && opt_prof_stats);
14 assert(ind < SC_NSIZES);
15 malloc_mutex_lock(tsd_tsdn(tsd), &prof_stats_mtx);
16}
17
18static void
19prof_stats_leave(tsd_t *tsd) {
20 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_stats_mtx);
21}
22
23void
24prof_stats_inc(tsd_t *tsd, szind_t ind, size_t size) {
25 cassert(config_prof);
26 prof_stats_enter(tsd, ind);
27 prof_stats_live[ind].req_sum += size;
28 prof_stats_live[ind].count++;
29 prof_stats_accum[ind].req_sum += size;
30 prof_stats_accum[ind].count++;
31 prof_stats_leave(tsd);
32}
33
34void
35prof_stats_dec(tsd_t *tsd, szind_t ind, size_t size) {
36 cassert(config_prof);
37 prof_stats_enter(tsd, ind);
38 prof_stats_live[ind].req_sum -= size;
39 prof_stats_live[ind].count--;
40 prof_stats_leave(tsd);
41}
42
43void
44prof_stats_get_live(tsd_t *tsd, szind_t ind, prof_stats_t *stats) {
45 cassert(config_prof);
46 prof_stats_enter(tsd, ind);
47 memcpy(stats, &prof_stats_live[ind], sizeof(prof_stats_t));
48 prof_stats_leave(tsd);
49}
50
51void
52prof_stats_get_accum(tsd_t *tsd, szind_t ind, prof_stats_t *stats) {
53 cassert(config_prof);
54 prof_stats_enter(tsd, ind);
55 memcpy(stats, &prof_stats_accum[ind], sizeof(prof_stats_t));
56 prof_stats_leave(tsd);
57}
diff --git a/examples/redis-unstable/deps/jemalloc/src/prof_sys.c b/examples/redis-unstable/deps/jemalloc/src/prof_sys.c
deleted file mode 100644
index b5f1f5b..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/prof_sys.c
+++ /dev/null
@@ -1,669 +0,0 @@
1#define JEMALLOC_PROF_SYS_C_
2#include "jemalloc/internal/jemalloc_preamble.h"
3#include "jemalloc/internal/jemalloc_internal_includes.h"
4
5#include "jemalloc/internal/buf_writer.h"
6#include "jemalloc/internal/ctl.h"
7#include "jemalloc/internal/prof_data.h"
8#include "jemalloc/internal/prof_sys.h"
9
10#ifdef JEMALLOC_PROF_LIBUNWIND
11#define UNW_LOCAL_ONLY
12#include <libunwind.h>
13#endif
14
15#ifdef JEMALLOC_PROF_LIBGCC
16/*
17 * We have a circular dependency -- jemalloc_internal.h tells us if we should
18 * use libgcc's unwinding functionality, but after we've included that, we've
19 * already hooked _Unwind_Backtrace. We'll temporarily disable hooking.
20 */
21#undef _Unwind_Backtrace
22#include <unwind.h>
23#define _Unwind_Backtrace JEMALLOC_TEST_HOOK(_Unwind_Backtrace, test_hooks_libc_hook)
24#endif
25
26/******************************************************************************/
27
28malloc_mutex_t prof_dump_filename_mtx;
29
30bool prof_do_mock = false;
31
32static uint64_t prof_dump_seq;
33static uint64_t prof_dump_iseq;
34static uint64_t prof_dump_mseq;
35static uint64_t prof_dump_useq;
36
37static char *prof_prefix = NULL;
38
39/* The fallback allocator profiling functionality will use. */
40base_t *prof_base;
41
42void
43bt_init(prof_bt_t *bt, void **vec) {
44 cassert(config_prof);
45
46 bt->vec = vec;
47 bt->len = 0;
48}
49
50#ifdef JEMALLOC_PROF_LIBUNWIND
51static void
52prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) {
53 int nframes;
54
55 cassert(config_prof);
56 assert(*len == 0);
57 assert(vec != NULL);
58 assert(max_len == PROF_BT_MAX);
59
60 nframes = unw_backtrace(vec, PROF_BT_MAX);
61 if (nframes <= 0) {
62 return;
63 }
64 *len = nframes;
65}
66#elif (defined(JEMALLOC_PROF_LIBGCC))
67static _Unwind_Reason_Code
68prof_unwind_init_callback(struct _Unwind_Context *context, void *arg) {
69 cassert(config_prof);
70
71 return _URC_NO_REASON;
72}
73
74static _Unwind_Reason_Code
75prof_unwind_callback(struct _Unwind_Context *context, void *arg) {
76 prof_unwind_data_t *data = (prof_unwind_data_t *)arg;
77 void *ip;
78
79 cassert(config_prof);
80
81 ip = (void *)_Unwind_GetIP(context);
82 if (ip == NULL) {
83 return _URC_END_OF_STACK;
84 }
85 data->vec[*data->len] = ip;
86 (*data->len)++;
87 if (*data->len == data->max) {
88 return _URC_END_OF_STACK;
89 }
90
91 return _URC_NO_REASON;
92}
93
94static void
95prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) {
96 prof_unwind_data_t data = {vec, len, max_len};
97
98 cassert(config_prof);
99 assert(vec != NULL);
100 assert(max_len == PROF_BT_MAX);
101
102 _Unwind_Backtrace(prof_unwind_callback, &data);
103}
104#elif (defined(JEMALLOC_PROF_GCC))
105static void
106prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) {
107#define BT_FRAME(i) \
108 if ((i) < max_len) { \
109 void *p; \
110 if (__builtin_frame_address(i) == 0) { \
111 return; \
112 } \
113 p = __builtin_return_address(i); \
114 if (p == NULL) { \
115 return; \
116 } \
117 vec[(i)] = p; \
118 *len = (i) + 1; \
119 } else { \
120 return; \
121 }
122
123 cassert(config_prof);
124 assert(vec != NULL);
125 assert(max_len == PROF_BT_MAX);
126
127 BT_FRAME(0)
128 BT_FRAME(1)
129 BT_FRAME(2)
130 BT_FRAME(3)
131 BT_FRAME(4)
132 BT_FRAME(5)
133 BT_FRAME(6)
134 BT_FRAME(7)
135 BT_FRAME(8)
136 BT_FRAME(9)
137
138 BT_FRAME(10)
139 BT_FRAME(11)
140 BT_FRAME(12)
141 BT_FRAME(13)
142 BT_FRAME(14)
143 BT_FRAME(15)
144 BT_FRAME(16)
145 BT_FRAME(17)
146 BT_FRAME(18)
147 BT_FRAME(19)
148
149 BT_FRAME(20)
150 BT_FRAME(21)
151 BT_FRAME(22)
152 BT_FRAME(23)
153 BT_FRAME(24)
154 BT_FRAME(25)
155 BT_FRAME(26)
156 BT_FRAME(27)
157 BT_FRAME(28)
158 BT_FRAME(29)
159
160 BT_FRAME(30)
161 BT_FRAME(31)
162 BT_FRAME(32)
163 BT_FRAME(33)
164 BT_FRAME(34)
165 BT_FRAME(35)
166 BT_FRAME(36)
167 BT_FRAME(37)
168 BT_FRAME(38)
169 BT_FRAME(39)
170
171 BT_FRAME(40)
172 BT_FRAME(41)
173 BT_FRAME(42)
174 BT_FRAME(43)
175 BT_FRAME(44)
176 BT_FRAME(45)
177 BT_FRAME(46)
178 BT_FRAME(47)
179 BT_FRAME(48)
180 BT_FRAME(49)
181
182 BT_FRAME(50)
183 BT_FRAME(51)
184 BT_FRAME(52)
185 BT_FRAME(53)
186 BT_FRAME(54)
187 BT_FRAME(55)
188 BT_FRAME(56)
189 BT_FRAME(57)
190 BT_FRAME(58)
191 BT_FRAME(59)
192
193 BT_FRAME(60)
194 BT_FRAME(61)
195 BT_FRAME(62)
196 BT_FRAME(63)
197 BT_FRAME(64)
198 BT_FRAME(65)
199 BT_FRAME(66)
200 BT_FRAME(67)
201 BT_FRAME(68)
202 BT_FRAME(69)
203
204 BT_FRAME(70)
205 BT_FRAME(71)
206 BT_FRAME(72)
207 BT_FRAME(73)
208 BT_FRAME(74)
209 BT_FRAME(75)
210 BT_FRAME(76)
211 BT_FRAME(77)
212 BT_FRAME(78)
213 BT_FRAME(79)
214
215 BT_FRAME(80)
216 BT_FRAME(81)
217 BT_FRAME(82)
218 BT_FRAME(83)
219 BT_FRAME(84)
220 BT_FRAME(85)
221 BT_FRAME(86)
222 BT_FRAME(87)
223 BT_FRAME(88)
224 BT_FRAME(89)
225
226 BT_FRAME(90)
227 BT_FRAME(91)
228 BT_FRAME(92)
229 BT_FRAME(93)
230 BT_FRAME(94)
231 BT_FRAME(95)
232 BT_FRAME(96)
233 BT_FRAME(97)
234 BT_FRAME(98)
235 BT_FRAME(99)
236
237 BT_FRAME(100)
238 BT_FRAME(101)
239 BT_FRAME(102)
240 BT_FRAME(103)
241 BT_FRAME(104)
242 BT_FRAME(105)
243 BT_FRAME(106)
244 BT_FRAME(107)
245 BT_FRAME(108)
246 BT_FRAME(109)
247
248 BT_FRAME(110)
249 BT_FRAME(111)
250 BT_FRAME(112)
251 BT_FRAME(113)
252 BT_FRAME(114)
253 BT_FRAME(115)
254 BT_FRAME(116)
255 BT_FRAME(117)
256 BT_FRAME(118)
257 BT_FRAME(119)
258
259 BT_FRAME(120)
260 BT_FRAME(121)
261 BT_FRAME(122)
262 BT_FRAME(123)
263 BT_FRAME(124)
264 BT_FRAME(125)
265 BT_FRAME(126)
266 BT_FRAME(127)
267#undef BT_FRAME
268}
269#else
270static void
271prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) {
272 cassert(config_prof);
273 not_reached();
274}
275#endif
276
277void
278prof_backtrace(tsd_t *tsd, prof_bt_t *bt) {
279 cassert(config_prof);
280 prof_backtrace_hook_t prof_backtrace_hook = prof_backtrace_hook_get();
281 assert(prof_backtrace_hook != NULL);
282
283 pre_reentrancy(tsd, NULL);
284 prof_backtrace_hook(bt->vec, &bt->len, PROF_BT_MAX);
285 post_reentrancy(tsd);
286}
287
288void
289prof_hooks_init() {
290 prof_backtrace_hook_set(&prof_backtrace_impl);
291 prof_dump_hook_set(NULL);
292}
293
294void
295prof_unwind_init() {
296#ifdef JEMALLOC_PROF_LIBGCC
297 /*
298 * Cause the backtracing machinery to allocate its internal
299 * state before enabling profiling.
300 */
301 _Unwind_Backtrace(prof_unwind_init_callback, NULL);
302#endif
303}
304
305static int
306prof_sys_thread_name_read_impl(char *buf, size_t limit) {
307#if defined(JEMALLOC_HAVE_PTHREAD_GETNAME_NP)
308 return pthread_getname_np(pthread_self(), buf, limit);
309#elif defined(JEMALLOC_HAVE_PTHREAD_GET_NAME_NP)
310 pthread_get_name_np(pthread_self(), buf, limit);
311 return 0;
312#else
313 return ENOSYS;
314#endif
315}
316prof_sys_thread_name_read_t *JET_MUTABLE prof_sys_thread_name_read =
317 prof_sys_thread_name_read_impl;
318
319void
320prof_sys_thread_name_fetch(tsd_t *tsd) {
321#define THREAD_NAME_MAX_LEN 16
322 char buf[THREAD_NAME_MAX_LEN];
323 if (!prof_sys_thread_name_read(buf, THREAD_NAME_MAX_LEN)) {
324 prof_thread_name_set_impl(tsd, buf);
325 }
326#undef THREAD_NAME_MAX_LEN
327}
328
329int
330prof_getpid(void) {
331#ifdef _WIN32
332 return GetCurrentProcessId();
333#else
334 return getpid();
335#endif
336}
337
338/*
339 * This buffer is rather large for stack allocation, so use a single buffer for
340 * all profile dumps; protected by prof_dump_mtx.
341 */
342static char prof_dump_buf[PROF_DUMP_BUFSIZE];
343
344typedef struct prof_dump_arg_s prof_dump_arg_t;
345struct prof_dump_arg_s {
346 /*
347 * Whether error should be handled locally: if true, then we print out
348 * error message as well as abort (if opt_abort is true) when an error
349 * occurred, and we also report the error back to the caller in the end;
350 * if false, then we only report the error back to the caller in the
351 * end.
352 */
353 const bool handle_error_locally;
354 /*
355 * Whether there has been an error in the dumping process, which could
356 * have happened either in file opening or in file writing. When an
357 * error has already occurred, we will stop further writing to the file.
358 */
359 bool error;
360 /* File descriptor of the dump file. */
361 int prof_dump_fd;
362};
363
364static void
365prof_dump_check_possible_error(prof_dump_arg_t *arg, bool err_cond,
366 const char *format, ...) {
367 assert(!arg->error);
368 if (!err_cond) {
369 return;
370 }
371
372 arg->error = true;
373 if (!arg->handle_error_locally) {
374 return;
375 }
376
377 va_list ap;
378 char buf[PROF_PRINTF_BUFSIZE];
379 va_start(ap, format);
380 malloc_vsnprintf(buf, sizeof(buf), format, ap);
381 va_end(ap);
382 malloc_write(buf);
383
384 if (opt_abort) {
385 abort();
386 }
387}
388
389static int
390prof_dump_open_file_impl(const char *filename, int mode) {
391 return creat(filename, mode);
392}
393prof_dump_open_file_t *JET_MUTABLE prof_dump_open_file =
394 prof_dump_open_file_impl;
395
396static void
397prof_dump_open(prof_dump_arg_t *arg, const char *filename) {
398 arg->prof_dump_fd = prof_dump_open_file(filename, 0644);
399 prof_dump_check_possible_error(arg, arg->prof_dump_fd == -1,
400 "<jemalloc>: failed to open \"%s\"\n", filename);
401}
402
403prof_dump_write_file_t *JET_MUTABLE prof_dump_write_file = malloc_write_fd;
404
405static void
406prof_dump_flush(void *opaque, const char *s) {
407 cassert(config_prof);
408 prof_dump_arg_t *arg = (prof_dump_arg_t *)opaque;
409 if (!arg->error) {
410 ssize_t err = prof_dump_write_file(arg->prof_dump_fd, s,
411 strlen(s));
412 prof_dump_check_possible_error(arg, err == -1,
413 "<jemalloc>: failed to write during heap profile flush\n");
414 }
415}
416
417static void
418prof_dump_close(prof_dump_arg_t *arg) {
419 if (arg->prof_dump_fd != -1) {
420 close(arg->prof_dump_fd);
421 }
422}
423
424#ifndef _WIN32
425JEMALLOC_FORMAT_PRINTF(1, 2)
426static int
427prof_open_maps_internal(const char *format, ...) {
428 int mfd;
429 va_list ap;
430 char filename[PATH_MAX + 1];
431
432 va_start(ap, format);
433 malloc_vsnprintf(filename, sizeof(filename), format, ap);
434 va_end(ap);
435
436#if defined(O_CLOEXEC)
437 mfd = open(filename, O_RDONLY | O_CLOEXEC);
438#else
439 mfd = open(filename, O_RDONLY);
440 if (mfd != -1) {
441 fcntl(mfd, F_SETFD, fcntl(mfd, F_GETFD) | FD_CLOEXEC);
442 }
443#endif
444
445 return mfd;
446}
447#endif
448
449static int
450prof_dump_open_maps_impl() {
451 int mfd;
452
453 cassert(config_prof);
454#if defined(__FreeBSD__) || defined(__DragonFly__)
455 mfd = prof_open_maps_internal("/proc/curproc/map");
456#elif defined(_WIN32)
457 mfd = -1; // Not implemented
458#else
459 int pid = prof_getpid();
460
461 mfd = prof_open_maps_internal("/proc/%d/task/%d/maps", pid, pid);
462 if (mfd == -1) {
463 mfd = prof_open_maps_internal("/proc/%d/maps", pid);
464 }
465#endif
466 return mfd;
467}
468prof_dump_open_maps_t *JET_MUTABLE prof_dump_open_maps =
469 prof_dump_open_maps_impl;
470
471static ssize_t
472prof_dump_read_maps_cb(void *read_cbopaque, void *buf, size_t limit) {
473 int mfd = *(int *)read_cbopaque;
474 assert(mfd != -1);
475 return malloc_read_fd(mfd, buf, limit);
476}
477
478static void
479prof_dump_maps(buf_writer_t *buf_writer) {
480 int mfd = prof_dump_open_maps();
481 if (mfd == -1) {
482 return;
483 }
484
485 buf_writer_cb(buf_writer, "\nMAPPED_LIBRARIES:\n");
486 buf_writer_pipe(buf_writer, prof_dump_read_maps_cb, &mfd);
487 close(mfd);
488}
489
490static bool
491prof_dump(tsd_t *tsd, bool propagate_err, const char *filename,
492 bool leakcheck) {
493 cassert(config_prof);
494 assert(tsd_reentrancy_level_get(tsd) == 0);
495
496 prof_tdata_t * tdata = prof_tdata_get(tsd, true);
497 if (tdata == NULL) {
498 return true;
499 }
500
501 prof_dump_arg_t arg = {/* handle_error_locally */ !propagate_err,
502 /* error */ false, /* prof_dump_fd */ -1};
503
504 pre_reentrancy(tsd, NULL);
505 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);
506
507 prof_dump_open(&arg, filename);
508 buf_writer_t buf_writer;
509 bool err = buf_writer_init(tsd_tsdn(tsd), &buf_writer, prof_dump_flush,
510 &arg, prof_dump_buf, PROF_DUMP_BUFSIZE);
511 assert(!err);
512 prof_dump_impl(tsd, buf_writer_cb, &buf_writer, tdata, leakcheck);
513 prof_dump_maps(&buf_writer);
514 buf_writer_terminate(tsd_tsdn(tsd), &buf_writer);
515 prof_dump_close(&arg);
516
517 prof_dump_hook_t dump_hook = prof_dump_hook_get();
518 if (dump_hook != NULL) {
519 dump_hook(filename);
520 }
521 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);
522 post_reentrancy(tsd);
523
524 return arg.error;
525}
526
527/*
528 * If profiling is off, then PROF_DUMP_FILENAME_LEN is 1, so we'll end up
529 * calling strncpy with a size of 0, which triggers a -Wstringop-truncation
530 * warning (strncpy can never actually be called in this case, since we bail out
531 * much earlier when config_prof is false). This function works around the
532 * warning to let us leave the warning on.
533 */
534static inline void
535prof_strncpy(char *UNUSED dest, const char *UNUSED src, size_t UNUSED size) {
536 cassert(config_prof);
537#ifdef JEMALLOC_PROF
538 strncpy(dest, src, size);
539#endif
540}
541
542static const char *
543prof_prefix_get(tsdn_t* tsdn) {
544 malloc_mutex_assert_owner(tsdn, &prof_dump_filename_mtx);
545
546 return prof_prefix == NULL ? opt_prof_prefix : prof_prefix;
547}
548
549static bool
550prof_prefix_is_empty(tsdn_t *tsdn) {
551 malloc_mutex_lock(tsdn, &prof_dump_filename_mtx);
552 bool ret = (prof_prefix_get(tsdn)[0] == '\0');
553 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx);
554 return ret;
555}
556
557#define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1)
558#define VSEQ_INVALID UINT64_C(0xffffffffffffffff)
559static void
560prof_dump_filename(tsd_t *tsd, char *filename, char v, uint64_t vseq) {
561 cassert(config_prof);
562
563 assert(tsd_reentrancy_level_get(tsd) == 0);
564 const char *prefix = prof_prefix_get(tsd_tsdn(tsd));
565
566 if (vseq != VSEQ_INVALID) {
567 /* "<prefix>.<pid>.<seq>.v<vseq>.heap" */
568 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
569 "%s.%d.%"FMTu64".%c%"FMTu64".heap", prefix, prof_getpid(),
570 prof_dump_seq, v, vseq);
571 } else {
572 /* "<prefix>.<pid>.<seq>.<v>.heap" */
573 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
574 "%s.%d.%"FMTu64".%c.heap", prefix, prof_getpid(),
575 prof_dump_seq, v);
576 }
577 prof_dump_seq++;
578}
579
580void
581prof_get_default_filename(tsdn_t *tsdn, char *filename, uint64_t ind) {
582 malloc_mutex_lock(tsdn, &prof_dump_filename_mtx);
583 malloc_snprintf(filename, PROF_DUMP_FILENAME_LEN,
584 "%s.%d.%"FMTu64".json", prof_prefix_get(tsdn), prof_getpid(), ind);
585 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx);
586}
587
588void
589prof_fdump_impl(tsd_t *tsd) {
590 char filename[DUMP_FILENAME_BUFSIZE];
591
592 assert(!prof_prefix_is_empty(tsd_tsdn(tsd)));
593 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_filename_mtx);
594 prof_dump_filename(tsd, filename, 'f', VSEQ_INVALID);
595 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_filename_mtx);
596 prof_dump(tsd, false, filename, opt_prof_leak);
597}
598
599bool
600prof_prefix_set(tsdn_t *tsdn, const char *prefix) {
601 cassert(config_prof);
602 ctl_mtx_assert_held(tsdn);
603 malloc_mutex_lock(tsdn, &prof_dump_filename_mtx);
604 if (prof_prefix == NULL) {
605 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx);
606 /* Everything is still guarded by ctl_mtx. */
607 char *buffer = base_alloc(tsdn, prof_base,
608 PROF_DUMP_FILENAME_LEN, QUANTUM);
609 if (buffer == NULL) {
610 return true;
611 }
612 malloc_mutex_lock(tsdn, &prof_dump_filename_mtx);
613 prof_prefix = buffer;
614 }
615 assert(prof_prefix != NULL);
616
617 prof_strncpy(prof_prefix, prefix, PROF_DUMP_FILENAME_LEN - 1);
618 prof_prefix[PROF_DUMP_FILENAME_LEN - 1] = '\0';
619 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx);
620
621 return false;
622}
623
624void
625prof_idump_impl(tsd_t *tsd) {
626 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_filename_mtx);
627 if (prof_prefix_get(tsd_tsdn(tsd))[0] == '\0') {
628 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_filename_mtx);
629 return;
630 }
631 char filename[PATH_MAX + 1];
632 prof_dump_filename(tsd, filename, 'i', prof_dump_iseq);
633 prof_dump_iseq++;
634 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_filename_mtx);
635 prof_dump(tsd, false, filename, false);
636}
637
638bool
639prof_mdump_impl(tsd_t *tsd, const char *filename) {
640 char filename_buf[DUMP_FILENAME_BUFSIZE];
641 if (filename == NULL) {
642 /* No filename specified, so automatically generate one. */
643 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_filename_mtx);
644 if (prof_prefix_get(tsd_tsdn(tsd))[0] == '\0') {
645 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_filename_mtx);
646 return true;
647 }
648 prof_dump_filename(tsd, filename_buf, 'm', prof_dump_mseq);
649 prof_dump_mseq++;
650 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_filename_mtx);
651 filename = filename_buf;
652 }
653 return prof_dump(tsd, true, filename, false);
654}
655
656void
657prof_gdump_impl(tsd_t *tsd) {
658 tsdn_t *tsdn = tsd_tsdn(tsd);
659 malloc_mutex_lock(tsdn, &prof_dump_filename_mtx);
660 if (prof_prefix_get(tsdn)[0] == '\0') {
661 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx);
662 return;
663 }
664 char filename[DUMP_FILENAME_BUFSIZE];
665 prof_dump_filename(tsd, filename, 'u', prof_dump_useq);
666 prof_dump_useq++;
667 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx);
668 prof_dump(tsd, false, filename, false);
669}
diff --git a/examples/redis-unstable/deps/jemalloc/src/psset.c b/examples/redis-unstable/deps/jemalloc/src/psset.c
deleted file mode 100644
index 9a8f054..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/psset.c
+++ /dev/null
@@ -1,385 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/psset.h"
5
6#include "jemalloc/internal/fb.h"
7
8void
9psset_init(psset_t *psset) {
10 for (unsigned i = 0; i < PSSET_NPSIZES; i++) {
11 hpdata_age_heap_new(&psset->pageslabs[i]);
12 }
13 fb_init(psset->pageslab_bitmap, PSSET_NPSIZES);
14 memset(&psset->merged_stats, 0, sizeof(psset->merged_stats));
15 memset(&psset->stats, 0, sizeof(psset->stats));
16 hpdata_empty_list_init(&psset->empty);
17 for (int i = 0; i < PSSET_NPURGE_LISTS; i++) {
18 hpdata_purge_list_init(&psset->to_purge[i]);
19 }
20 fb_init(psset->purge_bitmap, PSSET_NPURGE_LISTS);
21 hpdata_hugify_list_init(&psset->to_hugify);
22}
23
24static void
25psset_bin_stats_accum(psset_bin_stats_t *dst, psset_bin_stats_t *src) {
26 dst->npageslabs += src->npageslabs;
27 dst->nactive += src->nactive;
28 dst->ndirty += src->ndirty;
29}
30
31void
32psset_stats_accum(psset_stats_t *dst, psset_stats_t *src) {
33 psset_bin_stats_accum(&dst->full_slabs[0], &src->full_slabs[0]);
34 psset_bin_stats_accum(&dst->full_slabs[1], &src->full_slabs[1]);
35 psset_bin_stats_accum(&dst->empty_slabs[0], &src->empty_slabs[0]);
36 psset_bin_stats_accum(&dst->empty_slabs[1], &src->empty_slabs[1]);
37 for (pszind_t i = 0; i < PSSET_NPSIZES; i++) {
38 psset_bin_stats_accum(&dst->nonfull_slabs[i][0],
39 &src->nonfull_slabs[i][0]);
40 psset_bin_stats_accum(&dst->nonfull_slabs[i][1],
41 &src->nonfull_slabs[i][1]);
42 }
43}
44
45/*
46 * The stats maintenance strategy is to remove a pageslab's contribution to the
47 * stats when we call psset_update_begin, and re-add it (to a potentially new
48 * bin) when we call psset_update_end.
49 */
50JEMALLOC_ALWAYS_INLINE void
51psset_bin_stats_insert_remove(psset_t *psset, psset_bin_stats_t *binstats,
52 hpdata_t *ps, bool insert) {
53 size_t mul = insert ? (size_t)1 : (size_t)-1;
54 size_t huge_idx = (size_t)hpdata_huge_get(ps);
55
56 binstats[huge_idx].npageslabs += mul * 1;
57 binstats[huge_idx].nactive += mul * hpdata_nactive_get(ps);
58 binstats[huge_idx].ndirty += mul * hpdata_ndirty_get(ps);
59
60 psset->merged_stats.npageslabs += mul * 1;
61 psset->merged_stats.nactive += mul * hpdata_nactive_get(ps);
62 psset->merged_stats.ndirty += mul * hpdata_ndirty_get(ps);
63
64 if (config_debug) {
65 psset_bin_stats_t check_stats = {0};
66 for (size_t huge = 0; huge <= 1; huge++) {
67 psset_bin_stats_accum(&check_stats,
68 &psset->stats.full_slabs[huge]);
69 psset_bin_stats_accum(&check_stats,
70 &psset->stats.empty_slabs[huge]);
71 for (pszind_t pind = 0; pind < PSSET_NPSIZES; pind++) {
72 psset_bin_stats_accum(&check_stats,
73 &psset->stats.nonfull_slabs[pind][huge]);
74 }
75 }
76 assert(psset->merged_stats.npageslabs
77 == check_stats.npageslabs);
78 assert(psset->merged_stats.nactive == check_stats.nactive);
79 assert(psset->merged_stats.ndirty == check_stats.ndirty);
80 }
81}
82
83static void
84psset_bin_stats_insert(psset_t *psset, psset_bin_stats_t *binstats,
85 hpdata_t *ps) {
86 psset_bin_stats_insert_remove(psset, binstats, ps, true);
87}
88
89static void
90psset_bin_stats_remove(psset_t *psset, psset_bin_stats_t *binstats,
91 hpdata_t *ps) {
92 psset_bin_stats_insert_remove(psset, binstats, ps, false);
93}
94
95static void
96psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) {
97 hpdata_age_heap_remove(&psset->pageslabs[pind], ps);
98 if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
99 fb_unset(psset->pageslab_bitmap, PSSET_NPSIZES, (size_t)pind);
100 }
101}
102
103static void
104psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) {
105 if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
106 fb_set(psset->pageslab_bitmap, PSSET_NPSIZES, (size_t)pind);
107 }
108 hpdata_age_heap_insert(&psset->pageslabs[pind], ps);
109}
110
111static void
112psset_stats_insert(psset_t* psset, hpdata_t *ps) {
113 if (hpdata_empty(ps)) {
114 psset_bin_stats_insert(psset, psset->stats.empty_slabs, ps);
115 } else if (hpdata_full(ps)) {
116 psset_bin_stats_insert(psset, psset->stats.full_slabs, ps);
117 } else {
118 size_t longest_free_range = hpdata_longest_free_range_get(ps);
119
120 pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(
121 longest_free_range << LG_PAGE));
122 assert(pind < PSSET_NPSIZES);
123
124 psset_bin_stats_insert(psset, psset->stats.nonfull_slabs[pind],
125 ps);
126 }
127}
128
129static void
130psset_stats_remove(psset_t *psset, hpdata_t *ps) {
131 if (hpdata_empty(ps)) {
132 psset_bin_stats_remove(psset, psset->stats.empty_slabs, ps);
133 } else if (hpdata_full(ps)) {
134 psset_bin_stats_remove(psset, psset->stats.full_slabs, ps);
135 } else {
136 size_t longest_free_range = hpdata_longest_free_range_get(ps);
137
138 pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(
139 longest_free_range << LG_PAGE));
140 assert(pind < PSSET_NPSIZES);
141
142 psset_bin_stats_remove(psset, psset->stats.nonfull_slabs[pind],
143 ps);
144 }
145}
146
147/*
148 * Put ps into some container so that it can be found during future allocation
149 * requests.
150 */
151static void
152psset_alloc_container_insert(psset_t *psset, hpdata_t *ps) {
153 assert(!hpdata_in_psset_alloc_container_get(ps));
154 hpdata_in_psset_alloc_container_set(ps, true);
155 if (hpdata_empty(ps)) {
156 /*
157 * This prepend, paired with popping the head in psset_fit,
158 * means we implement LIFO ordering for the empty slabs set,
159 * which seems reasonable.
160 */
161 hpdata_empty_list_prepend(&psset->empty, ps);
162 } else if (hpdata_full(ps)) {
163 /*
164 * We don't need to keep track of the full slabs; we're never
165 * going to return them from a psset_pick_alloc call.
166 */
167 } else {
168 size_t longest_free_range = hpdata_longest_free_range_get(ps);
169
170 pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(
171 longest_free_range << LG_PAGE));
172 assert(pind < PSSET_NPSIZES);
173
174 psset_hpdata_heap_insert(psset, pind, ps);
175 }
176}
177
178/* Remove ps from those collections. */
179static void
180psset_alloc_container_remove(psset_t *psset, hpdata_t *ps) {
181 assert(hpdata_in_psset_alloc_container_get(ps));
182 hpdata_in_psset_alloc_container_set(ps, false);
183
184 if (hpdata_empty(ps)) {
185 hpdata_empty_list_remove(&psset->empty, ps);
186 } else if (hpdata_full(ps)) {
187 /* Same as above -- do nothing in this case. */
188 } else {
189 size_t longest_free_range = hpdata_longest_free_range_get(ps);
190
191 pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(
192 longest_free_range << LG_PAGE));
193 assert(pind < PSSET_NPSIZES);
194
195 psset_hpdata_heap_remove(psset, pind, ps);
196 }
197}
198
199static size_t
200psset_purge_list_ind(hpdata_t *ps) {
201 size_t ndirty = hpdata_ndirty_get(ps);
202 /* Shouldn't have something with no dirty pages purgeable. */
203 assert(ndirty > 0);
204 /*
205 * Higher indices correspond to lists we'd like to purge earlier; make
206 * the two highest indices correspond to empty lists, which we attempt
207 * to purge before purging any non-empty list. This has two advantages:
208 * - Empty page slabs are the least likely to get reused (we'll only
209 * pick them for an allocation if we have no other choice).
210 * - Empty page slabs can purge every dirty page they contain in a
211 * single call, which is not usually the case.
212 *
213 * We purge hugeified empty slabs before nonhugeified ones, on the basis
214 * that they are fully dirty, while nonhugified slabs might not be, so
215 * we free up more pages more easily.
216 */
217 if (hpdata_nactive_get(ps) == 0) {
218 if (hpdata_huge_get(ps)) {
219 return PSSET_NPURGE_LISTS - 1;
220 } else {
221 return PSSET_NPURGE_LISTS - 2;
222 }
223 }
224
225 pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(ndirty << LG_PAGE));
226 /*
227 * For non-empty slabs, we may reuse them again. Prefer purging
228 * non-hugeified slabs before hugeified ones then, among pages of
229 * similar dirtiness. We still get some benefit from the hugification.
230 */
231 return (size_t)pind * 2 + (hpdata_huge_get(ps) ? 0 : 1);
232}
233
234static void
235psset_maybe_remove_purge_list(psset_t *psset, hpdata_t *ps) {
236 /*
237 * Remove the hpdata from its purge list (if it's in one). Even if it's
238 * going to stay in the same one, by appending it during
239 * psset_update_end, we move it to the end of its queue, so that we
240 * purge LRU within a given dirtiness bucket.
241 */
242 if (hpdata_purge_allowed_get(ps)) {
243 size_t ind = psset_purge_list_ind(ps);
244 hpdata_purge_list_t *purge_list = &psset->to_purge[ind];
245 hpdata_purge_list_remove(purge_list, ps);
246 if (hpdata_purge_list_empty(purge_list)) {
247 fb_unset(psset->purge_bitmap, PSSET_NPURGE_LISTS, ind);
248 }
249 }
250}
251
252static void
253psset_maybe_insert_purge_list(psset_t *psset, hpdata_t *ps) {
254 if (hpdata_purge_allowed_get(ps)) {
255 size_t ind = psset_purge_list_ind(ps);
256 hpdata_purge_list_t *purge_list = &psset->to_purge[ind];
257 if (hpdata_purge_list_empty(purge_list)) {
258 fb_set(psset->purge_bitmap, PSSET_NPURGE_LISTS, ind);
259 }
260 hpdata_purge_list_append(purge_list, ps);
261 }
262
263}
264
265void
266psset_update_begin(psset_t *psset, hpdata_t *ps) {
267 hpdata_assert_consistent(ps);
268 assert(hpdata_in_psset_get(ps));
269 hpdata_updating_set(ps, true);
270 psset_stats_remove(psset, ps);
271 if (hpdata_in_psset_alloc_container_get(ps)) {
272 /*
273 * Some metadata updates can break alloc container invariants
274 * (e.g. the longest free range determines the hpdata_heap_t the
275 * pageslab lives in).
276 */
277 assert(hpdata_alloc_allowed_get(ps));
278 psset_alloc_container_remove(psset, ps);
279 }
280 psset_maybe_remove_purge_list(psset, ps);
281 /*
282 * We don't update presence in the hugify list; we try to keep it FIFO,
283 * even in the presence of other metadata updates. We'll update
284 * presence at the end of the metadata update if necessary.
285 */
286}
287
288void
289psset_update_end(psset_t *psset, hpdata_t *ps) {
290 assert(hpdata_in_psset_get(ps));
291 hpdata_updating_set(ps, false);
292 psset_stats_insert(psset, ps);
293
294 /*
295 * The update begin should have removed ps from whatever alloc container
296 * it was in.
297 */
298 assert(!hpdata_in_psset_alloc_container_get(ps));
299 if (hpdata_alloc_allowed_get(ps)) {
300 psset_alloc_container_insert(psset, ps);
301 }
302 psset_maybe_insert_purge_list(psset, ps);
303
304 if (hpdata_hugify_allowed_get(ps)
305 && !hpdata_in_psset_hugify_container_get(ps)) {
306 hpdata_in_psset_hugify_container_set(ps, true);
307 hpdata_hugify_list_append(&psset->to_hugify, ps);
308 } else if (!hpdata_hugify_allowed_get(ps)
309 && hpdata_in_psset_hugify_container_get(ps)) {
310 hpdata_in_psset_hugify_container_set(ps, false);
311 hpdata_hugify_list_remove(&psset->to_hugify, ps);
312 }
313 hpdata_assert_consistent(ps);
314}
315
316hpdata_t *
317psset_pick_alloc(psset_t *psset, size_t size) {
318 assert((size & PAGE_MASK) == 0);
319 assert(size <= HUGEPAGE);
320
321 pszind_t min_pind = sz_psz2ind(sz_psz_quantize_ceil(size));
322 pszind_t pind = (pszind_t)fb_ffs(psset->pageslab_bitmap, PSSET_NPSIZES,
323 (size_t)min_pind);
324 if (pind == PSSET_NPSIZES) {
325 return hpdata_empty_list_first(&psset->empty);
326 }
327 hpdata_t *ps = hpdata_age_heap_first(&psset->pageslabs[pind]);
328 if (ps == NULL) {
329 return NULL;
330 }
331
332 hpdata_assert_consistent(ps);
333
334 return ps;
335}
336
337hpdata_t *
338psset_pick_purge(psset_t *psset) {
339 ssize_t ind_ssz = fb_fls(psset->purge_bitmap, PSSET_NPURGE_LISTS,
340 PSSET_NPURGE_LISTS - 1);
341 if (ind_ssz < 0) {
342 return NULL;
343 }
344 pszind_t ind = (pszind_t)ind_ssz;
345 assert(ind < PSSET_NPURGE_LISTS);
346 hpdata_t *ps = hpdata_purge_list_first(&psset->to_purge[ind]);
347 assert(ps != NULL);
348 return ps;
349}
350
351hpdata_t *
352psset_pick_hugify(psset_t *psset) {
353 return hpdata_hugify_list_first(&psset->to_hugify);
354}
355
356void
357psset_insert(psset_t *psset, hpdata_t *ps) {
358 hpdata_in_psset_set(ps, true);
359
360 psset_stats_insert(psset, ps);
361 if (hpdata_alloc_allowed_get(ps)) {
362 psset_alloc_container_insert(psset, ps);
363 }
364 psset_maybe_insert_purge_list(psset, ps);
365
366 if (hpdata_hugify_allowed_get(ps)) {
367 hpdata_in_psset_hugify_container_set(ps, true);
368 hpdata_hugify_list_append(&psset->to_hugify, ps);
369 }
370}
371
372void
373psset_remove(psset_t *psset, hpdata_t *ps) {
374 hpdata_in_psset_set(ps, false);
375
376 psset_stats_remove(psset, ps);
377 if (hpdata_in_psset_alloc_container_get(ps)) {
378 psset_alloc_container_remove(psset, ps);
379 }
380 psset_maybe_remove_purge_list(psset, ps);
381 if (hpdata_in_psset_hugify_container_get(ps)) {
382 hpdata_in_psset_hugify_container_set(ps, false);
383 hpdata_hugify_list_remove(&psset->to_hugify, ps);
384 }
385}
diff --git a/examples/redis-unstable/deps/jemalloc/src/rtree.c b/examples/redis-unstable/deps/jemalloc/src/rtree.c
deleted file mode 100644
index 6496b5a..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/rtree.c
+++ /dev/null
@@ -1,261 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/mutex.h"
6
7/*
8 * Only the most significant bits of keys passed to rtree_{read,write}() are
9 * used.
10 */
11bool
12rtree_new(rtree_t *rtree, base_t *base, bool zeroed) {
13#ifdef JEMALLOC_JET
14 if (!zeroed) {
15 memset(rtree, 0, sizeof(rtree_t)); /* Clear root. */
16 }
17#else
18 assert(zeroed);
19#endif
20 rtree->base = base;
21
22 if (malloc_mutex_init(&rtree->init_lock, "rtree", WITNESS_RANK_RTREE,
23 malloc_mutex_rank_exclusive)) {
24 return true;
25 }
26
27 return false;
28}
29
30static rtree_node_elm_t *
31rtree_node_alloc(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) {
32 return (rtree_node_elm_t *)base_alloc(tsdn, rtree->base,
33 nelms * sizeof(rtree_node_elm_t), CACHELINE);
34}
35
36static rtree_leaf_elm_t *
37rtree_leaf_alloc(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) {
38 return (rtree_leaf_elm_t *)base_alloc(tsdn, rtree->base,
39 nelms * sizeof(rtree_leaf_elm_t), CACHELINE);
40}
41
42static rtree_node_elm_t *
43rtree_node_init(tsdn_t *tsdn, rtree_t *rtree, unsigned level,
44 atomic_p_t *elmp) {
45 malloc_mutex_lock(tsdn, &rtree->init_lock);
46 /*
47 * If *elmp is non-null, then it was initialized with the init lock
48 * held, so we can get by with 'relaxed' here.
49 */
50 rtree_node_elm_t *node = atomic_load_p(elmp, ATOMIC_RELAXED);
51 if (node == NULL) {
52 node = rtree_node_alloc(tsdn, rtree, ZU(1) <<
53 rtree_levels[level].bits);
54 if (node == NULL) {
55 malloc_mutex_unlock(tsdn, &rtree->init_lock);
56 return NULL;
57 }
58 /*
59 * Even though we hold the lock, a later reader might not; we
60 * need release semantics.
61 */
62 atomic_store_p(elmp, node, ATOMIC_RELEASE);
63 }
64 malloc_mutex_unlock(tsdn, &rtree->init_lock);
65
66 return node;
67}
68
69static rtree_leaf_elm_t *
70rtree_leaf_init(tsdn_t *tsdn, rtree_t *rtree, atomic_p_t *elmp) {
71 malloc_mutex_lock(tsdn, &rtree->init_lock);
72 /*
73 * If *elmp is non-null, then it was initialized with the init lock
74 * held, so we can get by with 'relaxed' here.
75 */
76 rtree_leaf_elm_t *leaf = atomic_load_p(elmp, ATOMIC_RELAXED);
77 if (leaf == NULL) {
78 leaf = rtree_leaf_alloc(tsdn, rtree, ZU(1) <<
79 rtree_levels[RTREE_HEIGHT-1].bits);
80 if (leaf == NULL) {
81 malloc_mutex_unlock(tsdn, &rtree->init_lock);
82 return NULL;
83 }
84 /*
85 * Even though we hold the lock, a later reader might not; we
86 * need release semantics.
87 */
88 atomic_store_p(elmp, leaf, ATOMIC_RELEASE);
89 }
90 malloc_mutex_unlock(tsdn, &rtree->init_lock);
91
92 return leaf;
93}
94
95static bool
96rtree_node_valid(rtree_node_elm_t *node) {
97 return ((uintptr_t)node != (uintptr_t)0);
98}
99
100static bool
101rtree_leaf_valid(rtree_leaf_elm_t *leaf) {
102 return ((uintptr_t)leaf != (uintptr_t)0);
103}
104
105static rtree_node_elm_t *
106rtree_child_node_tryread(rtree_node_elm_t *elm, bool dependent) {
107 rtree_node_elm_t *node;
108
109 if (dependent) {
110 node = (rtree_node_elm_t *)atomic_load_p(&elm->child,
111 ATOMIC_RELAXED);
112 } else {
113 node = (rtree_node_elm_t *)atomic_load_p(&elm->child,
114 ATOMIC_ACQUIRE);
115 }
116
117 assert(!dependent || node != NULL);
118 return node;
119}
120
121static rtree_node_elm_t *
122rtree_child_node_read(tsdn_t *tsdn, rtree_t *rtree, rtree_node_elm_t *elm,
123 unsigned level, bool dependent) {
124 rtree_node_elm_t *node;
125
126 node = rtree_child_node_tryread(elm, dependent);
127 if (!dependent && unlikely(!rtree_node_valid(node))) {
128 node = rtree_node_init(tsdn, rtree, level + 1, &elm->child);
129 }
130 assert(!dependent || node != NULL);
131 return node;
132}
133
134static rtree_leaf_elm_t *
135rtree_child_leaf_tryread(rtree_node_elm_t *elm, bool dependent) {
136 rtree_leaf_elm_t *leaf;
137
138 if (dependent) {
139 leaf = (rtree_leaf_elm_t *)atomic_load_p(&elm->child,
140 ATOMIC_RELAXED);
141 } else {
142 leaf = (rtree_leaf_elm_t *)atomic_load_p(&elm->child,
143 ATOMIC_ACQUIRE);
144 }
145
146 assert(!dependent || leaf != NULL);
147 return leaf;
148}
149
150static rtree_leaf_elm_t *
151rtree_child_leaf_read(tsdn_t *tsdn, rtree_t *rtree, rtree_node_elm_t *elm,
152 unsigned level, bool dependent) {
153 rtree_leaf_elm_t *leaf;
154
155 leaf = rtree_child_leaf_tryread(elm, dependent);
156 if (!dependent && unlikely(!rtree_leaf_valid(leaf))) {
157 leaf = rtree_leaf_init(tsdn, rtree, &elm->child);
158 }
159 assert(!dependent || leaf != NULL);
160 return leaf;
161}
162
163rtree_leaf_elm_t *
164rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
165 uintptr_t key, bool dependent, bool init_missing) {
166 rtree_node_elm_t *node;
167 rtree_leaf_elm_t *leaf;
168#if RTREE_HEIGHT > 1
169 node = rtree->root;
170#else
171 leaf = rtree->root;
172#endif
173
174 if (config_debug) {
175 uintptr_t leafkey = rtree_leafkey(key);
176 for (unsigned i = 0; i < RTREE_CTX_NCACHE; i++) {
177 assert(rtree_ctx->cache[i].leafkey != leafkey);
178 }
179 for (unsigned i = 0; i < RTREE_CTX_NCACHE_L2; i++) {
180 assert(rtree_ctx->l2_cache[i].leafkey != leafkey);
181 }
182 }
183
184#define RTREE_GET_CHILD(level) { \
185 assert(level < RTREE_HEIGHT-1); \
186 if (level != 0 && !dependent && \
187 unlikely(!rtree_node_valid(node))) { \
188 return NULL; \
189 } \
190 uintptr_t subkey = rtree_subkey(key, level); \
191 if (level + 2 < RTREE_HEIGHT) { \
192 node = init_missing ? \
193 rtree_child_node_read(tsdn, rtree, \
194 &node[subkey], level, dependent) : \
195 rtree_child_node_tryread(&node[subkey], \
196 dependent); \
197 } else { \
198 leaf = init_missing ? \
199 rtree_child_leaf_read(tsdn, rtree, \
200 &node[subkey], level, dependent) : \
201 rtree_child_leaf_tryread(&node[subkey], \
202 dependent); \
203 } \
204 }
205 /*
206 * Cache replacement upon hard lookup (i.e. L1 & L2 rtree cache miss):
207 * (1) evict last entry in L2 cache; (2) move the collision slot from L1
208 * cache down to L2; and 3) fill L1.
209 */
210#define RTREE_GET_LEAF(level) { \
211 assert(level == RTREE_HEIGHT-1); \
212 if (!dependent && unlikely(!rtree_leaf_valid(leaf))) { \
213 return NULL; \
214 } \
215 if (RTREE_CTX_NCACHE_L2 > 1) { \
216 memmove(&rtree_ctx->l2_cache[1], \
217 &rtree_ctx->l2_cache[0], \
218 sizeof(rtree_ctx_cache_elm_t) * \
219 (RTREE_CTX_NCACHE_L2 - 1)); \
220 } \
221 size_t slot = rtree_cache_direct_map(key); \
222 rtree_ctx->l2_cache[0].leafkey = \
223 rtree_ctx->cache[slot].leafkey; \
224 rtree_ctx->l2_cache[0].leaf = \
225 rtree_ctx->cache[slot].leaf; \
226 uintptr_t leafkey = rtree_leafkey(key); \
227 rtree_ctx->cache[slot].leafkey = leafkey; \
228 rtree_ctx->cache[slot].leaf = leaf; \
229 uintptr_t subkey = rtree_subkey(key, level); \
230 return &leaf[subkey]; \
231 }
232 if (RTREE_HEIGHT > 1) {
233 RTREE_GET_CHILD(0)
234 }
235 if (RTREE_HEIGHT > 2) {
236 RTREE_GET_CHILD(1)
237 }
238 if (RTREE_HEIGHT > 3) {
239 for (unsigned i = 2; i < RTREE_HEIGHT-1; i++) {
240 RTREE_GET_CHILD(i)
241 }
242 }
243 RTREE_GET_LEAF(RTREE_HEIGHT-1)
244#undef RTREE_GET_CHILD
245#undef RTREE_GET_LEAF
246 not_reached();
247}
248
249void
250rtree_ctx_data_init(rtree_ctx_t *ctx) {
251 for (unsigned i = 0; i < RTREE_CTX_NCACHE; i++) {
252 rtree_ctx_cache_elm_t *cache = &ctx->cache[i];
253 cache->leafkey = RTREE_LEAFKEY_INVALID;
254 cache->leaf = NULL;
255 }
256 for (unsigned i = 0; i < RTREE_CTX_NCACHE_L2; i++) {
257 rtree_ctx_cache_elm_t *cache = &ctx->l2_cache[i];
258 cache->leafkey = RTREE_LEAFKEY_INVALID;
259 cache->leaf = NULL;
260 }
261}
diff --git a/examples/redis-unstable/deps/jemalloc/src/safety_check.c b/examples/redis-unstable/deps/jemalloc/src/safety_check.c
deleted file mode 100644
index 209fdda..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/safety_check.c
+++ /dev/null
@@ -1,36 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4static safety_check_abort_hook_t safety_check_abort;
5
6void safety_check_fail_sized_dealloc(bool current_dealloc, const void *ptr,
7 size_t true_size, size_t input_size) {
8 char *src = current_dealloc ? "the current pointer being freed" :
9 "in thread cache, possibly from previous deallocations";
10
11 safety_check_fail("<jemalloc>: size mismatch detected (true size %zu "
12 "vs input size %zu), likely caused by application sized "
13 "deallocation bugs (source address: %p, %s). Suggest building with "
14 "--enable-debug or address sanitizer for debugging. Abort.\n",
15 true_size, input_size, ptr, src);
16}
17
18void safety_check_set_abort(safety_check_abort_hook_t abort_fn) {
19 safety_check_abort = abort_fn;
20}
21
22void safety_check_fail(const char *format, ...) {
23 char buf[MALLOC_PRINTF_BUFSIZE];
24
25 va_list ap;
26 va_start(ap, format);
27 malloc_vsnprintf(buf, MALLOC_PRINTF_BUFSIZE, format, ap);
28 va_end(ap);
29
30 if (safety_check_abort == NULL) {
31 malloc_write(buf);
32 abort();
33 } else {
34 safety_check_abort(buf);
35 }
36}
diff --git a/examples/redis-unstable/deps/jemalloc/src/san.c b/examples/redis-unstable/deps/jemalloc/src/san.c
deleted file mode 100644
index 6e51291..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/san.c
+++ /dev/null
@@ -1,208 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/ehooks.h"
6#include "jemalloc/internal/san.h"
7#include "jemalloc/internal/tsd.h"
8
9/* The sanitizer options. */
10size_t opt_san_guard_large = SAN_GUARD_LARGE_EVERY_N_EXTENTS_DEFAULT;
11size_t opt_san_guard_small = SAN_GUARD_SMALL_EVERY_N_EXTENTS_DEFAULT;
12
13/* Aligned (-1 is off) ptrs will be junked & stashed on dealloc. */
14ssize_t opt_lg_san_uaf_align = SAN_LG_UAF_ALIGN_DEFAULT;
15
16/*
17 * Initialized in san_init(). When disabled, the mask is set to (uintptr_t)-1
18 * to always fail the nonfast_align check.
19 */
20uintptr_t san_cache_bin_nonfast_mask = SAN_CACHE_BIN_NONFAST_MASK_DEFAULT;
21
22static inline void
23san_find_guarded_addr(edata_t *edata, uintptr_t *guard1, uintptr_t *guard2,
24 uintptr_t *addr, size_t size, bool left, bool right) {
25 assert(!edata_guarded_get(edata));
26 assert(size % PAGE == 0);
27 *addr = (uintptr_t)edata_base_get(edata);
28 if (left) {
29 *guard1 = *addr;
30 *addr += SAN_PAGE_GUARD;
31 } else {
32 *guard1 = 0;
33 }
34
35 if (right) {
36 *guard2 = *addr + size;
37 } else {
38 *guard2 = 0;
39 }
40}
41
42static inline void
43san_find_unguarded_addr(edata_t *edata, uintptr_t *guard1, uintptr_t *guard2,
44 uintptr_t *addr, size_t size, bool left, bool right) {
45 assert(edata_guarded_get(edata));
46 assert(size % PAGE == 0);
47 *addr = (uintptr_t)edata_base_get(edata);
48 if (right) {
49 *guard2 = *addr + size;
50 } else {
51 *guard2 = 0;
52 }
53
54 if (left) {
55 *guard1 = *addr - SAN_PAGE_GUARD;
56 assert(*guard1 != 0);
57 *addr = *guard1;
58 } else {
59 *guard1 = 0;
60 }
61}
62
63void
64san_guard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap,
65 bool left, bool right, bool remap) {
66 assert(left || right);
67 if (remap) {
68 emap_deregister_boundary(tsdn, emap, edata);
69 }
70
71 size_t size_with_guards = edata_size_get(edata);
72 size_t usize = (left && right)
73 ? san_two_side_unguarded_sz(size_with_guards)
74 : san_one_side_unguarded_sz(size_with_guards);
75
76 uintptr_t guard1, guard2, addr;
77 san_find_guarded_addr(edata, &guard1, &guard2, &addr, usize, left,
78 right);
79
80 assert(edata_state_get(edata) == extent_state_active);
81 ehooks_guard(tsdn, ehooks, (void *)guard1, (void *)guard2);
82
83 /* Update the guarded addr and usable size of the edata. */
84 edata_size_set(edata, usize);
85 edata_addr_set(edata, (void *)addr);
86 edata_guarded_set(edata, true);
87
88 if (remap) {
89 emap_register_boundary(tsdn, emap, edata, SC_NSIZES,
90 /* slab */ false);
91 }
92}
93
94static void
95san_unguard_pages_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
96 emap_t *emap, bool left, bool right, bool remap) {
97 assert(left || right);
98 /* Remove the inner boundary which no longer exists. */
99 if (remap) {
100 assert(edata_state_get(edata) == extent_state_active);
101 emap_deregister_boundary(tsdn, emap, edata);
102 } else {
103 assert(edata_state_get(edata) == extent_state_retained);
104 }
105
106 size_t size = edata_size_get(edata);
107 size_t size_with_guards = (left && right)
108 ? san_two_side_guarded_sz(size)
109 : san_one_side_guarded_sz(size);
110
111 uintptr_t guard1, guard2, addr;
112 san_find_unguarded_addr(edata, &guard1, &guard2, &addr, size, left,
113 right);
114
115 ehooks_unguard(tsdn, ehooks, (void *)guard1, (void *)guard2);
116
117 /* Update the true addr and usable size of the edata. */
118 edata_size_set(edata, size_with_guards);
119 edata_addr_set(edata, (void *)addr);
120 edata_guarded_set(edata, false);
121
122 /*
123 * Then re-register the outer boundary including the guards, if
124 * requested.
125 */
126 if (remap) {
127 emap_register_boundary(tsdn, emap, edata, SC_NSIZES,
128 /* slab */ false);
129 }
130}
131
132void
133san_unguard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
134 emap_t *emap, bool left, bool right) {
135 san_unguard_pages_impl(tsdn, ehooks, edata, emap, left, right,
136 /* remap */ true);
137}
138
139void
140san_unguard_pages_pre_destroy(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
141 emap_t *emap) {
142 emap_assert_not_mapped(tsdn, emap, edata);
143 /*
144 * We don't want to touch the emap of about to be destroyed extents, as
145 * they have been unmapped upon eviction from the retained ecache. Also,
146 * we unguard the extents to the right, because retained extents only
147 * own their right guard page per san_bump_alloc's logic.
148 */
149 san_unguard_pages_impl(tsdn, ehooks, edata, emap, /* left */ false,
150 /* right */ true, /* remap */ false);
151}
152
153static bool
154san_stashed_corrupted(void *ptr, size_t size) {
155 if (san_junk_ptr_should_slow()) {
156 for (size_t i = 0; i < size; i++) {
157 if (((char *)ptr)[i] != (char)uaf_detect_junk) {
158 return true;
159 }
160 }
161 return false;
162 }
163
164 void *first, *mid, *last;
165 san_junk_ptr_locations(ptr, size, &first, &mid, &last);
166 if (*(uintptr_t *)first != uaf_detect_junk ||
167 *(uintptr_t *)mid != uaf_detect_junk ||
168 *(uintptr_t *)last != uaf_detect_junk) {
169 return true;
170 }
171
172 return false;
173}
174
175void
176san_check_stashed_ptrs(void **ptrs, size_t nstashed, size_t usize) {
177 /*
178 * Verify that the junked-filled & stashed pointers remain unchanged, to
179 * detect write-after-free.
180 */
181 for (size_t n = 0; n < nstashed; n++) {
182 void *stashed = ptrs[n];
183 assert(stashed != NULL);
184 assert(cache_bin_nonfast_aligned(stashed));
185 if (unlikely(san_stashed_corrupted(stashed, usize))) {
186 safety_check_fail("<jemalloc>: Write-after-free "
187 "detected on deallocated pointer %p (size %zu).\n",
188 stashed, usize);
189 }
190 }
191}
192
193void
194tsd_san_init(tsd_t *tsd) {
195 *tsd_san_extents_until_guard_smallp_get(tsd) = opt_san_guard_small;
196 *tsd_san_extents_until_guard_largep_get(tsd) = opt_san_guard_large;
197}
198
199void
200san_init(ssize_t lg_san_uaf_align) {
201 assert(lg_san_uaf_align == -1 || lg_san_uaf_align >= LG_PAGE);
202 if (lg_san_uaf_align == -1) {
203 san_cache_bin_nonfast_mask = (uintptr_t)-1;
204 return;
205 }
206
207 san_cache_bin_nonfast_mask = ((uintptr_t)1 << lg_san_uaf_align) - 1;
208}
diff --git a/examples/redis-unstable/deps/jemalloc/src/san_bump.c b/examples/redis-unstable/deps/jemalloc/src/san_bump.c
deleted file mode 100644
index 8889745..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/san_bump.c
+++ /dev/null
@@ -1,104 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/san_bump.h"
5#include "jemalloc/internal/pac.h"
6#include "jemalloc/internal/san.h"
7#include "jemalloc/internal/ehooks.h"
8#include "jemalloc/internal/edata_cache.h"
9
10static bool
11san_bump_grow_locked(tsdn_t *tsdn, san_bump_alloc_t *sba, pac_t *pac,
12 ehooks_t *ehooks, size_t size);
13
14edata_t *
15san_bump_alloc(tsdn_t *tsdn, san_bump_alloc_t* sba, pac_t *pac,
16 ehooks_t *ehooks, size_t size, bool zero) {
17 assert(san_bump_enabled());
18
19 edata_t* to_destroy;
20 size_t guarded_size = san_one_side_guarded_sz(size);
21
22 malloc_mutex_lock(tsdn, &sba->mtx);
23
24 if (sba->curr_reg == NULL ||
25 edata_size_get(sba->curr_reg) < guarded_size) {
26 /*
27 * If the current region can't accommodate the allocation,
28 * try replacing it with a larger one and destroy current if the
29 * replacement succeeds.
30 */
31 to_destroy = sba->curr_reg;
32 bool err = san_bump_grow_locked(tsdn, sba, pac, ehooks,
33 guarded_size);
34 if (err) {
35 goto label_err;
36 }
37 } else {
38 to_destroy = NULL;
39 }
40 assert(guarded_size <= edata_size_get(sba->curr_reg));
41 size_t trail_size = edata_size_get(sba->curr_reg) - guarded_size;
42
43 edata_t* edata;
44 if (trail_size != 0) {
45 edata_t* curr_reg_trail = extent_split_wrapper(tsdn, pac,
46 ehooks, sba->curr_reg, guarded_size, trail_size,
47 /* holding_core_locks */ true);
48 if (curr_reg_trail == NULL) {
49 goto label_err;
50 }
51 edata = sba->curr_reg;
52 sba->curr_reg = curr_reg_trail;
53 } else {
54 edata = sba->curr_reg;
55 sba->curr_reg = NULL;
56 }
57
58 malloc_mutex_unlock(tsdn, &sba->mtx);
59
60 assert(!edata_guarded_get(edata));
61 assert(sba->curr_reg == NULL || !edata_guarded_get(sba->curr_reg));
62 assert(to_destroy == NULL || !edata_guarded_get(to_destroy));
63
64 if (to_destroy != NULL) {
65 extent_destroy_wrapper(tsdn, pac, ehooks, to_destroy);
66 }
67
68 san_guard_pages(tsdn, ehooks, edata, pac->emap, /* left */ false,
69 /* right */ true, /* remap */ true);
70
71 if (extent_commit_zero(tsdn, ehooks, edata, /* commit */ true, zero,
72 /* growing_retained */ false)) {
73 extent_record(tsdn, pac, ehooks, &pac->ecache_retained,
74 edata);
75 return NULL;
76 }
77
78 if (config_prof) {
79 extent_gdump_add(tsdn, edata);
80 }
81
82 return edata;
83label_err:
84 malloc_mutex_unlock(tsdn, &sba->mtx);
85 return NULL;
86}
87
88static bool
89san_bump_grow_locked(tsdn_t *tsdn, san_bump_alloc_t *sba, pac_t *pac,
90 ehooks_t *ehooks, size_t size) {
91 malloc_mutex_assert_owner(tsdn, &sba->mtx);
92
93 bool committed = false, zeroed = false;
94 size_t alloc_size = size > SBA_RETAINED_ALLOC_SIZE ? size :
95 SBA_RETAINED_ALLOC_SIZE;
96 assert((alloc_size & PAGE_MASK) == 0);
97 sba->curr_reg = extent_alloc_wrapper(tsdn, pac, ehooks, NULL,
98 alloc_size, PAGE, zeroed, &committed,
99 /* growing_retained */ true);
100 if (sba->curr_reg == NULL) {
101 return true;
102 }
103 return false;
104}
diff --git a/examples/redis-unstable/deps/jemalloc/src/sc.c b/examples/redis-unstable/deps/jemalloc/src/sc.c
deleted file mode 100644
index e4a94d8..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/sc.c
+++ /dev/null
@@ -1,306 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2
3#include "jemalloc/internal/assert.h"
4#include "jemalloc/internal/bit_util.h"
5#include "jemalloc/internal/bitmap.h"
6#include "jemalloc/internal/pages.h"
7#include "jemalloc/internal/sc.h"
8
9/*
10 * This module computes the size classes used to satisfy allocations. The logic
11 * here was ported more or less line-by-line from a shell script, and because of
12 * that is not the most idiomatic C. Eventually we should fix this, but for now
13 * at least the damage is compartmentalized to this file.
14 */
15
16size_t
17reg_size_compute(int lg_base, int lg_delta, int ndelta) {
18 return (ZU(1) << lg_base) + (ZU(ndelta) << lg_delta);
19}
20
21/* Returns the number of pages in the slab. */
22static int
23slab_size(int lg_page, int lg_base, int lg_delta, int ndelta) {
24 size_t page = (ZU(1) << lg_page);
25 size_t reg_size = reg_size_compute(lg_base, lg_delta, ndelta);
26
27 size_t try_slab_size = page;
28 size_t try_nregs = try_slab_size / reg_size;
29 size_t perfect_slab_size = 0;
30 bool perfect = false;
31 /*
32 * This loop continues until we find the least common multiple of the
33 * page size and size class size. Size classes are all of the form
34 * base + ndelta * delta == (ndelta + base/ndelta) * delta, which is
35 * (ndelta + ngroup) * delta. The way we choose slabbing strategies
36 * means that delta is at most the page size and ndelta < ngroup. So
37 * the loop executes for at most 2 * ngroup - 1 iterations, which is
38 * also the bound on the number of pages in a slab chosen by default.
39 * With the current default settings, this is at most 7.
40 */
41 while (!perfect) {
42 perfect_slab_size = try_slab_size;
43 size_t perfect_nregs = try_nregs;
44 try_slab_size += page;
45 try_nregs = try_slab_size / reg_size;
46 if (perfect_slab_size == perfect_nregs * reg_size) {
47 perfect = true;
48 }
49 }
50 return (int)(perfect_slab_size / page);
51}
52
53static void
54size_class(
55 /* Output. */
56 sc_t *sc,
57 /* Configuration decisions. */
58 int lg_max_lookup, int lg_page, int lg_ngroup,
59 /* Inputs specific to the size class. */
60 int index, int lg_base, int lg_delta, int ndelta) {
61 sc->index = index;
62 sc->lg_base = lg_base;
63 sc->lg_delta = lg_delta;
64 sc->ndelta = ndelta;
65 size_t size = reg_size_compute(lg_base, lg_delta, ndelta);
66 sc->psz = (size % (ZU(1) << lg_page) == 0);
67 if (index == 0) {
68 assert(!sc->psz);
69 }
70 if (size < (ZU(1) << (lg_page + lg_ngroup))) {
71 sc->bin = true;
72 sc->pgs = slab_size(lg_page, lg_base, lg_delta, ndelta);
73 } else {
74 sc->bin = false;
75 sc->pgs = 0;
76 }
77 if (size <= (ZU(1) << lg_max_lookup)) {
78 sc->lg_delta_lookup = lg_delta;
79 } else {
80 sc->lg_delta_lookup = 0;
81 }
82}
83
84static void
85size_classes(
86 /* Output. */
87 sc_data_t *sc_data,
88 /* Determined by the system. */
89 size_t lg_ptr_size, int lg_quantum,
90 /* Configuration decisions. */
91 int lg_tiny_min, int lg_max_lookup, int lg_page, int lg_ngroup) {
92 int ptr_bits = (1 << lg_ptr_size) * 8;
93 int ngroup = (1 << lg_ngroup);
94 int ntiny = 0;
95 int nlbins = 0;
96 int lg_tiny_maxclass = (unsigned)-1;
97 int nbins = 0;
98 int npsizes = 0;
99
100 int index = 0;
101
102 int ndelta = 0;
103 int lg_base = lg_tiny_min;
104 int lg_delta = lg_base;
105
106 /* Outputs that we update as we go. */
107 size_t lookup_maxclass = 0;
108 size_t small_maxclass = 0;
109 int lg_large_minclass = 0;
110 size_t large_maxclass = 0;
111
112 /* Tiny size classes. */
113 while (lg_base < lg_quantum) {
114 sc_t *sc = &sc_data->sc[index];
115 size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index,
116 lg_base, lg_delta, ndelta);
117 if (sc->lg_delta_lookup != 0) {
118 nlbins = index + 1;
119 }
120 if (sc->psz) {
121 npsizes++;
122 }
123 if (sc->bin) {
124 nbins++;
125 }
126 ntiny++;
127 /* Final written value is correct. */
128 lg_tiny_maxclass = lg_base;
129 index++;
130 lg_delta = lg_base;
131 lg_base++;
132 }
133
134 /* First non-tiny (pseudo) group. */
135 if (ntiny != 0) {
136 sc_t *sc = &sc_data->sc[index];
137 /*
138 * See the note in sc.h; the first non-tiny size class has an
139 * unusual encoding.
140 */
141 lg_base--;
142 ndelta = 1;
143 size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index,
144 lg_base, lg_delta, ndelta);
145 index++;
146 lg_base++;
147 lg_delta++;
148 if (sc->psz) {
149 npsizes++;
150 }
151 if (sc->bin) {
152 nbins++;
153 }
154 }
155 while (ndelta < ngroup) {
156 sc_t *sc = &sc_data->sc[index];
157 size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index,
158 lg_base, lg_delta, ndelta);
159 index++;
160 ndelta++;
161 if (sc->psz) {
162 npsizes++;
163 }
164 if (sc->bin) {
165 nbins++;
166 }
167 }
168
169 /* All remaining groups. */
170 lg_base = lg_base + lg_ngroup;
171 while (lg_base < ptr_bits - 1) {
172 ndelta = 1;
173 int ndelta_limit;
174 if (lg_base == ptr_bits - 2) {
175 ndelta_limit = ngroup - 1;
176 } else {
177 ndelta_limit = ngroup;
178 }
179 while (ndelta <= ndelta_limit) {
180 sc_t *sc = &sc_data->sc[index];
181 size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index,
182 lg_base, lg_delta, ndelta);
183 if (sc->lg_delta_lookup != 0) {
184 nlbins = index + 1;
185 /* Final written value is correct. */
186 lookup_maxclass = (ZU(1) << lg_base)
187 + (ZU(ndelta) << lg_delta);
188 }
189 if (sc->psz) {
190 npsizes++;
191 }
192 if (sc->bin) {
193 nbins++;
194 /* Final written value is correct. */
195 small_maxclass = (ZU(1) << lg_base)
196 + (ZU(ndelta) << lg_delta);
197 if (lg_ngroup > 0) {
198 lg_large_minclass = lg_base + 1;
199 } else {
200 lg_large_minclass = lg_base + 2;
201 }
202 }
203 large_maxclass = (ZU(1) << lg_base)
204 + (ZU(ndelta) << lg_delta);
205 index++;
206 ndelta++;
207 }
208 lg_base++;
209 lg_delta++;
210 }
211 /* Additional outputs. */
212 int nsizes = index;
213 unsigned lg_ceil_nsizes = lg_ceil(nsizes);
214
215 /* Fill in the output data. */
216 sc_data->ntiny = ntiny;
217 sc_data->nlbins = nlbins;
218 sc_data->nbins = nbins;
219 sc_data->nsizes = nsizes;
220 sc_data->lg_ceil_nsizes = lg_ceil_nsizes;
221 sc_data->npsizes = npsizes;
222 sc_data->lg_tiny_maxclass = lg_tiny_maxclass;
223 sc_data->lookup_maxclass = lookup_maxclass;
224 sc_data->small_maxclass = small_maxclass;
225 sc_data->lg_large_minclass = lg_large_minclass;
226 sc_data->large_minclass = (ZU(1) << lg_large_minclass);
227 sc_data->large_maxclass = large_maxclass;
228
229 /*
230 * We compute these values in two ways:
231 * - Incrementally, as above.
232 * - In macros, in sc.h.
233 * The computation is easier when done incrementally, but putting it in
234 * a constant makes it available to the fast paths without having to
235 * touch the extra global cacheline. We assert, however, that the two
236 * computations are equivalent.
237 */
238 assert(sc_data->npsizes == SC_NPSIZES);
239 assert(sc_data->lg_tiny_maxclass == SC_LG_TINY_MAXCLASS);
240 assert(sc_data->small_maxclass == SC_SMALL_MAXCLASS);
241 assert(sc_data->large_minclass == SC_LARGE_MINCLASS);
242 assert(sc_data->lg_large_minclass == SC_LG_LARGE_MINCLASS);
243 assert(sc_data->large_maxclass == SC_LARGE_MAXCLASS);
244
245 /*
246 * In the allocation fastpath, we want to assume that we can
247 * unconditionally subtract the requested allocation size from
248 * a ssize_t, and detect passing through 0 correctly. This
249 * results in optimal generated code. For this to work, the
250 * maximum allocation size must be less than SSIZE_MAX.
251 */
252 assert(SC_LARGE_MAXCLASS < SSIZE_MAX);
253}
254
255void
256sc_data_init(sc_data_t *sc_data) {
257 size_classes(sc_data, LG_SIZEOF_PTR, LG_QUANTUM, SC_LG_TINY_MIN,
258 SC_LG_MAX_LOOKUP, LG_PAGE, SC_LG_NGROUP);
259
260 sc_data->initialized = true;
261}
262
263static void
264sc_data_update_sc_slab_size(sc_t *sc, size_t reg_size, size_t pgs_guess) {
265 size_t min_pgs = reg_size / PAGE;
266 if (reg_size % PAGE != 0) {
267 min_pgs++;
268 }
269 /*
270 * BITMAP_MAXBITS is actually determined by putting the smallest
271 * possible size-class on one page, so this can never be 0.
272 */
273 size_t max_pgs = BITMAP_MAXBITS * reg_size / PAGE;
274
275 assert(min_pgs <= max_pgs);
276 assert(min_pgs > 0);
277 assert(max_pgs >= 1);
278 if (pgs_guess < min_pgs) {
279 sc->pgs = (int)min_pgs;
280 } else if (pgs_guess > max_pgs) {
281 sc->pgs = (int)max_pgs;
282 } else {
283 sc->pgs = (int)pgs_guess;
284 }
285}
286
287void
288sc_data_update_slab_size(sc_data_t *data, size_t begin, size_t end, int pgs) {
289 assert(data->initialized);
290 for (int i = 0; i < data->nsizes; i++) {
291 sc_t *sc = &data->sc[i];
292 if (!sc->bin) {
293 break;
294 }
295 size_t reg_size = reg_size_compute(sc->lg_base, sc->lg_delta,
296 sc->ndelta);
297 if (begin <= reg_size && reg_size <= end) {
298 sc_data_update_sc_slab_size(sc, reg_size, pgs);
299 }
300 }
301}
302
303void
304sc_boot(sc_data_t *data) {
305 sc_data_init(data);
306}
diff --git a/examples/redis-unstable/deps/jemalloc/src/sec.c b/examples/redis-unstable/deps/jemalloc/src/sec.c
deleted file mode 100644
index df67559..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/sec.c
+++ /dev/null
@@ -1,422 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/sec.h"
5
6static edata_t *sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
7 size_t alignment, bool zero, bool guarded, bool frequent_reuse,
8 bool *deferred_work_generated);
9static bool sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
10 size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
11static bool sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
12 size_t old_size, size_t new_size, bool *deferred_work_generated);
13static void sec_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata,
14 bool *deferred_work_generated);
15
16static void
17sec_bin_init(sec_bin_t *bin) {
18 bin->being_batch_filled = false;
19 bin->bytes_cur = 0;
20 edata_list_active_init(&bin->freelist);
21}
22
23bool
24sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, pai_t *fallback,
25 const sec_opts_t *opts) {
26 assert(opts->max_alloc >= PAGE);
27
28 size_t max_alloc = PAGE_FLOOR(opts->max_alloc);
29 pszind_t npsizes = sz_psz2ind(max_alloc) + 1;
30
31 size_t sz_shards = opts->nshards * sizeof(sec_shard_t);
32 size_t sz_bins = opts->nshards * (size_t)npsizes * sizeof(sec_bin_t);
33 size_t sz_alloc = sz_shards + sz_bins;
34 void *dynalloc = base_alloc(tsdn, base, sz_alloc, CACHELINE);
35 if (dynalloc == NULL) {
36 return true;
37 }
38 sec_shard_t *shard_cur = (sec_shard_t *)dynalloc;
39 sec->shards = shard_cur;
40 sec_bin_t *bin_cur = (sec_bin_t *)&shard_cur[opts->nshards];
41 /* Just for asserts, below. */
42 sec_bin_t *bin_start = bin_cur;
43
44 for (size_t i = 0; i < opts->nshards; i++) {
45 sec_shard_t *shard = shard_cur;
46 shard_cur++;
47 bool err = malloc_mutex_init(&shard->mtx, "sec_shard",
48 WITNESS_RANK_SEC_SHARD, malloc_mutex_rank_exclusive);
49 if (err) {
50 return true;
51 }
52 shard->enabled = true;
53 shard->bins = bin_cur;
54 for (pszind_t j = 0; j < npsizes; j++) {
55 sec_bin_init(&shard->bins[j]);
56 bin_cur++;
57 }
58 shard->bytes_cur = 0;
59 shard->to_flush_next = 0;
60 }
61 /*
62 * Should have exactly matched the bin_start to the first unused byte
63 * after the shards.
64 */
65 assert((void *)shard_cur == (void *)bin_start);
66 /* And the last bin to use up the last bytes of the allocation. */
67 assert((char *)bin_cur == ((char *)dynalloc + sz_alloc));
68 sec->fallback = fallback;
69
70
71 sec->opts = *opts;
72 sec->npsizes = npsizes;
73
74 /*
75 * Initialize these last so that an improper use of an SEC whose
76 * initialization failed will segfault in an easy-to-spot way.
77 */
78 sec->pai.alloc = &sec_alloc;
79 sec->pai.alloc_batch = &pai_alloc_batch_default;
80 sec->pai.expand = &sec_expand;
81 sec->pai.shrink = &sec_shrink;
82 sec->pai.dalloc = &sec_dalloc;
83 sec->pai.dalloc_batch = &pai_dalloc_batch_default;
84
85 return false;
86}
87
88static sec_shard_t *
89sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
90 /*
91 * Eventually, we should implement affinity, tracking source shard using
92 * the edata_t's newly freed up fields. For now, just randomly
93 * distribute across all shards.
94 */
95 if (tsdn_null(tsdn)) {
96 return &sec->shards[0];
97 }
98 tsd_t *tsd = tsdn_tsd(tsdn);
99 uint8_t *idxp = tsd_sec_shardp_get(tsd);
100 if (*idxp == (uint8_t)-1) {
101 /*
102 * First use; initialize using the trick from Daniel Lemire's
103 * "A fast alternative to the modulo reduction. Use a 64 bit
104 * number to store 32 bits, since we'll deliberately overflow
105 * when we multiply by the number of shards.
106 */
107 uint64_t rand32 = prng_lg_range_u64(tsd_prng_statep_get(tsd), 32);
108 uint32_t idx =
109 (uint32_t)((rand32 * (uint64_t)sec->opts.nshards) >> 32);
110 assert(idx < (uint32_t)sec->opts.nshards);
111 *idxp = (uint8_t)idx;
112 }
113 return &sec->shards[*idxp];
114}
115
116/*
117 * Perhaps surprisingly, this can be called on the alloc pathways; if we hit an
118 * empty cache, we'll try to fill it, which can push the shard over it's limit.
119 */
120static void
121sec_flush_some_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) {
122 malloc_mutex_assert_owner(tsdn, &shard->mtx);
123 edata_list_active_t to_flush;
124 edata_list_active_init(&to_flush);
125 while (shard->bytes_cur > sec->opts.bytes_after_flush) {
126 /* Pick a victim. */
127 sec_bin_t *bin = &shard->bins[shard->to_flush_next];
128
129 /* Update our victim-picking state. */
130 shard->to_flush_next++;
131 if (shard->to_flush_next == sec->npsizes) {
132 shard->to_flush_next = 0;
133 }
134
135 assert(shard->bytes_cur >= bin->bytes_cur);
136 if (bin->bytes_cur != 0) {
137 shard->bytes_cur -= bin->bytes_cur;
138 bin->bytes_cur = 0;
139 edata_list_active_concat(&to_flush, &bin->freelist);
140 }
141 /*
142 * Either bin->bytes_cur was 0, in which case we didn't touch
143 * the bin list but it should be empty anyways (or else we
144 * missed a bytes_cur update on a list modification), or it
145 * *was* 0 and we emptied it ourselves. Either way, it should
146 * be empty now.
147 */
148 assert(edata_list_active_empty(&bin->freelist));
149 }
150
151 malloc_mutex_unlock(tsdn, &shard->mtx);
152 bool deferred_work_generated = false;
153 pai_dalloc_batch(tsdn, sec->fallback, &to_flush,
154 &deferred_work_generated);
155}
156
157static edata_t *
158sec_shard_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard,
159 sec_bin_t *bin) {
160 malloc_mutex_assert_owner(tsdn, &shard->mtx);
161 if (!shard->enabled) {
162 return NULL;
163 }
164 edata_t *edata = edata_list_active_first(&bin->freelist);
165 if (edata != NULL) {
166 edata_list_active_remove(&bin->freelist, edata);
167 assert(edata_size_get(edata) <= bin->bytes_cur);
168 bin->bytes_cur -= edata_size_get(edata);
169 assert(edata_size_get(edata) <= shard->bytes_cur);
170 shard->bytes_cur -= edata_size_get(edata);
171 }
172 return edata;
173}
174
175static edata_t *
176sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard,
177 sec_bin_t *bin, size_t size) {
178 malloc_mutex_assert_not_owner(tsdn, &shard->mtx);
179
180 edata_list_active_t result;
181 edata_list_active_init(&result);
182 bool deferred_work_generated = false;
183 size_t nalloc = pai_alloc_batch(tsdn, sec->fallback, size,
184 1 + sec->opts.batch_fill_extra, &result, &deferred_work_generated);
185
186 edata_t *ret = edata_list_active_first(&result);
187 if (ret != NULL) {
188 edata_list_active_remove(&result, ret);
189 }
190
191 malloc_mutex_lock(tsdn, &shard->mtx);
192 bin->being_batch_filled = false;
193 /*
194 * Handle the easy case first: nothing to cache. Note that this can
195 * only happen in case of OOM, since sec_alloc checks the expected
196 * number of allocs, and doesn't bother going down the batch_fill
197 * pathway if there won't be anything left to cache. So to be in this
198 * code path, we must have asked for > 1 alloc, but only gotten 1 back.
199 */
200 if (nalloc <= 1) {
201 malloc_mutex_unlock(tsdn, &shard->mtx);
202 return ret;
203 }
204
205 size_t new_cached_bytes = (nalloc - 1) * size;
206
207 edata_list_active_concat(&bin->freelist, &result);
208 bin->bytes_cur += new_cached_bytes;
209 shard->bytes_cur += new_cached_bytes;
210
211 if (shard->bytes_cur > sec->opts.max_bytes) {
212 sec_flush_some_and_unlock(tsdn, sec, shard);
213 } else {
214 malloc_mutex_unlock(tsdn, &shard->mtx);
215 }
216
217 return ret;
218}
219
220static edata_t *
221sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
222 bool guarded, bool frequent_reuse, bool *deferred_work_generated) {
223 assert((size & PAGE_MASK) == 0);
224 assert(!guarded);
225
226 sec_t *sec = (sec_t *)self;
227
228 if (zero || alignment > PAGE || sec->opts.nshards == 0
229 || size > sec->opts.max_alloc) {
230 return pai_alloc(tsdn, sec->fallback, size, alignment, zero,
231 /* guarded */ false, frequent_reuse,
232 deferred_work_generated);
233 }
234 pszind_t pszind = sz_psz2ind(size);
235 assert(pszind < sec->npsizes);
236
237 sec_shard_t *shard = sec_shard_pick(tsdn, sec);
238 sec_bin_t *bin = &shard->bins[pszind];
239 bool do_batch_fill = false;
240
241 malloc_mutex_lock(tsdn, &shard->mtx);
242 edata_t *edata = sec_shard_alloc_locked(tsdn, sec, shard, bin);
243 if (edata == NULL) {
244 if (!bin->being_batch_filled
245 && sec->opts.batch_fill_extra > 0) {
246 bin->being_batch_filled = true;
247 do_batch_fill = true;
248 }
249 }
250 malloc_mutex_unlock(tsdn, &shard->mtx);
251 if (edata == NULL) {
252 if (do_batch_fill) {
253 edata = sec_batch_fill_and_alloc(tsdn, sec, shard, bin,
254 size);
255 } else {
256 edata = pai_alloc(tsdn, sec->fallback, size, alignment,
257 zero, /* guarded */ false, frequent_reuse,
258 deferred_work_generated);
259 }
260 }
261 return edata;
262}
263
264static bool
265sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
266 size_t new_size, bool zero, bool *deferred_work_generated) {
267 sec_t *sec = (sec_t *)self;
268 return pai_expand(tsdn, sec->fallback, edata, old_size, new_size, zero,
269 deferred_work_generated);
270}
271
272static bool
273sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
274 size_t new_size, bool *deferred_work_generated) {
275 sec_t *sec = (sec_t *)self;
276 return pai_shrink(tsdn, sec->fallback, edata, old_size, new_size,
277 deferred_work_generated);
278}
279
280static void
281sec_flush_all_locked(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) {
282 malloc_mutex_assert_owner(tsdn, &shard->mtx);
283 shard->bytes_cur = 0;
284 edata_list_active_t to_flush;
285 edata_list_active_init(&to_flush);
286 for (pszind_t i = 0; i < sec->npsizes; i++) {
287 sec_bin_t *bin = &shard->bins[i];
288 bin->bytes_cur = 0;
289 edata_list_active_concat(&to_flush, &bin->freelist);
290 }
291
292 /*
293 * Ordinarily we would try to avoid doing the batch deallocation while
294 * holding the shard mutex, but the flush_all pathways only happen when
295 * we're disabling the HPA or resetting the arena, both of which are
296 * rare pathways.
297 */
298 bool deferred_work_generated = false;
299 pai_dalloc_batch(tsdn, sec->fallback, &to_flush,
300 &deferred_work_generated);
301}
302
303static void
304sec_shard_dalloc_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard,
305 edata_t *edata) {
306 malloc_mutex_assert_owner(tsdn, &shard->mtx);
307 assert(shard->bytes_cur <= sec->opts.max_bytes);
308 size_t size = edata_size_get(edata);
309 pszind_t pszind = sz_psz2ind(size);
310 assert(pszind < sec->npsizes);
311 /*
312 * Prepending here results in LIFO allocation per bin, which seems
313 * reasonable.
314 */
315 sec_bin_t *bin = &shard->bins[pszind];
316 edata_list_active_prepend(&bin->freelist, edata);
317 bin->bytes_cur += size;
318 shard->bytes_cur += size;
319 if (shard->bytes_cur > sec->opts.max_bytes) {
320 /*
321 * We've exceeded the shard limit. We make two nods in the
322 * direction of fragmentation avoidance: we flush everything in
323 * the shard, rather than one particular bin, and we hold the
324 * lock while flushing (in case one of the extents we flush is
325 * highly preferred from a fragmentation-avoidance perspective
326 * in the backing allocator). This has the extra advantage of
327 * not requiring advanced cache balancing strategies.
328 */
329 sec_flush_some_and_unlock(tsdn, sec, shard);
330 malloc_mutex_assert_not_owner(tsdn, &shard->mtx);
331 } else {
332 malloc_mutex_unlock(tsdn, &shard->mtx);
333 }
334}
335
336static void
337sec_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata,
338 bool *deferred_work_generated) {
339 sec_t *sec = (sec_t *)self;
340 if (sec->opts.nshards == 0
341 || edata_size_get(edata) > sec->opts.max_alloc) {
342 pai_dalloc(tsdn, sec->fallback, edata,
343 deferred_work_generated);
344 return;
345 }
346 sec_shard_t *shard = sec_shard_pick(tsdn, sec);
347 malloc_mutex_lock(tsdn, &shard->mtx);
348 if (shard->enabled) {
349 sec_shard_dalloc_and_unlock(tsdn, sec, shard, edata);
350 } else {
351 malloc_mutex_unlock(tsdn, &shard->mtx);
352 pai_dalloc(tsdn, sec->fallback, edata,
353 deferred_work_generated);
354 }
355}
356
357void
358sec_flush(tsdn_t *tsdn, sec_t *sec) {
359 for (size_t i = 0; i < sec->opts.nshards; i++) {
360 malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
361 sec_flush_all_locked(tsdn, sec, &sec->shards[i]);
362 malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
363 }
364}
365
366void
367sec_disable(tsdn_t *tsdn, sec_t *sec) {
368 for (size_t i = 0; i < sec->opts.nshards; i++) {
369 malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
370 sec->shards[i].enabled = false;
371 sec_flush_all_locked(tsdn, sec, &sec->shards[i]);
372 malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
373 }
374}
375
376void
377sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) {
378 size_t sum = 0;
379 for (size_t i = 0; i < sec->opts.nshards; i++) {
380 /*
381 * We could save these lock acquisitions by making bytes_cur
382 * atomic, but stats collection is rare anyways and we expect
383 * the number and type of stats to get more interesting.
384 */
385 malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
386 sum += sec->shards[i].bytes_cur;
387 malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
388 }
389 stats->bytes += sum;
390}
391
392void
393sec_mutex_stats_read(tsdn_t *tsdn, sec_t *sec,
394 mutex_prof_data_t *mutex_prof_data) {
395 for (size_t i = 0; i < sec->opts.nshards; i++) {
396 malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
397 malloc_mutex_prof_accum(tsdn, mutex_prof_data,
398 &sec->shards[i].mtx);
399 malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
400 }
401}
402
403void
404sec_prefork2(tsdn_t *tsdn, sec_t *sec) {
405 for (size_t i = 0; i < sec->opts.nshards; i++) {
406 malloc_mutex_prefork(tsdn, &sec->shards[i].mtx);
407 }
408}
409
410void
411sec_postfork_parent(tsdn_t *tsdn, sec_t *sec) {
412 for (size_t i = 0; i < sec->opts.nshards; i++) {
413 malloc_mutex_postfork_parent(tsdn, &sec->shards[i].mtx);
414 }
415}
416
417void
418sec_postfork_child(tsdn_t *tsdn, sec_t *sec) {
419 for (size_t i = 0; i < sec->opts.nshards; i++) {
420 malloc_mutex_postfork_child(tsdn, &sec->shards[i].mtx);
421 }
422}
diff --git a/examples/redis-unstable/deps/jemalloc/src/stats.c b/examples/redis-unstable/deps/jemalloc/src/stats.c
deleted file mode 100644
index efc70fd..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/stats.c
+++ /dev/null
@@ -1,1973 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/ctl.h"
6#include "jemalloc/internal/emitter.h"
7#include "jemalloc/internal/fxp.h"
8#include "jemalloc/internal/mutex.h"
9#include "jemalloc/internal/mutex_prof.h"
10#include "jemalloc/internal/prof_stats.h"
11
12const char *global_mutex_names[mutex_prof_num_global_mutexes] = {
13#define OP(mtx) #mtx,
14 MUTEX_PROF_GLOBAL_MUTEXES
15#undef OP
16};
17
18const char *arena_mutex_names[mutex_prof_num_arena_mutexes] = {
19#define OP(mtx) #mtx,
20 MUTEX_PROF_ARENA_MUTEXES
21#undef OP
22};
23
24#define CTL_GET(n, v, t) do { \
25 size_t sz = sizeof(t); \
26 xmallctl(n, (void *)v, &sz, NULL, 0); \
27} while (0)
28
29#define CTL_LEAF_PREPARE(mib, miblen, name) do { \
30 assert(miblen < CTL_MAX_DEPTH); \
31 size_t miblen_new = CTL_MAX_DEPTH; \
32 xmallctlmibnametomib(mib, miblen, name, &miblen_new); \
33 assert(miblen_new > miblen); \
34} while (0)
35
36#define CTL_LEAF(mib, miblen, leaf, v, t) do { \
37 assert(miblen < CTL_MAX_DEPTH); \
38 size_t miblen_new = CTL_MAX_DEPTH; \
39 size_t sz = sizeof(t); \
40 xmallctlbymibname(mib, miblen, leaf, &miblen_new, (void *)v, \
41 &sz, NULL, 0); \
42 assert(miblen_new == miblen + 1); \
43} while (0)
44
45#define CTL_M2_GET(n, i, v, t) do { \
46 size_t mib[CTL_MAX_DEPTH]; \
47 size_t miblen = sizeof(mib) / sizeof(size_t); \
48 size_t sz = sizeof(t); \
49 xmallctlnametomib(n, mib, &miblen); \
50 mib[2] = (i); \
51 xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0); \
52} while (0)
53
54/******************************************************************************/
55/* Data. */
56
57bool opt_stats_print = false;
58char opt_stats_print_opts[stats_print_tot_num_options+1] = "";
59
60int64_t opt_stats_interval = STATS_INTERVAL_DEFAULT;
61char opt_stats_interval_opts[stats_print_tot_num_options+1] = "";
62
63static counter_accum_t stats_interval_accumulated;
64/* Per thread batch accum size for stats_interval. */
65static uint64_t stats_interval_accum_batch;
66
67/******************************************************************************/
68
69static uint64_t
70rate_per_second(uint64_t value, uint64_t uptime_ns) {
71 uint64_t billion = 1000000000;
72 if (uptime_ns == 0 || value == 0) {
73 return 0;
74 }
75 if (uptime_ns < billion) {
76 return value;
77 } else {
78 uint64_t uptime_s = uptime_ns / billion;
79 return value / uptime_s;
80 }
81}
82
83/* Calculate x.yyy and output a string (takes a fixed sized char array). */
84static bool
85get_rate_str(uint64_t dividend, uint64_t divisor, char str[6]) {
86 if (divisor == 0 || dividend > divisor) {
87 /* The rate is not supposed to be greater than 1. */
88 return true;
89 }
90 if (dividend > 0) {
91 assert(UINT64_MAX / dividend >= 1000);
92 }
93
94 unsigned n = (unsigned)((dividend * 1000) / divisor);
95 if (n < 10) {
96 malloc_snprintf(str, 6, "0.00%u", n);
97 } else if (n < 100) {
98 malloc_snprintf(str, 6, "0.0%u", n);
99 } else if (n < 1000) {
100 malloc_snprintf(str, 6, "0.%u", n);
101 } else {
102 malloc_snprintf(str, 6, "1");
103 }
104
105 return false;
106}
107
108static void
109mutex_stats_init_cols(emitter_row_t *row, const char *table_name,
110 emitter_col_t *name,
111 emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters],
112 emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) {
113 mutex_prof_uint64_t_counter_ind_t k_uint64_t = 0;
114 mutex_prof_uint32_t_counter_ind_t k_uint32_t = 0;
115
116 emitter_col_t *col;
117
118 if (name != NULL) {
119 emitter_col_init(name, row);
120 name->justify = emitter_justify_left;
121 name->width = 21;
122 name->type = emitter_type_title;
123 name->str_val = table_name;
124 }
125
126#define WIDTH_uint32_t 12
127#define WIDTH_uint64_t 16
128#define OP(counter, counter_type, human, derived, base_counter) \
129 col = &col_##counter_type[k_##counter_type]; \
130 ++k_##counter_type; \
131 emitter_col_init(col, row); \
132 col->justify = emitter_justify_right; \
133 col->width = derived ? 8 : WIDTH_##counter_type; \
134 col->type = emitter_type_title; \
135 col->str_val = human;
136 MUTEX_PROF_COUNTERS
137#undef OP
138#undef WIDTH_uint32_t
139#undef WIDTH_uint64_t
140 col_uint64_t[mutex_counter_total_wait_time_ps].width = 10;
141}
142
143static void
144mutex_stats_read_global(size_t mib[], size_t miblen, const char *name,
145 emitter_col_t *col_name,
146 emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters],
147 emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters],
148 uint64_t uptime) {
149 CTL_LEAF_PREPARE(mib, miblen, name);
150 size_t miblen_name = miblen + 1;
151
152 col_name->str_val = name;
153
154 emitter_col_t *dst;
155#define EMITTER_TYPE_uint32_t emitter_type_uint32
156#define EMITTER_TYPE_uint64_t emitter_type_uint64
157#define OP(counter, counter_type, human, derived, base_counter) \
158 dst = &col_##counter_type[mutex_counter_##counter]; \
159 dst->type = EMITTER_TYPE_##counter_type; \
160 if (!derived) { \
161 CTL_LEAF(mib, miblen_name, #counter, \
162 (counter_type *)&dst->bool_val, counter_type); \
163 } else { \
164 emitter_col_t *base = \
165 &col_##counter_type[mutex_counter_##base_counter]; \
166 dst->counter_type##_val = \
167 (counter_type)rate_per_second( \
168 base->counter_type##_val, uptime); \
169 }
170 MUTEX_PROF_COUNTERS
171#undef OP
172#undef EMITTER_TYPE_uint32_t
173#undef EMITTER_TYPE_uint64_t
174}
175
176static void
177mutex_stats_read_arena(size_t mib[], size_t miblen, const char *name,
178 emitter_col_t *col_name,
179 emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters],
180 emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters],
181 uint64_t uptime) {
182 CTL_LEAF_PREPARE(mib, miblen, name);
183 size_t miblen_name = miblen + 1;
184
185 col_name->str_val = name;
186
187 emitter_col_t *dst;
188#define EMITTER_TYPE_uint32_t emitter_type_uint32
189#define EMITTER_TYPE_uint64_t emitter_type_uint64
190#define OP(counter, counter_type, human, derived, base_counter) \
191 dst = &col_##counter_type[mutex_counter_##counter]; \
192 dst->type = EMITTER_TYPE_##counter_type; \
193 if (!derived) { \
194 CTL_LEAF(mib, miblen_name, #counter, \
195 (counter_type *)&dst->bool_val, counter_type); \
196 } else { \
197 emitter_col_t *base = \
198 &col_##counter_type[mutex_counter_##base_counter]; \
199 dst->counter_type##_val = \
200 (counter_type)rate_per_second( \
201 base->counter_type##_val, uptime); \
202 }
203 MUTEX_PROF_COUNTERS
204#undef OP
205#undef EMITTER_TYPE_uint32_t
206#undef EMITTER_TYPE_uint64_t
207}
208
209static void
210mutex_stats_read_arena_bin(size_t mib[], size_t miblen,
211 emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters],
212 emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters],
213 uint64_t uptime) {
214 CTL_LEAF_PREPARE(mib, miblen, "mutex");
215 size_t miblen_mutex = miblen + 1;
216
217 emitter_col_t *dst;
218
219#define EMITTER_TYPE_uint32_t emitter_type_uint32
220#define EMITTER_TYPE_uint64_t emitter_type_uint64
221#define OP(counter, counter_type, human, derived, base_counter) \
222 dst = &col_##counter_type[mutex_counter_##counter]; \
223 dst->type = EMITTER_TYPE_##counter_type; \
224 if (!derived) { \
225 CTL_LEAF(mib, miblen_mutex, #counter, \
226 (counter_type *)&dst->bool_val, counter_type); \
227 } else { \
228 emitter_col_t *base = \
229 &col_##counter_type[mutex_counter_##base_counter]; \
230 dst->counter_type##_val = \
231 (counter_type)rate_per_second( \
232 base->counter_type##_val, uptime); \
233 }
234 MUTEX_PROF_COUNTERS
235#undef OP
236#undef EMITTER_TYPE_uint32_t
237#undef EMITTER_TYPE_uint64_t
238}
239
240/* "row" can be NULL to avoid emitting in table mode. */
241static void
242mutex_stats_emit(emitter_t *emitter, emitter_row_t *row,
243 emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters],
244 emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) {
245 if (row != NULL) {
246 emitter_table_row(emitter, row);
247 }
248
249 mutex_prof_uint64_t_counter_ind_t k_uint64_t = 0;
250 mutex_prof_uint32_t_counter_ind_t k_uint32_t = 0;
251
252 emitter_col_t *col;
253
254#define EMITTER_TYPE_uint32_t emitter_type_uint32
255#define EMITTER_TYPE_uint64_t emitter_type_uint64
256#define OP(counter, type, human, derived, base_counter) \
257 if (!derived) { \
258 col = &col_##type[k_##type]; \
259 ++k_##type; \
260 emitter_json_kv(emitter, #counter, EMITTER_TYPE_##type, \
261 (const void *)&col->bool_val); \
262 }
263 MUTEX_PROF_COUNTERS;
264#undef OP
265#undef EMITTER_TYPE_uint32_t
266#undef EMITTER_TYPE_uint64_t
267}
268
269#define COL_DECLARE(column_name) \
270 emitter_col_t col_##column_name;
271
272#define COL_INIT(row_name, column_name, left_or_right, col_width, etype)\
273 emitter_col_init(&col_##column_name, &row_name); \
274 col_##column_name.justify = emitter_justify_##left_or_right; \
275 col_##column_name.width = col_width; \
276 col_##column_name.type = emitter_type_##etype;
277
278#define COL(row_name, column_name, left_or_right, col_width, etype) \
279 COL_DECLARE(column_name); \
280 COL_INIT(row_name, column_name, left_or_right, col_width, etype)
281
282#define COL_HDR_DECLARE(column_name) \
283 COL_DECLARE(column_name); \
284 emitter_col_t header_##column_name;
285
286#define COL_HDR_INIT(row_name, column_name, human, left_or_right, \
287 col_width, etype) \
288 COL_INIT(row_name, column_name, left_or_right, col_width, etype)\
289 emitter_col_init(&header_##column_name, &header_##row_name); \
290 header_##column_name.justify = emitter_justify_##left_or_right; \
291 header_##column_name.width = col_width; \
292 header_##column_name.type = emitter_type_title; \
293 header_##column_name.str_val = human ? human : #column_name;
294
295#define COL_HDR(row_name, column_name, human, left_or_right, col_width, \
296 etype) \
297 COL_HDR_DECLARE(column_name) \
298 COL_HDR_INIT(row_name, column_name, human, left_or_right, \
299 col_width, etype)
300
301JEMALLOC_COLD
302static void
303stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i,
304 uint64_t uptime) {
305 size_t page;
306 bool in_gap, in_gap_prev;
307 unsigned nbins, j;
308
309 CTL_GET("arenas.page", &page, size_t);
310
311 CTL_GET("arenas.nbins", &nbins, unsigned);
312
313 emitter_row_t header_row;
314 emitter_row_init(&header_row);
315
316 emitter_row_t row;
317 emitter_row_init(&row);
318
319 bool prof_stats_on = config_prof && opt_prof && opt_prof_stats
320 && i == MALLCTL_ARENAS_ALL;
321
322 COL_HDR(row, size, NULL, right, 20, size)
323 COL_HDR(row, ind, NULL, right, 4, unsigned)
324 COL_HDR(row, allocated, NULL, right, 13, uint64)
325 COL_HDR(row, nmalloc, NULL, right, 13, uint64)
326 COL_HDR(row, nmalloc_ps, "(#/sec)", right, 8, uint64)
327 COL_HDR(row, ndalloc, NULL, right, 13, uint64)
328 COL_HDR(row, ndalloc_ps, "(#/sec)", right, 8, uint64)
329 COL_HDR(row, nrequests, NULL, right, 13, uint64)
330 COL_HDR(row, nrequests_ps, "(#/sec)", right, 10, uint64)
331 COL_HDR_DECLARE(prof_live_requested);
332 COL_HDR_DECLARE(prof_live_count);
333 COL_HDR_DECLARE(prof_accum_requested);
334 COL_HDR_DECLARE(prof_accum_count);
335 if (prof_stats_on) {
336 COL_HDR_INIT(row, prof_live_requested, NULL, right, 21, uint64)
337 COL_HDR_INIT(row, prof_live_count, NULL, right, 17, uint64)
338 COL_HDR_INIT(row, prof_accum_requested, NULL, right, 21, uint64)
339 COL_HDR_INIT(row, prof_accum_count, NULL, right, 17, uint64)
340 }
341 COL_HDR(row, nshards, NULL, right, 9, unsigned)
342 COL_HDR(row, curregs, NULL, right, 13, size)
343 COL_HDR(row, curslabs, NULL, right, 13, size)
344 COL_HDR(row, nonfull_slabs, NULL, right, 15, size)
345 COL_HDR(row, regs, NULL, right, 5, unsigned)
346 COL_HDR(row, pgs, NULL, right, 4, size)
347 /* To buffer a right- and left-justified column. */
348 COL_HDR(row, justify_spacer, NULL, right, 1, title)
349 COL_HDR(row, util, NULL, right, 6, title)
350 COL_HDR(row, nfills, NULL, right, 13, uint64)
351 COL_HDR(row, nfills_ps, "(#/sec)", right, 8, uint64)
352 COL_HDR(row, nflushes, NULL, right, 13, uint64)
353 COL_HDR(row, nflushes_ps, "(#/sec)", right, 8, uint64)
354 COL_HDR(row, nslabs, NULL, right, 13, uint64)
355 COL_HDR(row, nreslabs, NULL, right, 13, uint64)
356 COL_HDR(row, nreslabs_ps, "(#/sec)", right, 8, uint64)
357
358 /* Don't want to actually print the name. */
359 header_justify_spacer.str_val = " ";
360 col_justify_spacer.str_val = " ";
361
362 emitter_col_t col_mutex64[mutex_prof_num_uint64_t_counters];
363 emitter_col_t col_mutex32[mutex_prof_num_uint32_t_counters];
364
365 emitter_col_t header_mutex64[mutex_prof_num_uint64_t_counters];
366 emitter_col_t header_mutex32[mutex_prof_num_uint32_t_counters];
367
368 if (mutex) {
369 mutex_stats_init_cols(&row, NULL, NULL, col_mutex64,
370 col_mutex32);
371 mutex_stats_init_cols(&header_row, NULL, NULL, header_mutex64,
372 header_mutex32);
373 }
374
375 /*
376 * We print a "bins:" header as part of the table row; we need to adjust
377 * the header size column to compensate.
378 */
379 header_size.width -=5;
380 emitter_table_printf(emitter, "bins:");
381 emitter_table_row(emitter, &header_row);
382 emitter_json_array_kv_begin(emitter, "bins");
383
384 size_t stats_arenas_mib[CTL_MAX_DEPTH];
385 CTL_LEAF_PREPARE(stats_arenas_mib, 0, "stats.arenas");
386 stats_arenas_mib[2] = i;
387 CTL_LEAF_PREPARE(stats_arenas_mib, 3, "bins");
388
389 size_t arenas_bin_mib[CTL_MAX_DEPTH];
390 CTL_LEAF_PREPARE(arenas_bin_mib, 0, "arenas.bin");
391
392 size_t prof_stats_mib[CTL_MAX_DEPTH];
393 if (prof_stats_on) {
394 CTL_LEAF_PREPARE(prof_stats_mib, 0, "prof.stats.bins");
395 }
396
397 for (j = 0, in_gap = false; j < nbins; j++) {
398 uint64_t nslabs;
399 size_t reg_size, slab_size, curregs;
400 size_t curslabs;
401 size_t nonfull_slabs;
402 uint32_t nregs, nshards;
403 uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes;
404 uint64_t nreslabs;
405 prof_stats_t prof_live;
406 prof_stats_t prof_accum;
407
408 stats_arenas_mib[4] = j;
409 arenas_bin_mib[2] = j;
410
411 CTL_LEAF(stats_arenas_mib, 5, "nslabs", &nslabs, uint64_t);
412
413 if (prof_stats_on) {
414 prof_stats_mib[3] = j;
415 CTL_LEAF(prof_stats_mib, 4, "live", &prof_live,
416 prof_stats_t);
417 CTL_LEAF(prof_stats_mib, 4, "accum", &prof_accum,
418 prof_stats_t);
419 }
420
421 in_gap_prev = in_gap;
422 if (prof_stats_on) {
423 in_gap = (nslabs == 0 && prof_accum.count == 0);
424 } else {
425 in_gap = (nslabs == 0);
426 }
427
428 if (in_gap_prev && !in_gap) {
429 emitter_table_printf(emitter,
430 " ---\n");
431 }
432
433 if (in_gap && !emitter_outputs_json(emitter)) {
434 continue;
435 }
436
437 CTL_LEAF(arenas_bin_mib, 3, "size", &reg_size, size_t);
438 CTL_LEAF(arenas_bin_mib, 3, "nregs", &nregs, uint32_t);
439 CTL_LEAF(arenas_bin_mib, 3, "slab_size", &slab_size, size_t);
440 CTL_LEAF(arenas_bin_mib, 3, "nshards", &nshards, uint32_t);
441 CTL_LEAF(stats_arenas_mib, 5, "nmalloc", &nmalloc, uint64_t);
442 CTL_LEAF(stats_arenas_mib, 5, "ndalloc", &ndalloc, uint64_t);
443 CTL_LEAF(stats_arenas_mib, 5, "curregs", &curregs, size_t);
444 CTL_LEAF(stats_arenas_mib, 5, "nrequests", &nrequests,
445 uint64_t);
446 CTL_LEAF(stats_arenas_mib, 5, "nfills", &nfills, uint64_t);
447 CTL_LEAF(stats_arenas_mib, 5, "nflushes", &nflushes, uint64_t);
448 CTL_LEAF(stats_arenas_mib, 5, "nreslabs", &nreslabs, uint64_t);
449 CTL_LEAF(stats_arenas_mib, 5, "curslabs", &curslabs, size_t);
450 CTL_LEAF(stats_arenas_mib, 5, "nonfull_slabs", &nonfull_slabs,
451 size_t);
452
453 if (mutex) {
454 mutex_stats_read_arena_bin(stats_arenas_mib, 5,
455 col_mutex64, col_mutex32, uptime);
456 }
457
458 emitter_json_object_begin(emitter);
459 emitter_json_kv(emitter, "nmalloc", emitter_type_uint64,
460 &nmalloc);
461 emitter_json_kv(emitter, "ndalloc", emitter_type_uint64,
462 &ndalloc);
463 emitter_json_kv(emitter, "curregs", emitter_type_size,
464 &curregs);
465 emitter_json_kv(emitter, "nrequests", emitter_type_uint64,
466 &nrequests);
467 if (prof_stats_on) {
468 emitter_json_kv(emitter, "prof_live_requested",
469 emitter_type_uint64, &prof_live.req_sum);
470 emitter_json_kv(emitter, "prof_live_count",
471 emitter_type_uint64, &prof_live.count);
472 emitter_json_kv(emitter, "prof_accum_requested",
473 emitter_type_uint64, &prof_accum.req_sum);
474 emitter_json_kv(emitter, "prof_accum_count",
475 emitter_type_uint64, &prof_accum.count);
476 }
477 emitter_json_kv(emitter, "nfills", emitter_type_uint64,
478 &nfills);
479 emitter_json_kv(emitter, "nflushes", emitter_type_uint64,
480 &nflushes);
481 emitter_json_kv(emitter, "nreslabs", emitter_type_uint64,
482 &nreslabs);
483 emitter_json_kv(emitter, "curslabs", emitter_type_size,
484 &curslabs);
485 emitter_json_kv(emitter, "nonfull_slabs", emitter_type_size,
486 &nonfull_slabs);
487 if (mutex) {
488 emitter_json_object_kv_begin(emitter, "mutex");
489 mutex_stats_emit(emitter, NULL, col_mutex64,
490 col_mutex32);
491 emitter_json_object_end(emitter);
492 }
493 emitter_json_object_end(emitter);
494
495 size_t availregs = nregs * curslabs;
496 char util[6];
497 if (get_rate_str((uint64_t)curregs, (uint64_t)availregs, util))
498 {
499 if (availregs == 0) {
500 malloc_snprintf(util, sizeof(util), "1");
501 } else if (curregs > availregs) {
502 /*
503 * Race detected: the counters were read in
504 * separate mallctl calls and concurrent
505 * operations happened in between. In this case
506 * no meaningful utilization can be computed.
507 */
508 malloc_snprintf(util, sizeof(util), " race");
509 } else {
510 not_reached();
511 }
512 }
513
514 col_size.size_val = reg_size;
515 col_ind.unsigned_val = j;
516 col_allocated.size_val = curregs * reg_size;
517 col_nmalloc.uint64_val = nmalloc;
518 col_nmalloc_ps.uint64_val = rate_per_second(nmalloc, uptime);
519 col_ndalloc.uint64_val = ndalloc;
520 col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime);
521 col_nrequests.uint64_val = nrequests;
522 col_nrequests_ps.uint64_val = rate_per_second(nrequests, uptime);
523 if (prof_stats_on) {
524 col_prof_live_requested.uint64_val = prof_live.req_sum;
525 col_prof_live_count.uint64_val = prof_live.count;
526 col_prof_accum_requested.uint64_val =
527 prof_accum.req_sum;
528 col_prof_accum_count.uint64_val = prof_accum.count;
529 }
530 col_nshards.unsigned_val = nshards;
531 col_curregs.size_val = curregs;
532 col_curslabs.size_val = curslabs;
533 col_nonfull_slabs.size_val = nonfull_slabs;
534 col_regs.unsigned_val = nregs;
535 col_pgs.size_val = slab_size / page;
536 col_util.str_val = util;
537 col_nfills.uint64_val = nfills;
538 col_nfills_ps.uint64_val = rate_per_second(nfills, uptime);
539 col_nflushes.uint64_val = nflushes;
540 col_nflushes_ps.uint64_val = rate_per_second(nflushes, uptime);
541 col_nslabs.uint64_val = nslabs;
542 col_nreslabs.uint64_val = nreslabs;
543 col_nreslabs_ps.uint64_val = rate_per_second(nreslabs, uptime);
544
545 /*
546 * Note that mutex columns were initialized above, if mutex ==
547 * true.
548 */
549
550 emitter_table_row(emitter, &row);
551 }
552 emitter_json_array_end(emitter); /* Close "bins". */
553
554 if (in_gap) {
555 emitter_table_printf(emitter, " ---\n");
556 }
557}
558
559JEMALLOC_COLD
560static void
561stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) {
562 unsigned nbins, nlextents, j;
563 bool in_gap, in_gap_prev;
564
565 CTL_GET("arenas.nbins", &nbins, unsigned);
566 CTL_GET("arenas.nlextents", &nlextents, unsigned);
567
568 emitter_row_t header_row;
569 emitter_row_init(&header_row);
570 emitter_row_t row;
571 emitter_row_init(&row);
572
573 bool prof_stats_on = config_prof && opt_prof && opt_prof_stats
574 && i == MALLCTL_ARENAS_ALL;
575
576 COL_HDR(row, size, NULL, right, 20, size)
577 COL_HDR(row, ind, NULL, right, 4, unsigned)
578 COL_HDR(row, allocated, NULL, right, 13, size)
579 COL_HDR(row, nmalloc, NULL, right, 13, uint64)
580 COL_HDR(row, nmalloc_ps, "(#/sec)", right, 8, uint64)
581 COL_HDR(row, ndalloc, NULL, right, 13, uint64)
582 COL_HDR(row, ndalloc_ps, "(#/sec)", right, 8, uint64)
583 COL_HDR(row, nrequests, NULL, right, 13, uint64)
584 COL_HDR(row, nrequests_ps, "(#/sec)", right, 8, uint64)
585 COL_HDR_DECLARE(prof_live_requested)
586 COL_HDR_DECLARE(prof_live_count)
587 COL_HDR_DECLARE(prof_accum_requested)
588 COL_HDR_DECLARE(prof_accum_count)
589 if (prof_stats_on) {
590 COL_HDR_INIT(row, prof_live_requested, NULL, right, 21, uint64)
591 COL_HDR_INIT(row, prof_live_count, NULL, right, 17, uint64)
592 COL_HDR_INIT(row, prof_accum_requested, NULL, right, 21, uint64)
593 COL_HDR_INIT(row, prof_accum_count, NULL, right, 17, uint64)
594 }
595 COL_HDR(row, curlextents, NULL, right, 13, size)
596
597 /* As with bins, we label the large extents table. */
598 header_size.width -= 6;
599 emitter_table_printf(emitter, "large:");
600 emitter_table_row(emitter, &header_row);
601 emitter_json_array_kv_begin(emitter, "lextents");
602
603 size_t stats_arenas_mib[CTL_MAX_DEPTH];
604 CTL_LEAF_PREPARE(stats_arenas_mib, 0, "stats.arenas");
605 stats_arenas_mib[2] = i;
606 CTL_LEAF_PREPARE(stats_arenas_mib, 3, "lextents");
607
608 size_t arenas_lextent_mib[CTL_MAX_DEPTH];
609 CTL_LEAF_PREPARE(arenas_lextent_mib, 0, "arenas.lextent");
610
611 size_t prof_stats_mib[CTL_MAX_DEPTH];
612 if (prof_stats_on) {
613 CTL_LEAF_PREPARE(prof_stats_mib, 0, "prof.stats.lextents");
614 }
615
616 for (j = 0, in_gap = false; j < nlextents; j++) {
617 uint64_t nmalloc, ndalloc, nrequests;
618 size_t lextent_size, curlextents;
619 prof_stats_t prof_live;
620 prof_stats_t prof_accum;
621
622 stats_arenas_mib[4] = j;
623 arenas_lextent_mib[2] = j;
624
625 CTL_LEAF(stats_arenas_mib, 5, "nmalloc", &nmalloc, uint64_t);
626 CTL_LEAF(stats_arenas_mib, 5, "ndalloc", &ndalloc, uint64_t);
627 CTL_LEAF(stats_arenas_mib, 5, "nrequests", &nrequests,
628 uint64_t);
629
630 in_gap_prev = in_gap;
631 in_gap = (nrequests == 0);
632
633 if (in_gap_prev && !in_gap) {
634 emitter_table_printf(emitter,
635 " ---\n");
636 }
637
638 CTL_LEAF(arenas_lextent_mib, 3, "size", &lextent_size, size_t);
639 CTL_LEAF(stats_arenas_mib, 5, "curlextents", &curlextents,
640 size_t);
641
642 if (prof_stats_on) {
643 prof_stats_mib[3] = j;
644 CTL_LEAF(prof_stats_mib, 4, "live", &prof_live,
645 prof_stats_t);
646 CTL_LEAF(prof_stats_mib, 4, "accum", &prof_accum,
647 prof_stats_t);
648 }
649
650 emitter_json_object_begin(emitter);
651 if (prof_stats_on) {
652 emitter_json_kv(emitter, "prof_live_requested",
653 emitter_type_uint64, &prof_live.req_sum);
654 emitter_json_kv(emitter, "prof_live_count",
655 emitter_type_uint64, &prof_live.count);
656 emitter_json_kv(emitter, "prof_accum_requested",
657 emitter_type_uint64, &prof_accum.req_sum);
658 emitter_json_kv(emitter, "prof_accum_count",
659 emitter_type_uint64, &prof_accum.count);
660 }
661 emitter_json_kv(emitter, "curlextents", emitter_type_size,
662 &curlextents);
663 emitter_json_object_end(emitter);
664
665 col_size.size_val = lextent_size;
666 col_ind.unsigned_val = nbins + j;
667 col_allocated.size_val = curlextents * lextent_size;
668 col_nmalloc.uint64_val = nmalloc;
669 col_nmalloc_ps.uint64_val = rate_per_second(nmalloc, uptime);
670 col_ndalloc.uint64_val = ndalloc;
671 col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime);
672 col_nrequests.uint64_val = nrequests;
673 col_nrequests_ps.uint64_val = rate_per_second(nrequests, uptime);
674 if (prof_stats_on) {
675 col_prof_live_requested.uint64_val = prof_live.req_sum;
676 col_prof_live_count.uint64_val = prof_live.count;
677 col_prof_accum_requested.uint64_val =
678 prof_accum.req_sum;
679 col_prof_accum_count.uint64_val = prof_accum.count;
680 }
681 col_curlextents.size_val = curlextents;
682
683 if (!in_gap) {
684 emitter_table_row(emitter, &row);
685 }
686 }
687 emitter_json_array_end(emitter); /* Close "lextents". */
688 if (in_gap) {
689 emitter_table_printf(emitter, " ---\n");
690 }
691}
692
693JEMALLOC_COLD
694static void
695stats_arena_extents_print(emitter_t *emitter, unsigned i) {
696 unsigned j;
697 bool in_gap, in_gap_prev;
698 emitter_row_t header_row;
699 emitter_row_init(&header_row);
700 emitter_row_t row;
701 emitter_row_init(&row);
702
703 COL_HDR(row, size, NULL, right, 20, size)
704 COL_HDR(row, ind, NULL, right, 4, unsigned)
705 COL_HDR(row, ndirty, NULL, right, 13, size)
706 COL_HDR(row, dirty, NULL, right, 13, size)
707 COL_HDR(row, nmuzzy, NULL, right, 13, size)
708 COL_HDR(row, muzzy, NULL, right, 13, size)
709 COL_HDR(row, nretained, NULL, right, 13, size)
710 COL_HDR(row, retained, NULL, right, 13, size)
711 COL_HDR(row, ntotal, NULL, right, 13, size)
712 COL_HDR(row, total, NULL, right, 13, size)
713
714 /* Label this section. */
715 header_size.width -= 8;
716 emitter_table_printf(emitter, "extents:");
717 emitter_table_row(emitter, &header_row);
718 emitter_json_array_kv_begin(emitter, "extents");
719
720 size_t stats_arenas_mib[CTL_MAX_DEPTH];
721 CTL_LEAF_PREPARE(stats_arenas_mib, 0, "stats.arenas");
722 stats_arenas_mib[2] = i;
723 CTL_LEAF_PREPARE(stats_arenas_mib, 3, "extents");
724
725 in_gap = false;
726 for (j = 0; j < SC_NPSIZES; j++) {
727 size_t ndirty, nmuzzy, nretained, total, dirty_bytes,
728 muzzy_bytes, retained_bytes, total_bytes;
729 stats_arenas_mib[4] = j;
730
731 CTL_LEAF(stats_arenas_mib, 5, "ndirty", &ndirty, size_t);
732 CTL_LEAF(stats_arenas_mib, 5, "nmuzzy", &nmuzzy, size_t);
733 CTL_LEAF(stats_arenas_mib, 5, "nretained", &nretained, size_t);
734 CTL_LEAF(stats_arenas_mib, 5, "dirty_bytes", &dirty_bytes,
735 size_t);
736 CTL_LEAF(stats_arenas_mib, 5, "muzzy_bytes", &muzzy_bytes,
737 size_t);
738 CTL_LEAF(stats_arenas_mib, 5, "retained_bytes",
739 &retained_bytes, size_t);
740
741 total = ndirty + nmuzzy + nretained;
742 total_bytes = dirty_bytes + muzzy_bytes + retained_bytes;
743
744 in_gap_prev = in_gap;
745 in_gap = (total == 0);
746
747 if (in_gap_prev && !in_gap) {
748 emitter_table_printf(emitter,
749 " ---\n");
750 }
751
752 emitter_json_object_begin(emitter);
753 emitter_json_kv(emitter, "ndirty", emitter_type_size, &ndirty);
754 emitter_json_kv(emitter, "nmuzzy", emitter_type_size, &nmuzzy);
755 emitter_json_kv(emitter, "nretained", emitter_type_size,
756 &nretained);
757
758 emitter_json_kv(emitter, "dirty_bytes", emitter_type_size,
759 &dirty_bytes);
760 emitter_json_kv(emitter, "muzzy_bytes", emitter_type_size,
761 &muzzy_bytes);
762 emitter_json_kv(emitter, "retained_bytes", emitter_type_size,
763 &retained_bytes);
764 emitter_json_object_end(emitter);
765
766 col_size.size_val = sz_pind2sz(j);
767 col_ind.size_val = j;
768 col_ndirty.size_val = ndirty;
769 col_dirty.size_val = dirty_bytes;
770 col_nmuzzy.size_val = nmuzzy;
771 col_muzzy.size_val = muzzy_bytes;
772 col_nretained.size_val = nretained;
773 col_retained.size_val = retained_bytes;
774 col_ntotal.size_val = total;
775 col_total.size_val = total_bytes;
776
777 if (!in_gap) {
778 emitter_table_row(emitter, &row);
779 }
780 }
781 emitter_json_array_end(emitter); /* Close "extents". */
782 if (in_gap) {
783 emitter_table_printf(emitter, " ---\n");
784 }
785}
786
787static void
788stats_arena_hpa_shard_print(emitter_t *emitter, unsigned i, uint64_t uptime) {
789 emitter_row_t header_row;
790 emitter_row_init(&header_row);
791 emitter_row_t row;
792 emitter_row_init(&row);
793
794 uint64_t npurge_passes;
795 uint64_t npurges;
796 uint64_t nhugifies;
797 uint64_t ndehugifies;
798
799 CTL_M2_GET("stats.arenas.0.hpa_shard.npurge_passes",
800 i, &npurge_passes, uint64_t);
801 CTL_M2_GET("stats.arenas.0.hpa_shard.npurges",
802 i, &npurges, uint64_t);
803 CTL_M2_GET("stats.arenas.0.hpa_shard.nhugifies",
804 i, &nhugifies, uint64_t);
805 CTL_M2_GET("stats.arenas.0.hpa_shard.ndehugifies",
806 i, &ndehugifies, uint64_t);
807
808 size_t npageslabs_huge;
809 size_t nactive_huge;
810 size_t ndirty_huge;
811
812 size_t npageslabs_nonhuge;
813 size_t nactive_nonhuge;
814 size_t ndirty_nonhuge;
815 size_t nretained_nonhuge;
816
817 size_t sec_bytes;
818 CTL_M2_GET("stats.arenas.0.hpa_sec_bytes", i, &sec_bytes, size_t);
819 emitter_kv(emitter, "sec_bytes", "Bytes in small extent cache",
820 emitter_type_size, &sec_bytes);
821
822 /* First, global stats. */
823 emitter_table_printf(emitter,
824 "HPA shard stats:\n"
825 " Purge passes: %" FMTu64 " (%" FMTu64 " / sec)\n"
826 " Purges: %" FMTu64 " (%" FMTu64 " / sec)\n"
827 " Hugeifies: %" FMTu64 " (%" FMTu64 " / sec)\n"
828 " Dehugifies: %" FMTu64 " (%" FMTu64 " / sec)\n"
829 "\n",
830 npurge_passes, rate_per_second(npurge_passes, uptime),
831 npurges, rate_per_second(npurges, uptime),
832 nhugifies, rate_per_second(nhugifies, uptime),
833 ndehugifies, rate_per_second(ndehugifies, uptime));
834
835 emitter_json_object_kv_begin(emitter, "hpa_shard");
836 emitter_json_kv(emitter, "npurge_passes", emitter_type_uint64,
837 &npurge_passes);
838 emitter_json_kv(emitter, "npurges", emitter_type_uint64,
839 &npurges);
840 emitter_json_kv(emitter, "nhugifies", emitter_type_uint64,
841 &nhugifies);
842 emitter_json_kv(emitter, "ndehugifies", emitter_type_uint64,
843 &ndehugifies);
844
845 /* Next, full slab stats. */
846 CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.npageslabs_huge",
847 i, &npageslabs_huge, size_t);
848 CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.nactive_huge",
849 i, &nactive_huge, size_t);
850 CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.ndirty_huge",
851 i, &ndirty_huge, size_t);
852
853 CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.npageslabs_nonhuge",
854 i, &npageslabs_nonhuge, size_t);
855 CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.nactive_nonhuge",
856 i, &nactive_nonhuge, size_t);
857 CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.ndirty_nonhuge",
858 i, &ndirty_nonhuge, size_t);
859 nretained_nonhuge = npageslabs_nonhuge * HUGEPAGE_PAGES
860 - nactive_nonhuge - ndirty_nonhuge;
861
862 emitter_table_printf(emitter,
863 " In full slabs:\n"
864 " npageslabs: %zu huge, %zu nonhuge\n"
865 " nactive: %zu huge, %zu nonhuge \n"
866 " ndirty: %zu huge, %zu nonhuge \n"
867 " nretained: 0 huge, %zu nonhuge \n",
868 npageslabs_huge, npageslabs_nonhuge,
869 nactive_huge, nactive_nonhuge,
870 ndirty_huge, ndirty_nonhuge,
871 nretained_nonhuge);
872
873 emitter_json_object_kv_begin(emitter, "full_slabs");
874 emitter_json_kv(emitter, "npageslabs_huge", emitter_type_size,
875 &npageslabs_huge);
876 emitter_json_kv(emitter, "nactive_huge", emitter_type_size,
877 &nactive_huge);
878 emitter_json_kv(emitter, "nactive_huge", emitter_type_size,
879 &nactive_huge);
880 emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size,
881 &npageslabs_nonhuge);
882 emitter_json_kv(emitter, "nactive_nonhuge", emitter_type_size,
883 &nactive_nonhuge);
884 emitter_json_kv(emitter, "ndirty_nonhuge", emitter_type_size,
885 &ndirty_nonhuge);
886 emitter_json_object_end(emitter); /* End "full_slabs" */
887
888 /* Next, empty slab stats. */
889 CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.npageslabs_huge",
890 i, &npageslabs_huge, size_t);
891 CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.nactive_huge",
892 i, &nactive_huge, size_t);
893 CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.ndirty_huge",
894 i, &ndirty_huge, size_t);
895
896 CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.npageslabs_nonhuge",
897 i, &npageslabs_nonhuge, size_t);
898 CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.nactive_nonhuge",
899 i, &nactive_nonhuge, size_t);
900 CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.ndirty_nonhuge",
901 i, &ndirty_nonhuge, size_t);
902 nretained_nonhuge = npageslabs_nonhuge * HUGEPAGE_PAGES
903 - nactive_nonhuge - ndirty_nonhuge;
904
905 emitter_table_printf(emitter,
906 " In empty slabs:\n"
907 " npageslabs: %zu huge, %zu nonhuge\n"
908 " nactive: %zu huge, %zu nonhuge \n"
909 " ndirty: %zu huge, %zu nonhuge \n"
910 " nretained: 0 huge, %zu nonhuge \n"
911 "\n",
912 npageslabs_huge, npageslabs_nonhuge,
913 nactive_huge, nactive_nonhuge,
914 ndirty_huge, ndirty_nonhuge,
915 nretained_nonhuge);
916
917 emitter_json_object_kv_begin(emitter, "empty_slabs");
918 emitter_json_kv(emitter, "npageslabs_huge", emitter_type_size,
919 &npageslabs_huge);
920 emitter_json_kv(emitter, "nactive_huge", emitter_type_size,
921 &nactive_huge);
922 emitter_json_kv(emitter, "nactive_huge", emitter_type_size,
923 &nactive_huge);
924 emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size,
925 &npageslabs_nonhuge);
926 emitter_json_kv(emitter, "nactive_nonhuge", emitter_type_size,
927 &nactive_nonhuge);
928 emitter_json_kv(emitter, "ndirty_nonhuge", emitter_type_size,
929 &ndirty_nonhuge);
930 emitter_json_object_end(emitter); /* End "empty_slabs" */
931
932 COL_HDR(row, size, NULL, right, 20, size)
933 COL_HDR(row, ind, NULL, right, 4, unsigned)
934 COL_HDR(row, npageslabs_huge, NULL, right, 16, size)
935 COL_HDR(row, nactive_huge, NULL, right, 16, size)
936 COL_HDR(row, ndirty_huge, NULL, right, 16, size)
937 COL_HDR(row, npageslabs_nonhuge, NULL, right, 20, size)
938 COL_HDR(row, nactive_nonhuge, NULL, right, 20, size)
939 COL_HDR(row, ndirty_nonhuge, NULL, right, 20, size)
940 COL_HDR(row, nretained_nonhuge, NULL, right, 20, size)
941
942 size_t stats_arenas_mib[CTL_MAX_DEPTH];
943 CTL_LEAF_PREPARE(stats_arenas_mib, 0, "stats.arenas");
944 stats_arenas_mib[2] = i;
945 CTL_LEAF_PREPARE(stats_arenas_mib, 3, "hpa_shard.nonfull_slabs");
946
947 emitter_table_row(emitter, &header_row);
948 emitter_json_array_kv_begin(emitter, "nonfull_slabs");
949 bool in_gap = false;
950 for (pszind_t j = 0; j < PSSET_NPSIZES && j < SC_NPSIZES; j++) {
951 stats_arenas_mib[5] = j;
952
953 CTL_LEAF(stats_arenas_mib, 6, "npageslabs_huge",
954 &npageslabs_huge, size_t);
955 CTL_LEAF(stats_arenas_mib, 6, "nactive_huge",
956 &nactive_huge, size_t);
957 CTL_LEAF(stats_arenas_mib, 6, "ndirty_huge",
958 &ndirty_huge, size_t);
959
960 CTL_LEAF(stats_arenas_mib, 6, "npageslabs_nonhuge",
961 &npageslabs_nonhuge, size_t);
962 CTL_LEAF(stats_arenas_mib, 6, "nactive_nonhuge",
963 &nactive_nonhuge, size_t);
964 CTL_LEAF(stats_arenas_mib, 6, "ndirty_nonhuge",
965 &ndirty_nonhuge, size_t);
966 nretained_nonhuge = npageslabs_nonhuge * HUGEPAGE_PAGES
967 - nactive_nonhuge - ndirty_nonhuge;
968
969 bool in_gap_prev = in_gap;
970 in_gap = (npageslabs_huge == 0 && npageslabs_nonhuge == 0);
971 if (in_gap_prev && !in_gap) {
972 emitter_table_printf(emitter,
973 " ---\n");
974 }
975
976 col_size.size_val = sz_pind2sz(j);
977 col_ind.size_val = j;
978 col_npageslabs_huge.size_val = npageslabs_huge;
979 col_nactive_huge.size_val = nactive_huge;
980 col_ndirty_huge.size_val = ndirty_huge;
981 col_npageslabs_nonhuge.size_val = npageslabs_nonhuge;
982 col_nactive_nonhuge.size_val = nactive_nonhuge;
983 col_ndirty_nonhuge.size_val = ndirty_nonhuge;
984 col_nretained_nonhuge.size_val = nretained_nonhuge;
985 if (!in_gap) {
986 emitter_table_row(emitter, &row);
987 }
988
989 emitter_json_object_begin(emitter);
990 emitter_json_kv(emitter, "npageslabs_huge", emitter_type_size,
991 &npageslabs_huge);
992 emitter_json_kv(emitter, "nactive_huge", emitter_type_size,
993 &nactive_huge);
994 emitter_json_kv(emitter, "ndirty_huge", emitter_type_size,
995 &ndirty_huge);
996 emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size,
997 &npageslabs_nonhuge);
998 emitter_json_kv(emitter, "nactive_nonhuge", emitter_type_size,
999 &nactive_nonhuge);
1000 emitter_json_kv(emitter, "ndirty_nonhuge", emitter_type_size,
1001 &ndirty_nonhuge);
1002 emitter_json_object_end(emitter);
1003 }
1004 emitter_json_array_end(emitter); /* End "nonfull_slabs" */
1005 emitter_json_object_end(emitter); /* End "hpa_shard" */
1006 if (in_gap) {
1007 emitter_table_printf(emitter, " ---\n");
1008 }
1009}
1010
1011static void
1012stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind, uint64_t uptime) {
1013 emitter_row_t row;
1014 emitter_col_t col_name;
1015 emitter_col_t col64[mutex_prof_num_uint64_t_counters];
1016 emitter_col_t col32[mutex_prof_num_uint32_t_counters];
1017
1018 emitter_row_init(&row);
1019 mutex_stats_init_cols(&row, "", &col_name, col64, col32);
1020
1021 emitter_json_object_kv_begin(emitter, "mutexes");
1022 emitter_table_row(emitter, &row);
1023
1024 size_t stats_arenas_mib[CTL_MAX_DEPTH];
1025 CTL_LEAF_PREPARE(stats_arenas_mib, 0, "stats.arenas");
1026 stats_arenas_mib[2] = arena_ind;
1027 CTL_LEAF_PREPARE(stats_arenas_mib, 3, "mutexes");
1028
1029 for (mutex_prof_arena_ind_t i = 0; i < mutex_prof_num_arena_mutexes;
1030 i++) {
1031 const char *name = arena_mutex_names[i];
1032 emitter_json_object_kv_begin(emitter, name);
1033 mutex_stats_read_arena(stats_arenas_mib, 4, name, &col_name,
1034 col64, col32, uptime);
1035 mutex_stats_emit(emitter, &row, col64, col32);
1036 emitter_json_object_end(emitter); /* Close the mutex dict. */
1037 }
1038 emitter_json_object_end(emitter); /* End "mutexes". */
1039}
1040
1041JEMALLOC_COLD
1042static void
1043stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
1044 bool mutex, bool extents, bool hpa) {
1045 unsigned nthreads;
1046 const char *dss;
1047 ssize_t dirty_decay_ms, muzzy_decay_ms;
1048 size_t page, pactive, pdirty, pmuzzy, mapped, retained;
1049 size_t base, internal, resident, metadata_thp, extent_avail;
1050 uint64_t dirty_npurge, dirty_nmadvise, dirty_purged;
1051 uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged;
1052 size_t small_allocated;
1053 uint64_t small_nmalloc, small_ndalloc, small_nrequests, small_nfills,
1054 small_nflushes;
1055 size_t large_allocated;
1056 uint64_t large_nmalloc, large_ndalloc, large_nrequests, large_nfills,
1057 large_nflushes;
1058 size_t tcache_bytes, tcache_stashed_bytes, abandoned_vm;
1059 uint64_t uptime;
1060
1061 CTL_GET("arenas.page", &page, size_t);
1062
1063 CTL_M2_GET("stats.arenas.0.nthreads", i, &nthreads, unsigned);
1064 emitter_kv(emitter, "nthreads", "assigned threads",
1065 emitter_type_unsigned, &nthreads);
1066
1067 CTL_M2_GET("stats.arenas.0.uptime", i, &uptime, uint64_t);
1068 emitter_kv(emitter, "uptime_ns", "uptime", emitter_type_uint64,
1069 &uptime);
1070
1071 CTL_M2_GET("stats.arenas.0.dss", i, &dss, const char *);
1072 emitter_kv(emitter, "dss", "dss allocation precedence",
1073 emitter_type_string, &dss);
1074
1075 CTL_M2_GET("stats.arenas.0.dirty_decay_ms", i, &dirty_decay_ms,
1076 ssize_t);
1077 CTL_M2_GET("stats.arenas.0.muzzy_decay_ms", i, &muzzy_decay_ms,
1078 ssize_t);
1079 CTL_M2_GET("stats.arenas.0.pactive", i, &pactive, size_t);
1080 CTL_M2_GET("stats.arenas.0.pdirty", i, &pdirty, size_t);
1081 CTL_M2_GET("stats.arenas.0.pmuzzy", i, &pmuzzy, size_t);
1082 CTL_M2_GET("stats.arenas.0.dirty_npurge", i, &dirty_npurge, uint64_t);
1083 CTL_M2_GET("stats.arenas.0.dirty_nmadvise", i, &dirty_nmadvise,
1084 uint64_t);
1085 CTL_M2_GET("stats.arenas.0.dirty_purged", i, &dirty_purged, uint64_t);
1086 CTL_M2_GET("stats.arenas.0.muzzy_npurge", i, &muzzy_npurge, uint64_t);
1087 CTL_M2_GET("stats.arenas.0.muzzy_nmadvise", i, &muzzy_nmadvise,
1088 uint64_t);
1089 CTL_M2_GET("stats.arenas.0.muzzy_purged", i, &muzzy_purged, uint64_t);
1090
1091 emitter_row_t decay_row;
1092 emitter_row_init(&decay_row);
1093
1094 /* JSON-style emission. */
1095 emitter_json_kv(emitter, "dirty_decay_ms", emitter_type_ssize,
1096 &dirty_decay_ms);
1097 emitter_json_kv(emitter, "muzzy_decay_ms", emitter_type_ssize,
1098 &muzzy_decay_ms);
1099
1100 emitter_json_kv(emitter, "pactive", emitter_type_size, &pactive);
1101 emitter_json_kv(emitter, "pdirty", emitter_type_size, &pdirty);
1102 emitter_json_kv(emitter, "pmuzzy", emitter_type_size, &pmuzzy);
1103
1104 emitter_json_kv(emitter, "dirty_npurge", emitter_type_uint64,
1105 &dirty_npurge);
1106 emitter_json_kv(emitter, "dirty_nmadvise", emitter_type_uint64,
1107 &dirty_nmadvise);
1108 emitter_json_kv(emitter, "dirty_purged", emitter_type_uint64,
1109 &dirty_purged);
1110
1111 emitter_json_kv(emitter, "muzzy_npurge", emitter_type_uint64,
1112 &muzzy_npurge);
1113 emitter_json_kv(emitter, "muzzy_nmadvise", emitter_type_uint64,
1114 &muzzy_nmadvise);
1115 emitter_json_kv(emitter, "muzzy_purged", emitter_type_uint64,
1116 &muzzy_purged);
1117
1118 /* Table-style emission. */
1119 COL(decay_row, decay_type, right, 9, title);
1120 col_decay_type.str_val = "decaying:";
1121
1122 COL(decay_row, decay_time, right, 6, title);
1123 col_decay_time.str_val = "time";
1124
1125 COL(decay_row, decay_npages, right, 13, title);
1126 col_decay_npages.str_val = "npages";
1127
1128 COL(decay_row, decay_sweeps, right, 13, title);
1129 col_decay_sweeps.str_val = "sweeps";
1130
1131 COL(decay_row, decay_madvises, right, 13, title);
1132 col_decay_madvises.str_val = "madvises";
1133
1134 COL(decay_row, decay_purged, right, 13, title);
1135 col_decay_purged.str_val = "purged";
1136
1137 /* Title row. */
1138 emitter_table_row(emitter, &decay_row);
1139
1140 /* Dirty row. */
1141 col_decay_type.str_val = "dirty:";
1142
1143 if (dirty_decay_ms >= 0) {
1144 col_decay_time.type = emitter_type_ssize;
1145 col_decay_time.ssize_val = dirty_decay_ms;
1146 } else {
1147 col_decay_time.type = emitter_type_title;
1148 col_decay_time.str_val = "N/A";
1149 }
1150
1151 col_decay_npages.type = emitter_type_size;
1152 col_decay_npages.size_val = pdirty;
1153
1154 col_decay_sweeps.type = emitter_type_uint64;
1155 col_decay_sweeps.uint64_val = dirty_npurge;
1156
1157 col_decay_madvises.type = emitter_type_uint64;
1158 col_decay_madvises.uint64_val = dirty_nmadvise;
1159
1160 col_decay_purged.type = emitter_type_uint64;
1161 col_decay_purged.uint64_val = dirty_purged;
1162
1163 emitter_table_row(emitter, &decay_row);
1164
1165 /* Muzzy row. */
1166 col_decay_type.str_val = "muzzy:";
1167
1168 if (muzzy_decay_ms >= 0) {
1169 col_decay_time.type = emitter_type_ssize;
1170 col_decay_time.ssize_val = muzzy_decay_ms;
1171 } else {
1172 col_decay_time.type = emitter_type_title;
1173 col_decay_time.str_val = "N/A";
1174 }
1175
1176 col_decay_npages.type = emitter_type_size;
1177 col_decay_npages.size_val = pmuzzy;
1178
1179 col_decay_sweeps.type = emitter_type_uint64;
1180 col_decay_sweeps.uint64_val = muzzy_npurge;
1181
1182 col_decay_madvises.type = emitter_type_uint64;
1183 col_decay_madvises.uint64_val = muzzy_nmadvise;
1184
1185 col_decay_purged.type = emitter_type_uint64;
1186 col_decay_purged.uint64_val = muzzy_purged;
1187
1188 emitter_table_row(emitter, &decay_row);
1189
1190 /* Small / large / total allocation counts. */
1191 emitter_row_t alloc_count_row;
1192 emitter_row_init(&alloc_count_row);
1193
1194 COL(alloc_count_row, count_title, left, 21, title);
1195 col_count_title.str_val = "";
1196
1197 COL(alloc_count_row, count_allocated, right, 16, title);
1198 col_count_allocated.str_val = "allocated";
1199
1200 COL(alloc_count_row, count_nmalloc, right, 16, title);
1201 col_count_nmalloc.str_val = "nmalloc";
1202 COL(alloc_count_row, count_nmalloc_ps, right, 10, title);
1203 col_count_nmalloc_ps.str_val = "(#/sec)";
1204
1205 COL(alloc_count_row, count_ndalloc, right, 16, title);
1206 col_count_ndalloc.str_val = "ndalloc";
1207 COL(alloc_count_row, count_ndalloc_ps, right, 10, title);
1208 col_count_ndalloc_ps.str_val = "(#/sec)";
1209
1210 COL(alloc_count_row, count_nrequests, right, 16, title);
1211 col_count_nrequests.str_val = "nrequests";
1212 COL(alloc_count_row, count_nrequests_ps, right, 10, title);
1213 col_count_nrequests_ps.str_val = "(#/sec)";
1214
1215 COL(alloc_count_row, count_nfills, right, 16, title);
1216 col_count_nfills.str_val = "nfill";
1217 COL(alloc_count_row, count_nfills_ps, right, 10, title);
1218 col_count_nfills_ps.str_val = "(#/sec)";
1219
1220 COL(alloc_count_row, count_nflushes, right, 16, title);
1221 col_count_nflushes.str_val = "nflush";
1222 COL(alloc_count_row, count_nflushes_ps, right, 10, title);
1223 col_count_nflushes_ps.str_val = "(#/sec)";
1224
1225 emitter_table_row(emitter, &alloc_count_row);
1226
1227 col_count_nmalloc_ps.type = emitter_type_uint64;
1228 col_count_ndalloc_ps.type = emitter_type_uint64;
1229 col_count_nrequests_ps.type = emitter_type_uint64;
1230 col_count_nfills_ps.type = emitter_type_uint64;
1231 col_count_nflushes_ps.type = emitter_type_uint64;
1232
1233#define GET_AND_EMIT_ALLOC_STAT(small_or_large, name, valtype) \
1234 CTL_M2_GET("stats.arenas.0." #small_or_large "." #name, i, \
1235 &small_or_large##_##name, valtype##_t); \
1236 emitter_json_kv(emitter, #name, emitter_type_##valtype, \
1237 &small_or_large##_##name); \
1238 col_count_##name.type = emitter_type_##valtype; \
1239 col_count_##name.valtype##_val = small_or_large##_##name;
1240
1241 emitter_json_object_kv_begin(emitter, "small");
1242 col_count_title.str_val = "small:";
1243
1244 GET_AND_EMIT_ALLOC_STAT(small, allocated, size)
1245 GET_AND_EMIT_ALLOC_STAT(small, nmalloc, uint64)
1246 col_count_nmalloc_ps.uint64_val =
1247 rate_per_second(col_count_nmalloc.uint64_val, uptime);
1248 GET_AND_EMIT_ALLOC_STAT(small, ndalloc, uint64)
1249 col_count_ndalloc_ps.uint64_val =
1250 rate_per_second(col_count_ndalloc.uint64_val, uptime);
1251 GET_AND_EMIT_ALLOC_STAT(small, nrequests, uint64)
1252 col_count_nrequests_ps.uint64_val =
1253 rate_per_second(col_count_nrequests.uint64_val, uptime);
1254 GET_AND_EMIT_ALLOC_STAT(small, nfills, uint64)
1255 col_count_nfills_ps.uint64_val =
1256 rate_per_second(col_count_nfills.uint64_val, uptime);
1257 GET_AND_EMIT_ALLOC_STAT(small, nflushes, uint64)
1258 col_count_nflushes_ps.uint64_val =
1259 rate_per_second(col_count_nflushes.uint64_val, uptime);
1260
1261 emitter_table_row(emitter, &alloc_count_row);
1262 emitter_json_object_end(emitter); /* Close "small". */
1263
1264 emitter_json_object_kv_begin(emitter, "large");
1265 col_count_title.str_val = "large:";
1266
1267 GET_AND_EMIT_ALLOC_STAT(large, allocated, size)
1268 GET_AND_EMIT_ALLOC_STAT(large, nmalloc, uint64)
1269 col_count_nmalloc_ps.uint64_val =
1270 rate_per_second(col_count_nmalloc.uint64_val, uptime);
1271 GET_AND_EMIT_ALLOC_STAT(large, ndalloc, uint64)
1272 col_count_ndalloc_ps.uint64_val =
1273 rate_per_second(col_count_ndalloc.uint64_val, uptime);
1274 GET_AND_EMIT_ALLOC_STAT(large, nrequests, uint64)
1275 col_count_nrequests_ps.uint64_val =
1276 rate_per_second(col_count_nrequests.uint64_val, uptime);
1277 GET_AND_EMIT_ALLOC_STAT(large, nfills, uint64)
1278 col_count_nfills_ps.uint64_val =
1279 rate_per_second(col_count_nfills.uint64_val, uptime);
1280 GET_AND_EMIT_ALLOC_STAT(large, nflushes, uint64)
1281 col_count_nflushes_ps.uint64_val =
1282 rate_per_second(col_count_nflushes.uint64_val, uptime);
1283
1284 emitter_table_row(emitter, &alloc_count_row);
1285 emitter_json_object_end(emitter); /* Close "large". */
1286
1287#undef GET_AND_EMIT_ALLOC_STAT
1288
1289 /* Aggregated small + large stats are emitter only in table mode. */
1290 col_count_title.str_val = "total:";
1291 col_count_allocated.size_val = small_allocated + large_allocated;
1292 col_count_nmalloc.uint64_val = small_nmalloc + large_nmalloc;
1293 col_count_ndalloc.uint64_val = small_ndalloc + large_ndalloc;
1294 col_count_nrequests.uint64_val = small_nrequests + large_nrequests;
1295 col_count_nfills.uint64_val = small_nfills + large_nfills;
1296 col_count_nflushes.uint64_val = small_nflushes + large_nflushes;
1297 col_count_nmalloc_ps.uint64_val =
1298 rate_per_second(col_count_nmalloc.uint64_val, uptime);
1299 col_count_ndalloc_ps.uint64_val =
1300 rate_per_second(col_count_ndalloc.uint64_val, uptime);
1301 col_count_nrequests_ps.uint64_val =
1302 rate_per_second(col_count_nrequests.uint64_val, uptime);
1303 col_count_nfills_ps.uint64_val =
1304 rate_per_second(col_count_nfills.uint64_val, uptime);
1305 col_count_nflushes_ps.uint64_val =
1306 rate_per_second(col_count_nflushes.uint64_val, uptime);
1307 emitter_table_row(emitter, &alloc_count_row);
1308
1309 emitter_row_t mem_count_row;
1310 emitter_row_init(&mem_count_row);
1311
1312 emitter_col_t mem_count_title;
1313 emitter_col_init(&mem_count_title, &mem_count_row);
1314 mem_count_title.justify = emitter_justify_left;
1315 mem_count_title.width = 21;
1316 mem_count_title.type = emitter_type_title;
1317 mem_count_title.str_val = "";
1318
1319 emitter_col_t mem_count_val;
1320 emitter_col_init(&mem_count_val, &mem_count_row);
1321 mem_count_val.justify = emitter_justify_right;
1322 mem_count_val.width = 16;
1323 mem_count_val.type = emitter_type_title;
1324 mem_count_val.str_val = "";
1325
1326 emitter_table_row(emitter, &mem_count_row);
1327 mem_count_val.type = emitter_type_size;
1328
1329 /* Active count in bytes is emitted only in table mode. */
1330 mem_count_title.str_val = "active:";
1331 mem_count_val.size_val = pactive * page;
1332 emitter_table_row(emitter, &mem_count_row);
1333
1334#define GET_AND_EMIT_MEM_STAT(stat) \
1335 CTL_M2_GET("stats.arenas.0."#stat, i, &stat, size_t); \
1336 emitter_json_kv(emitter, #stat, emitter_type_size, &stat); \
1337 mem_count_title.str_val = #stat":"; \
1338 mem_count_val.size_val = stat; \
1339 emitter_table_row(emitter, &mem_count_row);
1340
1341 GET_AND_EMIT_MEM_STAT(mapped)
1342 GET_AND_EMIT_MEM_STAT(retained)
1343 GET_AND_EMIT_MEM_STAT(base)
1344 GET_AND_EMIT_MEM_STAT(internal)
1345 GET_AND_EMIT_MEM_STAT(metadata_thp)
1346 GET_AND_EMIT_MEM_STAT(tcache_bytes)
1347 GET_AND_EMIT_MEM_STAT(tcache_stashed_bytes)
1348 GET_AND_EMIT_MEM_STAT(resident)
1349 GET_AND_EMIT_MEM_STAT(abandoned_vm)
1350 GET_AND_EMIT_MEM_STAT(extent_avail)
1351#undef GET_AND_EMIT_MEM_STAT
1352
1353 if (mutex) {
1354 stats_arena_mutexes_print(emitter, i, uptime);
1355 }
1356 if (bins) {
1357 stats_arena_bins_print(emitter, mutex, i, uptime);
1358 }
1359 if (large) {
1360 stats_arena_lextents_print(emitter, i, uptime);
1361 }
1362 if (extents) {
1363 stats_arena_extents_print(emitter, i);
1364 }
1365 if (hpa) {
1366 stats_arena_hpa_shard_print(emitter, i, uptime);
1367 }
1368}
1369
1370JEMALLOC_COLD
1371static void
1372stats_general_print(emitter_t *emitter) {
1373 const char *cpv;
1374 bool bv, bv2;
1375 unsigned uv;
1376 uint32_t u32v;
1377 uint64_t u64v;
1378 int64_t i64v;
1379 ssize_t ssv, ssv2;
1380 size_t sv, bsz, usz, u32sz, u64sz, i64sz, ssz, sssz, cpsz;
1381
1382 bsz = sizeof(bool);
1383 usz = sizeof(unsigned);
1384 ssz = sizeof(size_t);
1385 sssz = sizeof(ssize_t);
1386 cpsz = sizeof(const char *);
1387 u32sz = sizeof(uint32_t);
1388 i64sz = sizeof(int64_t);
1389 u64sz = sizeof(uint64_t);
1390
1391 CTL_GET("version", &cpv, const char *);
1392 emitter_kv(emitter, "version", "Version", emitter_type_string, &cpv);
1393
1394 /* config. */
1395 emitter_dict_begin(emitter, "config", "Build-time option settings");
1396#define CONFIG_WRITE_BOOL(name) \
1397 do { \
1398 CTL_GET("config."#name, &bv, bool); \
1399 emitter_kv(emitter, #name, "config."#name, \
1400 emitter_type_bool, &bv); \
1401 } while (0)
1402
1403 CONFIG_WRITE_BOOL(cache_oblivious);
1404 CONFIG_WRITE_BOOL(debug);
1405 CONFIG_WRITE_BOOL(fill);
1406 CONFIG_WRITE_BOOL(lazy_lock);
1407 emitter_kv(emitter, "malloc_conf", "config.malloc_conf",
1408 emitter_type_string, &config_malloc_conf);
1409
1410 CONFIG_WRITE_BOOL(opt_safety_checks);
1411 CONFIG_WRITE_BOOL(prof);
1412 CONFIG_WRITE_BOOL(prof_libgcc);
1413 CONFIG_WRITE_BOOL(prof_libunwind);
1414 CONFIG_WRITE_BOOL(stats);
1415 CONFIG_WRITE_BOOL(utrace);
1416 CONFIG_WRITE_BOOL(xmalloc);
1417#undef CONFIG_WRITE_BOOL
1418 emitter_dict_end(emitter); /* Close "config" dict. */
1419
1420 /* opt. */
1421#define OPT_WRITE(name, var, size, emitter_type) \
1422 if (je_mallctl("opt."name, (void *)&var, &size, NULL, 0) == \
1423 0) { \
1424 emitter_kv(emitter, name, "opt."name, emitter_type, \
1425 &var); \
1426 }
1427
1428#define OPT_WRITE_MUTABLE(name, var1, var2, size, emitter_type, \
1429 altname) \
1430 if (je_mallctl("opt."name, (void *)&var1, &size, NULL, 0) == \
1431 0 && je_mallctl(altname, (void *)&var2, &size, NULL, 0) \
1432 == 0) { \
1433 emitter_kv_note(emitter, name, "opt."name, \
1434 emitter_type, &var1, altname, emitter_type, \
1435 &var2); \
1436 }
1437
1438#define OPT_WRITE_BOOL(name) OPT_WRITE(name, bv, bsz, emitter_type_bool)
1439#define OPT_WRITE_BOOL_MUTABLE(name, altname) \
1440 OPT_WRITE_MUTABLE(name, bv, bv2, bsz, emitter_type_bool, altname)
1441
1442#define OPT_WRITE_UNSIGNED(name) \
1443 OPT_WRITE(name, uv, usz, emitter_type_unsigned)
1444
1445#define OPT_WRITE_INT64(name) \
1446 OPT_WRITE(name, i64v, i64sz, emitter_type_int64)
1447#define OPT_WRITE_UINT64(name) \
1448 OPT_WRITE(name, u64v, u64sz, emitter_type_uint64)
1449
1450#define OPT_WRITE_SIZE_T(name) \
1451 OPT_WRITE(name, sv, ssz, emitter_type_size)
1452#define OPT_WRITE_SSIZE_T(name) \
1453 OPT_WRITE(name, ssv, sssz, emitter_type_ssize)
1454#define OPT_WRITE_SSIZE_T_MUTABLE(name, altname) \
1455 OPT_WRITE_MUTABLE(name, ssv, ssv2, sssz, emitter_type_ssize, \
1456 altname)
1457
1458#define OPT_WRITE_CHAR_P(name) \
1459 OPT_WRITE(name, cpv, cpsz, emitter_type_string)
1460
1461 emitter_dict_begin(emitter, "opt", "Run-time option settings");
1462
1463 OPT_WRITE_BOOL("abort")
1464 OPT_WRITE_BOOL("abort_conf")
1465 OPT_WRITE_BOOL("cache_oblivious")
1466 OPT_WRITE_BOOL("confirm_conf")
1467 OPT_WRITE_BOOL("retain")
1468 OPT_WRITE_CHAR_P("dss")
1469 OPT_WRITE_UNSIGNED("narenas")
1470 OPT_WRITE_CHAR_P("percpu_arena")
1471 OPT_WRITE_SIZE_T("oversize_threshold")
1472 OPT_WRITE_BOOL("hpa")
1473 OPT_WRITE_SIZE_T("hpa_slab_max_alloc")
1474 OPT_WRITE_SIZE_T("hpa_hugification_threshold")
1475 OPT_WRITE_UINT64("hpa_hugify_delay_ms")
1476 OPT_WRITE_UINT64("hpa_min_purge_interval_ms")
1477 if (je_mallctl("opt.hpa_dirty_mult", (void *)&u32v, &u32sz, NULL, 0)
1478 == 0) {
1479 /*
1480 * We cheat a little and "know" the secret meaning of this
1481 * representation.
1482 */
1483 if (u32v == (uint32_t)-1) {
1484 const char *neg1 = "-1";
1485 emitter_kv(emitter, "hpa_dirty_mult",
1486 "opt.hpa_dirty_mult", emitter_type_string, &neg1);
1487 } else {
1488 char buf[FXP_BUF_SIZE];
1489 fxp_print(u32v, buf);
1490 const char *bufp = buf;
1491 emitter_kv(emitter, "hpa_dirty_mult",
1492 "opt.hpa_dirty_mult", emitter_type_string, &bufp);
1493 }
1494 }
1495 OPT_WRITE_SIZE_T("hpa_sec_nshards")
1496 OPT_WRITE_SIZE_T("hpa_sec_max_alloc")
1497 OPT_WRITE_SIZE_T("hpa_sec_max_bytes")
1498 OPT_WRITE_SIZE_T("hpa_sec_bytes_after_flush")
1499 OPT_WRITE_SIZE_T("hpa_sec_batch_fill_extra")
1500 OPT_WRITE_CHAR_P("metadata_thp")
1501 OPT_WRITE_INT64("mutex_max_spin")
1502 OPT_WRITE_BOOL_MUTABLE("background_thread", "background_thread")
1503 OPT_WRITE_SSIZE_T_MUTABLE("dirty_decay_ms", "arenas.dirty_decay_ms")
1504 OPT_WRITE_SSIZE_T_MUTABLE("muzzy_decay_ms", "arenas.muzzy_decay_ms")
1505 OPT_WRITE_SIZE_T("lg_extent_max_active_fit")
1506 OPT_WRITE_CHAR_P("junk")
1507 OPT_WRITE_BOOL("zero")
1508 OPT_WRITE_BOOL("utrace")
1509 OPT_WRITE_BOOL("xmalloc")
1510 OPT_WRITE_BOOL("experimental_infallible_new")
1511 OPT_WRITE_BOOL("tcache")
1512 OPT_WRITE_SIZE_T("tcache_max")
1513 OPT_WRITE_UNSIGNED("tcache_nslots_small_min")
1514 OPT_WRITE_UNSIGNED("tcache_nslots_small_max")
1515 OPT_WRITE_UNSIGNED("tcache_nslots_large")
1516 OPT_WRITE_SSIZE_T("lg_tcache_nslots_mul")
1517 OPT_WRITE_SIZE_T("tcache_gc_incr_bytes")
1518 OPT_WRITE_SIZE_T("tcache_gc_delay_bytes")
1519 OPT_WRITE_UNSIGNED("lg_tcache_flush_small_div")
1520 OPT_WRITE_UNSIGNED("lg_tcache_flush_large_div")
1521 OPT_WRITE_CHAR_P("thp")
1522 OPT_WRITE_BOOL("prof")
1523 OPT_WRITE_CHAR_P("prof_prefix")
1524 OPT_WRITE_BOOL_MUTABLE("prof_active", "prof.active")
1525 OPT_WRITE_BOOL_MUTABLE("prof_thread_active_init",
1526 "prof.thread_active_init")
1527 OPT_WRITE_SSIZE_T_MUTABLE("lg_prof_sample", "prof.lg_sample")
1528 OPT_WRITE_BOOL("prof_accum")
1529 OPT_WRITE_SSIZE_T("lg_prof_interval")
1530 OPT_WRITE_BOOL("prof_gdump")
1531 OPT_WRITE_BOOL("prof_final")
1532 OPT_WRITE_BOOL("prof_leak")
1533 OPT_WRITE_BOOL("prof_leak_error")
1534 OPT_WRITE_BOOL("stats_print")
1535 OPT_WRITE_CHAR_P("stats_print_opts")
1536 OPT_WRITE_BOOL("stats_print")
1537 OPT_WRITE_CHAR_P("stats_print_opts")
1538 OPT_WRITE_INT64("stats_interval")
1539 OPT_WRITE_CHAR_P("stats_interval_opts")
1540 OPT_WRITE_CHAR_P("zero_realloc")
1541
1542 emitter_dict_end(emitter);
1543
1544#undef OPT_WRITE
1545#undef OPT_WRITE_MUTABLE
1546#undef OPT_WRITE_BOOL
1547#undef OPT_WRITE_BOOL_MUTABLE
1548#undef OPT_WRITE_UNSIGNED
1549#undef OPT_WRITE_SSIZE_T
1550#undef OPT_WRITE_SSIZE_T_MUTABLE
1551#undef OPT_WRITE_CHAR_P
1552
1553 /* prof. */
1554 if (config_prof) {
1555 emitter_dict_begin(emitter, "prof", "Profiling settings");
1556
1557 CTL_GET("prof.thread_active_init", &bv, bool);
1558 emitter_kv(emitter, "thread_active_init",
1559 "prof.thread_active_init", emitter_type_bool, &bv);
1560
1561 CTL_GET("prof.active", &bv, bool);
1562 emitter_kv(emitter, "active", "prof.active", emitter_type_bool,
1563 &bv);
1564
1565 CTL_GET("prof.gdump", &bv, bool);
1566 emitter_kv(emitter, "gdump", "prof.gdump", emitter_type_bool,
1567 &bv);
1568
1569 CTL_GET("prof.interval", &u64v, uint64_t);
1570 emitter_kv(emitter, "interval", "prof.interval",
1571 emitter_type_uint64, &u64v);
1572
1573 CTL_GET("prof.lg_sample", &ssv, ssize_t);
1574 emitter_kv(emitter, "lg_sample", "prof.lg_sample",
1575 emitter_type_ssize, &ssv);
1576
1577 emitter_dict_end(emitter); /* Close "prof". */
1578 }
1579
1580 /* arenas. */
1581 /*
1582 * The json output sticks arena info into an "arenas" dict; the table
1583 * output puts them at the top-level.
1584 */
1585 emitter_json_object_kv_begin(emitter, "arenas");
1586
1587 CTL_GET("arenas.narenas", &uv, unsigned);
1588 emitter_kv(emitter, "narenas", "Arenas", emitter_type_unsigned, &uv);
1589
1590 /*
1591 * Decay settings are emitted only in json mode; in table mode, they're
1592 * emitted as notes with the opt output, above.
1593 */
1594 CTL_GET("arenas.dirty_decay_ms", &ssv, ssize_t);
1595 emitter_json_kv(emitter, "dirty_decay_ms", emitter_type_ssize, &ssv);
1596
1597 CTL_GET("arenas.muzzy_decay_ms", &ssv, ssize_t);
1598 emitter_json_kv(emitter, "muzzy_decay_ms", emitter_type_ssize, &ssv);
1599
1600 CTL_GET("arenas.quantum", &sv, size_t);
1601 emitter_kv(emitter, "quantum", "Quantum size", emitter_type_size, &sv);
1602
1603 CTL_GET("arenas.page", &sv, size_t);
1604 emitter_kv(emitter, "page", "Page size", emitter_type_size, &sv);
1605
1606 if (je_mallctl("arenas.tcache_max", (void *)&sv, &ssz, NULL, 0) == 0) {
1607 emitter_kv(emitter, "tcache_max",
1608 "Maximum thread-cached size class", emitter_type_size, &sv);
1609 }
1610
1611 unsigned arenas_nbins;
1612 CTL_GET("arenas.nbins", &arenas_nbins, unsigned);
1613 emitter_kv(emitter, "nbins", "Number of bin size classes",
1614 emitter_type_unsigned, &arenas_nbins);
1615
1616 unsigned arenas_nhbins;
1617 CTL_GET("arenas.nhbins", &arenas_nhbins, unsigned);
1618 emitter_kv(emitter, "nhbins", "Number of thread-cache bin size classes",
1619 emitter_type_unsigned, &arenas_nhbins);
1620
1621 /*
1622 * We do enough mallctls in a loop that we actually want to omit them
1623 * (not just omit the printing).
1624 */
1625 if (emitter_outputs_json(emitter)) {
1626 emitter_json_array_kv_begin(emitter, "bin");
1627 size_t arenas_bin_mib[CTL_MAX_DEPTH];
1628 CTL_LEAF_PREPARE(arenas_bin_mib, 0, "arenas.bin");
1629 for (unsigned i = 0; i < arenas_nbins; i++) {
1630 arenas_bin_mib[2] = i;
1631 emitter_json_object_begin(emitter);
1632
1633 CTL_LEAF(arenas_bin_mib, 3, "size", &sv, size_t);
1634 emitter_json_kv(emitter, "size", emitter_type_size,
1635 &sv);
1636
1637 CTL_LEAF(arenas_bin_mib, 3, "nregs", &u32v, uint32_t);
1638 emitter_json_kv(emitter, "nregs", emitter_type_uint32,
1639 &u32v);
1640
1641 CTL_LEAF(arenas_bin_mib, 3, "slab_size", &sv, size_t);
1642 emitter_json_kv(emitter, "slab_size", emitter_type_size,
1643 &sv);
1644
1645 CTL_LEAF(arenas_bin_mib, 3, "nshards", &u32v, uint32_t);
1646 emitter_json_kv(emitter, "nshards", emitter_type_uint32,
1647 &u32v);
1648
1649 emitter_json_object_end(emitter);
1650 }
1651 emitter_json_array_end(emitter); /* Close "bin". */
1652 }
1653
1654 unsigned nlextents;
1655 CTL_GET("arenas.nlextents", &nlextents, unsigned);
1656 emitter_kv(emitter, "nlextents", "Number of large size classes",
1657 emitter_type_unsigned, &nlextents);
1658
1659 if (emitter_outputs_json(emitter)) {
1660 emitter_json_array_kv_begin(emitter, "lextent");
1661 size_t arenas_lextent_mib[CTL_MAX_DEPTH];
1662 CTL_LEAF_PREPARE(arenas_lextent_mib, 0, "arenas.lextent");
1663 for (unsigned i = 0; i < nlextents; i++) {
1664 arenas_lextent_mib[2] = i;
1665 emitter_json_object_begin(emitter);
1666
1667 CTL_LEAF(arenas_lextent_mib, 3, "size", &sv, size_t);
1668 emitter_json_kv(emitter, "size", emitter_type_size,
1669 &sv);
1670
1671 emitter_json_object_end(emitter);
1672 }
1673 emitter_json_array_end(emitter); /* Close "lextent". */
1674 }
1675
1676 emitter_json_object_end(emitter); /* Close "arenas" */
1677}
1678
1679JEMALLOC_COLD
1680static void
1681stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
1682 bool unmerged, bool bins, bool large, bool mutex, bool extents, bool hpa) {
1683 /*
1684 * These should be deleted. We keep them around for a while, to aid in
1685 * the transition to the emitter code.
1686 */
1687 size_t allocated, active, metadata, metadata_thp, resident, mapped,
1688 retained;
1689 size_t num_background_threads;
1690 size_t zero_reallocs;
1691 uint64_t background_thread_num_runs, background_thread_run_interval;
1692
1693 CTL_GET("stats.allocated", &allocated, size_t);
1694 CTL_GET("stats.active", &active, size_t);
1695 CTL_GET("stats.metadata", &metadata, size_t);
1696 CTL_GET("stats.metadata_thp", &metadata_thp, size_t);
1697 CTL_GET("stats.resident", &resident, size_t);
1698 CTL_GET("stats.mapped", &mapped, size_t);
1699 CTL_GET("stats.retained", &retained, size_t);
1700
1701 CTL_GET("stats.zero_reallocs", &zero_reallocs, size_t);
1702
1703 if (have_background_thread) {
1704 CTL_GET("stats.background_thread.num_threads",
1705 &num_background_threads, size_t);
1706 CTL_GET("stats.background_thread.num_runs",
1707 &background_thread_num_runs, uint64_t);
1708 CTL_GET("stats.background_thread.run_interval",
1709 &background_thread_run_interval, uint64_t);
1710 } else {
1711 num_background_threads = 0;
1712 background_thread_num_runs = 0;
1713 background_thread_run_interval = 0;
1714 }
1715
1716 /* Generic global stats. */
1717 emitter_json_object_kv_begin(emitter, "stats");
1718 emitter_json_kv(emitter, "allocated", emitter_type_size, &allocated);
1719 emitter_json_kv(emitter, "active", emitter_type_size, &active);
1720 emitter_json_kv(emitter, "metadata", emitter_type_size, &metadata);
1721 emitter_json_kv(emitter, "metadata_thp", emitter_type_size,
1722 &metadata_thp);
1723 emitter_json_kv(emitter, "resident", emitter_type_size, &resident);
1724 emitter_json_kv(emitter, "mapped", emitter_type_size, &mapped);
1725 emitter_json_kv(emitter, "retained", emitter_type_size, &retained);
1726 emitter_json_kv(emitter, "zero_reallocs", emitter_type_size,
1727 &zero_reallocs);
1728
1729 emitter_table_printf(emitter, "Allocated: %zu, active: %zu, "
1730 "metadata: %zu (n_thp %zu), resident: %zu, mapped: %zu, "
1731 "retained: %zu\n", allocated, active, metadata, metadata_thp,
1732 resident, mapped, retained);
1733
1734 /* Strange behaviors */
1735 emitter_table_printf(emitter,
1736 "Count of realloc(non-null-ptr, 0) calls: %zu\n", zero_reallocs);
1737
1738 /* Background thread stats. */
1739 emitter_json_object_kv_begin(emitter, "background_thread");
1740 emitter_json_kv(emitter, "num_threads", emitter_type_size,
1741 &num_background_threads);
1742 emitter_json_kv(emitter, "num_runs", emitter_type_uint64,
1743 &background_thread_num_runs);
1744 emitter_json_kv(emitter, "run_interval", emitter_type_uint64,
1745 &background_thread_run_interval);
1746 emitter_json_object_end(emitter); /* Close "background_thread". */
1747
1748 emitter_table_printf(emitter, "Background threads: %zu, "
1749 "num_runs: %"FMTu64", run_interval: %"FMTu64" ns\n",
1750 num_background_threads, background_thread_num_runs,
1751 background_thread_run_interval);
1752
1753 if (mutex) {
1754 emitter_row_t row;
1755 emitter_col_t name;
1756 emitter_col_t col64[mutex_prof_num_uint64_t_counters];
1757 emitter_col_t col32[mutex_prof_num_uint32_t_counters];
1758 uint64_t uptime;
1759
1760 emitter_row_init(&row);
1761 mutex_stats_init_cols(&row, "", &name, col64, col32);
1762
1763 emitter_table_row(emitter, &row);
1764 emitter_json_object_kv_begin(emitter, "mutexes");
1765
1766 CTL_M2_GET("stats.arenas.0.uptime", 0, &uptime, uint64_t);
1767
1768 size_t stats_mutexes_mib[CTL_MAX_DEPTH];
1769 CTL_LEAF_PREPARE(stats_mutexes_mib, 0, "stats.mutexes");
1770 for (int i = 0; i < mutex_prof_num_global_mutexes; i++) {
1771 mutex_stats_read_global(stats_mutexes_mib, 2,
1772 global_mutex_names[i], &name, col64, col32, uptime);
1773 emitter_json_object_kv_begin(emitter, global_mutex_names[i]);
1774 mutex_stats_emit(emitter, &row, col64, col32);
1775 emitter_json_object_end(emitter);
1776 }
1777
1778 emitter_json_object_end(emitter); /* Close "mutexes". */
1779 }
1780
1781 emitter_json_object_end(emitter); /* Close "stats". */
1782
1783 if (merged || destroyed || unmerged) {
1784 unsigned narenas;
1785
1786 emitter_json_object_kv_begin(emitter, "stats.arenas");
1787
1788 CTL_GET("arenas.narenas", &narenas, unsigned);
1789 size_t mib[3];
1790 size_t miblen = sizeof(mib) / sizeof(size_t);
1791 size_t sz;
1792 VARIABLE_ARRAY(bool, initialized, narenas);
1793 bool destroyed_initialized;
1794 unsigned i, j, ninitialized;
1795
1796 xmallctlnametomib("arena.0.initialized", mib, &miblen);
1797 for (i = ninitialized = 0; i < narenas; i++) {
1798 mib[1] = i;
1799 sz = sizeof(bool);
1800 xmallctlbymib(mib, miblen, &initialized[i], &sz,
1801 NULL, 0);
1802 if (initialized[i]) {
1803 ninitialized++;
1804 }
1805 }
1806 mib[1] = MALLCTL_ARENAS_DESTROYED;
1807 sz = sizeof(bool);
1808 xmallctlbymib(mib, miblen, &destroyed_initialized, &sz,
1809 NULL, 0);
1810
1811 /* Merged stats. */
1812 if (merged && (ninitialized > 1 || !unmerged)) {
1813 /* Print merged arena stats. */
1814 emitter_table_printf(emitter, "Merged arenas stats:\n");
1815 emitter_json_object_kv_begin(emitter, "merged");
1816 stats_arena_print(emitter, MALLCTL_ARENAS_ALL, bins,
1817 large, mutex, extents, hpa);
1818 emitter_json_object_end(emitter); /* Close "merged". */
1819 }
1820
1821 /* Destroyed stats. */
1822 if (destroyed_initialized && destroyed) {
1823 /* Print destroyed arena stats. */
1824 emitter_table_printf(emitter,
1825 "Destroyed arenas stats:\n");
1826 emitter_json_object_kv_begin(emitter, "destroyed");
1827 stats_arena_print(emitter, MALLCTL_ARENAS_DESTROYED,
1828 bins, large, mutex, extents, hpa);
1829 emitter_json_object_end(emitter); /* Close "destroyed". */
1830 }
1831
1832 /* Unmerged stats. */
1833 if (unmerged) {
1834 for (i = j = 0; i < narenas; i++) {
1835 if (initialized[i]) {
1836 char arena_ind_str[20];
1837 malloc_snprintf(arena_ind_str,
1838 sizeof(arena_ind_str), "%u", i);
1839 emitter_json_object_kv_begin(emitter,
1840 arena_ind_str);
1841 emitter_table_printf(emitter,
1842 "arenas[%s]:\n", arena_ind_str);
1843 stats_arena_print(emitter, i, bins,
1844 large, mutex, extents, hpa);
1845 /* Close "<arena-ind>". */
1846 emitter_json_object_end(emitter);
1847 }
1848 }
1849 }
1850 emitter_json_object_end(emitter); /* Close "stats.arenas". */
1851 }
1852}
1853
1854void
1855stats_print(write_cb_t *write_cb, void *cbopaque, const char *opts) {
1856 int err;
1857 uint64_t epoch;
1858 size_t u64sz;
1859#define OPTION(o, v, d, s) bool v = d;
1860 STATS_PRINT_OPTIONS
1861#undef OPTION
1862
1863 /*
1864 * Refresh stats, in case mallctl() was called by the application.
1865 *
1866 * Check for OOM here, since refreshing the ctl cache can trigger
1867 * allocation. In practice, none of the subsequent mallctl()-related
1868 * calls in this function will cause OOM if this one succeeds.
1869 * */
1870 epoch = 1;
1871 u64sz = sizeof(uint64_t);
1872 err = je_mallctl("epoch", (void *)&epoch, &u64sz, (void *)&epoch,
1873 sizeof(uint64_t));
1874 if (err != 0) {
1875 if (err == EAGAIN) {
1876 malloc_write("<jemalloc>: Memory allocation failure in "
1877 "mallctl(\"epoch\", ...)\n");
1878 return;
1879 }
1880 malloc_write("<jemalloc>: Failure in mallctl(\"epoch\", "
1881 "...)\n");
1882 abort();
1883 }
1884
1885 if (opts != NULL) {
1886 for (unsigned i = 0; opts[i] != '\0'; i++) {
1887 switch (opts[i]) {
1888#define OPTION(o, v, d, s) case o: v = s; break;
1889 STATS_PRINT_OPTIONS
1890#undef OPTION
1891 default:;
1892 }
1893 }
1894 }
1895
1896 emitter_t emitter;
1897 emitter_init(&emitter,
1898 json ? emitter_output_json_compact : emitter_output_table,
1899 write_cb, cbopaque);
1900 emitter_begin(&emitter);
1901 emitter_table_printf(&emitter, "___ Begin jemalloc statistics ___\n");
1902 emitter_json_object_kv_begin(&emitter, "jemalloc");
1903
1904 if (general) {
1905 stats_general_print(&emitter);
1906 }
1907 if (config_stats) {
1908 stats_print_helper(&emitter, merged, destroyed, unmerged,
1909 bins, large, mutex, extents, hpa);
1910 }
1911
1912 emitter_json_object_end(&emitter); /* Closes the "jemalloc" dict. */
1913 emitter_table_printf(&emitter, "--- End jemalloc statistics ---\n");
1914 emitter_end(&emitter);
1915}
1916
1917uint64_t
1918stats_interval_new_event_wait(tsd_t *tsd) {
1919 return stats_interval_accum_batch;
1920}
1921
1922uint64_t
1923stats_interval_postponed_event_wait(tsd_t *tsd) {
1924 return TE_MIN_START_WAIT;
1925}
1926
1927void
1928stats_interval_event_handler(tsd_t *tsd, uint64_t elapsed) {
1929 assert(elapsed > 0 && elapsed != TE_INVALID_ELAPSED);
1930 if (counter_accum(tsd_tsdn(tsd), &stats_interval_accumulated,
1931 elapsed)) {
1932 je_malloc_stats_print(NULL, NULL, opt_stats_interval_opts);
1933 }
1934}
1935
1936bool
1937stats_boot(void) {
1938 uint64_t stats_interval;
1939 if (opt_stats_interval < 0) {
1940 assert(opt_stats_interval == -1);
1941 stats_interval = 0;
1942 stats_interval_accum_batch = 0;
1943 } else{
1944 /* See comments in stats.h */
1945 stats_interval = (opt_stats_interval > 0) ?
1946 opt_stats_interval : 1;
1947 uint64_t batch = stats_interval >>
1948 STATS_INTERVAL_ACCUM_LG_BATCH_SIZE;
1949 if (batch > STATS_INTERVAL_ACCUM_BATCH_MAX) {
1950 batch = STATS_INTERVAL_ACCUM_BATCH_MAX;
1951 } else if (batch == 0) {
1952 batch = 1;
1953 }
1954 stats_interval_accum_batch = batch;
1955 }
1956
1957 return counter_accum_init(&stats_interval_accumulated, stats_interval);
1958}
1959
1960void
1961stats_prefork(tsdn_t *tsdn) {
1962 counter_prefork(tsdn, &stats_interval_accumulated);
1963}
1964
1965void
1966stats_postfork_parent(tsdn_t *tsdn) {
1967 counter_postfork_parent(tsdn, &stats_interval_accumulated);
1968}
1969
1970void
1971stats_postfork_child(tsdn_t *tsdn) {
1972 counter_postfork_child(tsdn, &stats_interval_accumulated);
1973}
diff --git a/examples/redis-unstable/deps/jemalloc/src/sz.c b/examples/redis-unstable/deps/jemalloc/src/sz.c
deleted file mode 100644
index d3115dd..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/sz.c
+++ /dev/null
@@ -1,114 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3#include "jemalloc/internal/sz.h"
4
5JEMALLOC_ALIGNED(CACHELINE)
6size_t sz_pind2sz_tab[SC_NPSIZES+1];
7size_t sz_large_pad;
8
9size_t
10sz_psz_quantize_floor(size_t size) {
11 size_t ret;
12 pszind_t pind;
13
14 assert(size > 0);
15 assert((size & PAGE_MASK) == 0);
16
17 pind = sz_psz2ind(size - sz_large_pad + 1);
18 if (pind == 0) {
19 /*
20 * Avoid underflow. This short-circuit would also do the right
21 * thing for all sizes in the range for which there are
22 * PAGE-spaced size classes, but it's simplest to just handle
23 * the one case that would cause erroneous results.
24 */
25 return size;
26 }
27 ret = sz_pind2sz(pind - 1) + sz_large_pad;
28 assert(ret <= size);
29 return ret;
30}
31
32size_t
33sz_psz_quantize_ceil(size_t size) {
34 size_t ret;
35
36 assert(size > 0);
37 assert(size - sz_large_pad <= SC_LARGE_MAXCLASS);
38 assert((size & PAGE_MASK) == 0);
39
40 ret = sz_psz_quantize_floor(size);
41 if (ret < size) {
42 /*
43 * Skip a quantization that may have an adequately large extent,
44 * because under-sized extents may be mixed in. This only
45 * happens when an unusual size is requested, i.e. for aligned
46 * allocation, and is just one of several places where linear
47 * search would potentially find sufficiently aligned available
48 * memory somewhere lower.
49 */
50 ret = sz_pind2sz(sz_psz2ind(ret - sz_large_pad + 1)) +
51 sz_large_pad;
52 }
53 return ret;
54}
55
56static void
57sz_boot_pind2sz_tab(const sc_data_t *sc_data) {
58 int pind = 0;
59 for (unsigned i = 0; i < SC_NSIZES; i++) {
60 const sc_t *sc = &sc_data->sc[i];
61 if (sc->psz) {
62 sz_pind2sz_tab[pind] = (ZU(1) << sc->lg_base)
63 + (ZU(sc->ndelta) << sc->lg_delta);
64 pind++;
65 }
66 }
67 for (int i = pind; i <= (int)SC_NPSIZES; i++) {
68 sz_pind2sz_tab[pind] = sc_data->large_maxclass + PAGE;
69 }
70}
71
72JEMALLOC_ALIGNED(CACHELINE)
73size_t sz_index2size_tab[SC_NSIZES];
74
75static void
76sz_boot_index2size_tab(const sc_data_t *sc_data) {
77 for (unsigned i = 0; i < SC_NSIZES; i++) {
78 const sc_t *sc = &sc_data->sc[i];
79 sz_index2size_tab[i] = (ZU(1) << sc->lg_base)
80 + (ZU(sc->ndelta) << (sc->lg_delta));
81 }
82}
83
84/*
85 * To keep this table small, we divide sizes by the tiny min size, which gives
86 * the smallest interval for which the result can change.
87 */
88JEMALLOC_ALIGNED(CACHELINE)
89uint8_t sz_size2index_tab[(SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1];
90
91static void
92sz_boot_size2index_tab(const sc_data_t *sc_data) {
93 size_t dst_max = (SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1;
94 size_t dst_ind = 0;
95 for (unsigned sc_ind = 0; sc_ind < SC_NSIZES && dst_ind < dst_max;
96 sc_ind++) {
97 const sc_t *sc = &sc_data->sc[sc_ind];
98 size_t sz = (ZU(1) << sc->lg_base)
99 + (ZU(sc->ndelta) << sc->lg_delta);
100 size_t max_ind = ((sz + (ZU(1) << SC_LG_TINY_MIN) - 1)
101 >> SC_LG_TINY_MIN);
102 for (; dst_ind <= max_ind && dst_ind < dst_max; dst_ind++) {
103 sz_size2index_tab[dst_ind] = sc_ind;
104 }
105 }
106}
107
108void
109sz_boot(const sc_data_t *sc_data, bool cache_oblivious) {
110 sz_large_pad = cache_oblivious ? PAGE : 0;
111 sz_boot_pind2sz_tab(sc_data);
112 sz_boot_index2size_tab(sc_data);
113 sz_boot_size2index_tab(sc_data);
114}
diff --git a/examples/redis-unstable/deps/jemalloc/src/tcache.c b/examples/redis-unstable/deps/jemalloc/src/tcache.c
deleted file mode 100644
index fa16732..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/tcache.c
+++ /dev/null
@@ -1,1101 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/mutex.h"
6#include "jemalloc/internal/safety_check.h"
7#include "jemalloc/internal/san.h"
8#include "jemalloc/internal/sc.h"
9
10/******************************************************************************/
11/* Data. */
12
13bool opt_tcache = true;
14
15/* tcache_maxclass is set to 32KB by default. */
16size_t opt_tcache_max = ((size_t)1) << 15;
17
18/* Reasonable defaults for min and max values. */
19unsigned opt_tcache_nslots_small_min = 20;
20unsigned opt_tcache_nslots_small_max = 200;
21unsigned opt_tcache_nslots_large = 20;
22
23/*
24 * We attempt to make the number of slots in a tcache bin for a given size class
25 * equal to the number of objects in a slab times some multiplier. By default,
26 * the multiplier is 2 (i.e. we set the maximum number of objects in the tcache
27 * to twice the number of objects in a slab).
28 * This is bounded by some other constraints as well, like the fact that it
29 * must be even, must be less than opt_tcache_nslots_small_max, etc..
30 */
31ssize_t opt_lg_tcache_nslots_mul = 1;
32
33/*
34 * Number of allocation bytes between tcache incremental GCs. Again, this
35 * default just seems to work well; more tuning is possible.
36 */
37size_t opt_tcache_gc_incr_bytes = 65536;
38
39/*
40 * With default settings, we may end up flushing small bins frequently with
41 * small flush amounts. To limit this tendency, we can set a number of bytes to
42 * "delay" by. If we try to flush N M-byte items, we decrease that size-class's
43 * delay by N * M. So, if delay is 1024 and we're looking at the 64-byte size
44 * class, we won't do any flushing until we've been asked to flush 1024/64 == 16
45 * items. This can happen in any configuration (i.e. being asked to flush 16
46 * items once, or 4 items 4 times).
47 *
48 * Practically, this is stored as a count of items in a uint8_t, so the
49 * effective maximum value for a size class is 255 * sz.
50 */
51size_t opt_tcache_gc_delay_bytes = 0;
52
53/*
54 * When a cache bin is flushed because it's full, how much of it do we flush?
55 * By default, we flush half the maximum number of items.
56 */
57unsigned opt_lg_tcache_flush_small_div = 1;
58unsigned opt_lg_tcache_flush_large_div = 1;
59
60cache_bin_info_t *tcache_bin_info;
61
62/* Total stack size required (per tcache). Include the padding above. */
63static size_t tcache_bin_alloc_size;
64static size_t tcache_bin_alloc_alignment;
65
66/* Number of cache bins enabled, including both large and small. */
67unsigned nhbins;
68/* Max size class to be cached (can be small or large). */
69size_t tcache_maxclass;
70
71tcaches_t *tcaches;
72
73/* Index of first element within tcaches that has never been used. */
74static unsigned tcaches_past;
75
76/* Head of singly linked list tracking available tcaches elements. */
77static tcaches_t *tcaches_avail;
78
79/* Protects tcaches{,_past,_avail}. */
80static malloc_mutex_t tcaches_mtx;
81
82/******************************************************************************/
83
84size_t
85tcache_salloc(tsdn_t *tsdn, const void *ptr) {
86 return arena_salloc(tsdn, ptr);
87}
88
89uint64_t
90tcache_gc_new_event_wait(tsd_t *tsd) {
91 return opt_tcache_gc_incr_bytes;
92}
93
94uint64_t
95tcache_gc_postponed_event_wait(tsd_t *tsd) {
96 return TE_MIN_START_WAIT;
97}
98
99uint64_t
100tcache_gc_dalloc_new_event_wait(tsd_t *tsd) {
101 return opt_tcache_gc_incr_bytes;
102}
103
104uint64_t
105tcache_gc_dalloc_postponed_event_wait(tsd_t *tsd) {
106 return TE_MIN_START_WAIT;
107}
108
109static uint8_t
110tcache_gc_item_delay_compute(szind_t szind) {
111 assert(szind < SC_NBINS);
112 size_t sz = sz_index2size(szind);
113 size_t item_delay = opt_tcache_gc_delay_bytes / sz;
114 size_t delay_max = ZU(1)
115 << (sizeof(((tcache_slow_t *)NULL)->bin_flush_delay_items[0]) * 8);
116 if (item_delay >= delay_max) {
117 item_delay = delay_max - 1;
118 }
119 return (uint8_t)item_delay;
120}
121
122static void
123tcache_gc_small(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
124 szind_t szind) {
125 /* Aim to flush 3/4 of items below low-water. */
126 assert(szind < SC_NBINS);
127
128 cache_bin_t *cache_bin = &tcache->bins[szind];
129 cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin,
130 &tcache_bin_info[szind]);
131 cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin,
132 &tcache_bin_info[szind]);
133 assert(!tcache_slow->bin_refilled[szind]);
134
135 size_t nflush = low_water - (low_water >> 2);
136 if (nflush < tcache_slow->bin_flush_delay_items[szind]) {
137 /* Workaround for a conversion warning. */
138 uint8_t nflush_uint8 = (uint8_t)nflush;
139 assert(sizeof(tcache_slow->bin_flush_delay_items[0]) ==
140 sizeof(nflush_uint8));
141 tcache_slow->bin_flush_delay_items[szind] -= nflush_uint8;
142 return;
143 } else {
144 tcache_slow->bin_flush_delay_items[szind]
145 = tcache_gc_item_delay_compute(szind);
146 }
147
148 tcache_bin_flush_small(tsd, tcache, cache_bin, szind,
149 (unsigned)(ncached - nflush));
150
151 /*
152 * Reduce fill count by 2X. Limit lg_fill_div such that
153 * the fill count is always at least 1.
154 */
155 if ((cache_bin_info_ncached_max(&tcache_bin_info[szind])
156 >> (tcache_slow->lg_fill_div[szind] + 1)) >= 1) {
157 tcache_slow->lg_fill_div[szind]++;
158 }
159}
160
161static void
162tcache_gc_large(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
163 szind_t szind) {
164 /* Like the small GC; flush 3/4 of untouched items. */
165 assert(szind >= SC_NBINS);
166 cache_bin_t *cache_bin = &tcache->bins[szind];
167 cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin,
168 &tcache_bin_info[szind]);
169 cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin,
170 &tcache_bin_info[szind]);
171 tcache_bin_flush_large(tsd, tcache, cache_bin, szind,
172 (unsigned)(ncached - low_water + (low_water >> 2)));
173}
174
175static void
176tcache_event(tsd_t *tsd) {
177 tcache_t *tcache = tcache_get(tsd);
178 if (tcache == NULL) {
179 return;
180 }
181
182 tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
183 szind_t szind = tcache_slow->next_gc_bin;
184 bool is_small = (szind < SC_NBINS);
185 cache_bin_t *cache_bin = &tcache->bins[szind];
186
187 tcache_bin_flush_stashed(tsd, tcache, cache_bin, szind, is_small);
188
189 cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin,
190 &tcache_bin_info[szind]);
191 if (low_water > 0) {
192 if (is_small) {
193 tcache_gc_small(tsd, tcache_slow, tcache, szind);
194 } else {
195 tcache_gc_large(tsd, tcache_slow, tcache, szind);
196 }
197 } else if (is_small && tcache_slow->bin_refilled[szind]) {
198 assert(low_water == 0);
199 /*
200 * Increase fill count by 2X for small bins. Make sure
201 * lg_fill_div stays greater than 0.
202 */
203 if (tcache_slow->lg_fill_div[szind] > 1) {
204 tcache_slow->lg_fill_div[szind]--;
205 }
206 tcache_slow->bin_refilled[szind] = false;
207 }
208 cache_bin_low_water_set(cache_bin);
209
210 tcache_slow->next_gc_bin++;
211 if (tcache_slow->next_gc_bin == nhbins) {
212 tcache_slow->next_gc_bin = 0;
213 }
214}
215
216void
217tcache_gc_event_handler(tsd_t *tsd, uint64_t elapsed) {
218 assert(elapsed == TE_INVALID_ELAPSED);
219 tcache_event(tsd);
220}
221
222void
223tcache_gc_dalloc_event_handler(tsd_t *tsd, uint64_t elapsed) {
224 assert(elapsed == TE_INVALID_ELAPSED);
225 tcache_event(tsd);
226}
227
228void *
229tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena,
230 tcache_t *tcache, cache_bin_t *cache_bin, szind_t binind,
231 bool *tcache_success) {
232 tcache_slow_t *tcache_slow = tcache->tcache_slow;
233 void *ret;
234
235 assert(tcache_slow->arena != NULL);
236 unsigned nfill = cache_bin_info_ncached_max(&tcache_bin_info[binind])
237 >> tcache_slow->lg_fill_div[binind];
238 arena_cache_bin_fill_small(tsdn, arena, cache_bin,
239 &tcache_bin_info[binind], binind, nfill);
240 tcache_slow->bin_refilled[binind] = true;
241 ret = cache_bin_alloc(cache_bin, tcache_success);
242
243 return ret;
244}
245
246static const void *
247tcache_bin_flush_ptr_getter(void *arr_ctx, size_t ind) {
248 cache_bin_ptr_array_t *arr = (cache_bin_ptr_array_t *)arr_ctx;
249 return arr->ptr[ind];
250}
251
252static void
253tcache_bin_flush_metadata_visitor(void *szind_sum_ctx,
254 emap_full_alloc_ctx_t *alloc_ctx) {
255 size_t *szind_sum = (size_t *)szind_sum_ctx;
256 *szind_sum -= alloc_ctx->szind;
257 util_prefetch_write_range(alloc_ctx->edata, sizeof(edata_t));
258}
259
260JEMALLOC_NOINLINE static void
261tcache_bin_flush_size_check_fail(cache_bin_ptr_array_t *arr, szind_t szind,
262 size_t nptrs, emap_batch_lookup_result_t *edatas) {
263 bool found_mismatch = false;
264 for (size_t i = 0; i < nptrs; i++) {
265 szind_t true_szind = edata_szind_get(edatas[i].edata);
266 if (true_szind != szind) {
267 found_mismatch = true;
268 safety_check_fail_sized_dealloc(
269 /* current_dealloc */ false,
270 /* ptr */ tcache_bin_flush_ptr_getter(arr, i),
271 /* true_size */ sz_index2size(true_szind),
272 /* input_size */ sz_index2size(szind));
273 }
274 }
275 assert(found_mismatch);
276}
277
278static void
279tcache_bin_flush_edatas_lookup(tsd_t *tsd, cache_bin_ptr_array_t *arr,
280 szind_t binind, size_t nflush, emap_batch_lookup_result_t *edatas) {
281
282 /*
283 * This gets compiled away when config_opt_safety_checks is false.
284 * Checks for sized deallocation bugs, failing early rather than
285 * corrupting metadata.
286 */
287 size_t szind_sum = binind * nflush;
288 emap_edata_lookup_batch(tsd, &arena_emap_global, nflush,
289 &tcache_bin_flush_ptr_getter, (void *)arr,
290 &tcache_bin_flush_metadata_visitor, (void *)&szind_sum,
291 edatas);
292 if (config_opt_safety_checks && unlikely(szind_sum != 0)) {
293 tcache_bin_flush_size_check_fail(arr, binind, nflush, edatas);
294 }
295}
296
297JEMALLOC_ALWAYS_INLINE bool
298tcache_bin_flush_match(edata_t *edata, unsigned cur_arena_ind,
299 unsigned cur_binshard, bool small) {
300 if (small) {
301 return edata_arena_ind_get(edata) == cur_arena_ind
302 && edata_binshard_get(edata) == cur_binshard;
303 } else {
304 return edata_arena_ind_get(edata) == cur_arena_ind;
305 }
306}
307
308JEMALLOC_ALWAYS_INLINE void
309tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
310 szind_t binind, cache_bin_ptr_array_t *ptrs, unsigned nflush, bool small) {
311 tcache_slow_t *tcache_slow = tcache->tcache_slow;
312 /*
313 * A couple lookup calls take tsdn; declare it once for convenience
314 * instead of calling tsd_tsdn(tsd) all the time.
315 */
316 tsdn_t *tsdn = tsd_tsdn(tsd);
317
318 if (small) {
319 assert(binind < SC_NBINS);
320 } else {
321 assert(binind < nhbins);
322 }
323 arena_t *tcache_arena = tcache_slow->arena;
324 assert(tcache_arena != NULL);
325
326 /*
327 * Variable length array must have > 0 length; the last element is never
328 * touched (it's just included to satisfy the no-zero-length rule).
329 */
330 VARIABLE_ARRAY(emap_batch_lookup_result_t, item_edata, nflush + 1);
331 tcache_bin_flush_edatas_lookup(tsd, ptrs, binind, nflush, item_edata);
332
333 /*
334 * The slabs where we freed the last remaining object in the slab (and
335 * so need to free the slab itself).
336 * Used only if small == true.
337 */
338 unsigned dalloc_count = 0;
339 VARIABLE_ARRAY(edata_t *, dalloc_slabs, nflush + 1);
340
341 /*
342 * We're about to grab a bunch of locks. If one of them happens to be
343 * the one guarding the arena-level stats counters we flush our
344 * thread-local ones to, we do so under one critical section.
345 */
346 bool merged_stats = false;
347 while (nflush > 0) {
348 /* Lock the arena, or bin, associated with the first object. */
349 edata_t *edata = item_edata[0].edata;
350 unsigned cur_arena_ind = edata_arena_ind_get(edata);
351 arena_t *cur_arena = arena_get(tsdn, cur_arena_ind, false);
352
353 /*
354 * These assignments are always overwritten when small is true,
355 * and their values are always ignored when small is false, but
356 * to avoid the technical UB when we pass them as parameters, we
357 * need to intialize them.
358 */
359 unsigned cur_binshard = 0;
360 bin_t *cur_bin = NULL;
361 if (small) {
362 cur_binshard = edata_binshard_get(edata);
363 cur_bin = arena_get_bin(cur_arena, binind,
364 cur_binshard);
365 assert(cur_binshard < bin_infos[binind].n_shards);
366 /*
367 * If you're looking at profiles, you might think this
368 * is a good place to prefetch the bin stats, which are
369 * often a cache miss. This turns out not to be
370 * helpful on the workloads we've looked at, with moving
371 * the bin stats next to the lock seeming to do better.
372 */
373 }
374
375 if (small) {
376 malloc_mutex_lock(tsdn, &cur_bin->lock);
377 }
378 if (!small && !arena_is_auto(cur_arena)) {
379 malloc_mutex_lock(tsdn, &cur_arena->large_mtx);
380 }
381
382 /*
383 * If we acquired the right lock and have some stats to flush,
384 * flush them.
385 */
386 if (config_stats && tcache_arena == cur_arena
387 && !merged_stats) {
388 merged_stats = true;
389 if (small) {
390 cur_bin->stats.nflushes++;
391 cur_bin->stats.nrequests +=
392 cache_bin->tstats.nrequests;
393 cache_bin->tstats.nrequests = 0;
394 } else {
395 arena_stats_large_flush_nrequests_add(tsdn,
396 &tcache_arena->stats, binind,
397 cache_bin->tstats.nrequests);
398 cache_bin->tstats.nrequests = 0;
399 }
400 }
401
402 /*
403 * Large allocations need special prep done. Afterwards, we can
404 * drop the large lock.
405 */
406 if (!small) {
407 for (unsigned i = 0; i < nflush; i++) {
408 void *ptr = ptrs->ptr[i];
409 edata = item_edata[i].edata;
410 assert(ptr != NULL && edata != NULL);
411
412 if (tcache_bin_flush_match(edata, cur_arena_ind,
413 cur_binshard, small)) {
414 large_dalloc_prep_locked(tsdn,
415 edata);
416 }
417 }
418 }
419 if (!small && !arena_is_auto(cur_arena)) {
420 malloc_mutex_unlock(tsdn, &cur_arena->large_mtx);
421 }
422
423 /* Deallocate whatever we can. */
424 unsigned ndeferred = 0;
425 /* Init only to avoid used-uninitialized warning. */
426 arena_dalloc_bin_locked_info_t dalloc_bin_info = {0};
427 if (small) {
428 arena_dalloc_bin_locked_begin(&dalloc_bin_info, binind);
429 }
430 for (unsigned i = 0; i < nflush; i++) {
431 void *ptr = ptrs->ptr[i];
432 edata = item_edata[i].edata;
433 assert(ptr != NULL && edata != NULL);
434 if (!tcache_bin_flush_match(edata, cur_arena_ind,
435 cur_binshard, small)) {
436 /*
437 * The object was allocated either via a
438 * different arena, or a different bin in this
439 * arena. Either way, stash the object so that
440 * it can be handled in a future pass.
441 */
442 ptrs->ptr[ndeferred] = ptr;
443 item_edata[ndeferred].edata = edata;
444 ndeferred++;
445 continue;
446 }
447 if (small) {
448 if (arena_dalloc_bin_locked_step(tsdn,
449 cur_arena, cur_bin, &dalloc_bin_info,
450 binind, edata, ptr)) {
451 dalloc_slabs[dalloc_count] = edata;
452 dalloc_count++;
453 }
454 } else {
455 if (large_dalloc_safety_checks(edata, ptr,
456 binind)) {
457 /* See the comment in isfree. */
458 continue;
459 }
460 large_dalloc_finish(tsdn, edata);
461 }
462 }
463
464 if (small) {
465 arena_dalloc_bin_locked_finish(tsdn, cur_arena, cur_bin,
466 &dalloc_bin_info);
467 malloc_mutex_unlock(tsdn, &cur_bin->lock);
468 }
469 arena_decay_ticks(tsdn, cur_arena, nflush - ndeferred);
470 nflush = ndeferred;
471 }
472
473 /* Handle all deferred slab dalloc. */
474 assert(small || dalloc_count == 0);
475 for (unsigned i = 0; i < dalloc_count; i++) {
476 edata_t *slab = dalloc_slabs[i];
477 arena_slab_dalloc(tsdn, arena_get_from_edata(slab), slab);
478
479 }
480
481 if (config_stats && !merged_stats) {
482 if (small) {
483 /*
484 * The flush loop didn't happen to flush to this
485 * thread's arena, so the stats didn't get merged.
486 * Manually do so now.
487 */
488 bin_t *bin = arena_bin_choose(tsdn, tcache_arena,
489 binind, NULL);
490 malloc_mutex_lock(tsdn, &bin->lock);
491 bin->stats.nflushes++;
492 bin->stats.nrequests += cache_bin->tstats.nrequests;
493 cache_bin->tstats.nrequests = 0;
494 malloc_mutex_unlock(tsdn, &bin->lock);
495 } else {
496 arena_stats_large_flush_nrequests_add(tsdn,
497 &tcache_arena->stats, binind,
498 cache_bin->tstats.nrequests);
499 cache_bin->tstats.nrequests = 0;
500 }
501 }
502
503}
504
505JEMALLOC_ALWAYS_INLINE void
506tcache_bin_flush_bottom(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
507 szind_t binind, unsigned rem, bool small) {
508 tcache_bin_flush_stashed(tsd, tcache, cache_bin, binind, small);
509
510 cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin,
511 &tcache_bin_info[binind]);
512 assert((cache_bin_sz_t)rem <= ncached);
513 unsigned nflush = ncached - rem;
514
515 CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nflush);
516 cache_bin_init_ptr_array_for_flush(cache_bin, &tcache_bin_info[binind],
517 &ptrs, nflush);
518
519 tcache_bin_flush_impl(tsd, tcache, cache_bin, binind, &ptrs, nflush,
520 small);
521
522 cache_bin_finish_flush(cache_bin, &tcache_bin_info[binind], &ptrs,
523 ncached - rem);
524}
525
526void
527tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
528 szind_t binind, unsigned rem) {
529 tcache_bin_flush_bottom(tsd, tcache, cache_bin, binind, rem, true);
530}
531
532void
533tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
534 szind_t binind, unsigned rem) {
535 tcache_bin_flush_bottom(tsd, tcache, cache_bin, binind, rem, false);
536}
537
538/*
539 * Flushing stashed happens when 1) tcache fill, 2) tcache flush, or 3) tcache
540 * GC event. This makes sure that the stashed items do not hold memory for too
541 * long, and new buffers can only be allocated when nothing is stashed.
542 *
543 * The downside is, the time between stash and flush may be relatively short,
544 * especially when the request rate is high. It lowers the chance of detecting
545 * write-after-free -- however that is a delayed detection anyway, and is less
546 * of a focus than the memory overhead.
547 */
548void
549tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
550 szind_t binind, bool is_small) {
551 cache_bin_info_t *info = &tcache_bin_info[binind];
552 /*
553 * The two below are for assertion only. The content of original cached
554 * items remain unchanged -- the stashed items reside on the other end
555 * of the stack. Checking the stack head and ncached to verify.
556 */
557 void *head_content = *cache_bin->stack_head;
558 cache_bin_sz_t orig_cached = cache_bin_ncached_get_local(cache_bin,
559 info);
560
561 cache_bin_sz_t nstashed = cache_bin_nstashed_get_local(cache_bin, info);
562 assert(orig_cached + nstashed <= cache_bin_info_ncached_max(info));
563 if (nstashed == 0) {
564 return;
565 }
566
567 CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nstashed);
568 cache_bin_init_ptr_array_for_stashed(cache_bin, binind, info, &ptrs,
569 nstashed);
570 san_check_stashed_ptrs(ptrs.ptr, nstashed, sz_index2size(binind));
571 tcache_bin_flush_impl(tsd, tcache, cache_bin, binind, &ptrs, nstashed,
572 is_small);
573 cache_bin_finish_flush_stashed(cache_bin, info);
574
575 assert(cache_bin_nstashed_get_local(cache_bin, info) == 0);
576 assert(cache_bin_ncached_get_local(cache_bin, info) == orig_cached);
577 assert(head_content == *cache_bin->stack_head);
578}
579
580void
581tcache_arena_associate(tsdn_t *tsdn, tcache_slow_t *tcache_slow,
582 tcache_t *tcache, arena_t *arena) {
583 assert(tcache_slow->arena == NULL);
584 tcache_slow->arena = arena;
585
586 if (config_stats) {
587 /* Link into list of extant tcaches. */
588 malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
589
590 ql_elm_new(tcache_slow, link);
591 ql_tail_insert(&arena->tcache_ql, tcache_slow, link);
592 cache_bin_array_descriptor_init(
593 &tcache_slow->cache_bin_array_descriptor, tcache->bins);
594 ql_tail_insert(&arena->cache_bin_array_descriptor_ql,
595 &tcache_slow->cache_bin_array_descriptor, link);
596
597 malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
598 }
599}
600
601static void
602tcache_arena_dissociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow,
603 tcache_t *tcache) {
604 arena_t *arena = tcache_slow->arena;
605 assert(arena != NULL);
606 if (config_stats) {
607 /* Unlink from list of extant tcaches. */
608 malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
609 if (config_debug) {
610 bool in_ql = false;
611 tcache_slow_t *iter;
612 ql_foreach(iter, &arena->tcache_ql, link) {
613 if (iter == tcache_slow) {
614 in_ql = true;
615 break;
616 }
617 }
618 assert(in_ql);
619 }
620 ql_remove(&arena->tcache_ql, tcache_slow, link);
621 ql_remove(&arena->cache_bin_array_descriptor_ql,
622 &tcache_slow->cache_bin_array_descriptor, link);
623 tcache_stats_merge(tsdn, tcache_slow->tcache, arena);
624 malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
625 }
626 tcache_slow->arena = NULL;
627}
628
629void
630tcache_arena_reassociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow,
631 tcache_t *tcache, arena_t *arena) {
632 tcache_arena_dissociate(tsdn, tcache_slow, tcache);
633 tcache_arena_associate(tsdn, tcache_slow, tcache, arena);
634}
635
636bool
637tsd_tcache_enabled_data_init(tsd_t *tsd) {
638 /* Called upon tsd initialization. */
639 tsd_tcache_enabled_set(tsd, opt_tcache);
640 tsd_slow_update(tsd);
641
642 if (opt_tcache) {
643 /* Trigger tcache init. */
644 tsd_tcache_data_init(tsd);
645 }
646
647 return false;
648}
649
650static void
651tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
652 void *mem) {
653 tcache->tcache_slow = tcache_slow;
654 tcache_slow->tcache = tcache;
655
656 memset(&tcache_slow->link, 0, sizeof(ql_elm(tcache_t)));
657 tcache_slow->next_gc_bin = 0;
658 tcache_slow->arena = NULL;
659 tcache_slow->dyn_alloc = mem;
660
661 /*
662 * We reserve cache bins for all small size classes, even if some may
663 * not get used (i.e. bins higher than nhbins). This allows the fast
664 * and common paths to access cache bin metadata safely w/o worrying
665 * about which ones are disabled.
666 */
667 unsigned n_reserved_bins = nhbins < SC_NBINS ? SC_NBINS : nhbins;
668 memset(tcache->bins, 0, sizeof(cache_bin_t) * n_reserved_bins);
669
670 size_t cur_offset = 0;
671 cache_bin_preincrement(tcache_bin_info, nhbins, mem,
672 &cur_offset);
673 for (unsigned i = 0; i < nhbins; i++) {
674 if (i < SC_NBINS) {
675 tcache_slow->lg_fill_div[i] = 1;
676 tcache_slow->bin_refilled[i] = false;
677 tcache_slow->bin_flush_delay_items[i]
678 = tcache_gc_item_delay_compute(i);
679 }
680 cache_bin_t *cache_bin = &tcache->bins[i];
681 cache_bin_init(cache_bin, &tcache_bin_info[i], mem,
682 &cur_offset);
683 }
684 /*
685 * For small size classes beyond tcache_maxclass (i.e. nhbins < NBINS),
686 * their cache bins are initialized to a state to safely and efficiently
687 * fail all fastpath alloc / free, so that no additional check around
688 * nhbins is needed on fastpath.
689 */
690 for (unsigned i = nhbins; i < SC_NBINS; i++) {
691 /* Disabled small bins. */
692 cache_bin_t *cache_bin = &tcache->bins[i];
693 void *fake_stack = mem;
694 size_t fake_offset = 0;
695
696 cache_bin_init(cache_bin, &tcache_bin_info[i], fake_stack,
697 &fake_offset);
698 assert(tcache_small_bin_disabled(i, cache_bin));
699 }
700
701 cache_bin_postincrement(tcache_bin_info, nhbins, mem,
702 &cur_offset);
703 /* Sanity check that the whole stack is used. */
704 assert(cur_offset == tcache_bin_alloc_size);
705}
706
707/* Initialize auto tcache (embedded in TSD). */
708bool
709tsd_tcache_data_init(tsd_t *tsd) {
710 tcache_slow_t *tcache_slow = tsd_tcache_slowp_get_unsafe(tsd);
711 tcache_t *tcache = tsd_tcachep_get_unsafe(tsd);
712
713 assert(cache_bin_still_zero_initialized(&tcache->bins[0]));
714 size_t alignment = tcache_bin_alloc_alignment;
715 size_t size = sz_sa2u(tcache_bin_alloc_size, alignment);
716
717 void *mem = ipallocztm(tsd_tsdn(tsd), size, alignment, true, NULL,
718 true, arena_get(TSDN_NULL, 0, true));
719 if (mem == NULL) {
720 return true;
721 }
722
723 tcache_init(tsd, tcache_slow, tcache, mem);
724 /*
725 * Initialization is a bit tricky here. After malloc init is done, all
726 * threads can rely on arena_choose and associate tcache accordingly.
727 * However, the thread that does actual malloc bootstrapping relies on
728 * functional tsd, and it can only rely on a0. In that case, we
729 * associate its tcache to a0 temporarily, and later on
730 * arena_choose_hard() will re-associate properly.
731 */
732 tcache_slow->arena = NULL;
733 arena_t *arena;
734 if (!malloc_initialized()) {
735 /* If in initialization, assign to a0. */
736 arena = arena_get(tsd_tsdn(tsd), 0, false);
737 tcache_arena_associate(tsd_tsdn(tsd), tcache_slow, tcache,
738 arena);
739 } else {
740 arena = arena_choose(tsd, NULL);
741 /* This may happen if thread.tcache.enabled is used. */
742 if (tcache_slow->arena == NULL) {
743 tcache_arena_associate(tsd_tsdn(tsd), tcache_slow,
744 tcache, arena);
745 }
746 }
747 assert(arena == tcache_slow->arena);
748
749 return false;
750}
751
752/* Created manual tcache for tcache.create mallctl. */
753tcache_t *
754tcache_create_explicit(tsd_t *tsd) {
755 /*
756 * We place the cache bin stacks, then the tcache_t, then a pointer to
757 * the beginning of the whole allocation (for freeing). The makes sure
758 * the cache bins have the requested alignment.
759 */
760 size_t size = tcache_bin_alloc_size + sizeof(tcache_t)
761 + sizeof(tcache_slow_t);
762 /* Naturally align the pointer stacks. */
763 size = PTR_CEILING(size);
764 size = sz_sa2u(size, tcache_bin_alloc_alignment);
765
766 void *mem = ipallocztm(tsd_tsdn(tsd), size, tcache_bin_alloc_alignment,
767 true, NULL, true, arena_get(TSDN_NULL, 0, true));
768 if (mem == NULL) {
769 return NULL;
770 }
771 tcache_t *tcache = (void *)((uintptr_t)mem + tcache_bin_alloc_size);
772 tcache_slow_t *tcache_slow =
773 (void *)((uintptr_t)mem + tcache_bin_alloc_size + sizeof(tcache_t));
774 tcache_init(tsd, tcache_slow, tcache, mem);
775
776 tcache_arena_associate(tsd_tsdn(tsd), tcache_slow, tcache,
777 arena_ichoose(tsd, NULL));
778
779 return tcache;
780}
781
782static void
783tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) {
784 tcache_slow_t *tcache_slow = tcache->tcache_slow;
785 assert(tcache_slow->arena != NULL);
786
787 for (unsigned i = 0; i < nhbins; i++) {
788 cache_bin_t *cache_bin = &tcache->bins[i];
789 if (i < SC_NBINS) {
790 tcache_bin_flush_small(tsd, tcache, cache_bin, i, 0);
791 } else {
792 tcache_bin_flush_large(tsd, tcache, cache_bin, i, 0);
793 }
794 if (config_stats) {
795 assert(cache_bin->tstats.nrequests == 0);
796 }
797 }
798}
799
800void
801tcache_flush(tsd_t *tsd) {
802 assert(tcache_available(tsd));
803 tcache_flush_cache(tsd, tsd_tcachep_get(tsd));
804}
805
806static void
807tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
808 tcache_slow_t *tcache_slow = tcache->tcache_slow;
809 tcache_flush_cache(tsd, tcache);
810 arena_t *arena = tcache_slow->arena;
811 tcache_arena_dissociate(tsd_tsdn(tsd), tcache_slow, tcache);
812
813 if (tsd_tcache) {
814 cache_bin_t *cache_bin = &tcache->bins[0];
815 cache_bin_assert_empty(cache_bin, &tcache_bin_info[0]);
816 }
817 idalloctm(tsd_tsdn(tsd), tcache_slow->dyn_alloc, NULL, NULL, true,
818 true);
819
820 /*
821 * The deallocation and tcache flush above may not trigger decay since
822 * we are on the tcache shutdown path (potentially with non-nominal
823 * tsd). Manually trigger decay to avoid pathological cases. Also
824 * include arena 0 because the tcache array is allocated from it.
825 */
826 arena_decay(tsd_tsdn(tsd), arena_get(tsd_tsdn(tsd), 0, false),
827 false, false);
828
829 if (arena_nthreads_get(arena, false) == 0 &&
830 !background_thread_enabled()) {
831 /* Force purging when no threads assigned to the arena anymore. */
832 arena_decay(tsd_tsdn(tsd), arena,
833 /* is_background_thread */ false, /* all */ true);
834 } else {
835 arena_decay(tsd_tsdn(tsd), arena,
836 /* is_background_thread */ false, /* all */ false);
837 }
838}
839
840/* For auto tcache (embedded in TSD) only. */
841void
842tcache_cleanup(tsd_t *tsd) {
843 tcache_t *tcache = tsd_tcachep_get(tsd);
844 if (!tcache_available(tsd)) {
845 assert(tsd_tcache_enabled_get(tsd) == false);
846 assert(cache_bin_still_zero_initialized(&tcache->bins[0]));
847 return;
848 }
849 assert(tsd_tcache_enabled_get(tsd));
850 assert(!cache_bin_still_zero_initialized(&tcache->bins[0]));
851
852 tcache_destroy(tsd, tcache, true);
853 if (config_debug) {
854 /*
855 * For debug testing only, we want to pretend we're still in the
856 * zero-initialized state.
857 */
858 memset(tcache->bins, 0, sizeof(cache_bin_t) * nhbins);
859 }
860}
861
862void
863tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
864 cassert(config_stats);
865
866 /* Merge and reset tcache stats. */
867 for (unsigned i = 0; i < nhbins; i++) {
868 cache_bin_t *cache_bin = &tcache->bins[i];
869 if (i < SC_NBINS) {
870 bin_t *bin = arena_bin_choose(tsdn, arena, i, NULL);
871 malloc_mutex_lock(tsdn, &bin->lock);
872 bin->stats.nrequests += cache_bin->tstats.nrequests;
873 malloc_mutex_unlock(tsdn, &bin->lock);
874 } else {
875 arena_stats_large_flush_nrequests_add(tsdn,
876 &arena->stats, i, cache_bin->tstats.nrequests);
877 }
878 cache_bin->tstats.nrequests = 0;
879 }
880}
881
882static bool
883tcaches_create_prep(tsd_t *tsd, base_t *base) {
884 bool err;
885
886 malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx);
887
888 if (tcaches == NULL) {
889 tcaches = base_alloc(tsd_tsdn(tsd), base,
890 sizeof(tcache_t *) * (MALLOCX_TCACHE_MAX+1), CACHELINE);
891 if (tcaches == NULL) {
892 err = true;
893 goto label_return;
894 }
895 }
896
897 if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX) {
898 err = true;
899 goto label_return;
900 }
901
902 err = false;
903label_return:
904 return err;
905}
906
907bool
908tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind) {
909 witness_assert_depth(tsdn_witness_tsdp_get(tsd_tsdn(tsd)), 0);
910
911 bool err;
912
913 malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
914
915 if (tcaches_create_prep(tsd, base)) {
916 err = true;
917 goto label_return;
918 }
919
920 tcache_t *tcache = tcache_create_explicit(tsd);
921 if (tcache == NULL) {
922 err = true;
923 goto label_return;
924 }
925
926 tcaches_t *elm;
927 if (tcaches_avail != NULL) {
928 elm = tcaches_avail;
929 tcaches_avail = tcaches_avail->next;
930 elm->tcache = tcache;
931 *r_ind = (unsigned)(elm - tcaches);
932 } else {
933 elm = &tcaches[tcaches_past];
934 elm->tcache = tcache;
935 *r_ind = tcaches_past;
936 tcaches_past++;
937 }
938
939 err = false;
940label_return:
941 malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
942 witness_assert_depth(tsdn_witness_tsdp_get(tsd_tsdn(tsd)), 0);
943 return err;
944}
945
946static tcache_t *
947tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm, bool allow_reinit) {
948 malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx);
949
950 if (elm->tcache == NULL) {
951 return NULL;
952 }
953 tcache_t *tcache = elm->tcache;
954 if (allow_reinit) {
955 elm->tcache = TCACHES_ELM_NEED_REINIT;
956 } else {
957 elm->tcache = NULL;
958 }
959
960 if (tcache == TCACHES_ELM_NEED_REINIT) {
961 return NULL;
962 }
963 return tcache;
964}
965
966void
967tcaches_flush(tsd_t *tsd, unsigned ind) {
968 malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
969 tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind], true);
970 malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
971 if (tcache != NULL) {
972 /* Destroy the tcache; recreate in tcaches_get() if needed. */
973 tcache_destroy(tsd, tcache, false);
974 }
975}
976
977void
978tcaches_destroy(tsd_t *tsd, unsigned ind) {
979 malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
980 tcaches_t *elm = &tcaches[ind];
981 tcache_t *tcache = tcaches_elm_remove(tsd, elm, false);
982 elm->next = tcaches_avail;
983 tcaches_avail = elm;
984 malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
985 if (tcache != NULL) {
986 tcache_destroy(tsd, tcache, false);
987 }
988}
989
990static unsigned
991tcache_ncached_max_compute(szind_t szind) {
992 if (szind >= SC_NBINS) {
993 assert(szind < nhbins);
994 return opt_tcache_nslots_large;
995 }
996 unsigned slab_nregs = bin_infos[szind].nregs;
997
998 /* We may modify these values; start with the opt versions. */
999 unsigned nslots_small_min = opt_tcache_nslots_small_min;
1000 unsigned nslots_small_max = opt_tcache_nslots_small_max;
1001
1002 /*
1003 * Clamp values to meet our constraints -- even, nonzero, min < max, and
1004 * suitable for a cache bin size.
1005 */
1006 if (opt_tcache_nslots_small_max > CACHE_BIN_NCACHED_MAX) {
1007 nslots_small_max = CACHE_BIN_NCACHED_MAX;
1008 }
1009 if (nslots_small_min % 2 != 0) {
1010 nslots_small_min++;
1011 }
1012 if (nslots_small_max % 2 != 0) {
1013 nslots_small_max--;
1014 }
1015 if (nslots_small_min < 2) {
1016 nslots_small_min = 2;
1017 }
1018 if (nslots_small_max < 2) {
1019 nslots_small_max = 2;
1020 }
1021 if (nslots_small_min > nslots_small_max) {
1022 nslots_small_min = nslots_small_max;
1023 }
1024
1025 unsigned candidate;
1026 if (opt_lg_tcache_nslots_mul < 0) {
1027 candidate = slab_nregs >> (-opt_lg_tcache_nslots_mul);
1028 } else {
1029 candidate = slab_nregs << opt_lg_tcache_nslots_mul;
1030 }
1031 if (candidate % 2 != 0) {
1032 /*
1033 * We need the candidate size to be even -- we assume that we
1034 * can divide by two and get a positive number (e.g. when
1035 * flushing).
1036 */
1037 ++candidate;
1038 }
1039 if (candidate <= nslots_small_min) {
1040 return nslots_small_min;
1041 } else if (candidate <= nslots_small_max) {
1042 return candidate;
1043 } else {
1044 return nslots_small_max;
1045 }
1046}
1047
1048bool
1049tcache_boot(tsdn_t *tsdn, base_t *base) {
1050 tcache_maxclass = sz_s2u(opt_tcache_max);
1051 assert(tcache_maxclass <= TCACHE_MAXCLASS_LIMIT);
1052 nhbins = sz_size2index(tcache_maxclass) + 1;
1053
1054 if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES,
1055 malloc_mutex_rank_exclusive)) {
1056 return true;
1057 }
1058
1059 /* Initialize tcache_bin_info. See comments in tcache_init(). */
1060 unsigned n_reserved_bins = nhbins < SC_NBINS ? SC_NBINS : nhbins;
1061 size_t size = n_reserved_bins * sizeof(cache_bin_info_t);
1062 tcache_bin_info = (cache_bin_info_t *)base_alloc(tsdn, base, size,
1063 CACHELINE);
1064 if (tcache_bin_info == NULL) {
1065 return true;
1066 }
1067
1068 for (szind_t i = 0; i < nhbins; i++) {
1069 unsigned ncached_max = tcache_ncached_max_compute(i);
1070 cache_bin_info_init(&tcache_bin_info[i], ncached_max);
1071 }
1072 for (szind_t i = nhbins; i < SC_NBINS; i++) {
1073 /* Disabled small bins. */
1074 cache_bin_info_init(&tcache_bin_info[i], 0);
1075 assert(tcache_small_bin_disabled(i, NULL));
1076 }
1077
1078 cache_bin_info_compute_alloc(tcache_bin_info, nhbins,
1079 &tcache_bin_alloc_size, &tcache_bin_alloc_alignment);
1080
1081 return false;
1082}
1083
1084void
1085tcache_prefork(tsdn_t *tsdn) {
1086 malloc_mutex_prefork(tsdn, &tcaches_mtx);
1087}
1088
1089void
1090tcache_postfork_parent(tsdn_t *tsdn) {
1091 malloc_mutex_postfork_parent(tsdn, &tcaches_mtx);
1092}
1093
1094void
1095tcache_postfork_child(tsdn_t *tsdn) {
1096 malloc_mutex_postfork_child(tsdn, &tcaches_mtx);
1097}
1098
1099void tcache_assert_initialized(tcache_t *tcache) {
1100 assert(!cache_bin_still_zero_initialized(&tcache->bins[0]));
1101}
diff --git a/examples/redis-unstable/deps/jemalloc/src/test_hooks.c b/examples/redis-unstable/deps/jemalloc/src/test_hooks.c
deleted file mode 100644
index ace00d9..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/test_hooks.c
+++ /dev/null
@@ -1,12 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2
3/*
4 * The hooks are a little bit screwy -- they're not genuinely exported in the
5 * sense that we want them available to end-users, but we do want them visible
6 * from outside the generated library, so that we can use them in test code.
7 */
8JEMALLOC_EXPORT
9void (*test_hooks_arena_new_hook)() = NULL;
10
11JEMALLOC_EXPORT
12void (*test_hooks_libc_hook)() = NULL;
diff --git a/examples/redis-unstable/deps/jemalloc/src/thread_event.c b/examples/redis-unstable/deps/jemalloc/src/thread_event.c
deleted file mode 100644
index 37eb582..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/thread_event.c
+++ /dev/null
@@ -1,343 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/thread_event.h"
5
6/*
7 * Signatures for event specific functions. These functions should be defined
8 * by the modules owning each event. The signatures here verify that the
9 * definitions follow the right format.
10 *
11 * The first two are functions computing new / postponed event wait time. New
12 * event wait time is the time till the next event if an event is currently
13 * being triggered; postponed event wait time is the time till the next event
14 * if an event should be triggered but needs to be postponed, e.g. when the TSD
15 * is not nominal or during reentrancy.
16 *
17 * The third is the event handler function, which is called whenever an event
18 * is triggered. The parameter is the elapsed time since the last time an
19 * event of the same type was triggered.
20 */
21#define E(event, condition_unused, is_alloc_event_unused) \
22uint64_t event##_new_event_wait(tsd_t *tsd); \
23uint64_t event##_postponed_event_wait(tsd_t *tsd); \
24void event##_event_handler(tsd_t *tsd, uint64_t elapsed);
25
26ITERATE_OVER_ALL_EVENTS
27#undef E
28
29/* Signatures for internal functions fetching elapsed time. */
30#define E(event, condition_unused, is_alloc_event_unused) \
31static uint64_t event##_fetch_elapsed(tsd_t *tsd);
32
33ITERATE_OVER_ALL_EVENTS
34#undef E
35
36static uint64_t
37tcache_gc_fetch_elapsed(tsd_t *tsd) {
38 return TE_INVALID_ELAPSED;
39}
40
41static uint64_t
42tcache_gc_dalloc_fetch_elapsed(tsd_t *tsd) {
43 return TE_INVALID_ELAPSED;
44}
45
46static uint64_t
47prof_sample_fetch_elapsed(tsd_t *tsd) {
48 uint64_t last_event = thread_allocated_last_event_get(tsd);
49 uint64_t last_sample_event = prof_sample_last_event_get(tsd);
50 prof_sample_last_event_set(tsd, last_event);
51 return last_event - last_sample_event;
52}
53
54static uint64_t
55stats_interval_fetch_elapsed(tsd_t *tsd) {
56 uint64_t last_event = thread_allocated_last_event_get(tsd);
57 uint64_t last_stats_event = stats_interval_last_event_get(tsd);
58 stats_interval_last_event_set(tsd, last_event);
59 return last_event - last_stats_event;
60}
61
62static uint64_t
63peak_alloc_fetch_elapsed(tsd_t *tsd) {
64 return TE_INVALID_ELAPSED;
65}
66
67static uint64_t
68peak_dalloc_fetch_elapsed(tsd_t *tsd) {
69 return TE_INVALID_ELAPSED;
70}
71
72/* Per event facilities done. */
73
74static bool
75te_ctx_has_active_events(te_ctx_t *ctx) {
76 assert(config_debug);
77#define E(event, condition, alloc_event) \
78 if (condition && alloc_event == ctx->is_alloc) { \
79 return true; \
80 }
81 ITERATE_OVER_ALL_EVENTS
82#undef E
83 return false;
84}
85
86static uint64_t
87te_next_event_compute(tsd_t *tsd, bool is_alloc) {
88 uint64_t wait = TE_MAX_START_WAIT;
89#define E(event, condition, alloc_event) \
90 if (is_alloc == alloc_event && condition) { \
91 uint64_t event_wait = \
92 event##_event_wait_get(tsd); \
93 assert(event_wait <= TE_MAX_START_WAIT); \
94 if (event_wait > 0U && event_wait < wait) { \
95 wait = event_wait; \
96 } \
97 }
98
99 ITERATE_OVER_ALL_EVENTS
100#undef E
101 assert(wait <= TE_MAX_START_WAIT);
102 return wait;
103}
104
105static void
106te_assert_invariants_impl(tsd_t *tsd, te_ctx_t *ctx) {
107 uint64_t current_bytes = te_ctx_current_bytes_get(ctx);
108 uint64_t last_event = te_ctx_last_event_get(ctx);
109 uint64_t next_event = te_ctx_next_event_get(ctx);
110 uint64_t next_event_fast = te_ctx_next_event_fast_get(ctx);
111
112 assert(last_event != next_event);
113 if (next_event > TE_NEXT_EVENT_FAST_MAX || !tsd_fast(tsd)) {
114 assert(next_event_fast == 0U);
115 } else {
116 assert(next_event_fast == next_event);
117 }
118
119 /* The subtraction is intentionally susceptible to underflow. */
120 uint64_t interval = next_event - last_event;
121
122 /* The subtraction is intentionally susceptible to underflow. */
123 assert(current_bytes - last_event < interval);
124 uint64_t min_wait = te_next_event_compute(tsd, te_ctx_is_alloc(ctx));
125 /*
126 * next_event should have been pushed up only except when no event is
127 * on and the TSD is just initialized. The last_event == 0U guard
128 * below is stronger than needed, but having an exactly accurate guard
129 * is more complicated to implement.
130 */
131 assert((!te_ctx_has_active_events(ctx) && last_event == 0U) ||
132 interval == min_wait ||
133 (interval < min_wait && interval == TE_MAX_INTERVAL));
134}
135
136void
137te_assert_invariants_debug(tsd_t *tsd) {
138 te_ctx_t ctx;
139 te_ctx_get(tsd, &ctx, true);
140 te_assert_invariants_impl(tsd, &ctx);
141
142 te_ctx_get(tsd, &ctx, false);
143 te_assert_invariants_impl(tsd, &ctx);
144}
145
146/*
147 * Synchronization around the fast threshold in tsd --
148 * There are two threads to consider in the synchronization here:
149 * - The owner of the tsd being updated by a slow path change
150 * - The remote thread, doing that slow path change.
151 *
152 * As a design constraint, we want to ensure that a slow-path transition cannot
153 * be ignored for arbitrarily long, and that if the remote thread causes a
154 * slow-path transition and then communicates with the owner thread that it has
155 * occurred, then the owner will go down the slow path on the next allocator
156 * operation (so that we don't want to just wait until the owner hits its slow
157 * path reset condition on its own).
158 *
159 * Here's our strategy to do that:
160 *
161 * The remote thread will update the slow-path stores to TSD variables, issue a
162 * SEQ_CST fence, and then update the TSD next_event_fast counter. The owner
163 * thread will update next_event_fast, issue an SEQ_CST fence, and then check
164 * its TSD to see if it's on the slow path.
165
166 * This is fairly straightforward when 64-bit atomics are supported. Assume that
167 * the remote fence is sandwiched between two owner fences in the reset pathway.
168 * The case where there is no preceding or trailing owner fence (i.e. because
169 * the owner thread is near the beginning or end of its life) can be analyzed
170 * similarly. The owner store to next_event_fast preceding the earlier owner
171 * fence will be earlier in coherence order than the remote store to it, so that
172 * the owner thread will go down the slow path once the store becomes visible to
173 * it, which is no later than the time of the second fence.
174
175 * The case where we don't support 64-bit atomics is trickier, since word
176 * tearing is possible. We'll repeat the same analysis, and look at the two
177 * owner fences sandwiching the remote fence. The next_event_fast stores done
178 * alongside the earlier owner fence cannot overwrite any of the remote stores
179 * (since they precede the earlier owner fence in sb, which precedes the remote
180 * fence in sc, which precedes the remote stores in sb). After the second owner
181 * fence there will be a re-check of the slow-path variables anyways, so the
182 * "owner will notice that it's on the slow path eventually" guarantee is
183 * satisfied. To make sure that the out-of-band-messaging constraint is as well,
184 * note that either the message passing is sequenced before the second owner
185 * fence (in which case the remote stores happen before the second set of owner
186 * stores, so malloc sees a value of zero for next_event_fast and goes down the
187 * slow path), or it is not (in which case the owner sees the tsd slow-path
188 * writes on its previous update). This leaves open the possibility that the
189 * remote thread will (at some arbitrary point in the future) zero out one half
190 * of the owner thread's next_event_fast, but that's always safe (it just sends
191 * it down the slow path earlier).
192 */
193static void
194te_ctx_next_event_fast_update(te_ctx_t *ctx) {
195 uint64_t next_event = te_ctx_next_event_get(ctx);
196 uint64_t next_event_fast = (next_event <= TE_NEXT_EVENT_FAST_MAX) ?
197 next_event : 0U;
198 te_ctx_next_event_fast_set(ctx, next_event_fast);
199}
200
201void
202te_recompute_fast_threshold(tsd_t *tsd) {
203 if (tsd_state_get(tsd) != tsd_state_nominal) {
204 /* Check first because this is also called on purgatory. */
205 te_next_event_fast_set_non_nominal(tsd);
206 return;
207 }
208
209 te_ctx_t ctx;
210 te_ctx_get(tsd, &ctx, true);
211 te_ctx_next_event_fast_update(&ctx);
212 te_ctx_get(tsd, &ctx, false);
213 te_ctx_next_event_fast_update(&ctx);
214
215 atomic_fence(ATOMIC_SEQ_CST);
216 if (tsd_state_get(tsd) != tsd_state_nominal) {
217 te_next_event_fast_set_non_nominal(tsd);
218 }
219}
220
221static void
222te_adjust_thresholds_helper(tsd_t *tsd, te_ctx_t *ctx,
223 uint64_t wait) {
224 /*
225 * The next threshold based on future events can only be adjusted after
226 * progressing the last_event counter (which is set to current).
227 */
228 assert(te_ctx_current_bytes_get(ctx) == te_ctx_last_event_get(ctx));
229 assert(wait <= TE_MAX_START_WAIT);
230
231 uint64_t next_event = te_ctx_last_event_get(ctx) + (wait <=
232 TE_MAX_INTERVAL ? wait : TE_MAX_INTERVAL);
233 te_ctx_next_event_set(tsd, ctx, next_event);
234}
235
236static uint64_t
237te_clip_event_wait(uint64_t event_wait) {
238 assert(event_wait > 0U);
239 if (TE_MIN_START_WAIT > 1U &&
240 unlikely(event_wait < TE_MIN_START_WAIT)) {
241 event_wait = TE_MIN_START_WAIT;
242 }
243 if (TE_MAX_START_WAIT < UINT64_MAX &&
244 unlikely(event_wait > TE_MAX_START_WAIT)) {
245 event_wait = TE_MAX_START_WAIT;
246 }
247 return event_wait;
248}
249
250void
251te_event_trigger(tsd_t *tsd, te_ctx_t *ctx) {
252 /* usize has already been added to thread_allocated. */
253 uint64_t bytes_after = te_ctx_current_bytes_get(ctx);
254 /* The subtraction is intentionally susceptible to underflow. */
255 uint64_t accumbytes = bytes_after - te_ctx_last_event_get(ctx);
256
257 te_ctx_last_event_set(ctx, bytes_after);
258
259 bool allow_event_trigger = tsd_nominal(tsd) &&
260 tsd_reentrancy_level_get(tsd) == 0;
261 bool is_alloc = ctx->is_alloc;
262 uint64_t wait = TE_MAX_START_WAIT;
263
264#define E(event, condition, alloc_event) \
265 bool is_##event##_triggered = false; \
266 if (is_alloc == alloc_event && condition) { \
267 uint64_t event_wait = event##_event_wait_get(tsd); \
268 assert(event_wait <= TE_MAX_START_WAIT); \
269 if (event_wait > accumbytes) { \
270 event_wait -= accumbytes; \
271 } else if (!allow_event_trigger) { \
272 event_wait = event##_postponed_event_wait(tsd); \
273 } else { \
274 is_##event##_triggered = true; \
275 event_wait = event##_new_event_wait(tsd); \
276 } \
277 event_wait = te_clip_event_wait(event_wait); \
278 event##_event_wait_set(tsd, event_wait); \
279 if (event_wait < wait) { \
280 wait = event_wait; \
281 } \
282 }
283
284 ITERATE_OVER_ALL_EVENTS
285#undef E
286
287 assert(wait <= TE_MAX_START_WAIT);
288 te_adjust_thresholds_helper(tsd, ctx, wait);
289 te_assert_invariants(tsd);
290
291#define E(event, condition, alloc_event) \
292 if (is_alloc == alloc_event && condition && \
293 is_##event##_triggered) { \
294 assert(allow_event_trigger); \
295 uint64_t elapsed = event##_fetch_elapsed(tsd); \
296 event##_event_handler(tsd, elapsed); \
297 }
298
299 ITERATE_OVER_ALL_EVENTS
300#undef E
301
302 te_assert_invariants(tsd);
303}
304
305static void
306te_init(tsd_t *tsd, bool is_alloc) {
307 te_ctx_t ctx;
308 te_ctx_get(tsd, &ctx, is_alloc);
309 /*
310 * Reset the last event to current, which starts the events from a clean
311 * state. This is necessary when re-init the tsd event counters.
312 *
313 * The event counters maintain a relationship with the current bytes:
314 * last_event <= current < next_event. When a reinit happens (e.g.
315 * reincarnated tsd), the last event needs progressing because all
316 * events start fresh from the current bytes.
317 */
318 te_ctx_last_event_set(&ctx, te_ctx_current_bytes_get(&ctx));
319
320 uint64_t wait = TE_MAX_START_WAIT;
321#define E(event, condition, alloc_event) \
322 if (is_alloc == alloc_event && condition) { \
323 uint64_t event_wait = event##_new_event_wait(tsd); \
324 event_wait = te_clip_event_wait(event_wait); \
325 event##_event_wait_set(tsd, event_wait); \
326 if (event_wait < wait) { \
327 wait = event_wait; \
328 } \
329 }
330
331 ITERATE_OVER_ALL_EVENTS
332#undef E
333 te_adjust_thresholds_helper(tsd, &ctx, wait);
334}
335
336void
337tsd_te_init(tsd_t *tsd) {
338 /* Make sure no overflow for the bytes accumulated on event_trigger. */
339 assert(TE_MAX_INTERVAL <= UINT64_MAX - SC_LARGE_MAXCLASS + 1);
340 te_init(tsd, true);
341 te_init(tsd, false);
342 te_assert_invariants(tsd);
343}
diff --git a/examples/redis-unstable/deps/jemalloc/src/ticker.c b/examples/redis-unstable/deps/jemalloc/src/ticker.c
deleted file mode 100644
index 790b5c2..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/ticker.c
+++ /dev/null
@@ -1,32 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4/*
5 * To avoid using floating point math down core paths (still necessary because
6 * versions of the glibc dynamic loader that did not preserve xmm registers are
7 * still somewhat common, requiring us to be compilable with -mno-sse), and also
8 * to avoid generally expensive library calls, we use a precomputed table of
9 * values. We want to sample U uniformly on [0, 1], and then compute
10 * ceil(log(u)/log(1-1/nticks)). We're mostly interested in the case where
11 * nticks is reasonably big, so 1/log(1-1/nticks) is well-approximated by
12 * -nticks.
13 *
14 * To compute log(u), we sample an integer in [1, 64] and divide, then just look
15 * up results in a table. As a space-compression mechanism, we store these as
16 * uint8_t by dividing the range (255) by the highest-magnitude value the log
17 * can take on, and using that as a multiplier. We then have to divide by that
18 * multiplier at the end of the computation.
19 *
20 * The values here are computed in src/ticker.py
21 */
22
23const uint8_t ticker_geom_table[1 << TICKER_GEOM_NBITS] = {
24 254, 211, 187, 169, 156, 144, 135, 127,
25 120, 113, 107, 102, 97, 93, 89, 85,
26 81, 77, 74, 71, 68, 65, 62, 60,
27 57, 55, 53, 50, 48, 46, 44, 42,
28 40, 39, 37, 35, 33, 32, 30, 29,
29 27, 26, 24, 23, 21, 20, 19, 18,
30 16, 15, 14, 13, 12, 10, 9, 8,
31 7, 6, 5, 4, 3, 2, 1, 0
32};
diff --git a/examples/redis-unstable/deps/jemalloc/src/ticker.py b/examples/redis-unstable/deps/jemalloc/src/ticker.py
deleted file mode 100755
index 3807740..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/ticker.py
+++ /dev/null
@@ -1,15 +0,0 @@
1#!/usr/bin/env python3
2
3import math
4
5# Must match TICKER_GEOM_NBITS
6lg_table_size = 6
7table_size = 2**lg_table_size
8byte_max = 255
9mul = math.floor(-byte_max/math.log(1 / table_size))
10values = [round(-mul * math.log(i / table_size))
11 for i in range(1, table_size+1)]
12print("mul =", mul)
13print("values:")
14for i in range(table_size // 8):
15 print(", ".join((str(x) for x in values[i*8 : i*8 + 8])))
diff --git a/examples/redis-unstable/deps/jemalloc/src/tsd.c b/examples/redis-unstable/deps/jemalloc/src/tsd.c
deleted file mode 100644
index e8e4f3a..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/tsd.c
+++ /dev/null
@@ -1,549 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/san.h"
6#include "jemalloc/internal/mutex.h"
7#include "jemalloc/internal/rtree.h"
8
9/******************************************************************************/
10/* Data. */
11
12/* TSD_INITIALIZER triggers "-Wmissing-field-initializer" */
13JEMALLOC_DIAGNOSTIC_PUSH
14JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
15
16#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
17JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls = TSD_INITIALIZER;
18JEMALLOC_TSD_TYPE_ATTR(bool) JEMALLOC_TLS_MODEL tsd_initialized = false;
19bool tsd_booted = false;
20#elif (defined(JEMALLOC_TLS))
21JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls = TSD_INITIALIZER;
22pthread_key_t tsd_tsd;
23bool tsd_booted = false;
24#elif (defined(_WIN32))
25DWORD tsd_tsd;
26tsd_wrapper_t tsd_boot_wrapper = {false, TSD_INITIALIZER};
27bool tsd_booted = false;
28#else
29
30/*
31 * This contains a mutex, but it's pretty convenient to allow the mutex code to
32 * have a dependency on tsd. So we define the struct here, and only refer to it
33 * by pointer in the header.
34 */
35struct tsd_init_head_s {
36 ql_head(tsd_init_block_t) blocks;
37 malloc_mutex_t lock;
38};
39
40pthread_key_t tsd_tsd;
41tsd_init_head_t tsd_init_head = {
42 ql_head_initializer(blocks),
43 MALLOC_MUTEX_INITIALIZER
44};
45
46tsd_wrapper_t tsd_boot_wrapper = {
47 false,
48 TSD_INITIALIZER
49};
50bool tsd_booted = false;
51#endif
52
53JEMALLOC_DIAGNOSTIC_POP
54
55/******************************************************************************/
56
57/* A list of all the tsds in the nominal state. */
58typedef ql_head(tsd_t) tsd_list_t;
59static tsd_list_t tsd_nominal_tsds = ql_head_initializer(tsd_nominal_tsds);
60static malloc_mutex_t tsd_nominal_tsds_lock;
61
62/* How many slow-path-enabling features are turned on. */
63static atomic_u32_t tsd_global_slow_count = ATOMIC_INIT(0);
64
65static bool
66tsd_in_nominal_list(tsd_t *tsd) {
67 tsd_t *tsd_list;
68 bool found = false;
69 /*
70 * We don't know that tsd is nominal; it might not be safe to get data
71 * out of it here.
72 */
73 malloc_mutex_lock(TSDN_NULL, &tsd_nominal_tsds_lock);
74 ql_foreach(tsd_list, &tsd_nominal_tsds, TSD_MANGLE(tsd_link)) {
75 if (tsd == tsd_list) {
76 found = true;
77 break;
78 }
79 }
80 malloc_mutex_unlock(TSDN_NULL, &tsd_nominal_tsds_lock);
81 return found;
82}
83
84static void
85tsd_add_nominal(tsd_t *tsd) {
86 assert(!tsd_in_nominal_list(tsd));
87 assert(tsd_state_get(tsd) <= tsd_state_nominal_max);
88 ql_elm_new(tsd, TSD_MANGLE(tsd_link));
89 malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
90 ql_tail_insert(&tsd_nominal_tsds, tsd, TSD_MANGLE(tsd_link));
91 malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
92}
93
94static void
95tsd_remove_nominal(tsd_t *tsd) {
96 assert(tsd_in_nominal_list(tsd));
97 assert(tsd_state_get(tsd) <= tsd_state_nominal_max);
98 malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
99 ql_remove(&tsd_nominal_tsds, tsd, TSD_MANGLE(tsd_link));
100 malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
101}
102
103static void
104tsd_force_recompute(tsdn_t *tsdn) {
105 /*
106 * The stores to tsd->state here need to synchronize with the exchange
107 * in tsd_slow_update.
108 */
109 atomic_fence(ATOMIC_RELEASE);
110 malloc_mutex_lock(tsdn, &tsd_nominal_tsds_lock);
111 tsd_t *remote_tsd;
112 ql_foreach(remote_tsd, &tsd_nominal_tsds, TSD_MANGLE(tsd_link)) {
113 assert(tsd_atomic_load(&remote_tsd->state, ATOMIC_RELAXED)
114 <= tsd_state_nominal_max);
115 tsd_atomic_store(&remote_tsd->state,
116 tsd_state_nominal_recompute, ATOMIC_RELAXED);
117 /* See comments in te_recompute_fast_threshold(). */
118 atomic_fence(ATOMIC_SEQ_CST);
119 te_next_event_fast_set_non_nominal(remote_tsd);
120 }
121 malloc_mutex_unlock(tsdn, &tsd_nominal_tsds_lock);
122}
123
124void
125tsd_global_slow_inc(tsdn_t *tsdn) {
126 atomic_fetch_add_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED);
127 /*
128 * We unconditionally force a recompute, even if the global slow count
129 * was already positive. If we didn't, then it would be possible for us
130 * to return to the user, have the user synchronize externally with some
131 * other thread, and then have that other thread not have picked up the
132 * update yet (since the original incrementing thread might still be
133 * making its way through the tsd list).
134 */
135 tsd_force_recompute(tsdn);
136}
137
138void tsd_global_slow_dec(tsdn_t *tsdn) {
139 atomic_fetch_sub_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED);
140 /* See the note in ..._inc(). */
141 tsd_force_recompute(tsdn);
142}
143
144static bool
145tsd_local_slow(tsd_t *tsd) {
146 return !tsd_tcache_enabled_get(tsd)
147 || tsd_reentrancy_level_get(tsd) > 0;
148}
149
150bool
151tsd_global_slow() {
152 return atomic_load_u32(&tsd_global_slow_count, ATOMIC_RELAXED) > 0;
153}
154
155/******************************************************************************/
156
157static uint8_t
158tsd_state_compute(tsd_t *tsd) {
159 if (!tsd_nominal(tsd)) {
160 return tsd_state_get(tsd);
161 }
162 /* We're in *a* nominal state; but which one? */
163 if (malloc_slow || tsd_local_slow(tsd) || tsd_global_slow()) {
164 return tsd_state_nominal_slow;
165 } else {
166 return tsd_state_nominal;
167 }
168}
169
170void
171tsd_slow_update(tsd_t *tsd) {
172 uint8_t old_state;
173 do {
174 uint8_t new_state = tsd_state_compute(tsd);
175 old_state = tsd_atomic_exchange(&tsd->state, new_state,
176 ATOMIC_ACQUIRE);
177 } while (old_state == tsd_state_nominal_recompute);
178
179 te_recompute_fast_threshold(tsd);
180}
181
182void
183tsd_state_set(tsd_t *tsd, uint8_t new_state) {
184 /* Only the tsd module can change the state *to* recompute. */
185 assert(new_state != tsd_state_nominal_recompute);
186 uint8_t old_state = tsd_atomic_load(&tsd->state, ATOMIC_RELAXED);
187 if (old_state > tsd_state_nominal_max) {
188 /*
189 * Not currently in the nominal list, but it might need to be
190 * inserted there.
191 */
192 assert(!tsd_in_nominal_list(tsd));
193 tsd_atomic_store(&tsd->state, new_state, ATOMIC_RELAXED);
194 if (new_state <= tsd_state_nominal_max) {
195 tsd_add_nominal(tsd);
196 }
197 } else {
198 /*
199 * We're currently nominal. If the new state is non-nominal,
200 * great; we take ourselves off the list and just enter the new
201 * state.
202 */
203 assert(tsd_in_nominal_list(tsd));
204 if (new_state > tsd_state_nominal_max) {
205 tsd_remove_nominal(tsd);
206 tsd_atomic_store(&tsd->state, new_state,
207 ATOMIC_RELAXED);
208 } else {
209 /*
210 * This is the tricky case. We're transitioning from
211 * one nominal state to another. The caller can't know
212 * about any races that are occurring at the same time,
213 * so we always have to recompute no matter what.
214 */
215 tsd_slow_update(tsd);
216 }
217 }
218 te_recompute_fast_threshold(tsd);
219}
220
221static void
222tsd_prng_state_init(tsd_t *tsd) {
223 /*
224 * A nondeterministic seed based on the address of tsd reduces
225 * the likelihood of lockstep non-uniform cache index
226 * utilization among identical concurrent processes, but at the
227 * cost of test repeatability. For debug builds, instead use a
228 * deterministic seed.
229 */
230 *tsd_prng_statep_get(tsd) = config_debug ? 0 :
231 (uint64_t)(uintptr_t)tsd;
232}
233
234static bool
235tsd_data_init(tsd_t *tsd) {
236 /*
237 * We initialize the rtree context first (before the tcache), since the
238 * tcache initialization depends on it.
239 */
240 rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd));
241 tsd_prng_state_init(tsd);
242 tsd_te_init(tsd); /* event_init may use the prng state above. */
243 tsd_san_init(tsd);
244 return tsd_tcache_enabled_data_init(tsd);
245}
246
247static void
248assert_tsd_data_cleanup_done(tsd_t *tsd) {
249 assert(!tsd_nominal(tsd));
250 assert(!tsd_in_nominal_list(tsd));
251 assert(*tsd_arenap_get_unsafe(tsd) == NULL);
252 assert(*tsd_iarenap_get_unsafe(tsd) == NULL);
253 assert(*tsd_tcache_enabledp_get_unsafe(tsd) == false);
254 assert(*tsd_prof_tdatap_get_unsafe(tsd) == NULL);
255}
256
257static bool
258tsd_data_init_nocleanup(tsd_t *tsd) {
259 assert(tsd_state_get(tsd) == tsd_state_reincarnated ||
260 tsd_state_get(tsd) == tsd_state_minimal_initialized);
261 /*
262 * During reincarnation, there is no guarantee that the cleanup function
263 * will be called (deallocation may happen after all tsd destructors).
264 * We set up tsd in a way that no cleanup is needed.
265 */
266 rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd));
267 *tsd_tcache_enabledp_get_unsafe(tsd) = false;
268 *tsd_reentrancy_levelp_get(tsd) = 1;
269 tsd_prng_state_init(tsd);
270 tsd_te_init(tsd); /* event_init may use the prng state above. */
271 tsd_san_init(tsd);
272 assert_tsd_data_cleanup_done(tsd);
273
274 return false;
275}
276
277tsd_t *
278tsd_fetch_slow(tsd_t *tsd, bool minimal) {
279 assert(!tsd_fast(tsd));
280
281 if (tsd_state_get(tsd) == tsd_state_nominal_slow) {
282 /*
283 * On slow path but no work needed. Note that we can't
284 * necessarily *assert* that we're slow, because we might be
285 * slow because of an asynchronous modification to global state,
286 * which might be asynchronously modified *back*.
287 */
288 } else if (tsd_state_get(tsd) == tsd_state_nominal_recompute) {
289 tsd_slow_update(tsd);
290 } else if (tsd_state_get(tsd) == tsd_state_uninitialized) {
291 if (!minimal) {
292 if (tsd_booted) {
293 tsd_state_set(tsd, tsd_state_nominal);
294 tsd_slow_update(tsd);
295 /* Trigger cleanup handler registration. */
296 tsd_set(tsd);
297 tsd_data_init(tsd);
298 }
299 } else {
300 tsd_state_set(tsd, tsd_state_minimal_initialized);
301 tsd_set(tsd);
302 tsd_data_init_nocleanup(tsd);
303 }
304 } else if (tsd_state_get(tsd) == tsd_state_minimal_initialized) {
305 if (!minimal) {
306 /* Switch to fully initialized. */
307 tsd_state_set(tsd, tsd_state_nominal);
308 assert(*tsd_reentrancy_levelp_get(tsd) >= 1);
309 (*tsd_reentrancy_levelp_get(tsd))--;
310 tsd_slow_update(tsd);
311 tsd_data_init(tsd);
312 } else {
313 assert_tsd_data_cleanup_done(tsd);
314 }
315 } else if (tsd_state_get(tsd) == tsd_state_purgatory) {
316 tsd_state_set(tsd, tsd_state_reincarnated);
317 tsd_set(tsd);
318 tsd_data_init_nocleanup(tsd);
319 } else {
320 assert(tsd_state_get(tsd) == tsd_state_reincarnated);
321 }
322
323 return tsd;
324}
325
326void *
327malloc_tsd_malloc(size_t size) {
328 return a0malloc(CACHELINE_CEILING(size));
329}
330
331void
332malloc_tsd_dalloc(void *wrapper) {
333 a0dalloc(wrapper);
334}
335
336#if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32)
337static unsigned ncleanups;
338static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX];
339
340#ifndef _WIN32
341JEMALLOC_EXPORT
342#endif
343void
344_malloc_thread_cleanup(void) {
345 bool pending[MALLOC_TSD_CLEANUPS_MAX], again;
346 unsigned i;
347
348 for (i = 0; i < ncleanups; i++) {
349 pending[i] = true;
350 }
351
352 do {
353 again = false;
354 for (i = 0; i < ncleanups; i++) {
355 if (pending[i]) {
356 pending[i] = cleanups[i]();
357 if (pending[i]) {
358 again = true;
359 }
360 }
361 }
362 } while (again);
363}
364
365#ifndef _WIN32
366JEMALLOC_EXPORT
367#endif
368void
369_malloc_tsd_cleanup_register(bool (*f)(void)) {
370 assert(ncleanups < MALLOC_TSD_CLEANUPS_MAX);
371 cleanups[ncleanups] = f;
372 ncleanups++;
373}
374
375#endif
376
377static void
378tsd_do_data_cleanup(tsd_t *tsd) {
379 prof_tdata_cleanup(tsd);
380 iarena_cleanup(tsd);
381 arena_cleanup(tsd);
382 tcache_cleanup(tsd);
383 witnesses_cleanup(tsd_witness_tsdp_get_unsafe(tsd));
384 *tsd_reentrancy_levelp_get(tsd) = 1;
385}
386
387void
388tsd_cleanup(void *arg) {
389 tsd_t *tsd = (tsd_t *)arg;
390
391 switch (tsd_state_get(tsd)) {
392 case tsd_state_uninitialized:
393 /* Do nothing. */
394 break;
395 case tsd_state_minimal_initialized:
396 /* This implies the thread only did free() in its life time. */
397 /* Fall through. */
398 case tsd_state_reincarnated:
399 /*
400 * Reincarnated means another destructor deallocated memory
401 * after the destructor was called. Cleanup isn't required but
402 * is still called for testing and completeness.
403 */
404 assert_tsd_data_cleanup_done(tsd);
405 JEMALLOC_FALLTHROUGH;
406 case tsd_state_nominal:
407 case tsd_state_nominal_slow:
408 tsd_do_data_cleanup(tsd);
409 tsd_state_set(tsd, tsd_state_purgatory);
410 tsd_set(tsd);
411 break;
412 case tsd_state_purgatory:
413 /*
414 * The previous time this destructor was called, we set the
415 * state to tsd_state_purgatory so that other destructors
416 * wouldn't cause re-creation of the tsd. This time, do
417 * nothing, and do not request another callback.
418 */
419 break;
420 default:
421 not_reached();
422 }
423#ifdef JEMALLOC_JET
424 test_callback_t test_callback = *tsd_test_callbackp_get_unsafe(tsd);
425 int *data = tsd_test_datap_get_unsafe(tsd);
426 if (test_callback != NULL) {
427 test_callback(data);
428 }
429#endif
430}
431
432tsd_t *
433malloc_tsd_boot0(void) {
434 tsd_t *tsd;
435
436#if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32)
437 ncleanups = 0;
438#endif
439 if (malloc_mutex_init(&tsd_nominal_tsds_lock, "tsd_nominal_tsds_lock",
440 WITNESS_RANK_OMIT, malloc_mutex_rank_exclusive)) {
441 return NULL;
442 }
443 if (tsd_boot0()) {
444 return NULL;
445 }
446 tsd = tsd_fetch();
447 return tsd;
448}
449
450void
451malloc_tsd_boot1(void) {
452 tsd_boot1();
453 tsd_t *tsd = tsd_fetch();
454 /* malloc_slow has been set properly. Update tsd_slow. */
455 tsd_slow_update(tsd);
456}
457
458#ifdef _WIN32
459static BOOL WINAPI
460_tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) {
461 switch (fdwReason) {
462#ifdef JEMALLOC_LAZY_LOCK
463 case DLL_THREAD_ATTACH:
464 isthreaded = true;
465 break;
466#endif
467 case DLL_THREAD_DETACH:
468 _malloc_thread_cleanup();
469 break;
470 default:
471 break;
472 }
473 return true;
474}
475
476/*
477 * We need to be able to say "read" here (in the "pragma section"), but have
478 * hooked "read". We won't read for the rest of the file, so we can get away
479 * with unhooking.
480 */
481#ifdef read
482# undef read
483#endif
484
485#ifdef _MSC_VER
486# ifdef _M_IX86
487# pragma comment(linker, "/INCLUDE:__tls_used")
488# pragma comment(linker, "/INCLUDE:_tls_callback")
489# else
490# pragma comment(linker, "/INCLUDE:_tls_used")
491# pragma comment(linker, "/INCLUDE:" STRINGIFY(tls_callback) )
492# endif
493# pragma section(".CRT$XLY",long,read)
494#endif
495JEMALLOC_SECTION(".CRT$XLY") JEMALLOC_ATTR(used)
496BOOL (WINAPI *const tls_callback)(HINSTANCE hinstDLL,
497 DWORD fdwReason, LPVOID lpvReserved) = _tls_callback;
498#endif
499
500#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
501 !defined(_WIN32))
502void *
503tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block) {
504 pthread_t self = pthread_self();
505 tsd_init_block_t *iter;
506
507 /* Check whether this thread has already inserted into the list. */
508 malloc_mutex_lock(TSDN_NULL, &head->lock);
509 ql_foreach(iter, &head->blocks, link) {
510 if (iter->thread == self) {
511 malloc_mutex_unlock(TSDN_NULL, &head->lock);
512 return iter->data;
513 }
514 }
515 /* Insert block into list. */
516 ql_elm_new(block, link);
517 block->thread = self;
518 ql_tail_insert(&head->blocks, block, link);
519 malloc_mutex_unlock(TSDN_NULL, &head->lock);
520 return NULL;
521}
522
523void
524tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block) {
525 malloc_mutex_lock(TSDN_NULL, &head->lock);
526 ql_remove(&head->blocks, block, link);
527 malloc_mutex_unlock(TSDN_NULL, &head->lock);
528}
529#endif
530
531void
532tsd_prefork(tsd_t *tsd) {
533 malloc_mutex_prefork(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
534}
535
536void
537tsd_postfork_parent(tsd_t *tsd) {
538 malloc_mutex_postfork_parent(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
539}
540
541void
542tsd_postfork_child(tsd_t *tsd) {
543 malloc_mutex_postfork_child(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
544 ql_new(&tsd_nominal_tsds);
545
546 if (tsd_state_get(tsd) <= tsd_state_nominal_max) {
547 tsd_add_nominal(tsd);
548 }
549}
diff --git a/examples/redis-unstable/deps/jemalloc/src/witness.c b/examples/redis-unstable/deps/jemalloc/src/witness.c
deleted file mode 100644
index 4474af0..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/witness.c
+++ /dev/null
@@ -1,122 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/malloc_io.h"
6
7void
8witness_init(witness_t *witness, const char *name, witness_rank_t rank,
9 witness_comp_t *comp, void *opaque) {
10 witness->name = name;
11 witness->rank = rank;
12 witness->comp = comp;
13 witness->opaque = opaque;
14}
15
16static void
17witness_print_witness(witness_t *w, unsigned n) {
18 assert(n > 0);
19 if (n == 1) {
20 malloc_printf(" %s(%u)", w->name, w->rank);
21 } else {
22 malloc_printf(" %s(%u)X%u", w->name, w->rank, n);
23 }
24}
25
26static void
27witness_print_witnesses(const witness_list_t *witnesses) {
28 witness_t *w, *last = NULL;
29 unsigned n = 0;
30 ql_foreach(w, witnesses, link) {
31 if (last != NULL && w->rank > last->rank) {
32 assert(w->name != last->name);
33 witness_print_witness(last, n);
34 n = 0;
35 } else if (last != NULL) {
36 assert(w->rank == last->rank);
37 assert(w->name == last->name);
38 }
39 last = w;
40 ++n;
41 }
42 if (last != NULL) {
43 witness_print_witness(last, n);
44 }
45}
46
47static void
48witness_lock_error_impl(const witness_list_t *witnesses,
49 const witness_t *witness) {
50 malloc_printf("<jemalloc>: Lock rank order reversal:");
51 witness_print_witnesses(witnesses);
52 malloc_printf(" %s(%u)\n", witness->name, witness->rank);
53 abort();
54}
55witness_lock_error_t *JET_MUTABLE witness_lock_error = witness_lock_error_impl;
56
57static void
58witness_owner_error_impl(const witness_t *witness) {
59 malloc_printf("<jemalloc>: Should own %s(%u)\n", witness->name,
60 witness->rank);
61 abort();
62}
63witness_owner_error_t *JET_MUTABLE witness_owner_error =
64 witness_owner_error_impl;
65
66static void
67witness_not_owner_error_impl(const witness_t *witness) {
68 malloc_printf("<jemalloc>: Should not own %s(%u)\n", witness->name,
69 witness->rank);
70 abort();
71}
72witness_not_owner_error_t *JET_MUTABLE witness_not_owner_error =
73 witness_not_owner_error_impl;
74
75static void
76witness_depth_error_impl(const witness_list_t *witnesses,
77 witness_rank_t rank_inclusive, unsigned depth) {
78 malloc_printf("<jemalloc>: Should own %u lock%s of rank >= %u:", depth,
79 (depth != 1) ? "s" : "", rank_inclusive);
80 witness_print_witnesses(witnesses);
81 malloc_printf("\n");
82 abort();
83}
84witness_depth_error_t *JET_MUTABLE witness_depth_error =
85 witness_depth_error_impl;
86
87void
88witnesses_cleanup(witness_tsd_t *witness_tsd) {
89 witness_assert_lockless(witness_tsd_tsdn(witness_tsd));
90
91 /* Do nothing. */
92}
93
94void
95witness_prefork(witness_tsd_t *witness_tsd) {
96 if (!config_debug) {
97 return;
98 }
99 witness_tsd->forking = true;
100}
101
102void
103witness_postfork_parent(witness_tsd_t *witness_tsd) {
104 if (!config_debug) {
105 return;
106 }
107 witness_tsd->forking = false;
108}
109
110void
111witness_postfork_child(witness_tsd_t *witness_tsd) {
112 if (!config_debug) {
113 return;
114 }
115#ifndef JEMALLOC_MUTEX_INIT_CB
116 witness_list_t *witnesses;
117
118 witnesses = &witness_tsd->witnesses;
119 ql_new(witnesses);
120#endif
121 witness_tsd->forking = false;
122}
diff --git a/examples/redis-unstable/deps/jemalloc/src/zone.c b/examples/redis-unstable/deps/jemalloc/src/zone.c
deleted file mode 100644
index 23dfdd0..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/zone.c
+++ /dev/null
@@ -1,469 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5
6#ifndef JEMALLOC_ZONE
7# error "This source file is for zones on Darwin (OS X)."
8#endif
9
10/* Definitions of the following structs in malloc/malloc.h might be too old
11 * for the built binary to run on newer versions of OSX. So use the newest
12 * possible version of those structs.
13 */
14typedef struct _malloc_zone_t {
15 void *reserved1;
16 void *reserved2;
17 size_t (*size)(struct _malloc_zone_t *, const void *);
18 void *(*malloc)(struct _malloc_zone_t *, size_t);
19 void *(*calloc)(struct _malloc_zone_t *, size_t, size_t);
20 void *(*valloc)(struct _malloc_zone_t *, size_t);
21 void (*free)(struct _malloc_zone_t *, void *);
22 void *(*realloc)(struct _malloc_zone_t *, void *, size_t);
23 void (*destroy)(struct _malloc_zone_t *);
24 const char *zone_name;
25 unsigned (*batch_malloc)(struct _malloc_zone_t *, size_t, void **, unsigned);
26 void (*batch_free)(struct _malloc_zone_t *, void **, unsigned);
27 struct malloc_introspection_t *introspect;
28 unsigned version;
29 void *(*memalign)(struct _malloc_zone_t *, size_t, size_t);
30 void (*free_definite_size)(struct _malloc_zone_t *, void *, size_t);
31 size_t (*pressure_relief)(struct _malloc_zone_t *, size_t);
32} malloc_zone_t;
33
34typedef struct {
35 vm_address_t address;
36 vm_size_t size;
37} vm_range_t;
38
39typedef struct malloc_statistics_t {
40 unsigned blocks_in_use;
41 size_t size_in_use;
42 size_t max_size_in_use;
43 size_t size_allocated;
44} malloc_statistics_t;
45
46typedef kern_return_t memory_reader_t(task_t, vm_address_t, vm_size_t, void **);
47
48typedef void vm_range_recorder_t(task_t, void *, unsigned type, vm_range_t *, unsigned);
49
50typedef struct malloc_introspection_t {
51 kern_return_t (*enumerator)(task_t, void *, unsigned, vm_address_t, memory_reader_t, vm_range_recorder_t);
52 size_t (*good_size)(malloc_zone_t *, size_t);
53 boolean_t (*check)(malloc_zone_t *);
54 void (*print)(malloc_zone_t *, boolean_t);
55 void (*log)(malloc_zone_t *, void *);
56 void (*force_lock)(malloc_zone_t *);
57 void (*force_unlock)(malloc_zone_t *);
58 void (*statistics)(malloc_zone_t *, malloc_statistics_t *);
59 boolean_t (*zone_locked)(malloc_zone_t *);
60 boolean_t (*enable_discharge_checking)(malloc_zone_t *);
61 boolean_t (*disable_discharge_checking)(malloc_zone_t *);
62 void (*discharge)(malloc_zone_t *, void *);
63#ifdef __BLOCKS__
64 void (*enumerate_discharged_pointers)(malloc_zone_t *, void (^)(void *, void *));
65#else
66 void *enumerate_unavailable_without_blocks;
67#endif
68 void (*reinit_lock)(malloc_zone_t *);
69} malloc_introspection_t;
70
71extern kern_return_t malloc_get_all_zones(task_t, memory_reader_t, vm_address_t **, unsigned *);
72
73extern malloc_zone_t *malloc_default_zone(void);
74
75extern void malloc_zone_register(malloc_zone_t *zone);
76
77extern void malloc_zone_unregister(malloc_zone_t *zone);
78
79/*
80 * The malloc_default_purgeable_zone() function is only available on >= 10.6.
81 * We need to check whether it is present at runtime, thus the weak_import.
82 */
83extern malloc_zone_t *malloc_default_purgeable_zone(void)
84JEMALLOC_ATTR(weak_import);
85
86/******************************************************************************/
87/* Data. */
88
89static malloc_zone_t *default_zone, *purgeable_zone;
90static malloc_zone_t jemalloc_zone;
91static struct malloc_introspection_t jemalloc_zone_introspect;
92static pid_t zone_force_lock_pid = -1;
93
94/******************************************************************************/
95/* Function prototypes for non-inline static functions. */
96
97static size_t zone_size(malloc_zone_t *zone, const void *ptr);
98static void *zone_malloc(malloc_zone_t *zone, size_t size);
99static void *zone_calloc(malloc_zone_t *zone, size_t num, size_t size);
100static void *zone_valloc(malloc_zone_t *zone, size_t size);
101static void zone_free(malloc_zone_t *zone, void *ptr);
102static void *zone_realloc(malloc_zone_t *zone, void *ptr, size_t size);
103static void *zone_memalign(malloc_zone_t *zone, size_t alignment,
104 size_t size);
105static void zone_free_definite_size(malloc_zone_t *zone, void *ptr,
106 size_t size);
107static void zone_destroy(malloc_zone_t *zone);
108static unsigned zone_batch_malloc(struct _malloc_zone_t *zone, size_t size,
109 void **results, unsigned num_requested);
110static void zone_batch_free(struct _malloc_zone_t *zone,
111 void **to_be_freed, unsigned num_to_be_freed);
112static size_t zone_pressure_relief(struct _malloc_zone_t *zone, size_t goal);
113static size_t zone_good_size(malloc_zone_t *zone, size_t size);
114static kern_return_t zone_enumerator(task_t task, void *data, unsigned type_mask,
115 vm_address_t zone_address, memory_reader_t reader,
116 vm_range_recorder_t recorder);
117static boolean_t zone_check(malloc_zone_t *zone);
118static void zone_print(malloc_zone_t *zone, boolean_t verbose);
119static void zone_log(malloc_zone_t *zone, void *address);
120static void zone_force_lock(malloc_zone_t *zone);
121static void zone_force_unlock(malloc_zone_t *zone);
122static void zone_statistics(malloc_zone_t *zone,
123 malloc_statistics_t *stats);
124static boolean_t zone_locked(malloc_zone_t *zone);
125static void zone_reinit_lock(malloc_zone_t *zone);
126
127/******************************************************************************/
128/*
129 * Functions.
130 */
131
132static size_t
133zone_size(malloc_zone_t *zone, const void *ptr) {
134 /*
135 * There appear to be places within Darwin (such as setenv(3)) that
136 * cause calls to this function with pointers that *no* zone owns. If
137 * we knew that all pointers were owned by *some* zone, we could split
138 * our zone into two parts, and use one as the default allocator and
139 * the other as the default deallocator/reallocator. Since that will
140 * not work in practice, we must check all pointers to assure that they
141 * reside within a mapped extent before determining size.
142 */
143 return ivsalloc(tsdn_fetch(), ptr);
144}
145
146static void *
147zone_malloc(malloc_zone_t *zone, size_t size) {
148 return je_malloc(size);
149}
150
151static void *
152zone_calloc(malloc_zone_t *zone, size_t num, size_t size) {
153 return je_calloc(num, size);
154}
155
156static void *
157zone_valloc(malloc_zone_t *zone, size_t size) {
158 void *ret = NULL; /* Assignment avoids useless compiler warning. */
159
160 je_posix_memalign(&ret, PAGE, size);
161
162 return ret;
163}
164
165static void
166zone_free(malloc_zone_t *zone, void *ptr) {
167 if (ivsalloc(tsdn_fetch(), ptr) != 0) {
168 je_free(ptr);
169 return;
170 }
171
172 free(ptr);
173}
174
175static void *
176zone_realloc(malloc_zone_t *zone, void *ptr, size_t size) {
177 if (ivsalloc(tsdn_fetch(), ptr) != 0) {
178 return je_realloc(ptr, size);
179 }
180
181 return realloc(ptr, size);
182}
183
184static void *
185zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size) {
186 void *ret = NULL; /* Assignment avoids useless compiler warning. */
187
188 je_posix_memalign(&ret, alignment, size);
189
190 return ret;
191}
192
193static void
194zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) {
195 size_t alloc_size;
196
197 alloc_size = ivsalloc(tsdn_fetch(), ptr);
198 if (alloc_size != 0) {
199 assert(alloc_size == size);
200 je_free(ptr);
201 return;
202 }
203
204 free(ptr);
205}
206
207static void
208zone_destroy(malloc_zone_t *zone) {
209 /* This function should never be called. */
210 not_reached();
211}
212
213static unsigned
214zone_batch_malloc(struct _malloc_zone_t *zone, size_t size, void **results,
215 unsigned num_requested) {
216 unsigned i;
217
218 for (i = 0; i < num_requested; i++) {
219 results[i] = je_malloc(size);
220 if (!results[i])
221 break;
222 }
223
224 return i;
225}
226
227static void
228zone_batch_free(struct _malloc_zone_t *zone, void **to_be_freed,
229 unsigned num_to_be_freed) {
230 unsigned i;
231
232 for (i = 0; i < num_to_be_freed; i++) {
233 zone_free(zone, to_be_freed[i]);
234 to_be_freed[i] = NULL;
235 }
236}
237
238static size_t
239zone_pressure_relief(struct _malloc_zone_t *zone, size_t goal) {
240 return 0;
241}
242
243static size_t
244zone_good_size(malloc_zone_t *zone, size_t size) {
245 if (size == 0) {
246 size = 1;
247 }
248 return sz_s2u(size);
249}
250
251static kern_return_t
252zone_enumerator(task_t task, void *data, unsigned type_mask,
253 vm_address_t zone_address, memory_reader_t reader,
254 vm_range_recorder_t recorder) {
255 return KERN_SUCCESS;
256}
257
258static boolean_t
259zone_check(malloc_zone_t *zone) {
260 return true;
261}
262
263static void
264zone_print(malloc_zone_t *zone, boolean_t verbose) {
265}
266
267static void
268zone_log(malloc_zone_t *zone, void *address) {
269}
270
271static void
272zone_force_lock(malloc_zone_t *zone) {
273 if (isthreaded) {
274 /*
275 * See the note in zone_force_unlock, below, to see why we need
276 * this.
277 */
278 assert(zone_force_lock_pid == -1);
279 zone_force_lock_pid = getpid();
280 jemalloc_prefork();
281 }
282}
283
284static void
285zone_force_unlock(malloc_zone_t *zone) {
286 /*
287 * zone_force_lock and zone_force_unlock are the entry points to the
288 * forking machinery on OS X. The tricky thing is, the child is not
289 * allowed to unlock mutexes locked in the parent, even if owned by the
290 * forking thread (and the mutex type we use in OS X will fail an assert
291 * if we try). In the child, we can get away with reinitializing all
292 * the mutexes, which has the effect of unlocking them. In the parent,
293 * doing this would mean we wouldn't wake any waiters blocked on the
294 * mutexes we unlock. So, we record the pid of the current thread in
295 * zone_force_lock, and use that to detect if we're in the parent or
296 * child here, to decide which unlock logic we need.
297 */
298 if (isthreaded) {
299 assert(zone_force_lock_pid != -1);
300 if (getpid() == zone_force_lock_pid) {
301 jemalloc_postfork_parent();
302 } else {
303 jemalloc_postfork_child();
304 }
305 zone_force_lock_pid = -1;
306 }
307}
308
309static void
310zone_statistics(malloc_zone_t *zone, malloc_statistics_t *stats) {
311 /* We make no effort to actually fill the values */
312 stats->blocks_in_use = 0;
313 stats->size_in_use = 0;
314 stats->max_size_in_use = 0;
315 stats->size_allocated = 0;
316}
317
318static boolean_t
319zone_locked(malloc_zone_t *zone) {
320 /* Pretend no lock is being held */
321 return false;
322}
323
324static void
325zone_reinit_lock(malloc_zone_t *zone) {
326 /* As of OSX 10.12, this function is only used when force_unlock would
327 * be used if the zone version were < 9. So just use force_unlock. */
328 zone_force_unlock(zone);
329}
330
331static void
332zone_init(void) {
333 jemalloc_zone.size = zone_size;
334 jemalloc_zone.malloc = zone_malloc;
335 jemalloc_zone.calloc = zone_calloc;
336 jemalloc_zone.valloc = zone_valloc;
337 jemalloc_zone.free = zone_free;
338 jemalloc_zone.realloc = zone_realloc;
339 jemalloc_zone.destroy = zone_destroy;
340 jemalloc_zone.zone_name = "jemalloc_zone";
341 jemalloc_zone.batch_malloc = zone_batch_malloc;
342 jemalloc_zone.batch_free = zone_batch_free;
343 jemalloc_zone.introspect = &jemalloc_zone_introspect;
344 jemalloc_zone.version = 9;
345 jemalloc_zone.memalign = zone_memalign;
346 jemalloc_zone.free_definite_size = zone_free_definite_size;
347 jemalloc_zone.pressure_relief = zone_pressure_relief;
348
349 jemalloc_zone_introspect.enumerator = zone_enumerator;
350 jemalloc_zone_introspect.good_size = zone_good_size;
351 jemalloc_zone_introspect.check = zone_check;
352 jemalloc_zone_introspect.print = zone_print;
353 jemalloc_zone_introspect.log = zone_log;
354 jemalloc_zone_introspect.force_lock = zone_force_lock;
355 jemalloc_zone_introspect.force_unlock = zone_force_unlock;
356 jemalloc_zone_introspect.statistics = zone_statistics;
357 jemalloc_zone_introspect.zone_locked = zone_locked;
358 jemalloc_zone_introspect.enable_discharge_checking = NULL;
359 jemalloc_zone_introspect.disable_discharge_checking = NULL;
360 jemalloc_zone_introspect.discharge = NULL;
361#ifdef __BLOCKS__
362 jemalloc_zone_introspect.enumerate_discharged_pointers = NULL;
363#else
364 jemalloc_zone_introspect.enumerate_unavailable_without_blocks = NULL;
365#endif
366 jemalloc_zone_introspect.reinit_lock = zone_reinit_lock;
367}
368
369static malloc_zone_t *
370zone_default_get(void) {
371 malloc_zone_t **zones = NULL;
372 unsigned int num_zones = 0;
373
374 /*
375 * On OSX 10.12, malloc_default_zone returns a special zone that is not
376 * present in the list of registered zones. That zone uses a "lite zone"
377 * if one is present (apparently enabled when malloc stack logging is
378 * enabled), or the first registered zone otherwise. In practice this
379 * means unless malloc stack logging is enabled, the first registered
380 * zone is the default. So get the list of zones to get the first one,
381 * instead of relying on malloc_default_zone.
382 */
383 if (KERN_SUCCESS != malloc_get_all_zones(0, NULL,
384 (vm_address_t**)&zones, &num_zones)) {
385 /*
386 * Reset the value in case the failure happened after it was
387 * set.
388 */
389 num_zones = 0;
390 }
391
392 if (num_zones) {
393 return zones[0];
394 }
395
396 return malloc_default_zone();
397}
398
399/* As written, this function can only promote jemalloc_zone. */
400static void
401zone_promote(void) {
402 malloc_zone_t *zone;
403
404 do {
405 /*
406 * Unregister and reregister the default zone. On OSX >= 10.6,
407 * unregistering takes the last registered zone and places it
408 * at the location of the specified zone. Unregistering the
409 * default zone thus makes the last registered one the default.
410 * On OSX < 10.6, unregistering shifts all registered zones.
411 * The first registered zone then becomes the default.
412 */
413 malloc_zone_unregister(default_zone);
414 malloc_zone_register(default_zone);
415
416 /*
417 * On OSX 10.6, having the default purgeable zone appear before
418 * the default zone makes some things crash because it thinks it
419 * owns the default zone allocated pointers. We thus
420 * unregister/re-register it in order to ensure it's always
421 * after the default zone. On OSX < 10.6, there is no purgeable
422 * zone, so this does nothing. On OSX >= 10.6, unregistering
423 * replaces the purgeable zone with the last registered zone
424 * above, i.e. the default zone. Registering it again then puts
425 * it at the end, obviously after the default zone.
426 */
427 if (purgeable_zone != NULL) {
428 malloc_zone_unregister(purgeable_zone);
429 malloc_zone_register(purgeable_zone);
430 }
431
432 zone = zone_default_get();
433 } while (zone != &jemalloc_zone);
434}
435
436JEMALLOC_ATTR(constructor)
437void
438zone_register(void) {
439 /*
440 * If something else replaced the system default zone allocator, don't
441 * register jemalloc's.
442 */
443 default_zone = zone_default_get();
444 if (!default_zone->zone_name || strcmp(default_zone->zone_name,
445 "DefaultMallocZone") != 0) {
446 return;
447 }
448
449 /*
450 * The default purgeable zone is created lazily by OSX's libc. It uses
451 * the default zone when it is created for "small" allocations
452 * (< 15 KiB), but assumes the default zone is a scalable_zone. This
453 * obviously fails when the default zone is the jemalloc zone, so
454 * malloc_default_purgeable_zone() is called beforehand so that the
455 * default purgeable zone is created when the default zone is still
456 * a scalable_zone. As purgeable zones only exist on >= 10.6, we need
457 * to check for the existence of malloc_default_purgeable_zone() at
458 * run time.
459 */
460 purgeable_zone = (malloc_default_purgeable_zone == NULL) ? NULL :
461 malloc_default_purgeable_zone();
462
463 /* Register the custom zone. At this point it won't be the default. */
464 zone_init();
465 malloc_zone_register(&jemalloc_zone);
466
467 /* Promote the custom zone to be default. */
468 zone_promote();
469}
generated by cgit v1.2.3 (git 2.46.0) at 2026-04-30 07:02:27 +0000