aboutsummaryrefslogtreecommitdiff
path: root/examples/redis-unstable/deps/jemalloc/src/ctl.c
diff options
context:
space:
mode:
Diffstat (limited to 'examples/redis-unstable/deps/jemalloc/src/ctl.c')
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/ctl.c4414
1 files changed, 4414 insertions, 0 deletions
diff --git a/examples/redis-unstable/deps/jemalloc/src/ctl.c b/examples/redis-unstable/deps/jemalloc/src/ctl.c
new file mode 100644
index 0000000..135271b
--- /dev/null
+++ b/examples/redis-unstable/deps/jemalloc/src/ctl.c
@@ -0,0 +1,4414 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/ctl.h"
6#include "jemalloc/internal/extent_dss.h"
7#include "jemalloc/internal/extent_mmap.h"
8#include "jemalloc/internal/inspect.h"
9#include "jemalloc/internal/mutex.h"
10#include "jemalloc/internal/nstime.h"
11#include "jemalloc/internal/peak_event.h"
12#include "jemalloc/internal/prof_data.h"
13#include "jemalloc/internal/prof_log.h"
14#include "jemalloc/internal/prof_recent.h"
15#include "jemalloc/internal/prof_stats.h"
16#include "jemalloc/internal/prof_sys.h"
17#include "jemalloc/internal/safety_check.h"
18#include "jemalloc/internal/sc.h"
19#include "jemalloc/internal/util.h"
20
21/******************************************************************************/
22/* Data. */
23
24/*
25 * ctl_mtx protects the following:
26 * - ctl_stats->*
27 */
28static malloc_mutex_t ctl_mtx;
29static bool ctl_initialized;
30static ctl_stats_t *ctl_stats;
31static ctl_arenas_t *ctl_arenas;
32
33/******************************************************************************/
34/* Helpers for named and indexed nodes. */
35
36static const ctl_named_node_t *
37ctl_named_node(const ctl_node_t *node) {
38 return ((node->named) ? (const ctl_named_node_t *)node : NULL);
39}
40
41static const ctl_named_node_t *
42ctl_named_children(const ctl_named_node_t *node, size_t index) {
43 const ctl_named_node_t *children = ctl_named_node(node->children);
44
45 return (children ? &children[index] : NULL);
46}
47
48static const ctl_indexed_node_t *
49ctl_indexed_node(const ctl_node_t *node) {
50 return (!node->named ? (const ctl_indexed_node_t *)node : NULL);
51}
52
53/******************************************************************************/
54/* Function prototypes for non-inline static functions. */
55
56#define CTL_PROTO(n) \
57static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
58 void *oldp, size_t *oldlenp, void *newp, size_t newlen);
59
60#define INDEX_PROTO(n) \
61static const ctl_named_node_t *n##_index(tsdn_t *tsdn, \
62 const size_t *mib, size_t miblen, size_t i);
63
64CTL_PROTO(version)
65CTL_PROTO(epoch)
66CTL_PROTO(background_thread)
67CTL_PROTO(max_background_threads)
68CTL_PROTO(thread_tcache_enabled)
69CTL_PROTO(thread_tcache_flush)
70CTL_PROTO(thread_peak_read)
71CTL_PROTO(thread_peak_reset)
72CTL_PROTO(thread_prof_name)
73CTL_PROTO(thread_prof_active)
74CTL_PROTO(thread_arena)
75CTL_PROTO(thread_allocated)
76CTL_PROTO(thread_allocatedp)
77CTL_PROTO(thread_deallocated)
78CTL_PROTO(thread_deallocatedp)
79CTL_PROTO(thread_idle)
80CTL_PROTO(config_cache_oblivious)
81CTL_PROTO(config_debug)
82CTL_PROTO(config_fill)
83CTL_PROTO(config_lazy_lock)
84CTL_PROTO(config_malloc_conf)
85CTL_PROTO(config_opt_safety_checks)
86CTL_PROTO(config_prof)
87CTL_PROTO(config_prof_libgcc)
88CTL_PROTO(config_prof_libunwind)
89CTL_PROTO(config_stats)
90CTL_PROTO(config_utrace)
91CTL_PROTO(config_xmalloc)
92CTL_PROTO(opt_abort)
93CTL_PROTO(opt_abort_conf)
94CTL_PROTO(opt_cache_oblivious)
95CTL_PROTO(opt_trust_madvise)
96CTL_PROTO(opt_confirm_conf)
97CTL_PROTO(opt_hpa)
98CTL_PROTO(opt_hpa_slab_max_alloc)
99CTL_PROTO(opt_hpa_hugification_threshold)
100CTL_PROTO(opt_hpa_hugify_delay_ms)
101CTL_PROTO(opt_hpa_min_purge_interval_ms)
102CTL_PROTO(opt_hpa_dirty_mult)
103CTL_PROTO(opt_hpa_sec_nshards)
104CTL_PROTO(opt_hpa_sec_max_alloc)
105CTL_PROTO(opt_hpa_sec_max_bytes)
106CTL_PROTO(opt_hpa_sec_bytes_after_flush)
107CTL_PROTO(opt_hpa_sec_batch_fill_extra)
108CTL_PROTO(opt_metadata_thp)
109CTL_PROTO(opt_retain)
110CTL_PROTO(opt_dss)
111CTL_PROTO(opt_narenas)
112CTL_PROTO(opt_percpu_arena)
113CTL_PROTO(opt_oversize_threshold)
114CTL_PROTO(opt_background_thread)
115CTL_PROTO(opt_mutex_max_spin)
116CTL_PROTO(opt_max_background_threads)
117CTL_PROTO(opt_dirty_decay_ms)
118CTL_PROTO(opt_muzzy_decay_ms)
119CTL_PROTO(opt_stats_print)
120CTL_PROTO(opt_stats_print_opts)
121CTL_PROTO(opt_stats_interval)
122CTL_PROTO(opt_stats_interval_opts)
123CTL_PROTO(opt_junk)
124CTL_PROTO(opt_zero)
125CTL_PROTO(opt_utrace)
126CTL_PROTO(opt_xmalloc)
127CTL_PROTO(opt_experimental_infallible_new)
128CTL_PROTO(opt_tcache)
129CTL_PROTO(opt_tcache_max)
130CTL_PROTO(opt_tcache_nslots_small_min)
131CTL_PROTO(opt_tcache_nslots_small_max)
132CTL_PROTO(opt_tcache_nslots_large)
133CTL_PROTO(opt_lg_tcache_nslots_mul)
134CTL_PROTO(opt_tcache_gc_incr_bytes)
135CTL_PROTO(opt_tcache_gc_delay_bytes)
136CTL_PROTO(opt_lg_tcache_flush_small_div)
137CTL_PROTO(opt_lg_tcache_flush_large_div)
138CTL_PROTO(opt_thp)
139CTL_PROTO(opt_lg_extent_max_active_fit)
140CTL_PROTO(opt_prof)
141CTL_PROTO(opt_prof_prefix)
142CTL_PROTO(opt_prof_active)
143CTL_PROTO(opt_prof_thread_active_init)
144CTL_PROTO(opt_lg_prof_sample)
145CTL_PROTO(opt_lg_prof_interval)
146CTL_PROTO(opt_prof_gdump)
147CTL_PROTO(opt_prof_final)
148CTL_PROTO(opt_prof_leak)
149CTL_PROTO(opt_prof_leak_error)
150CTL_PROTO(opt_prof_accum)
151CTL_PROTO(opt_prof_recent_alloc_max)
152CTL_PROTO(opt_prof_stats)
153CTL_PROTO(opt_prof_sys_thread_name)
154CTL_PROTO(opt_prof_time_res)
155CTL_PROTO(opt_lg_san_uaf_align)
156CTL_PROTO(opt_zero_realloc)
157CTL_PROTO(tcache_create)
158CTL_PROTO(tcache_flush)
159CTL_PROTO(tcache_destroy)
160CTL_PROTO(arena_i_initialized)
161CTL_PROTO(arena_i_decay)
162CTL_PROTO(arena_i_purge)
163CTL_PROTO(arena_i_reset)
164CTL_PROTO(arena_i_destroy)
165CTL_PROTO(arena_i_dss)
166CTL_PROTO(arena_i_oversize_threshold)
167CTL_PROTO(arena_i_dirty_decay_ms)
168CTL_PROTO(arena_i_muzzy_decay_ms)
169CTL_PROTO(arena_i_extent_hooks)
170CTL_PROTO(arena_i_retain_grow_limit)
171INDEX_PROTO(arena_i)
172CTL_PROTO(arenas_bin_i_size)
173CTL_PROTO(arenas_bin_i_nregs)
174CTL_PROTO(arenas_bin_i_slab_size)
175CTL_PROTO(arenas_bin_i_nshards)
176INDEX_PROTO(arenas_bin_i)
177CTL_PROTO(arenas_lextent_i_size)
178INDEX_PROTO(arenas_lextent_i)
179CTL_PROTO(arenas_narenas)
180CTL_PROTO(arenas_dirty_decay_ms)
181CTL_PROTO(arenas_muzzy_decay_ms)
182CTL_PROTO(arenas_quantum)
183CTL_PROTO(arenas_page)
184CTL_PROTO(arenas_tcache_max)
185CTL_PROTO(arenas_nbins)
186CTL_PROTO(arenas_nhbins)
187CTL_PROTO(arenas_nlextents)
188CTL_PROTO(arenas_create)
189CTL_PROTO(arenas_lookup)
190CTL_PROTO(prof_thread_active_init)
191CTL_PROTO(prof_active)
192CTL_PROTO(prof_dump)
193CTL_PROTO(prof_gdump)
194CTL_PROTO(prof_prefix)
195CTL_PROTO(prof_reset)
196CTL_PROTO(prof_interval)
197CTL_PROTO(lg_prof_sample)
198CTL_PROTO(prof_log_start)
199CTL_PROTO(prof_log_stop)
200CTL_PROTO(prof_stats_bins_i_live)
201CTL_PROTO(prof_stats_bins_i_accum)
202INDEX_PROTO(prof_stats_bins_i)
203CTL_PROTO(prof_stats_lextents_i_live)
204CTL_PROTO(prof_stats_lextents_i_accum)
205INDEX_PROTO(prof_stats_lextents_i)
206CTL_PROTO(stats_arenas_i_small_allocated)
207CTL_PROTO(stats_arenas_i_small_nmalloc)
208CTL_PROTO(stats_arenas_i_small_ndalloc)
209CTL_PROTO(stats_arenas_i_small_nrequests)
210CTL_PROTO(stats_arenas_i_small_nfills)
211CTL_PROTO(stats_arenas_i_small_nflushes)
212CTL_PROTO(stats_arenas_i_large_allocated)
213CTL_PROTO(stats_arenas_i_large_nmalloc)
214CTL_PROTO(stats_arenas_i_large_ndalloc)
215CTL_PROTO(stats_arenas_i_large_nrequests)
216CTL_PROTO(stats_arenas_i_large_nfills)
217CTL_PROTO(stats_arenas_i_large_nflushes)
218CTL_PROTO(stats_arenas_i_bins_j_nmalloc)
219CTL_PROTO(stats_arenas_i_bins_j_ndalloc)
220CTL_PROTO(stats_arenas_i_bins_j_nrequests)
221CTL_PROTO(stats_arenas_i_bins_j_curregs)
222CTL_PROTO(stats_arenas_i_bins_j_nfills)
223CTL_PROTO(stats_arenas_i_bins_j_nflushes)
224CTL_PROTO(stats_arenas_i_bins_j_nslabs)
225CTL_PROTO(stats_arenas_i_bins_j_nreslabs)
226CTL_PROTO(stats_arenas_i_bins_j_curslabs)
227CTL_PROTO(stats_arenas_i_bins_j_nonfull_slabs)
228INDEX_PROTO(stats_arenas_i_bins_j)
229CTL_PROTO(stats_arenas_i_lextents_j_nmalloc)
230CTL_PROTO(stats_arenas_i_lextents_j_ndalloc)
231CTL_PROTO(stats_arenas_i_lextents_j_nrequests)
232CTL_PROTO(stats_arenas_i_lextents_j_curlextents)
233INDEX_PROTO(stats_arenas_i_lextents_j)
234CTL_PROTO(stats_arenas_i_extents_j_ndirty)
235CTL_PROTO(stats_arenas_i_extents_j_nmuzzy)
236CTL_PROTO(stats_arenas_i_extents_j_nretained)
237CTL_PROTO(stats_arenas_i_extents_j_dirty_bytes)
238CTL_PROTO(stats_arenas_i_extents_j_muzzy_bytes)
239CTL_PROTO(stats_arenas_i_extents_j_retained_bytes)
240INDEX_PROTO(stats_arenas_i_extents_j)
241CTL_PROTO(stats_arenas_i_hpa_shard_npurge_passes)
242CTL_PROTO(stats_arenas_i_hpa_shard_npurges)
243CTL_PROTO(stats_arenas_i_hpa_shard_nhugifies)
244CTL_PROTO(stats_arenas_i_hpa_shard_ndehugifies)
245
246/* We have a set of stats for full slabs. */
247CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge)
248CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge)
249CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge)
250CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_nactive_huge)
251CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_ndirty_nonhuge)
252CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_ndirty_huge)
253
254/* A parallel set for the empty slabs. */
255CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_nonhuge)
256CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_huge)
257CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_nactive_nonhuge)
258CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_nactive_huge)
259CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_ndirty_nonhuge)
260CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_ndirty_huge)
261
262/*
263 * And one for the slabs that are neither empty nor full, but indexed by how
264 * full they are.
265 */
266CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge)
267CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge)
268CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge)
269CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge)
270CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge)
271CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge)
272
273INDEX_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j)
274CTL_PROTO(stats_arenas_i_nthreads)
275CTL_PROTO(stats_arenas_i_uptime)
276CTL_PROTO(stats_arenas_i_dss)
277CTL_PROTO(stats_arenas_i_dirty_decay_ms)
278CTL_PROTO(stats_arenas_i_muzzy_decay_ms)
279CTL_PROTO(stats_arenas_i_pactive)
280CTL_PROTO(stats_arenas_i_pdirty)
281CTL_PROTO(stats_arenas_i_pmuzzy)
282CTL_PROTO(stats_arenas_i_mapped)
283CTL_PROTO(stats_arenas_i_retained)
284CTL_PROTO(stats_arenas_i_extent_avail)
285CTL_PROTO(stats_arenas_i_dirty_npurge)
286CTL_PROTO(stats_arenas_i_dirty_nmadvise)
287CTL_PROTO(stats_arenas_i_dirty_purged)
288CTL_PROTO(stats_arenas_i_muzzy_npurge)
289CTL_PROTO(stats_arenas_i_muzzy_nmadvise)
290CTL_PROTO(stats_arenas_i_muzzy_purged)
291CTL_PROTO(stats_arenas_i_base)
292CTL_PROTO(stats_arenas_i_internal)
293CTL_PROTO(stats_arenas_i_metadata_thp)
294CTL_PROTO(stats_arenas_i_tcache_bytes)
295CTL_PROTO(stats_arenas_i_tcache_stashed_bytes)
296CTL_PROTO(stats_arenas_i_resident)
297CTL_PROTO(stats_arenas_i_abandoned_vm)
298CTL_PROTO(stats_arenas_i_hpa_sec_bytes)
299INDEX_PROTO(stats_arenas_i)
300CTL_PROTO(stats_allocated)
301CTL_PROTO(stats_active)
302CTL_PROTO(stats_background_thread_num_threads)
303CTL_PROTO(stats_background_thread_num_runs)
304CTL_PROTO(stats_background_thread_run_interval)
305CTL_PROTO(stats_metadata)
306CTL_PROTO(stats_metadata_thp)
307CTL_PROTO(stats_resident)
308CTL_PROTO(stats_mapped)
309CTL_PROTO(stats_retained)
310CTL_PROTO(stats_zero_reallocs)
311CTL_PROTO(experimental_hooks_install)
312CTL_PROTO(experimental_hooks_remove)
313CTL_PROTO(experimental_hooks_prof_backtrace)
314CTL_PROTO(experimental_hooks_prof_dump)
315CTL_PROTO(experimental_hooks_safety_check_abort)
316CTL_PROTO(experimental_thread_activity_callback)
317CTL_PROTO(experimental_utilization_query)
318CTL_PROTO(experimental_utilization_batch_query)
319CTL_PROTO(experimental_arenas_i_pactivep)
320INDEX_PROTO(experimental_arenas_i)
321CTL_PROTO(experimental_prof_recent_alloc_max)
322CTL_PROTO(experimental_prof_recent_alloc_dump)
323CTL_PROTO(experimental_batch_alloc)
324CTL_PROTO(experimental_arenas_create_ext)
325
326#define MUTEX_STATS_CTL_PROTO_GEN(n) \
327CTL_PROTO(stats_##n##_num_ops) \
328CTL_PROTO(stats_##n##_num_wait) \
329CTL_PROTO(stats_##n##_num_spin_acq) \
330CTL_PROTO(stats_##n##_num_owner_switch) \
331CTL_PROTO(stats_##n##_total_wait_time) \
332CTL_PROTO(stats_##n##_max_wait_time) \
333CTL_PROTO(stats_##n##_max_num_thds)
334
335/* Global mutexes. */
336#define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(mutexes_##mtx)
337MUTEX_PROF_GLOBAL_MUTEXES
338#undef OP
339
340/* Per arena mutexes. */
341#define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(arenas_i_mutexes_##mtx)
342MUTEX_PROF_ARENA_MUTEXES
343#undef OP
344
345/* Arena bin mutexes. */
346MUTEX_STATS_CTL_PROTO_GEN(arenas_i_bins_j_mutex)
347#undef MUTEX_STATS_CTL_PROTO_GEN
348
349CTL_PROTO(stats_mutexes_reset)
350
351/******************************************************************************/
352/* mallctl tree. */
353
354#define NAME(n) {true}, n
355#define CHILD(t, c) \
356 sizeof(c##_node) / sizeof(ctl_##t##_node_t), \
357 (ctl_node_t *)c##_node, \
358 NULL
359#define CTL(c) 0, NULL, c##_ctl
360
361/*
362 * Only handles internal indexed nodes, since there are currently no external
363 * ones.
364 */
365#define INDEX(i) {false}, i##_index
366
367static const ctl_named_node_t thread_tcache_node[] = {
368 {NAME("enabled"), CTL(thread_tcache_enabled)},
369 {NAME("flush"), CTL(thread_tcache_flush)}
370};
371
372static const ctl_named_node_t thread_peak_node[] = {
373 {NAME("read"), CTL(thread_peak_read)},
374 {NAME("reset"), CTL(thread_peak_reset)},
375};
376
377static const ctl_named_node_t thread_prof_node[] = {
378 {NAME("name"), CTL(thread_prof_name)},
379 {NAME("active"), CTL(thread_prof_active)}
380};
381
382static const ctl_named_node_t thread_node[] = {
383 {NAME("arena"), CTL(thread_arena)},
384 {NAME("allocated"), CTL(thread_allocated)},
385 {NAME("allocatedp"), CTL(thread_allocatedp)},
386 {NAME("deallocated"), CTL(thread_deallocated)},
387 {NAME("deallocatedp"), CTL(thread_deallocatedp)},
388 {NAME("tcache"), CHILD(named, thread_tcache)},
389 {NAME("peak"), CHILD(named, thread_peak)},
390 {NAME("prof"), CHILD(named, thread_prof)},
391 {NAME("idle"), CTL(thread_idle)}
392};
393
394static const ctl_named_node_t config_node[] = {
395 {NAME("cache_oblivious"), CTL(config_cache_oblivious)},
396 {NAME("debug"), CTL(config_debug)},
397 {NAME("fill"), CTL(config_fill)},
398 {NAME("lazy_lock"), CTL(config_lazy_lock)},
399 {NAME("malloc_conf"), CTL(config_malloc_conf)},
400 {NAME("opt_safety_checks"), CTL(config_opt_safety_checks)},
401 {NAME("prof"), CTL(config_prof)},
402 {NAME("prof_libgcc"), CTL(config_prof_libgcc)},
403 {NAME("prof_libunwind"), CTL(config_prof_libunwind)},
404 {NAME("stats"), CTL(config_stats)},
405 {NAME("utrace"), CTL(config_utrace)},
406 {NAME("xmalloc"), CTL(config_xmalloc)}
407};
408
409static const ctl_named_node_t opt_node[] = {
410 {NAME("abort"), CTL(opt_abort)},
411 {NAME("abort_conf"), CTL(opt_abort_conf)},
412 {NAME("cache_oblivious"), CTL(opt_cache_oblivious)},
413 {NAME("trust_madvise"), CTL(opt_trust_madvise)},
414 {NAME("confirm_conf"), CTL(opt_confirm_conf)},
415 {NAME("hpa"), CTL(opt_hpa)},
416 {NAME("hpa_slab_max_alloc"), CTL(opt_hpa_slab_max_alloc)},
417 {NAME("hpa_hugification_threshold"),
418 CTL(opt_hpa_hugification_threshold)},
419 {NAME("hpa_hugify_delay_ms"), CTL(opt_hpa_hugify_delay_ms)},
420 {NAME("hpa_min_purge_interval_ms"), CTL(opt_hpa_min_purge_interval_ms)},
421 {NAME("hpa_dirty_mult"), CTL(opt_hpa_dirty_mult)},
422 {NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)},
423 {NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)},
424 {NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)},
425 {NAME("hpa_sec_bytes_after_flush"),
426 CTL(opt_hpa_sec_bytes_after_flush)},
427 {NAME("hpa_sec_batch_fill_extra"),
428 CTL(opt_hpa_sec_batch_fill_extra)},
429 {NAME("metadata_thp"), CTL(opt_metadata_thp)},
430 {NAME("retain"), CTL(opt_retain)},
431 {NAME("dss"), CTL(opt_dss)},
432 {NAME("narenas"), CTL(opt_narenas)},
433 {NAME("percpu_arena"), CTL(opt_percpu_arena)},
434 {NAME("oversize_threshold"), CTL(opt_oversize_threshold)},
435 {NAME("mutex_max_spin"), CTL(opt_mutex_max_spin)},
436 {NAME("background_thread"), CTL(opt_background_thread)},
437 {NAME("max_background_threads"), CTL(opt_max_background_threads)},
438 {NAME("dirty_decay_ms"), CTL(opt_dirty_decay_ms)},
439 {NAME("muzzy_decay_ms"), CTL(opt_muzzy_decay_ms)},
440 {NAME("stats_print"), CTL(opt_stats_print)},
441 {NAME("stats_print_opts"), CTL(opt_stats_print_opts)},
442 {NAME("stats_interval"), CTL(opt_stats_interval)},
443 {NAME("stats_interval_opts"), CTL(opt_stats_interval_opts)},
444 {NAME("junk"), CTL(opt_junk)},
445 {NAME("zero"), CTL(opt_zero)},
446 {NAME("utrace"), CTL(opt_utrace)},
447 {NAME("xmalloc"), CTL(opt_xmalloc)},
448 {NAME("experimental_infallible_new"),
449 CTL(opt_experimental_infallible_new)},
450 {NAME("tcache"), CTL(opt_tcache)},
451 {NAME("tcache_max"), CTL(opt_tcache_max)},
452 {NAME("tcache_nslots_small_min"),
453 CTL(opt_tcache_nslots_small_min)},
454 {NAME("tcache_nslots_small_max"),
455 CTL(opt_tcache_nslots_small_max)},
456 {NAME("tcache_nslots_large"), CTL(opt_tcache_nslots_large)},
457 {NAME("lg_tcache_nslots_mul"), CTL(opt_lg_tcache_nslots_mul)},
458 {NAME("tcache_gc_incr_bytes"), CTL(opt_tcache_gc_incr_bytes)},
459 {NAME("tcache_gc_delay_bytes"), CTL(opt_tcache_gc_delay_bytes)},
460 {NAME("lg_tcache_flush_small_div"),
461 CTL(opt_lg_tcache_flush_small_div)},
462 {NAME("lg_tcache_flush_large_div"),
463 CTL(opt_lg_tcache_flush_large_div)},
464 {NAME("thp"), CTL(opt_thp)},
465 {NAME("lg_extent_max_active_fit"), CTL(opt_lg_extent_max_active_fit)},
466 {NAME("prof"), CTL(opt_prof)},
467 {NAME("prof_prefix"), CTL(opt_prof_prefix)},
468 {NAME("prof_active"), CTL(opt_prof_active)},
469 {NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)},
470 {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)},
471 {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)},
472 {NAME("prof_gdump"), CTL(opt_prof_gdump)},
473 {NAME("prof_final"), CTL(opt_prof_final)},
474 {NAME("prof_leak"), CTL(opt_prof_leak)},
475 {NAME("prof_leak_error"), CTL(opt_prof_leak_error)},
476 {NAME("prof_accum"), CTL(opt_prof_accum)},
477 {NAME("prof_recent_alloc_max"), CTL(opt_prof_recent_alloc_max)},
478 {NAME("prof_stats"), CTL(opt_prof_stats)},
479 {NAME("prof_sys_thread_name"), CTL(opt_prof_sys_thread_name)},
480 {NAME("prof_time_resolution"), CTL(opt_prof_time_res)},
481 {NAME("lg_san_uaf_align"), CTL(opt_lg_san_uaf_align)},
482 {NAME("zero_realloc"), CTL(opt_zero_realloc)}
483};
484
485static const ctl_named_node_t tcache_node[] = {
486 {NAME("create"), CTL(tcache_create)},
487 {NAME("flush"), CTL(tcache_flush)},
488 {NAME("destroy"), CTL(tcache_destroy)}
489};
490
491static const ctl_named_node_t arena_i_node[] = {
492 {NAME("initialized"), CTL(arena_i_initialized)},
493 {NAME("decay"), CTL(arena_i_decay)},
494 {NAME("purge"), CTL(arena_i_purge)},
495 {NAME("reset"), CTL(arena_i_reset)},
496 {NAME("destroy"), CTL(arena_i_destroy)},
497 {NAME("dss"), CTL(arena_i_dss)},
498 /*
499 * Undocumented for now, since we anticipate an arena API in flux after
500 * we cut the last 5-series release.
501 */
502 {NAME("oversize_threshold"), CTL(arena_i_oversize_threshold)},
503 {NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)},
504 {NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)},
505 {NAME("extent_hooks"), CTL(arena_i_extent_hooks)},
506 {NAME("retain_grow_limit"), CTL(arena_i_retain_grow_limit)}
507};
508static const ctl_named_node_t super_arena_i_node[] = {
509 {NAME(""), CHILD(named, arena_i)}
510};
511
512static const ctl_indexed_node_t arena_node[] = {
513 {INDEX(arena_i)}
514};
515
516static const ctl_named_node_t arenas_bin_i_node[] = {
517 {NAME("size"), CTL(arenas_bin_i_size)},
518 {NAME("nregs"), CTL(arenas_bin_i_nregs)},
519 {NAME("slab_size"), CTL(arenas_bin_i_slab_size)},
520 {NAME("nshards"), CTL(arenas_bin_i_nshards)}
521};
522static const ctl_named_node_t super_arenas_bin_i_node[] = {
523 {NAME(""), CHILD(named, arenas_bin_i)}
524};
525
526static const ctl_indexed_node_t arenas_bin_node[] = {
527 {INDEX(arenas_bin_i)}
528};
529
530static const ctl_named_node_t arenas_lextent_i_node[] = {
531 {NAME("size"), CTL(arenas_lextent_i_size)}
532};
533static const ctl_named_node_t super_arenas_lextent_i_node[] = {
534 {NAME(""), CHILD(named, arenas_lextent_i)}
535};
536
537static const ctl_indexed_node_t arenas_lextent_node[] = {
538 {INDEX(arenas_lextent_i)}
539};
540
541static const ctl_named_node_t arenas_node[] = {
542 {NAME("narenas"), CTL(arenas_narenas)},
543 {NAME("dirty_decay_ms"), CTL(arenas_dirty_decay_ms)},
544 {NAME("muzzy_decay_ms"), CTL(arenas_muzzy_decay_ms)},
545 {NAME("quantum"), CTL(arenas_quantum)},
546 {NAME("page"), CTL(arenas_page)},
547 {NAME("tcache_max"), CTL(arenas_tcache_max)},
548 {NAME("nbins"), CTL(arenas_nbins)},
549 {NAME("nhbins"), CTL(arenas_nhbins)},
550 {NAME("bin"), CHILD(indexed, arenas_bin)},
551 {NAME("nlextents"), CTL(arenas_nlextents)},
552 {NAME("lextent"), CHILD(indexed, arenas_lextent)},
553 {NAME("create"), CTL(arenas_create)},
554 {NAME("lookup"), CTL(arenas_lookup)}
555};
556
557static const ctl_named_node_t prof_stats_bins_i_node[] = {
558 {NAME("live"), CTL(prof_stats_bins_i_live)},
559 {NAME("accum"), CTL(prof_stats_bins_i_accum)}
560};
561
562static const ctl_named_node_t super_prof_stats_bins_i_node[] = {
563 {NAME(""), CHILD(named, prof_stats_bins_i)}
564};
565
566static const ctl_indexed_node_t prof_stats_bins_node[] = {
567 {INDEX(prof_stats_bins_i)}
568};
569
570static const ctl_named_node_t prof_stats_lextents_i_node[] = {
571 {NAME("live"), CTL(prof_stats_lextents_i_live)},
572 {NAME("accum"), CTL(prof_stats_lextents_i_accum)}
573};
574
575static const ctl_named_node_t super_prof_stats_lextents_i_node[] = {
576 {NAME(""), CHILD(named, prof_stats_lextents_i)}
577};
578
579static const ctl_indexed_node_t prof_stats_lextents_node[] = {
580 {INDEX(prof_stats_lextents_i)}
581};
582
583static const ctl_named_node_t prof_stats_node[] = {
584 {NAME("bins"), CHILD(indexed, prof_stats_bins)},
585 {NAME("lextents"), CHILD(indexed, prof_stats_lextents)},
586};
587
588static const ctl_named_node_t prof_node[] = {
589 {NAME("thread_active_init"), CTL(prof_thread_active_init)},
590 {NAME("active"), CTL(prof_active)},
591 {NAME("dump"), CTL(prof_dump)},
592 {NAME("gdump"), CTL(prof_gdump)},
593 {NAME("prefix"), CTL(prof_prefix)},
594 {NAME("reset"), CTL(prof_reset)},
595 {NAME("interval"), CTL(prof_interval)},
596 {NAME("lg_sample"), CTL(lg_prof_sample)},
597 {NAME("log_start"), CTL(prof_log_start)},
598 {NAME("log_stop"), CTL(prof_log_stop)},
599 {NAME("stats"), CHILD(named, prof_stats)}
600};
601
602static const ctl_named_node_t stats_arenas_i_small_node[] = {
603 {NAME("allocated"), CTL(stats_arenas_i_small_allocated)},
604 {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)},
605 {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)},
606 {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)},
607 {NAME("nfills"), CTL(stats_arenas_i_small_nfills)},
608 {NAME("nflushes"), CTL(stats_arenas_i_small_nflushes)}
609};
610
611static const ctl_named_node_t stats_arenas_i_large_node[] = {
612 {NAME("allocated"), CTL(stats_arenas_i_large_allocated)},
613 {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)},
614 {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)},
615 {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)},
616 {NAME("nfills"), CTL(stats_arenas_i_large_nfills)},
617 {NAME("nflushes"), CTL(stats_arenas_i_large_nflushes)}
618};
619
620#define MUTEX_PROF_DATA_NODE(prefix) \
621static const ctl_named_node_t stats_##prefix##_node[] = { \
622 {NAME("num_ops"), \
623 CTL(stats_##prefix##_num_ops)}, \
624 {NAME("num_wait"), \
625 CTL(stats_##prefix##_num_wait)}, \
626 {NAME("num_spin_acq"), \
627 CTL(stats_##prefix##_num_spin_acq)}, \
628 {NAME("num_owner_switch"), \
629 CTL(stats_##prefix##_num_owner_switch)}, \
630 {NAME("total_wait_time"), \
631 CTL(stats_##prefix##_total_wait_time)}, \
632 {NAME("max_wait_time"), \
633 CTL(stats_##prefix##_max_wait_time)}, \
634 {NAME("max_num_thds"), \
635 CTL(stats_##prefix##_max_num_thds)} \
636 /* Note that # of current waiting thread not provided. */ \
637};
638
639MUTEX_PROF_DATA_NODE(arenas_i_bins_j_mutex)
640
641static const ctl_named_node_t stats_arenas_i_bins_j_node[] = {
642 {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)},
643 {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)},
644 {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)},
645 {NAME("curregs"), CTL(stats_arenas_i_bins_j_curregs)},
646 {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)},
647 {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)},
648 {NAME("nslabs"), CTL(stats_arenas_i_bins_j_nslabs)},
649 {NAME("nreslabs"), CTL(stats_arenas_i_bins_j_nreslabs)},
650 {NAME("curslabs"), CTL(stats_arenas_i_bins_j_curslabs)},
651 {NAME("nonfull_slabs"), CTL(stats_arenas_i_bins_j_nonfull_slabs)},
652 {NAME("mutex"), CHILD(named, stats_arenas_i_bins_j_mutex)}
653};
654
655static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = {
656 {NAME(""), CHILD(named, stats_arenas_i_bins_j)}
657};
658
659static const ctl_indexed_node_t stats_arenas_i_bins_node[] = {
660 {INDEX(stats_arenas_i_bins_j)}
661};
662
663static const ctl_named_node_t stats_arenas_i_lextents_j_node[] = {
664 {NAME("nmalloc"), CTL(stats_arenas_i_lextents_j_nmalloc)},
665 {NAME("ndalloc"), CTL(stats_arenas_i_lextents_j_ndalloc)},
666 {NAME("nrequests"), CTL(stats_arenas_i_lextents_j_nrequests)},
667 {NAME("curlextents"), CTL(stats_arenas_i_lextents_j_curlextents)}
668};
669static const ctl_named_node_t super_stats_arenas_i_lextents_j_node[] = {
670 {NAME(""), CHILD(named, stats_arenas_i_lextents_j)}
671};
672
673static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = {
674 {INDEX(stats_arenas_i_lextents_j)}
675};
676
677static const ctl_named_node_t stats_arenas_i_extents_j_node[] = {
678 {NAME("ndirty"), CTL(stats_arenas_i_extents_j_ndirty)},
679 {NAME("nmuzzy"), CTL(stats_arenas_i_extents_j_nmuzzy)},
680 {NAME("nretained"), CTL(stats_arenas_i_extents_j_nretained)},
681 {NAME("dirty_bytes"), CTL(stats_arenas_i_extents_j_dirty_bytes)},
682 {NAME("muzzy_bytes"), CTL(stats_arenas_i_extents_j_muzzy_bytes)},
683 {NAME("retained_bytes"), CTL(stats_arenas_i_extents_j_retained_bytes)}
684};
685
686static const ctl_named_node_t super_stats_arenas_i_extents_j_node[] = {
687 {NAME(""), CHILD(named, stats_arenas_i_extents_j)}
688};
689
690static const ctl_indexed_node_t stats_arenas_i_extents_node[] = {
691 {INDEX(stats_arenas_i_extents_j)}
692};
693
694#define OP(mtx) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx)
695MUTEX_PROF_ARENA_MUTEXES
696#undef OP
697
698static const ctl_named_node_t stats_arenas_i_mutexes_node[] = {
699#define OP(mtx) {NAME(#mtx), CHILD(named, stats_arenas_i_mutexes_##mtx)},
700MUTEX_PROF_ARENA_MUTEXES
701#undef OP
702};
703
704static const ctl_named_node_t stats_arenas_i_hpa_shard_full_slabs_node[] = {
705 {NAME("npageslabs_nonhuge"),
706 CTL(stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge)},
707 {NAME("npageslabs_huge"),
708 CTL(stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge)},
709 {NAME("nactive_nonhuge"),
710 CTL(stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge)},
711 {NAME("nactive_huge"),
712 CTL(stats_arenas_i_hpa_shard_full_slabs_nactive_huge)},
713 {NAME("ndirty_nonhuge"),
714 CTL(stats_arenas_i_hpa_shard_full_slabs_ndirty_nonhuge)},
715 {NAME("ndirty_huge"),
716 CTL(stats_arenas_i_hpa_shard_full_slabs_ndirty_huge)}
717};
718
719static const ctl_named_node_t stats_arenas_i_hpa_shard_empty_slabs_node[] = {
720 {NAME("npageslabs_nonhuge"),
721 CTL(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_nonhuge)},
722 {NAME("npageslabs_huge"),
723 CTL(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_huge)},
724 {NAME("nactive_nonhuge"),
725 CTL(stats_arenas_i_hpa_shard_empty_slabs_nactive_nonhuge)},
726 {NAME("nactive_huge"),
727 CTL(stats_arenas_i_hpa_shard_empty_slabs_nactive_huge)},
728 {NAME("ndirty_nonhuge"),
729 CTL(stats_arenas_i_hpa_shard_empty_slabs_ndirty_nonhuge)},
730 {NAME("ndirty_huge"),
731 CTL(stats_arenas_i_hpa_shard_empty_slabs_ndirty_huge)}
732};
733
734static const ctl_named_node_t stats_arenas_i_hpa_shard_nonfull_slabs_j_node[] = {
735 {NAME("npageslabs_nonhuge"),
736 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge)},
737 {NAME("npageslabs_huge"),
738 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge)},
739 {NAME("nactive_nonhuge"),
740 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge)},
741 {NAME("nactive_huge"),
742 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge)},
743 {NAME("ndirty_nonhuge"),
744 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge)},
745 {NAME("ndirty_huge"),
746 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge)}
747};
748
749static const ctl_named_node_t super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node[] = {
750 {NAME(""),
751 CHILD(named, stats_arenas_i_hpa_shard_nonfull_slabs_j)}
752};
753
754static const ctl_indexed_node_t stats_arenas_i_hpa_shard_nonfull_slabs_node[] =
755{
756 {INDEX(stats_arenas_i_hpa_shard_nonfull_slabs_j)}
757};
758
759static const ctl_named_node_t stats_arenas_i_hpa_shard_node[] = {
760 {NAME("full_slabs"), CHILD(named,
761 stats_arenas_i_hpa_shard_full_slabs)},
762 {NAME("empty_slabs"), CHILD(named,
763 stats_arenas_i_hpa_shard_empty_slabs)},
764 {NAME("nonfull_slabs"), CHILD(indexed,
765 stats_arenas_i_hpa_shard_nonfull_slabs)},
766
767 {NAME("npurge_passes"), CTL(stats_arenas_i_hpa_shard_npurge_passes)},
768 {NAME("npurges"), CTL(stats_arenas_i_hpa_shard_npurges)},
769 {NAME("nhugifies"), CTL(stats_arenas_i_hpa_shard_nhugifies)},
770 {NAME("ndehugifies"), CTL(stats_arenas_i_hpa_shard_ndehugifies)}
771};
772
773static const ctl_named_node_t stats_arenas_i_node[] = {
774 {NAME("nthreads"), CTL(stats_arenas_i_nthreads)},
775 {NAME("uptime"), CTL(stats_arenas_i_uptime)},
776 {NAME("dss"), CTL(stats_arenas_i_dss)},
777 {NAME("dirty_decay_ms"), CTL(stats_arenas_i_dirty_decay_ms)},
778 {NAME("muzzy_decay_ms"), CTL(stats_arenas_i_muzzy_decay_ms)},
779 {NAME("pactive"), CTL(stats_arenas_i_pactive)},
780 {NAME("pdirty"), CTL(stats_arenas_i_pdirty)},
781 {NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)},
782 {NAME("mapped"), CTL(stats_arenas_i_mapped)},
783 {NAME("retained"), CTL(stats_arenas_i_retained)},
784 {NAME("extent_avail"), CTL(stats_arenas_i_extent_avail)},
785 {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)},
786 {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)},
787 {NAME("dirty_purged"), CTL(stats_arenas_i_dirty_purged)},
788 {NAME("muzzy_npurge"), CTL(stats_arenas_i_muzzy_npurge)},
789 {NAME("muzzy_nmadvise"), CTL(stats_arenas_i_muzzy_nmadvise)},
790 {NAME("muzzy_purged"), CTL(stats_arenas_i_muzzy_purged)},
791 {NAME("base"), CTL(stats_arenas_i_base)},
792 {NAME("internal"), CTL(stats_arenas_i_internal)},
793 {NAME("metadata_thp"), CTL(stats_arenas_i_metadata_thp)},
794 {NAME("tcache_bytes"), CTL(stats_arenas_i_tcache_bytes)},
795 {NAME("tcache_stashed_bytes"),
796 CTL(stats_arenas_i_tcache_stashed_bytes)},
797 {NAME("resident"), CTL(stats_arenas_i_resident)},
798 {NAME("abandoned_vm"), CTL(stats_arenas_i_abandoned_vm)},
799 {NAME("hpa_sec_bytes"), CTL(stats_arenas_i_hpa_sec_bytes)},
800 {NAME("small"), CHILD(named, stats_arenas_i_small)},
801 {NAME("large"), CHILD(named, stats_arenas_i_large)},
802 {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)},
803 {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)},
804 {NAME("extents"), CHILD(indexed, stats_arenas_i_extents)},
805 {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)},
806 {NAME("hpa_shard"), CHILD(named, stats_arenas_i_hpa_shard)}
807};
808static const ctl_named_node_t super_stats_arenas_i_node[] = {
809 {NAME(""), CHILD(named, stats_arenas_i)}
810};
811
812static const ctl_indexed_node_t stats_arenas_node[] = {
813 {INDEX(stats_arenas_i)}
814};
815
816static const ctl_named_node_t stats_background_thread_node[] = {
817 {NAME("num_threads"), CTL(stats_background_thread_num_threads)},
818 {NAME("num_runs"), CTL(stats_background_thread_num_runs)},
819 {NAME("run_interval"), CTL(stats_background_thread_run_interval)}
820};
821
822#define OP(mtx) MUTEX_PROF_DATA_NODE(mutexes_##mtx)
823MUTEX_PROF_GLOBAL_MUTEXES
824#undef OP
825
826static const ctl_named_node_t stats_mutexes_node[] = {
827#define OP(mtx) {NAME(#mtx), CHILD(named, stats_mutexes_##mtx)},
828MUTEX_PROF_GLOBAL_MUTEXES
829#undef OP
830 {NAME("reset"), CTL(stats_mutexes_reset)}
831};
832#undef MUTEX_PROF_DATA_NODE
833
834static const ctl_named_node_t stats_node[] = {
835 {NAME("allocated"), CTL(stats_allocated)},
836 {NAME("active"), CTL(stats_active)},
837 {NAME("metadata"), CTL(stats_metadata)},
838 {NAME("metadata_thp"), CTL(stats_metadata_thp)},
839 {NAME("resident"), CTL(stats_resident)},
840 {NAME("mapped"), CTL(stats_mapped)},
841 {NAME("retained"), CTL(stats_retained)},
842 {NAME("background_thread"),
843 CHILD(named, stats_background_thread)},
844 {NAME("mutexes"), CHILD(named, stats_mutexes)},
845 {NAME("arenas"), CHILD(indexed, stats_arenas)},
846 {NAME("zero_reallocs"), CTL(stats_zero_reallocs)},
847};
848
849static const ctl_named_node_t experimental_hooks_node[] = {
850 {NAME("install"), CTL(experimental_hooks_install)},
851 {NAME("remove"), CTL(experimental_hooks_remove)},
852 {NAME("prof_backtrace"), CTL(experimental_hooks_prof_backtrace)},
853 {NAME("prof_dump"), CTL(experimental_hooks_prof_dump)},
854 {NAME("safety_check_abort"), CTL(experimental_hooks_safety_check_abort)},
855};
856
857static const ctl_named_node_t experimental_thread_node[] = {
858 {NAME("activity_callback"),
859 CTL(experimental_thread_activity_callback)}
860};
861
862static const ctl_named_node_t experimental_utilization_node[] = {
863 {NAME("query"), CTL(experimental_utilization_query)},
864 {NAME("batch_query"), CTL(experimental_utilization_batch_query)}
865};
866
867static const ctl_named_node_t experimental_arenas_i_node[] = {
868 {NAME("pactivep"), CTL(experimental_arenas_i_pactivep)}
869};
870static const ctl_named_node_t super_experimental_arenas_i_node[] = {
871 {NAME(""), CHILD(named, experimental_arenas_i)}
872};
873
874static const ctl_indexed_node_t experimental_arenas_node[] = {
875 {INDEX(experimental_arenas_i)}
876};
877
878static const ctl_named_node_t experimental_prof_recent_node[] = {
879 {NAME("alloc_max"), CTL(experimental_prof_recent_alloc_max)},
880 {NAME("alloc_dump"), CTL(experimental_prof_recent_alloc_dump)},
881};
882
883static const ctl_named_node_t experimental_node[] = {
884 {NAME("hooks"), CHILD(named, experimental_hooks)},
885 {NAME("utilization"), CHILD(named, experimental_utilization)},
886 {NAME("arenas"), CHILD(indexed, experimental_arenas)},
887 {NAME("arenas_create_ext"), CTL(experimental_arenas_create_ext)},
888 {NAME("prof_recent"), CHILD(named, experimental_prof_recent)},
889 {NAME("batch_alloc"), CTL(experimental_batch_alloc)},
890 {NAME("thread"), CHILD(named, experimental_thread)}
891};
892
893static const ctl_named_node_t root_node[] = {
894 {NAME("version"), CTL(version)},
895 {NAME("epoch"), CTL(epoch)},
896 {NAME("background_thread"), CTL(background_thread)},
897 {NAME("max_background_threads"), CTL(max_background_threads)},
898 {NAME("thread"), CHILD(named, thread)},
899 {NAME("config"), CHILD(named, config)},
900 {NAME("opt"), CHILD(named, opt)},
901 {NAME("tcache"), CHILD(named, tcache)},
902 {NAME("arena"), CHILD(indexed, arena)},
903 {NAME("arenas"), CHILD(named, arenas)},
904 {NAME("prof"), CHILD(named, prof)},
905 {NAME("stats"), CHILD(named, stats)},
906 {NAME("experimental"), CHILD(named, experimental)}
907};
908static const ctl_named_node_t super_root_node[] = {
909 {NAME(""), CHILD(named, root)}
910};
911
912#undef NAME
913#undef CHILD
914#undef CTL
915#undef INDEX
916
917/******************************************************************************/
918
919/*
920 * Sets *dst + *src non-atomically. This is safe, since everything is
921 * synchronized by the ctl mutex.
922 */
923static void
924ctl_accum_locked_u64(locked_u64_t *dst, locked_u64_t *src) {
925 locked_inc_u64_unsynchronized(dst,
926 locked_read_u64_unsynchronized(src));
927}
928
929static void
930ctl_accum_atomic_zu(atomic_zu_t *dst, atomic_zu_t *src) {
931 size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED);
932 size_t cur_src = atomic_load_zu(src, ATOMIC_RELAXED);
933 atomic_store_zu(dst, cur_dst + cur_src, ATOMIC_RELAXED);
934}
935
936/******************************************************************************/
937
938static unsigned
939arenas_i2a_impl(size_t i, bool compat, bool validate) {
940 unsigned a;
941
942 switch (i) {
943 case MALLCTL_ARENAS_ALL:
944 a = 0;
945 break;
946 case MALLCTL_ARENAS_DESTROYED:
947 a = 1;
948 break;
949 default:
950 if (compat && i == ctl_arenas->narenas) {
951 /*
952 * Provide deprecated backward compatibility for
953 * accessing the merged stats at index narenas rather
954 * than via MALLCTL_ARENAS_ALL. This is scheduled for
955 * removal in 6.0.0.
956 */
957 a = 0;
958 } else if (validate && i >= ctl_arenas->narenas) {
959 a = UINT_MAX;
960 } else {
961 /*
962 * This function should never be called for an index
963 * more than one past the range of indices that have
964 * initialized ctl data.
965 */
966 assert(i < ctl_arenas->narenas || (!validate && i ==
967 ctl_arenas->narenas));
968 a = (unsigned)i + 2;
969 }
970 break;
971 }
972
973 return a;
974}
975
976static unsigned
977arenas_i2a(size_t i) {
978 return arenas_i2a_impl(i, true, false);
979}
980
981static ctl_arena_t *
982arenas_i_impl(tsd_t *tsd, size_t i, bool compat, bool init) {
983 ctl_arena_t *ret;
984
985 assert(!compat || !init);
986
987 ret = ctl_arenas->arenas[arenas_i2a_impl(i, compat, false)];
988 if (init && ret == NULL) {
989 if (config_stats) {
990 struct container_s {
991 ctl_arena_t ctl_arena;
992 ctl_arena_stats_t astats;
993 };
994 struct container_s *cont =
995 (struct container_s *)base_alloc(tsd_tsdn(tsd),
996 b0get(), sizeof(struct container_s), QUANTUM);
997 if (cont == NULL) {
998 return NULL;
999 }
1000 ret = &cont->ctl_arena;
1001 ret->astats = &cont->astats;
1002 } else {
1003 ret = (ctl_arena_t *)base_alloc(tsd_tsdn(tsd), b0get(),
1004 sizeof(ctl_arena_t), QUANTUM);
1005 if (ret == NULL) {
1006 return NULL;
1007 }
1008 }
1009 ret->arena_ind = (unsigned)i;
1010 ctl_arenas->arenas[arenas_i2a_impl(i, compat, false)] = ret;
1011 }
1012
1013 assert(ret == NULL || arenas_i2a(ret->arena_ind) == arenas_i2a(i));
1014 return ret;
1015}
1016
1017static ctl_arena_t *
1018arenas_i(size_t i) {
1019 ctl_arena_t *ret = arenas_i_impl(tsd_fetch(), i, true, false);
1020 assert(ret != NULL);
1021 return ret;
1022}
1023
1024static void
1025ctl_arena_clear(ctl_arena_t *ctl_arena) {
1026 ctl_arena->nthreads = 0;
1027 ctl_arena->dss = dss_prec_names[dss_prec_limit];
1028 ctl_arena->dirty_decay_ms = -1;
1029 ctl_arena->muzzy_decay_ms = -1;
1030 ctl_arena->pactive = 0;
1031 ctl_arena->pdirty = 0;
1032 ctl_arena->pmuzzy = 0;
1033 if (config_stats) {
1034 memset(&ctl_arena->astats->astats, 0, sizeof(arena_stats_t));
1035 ctl_arena->astats->allocated_small = 0;
1036 ctl_arena->astats->nmalloc_small = 0;
1037 ctl_arena->astats->ndalloc_small = 0;
1038 ctl_arena->astats->nrequests_small = 0;
1039 ctl_arena->astats->nfills_small = 0;
1040 ctl_arena->astats->nflushes_small = 0;
1041 memset(ctl_arena->astats->bstats, 0, SC_NBINS *
1042 sizeof(bin_stats_data_t));
1043 memset(ctl_arena->astats->lstats, 0, (SC_NSIZES - SC_NBINS) *
1044 sizeof(arena_stats_large_t));
1045 memset(ctl_arena->astats->estats, 0, SC_NPSIZES *
1046 sizeof(pac_estats_t));
1047 memset(&ctl_arena->astats->hpastats, 0,
1048 sizeof(hpa_shard_stats_t));
1049 memset(&ctl_arena->astats->secstats, 0,
1050 sizeof(sec_stats_t));
1051 }
1052}
1053
1054static void
1055ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) {
1056 unsigned i;
1057
1058 if (config_stats) {
1059 arena_stats_merge(tsdn, arena, &ctl_arena->nthreads,
1060 &ctl_arena->dss, &ctl_arena->dirty_decay_ms,
1061 &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive,
1062 &ctl_arena->pdirty, &ctl_arena->pmuzzy,
1063 &ctl_arena->astats->astats, ctl_arena->astats->bstats,
1064 ctl_arena->astats->lstats, ctl_arena->astats->estats,
1065 &ctl_arena->astats->hpastats, &ctl_arena->astats->secstats);
1066
1067 for (i = 0; i < SC_NBINS; i++) {
1068 bin_stats_t *bstats =
1069 &ctl_arena->astats->bstats[i].stats_data;
1070 ctl_arena->astats->allocated_small += bstats->curregs *
1071 sz_index2size(i);
1072 ctl_arena->astats->nmalloc_small += bstats->nmalloc;
1073 ctl_arena->astats->ndalloc_small += bstats->ndalloc;
1074 ctl_arena->astats->nrequests_small += bstats->nrequests;
1075 ctl_arena->astats->nfills_small += bstats->nfills;
1076 ctl_arena->astats->nflushes_small += bstats->nflushes;
1077 }
1078 } else {
1079 arena_basic_stats_merge(tsdn, arena, &ctl_arena->nthreads,
1080 &ctl_arena->dss, &ctl_arena->dirty_decay_ms,
1081 &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive,
1082 &ctl_arena->pdirty, &ctl_arena->pmuzzy);
1083 }
1084}
1085
1086static void
1087ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena,
1088 bool destroyed) {
1089 unsigned i;
1090
1091 if (!destroyed) {
1092 ctl_sdarena->nthreads += ctl_arena->nthreads;
1093 ctl_sdarena->pactive += ctl_arena->pactive;
1094 ctl_sdarena->pdirty += ctl_arena->pdirty;
1095 ctl_sdarena->pmuzzy += ctl_arena->pmuzzy;
1096 } else {
1097 assert(ctl_arena->nthreads == 0);
1098 assert(ctl_arena->pactive == 0);
1099 assert(ctl_arena->pdirty == 0);
1100 assert(ctl_arena->pmuzzy == 0);
1101 }
1102
1103 if (config_stats) {
1104 ctl_arena_stats_t *sdstats = ctl_sdarena->astats;
1105 ctl_arena_stats_t *astats = ctl_arena->astats;
1106
1107 if (!destroyed) {
1108 sdstats->astats.mapped += astats->astats.mapped;
1109 sdstats->astats.pa_shard_stats.pac_stats.retained
1110 += astats->astats.pa_shard_stats.pac_stats.retained;
1111 sdstats->astats.pa_shard_stats.edata_avail
1112 += astats->astats.pa_shard_stats.edata_avail;
1113 }
1114
1115 ctl_accum_locked_u64(
1116 &sdstats->astats.pa_shard_stats.pac_stats.decay_dirty.npurge,
1117 &astats->astats.pa_shard_stats.pac_stats.decay_dirty.npurge);
1118 ctl_accum_locked_u64(
1119 &sdstats->astats.pa_shard_stats.pac_stats.decay_dirty.nmadvise,
1120 &astats->astats.pa_shard_stats.pac_stats.decay_dirty.nmadvise);
1121 ctl_accum_locked_u64(
1122 &sdstats->astats.pa_shard_stats.pac_stats.decay_dirty.purged,
1123 &astats->astats.pa_shard_stats.pac_stats.decay_dirty.purged);
1124
1125 ctl_accum_locked_u64(
1126 &sdstats->astats.pa_shard_stats.pac_stats.decay_muzzy.npurge,
1127 &astats->astats.pa_shard_stats.pac_stats.decay_muzzy.npurge);
1128 ctl_accum_locked_u64(
1129 &sdstats->astats.pa_shard_stats.pac_stats.decay_muzzy.nmadvise,
1130 &astats->astats.pa_shard_stats.pac_stats.decay_muzzy.nmadvise);
1131 ctl_accum_locked_u64(
1132 &sdstats->astats.pa_shard_stats.pac_stats.decay_muzzy.purged,
1133 &astats->astats.pa_shard_stats.pac_stats.decay_muzzy.purged);
1134
1135#define OP(mtx) malloc_mutex_prof_merge( \
1136 &(sdstats->astats.mutex_prof_data[ \
1137 arena_prof_mutex_##mtx]), \
1138 &(astats->astats.mutex_prof_data[ \
1139 arena_prof_mutex_##mtx]));
1140MUTEX_PROF_ARENA_MUTEXES
1141#undef OP
1142 if (!destroyed) {
1143 sdstats->astats.base += astats->astats.base;
1144 sdstats->astats.resident += astats->astats.resident;
1145 sdstats->astats.metadata_thp += astats->astats.metadata_thp;
1146 ctl_accum_atomic_zu(&sdstats->astats.internal,
1147 &astats->astats.internal);
1148 } else {
1149 assert(atomic_load_zu(
1150 &astats->astats.internal, ATOMIC_RELAXED) == 0);
1151 }
1152
1153 if (!destroyed) {
1154 sdstats->allocated_small += astats->allocated_small;
1155 } else {
1156 assert(astats->allocated_small == 0);
1157 }
1158 sdstats->nmalloc_small += astats->nmalloc_small;
1159 sdstats->ndalloc_small += astats->ndalloc_small;
1160 sdstats->nrequests_small += astats->nrequests_small;
1161 sdstats->nfills_small += astats->nfills_small;
1162 sdstats->nflushes_small += astats->nflushes_small;
1163
1164 if (!destroyed) {
1165 sdstats->astats.allocated_large +=
1166 astats->astats.allocated_large;
1167 } else {
1168 assert(astats->astats.allocated_large == 0);
1169 }
1170 sdstats->astats.nmalloc_large += astats->astats.nmalloc_large;
1171 sdstats->astats.ndalloc_large += astats->astats.ndalloc_large;
1172 sdstats->astats.nrequests_large
1173 += astats->astats.nrequests_large;
1174 sdstats->astats.nflushes_large += astats->astats.nflushes_large;
1175 ctl_accum_atomic_zu(
1176 &sdstats->astats.pa_shard_stats.pac_stats.abandoned_vm,
1177 &astats->astats.pa_shard_stats.pac_stats.abandoned_vm);
1178
1179 sdstats->astats.tcache_bytes += astats->astats.tcache_bytes;
1180 sdstats->astats.tcache_stashed_bytes +=
1181 astats->astats.tcache_stashed_bytes;
1182
1183 if (ctl_arena->arena_ind == 0) {
1184 sdstats->astats.uptime = astats->astats.uptime;
1185 }
1186
1187 /* Merge bin stats. */
1188 for (i = 0; i < SC_NBINS; i++) {
1189 bin_stats_t *bstats = &astats->bstats[i].stats_data;
1190 bin_stats_t *merged = &sdstats->bstats[i].stats_data;
1191 merged->nmalloc += bstats->nmalloc;
1192 merged->ndalloc += bstats->ndalloc;
1193 merged->nrequests += bstats->nrequests;
1194 if (!destroyed) {
1195 merged->curregs += bstats->curregs;
1196 } else {
1197 assert(bstats->curregs == 0);
1198 }
1199 merged->nfills += bstats->nfills;
1200 merged->nflushes += bstats->nflushes;
1201 merged->nslabs += bstats->nslabs;
1202 merged->reslabs += bstats->reslabs;
1203 if (!destroyed) {
1204 merged->curslabs += bstats->curslabs;
1205 merged->nonfull_slabs += bstats->nonfull_slabs;
1206 } else {
1207 assert(bstats->curslabs == 0);
1208 assert(bstats->nonfull_slabs == 0);
1209 }
1210 malloc_mutex_prof_merge(&sdstats->bstats[i].mutex_data,
1211 &astats->bstats[i].mutex_data);
1212 }
1213
1214 /* Merge stats for large allocations. */
1215 for (i = 0; i < SC_NSIZES - SC_NBINS; i++) {
1216 ctl_accum_locked_u64(&sdstats->lstats[i].nmalloc,
1217 &astats->lstats[i].nmalloc);
1218 ctl_accum_locked_u64(&sdstats->lstats[i].ndalloc,
1219 &astats->lstats[i].ndalloc);
1220 ctl_accum_locked_u64(&sdstats->lstats[i].nrequests,
1221 &astats->lstats[i].nrequests);
1222 if (!destroyed) {
1223 sdstats->lstats[i].curlextents +=
1224 astats->lstats[i].curlextents;
1225 } else {
1226 assert(astats->lstats[i].curlextents == 0);
1227 }
1228 }
1229
1230 /* Merge extents stats. */
1231 for (i = 0; i < SC_NPSIZES; i++) {
1232 sdstats->estats[i].ndirty += astats->estats[i].ndirty;
1233 sdstats->estats[i].nmuzzy += astats->estats[i].nmuzzy;
1234 sdstats->estats[i].nretained
1235 += astats->estats[i].nretained;
1236 sdstats->estats[i].dirty_bytes
1237 += astats->estats[i].dirty_bytes;
1238 sdstats->estats[i].muzzy_bytes
1239 += astats->estats[i].muzzy_bytes;
1240 sdstats->estats[i].retained_bytes
1241 += astats->estats[i].retained_bytes;
1242 }
1243
1244 /* Merge HPA stats. */
1245 hpa_shard_stats_accum(&sdstats->hpastats, &astats->hpastats);
1246 sec_stats_accum(&sdstats->secstats, &astats->secstats);
1247 }
1248}
1249
1250static void
1251ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, ctl_arena_t *ctl_sdarena,
1252 unsigned i, bool destroyed) {
1253 ctl_arena_t *ctl_arena = arenas_i(i);
1254
1255 ctl_arena_clear(ctl_arena);
1256 ctl_arena_stats_amerge(tsdn, ctl_arena, arena);
1257 /* Merge into sum stats as well. */
1258 ctl_arena_stats_sdmerge(ctl_sdarena, ctl_arena, destroyed);
1259}
1260
1261static unsigned
1262ctl_arena_init(tsd_t *tsd, const arena_config_t *config) {
1263 unsigned arena_ind;
1264 ctl_arena_t *ctl_arena;
1265
1266 if ((ctl_arena = ql_last(&ctl_arenas->destroyed, destroyed_link)) !=
1267 NULL) {
1268 ql_remove(&ctl_arenas->destroyed, ctl_arena, destroyed_link);
1269 arena_ind = ctl_arena->arena_ind;
1270 } else {
1271 arena_ind = ctl_arenas->narenas;
1272 }
1273
1274 /* Trigger stats allocation. */
1275 if (arenas_i_impl(tsd, arena_ind, false, true) == NULL) {
1276 return UINT_MAX;
1277 }
1278
1279 /* Initialize new arena. */
1280 if (arena_init(tsd_tsdn(tsd), arena_ind, config) == NULL) {
1281 return UINT_MAX;
1282 }
1283
1284 if (arena_ind == ctl_arenas->narenas) {
1285 ctl_arenas->narenas++;
1286 }
1287
1288 return arena_ind;
1289}
1290
1291static void
1292ctl_background_thread_stats_read(tsdn_t *tsdn) {
1293 background_thread_stats_t *stats = &ctl_stats->background_thread;
1294 if (!have_background_thread ||
1295 background_thread_stats_read(tsdn, stats)) {
1296 memset(stats, 0, sizeof(background_thread_stats_t));
1297 nstime_init_zero(&stats->run_interval);
1298 }
1299 malloc_mutex_prof_copy(
1300 &ctl_stats->mutex_prof_data[global_prof_mutex_max_per_bg_thd],
1301 &stats->max_counter_per_bg_thd);
1302}
1303
1304static void
1305ctl_refresh(tsdn_t *tsdn) {
1306 unsigned i;
1307 ctl_arena_t *ctl_sarena = arenas_i(MALLCTL_ARENAS_ALL);
1308 VARIABLE_ARRAY(arena_t *, tarenas, ctl_arenas->narenas);
1309
1310 /*
1311 * Clear sum stats, since they will be merged into by
1312 * ctl_arena_refresh().
1313 */
1314 ctl_arena_clear(ctl_sarena);
1315
1316 for (i = 0; i < ctl_arenas->narenas; i++) {
1317 tarenas[i] = arena_get(tsdn, i, false);
1318 }
1319
1320 for (i = 0; i < ctl_arenas->narenas; i++) {
1321 ctl_arena_t *ctl_arena = arenas_i(i);
1322 bool initialized = (tarenas[i] != NULL);
1323
1324 ctl_arena->initialized = initialized;
1325 if (initialized) {
1326 ctl_arena_refresh(tsdn, tarenas[i], ctl_sarena, i,
1327 false);
1328 }
1329 }
1330
1331 if (config_stats) {
1332 ctl_stats->allocated = ctl_sarena->astats->allocated_small +
1333 ctl_sarena->astats->astats.allocated_large;
1334 ctl_stats->active = (ctl_sarena->pactive << LG_PAGE);
1335 ctl_stats->metadata = ctl_sarena->astats->astats.base +
1336 atomic_load_zu(&ctl_sarena->astats->astats.internal,
1337 ATOMIC_RELAXED);
1338 ctl_stats->resident = ctl_sarena->astats->astats.resident;
1339 ctl_stats->metadata_thp =
1340 ctl_sarena->astats->astats.metadata_thp;
1341 ctl_stats->mapped = ctl_sarena->astats->astats.mapped;
1342 ctl_stats->retained = ctl_sarena->astats->astats
1343 .pa_shard_stats.pac_stats.retained;
1344
1345 ctl_background_thread_stats_read(tsdn);
1346
1347#define READ_GLOBAL_MUTEX_PROF_DATA(i, mtx) \
1348 malloc_mutex_lock(tsdn, &mtx); \
1349 malloc_mutex_prof_read(tsdn, &ctl_stats->mutex_prof_data[i], &mtx); \
1350 malloc_mutex_unlock(tsdn, &mtx);
1351
1352 if (config_prof && opt_prof) {
1353 READ_GLOBAL_MUTEX_PROF_DATA(
1354 global_prof_mutex_prof, bt2gctx_mtx);
1355 READ_GLOBAL_MUTEX_PROF_DATA(
1356 global_prof_mutex_prof_thds_data, tdatas_mtx);
1357 READ_GLOBAL_MUTEX_PROF_DATA(
1358 global_prof_mutex_prof_dump, prof_dump_mtx);
1359 READ_GLOBAL_MUTEX_PROF_DATA(
1360 global_prof_mutex_prof_recent_alloc,
1361 prof_recent_alloc_mtx);
1362 READ_GLOBAL_MUTEX_PROF_DATA(
1363 global_prof_mutex_prof_recent_dump,
1364 prof_recent_dump_mtx);
1365 READ_GLOBAL_MUTEX_PROF_DATA(
1366 global_prof_mutex_prof_stats, prof_stats_mtx);
1367 }
1368 if (have_background_thread) {
1369 READ_GLOBAL_MUTEX_PROF_DATA(
1370 global_prof_mutex_background_thread,
1371 background_thread_lock);
1372 } else {
1373 memset(&ctl_stats->mutex_prof_data[
1374 global_prof_mutex_background_thread], 0,
1375 sizeof(mutex_prof_data_t));
1376 }
1377 /* We own ctl mutex already. */
1378 malloc_mutex_prof_read(tsdn,
1379 &ctl_stats->mutex_prof_data[global_prof_mutex_ctl],
1380 &ctl_mtx);
1381#undef READ_GLOBAL_MUTEX_PROF_DATA
1382 }
1383 ctl_arenas->epoch++;
1384}
1385
1386static bool
1387ctl_init(tsd_t *tsd) {
1388 bool ret;
1389 tsdn_t *tsdn = tsd_tsdn(tsd);
1390
1391 malloc_mutex_lock(tsdn, &ctl_mtx);
1392 if (!ctl_initialized) {
1393 ctl_arena_t *ctl_sarena, *ctl_darena;
1394 unsigned i;
1395
1396 /*
1397 * Allocate demand-zeroed space for pointers to the full
1398 * range of supported arena indices.
1399 */
1400 if (ctl_arenas == NULL) {
1401 ctl_arenas = (ctl_arenas_t *)base_alloc(tsdn,
1402 b0get(), sizeof(ctl_arenas_t), QUANTUM);
1403 if (ctl_arenas == NULL) {
1404 ret = true;
1405 goto label_return;
1406 }
1407 }
1408
1409 if (config_stats && ctl_stats == NULL) {
1410 ctl_stats = (ctl_stats_t *)base_alloc(tsdn, b0get(),
1411 sizeof(ctl_stats_t), QUANTUM);
1412 if (ctl_stats == NULL) {
1413 ret = true;
1414 goto label_return;
1415 }
1416 }
1417
1418 /*
1419 * Allocate space for the current full range of arenas
1420 * here rather than doing it lazily elsewhere, in order
1421 * to limit when OOM-caused errors can occur.
1422 */
1423 if ((ctl_sarena = arenas_i_impl(tsd, MALLCTL_ARENAS_ALL, false,
1424 true)) == NULL) {
1425 ret = true;
1426 goto label_return;
1427 }
1428 ctl_sarena->initialized = true;
1429
1430 if ((ctl_darena = arenas_i_impl(tsd, MALLCTL_ARENAS_DESTROYED,
1431 false, true)) == NULL) {
1432 ret = true;
1433 goto label_return;
1434 }
1435 ctl_arena_clear(ctl_darena);
1436 /*
1437 * Don't toggle ctl_darena to initialized until an arena is
1438 * actually destroyed, so that arena.<i>.initialized can be used
1439 * to query whether the stats are relevant.
1440 */
1441
1442 ctl_arenas->narenas = narenas_total_get();
1443 for (i = 0; i < ctl_arenas->narenas; i++) {
1444 if (arenas_i_impl(tsd, i, false, true) == NULL) {
1445 ret = true;
1446 goto label_return;
1447 }
1448 }
1449
1450 ql_new(&ctl_arenas->destroyed);
1451 ctl_refresh(tsdn);
1452
1453 ctl_initialized = true;
1454 }
1455
1456 ret = false;
1457label_return:
1458 malloc_mutex_unlock(tsdn, &ctl_mtx);
1459 return ret;
1460}
1461
1462static int
1463ctl_lookup(tsdn_t *tsdn, const ctl_named_node_t *starting_node,
1464 const char *name, const ctl_named_node_t **ending_nodep, size_t *mibp,
1465 size_t *depthp) {
1466 int ret;
1467 const char *elm, *tdot, *dot;
1468 size_t elen, i, j;
1469 const ctl_named_node_t *node;
1470
1471 elm = name;
1472 /* Equivalent to strchrnul(). */
1473 dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : strchr(elm, '\0');
1474 elen = (size_t)((uintptr_t)dot - (uintptr_t)elm);
1475 if (elen == 0) {
1476 ret = ENOENT;
1477 goto label_return;
1478 }
1479 node = starting_node;
1480 for (i = 0; i < *depthp; i++) {
1481 assert(node);
1482 assert(node->nchildren > 0);
1483 if (ctl_named_node(node->children) != NULL) {
1484 const ctl_named_node_t *pnode = node;
1485
1486 /* Children are named. */
1487 for (j = 0; j < node->nchildren; j++) {
1488 const ctl_named_node_t *child =
1489 ctl_named_children(node, j);
1490 if (strlen(child->name) == elen &&
1491 strncmp(elm, child->name, elen) == 0) {
1492 node = child;
1493 mibp[i] = j;
1494 break;
1495 }
1496 }
1497 if (node == pnode) {
1498 ret = ENOENT;
1499 goto label_return;
1500 }
1501 } else {
1502 uintmax_t index;
1503 const ctl_indexed_node_t *inode;
1504
1505 /* Children are indexed. */
1506 index = malloc_strtoumax(elm, NULL, 10);
1507 if (index == UINTMAX_MAX || index > SIZE_T_MAX) {
1508 ret = ENOENT;
1509 goto label_return;
1510 }
1511
1512 inode = ctl_indexed_node(node->children);
1513 node = inode->index(tsdn, mibp, *depthp, (size_t)index);
1514 if (node == NULL) {
1515 ret = ENOENT;
1516 goto label_return;
1517 }
1518
1519 mibp[i] = (size_t)index;
1520 }
1521
1522 /* Reached the end? */
1523 if (node->ctl != NULL || *dot == '\0') {
1524 /* Terminal node. */
1525 if (*dot != '\0') {
1526 /*
1527 * The name contains more elements than are
1528 * in this path through the tree.
1529 */
1530 ret = ENOENT;
1531 goto label_return;
1532 }
1533 /* Complete lookup successful. */
1534 *depthp = i + 1;
1535 break;
1536 }
1537
1538 /* Update elm. */
1539 elm = &dot[1];
1540 dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot :
1541 strchr(elm, '\0');
1542 elen = (size_t)((uintptr_t)dot - (uintptr_t)elm);
1543 }
1544 if (ending_nodep != NULL) {
1545 *ending_nodep = node;
1546 }
1547
1548 ret = 0;
1549label_return:
1550 return ret;
1551}
1552
1553int
1554ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp,
1555 void *newp, size_t newlen) {
1556 int ret;
1557 size_t depth;
1558 size_t mib[CTL_MAX_DEPTH];
1559 const ctl_named_node_t *node;
1560
1561 if (!ctl_initialized && ctl_init(tsd)) {
1562 ret = EAGAIN;
1563 goto label_return;
1564 }
1565
1566 depth = CTL_MAX_DEPTH;
1567 ret = ctl_lookup(tsd_tsdn(tsd), super_root_node, name, &node, mib,
1568 &depth);
1569 if (ret != 0) {
1570 goto label_return;
1571 }
1572
1573 if (node != NULL && node->ctl) {
1574 ret = node->ctl(tsd, mib, depth, oldp, oldlenp, newp, newlen);
1575 } else {
1576 /* The name refers to a partial path through the ctl tree. */
1577 ret = ENOENT;
1578 }
1579
1580label_return:
1581 return(ret);
1582}
1583
1584int
1585ctl_nametomib(tsd_t *tsd, const char *name, size_t *mibp, size_t *miblenp) {
1586 int ret;
1587
1588 if (!ctl_initialized && ctl_init(tsd)) {
1589 ret = EAGAIN;
1590 goto label_return;
1591 }
1592
1593 ret = ctl_lookup(tsd_tsdn(tsd), super_root_node, name, NULL, mibp,
1594 miblenp);
1595label_return:
1596 return(ret);
1597}
1598
1599static int
1600ctl_lookupbymib(tsdn_t *tsdn, const ctl_named_node_t **ending_nodep,
1601 const size_t *mib, size_t miblen) {
1602 int ret;
1603
1604 const ctl_named_node_t *node = super_root_node;
1605 for (size_t i = 0; i < miblen; i++) {
1606 assert(node);
1607 assert(node->nchildren > 0);
1608 if (ctl_named_node(node->children) != NULL) {
1609 /* Children are named. */
1610 if (node->nchildren <= mib[i]) {
1611 ret = ENOENT;
1612 goto label_return;
1613 }
1614 node = ctl_named_children(node, mib[i]);
1615 } else {
1616 const ctl_indexed_node_t *inode;
1617
1618 /* Indexed element. */
1619 inode = ctl_indexed_node(node->children);
1620 node = inode->index(tsdn, mib, miblen, mib[i]);
1621 if (node == NULL) {
1622 ret = ENOENT;
1623 goto label_return;
1624 }
1625 }
1626 }
1627 assert(ending_nodep != NULL);
1628 *ending_nodep = node;
1629 ret = 0;
1630
1631label_return:
1632 return(ret);
1633}
1634
1635int
1636ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
1637 size_t *oldlenp, void *newp, size_t newlen) {
1638 int ret;
1639 const ctl_named_node_t *node;
1640
1641 if (!ctl_initialized && ctl_init(tsd)) {
1642 ret = EAGAIN;
1643 goto label_return;
1644 }
1645
1646 ret = ctl_lookupbymib(tsd_tsdn(tsd), &node, mib, miblen);
1647 if (ret != 0) {
1648 goto label_return;
1649 }
1650
1651 /* Call the ctl function. */
1652 if (node && node->ctl) {
1653 ret = node->ctl(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
1654 } else {
1655 /* Partial MIB. */
1656 ret = ENOENT;
1657 }
1658
1659label_return:
1660 return(ret);
1661}
1662
1663int
1664ctl_mibnametomib(tsd_t *tsd, size_t *mib, size_t miblen, const char *name,
1665 size_t *miblenp) {
1666 int ret;
1667 const ctl_named_node_t *node;
1668
1669 if (!ctl_initialized && ctl_init(tsd)) {
1670 ret = EAGAIN;
1671 goto label_return;
1672 }
1673
1674 ret = ctl_lookupbymib(tsd_tsdn(tsd), &node, mib, miblen);
1675 if (ret != 0) {
1676 goto label_return;
1677 }
1678 if (node == NULL || node->ctl != NULL) {
1679 ret = ENOENT;
1680 goto label_return;
1681 }
1682
1683 assert(miblenp != NULL);
1684 assert(*miblenp >= miblen);
1685 *miblenp -= miblen;
1686 ret = ctl_lookup(tsd_tsdn(tsd), node, name, NULL, mib + miblen,
1687 miblenp);
1688 *miblenp += miblen;
1689label_return:
1690 return(ret);
1691}
1692
1693int
1694ctl_bymibname(tsd_t *tsd, size_t *mib, size_t miblen, const char *name,
1695 size_t *miblenp, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
1696 int ret;
1697 const ctl_named_node_t *node;
1698
1699 if (!ctl_initialized && ctl_init(tsd)) {
1700 ret = EAGAIN;
1701 goto label_return;
1702 }
1703
1704 ret = ctl_lookupbymib(tsd_tsdn(tsd), &node, mib, miblen);
1705 if (ret != 0) {
1706 goto label_return;
1707 }
1708 if (node == NULL || node->ctl != NULL) {
1709 ret = ENOENT;
1710 goto label_return;
1711 }
1712
1713 assert(miblenp != NULL);
1714 assert(*miblenp >= miblen);
1715 *miblenp -= miblen;
1716 /*
1717 * The same node supplies the starting node and stores the ending node.
1718 */
1719 ret = ctl_lookup(tsd_tsdn(tsd), node, name, &node, mib + miblen,
1720 miblenp);
1721 *miblenp += miblen;
1722 if (ret != 0) {
1723 goto label_return;
1724 }
1725
1726 if (node != NULL && node->ctl) {
1727 ret = node->ctl(tsd, mib, *miblenp, oldp, oldlenp, newp,
1728 newlen);
1729 } else {
1730 /* The name refers to a partial path through the ctl tree. */
1731 ret = ENOENT;
1732 }
1733
1734label_return:
1735 return(ret);
1736}
1737
1738bool
1739ctl_boot(void) {
1740 if (malloc_mutex_init(&ctl_mtx, "ctl", WITNESS_RANK_CTL,
1741 malloc_mutex_rank_exclusive)) {
1742 return true;
1743 }
1744
1745 ctl_initialized = false;
1746
1747 return false;
1748}
1749
1750void
1751ctl_prefork(tsdn_t *tsdn) {
1752 malloc_mutex_prefork(tsdn, &ctl_mtx);
1753}
1754
1755void
1756ctl_postfork_parent(tsdn_t *tsdn) {
1757 malloc_mutex_postfork_parent(tsdn, &ctl_mtx);
1758}
1759
1760void
1761ctl_postfork_child(tsdn_t *tsdn) {
1762 malloc_mutex_postfork_child(tsdn, &ctl_mtx);
1763}
1764
1765void
1766ctl_mtx_assert_held(tsdn_t *tsdn) {
1767 malloc_mutex_assert_owner(tsdn, &ctl_mtx);
1768}
1769
1770/******************************************************************************/
1771/* *_ctl() functions. */
1772
1773#define READONLY() do { \
1774 if (newp != NULL || newlen != 0) { \
1775 ret = EPERM; \
1776 goto label_return; \
1777 } \
1778} while (0)
1779
1780#define WRITEONLY() do { \
1781 if (oldp != NULL || oldlenp != NULL) { \
1782 ret = EPERM; \
1783 goto label_return; \
1784 } \
1785} while (0)
1786
1787/* Can read or write, but not both. */
1788#define READ_XOR_WRITE() do { \
1789 if ((oldp != NULL && oldlenp != NULL) && (newp != NULL || \
1790 newlen != 0)) { \
1791 ret = EPERM; \
1792 goto label_return; \
1793 } \
1794} while (0)
1795
1796/* Can neither read nor write. */
1797#define NEITHER_READ_NOR_WRITE() do { \
1798 if (oldp != NULL || oldlenp != NULL || newp != NULL || \
1799 newlen != 0) { \
1800 ret = EPERM; \
1801 goto label_return; \
1802 } \
1803} while (0)
1804
1805/* Verify that the space provided is enough. */
1806#define VERIFY_READ(t) do { \
1807 if (oldp == NULL || oldlenp == NULL || *oldlenp != sizeof(t)) { \
1808 *oldlenp = 0; \
1809 ret = EINVAL; \
1810 goto label_return; \
1811 } \
1812} while (0)
1813
1814#define READ(v, t) do { \
1815 if (oldp != NULL && oldlenp != NULL) { \
1816 if (*oldlenp != sizeof(t)) { \
1817 size_t copylen = (sizeof(t) <= *oldlenp) \
1818 ? sizeof(t) : *oldlenp; \
1819 memcpy(oldp, (void *)&(v), copylen); \
1820 *oldlenp = copylen; \
1821 ret = EINVAL; \
1822 goto label_return; \
1823 } \
1824 *(t *)oldp = (v); \
1825 } \
1826} while (0)
1827
1828#define WRITE(v, t) do { \
1829 if (newp != NULL) { \
1830 if (newlen != sizeof(t)) { \
1831 ret = EINVAL; \
1832 goto label_return; \
1833 } \
1834 (v) = *(t *)newp; \
1835 } \
1836} while (0)
1837
1838#define ASSURED_WRITE(v, t) do { \
1839 if (newp == NULL || newlen != sizeof(t)) { \
1840 ret = EINVAL; \
1841 goto label_return; \
1842 } \
1843 (v) = *(t *)newp; \
1844} while (0)
1845
1846#define MIB_UNSIGNED(v, i) do { \
1847 if (mib[i] > UINT_MAX) { \
1848 ret = EFAULT; \
1849 goto label_return; \
1850 } \
1851 v = (unsigned)mib[i]; \
1852} while (0)
1853
1854/*
1855 * There's a lot of code duplication in the following macros due to limitations
1856 * in how nested cpp macros are expanded.
1857 */
1858#define CTL_RO_CLGEN(c, l, n, v, t) \
1859static int \
1860n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
1861 size_t *oldlenp, void *newp, size_t newlen) { \
1862 int ret; \
1863 t oldval; \
1864 \
1865 if (!(c)) { \
1866 return ENOENT; \
1867 } \
1868 if (l) { \
1869 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \
1870 } \
1871 READONLY(); \
1872 oldval = (v); \
1873 READ(oldval, t); \
1874 \
1875 ret = 0; \
1876label_return: \
1877 if (l) { \
1878 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \
1879 } \
1880 return ret; \
1881}
1882
1883#define CTL_RO_CGEN(c, n, v, t) \
1884static int \
1885n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
1886 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \
1887 int ret; \
1888 t oldval; \
1889 \
1890 if (!(c)) { \
1891 return ENOENT; \
1892 } \
1893 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \
1894 READONLY(); \
1895 oldval = (v); \
1896 READ(oldval, t); \
1897 \
1898 ret = 0; \
1899label_return: \
1900 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \
1901 return ret; \
1902}
1903
1904#define CTL_RO_GEN(n, v, t) \
1905static int \
1906n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
1907 size_t *oldlenp, void *newp, size_t newlen) { \
1908 int ret; \
1909 t oldval; \
1910 \
1911 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \
1912 READONLY(); \
1913 oldval = (v); \
1914 READ(oldval, t); \
1915 \
1916 ret = 0; \
1917label_return: \
1918 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \
1919 return ret; \
1920}
1921
1922/*
1923 * ctl_mtx is not acquired, under the assumption that no pertinent data will
1924 * mutate during the call.
1925 */
1926#define CTL_RO_NL_CGEN(c, n, v, t) \
1927static int \
1928n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
1929 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \
1930 int ret; \
1931 t oldval; \
1932 \
1933 if (!(c)) { \
1934 return ENOENT; \
1935 } \
1936 READONLY(); \
1937 oldval = (v); \
1938 READ(oldval, t); \
1939 \
1940 ret = 0; \
1941label_return: \
1942 return ret; \
1943}
1944
1945#define CTL_RO_NL_GEN(n, v, t) \
1946static int \
1947n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
1948 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \
1949 int ret; \
1950 t oldval; \
1951 \
1952 READONLY(); \
1953 oldval = (v); \
1954 READ(oldval, t); \
1955 \
1956 ret = 0; \
1957label_return: \
1958 return ret; \
1959}
1960
1961#define CTL_RO_CONFIG_GEN(n, t) \
1962static int \
1963n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
1964 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \
1965 int ret; \
1966 t oldval; \
1967 \
1968 READONLY(); \
1969 oldval = n; \
1970 READ(oldval, t); \
1971 \
1972 ret = 0; \
1973label_return: \
1974 return ret; \
1975}
1976
1977/******************************************************************************/
1978
1979CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *)
1980
1981static int
1982epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
1983 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
1984 int ret;
1985 UNUSED uint64_t newval;
1986
1987 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
1988 WRITE(newval, uint64_t);
1989 if (newp != NULL) {
1990 ctl_refresh(tsd_tsdn(tsd));
1991 }
1992 READ(ctl_arenas->epoch, uint64_t);
1993
1994 ret = 0;
1995label_return:
1996 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
1997 return ret;
1998}
1999
2000static int
2001background_thread_ctl(tsd_t *tsd, const size_t *mib,
2002 size_t miblen, void *oldp, size_t *oldlenp,
2003 void *newp, size_t newlen) {
2004 int ret;
2005 bool oldval;
2006
2007 if (!have_background_thread) {
2008 return ENOENT;
2009 }
2010 background_thread_ctl_init(tsd_tsdn(tsd));
2011
2012 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2013 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
2014 if (newp == NULL) {
2015 oldval = background_thread_enabled();
2016 READ(oldval, bool);
2017 } else {
2018 if (newlen != sizeof(bool)) {
2019 ret = EINVAL;
2020 goto label_return;
2021 }
2022 oldval = background_thread_enabled();
2023 READ(oldval, bool);
2024
2025 bool newval = *(bool *)newp;
2026 if (newval == oldval) {
2027 ret = 0;
2028 goto label_return;
2029 }
2030
2031 background_thread_enabled_set(tsd_tsdn(tsd), newval);
2032 if (newval) {
2033 if (background_threads_enable(tsd)) {
2034 ret = EFAULT;
2035 goto label_return;
2036 }
2037 } else {
2038 if (background_threads_disable(tsd)) {
2039 ret = EFAULT;
2040 goto label_return;
2041 }
2042 }
2043 }
2044 ret = 0;
2045label_return:
2046 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
2047 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2048
2049 return ret;
2050}
2051
2052static int
2053max_background_threads_ctl(tsd_t *tsd, const size_t *mib,
2054 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2055 size_t newlen) {
2056 int ret;
2057 size_t oldval;
2058
2059 if (!have_background_thread) {
2060 return ENOENT;
2061 }
2062 background_thread_ctl_init(tsd_tsdn(tsd));
2063
2064 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2065 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
2066 if (newp == NULL) {
2067 oldval = max_background_threads;
2068 READ(oldval, size_t);
2069 } else {
2070 if (newlen != sizeof(size_t)) {
2071 ret = EINVAL;
2072 goto label_return;
2073 }
2074 oldval = max_background_threads;
2075 READ(oldval, size_t);
2076
2077 size_t newval = *(size_t *)newp;
2078 if (newval == oldval) {
2079 ret = 0;
2080 goto label_return;
2081 }
2082 if (newval > opt_max_background_threads) {
2083 ret = EINVAL;
2084 goto label_return;
2085 }
2086
2087 if (background_thread_enabled()) {
2088 background_thread_enabled_set(tsd_tsdn(tsd), false);
2089 if (background_threads_disable(tsd)) {
2090 ret = EFAULT;
2091 goto label_return;
2092 }
2093 max_background_threads = newval;
2094 background_thread_enabled_set(tsd_tsdn(tsd), true);
2095 if (background_threads_enable(tsd)) {
2096 ret = EFAULT;
2097 goto label_return;
2098 }
2099 } else {
2100 max_background_threads = newval;
2101 }
2102 }
2103 ret = 0;
2104label_return:
2105 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
2106 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2107
2108 return ret;
2109}
2110
2111/******************************************************************************/
2112
2113CTL_RO_CONFIG_GEN(config_cache_oblivious, bool)
2114CTL_RO_CONFIG_GEN(config_debug, bool)
2115CTL_RO_CONFIG_GEN(config_fill, bool)
2116CTL_RO_CONFIG_GEN(config_lazy_lock, bool)
2117CTL_RO_CONFIG_GEN(config_malloc_conf, const char *)
2118CTL_RO_CONFIG_GEN(config_opt_safety_checks, bool)
2119CTL_RO_CONFIG_GEN(config_prof, bool)
2120CTL_RO_CONFIG_GEN(config_prof_libgcc, bool)
2121CTL_RO_CONFIG_GEN(config_prof_libunwind, bool)
2122CTL_RO_CONFIG_GEN(config_stats, bool)
2123CTL_RO_CONFIG_GEN(config_utrace, bool)
2124CTL_RO_CONFIG_GEN(config_xmalloc, bool)
2125
2126/******************************************************************************/
2127
2128CTL_RO_NL_GEN(opt_abort, opt_abort, bool)
2129CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool)
2130CTL_RO_NL_GEN(opt_cache_oblivious, opt_cache_oblivious, bool)
2131CTL_RO_NL_GEN(opt_trust_madvise, opt_trust_madvise, bool)
2132CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool)
2133
2134/* HPA options. */
2135CTL_RO_NL_GEN(opt_hpa, opt_hpa, bool)
2136CTL_RO_NL_GEN(opt_hpa_hugification_threshold,
2137 opt_hpa_opts.hugification_threshold, size_t)
2138CTL_RO_NL_GEN(opt_hpa_hugify_delay_ms, opt_hpa_opts.hugify_delay_ms, uint64_t)
2139CTL_RO_NL_GEN(opt_hpa_min_purge_interval_ms, opt_hpa_opts.min_purge_interval_ms,
2140 uint64_t)
2141
2142/*
2143 * This will have to change before we publicly document this option; fxp_t and
2144 * its representation are internal implementation details.
2145 */
2146CTL_RO_NL_GEN(opt_hpa_dirty_mult, opt_hpa_opts.dirty_mult, fxp_t)
2147CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t)
2148
2149/* HPA SEC options */
2150CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_opts.nshards, size_t)
2151CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_opts.max_alloc, size_t)
2152CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_opts.max_bytes, size_t)
2153CTL_RO_NL_GEN(opt_hpa_sec_bytes_after_flush, opt_hpa_sec_opts.bytes_after_flush,
2154 size_t)
2155CTL_RO_NL_GEN(opt_hpa_sec_batch_fill_extra, opt_hpa_sec_opts.batch_fill_extra,
2156 size_t)
2157
2158CTL_RO_NL_GEN(opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp],
2159 const char *)
2160CTL_RO_NL_GEN(opt_retain, opt_retain, bool)
2161CTL_RO_NL_GEN(opt_dss, opt_dss, const char *)
2162CTL_RO_NL_GEN(opt_narenas, opt_narenas, unsigned)
2163CTL_RO_NL_GEN(opt_percpu_arena, percpu_arena_mode_names[opt_percpu_arena],
2164 const char *)
2165CTL_RO_NL_GEN(opt_mutex_max_spin, opt_mutex_max_spin, int64_t)
2166CTL_RO_NL_GEN(opt_oversize_threshold, opt_oversize_threshold, size_t)
2167CTL_RO_NL_GEN(opt_background_thread, opt_background_thread, bool)
2168CTL_RO_NL_GEN(opt_max_background_threads, opt_max_background_threads, size_t)
2169CTL_RO_NL_GEN(opt_dirty_decay_ms, opt_dirty_decay_ms, ssize_t)
2170CTL_RO_NL_GEN(opt_muzzy_decay_ms, opt_muzzy_decay_ms, ssize_t)
2171CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool)
2172CTL_RO_NL_GEN(opt_stats_print_opts, opt_stats_print_opts, const char *)
2173CTL_RO_NL_GEN(opt_stats_interval, opt_stats_interval, int64_t)
2174CTL_RO_NL_GEN(opt_stats_interval_opts, opt_stats_interval_opts, const char *)
2175CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, const char *)
2176CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool)
2177CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool)
2178CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool)
2179CTL_RO_NL_CGEN(config_enable_cxx, opt_experimental_infallible_new,
2180 opt_experimental_infallible_new, bool)
2181CTL_RO_NL_GEN(opt_tcache, opt_tcache, bool)
2182CTL_RO_NL_GEN(opt_tcache_max, opt_tcache_max, size_t)
2183CTL_RO_NL_GEN(opt_tcache_nslots_small_min, opt_tcache_nslots_small_min,
2184 unsigned)
2185CTL_RO_NL_GEN(opt_tcache_nslots_small_max, opt_tcache_nslots_small_max,
2186 unsigned)
2187CTL_RO_NL_GEN(opt_tcache_nslots_large, opt_tcache_nslots_large, unsigned)
2188CTL_RO_NL_GEN(opt_lg_tcache_nslots_mul, opt_lg_tcache_nslots_mul, ssize_t)
2189CTL_RO_NL_GEN(opt_tcache_gc_incr_bytes, opt_tcache_gc_incr_bytes, size_t)
2190CTL_RO_NL_GEN(opt_tcache_gc_delay_bytes, opt_tcache_gc_delay_bytes, size_t)
2191CTL_RO_NL_GEN(opt_lg_tcache_flush_small_div, opt_lg_tcache_flush_small_div,
2192 unsigned)
2193CTL_RO_NL_GEN(opt_lg_tcache_flush_large_div, opt_lg_tcache_flush_large_div,
2194 unsigned)
2195CTL_RO_NL_GEN(opt_thp, thp_mode_names[opt_thp], const char *)
2196CTL_RO_NL_GEN(opt_lg_extent_max_active_fit, opt_lg_extent_max_active_fit,
2197 size_t)
2198CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool)
2199CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *)
2200CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool)
2201CTL_RO_NL_CGEN(config_prof, opt_prof_thread_active_init,
2202 opt_prof_thread_active_init, bool)
2203CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t)
2204CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool)
2205CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t)
2206CTL_RO_NL_CGEN(config_prof, opt_prof_gdump, opt_prof_gdump, bool)
2207CTL_RO_NL_CGEN(config_prof, opt_prof_final, opt_prof_final, bool)
2208CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool)
2209CTL_RO_NL_CGEN(config_prof, opt_prof_leak_error, opt_prof_leak_error, bool)
2210CTL_RO_NL_CGEN(config_prof, opt_prof_recent_alloc_max,
2211 opt_prof_recent_alloc_max, ssize_t)
2212CTL_RO_NL_CGEN(config_prof, opt_prof_stats, opt_prof_stats, bool)
2213CTL_RO_NL_CGEN(config_prof, opt_prof_sys_thread_name, opt_prof_sys_thread_name,
2214 bool)
2215CTL_RO_NL_CGEN(config_prof, opt_prof_time_res,
2216 prof_time_res_mode_names[opt_prof_time_res], const char *)
2217CTL_RO_NL_CGEN(config_uaf_detection, opt_lg_san_uaf_align,
2218 opt_lg_san_uaf_align, ssize_t)
2219CTL_RO_NL_GEN(opt_zero_realloc,
2220 zero_realloc_mode_names[opt_zero_realloc_action], const char *)
2221
2222/******************************************************************************/
2223
2224static int
2225thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2226 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2227 int ret;
2228 arena_t *oldarena;
2229 unsigned newind, oldind;
2230
2231 oldarena = arena_choose(tsd, NULL);
2232 if (oldarena == NULL) {
2233 return EAGAIN;
2234 }
2235 newind = oldind = arena_ind_get(oldarena);
2236 WRITE(newind, unsigned);
2237 READ(oldind, unsigned);
2238
2239 if (newind != oldind) {
2240 arena_t *newarena;
2241
2242 if (newind >= narenas_total_get()) {
2243 /* New arena index is out of range. */
2244 ret = EFAULT;
2245 goto label_return;
2246 }
2247
2248 if (have_percpu_arena &&
2249 PERCPU_ARENA_ENABLED(opt_percpu_arena)) {
2250 if (newind < percpu_arena_ind_limit(opt_percpu_arena)) {
2251 /*
2252 * If perCPU arena is enabled, thread_arena
2253 * control is not allowed for the auto arena
2254 * range.
2255 */
2256 ret = EPERM;
2257 goto label_return;
2258 }
2259 }
2260
2261 /* Initialize arena if necessary. */
2262 newarena = arena_get(tsd_tsdn(tsd), newind, true);
2263 if (newarena == NULL) {
2264 ret = EAGAIN;
2265 goto label_return;
2266 }
2267 /* Set new arena/tcache associations. */
2268 arena_migrate(tsd, oldarena, newarena);
2269 if (tcache_available(tsd)) {
2270 tcache_arena_reassociate(tsd_tsdn(tsd),
2271 tsd_tcache_slowp_get(tsd), tsd_tcachep_get(tsd),
2272 newarena);
2273 }
2274 }
2275
2276 ret = 0;
2277label_return:
2278 return ret;
2279}
2280
2281CTL_RO_NL_GEN(thread_allocated, tsd_thread_allocated_get(tsd), uint64_t)
2282CTL_RO_NL_GEN(thread_allocatedp, tsd_thread_allocatedp_get(tsd), uint64_t *)
2283CTL_RO_NL_GEN(thread_deallocated, tsd_thread_deallocated_get(tsd), uint64_t)
2284CTL_RO_NL_GEN(thread_deallocatedp, tsd_thread_deallocatedp_get(tsd), uint64_t *)
2285
2286static int
2287thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib,
2288 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2289 size_t newlen) {
2290 int ret;
2291 bool oldval;
2292
2293 oldval = tcache_enabled_get(tsd);
2294 if (newp != NULL) {
2295 if (newlen != sizeof(bool)) {
2296 ret = EINVAL;
2297 goto label_return;
2298 }
2299 tcache_enabled_set(tsd, *(bool *)newp);
2300 }
2301 READ(oldval, bool);
2302
2303 ret = 0;
2304label_return:
2305 return ret;
2306}
2307
2308static int
2309thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib,
2310 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2311 size_t newlen) {
2312 int ret;
2313
2314 if (!tcache_available(tsd)) {
2315 ret = EFAULT;
2316 goto label_return;
2317 }
2318
2319 NEITHER_READ_NOR_WRITE();
2320
2321 tcache_flush(tsd);
2322
2323 ret = 0;
2324label_return:
2325 return ret;
2326}
2327
2328static int
2329thread_peak_read_ctl(tsd_t *tsd, const size_t *mib,
2330 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2331 size_t newlen) {
2332 int ret;
2333 if (!config_stats) {
2334 return ENOENT;
2335 }
2336 READONLY();
2337 peak_event_update(tsd);
2338 uint64_t result = peak_event_max(tsd);
2339 READ(result, uint64_t);
2340 ret = 0;
2341label_return:
2342 return ret;
2343}
2344
2345static int
2346thread_peak_reset_ctl(tsd_t *tsd, const size_t *mib,
2347 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2348 size_t newlen) {
2349 int ret;
2350 if (!config_stats) {
2351 return ENOENT;
2352 }
2353 NEITHER_READ_NOR_WRITE();
2354 peak_event_zero(tsd);
2355 ret = 0;
2356label_return:
2357 return ret;
2358}
2359
2360static int
2361thread_prof_name_ctl(tsd_t *tsd, const size_t *mib,
2362 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2363 size_t newlen) {
2364 int ret;
2365
2366 if (!config_prof || !opt_prof) {
2367 return ENOENT;
2368 }
2369
2370 READ_XOR_WRITE();
2371
2372 if (newp != NULL) {
2373 if (newlen != sizeof(const char *)) {
2374 ret = EINVAL;
2375 goto label_return;
2376 }
2377
2378 if ((ret = prof_thread_name_set(tsd, *(const char **)newp)) !=
2379 0) {
2380 goto label_return;
2381 }
2382 } else {
2383 const char *oldname = prof_thread_name_get(tsd);
2384 READ(oldname, const char *);
2385 }
2386
2387 ret = 0;
2388label_return:
2389 return ret;
2390}
2391
2392static int
2393thread_prof_active_ctl(tsd_t *tsd, const size_t *mib,
2394 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2395 size_t newlen) {
2396 int ret;
2397 bool oldval;
2398
2399 if (!config_prof) {
2400 return ENOENT;
2401 }
2402
2403 oldval = opt_prof ? prof_thread_active_get(tsd) : false;
2404 if (newp != NULL) {
2405 if (!opt_prof) {
2406 ret = ENOENT;
2407 goto label_return;
2408 }
2409 if (newlen != sizeof(bool)) {
2410 ret = EINVAL;
2411 goto label_return;
2412 }
2413 if (prof_thread_active_set(tsd, *(bool *)newp)) {
2414 ret = EAGAIN;
2415 goto label_return;
2416 }
2417 }
2418 READ(oldval, bool);
2419
2420 ret = 0;
2421label_return:
2422 return ret;
2423}
2424
2425static int
2426thread_idle_ctl(tsd_t *tsd, const size_t *mib,
2427 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2428 size_t newlen) {
2429 int ret;
2430
2431 NEITHER_READ_NOR_WRITE();
2432
2433 if (tcache_available(tsd)) {
2434 tcache_flush(tsd);
2435 }
2436 /*
2437 * This heuristic is perhaps not the most well-considered. But it
2438 * matches the only idling policy we have experience with in the status
2439 * quo. Over time we should investigate more principled approaches.
2440 */
2441 if (opt_narenas > ncpus * 2) {
2442 arena_t *arena = arena_choose(tsd, NULL);
2443 if (arena != NULL) {
2444 arena_decay(tsd_tsdn(tsd), arena, false, true);
2445 }
2446 /*
2447 * The missing arena case is not actually an error; a thread
2448 * might be idle before it associates itself to one. This is
2449 * unusual, but not wrong.
2450 */
2451 }
2452
2453 ret = 0;
2454label_return:
2455 return ret;
2456}
2457
2458/******************************************************************************/
2459
2460static int
2461tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2462 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2463 int ret;
2464 unsigned tcache_ind;
2465
2466 READONLY();
2467 VERIFY_READ(unsigned);
2468 if (tcaches_create(tsd, b0get(), &tcache_ind)) {
2469 ret = EFAULT;
2470 goto label_return;
2471 }
2472 READ(tcache_ind, unsigned);
2473
2474 ret = 0;
2475label_return:
2476 return ret;
2477}
2478
2479static int
2480tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2481 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2482 int ret;
2483 unsigned tcache_ind;
2484
2485 WRITEONLY();
2486 ASSURED_WRITE(tcache_ind, unsigned);
2487 tcaches_flush(tsd, tcache_ind);
2488
2489 ret = 0;
2490label_return:
2491 return ret;
2492}
2493
2494static int
2495tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2496 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2497 int ret;
2498 unsigned tcache_ind;
2499
2500 WRITEONLY();
2501 ASSURED_WRITE(tcache_ind, unsigned);
2502 tcaches_destroy(tsd, tcache_ind);
2503
2504 ret = 0;
2505label_return:
2506 return ret;
2507}
2508
2509/******************************************************************************/
2510
2511static int
2512arena_i_initialized_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2513 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2514 int ret;
2515 tsdn_t *tsdn = tsd_tsdn(tsd);
2516 unsigned arena_ind;
2517 bool initialized;
2518
2519 READONLY();
2520 MIB_UNSIGNED(arena_ind, 1);
2521
2522 malloc_mutex_lock(tsdn, &ctl_mtx);
2523 initialized = arenas_i(arena_ind)->initialized;
2524 malloc_mutex_unlock(tsdn, &ctl_mtx);
2525
2526 READ(initialized, bool);
2527
2528 ret = 0;
2529label_return:
2530 return ret;
2531}
2532
2533static void
2534arena_i_decay(tsdn_t *tsdn, unsigned arena_ind, bool all) {
2535 malloc_mutex_lock(tsdn, &ctl_mtx);
2536 {
2537 unsigned narenas = ctl_arenas->narenas;
2538
2539 /*
2540 * Access via index narenas is deprecated, and scheduled for
2541 * removal in 6.0.0.
2542 */
2543 if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind == narenas) {
2544 unsigned i;
2545 VARIABLE_ARRAY(arena_t *, tarenas, narenas);
2546
2547 for (i = 0; i < narenas; i++) {
2548 tarenas[i] = arena_get(tsdn, i, false);
2549 }
2550
2551 /*
2552 * No further need to hold ctl_mtx, since narenas and
2553 * tarenas contain everything needed below.
2554 */
2555 malloc_mutex_unlock(tsdn, &ctl_mtx);
2556
2557 for (i = 0; i < narenas; i++) {
2558 if (tarenas[i] != NULL) {
2559 arena_decay(tsdn, tarenas[i], false,
2560 all);
2561 }
2562 }
2563 } else {
2564 arena_t *tarena;
2565
2566 assert(arena_ind < narenas);
2567
2568 tarena = arena_get(tsdn, arena_ind, false);
2569
2570 /* No further need to hold ctl_mtx. */
2571 malloc_mutex_unlock(tsdn, &ctl_mtx);
2572
2573 if (tarena != NULL) {
2574 arena_decay(tsdn, tarena, false, all);
2575 }
2576 }
2577 }
2578}
2579
2580static int
2581arena_i_decay_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2582 size_t *oldlenp, void *newp, size_t newlen) {
2583 int ret;
2584 unsigned arena_ind;
2585
2586 NEITHER_READ_NOR_WRITE();
2587 MIB_UNSIGNED(arena_ind, 1);
2588 arena_i_decay(tsd_tsdn(tsd), arena_ind, false);
2589
2590 ret = 0;
2591label_return:
2592 return ret;
2593}
2594
2595static int
2596arena_i_purge_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2597 size_t *oldlenp, void *newp, size_t newlen) {
2598 int ret;
2599 unsigned arena_ind;
2600
2601 NEITHER_READ_NOR_WRITE();
2602 MIB_UNSIGNED(arena_ind, 1);
2603 arena_i_decay(tsd_tsdn(tsd), arena_ind, true);
2604
2605 ret = 0;
2606label_return:
2607 return ret;
2608}
2609
2610static int
2611arena_i_reset_destroy_helper(tsd_t *tsd, const size_t *mib, size_t miblen,
2612 void *oldp, size_t *oldlenp, void *newp, size_t newlen, unsigned *arena_ind,
2613 arena_t **arena) {
2614 int ret;
2615
2616 NEITHER_READ_NOR_WRITE();
2617 MIB_UNSIGNED(*arena_ind, 1);
2618
2619 *arena = arena_get(tsd_tsdn(tsd), *arena_ind, false);
2620 if (*arena == NULL || arena_is_auto(*arena)) {
2621 ret = EFAULT;
2622 goto label_return;
2623 }
2624
2625 ret = 0;
2626label_return:
2627 return ret;
2628}
2629
2630static void
2631arena_reset_prepare_background_thread(tsd_t *tsd, unsigned arena_ind) {
2632 /* Temporarily disable the background thread during arena reset. */
2633 if (have_background_thread) {
2634 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
2635 if (background_thread_enabled()) {
2636 background_thread_info_t *info =
2637 background_thread_info_get(arena_ind);
2638 assert(info->state == background_thread_started);
2639 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
2640 info->state = background_thread_paused;
2641 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
2642 }
2643 }
2644}
2645
2646static void
2647arena_reset_finish_background_thread(tsd_t *tsd, unsigned arena_ind) {
2648 if (have_background_thread) {
2649 if (background_thread_enabled()) {
2650 background_thread_info_t *info =
2651 background_thread_info_get(arena_ind);
2652 assert(info->state == background_thread_paused);
2653 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
2654 info->state = background_thread_started;
2655 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
2656 }
2657 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
2658 }
2659}
2660
2661static int
2662arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2663 size_t *oldlenp, void *newp, size_t newlen) {
2664 int ret;
2665 unsigned arena_ind;
2666 arena_t *arena;
2667
2668 ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp,
2669 newp, newlen, &arena_ind, &arena);
2670 if (ret != 0) {
2671 return ret;
2672 }
2673
2674 arena_reset_prepare_background_thread(tsd, arena_ind);
2675 arena_reset(tsd, arena);
2676 arena_reset_finish_background_thread(tsd, arena_ind);
2677
2678 return ret;
2679}
2680
2681static int
2682arena_i_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2683 size_t *oldlenp, void *newp, size_t newlen) {
2684 int ret;
2685 unsigned arena_ind;
2686 arena_t *arena;
2687 ctl_arena_t *ctl_darena, *ctl_arena;
2688
2689 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2690
2691 ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp,
2692 newp, newlen, &arena_ind, &arena);
2693 if (ret != 0) {
2694 goto label_return;
2695 }
2696
2697 if (arena_nthreads_get(arena, false) != 0 || arena_nthreads_get(arena,
2698 true) != 0) {
2699 ret = EFAULT;
2700 goto label_return;
2701 }
2702
2703 arena_reset_prepare_background_thread(tsd, arena_ind);
2704 /* Merge stats after resetting and purging arena. */
2705 arena_reset(tsd, arena);
2706 arena_decay(tsd_tsdn(tsd), arena, false, true);
2707 ctl_darena = arenas_i(MALLCTL_ARENAS_DESTROYED);
2708 ctl_darena->initialized = true;
2709 ctl_arena_refresh(tsd_tsdn(tsd), arena, ctl_darena, arena_ind, true);
2710 /* Destroy arena. */
2711 arena_destroy(tsd, arena);
2712 ctl_arena = arenas_i(arena_ind);
2713 ctl_arena->initialized = false;
2714 /* Record arena index for later recycling via arenas.create. */
2715 ql_elm_new(ctl_arena, destroyed_link);
2716 ql_tail_insert(&ctl_arenas->destroyed, ctl_arena, destroyed_link);
2717 arena_reset_finish_background_thread(tsd, arena_ind);
2718
2719 assert(ret == 0);
2720label_return:
2721 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2722
2723 return ret;
2724}
2725
2726static int
2727arena_i_dss_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2728 size_t *oldlenp, void *newp, size_t newlen) {
2729 int ret;
2730 const char *dss = NULL;
2731 unsigned arena_ind;
2732 dss_prec_t dss_prec_old = dss_prec_limit;
2733 dss_prec_t dss_prec = dss_prec_limit;
2734
2735 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2736 WRITE(dss, const char *);
2737 MIB_UNSIGNED(arena_ind, 1);
2738 if (dss != NULL) {
2739 int i;
2740 bool match = false;
2741
2742 for (i = 0; i < dss_prec_limit; i++) {
2743 if (strcmp(dss_prec_names[i], dss) == 0) {
2744 dss_prec = i;
2745 match = true;
2746 break;
2747 }
2748 }
2749
2750 if (!match) {
2751 ret = EINVAL;
2752 goto label_return;
2753 }
2754 }
2755
2756 /*
2757 * Access via index narenas is deprecated, and scheduled for removal in
2758 * 6.0.0.
2759 */
2760 if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind ==
2761 ctl_arenas->narenas) {
2762 if (dss_prec != dss_prec_limit &&
2763 extent_dss_prec_set(dss_prec)) {
2764 ret = EFAULT;
2765 goto label_return;
2766 }
2767 dss_prec_old = extent_dss_prec_get();
2768 } else {
2769 arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
2770 if (arena == NULL || (dss_prec != dss_prec_limit &&
2771 arena_dss_prec_set(arena, dss_prec))) {
2772 ret = EFAULT;
2773 goto label_return;
2774 }
2775 dss_prec_old = arena_dss_prec_get(arena);
2776 }
2777
2778 dss = dss_prec_names[dss_prec_old];
2779 READ(dss, const char *);
2780
2781 ret = 0;
2782label_return:
2783 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2784 return ret;
2785}
2786
2787static int
2788arena_i_oversize_threshold_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2789 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2790 int ret;
2791
2792 unsigned arena_ind;
2793 MIB_UNSIGNED(arena_ind, 1);
2794
2795 arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
2796 if (arena == NULL) {
2797 ret = EFAULT;
2798 goto label_return;
2799 }
2800
2801 if (oldp != NULL && oldlenp != NULL) {
2802 size_t oldval = atomic_load_zu(
2803 &arena->pa_shard.pac.oversize_threshold, ATOMIC_RELAXED);
2804 READ(oldval, size_t);
2805 }
2806 if (newp != NULL) {
2807 if (newlen != sizeof(size_t)) {
2808 ret = EINVAL;
2809 goto label_return;
2810 }
2811 atomic_store_zu(&arena->pa_shard.pac.oversize_threshold,
2812 *(size_t *)newp, ATOMIC_RELAXED);
2813 }
2814 ret = 0;
2815label_return:
2816 return ret;
2817}
2818
2819static int
2820arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen,
2821 void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) {
2822 int ret;
2823 unsigned arena_ind;
2824 arena_t *arena;
2825
2826 MIB_UNSIGNED(arena_ind, 1);
2827 arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
2828 if (arena == NULL) {
2829 ret = EFAULT;
2830 goto label_return;
2831 }
2832 extent_state_t state = dirty ? extent_state_dirty : extent_state_muzzy;
2833
2834 if (oldp != NULL && oldlenp != NULL) {
2835 size_t oldval = arena_decay_ms_get(arena, state);
2836 READ(oldval, ssize_t);
2837 }
2838 if (newp != NULL) {
2839 if (newlen != sizeof(ssize_t)) {
2840 ret = EINVAL;
2841 goto label_return;
2842 }
2843 if (arena_is_huge(arena_ind) && *(ssize_t *)newp > 0) {
2844 /*
2845 * By default the huge arena purges eagerly. If it is
2846 * set to non-zero decay time afterwards, background
2847 * thread might be needed.
2848 */
2849 if (background_thread_create(tsd, arena_ind)) {
2850 ret = EFAULT;
2851 goto label_return;
2852 }
2853 }
2854
2855 if (arena_decay_ms_set(tsd_tsdn(tsd), arena, state,
2856 *(ssize_t *)newp)) {
2857 ret = EFAULT;
2858 goto label_return;
2859 }
2860 }
2861
2862 ret = 0;
2863label_return:
2864 return ret;
2865}
2866
2867static int
2868arena_i_dirty_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2869 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2870 return arena_i_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp,
2871 newlen, true);
2872}
2873
2874static int
2875arena_i_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2876 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2877 return arena_i_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp,
2878 newlen, false);
2879}
2880
2881static int
2882arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2883 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2884 int ret;
2885 unsigned arena_ind;
2886 arena_t *arena;
2887
2888 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2889 MIB_UNSIGNED(arena_ind, 1);
2890 if (arena_ind < narenas_total_get()) {
2891 extent_hooks_t *old_extent_hooks;
2892 arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
2893 if (arena == NULL) {
2894 if (arena_ind >= narenas_auto) {
2895 ret = EFAULT;
2896 goto label_return;
2897 }
2898 old_extent_hooks =
2899 (extent_hooks_t *)&ehooks_default_extent_hooks;
2900 READ(old_extent_hooks, extent_hooks_t *);
2901 if (newp != NULL) {
2902 /* Initialize a new arena as a side effect. */
2903 extent_hooks_t *new_extent_hooks
2904 JEMALLOC_CC_SILENCE_INIT(NULL);
2905 WRITE(new_extent_hooks, extent_hooks_t *);
2906 arena_config_t config = arena_config_default;
2907 config.extent_hooks = new_extent_hooks;
2908
2909 arena = arena_init(tsd_tsdn(tsd), arena_ind,
2910 &config);
2911 if (arena == NULL) {
2912 ret = EFAULT;
2913 goto label_return;
2914 }
2915 }
2916 } else {
2917 if (newp != NULL) {
2918 extent_hooks_t *new_extent_hooks
2919 JEMALLOC_CC_SILENCE_INIT(NULL);
2920 WRITE(new_extent_hooks, extent_hooks_t *);
2921 old_extent_hooks = arena_set_extent_hooks(tsd,
2922 arena, new_extent_hooks);
2923 READ(old_extent_hooks, extent_hooks_t *);
2924 } else {
2925 old_extent_hooks =
2926 ehooks_get_extent_hooks_ptr(
2927 arena_get_ehooks(arena));
2928 READ(old_extent_hooks, extent_hooks_t *);
2929 }
2930 }
2931 } else {
2932 ret = EFAULT;
2933 goto label_return;
2934 }
2935 ret = 0;
2936label_return:
2937 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2938 return ret;
2939}
2940
2941static int
2942arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib,
2943 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2944 size_t newlen) {
2945 int ret;
2946 unsigned arena_ind;
2947 arena_t *arena;
2948
2949 if (!opt_retain) {
2950 /* Only relevant when retain is enabled. */
2951 return ENOENT;
2952 }
2953
2954 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2955 MIB_UNSIGNED(arena_ind, 1);
2956 if (arena_ind < narenas_total_get() && (arena =
2957 arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) {
2958 size_t old_limit, new_limit;
2959 if (newp != NULL) {
2960 WRITE(new_limit, size_t);
2961 }
2962 bool err = arena_retain_grow_limit_get_set(tsd, arena,
2963 &old_limit, newp != NULL ? &new_limit : NULL);
2964 if (!err) {
2965 READ(old_limit, size_t);
2966 ret = 0;
2967 } else {
2968 ret = EFAULT;
2969 }
2970 } else {
2971 ret = EFAULT;
2972 }
2973label_return:
2974 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2975 return ret;
2976}
2977
2978static const ctl_named_node_t *
2979arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen,
2980 size_t i) {
2981 const ctl_named_node_t *ret;
2982
2983 malloc_mutex_lock(tsdn, &ctl_mtx);
2984 switch (i) {
2985 case MALLCTL_ARENAS_ALL:
2986 case MALLCTL_ARENAS_DESTROYED:
2987 break;
2988 default:
2989 if (i > ctl_arenas->narenas) {
2990 ret = NULL;
2991 goto label_return;
2992 }
2993 break;
2994 }
2995
2996 ret = super_arena_i_node;
2997label_return:
2998 malloc_mutex_unlock(tsdn, &ctl_mtx);
2999 return ret;
3000}
3001
3002/******************************************************************************/
3003
3004static int
3005arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3006 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3007 int ret;
3008 unsigned narenas;
3009
3010 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
3011 READONLY();
3012 narenas = ctl_arenas->narenas;
3013 READ(narenas, unsigned);
3014
3015 ret = 0;
3016label_return:
3017 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
3018 return ret;
3019}
3020
3021static int
3022arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib,
3023 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
3024 size_t newlen, bool dirty) {
3025 int ret;
3026
3027 if (oldp != NULL && oldlenp != NULL) {
3028 size_t oldval = (dirty ? arena_dirty_decay_ms_default_get() :
3029 arena_muzzy_decay_ms_default_get());
3030 READ(oldval, ssize_t);
3031 }
3032 if (newp != NULL) {
3033 if (newlen != sizeof(ssize_t)) {
3034 ret = EINVAL;
3035 goto label_return;
3036 }
3037 if (dirty ? arena_dirty_decay_ms_default_set(*(ssize_t *)newp)
3038 : arena_muzzy_decay_ms_default_set(*(ssize_t *)newp)) {
3039 ret = EFAULT;
3040 goto label_return;
3041 }
3042 }
3043
3044 ret = 0;
3045label_return:
3046 return ret;
3047}
3048
3049static int
3050arenas_dirty_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3051 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3052 return arenas_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp,
3053 newlen, true);
3054}
3055
3056static int
3057arenas_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3058 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3059 return arenas_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp,
3060 newlen, false);
3061}
3062
3063CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t)
3064CTL_RO_NL_GEN(arenas_page, PAGE, size_t)
3065CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t)
3066CTL_RO_NL_GEN(arenas_nbins, SC_NBINS, unsigned)
3067CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned)
3068CTL_RO_NL_GEN(arenas_bin_i_size, bin_infos[mib[2]].reg_size, size_t)
3069CTL_RO_NL_GEN(arenas_bin_i_nregs, bin_infos[mib[2]].nregs, uint32_t)
3070CTL_RO_NL_GEN(arenas_bin_i_slab_size, bin_infos[mib[2]].slab_size, size_t)
3071CTL_RO_NL_GEN(arenas_bin_i_nshards, bin_infos[mib[2]].n_shards, uint32_t)
3072static const ctl_named_node_t *
3073arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib,
3074 size_t miblen, size_t i) {
3075 if (i > SC_NBINS) {
3076 return NULL;
3077 }
3078 return super_arenas_bin_i_node;
3079}
3080
3081CTL_RO_NL_GEN(arenas_nlextents, SC_NSIZES - SC_NBINS, unsigned)
3082CTL_RO_NL_GEN(arenas_lextent_i_size, sz_index2size(SC_NBINS+(szind_t)mib[2]),
3083 size_t)
3084static const ctl_named_node_t *
3085arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib,
3086 size_t miblen, size_t i) {
3087 if (i > SC_NSIZES - SC_NBINS) {
3088 return NULL;
3089 }
3090 return super_arenas_lextent_i_node;
3091}
3092
3093static int
3094arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3095 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3096 int ret;
3097 unsigned arena_ind;
3098
3099 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
3100
3101 VERIFY_READ(unsigned);
3102 arena_config_t config = arena_config_default;
3103 WRITE(config.extent_hooks, extent_hooks_t *);
3104 if ((arena_ind = ctl_arena_init(tsd, &config)) == UINT_MAX) {
3105 ret = EAGAIN;
3106 goto label_return;
3107 }
3108 READ(arena_ind, unsigned);
3109
3110 ret = 0;
3111label_return:
3112 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
3113 return ret;
3114}
3115
3116static int
3117experimental_arenas_create_ext_ctl(tsd_t *tsd,
3118 const size_t *mib, size_t miblen,
3119 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3120 int ret;
3121 unsigned arena_ind;
3122
3123 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
3124
3125 arena_config_t config = arena_config_default;
3126 VERIFY_READ(unsigned);
3127 WRITE(config, arena_config_t);
3128
3129 if ((arena_ind = ctl_arena_init(tsd, &config)) == UINT_MAX) {
3130 ret = EAGAIN;
3131 goto label_return;
3132 }
3133 READ(arena_ind, unsigned);
3134 ret = 0;
3135label_return:
3136 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
3137 return ret;
3138}
3139
3140static int
3141arenas_lookup_ctl(tsd_t *tsd, const size_t *mib,
3142 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
3143 size_t newlen) {
3144 int ret;
3145 unsigned arena_ind;
3146 void *ptr;
3147 edata_t *edata;
3148 arena_t *arena;
3149
3150 ptr = NULL;
3151 ret = EINVAL;
3152 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
3153 WRITE(ptr, void *);
3154 edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr);
3155 if (edata == NULL) {
3156 goto label_return;
3157 }
3158
3159 arena = arena_get_from_edata(edata);
3160 if (arena == NULL) {
3161 goto label_return;
3162 }
3163
3164 arena_ind = arena_ind_get(arena);
3165 READ(arena_ind, unsigned);
3166
3167 ret = 0;
3168label_return:
3169 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
3170 return ret;
3171}
3172
3173/******************************************************************************/
3174
3175static int
3176prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib,
3177 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
3178 size_t newlen) {
3179 int ret;
3180 bool oldval;
3181
3182 if (!config_prof) {
3183 return ENOENT;
3184 }
3185
3186 if (newp != NULL) {
3187 if (!opt_prof) {
3188 ret = ENOENT;
3189 goto label_return;
3190 }
3191 if (newlen != sizeof(bool)) {
3192 ret = EINVAL;
3193 goto label_return;
3194 }
3195 oldval = prof_thread_active_init_set(tsd_tsdn(tsd),
3196 *(bool *)newp);
3197 } else {
3198 oldval = opt_prof ? prof_thread_active_init_get(tsd_tsdn(tsd)) :
3199 false;
3200 }
3201 READ(oldval, bool);
3202
3203 ret = 0;
3204label_return:
3205 return ret;
3206}
3207
3208static int
3209prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3210 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3211 int ret;
3212 bool oldval;
3213
3214 if (!config_prof) {
3215 ret = ENOENT;
3216 goto label_return;
3217 }
3218
3219 if (newp != NULL) {
3220 if (newlen != sizeof(bool)) {
3221 ret = EINVAL;
3222 goto label_return;
3223 }
3224 bool val = *(bool *)newp;
3225 if (!opt_prof) {
3226 if (val) {
3227 ret = ENOENT;
3228 goto label_return;
3229 } else {
3230 /* No change needed (already off). */
3231 oldval = false;
3232 }
3233 } else {
3234 oldval = prof_active_set(tsd_tsdn(tsd), val);
3235 }
3236 } else {
3237 oldval = opt_prof ? prof_active_get(tsd_tsdn(tsd)) : false;
3238 }
3239 READ(oldval, bool);
3240
3241 ret = 0;
3242label_return:
3243 return ret;
3244}
3245
3246static int
3247prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3248 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3249 int ret;
3250 const char *filename = NULL;
3251
3252 if (!config_prof || !opt_prof) {
3253 return ENOENT;
3254 }
3255
3256 WRITEONLY();
3257 WRITE(filename, const char *);
3258
3259 if (prof_mdump(tsd, filename)) {
3260 ret = EFAULT;
3261 goto label_return;
3262 }
3263
3264 ret = 0;
3265label_return:
3266 return ret;
3267}
3268
3269static int
3270prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3271 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3272 int ret;
3273 bool oldval;
3274
3275 if (!config_prof) {
3276 return ENOENT;
3277 }
3278
3279 if (newp != NULL) {
3280 if (!opt_prof) {
3281 ret = ENOENT;
3282 goto label_return;
3283 }
3284 if (newlen != sizeof(bool)) {
3285 ret = EINVAL;
3286 goto label_return;
3287 }
3288 oldval = prof_gdump_set(tsd_tsdn(tsd), *(bool *)newp);
3289 } else {
3290 oldval = opt_prof ? prof_gdump_get(tsd_tsdn(tsd)) : false;
3291 }
3292 READ(oldval, bool);
3293
3294 ret = 0;
3295label_return:
3296 return ret;
3297}
3298
3299static int
3300prof_prefix_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3301 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3302 int ret;
3303 const char *prefix = NULL;
3304
3305 if (!config_prof || !opt_prof) {
3306 return ENOENT;
3307 }
3308
3309 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
3310 WRITEONLY();
3311 WRITE(prefix, const char *);
3312
3313 ret = prof_prefix_set(tsd_tsdn(tsd), prefix) ? EFAULT : 0;
3314label_return:
3315 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
3316 return ret;
3317}
3318
3319static int
3320prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3321 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3322 int ret;
3323 size_t lg_sample = lg_prof_sample;
3324
3325 if (!config_prof || !opt_prof) {
3326 return ENOENT;
3327 }
3328
3329 WRITEONLY();
3330 WRITE(lg_sample, size_t);
3331 if (lg_sample >= (sizeof(uint64_t) << 3)) {
3332 lg_sample = (sizeof(uint64_t) << 3) - 1;
3333 }
3334
3335 prof_reset(tsd, lg_sample);
3336
3337 ret = 0;
3338label_return:
3339 return ret;
3340}
3341
3342CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t)
3343CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t)
3344
3345static int
3346prof_log_start_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
3347 size_t *oldlenp, void *newp, size_t newlen) {
3348 int ret;
3349
3350 const char *filename = NULL;
3351
3352 if (!config_prof || !opt_prof) {
3353 return ENOENT;
3354 }
3355
3356 WRITEONLY();
3357 WRITE(filename, const char *);
3358
3359 if (prof_log_start(tsd_tsdn(tsd), filename)) {
3360 ret = EFAULT;
3361 goto label_return;
3362 }
3363
3364 ret = 0;
3365label_return:
3366 return ret;
3367}
3368
3369static int
3370prof_log_stop_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
3371 size_t *oldlenp, void *newp, size_t newlen) {
3372 if (!config_prof || !opt_prof) {
3373 return ENOENT;
3374 }
3375
3376 if (prof_log_stop(tsd_tsdn(tsd))) {
3377 return EFAULT;
3378 }
3379
3380 return 0;
3381}
3382
3383static int
3384experimental_hooks_prof_backtrace_ctl(tsd_t *tsd, const size_t *mib,
3385 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3386 int ret;
3387
3388 if (oldp == NULL && newp == NULL) {
3389 ret = EINVAL;
3390 goto label_return;
3391 }
3392 if (oldp != NULL) {
3393 prof_backtrace_hook_t old_hook =
3394 prof_backtrace_hook_get();
3395 READ(old_hook, prof_backtrace_hook_t);
3396 }
3397 if (newp != NULL) {
3398 if (!opt_prof) {
3399 ret = ENOENT;
3400 goto label_return;
3401 }
3402 prof_backtrace_hook_t new_hook JEMALLOC_CC_SILENCE_INIT(NULL);
3403 WRITE(new_hook, prof_backtrace_hook_t);
3404 if (new_hook == NULL) {
3405 ret = EINVAL;
3406 goto label_return;
3407 }
3408 prof_backtrace_hook_set(new_hook);
3409 }
3410 ret = 0;
3411label_return:
3412 return ret;
3413}
3414
3415static int
3416experimental_hooks_prof_dump_ctl(tsd_t *tsd, const size_t *mib,
3417 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3418 int ret;
3419
3420 if (oldp == NULL && newp == NULL) {
3421 ret = EINVAL;
3422 goto label_return;
3423 }
3424 if (oldp != NULL) {
3425 prof_dump_hook_t old_hook =
3426 prof_dump_hook_get();
3427 READ(old_hook, prof_dump_hook_t);
3428 }
3429 if (newp != NULL) {
3430 if (!opt_prof) {
3431 ret = ENOENT;
3432 goto label_return;
3433 }
3434 prof_dump_hook_t new_hook JEMALLOC_CC_SILENCE_INIT(NULL);
3435 WRITE(new_hook, prof_dump_hook_t);
3436 prof_dump_hook_set(new_hook);
3437 }
3438 ret = 0;
3439label_return:
3440 return ret;
3441}
3442
3443/* For integration test purpose only. No plan to move out of experimental. */
3444static int
3445experimental_hooks_safety_check_abort_ctl(tsd_t *tsd, const size_t *mib,
3446 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3447 int ret;
3448
3449 WRITEONLY();
3450 if (newp != NULL) {
3451 if (newlen != sizeof(safety_check_abort_hook_t)) {
3452 ret = EINVAL;
3453 goto label_return;
3454 }
3455 safety_check_abort_hook_t hook JEMALLOC_CC_SILENCE_INIT(NULL);
3456 WRITE(hook, safety_check_abort_hook_t);
3457 safety_check_set_abort(hook);
3458 }
3459 ret = 0;
3460label_return:
3461 return ret;
3462}
3463
3464/******************************************************************************/
3465
3466CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t)
3467CTL_RO_CGEN(config_stats, stats_active, ctl_stats->active, size_t)
3468CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats->metadata, size_t)
3469CTL_RO_CGEN(config_stats, stats_metadata_thp, ctl_stats->metadata_thp, size_t)
3470CTL_RO_CGEN(config_stats, stats_resident, ctl_stats->resident, size_t)
3471CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats->mapped, size_t)
3472CTL_RO_CGEN(config_stats, stats_retained, ctl_stats->retained, size_t)
3473
3474CTL_RO_CGEN(config_stats, stats_background_thread_num_threads,
3475 ctl_stats->background_thread.num_threads, size_t)
3476CTL_RO_CGEN(config_stats, stats_background_thread_num_runs,
3477 ctl_stats->background_thread.num_runs, uint64_t)
3478CTL_RO_CGEN(config_stats, stats_background_thread_run_interval,
3479 nstime_ns(&ctl_stats->background_thread.run_interval), uint64_t)
3480
3481CTL_RO_CGEN(config_stats, stats_zero_reallocs,
3482 atomic_load_zu(&zero_realloc_count, ATOMIC_RELAXED), size_t)
3483
3484CTL_RO_GEN(stats_arenas_i_dss, arenas_i(mib[2])->dss, const char *)
3485CTL_RO_GEN(stats_arenas_i_dirty_decay_ms, arenas_i(mib[2])->dirty_decay_ms,
3486 ssize_t)
3487CTL_RO_GEN(stats_arenas_i_muzzy_decay_ms, arenas_i(mib[2])->muzzy_decay_ms,
3488 ssize_t)
3489CTL_RO_GEN(stats_arenas_i_nthreads, arenas_i(mib[2])->nthreads, unsigned)
3490CTL_RO_GEN(stats_arenas_i_uptime,
3491 nstime_ns(&arenas_i(mib[2])->astats->astats.uptime), uint64_t)
3492CTL_RO_GEN(stats_arenas_i_pactive, arenas_i(mib[2])->pactive, size_t)
3493CTL_RO_GEN(stats_arenas_i_pdirty, arenas_i(mib[2])->pdirty, size_t)
3494CTL_RO_GEN(stats_arenas_i_pmuzzy, arenas_i(mib[2])->pmuzzy, size_t)
3495CTL_RO_CGEN(config_stats, stats_arenas_i_mapped,
3496 arenas_i(mib[2])->astats->astats.mapped, size_t)
3497CTL_RO_CGEN(config_stats, stats_arenas_i_retained,
3498 arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.retained, size_t)
3499CTL_RO_CGEN(config_stats, stats_arenas_i_extent_avail,
3500 arenas_i(mib[2])->astats->astats.pa_shard_stats.edata_avail, size_t)
3501
3502CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_npurge,
3503 locked_read_u64_unsynchronized(
3504 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_dirty.npurge),
3505 uint64_t)
3506CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_nmadvise,
3507 locked_read_u64_unsynchronized(
3508 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_dirty.nmadvise),
3509 uint64_t)
3510CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_purged,
3511 locked_read_u64_unsynchronized(
3512 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_dirty.purged),
3513 uint64_t)
3514
3515CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_npurge,
3516 locked_read_u64_unsynchronized(
3517 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.npurge),
3518 uint64_t)
3519CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_nmadvise,
3520 locked_read_u64_unsynchronized(
3521 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.nmadvise),
3522 uint64_t)
3523CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_purged,
3524 locked_read_u64_unsynchronized(
3525 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.purged),
3526 uint64_t)
3527
3528CTL_RO_CGEN(config_stats, stats_arenas_i_base,
3529 arenas_i(mib[2])->astats->astats.base,
3530 size_t)
3531CTL_RO_CGEN(config_stats, stats_arenas_i_internal,
3532 atomic_load_zu(&arenas_i(mib[2])->astats->astats.internal, ATOMIC_RELAXED),
3533 size_t)
3534CTL_RO_CGEN(config_stats, stats_arenas_i_metadata_thp,
3535 arenas_i(mib[2])->astats->astats.metadata_thp, size_t)
3536CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_bytes,
3537 arenas_i(mib[2])->astats->astats.tcache_bytes, size_t)
3538CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_stashed_bytes,
3539 arenas_i(mib[2])->astats->astats.tcache_stashed_bytes, size_t)
3540CTL_RO_CGEN(config_stats, stats_arenas_i_resident,
3541 arenas_i(mib[2])->astats->astats.resident,
3542 size_t)
3543CTL_RO_CGEN(config_stats, stats_arenas_i_abandoned_vm,
3544 atomic_load_zu(
3545 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.abandoned_vm,
3546 ATOMIC_RELAXED), size_t)
3547
3548CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_bytes,
3549 arenas_i(mib[2])->astats->secstats.bytes, size_t)
3550
3551CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated,
3552 arenas_i(mib[2])->astats->allocated_small, size_t)
3553CTL_RO_CGEN(config_stats, stats_arenas_i_small_nmalloc,
3554 arenas_i(mib[2])->astats->nmalloc_small, uint64_t)
3555CTL_RO_CGEN(config_stats, stats_arenas_i_small_ndalloc,
3556 arenas_i(mib[2])->astats->ndalloc_small, uint64_t)
3557CTL_RO_CGEN(config_stats, stats_arenas_i_small_nrequests,
3558 arenas_i(mib[2])->astats->nrequests_small, uint64_t)
3559CTL_RO_CGEN(config_stats, stats_arenas_i_small_nfills,
3560 arenas_i(mib[2])->astats->nfills_small, uint64_t)
3561CTL_RO_CGEN(config_stats, stats_arenas_i_small_nflushes,
3562 arenas_i(mib[2])->astats->nflushes_small, uint64_t)
3563CTL_RO_CGEN(config_stats, stats_arenas_i_large_allocated,
3564 arenas_i(mib[2])->astats->astats.allocated_large, size_t)
3565CTL_RO_CGEN(config_stats, stats_arenas_i_large_nmalloc,
3566 arenas_i(mib[2])->astats->astats.nmalloc_large, uint64_t)
3567CTL_RO_CGEN(config_stats, stats_arenas_i_large_ndalloc,
3568 arenas_i(mib[2])->astats->astats.ndalloc_large, uint64_t)
3569CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests,
3570 arenas_i(mib[2])->astats->astats.nrequests_large, uint64_t)
3571/*
3572 * Note: "nmalloc_large" here instead of "nfills" in the read. This is
3573 * intentional (large has no batch fill).
3574 */
3575CTL_RO_CGEN(config_stats, stats_arenas_i_large_nfills,
3576 arenas_i(mib[2])->astats->astats.nmalloc_large, uint64_t)
3577CTL_RO_CGEN(config_stats, stats_arenas_i_large_nflushes,
3578 arenas_i(mib[2])->astats->astats.nflushes_large, uint64_t)
3579
3580/* Lock profiling related APIs below. */
3581#define RO_MUTEX_CTL_GEN(n, l) \
3582CTL_RO_CGEN(config_stats, stats_##n##_num_ops, \
3583 l.n_lock_ops, uint64_t) \
3584CTL_RO_CGEN(config_stats, stats_##n##_num_wait, \
3585 l.n_wait_times, uint64_t) \
3586CTL_RO_CGEN(config_stats, stats_##n##_num_spin_acq, \
3587 l.n_spin_acquired, uint64_t) \
3588CTL_RO_CGEN(config_stats, stats_##n##_num_owner_switch, \
3589 l.n_owner_switches, uint64_t) \
3590CTL_RO_CGEN(config_stats, stats_##n##_total_wait_time, \
3591 nstime_ns(&l.tot_wait_time), uint64_t) \
3592CTL_RO_CGEN(config_stats, stats_##n##_max_wait_time, \
3593 nstime_ns(&l.max_wait_time), uint64_t) \
3594CTL_RO_CGEN(config_stats, stats_##n##_max_num_thds, \
3595 l.max_n_thds, uint32_t)
3596
3597/* Global mutexes. */
3598#define OP(mtx) \
3599 RO_MUTEX_CTL_GEN(mutexes_##mtx, \
3600 ctl_stats->mutex_prof_data[global_prof_mutex_##mtx])
3601MUTEX_PROF_GLOBAL_MUTEXES
3602#undef OP
3603
3604/* Per arena mutexes */
3605#define OP(mtx) RO_MUTEX_CTL_GEN(arenas_i_mutexes_##mtx, \
3606 arenas_i(mib[2])->astats->astats.mutex_prof_data[arena_prof_mutex_##mtx])
3607MUTEX_PROF_ARENA_MUTEXES
3608#undef OP
3609
3610/* tcache bin mutex */
3611RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex,
3612 arenas_i(mib[2])->astats->bstats[mib[4]].mutex_data)
3613#undef RO_MUTEX_CTL_GEN
3614
3615/* Resets all mutex stats, including global, arena and bin mutexes. */
3616static int
3617stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib,
3618 size_t miblen, void *oldp, size_t *oldlenp,
3619 void *newp, size_t newlen) {
3620 if (!config_stats) {
3621 return ENOENT;
3622 }
3623
3624 tsdn_t *tsdn = tsd_tsdn(tsd);
3625
3626#define MUTEX_PROF_RESET(mtx) \
3627 malloc_mutex_lock(tsdn, &mtx); \
3628 malloc_mutex_prof_data_reset(tsdn, &mtx); \
3629 malloc_mutex_unlock(tsdn, &mtx);
3630
3631 /* Global mutexes: ctl and prof. */
3632 MUTEX_PROF_RESET(ctl_mtx);
3633 if (have_background_thread) {
3634 MUTEX_PROF_RESET(background_thread_lock);
3635 }
3636 if (config_prof && opt_prof) {
3637 MUTEX_PROF_RESET(bt2gctx_mtx);
3638 MUTEX_PROF_RESET(tdatas_mtx);
3639 MUTEX_PROF_RESET(prof_dump_mtx);
3640 MUTEX_PROF_RESET(prof_recent_alloc_mtx);
3641 MUTEX_PROF_RESET(prof_recent_dump_mtx);
3642 MUTEX_PROF_RESET(prof_stats_mtx);
3643 }
3644
3645 /* Per arena mutexes. */
3646 unsigned n = narenas_total_get();
3647
3648 for (unsigned i = 0; i < n; i++) {
3649 arena_t *arena = arena_get(tsdn, i, false);
3650 if (!arena) {
3651 continue;
3652 }
3653 MUTEX_PROF_RESET(arena->large_mtx);
3654 MUTEX_PROF_RESET(arena->pa_shard.edata_cache.mtx);
3655 MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_dirty.mtx);
3656 MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_muzzy.mtx);
3657 MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_retained.mtx);
3658 MUTEX_PROF_RESET(arena->pa_shard.pac.decay_dirty.mtx);
3659 MUTEX_PROF_RESET(arena->pa_shard.pac.decay_muzzy.mtx);
3660 MUTEX_PROF_RESET(arena->tcache_ql_mtx);
3661 MUTEX_PROF_RESET(arena->base->mtx);
3662
3663 for (szind_t j = 0; j < SC_NBINS; j++) {
3664 for (unsigned k = 0; k < bin_infos[j].n_shards; k++) {
3665 bin_t *bin = arena_get_bin(arena, j, k);
3666 MUTEX_PROF_RESET(bin->lock);
3667 }
3668 }
3669 }
3670#undef MUTEX_PROF_RESET
3671 return 0;
3672}
3673
3674CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc,
3675 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nmalloc, uint64_t)
3676CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_ndalloc,
3677 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.ndalloc, uint64_t)
3678CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nrequests,
3679 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nrequests, uint64_t)
3680CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curregs,
3681 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.curregs, size_t)
3682CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nfills,
3683 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nfills, uint64_t)
3684CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nflushes,
3685 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nflushes, uint64_t)
3686CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nslabs,
3687 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nslabs, uint64_t)
3688CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreslabs,
3689 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.reslabs, uint64_t)
3690CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curslabs,
3691 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.curslabs, size_t)
3692CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nonfull_slabs,
3693 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nonfull_slabs, size_t)
3694
3695static const ctl_named_node_t *
3696stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib,
3697 size_t miblen, size_t j) {
3698 if (j > SC_NBINS) {
3699 return NULL;
3700 }
3701 return super_stats_arenas_i_bins_j_node;
3702}
3703
3704CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nmalloc,
3705 locked_read_u64_unsynchronized(
3706 &arenas_i(mib[2])->astats->lstats[mib[4]].nmalloc), uint64_t)
3707CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_ndalloc,
3708 locked_read_u64_unsynchronized(
3709 &arenas_i(mib[2])->astats->lstats[mib[4]].ndalloc), uint64_t)
3710CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nrequests,
3711 locked_read_u64_unsynchronized(
3712 &arenas_i(mib[2])->astats->lstats[mib[4]].nrequests), uint64_t)
3713CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents,
3714 arenas_i(mib[2])->astats->lstats[mib[4]].curlextents, size_t)
3715
3716static const ctl_named_node_t *
3717stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib,
3718 size_t miblen, size_t j) {
3719 if (j > SC_NSIZES - SC_NBINS) {
3720 return NULL;
3721 }
3722 return super_stats_arenas_i_lextents_j_node;
3723}
3724
3725CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_ndirty,
3726 arenas_i(mib[2])->astats->estats[mib[4]].ndirty, size_t);
3727CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nmuzzy,
3728 arenas_i(mib[2])->astats->estats[mib[4]].nmuzzy, size_t);
3729CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nretained,
3730 arenas_i(mib[2])->astats->estats[mib[4]].nretained, size_t);
3731CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_dirty_bytes,
3732 arenas_i(mib[2])->astats->estats[mib[4]].dirty_bytes, size_t);
3733CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_muzzy_bytes,
3734 arenas_i(mib[2])->astats->estats[mib[4]].muzzy_bytes, size_t);
3735CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_retained_bytes,
3736 arenas_i(mib[2])->astats->estats[mib[4]].retained_bytes, size_t);
3737
3738static const ctl_named_node_t *
3739stats_arenas_i_extents_j_index(tsdn_t *tsdn, const size_t *mib,
3740 size_t miblen, size_t j) {
3741 if (j >= SC_NPSIZES) {
3742 return NULL;
3743 }
3744 return super_stats_arenas_i_extents_j_node;
3745}
3746
3747CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_npurge_passes,
3748 arenas_i(mib[2])->astats->hpastats.nonderived_stats.npurge_passes, uint64_t);
3749CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_npurges,
3750 arenas_i(mib[2])->astats->hpastats.nonderived_stats.npurges, uint64_t);
3751CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nhugifies,
3752 arenas_i(mib[2])->astats->hpastats.nonderived_stats.nhugifies, uint64_t);
3753CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_ndehugifies,
3754 arenas_i(mib[2])->astats->hpastats.nonderived_stats.ndehugifies, uint64_t);
3755
3756/* Full, nonhuge */
3757CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge,
3758 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].npageslabs,
3759 size_t);
3760CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge,
3761 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].nactive, size_t);
3762CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ndirty_nonhuge,
3763 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].ndirty, size_t);
3764
3765/* Full, huge */
3766CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge,
3767 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].npageslabs,
3768 size_t);
3769CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_huge,
3770 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].nactive, size_t);
3771CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ndirty_huge,
3772 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].ndirty, size_t);
3773
3774/* Empty, nonhuge */
3775CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_npageslabs_nonhuge,
3776 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].npageslabs,
3777 size_t);
3778CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_nactive_nonhuge,
3779 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].nactive, size_t);
3780CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_ndirty_nonhuge,
3781 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].ndirty, size_t);
3782
3783/* Empty, huge */
3784CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_npageslabs_huge,
3785 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].npageslabs,
3786 size_t);
3787CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_nactive_huge,
3788 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].nactive, size_t);
3789CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_ndirty_huge,
3790 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].ndirty, size_t);
3791
3792/* Nonfull, nonhuge */
3793CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge,
3794 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].npageslabs,
3795 size_t);
3796CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge,
3797 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].nactive,
3798 size_t);
3799CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge,
3800 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].ndirty,
3801 size_t);
3802
3803/* Nonfull, huge */
3804CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge,
3805 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].npageslabs,
3806 size_t);
3807CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge,
3808 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].nactive,
3809 size_t);
3810CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge,
3811 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].ndirty,
3812 size_t);
3813
3814static const ctl_named_node_t *
3815stats_arenas_i_hpa_shard_nonfull_slabs_j_index(tsdn_t *tsdn, const size_t *mib,
3816 size_t miblen, size_t j) {
3817 if (j >= PSSET_NPSIZES) {
3818 return NULL;
3819 }
3820 return super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node;
3821}
3822
3823static bool
3824ctl_arenas_i_verify(size_t i) {
3825 size_t a = arenas_i2a_impl(i, true, true);
3826 if (a == UINT_MAX || !ctl_arenas->arenas[a]->initialized) {
3827 return true;
3828 }
3829
3830 return false;
3831}
3832
3833static const ctl_named_node_t *
3834stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib,
3835 size_t miblen, size_t i) {
3836 const ctl_named_node_t *ret;
3837
3838 malloc_mutex_lock(tsdn, &ctl_mtx);
3839 if (ctl_arenas_i_verify(i)) {
3840 ret = NULL;
3841 goto label_return;
3842 }
3843
3844 ret = super_stats_arenas_i_node;
3845label_return:
3846 malloc_mutex_unlock(tsdn, &ctl_mtx);
3847 return ret;
3848}
3849
3850static int
3851experimental_hooks_install_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3852 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3853 int ret;
3854 if (oldp == NULL || oldlenp == NULL|| newp == NULL) {
3855 ret = EINVAL;
3856 goto label_return;
3857 }
3858 /*
3859 * Note: this is a *private* struct. This is an experimental interface;
3860 * forcing the user to know the jemalloc internals well enough to
3861 * extract the ABI hopefully ensures nobody gets too comfortable with
3862 * this API, which can change at a moment's notice.
3863 */
3864 hooks_t hooks;
3865 WRITE(hooks, hooks_t);
3866 void *handle = hook_install(tsd_tsdn(tsd), &hooks);
3867 if (handle == NULL) {
3868 ret = EAGAIN;
3869 goto label_return;
3870 }
3871 READ(handle, void *);
3872
3873 ret = 0;
3874label_return:
3875 return ret;
3876}
3877
3878static int
3879experimental_hooks_remove_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3880 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3881 int ret;
3882 WRITEONLY();
3883 void *handle = NULL;
3884 WRITE(handle, void *);
3885 if (handle == NULL) {
3886 ret = EINVAL;
3887 goto label_return;
3888 }
3889 hook_remove(tsd_tsdn(tsd), handle);
3890 ret = 0;
3891label_return:
3892 return ret;
3893}
3894
3895static int
3896experimental_thread_activity_callback_ctl(tsd_t *tsd, const size_t *mib,
3897 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3898 int ret;
3899
3900 if (!config_stats) {
3901 return ENOENT;
3902 }
3903
3904 activity_callback_thunk_t t_old = tsd_activity_callback_thunk_get(tsd);
3905 READ(t_old, activity_callback_thunk_t);
3906
3907 if (newp != NULL) {
3908 /*
3909 * This initialization is unnecessary. If it's omitted, though,
3910 * clang gets confused and warns on the subsequent use of t_new.
3911 */
3912 activity_callback_thunk_t t_new = {NULL, NULL};
3913 WRITE(t_new, activity_callback_thunk_t);
3914 tsd_activity_callback_thunk_set(tsd, t_new);
3915 }
3916 ret = 0;
3917label_return:
3918 return ret;
3919}
3920
3921/*
3922 * Output six memory utilization entries for an input pointer, the first one of
3923 * type (void *) and the remaining five of type size_t, describing the following
3924 * (in the same order):
3925 *
3926 * (a) memory address of the extent a potential reallocation would go into,
3927 * == the five fields below describe about the extent the pointer resides in ==
3928 * (b) number of free regions in the extent,
3929 * (c) number of regions in the extent,
3930 * (d) size of the extent in terms of bytes,
3931 * (e) total number of free regions in the bin the extent belongs to, and
3932 * (f) total number of regions in the bin the extent belongs to.
3933 *
3934 * Note that "(e)" and "(f)" are only available when stats are enabled;
3935 * otherwise their values are undefined.
3936 *
3937 * This API is mainly intended for small class allocations, where extents are
3938 * used as slab. Note that if the bin the extent belongs to is completely
3939 * full, "(a)" will be NULL.
3940 *
3941 * In case of large class allocations, "(a)" will be NULL, and "(e)" and "(f)"
3942 * will be zero (if stats are enabled; otherwise undefined). The other three
3943 * fields will be properly set though the values are trivial: "(b)" will be 0,
3944 * "(c)" will be 1, and "(d)" will be the usable size.
3945 *
3946 * The input pointer and size are respectively passed in by newp and newlen,
3947 * and the output fields and size are respectively oldp and *oldlenp.
3948 *
3949 * It can be beneficial to define the following macros to make it easier to
3950 * access the output:
3951 *
3952 * #define SLABCUR_READ(out) (*(void **)out)
3953 * #define COUNTS(out) ((size_t *)((void **)out + 1))
3954 * #define NFREE_READ(out) COUNTS(out)[0]
3955 * #define NREGS_READ(out) COUNTS(out)[1]
3956 * #define SIZE_READ(out) COUNTS(out)[2]
3957 * #define BIN_NFREE_READ(out) COUNTS(out)[3]
3958 * #define BIN_NREGS_READ(out) COUNTS(out)[4]
3959 *
3960 * and then write e.g. NFREE_READ(oldp) to fetch the output. See the unit test
3961 * test_query in test/unit/extent_util.c for an example.
3962 *
3963 * For a typical defragmentation workflow making use of this API for
3964 * understanding the fragmentation level, please refer to the comment for
3965 * experimental_utilization_batch_query_ctl.
3966 *
3967 * It's up to the application how to determine the significance of
3968 * fragmentation relying on the outputs returned. Possible choices are:
3969 *
3970 * (a) if extent utilization ratio is below certain threshold,
3971 * (b) if extent memory consumption is above certain threshold,
3972 * (c) if extent utilization ratio is significantly below bin utilization ratio,
3973 * (d) if input pointer deviates a lot from potential reallocation address, or
3974 * (e) some selection/combination of the above.
3975 *
3976 * The caller needs to make sure that the input/output arguments are valid,
3977 * in particular, that the size of the output is correct, i.e.:
3978 *
3979 * *oldlenp = sizeof(void *) + sizeof(size_t) * 5
3980 *
3981 * Otherwise, the function immediately returns EINVAL without touching anything.
3982 *
3983 * In the rare case where there's no associated extent found for the input
3984 * pointer, the function zeros out all output fields and return. Please refer
3985 * to the comment for experimental_utilization_batch_query_ctl to understand the
3986 * motivation from C++.
3987 */
3988static int
3989experimental_utilization_query_ctl(tsd_t *tsd, const size_t *mib,
3990 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3991 int ret;
3992
3993 assert(sizeof(inspect_extent_util_stats_verbose_t)
3994 == sizeof(void *) + sizeof(size_t) * 5);
3995
3996 if (oldp == NULL || oldlenp == NULL
3997 || *oldlenp != sizeof(inspect_extent_util_stats_verbose_t)
3998 || newp == NULL) {
3999 ret = EINVAL;
4000 goto label_return;
4001 }
4002
4003 void *ptr = NULL;
4004 WRITE(ptr, void *);
4005 inspect_extent_util_stats_verbose_t *util_stats
4006 = (inspect_extent_util_stats_verbose_t *)oldp;
4007 inspect_extent_util_stats_verbose_get(tsd_tsdn(tsd), ptr,
4008 &util_stats->nfree, &util_stats->nregs, &util_stats->size,
4009 &util_stats->bin_nfree, &util_stats->bin_nregs,
4010 &util_stats->slabcur_addr);
4011 ret = 0;
4012
4013label_return:
4014 return ret;
4015}
4016
4017/*
4018 * Given an input array of pointers, output three memory utilization entries of
4019 * type size_t for each input pointer about the extent it resides in:
4020 *
4021 * (a) number of free regions in the extent,
4022 * (b) number of regions in the extent, and
4023 * (c) size of the extent in terms of bytes.
4024 *
4025 * This API is mainly intended for small class allocations, where extents are
4026 * used as slab. In case of large class allocations, the outputs are trivial:
4027 * "(a)" will be 0, "(b)" will be 1, and "(c)" will be the usable size.
4028 *
4029 * Note that multiple input pointers may reside on a same extent so the output
4030 * fields may contain duplicates.
4031 *
4032 * The format of the input/output looks like:
4033 *
4034 * input[0]: 1st_pointer_to_query | output[0]: 1st_extent_n_free_regions
4035 * | output[1]: 1st_extent_n_regions
4036 * | output[2]: 1st_extent_size
4037 * input[1]: 2nd_pointer_to_query | output[3]: 2nd_extent_n_free_regions
4038 * | output[4]: 2nd_extent_n_regions
4039 * | output[5]: 2nd_extent_size
4040 * ... | ...
4041 *
4042 * The input array and size are respectively passed in by newp and newlen, and
4043 * the output array and size are respectively oldp and *oldlenp.
4044 *
4045 * It can be beneficial to define the following macros to make it easier to
4046 * access the output:
4047 *
4048 * #define NFREE_READ(out, i) out[(i) * 3]
4049 * #define NREGS_READ(out, i) out[(i) * 3 + 1]
4050 * #define SIZE_READ(out, i) out[(i) * 3 + 2]
4051 *
4052 * and then write e.g. NFREE_READ(oldp, i) to fetch the output. See the unit
4053 * test test_batch in test/unit/extent_util.c for a concrete example.
4054 *
4055 * A typical workflow would be composed of the following steps:
4056 *
4057 * (1) flush tcache: mallctl("thread.tcache.flush", ...)
4058 * (2) initialize input array of pointers to query fragmentation
4059 * (3) allocate output array to hold utilization statistics
4060 * (4) query utilization: mallctl("experimental.utilization.batch_query", ...)
4061 * (5) (optional) decide if it's worthwhile to defragment; otherwise stop here
4062 * (6) disable tcache: mallctl("thread.tcache.enabled", ...)
4063 * (7) defragment allocations with significant fragmentation, e.g.:
4064 * for each allocation {
4065 * if it's fragmented {
4066 * malloc(...);
4067 * memcpy(...);
4068 * free(...);
4069 * }
4070 * }
4071 * (8) enable tcache: mallctl("thread.tcache.enabled", ...)
4072 *
4073 * The application can determine the significance of fragmentation themselves
4074 * relying on the statistics returned, both at the overall level i.e. step "(5)"
4075 * and at individual allocation level i.e. within step "(7)". Possible choices
4076 * are:
4077 *
4078 * (a) whether memory utilization ratio is below certain threshold,
4079 * (b) whether memory consumption is above certain threshold, or
4080 * (c) some combination of the two.
4081 *
4082 * The caller needs to make sure that the input/output arrays are valid and
4083 * their sizes are proper as well as matched, meaning:
4084 *
4085 * (a) newlen = n_pointers * sizeof(const void *)
4086 * (b) *oldlenp = n_pointers * sizeof(size_t) * 3
4087 * (c) n_pointers > 0
4088 *
4089 * Otherwise, the function immediately returns EINVAL without touching anything.
4090 *
4091 * In the rare case where there's no associated extent found for some pointers,
4092 * rather than immediately terminating the computation and raising an error,
4093 * the function simply zeros out the corresponding output fields and continues
4094 * the computation until all input pointers are handled. The motivations of
4095 * such a design are as follows:
4096 *
4097 * (a) The function always either processes nothing or processes everything, and
4098 * never leaves the output half touched and half untouched.
4099 *
4100 * (b) It facilitates usage needs especially common in C++. A vast variety of
4101 * C++ objects are instantiated with multiple dynamic memory allocations. For
4102 * example, std::string and std::vector typically use at least two allocations,
4103 * one for the metadata and one for the actual content. Other types may use
4104 * even more allocations. When inquiring about utilization statistics, the
4105 * caller often wants to examine into all such allocations, especially internal
4106 * one(s), rather than just the topmost one. The issue comes when some
4107 * implementations do certain optimizations to reduce/aggregate some internal
4108 * allocations, e.g. putting short strings directly into the metadata, and such
4109 * decisions are not known to the caller. Therefore, we permit pointers to
4110 * memory usages that may not be returned by previous malloc calls, and we
4111 * provide the caller a convenient way to identify such cases.
4112 */
4113static int
4114experimental_utilization_batch_query_ctl(tsd_t *tsd, const size_t *mib,
4115 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4116 int ret;
4117
4118 assert(sizeof(inspect_extent_util_stats_t) == sizeof(size_t) * 3);
4119
4120 const size_t len = newlen / sizeof(const void *);
4121 if (oldp == NULL || oldlenp == NULL || newp == NULL || newlen == 0
4122 || newlen != len * sizeof(const void *)
4123 || *oldlenp != len * sizeof(inspect_extent_util_stats_t)) {
4124 ret = EINVAL;
4125 goto label_return;
4126 }
4127
4128 void **ptrs = (void **)newp;
4129 inspect_extent_util_stats_t *util_stats =
4130 (inspect_extent_util_stats_t *)oldp;
4131 size_t i;
4132 for (i = 0; i < len; ++i) {
4133 inspect_extent_util_stats_get(tsd_tsdn(tsd), ptrs[i],
4134 &util_stats[i].nfree, &util_stats[i].nregs,
4135 &util_stats[i].size);
4136 }
4137 ret = 0;
4138
4139label_return:
4140 return ret;
4141}
4142
4143static const ctl_named_node_t *
4144experimental_arenas_i_index(tsdn_t *tsdn, const size_t *mib,
4145 size_t miblen, size_t i) {
4146 const ctl_named_node_t *ret;
4147
4148 malloc_mutex_lock(tsdn, &ctl_mtx);
4149 if (ctl_arenas_i_verify(i)) {
4150 ret = NULL;
4151 goto label_return;
4152 }
4153 ret = super_experimental_arenas_i_node;
4154label_return:
4155 malloc_mutex_unlock(tsdn, &ctl_mtx);
4156 return ret;
4157}
4158
4159static int
4160experimental_arenas_i_pactivep_ctl(tsd_t *tsd, const size_t *mib,
4161 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4162 if (!config_stats) {
4163 return ENOENT;
4164 }
4165 if (oldp == NULL || oldlenp == NULL || *oldlenp != sizeof(size_t *)) {
4166 return EINVAL;
4167 }
4168
4169 unsigned arena_ind;
4170 arena_t *arena;
4171 int ret;
4172 size_t *pactivep;
4173
4174 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
4175 READONLY();
4176 MIB_UNSIGNED(arena_ind, 2);
4177 if (arena_ind < narenas_total_get() && (arena =
4178 arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) {
4179#if defined(JEMALLOC_GCC_ATOMIC_ATOMICS) || \
4180 defined(JEMALLOC_GCC_SYNC_ATOMICS) || defined(_MSC_VER)
4181 /* Expose the underlying counter for fast read. */
4182 pactivep = (size_t *)&(arena->pa_shard.nactive.repr);
4183 READ(pactivep, size_t *);
4184 ret = 0;
4185#else
4186 ret = EFAULT;
4187#endif
4188 } else {
4189 ret = EFAULT;
4190 }
4191label_return:
4192 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
4193 return ret;
4194}
4195
4196static int
4197experimental_prof_recent_alloc_max_ctl(tsd_t *tsd, const size_t *mib,
4198 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4199 int ret;
4200
4201 if (!(config_prof && opt_prof)) {
4202 ret = ENOENT;
4203 goto label_return;
4204 }
4205
4206 ssize_t old_max;
4207 if (newp != NULL) {
4208 if (newlen != sizeof(ssize_t)) {
4209 ret = EINVAL;
4210 goto label_return;
4211 }
4212 ssize_t max = *(ssize_t *)newp;
4213 if (max < -1) {
4214 ret = EINVAL;
4215 goto label_return;
4216 }
4217 old_max = prof_recent_alloc_max_ctl_write(tsd, max);
4218 } else {
4219 old_max = prof_recent_alloc_max_ctl_read();
4220 }
4221 READ(old_max, ssize_t);
4222
4223 ret = 0;
4224
4225label_return:
4226 return ret;
4227}
4228
4229typedef struct write_cb_packet_s write_cb_packet_t;
4230struct write_cb_packet_s {
4231 write_cb_t *write_cb;
4232 void *cbopaque;
4233};
4234
4235static int
4236experimental_prof_recent_alloc_dump_ctl(tsd_t *tsd, const size_t *mib,
4237 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4238 int ret;
4239
4240 if (!(config_prof && opt_prof)) {
4241 ret = ENOENT;
4242 goto label_return;
4243 }
4244
4245 assert(sizeof(write_cb_packet_t) == sizeof(void *) * 2);
4246
4247 WRITEONLY();
4248 write_cb_packet_t write_cb_packet;
4249 ASSURED_WRITE(write_cb_packet, write_cb_packet_t);
4250
4251 prof_recent_alloc_dump(tsd, write_cb_packet.write_cb,
4252 write_cb_packet.cbopaque);
4253
4254 ret = 0;
4255
4256label_return:
4257 return ret;
4258}
4259
4260typedef struct batch_alloc_packet_s batch_alloc_packet_t;
4261struct batch_alloc_packet_s {
4262 void **ptrs;
4263 size_t num;
4264 size_t size;
4265 int flags;
4266};
4267
4268static int
4269experimental_batch_alloc_ctl(tsd_t *tsd, const size_t *mib,
4270 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4271 int ret;
4272
4273 VERIFY_READ(size_t);
4274
4275 batch_alloc_packet_t batch_alloc_packet;
4276 ASSURED_WRITE(batch_alloc_packet, batch_alloc_packet_t);
4277 size_t filled = batch_alloc(batch_alloc_packet.ptrs,
4278 batch_alloc_packet.num, batch_alloc_packet.size,
4279 batch_alloc_packet.flags);
4280 READ(filled, size_t);
4281
4282 ret = 0;
4283
4284label_return:
4285 return ret;
4286}
4287
4288static int
4289prof_stats_bins_i_live_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
4290 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4291 int ret;
4292 unsigned binind;
4293 prof_stats_t stats;
4294
4295 if (!(config_prof && opt_prof && opt_prof_stats)) {
4296 ret = ENOENT;
4297 goto label_return;
4298 }
4299
4300 READONLY();
4301 MIB_UNSIGNED(binind, 3);
4302 if (binind >= SC_NBINS) {
4303 ret = EINVAL;
4304 goto label_return;
4305 }
4306 prof_stats_get_live(tsd, (szind_t)binind, &stats);
4307 READ(stats, prof_stats_t);
4308
4309 ret = 0;
4310label_return:
4311 return ret;
4312}
4313
4314static int
4315prof_stats_bins_i_accum_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
4316 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4317 int ret;
4318 unsigned binind;
4319 prof_stats_t stats;
4320
4321 if (!(config_prof && opt_prof && opt_prof_stats)) {
4322 ret = ENOENT;
4323 goto label_return;
4324 }
4325
4326 READONLY();
4327 MIB_UNSIGNED(binind, 3);
4328 if (binind >= SC_NBINS) {
4329 ret = EINVAL;
4330 goto label_return;
4331 }
4332 prof_stats_get_accum(tsd, (szind_t)binind, &stats);
4333 READ(stats, prof_stats_t);
4334
4335 ret = 0;
4336label_return:
4337 return ret;
4338}
4339
4340static const ctl_named_node_t *
4341prof_stats_bins_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen,
4342 size_t i) {
4343 if (!(config_prof && opt_prof && opt_prof_stats)) {
4344 return NULL;
4345 }
4346 if (i >= SC_NBINS) {
4347 return NULL;
4348 }
4349 return super_prof_stats_bins_i_node;
4350}
4351
4352static int
4353prof_stats_lextents_i_live_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
4354 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4355 int ret;
4356 unsigned lextent_ind;
4357 prof_stats_t stats;
4358
4359 if (!(config_prof && opt_prof && opt_prof_stats)) {
4360 ret = ENOENT;
4361 goto label_return;
4362 }
4363
4364 READONLY();
4365 MIB_UNSIGNED(lextent_ind, 3);
4366 if (lextent_ind >= SC_NSIZES - SC_NBINS) {
4367 ret = EINVAL;
4368 goto label_return;
4369 }
4370 prof_stats_get_live(tsd, (szind_t)(lextent_ind + SC_NBINS), &stats);
4371 READ(stats, prof_stats_t);
4372
4373 ret = 0;
4374label_return:
4375 return ret;
4376}
4377
4378static int
4379prof_stats_lextents_i_accum_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
4380 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
4381 int ret;
4382 unsigned lextent_ind;
4383 prof_stats_t stats;
4384
4385 if (!(config_prof && opt_prof && opt_prof_stats)) {
4386 ret = ENOENT;
4387 goto label_return;
4388 }
4389
4390 READONLY();
4391 MIB_UNSIGNED(lextent_ind, 3);
4392 if (lextent_ind >= SC_NSIZES - SC_NBINS) {
4393 ret = EINVAL;
4394 goto label_return;
4395 }
4396 prof_stats_get_accum(tsd, (szind_t)(lextent_ind + SC_NBINS), &stats);
4397 READ(stats, prof_stats_t);
4398
4399 ret = 0;
4400label_return:
4401 return ret;
4402}
4403
4404static const ctl_named_node_t *
4405prof_stats_lextents_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen,
4406 size_t i) {
4407 if (!(config_prof && opt_prof && opt_prof_stats)) {
4408 return NULL;
4409 }
4410 if (i >= SC_NSIZES - SC_NBINS) {
4411 return NULL;
4412 }
4413 return super_prof_stats_lextents_i_node;
4414}
generated by cgit v1.2.3 (git 2.46.0) at 2026-04-30 00:56:01 +0000