aboutsummaryrefslogtreecommitdiff
path: root/examples/redis-unstable/deps/jemalloc/src/background_thread.c
diff options
context:
space:
mode:
Diffstat (limited to 'examples/redis-unstable/deps/jemalloc/src/background_thread.c')
-rw-r--r--examples/redis-unstable/deps/jemalloc/src/background_thread.c820
1 files changed, 0 insertions, 820 deletions
diff --git a/examples/redis-unstable/deps/jemalloc/src/background_thread.c b/examples/redis-unstable/deps/jemalloc/src/background_thread.c
deleted file mode 100644
index 3bb8d26..0000000
--- a/examples/redis-unstable/deps/jemalloc/src/background_thread.c
+++ /dev/null
@@ -1,820 +0,0 @@
1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5
6JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
7
8/******************************************************************************/
9/* Data. */
10
11/* This option should be opt-in only. */
12#define BACKGROUND_THREAD_DEFAULT false
13/* Read-only after initialization. */
14bool opt_background_thread = BACKGROUND_THREAD_DEFAULT;
15size_t opt_max_background_threads = MAX_BACKGROUND_THREAD_LIMIT + 1;
16
17/* Used for thread creation, termination and stats. */
18malloc_mutex_t background_thread_lock;
19/* Indicates global state. Atomic because decay reads this w/o locking. */
20atomic_b_t background_thread_enabled_state;
21size_t n_background_threads;
22size_t max_background_threads;
23/* Thread info per-index. */
24background_thread_info_t *background_thread_info;
25
26/******************************************************************************/
27
28#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
29
30static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *,
31 void *(*)(void *), void *__restrict);
32
33static void
34pthread_create_wrapper_init(void) {
35#ifdef JEMALLOC_LAZY_LOCK
36 if (!isthreaded) {
37 isthreaded = true;
38 }
39#endif
40}
41
42int
43pthread_create_wrapper(pthread_t *__restrict thread, const pthread_attr_t *attr,
44 void *(*start_routine)(void *), void *__restrict arg) {
45 pthread_create_wrapper_init();
46
47 return pthread_create_fptr(thread, attr, start_routine, arg);
48}
49#endif /* JEMALLOC_PTHREAD_CREATE_WRAPPER */
50
51#ifndef JEMALLOC_BACKGROUND_THREAD
52#define NOT_REACHED { not_reached(); }
53bool background_thread_create(tsd_t *tsd, unsigned arena_ind) NOT_REACHED
54bool background_threads_enable(tsd_t *tsd) NOT_REACHED
55bool background_threads_disable(tsd_t *tsd) NOT_REACHED
56bool background_thread_is_started(background_thread_info_t *info) NOT_REACHED
57void background_thread_wakeup_early(background_thread_info_t *info,
58 nstime_t *remaining_sleep) NOT_REACHED
59void background_thread_prefork0(tsdn_t *tsdn) NOT_REACHED
60void background_thread_prefork1(tsdn_t *tsdn) NOT_REACHED
61void background_thread_postfork_parent(tsdn_t *tsdn) NOT_REACHED
62void background_thread_postfork_child(tsdn_t *tsdn) NOT_REACHED
63bool background_thread_stats_read(tsdn_t *tsdn,
64 background_thread_stats_t *stats) NOT_REACHED
65void background_thread_ctl_init(tsdn_t *tsdn) NOT_REACHED
66#undef NOT_REACHED
67#else
68
69static bool background_thread_enabled_at_fork;
70
71static void
72background_thread_info_init(tsdn_t *tsdn, background_thread_info_t *info) {
73 background_thread_wakeup_time_set(tsdn, info, 0);
74 info->npages_to_purge_new = 0;
75 if (config_stats) {
76 info->tot_n_runs = 0;
77 nstime_init_zero(&info->tot_sleep_time);
78 }
79}
80
81static inline bool
82set_current_thread_affinity(int cpu) {
83#if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
84 cpu_set_t cpuset;
85#else
86# ifndef __NetBSD__
87 cpuset_t cpuset;
88# else
89 cpuset_t *cpuset;
90# endif
91#endif
92
93#ifndef __NetBSD__
94 CPU_ZERO(&cpuset);
95 CPU_SET(cpu, &cpuset);
96#else
97 cpuset = cpuset_create();
98#endif
99
100#if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
101 return (sched_setaffinity(0, sizeof(cpu_set_t), &cpuset) != 0);
102#else
103# ifndef __NetBSD__
104 int ret = pthread_setaffinity_np(pthread_self(), sizeof(cpuset_t),
105 &cpuset);
106# else
107 int ret = pthread_setaffinity_np(pthread_self(), cpuset_size(cpuset),
108 cpuset);
109 cpuset_destroy(cpuset);
110# endif
111 return ret != 0;
112#endif
113}
114
115#define BILLION UINT64_C(1000000000)
116/* Minimal sleep interval 100 ms. */
117#define BACKGROUND_THREAD_MIN_INTERVAL_NS (BILLION / 10)
118
119static void
120background_thread_sleep(tsdn_t *tsdn, background_thread_info_t *info,
121 uint64_t interval) {
122 if (config_stats) {
123 info->tot_n_runs++;
124 }
125 info->npages_to_purge_new = 0;
126
127 struct timeval tv;
128 /* Specific clock required by timedwait. */
129 gettimeofday(&tv, NULL);
130 nstime_t before_sleep;
131 nstime_init2(&before_sleep, tv.tv_sec, tv.tv_usec * 1000);
132
133 int ret;
134 if (interval == BACKGROUND_THREAD_INDEFINITE_SLEEP) {
135 background_thread_wakeup_time_set(tsdn, info,
136 BACKGROUND_THREAD_INDEFINITE_SLEEP);
137 ret = pthread_cond_wait(&info->cond, &info->mtx.lock);
138 assert(ret == 0);
139 } else {
140 assert(interval >= BACKGROUND_THREAD_MIN_INTERVAL_NS &&
141 interval <= BACKGROUND_THREAD_INDEFINITE_SLEEP);
142 /* We need malloc clock (can be different from tv). */
143 nstime_t next_wakeup;
144 nstime_init_update(&next_wakeup);
145 nstime_iadd(&next_wakeup, interval);
146 assert(nstime_ns(&next_wakeup) <
147 BACKGROUND_THREAD_INDEFINITE_SLEEP);
148 background_thread_wakeup_time_set(tsdn, info,
149 nstime_ns(&next_wakeup));
150
151 nstime_t ts_wakeup;
152 nstime_copy(&ts_wakeup, &before_sleep);
153 nstime_iadd(&ts_wakeup, interval);
154 struct timespec ts;
155 ts.tv_sec = (size_t)nstime_sec(&ts_wakeup);
156 ts.tv_nsec = (size_t)nstime_nsec(&ts_wakeup);
157
158 assert(!background_thread_indefinite_sleep(info));
159 ret = pthread_cond_timedwait(&info->cond, &info->mtx.lock, &ts);
160 assert(ret == ETIMEDOUT || ret == 0);
161 }
162 if (config_stats) {
163 gettimeofday(&tv, NULL);
164 nstime_t after_sleep;
165 nstime_init2(&after_sleep, tv.tv_sec, tv.tv_usec * 1000);
166 if (nstime_compare(&after_sleep, &before_sleep) > 0) {
167 nstime_subtract(&after_sleep, &before_sleep);
168 nstime_add(&info->tot_sleep_time, &after_sleep);
169 }
170 }
171}
172
173static bool
174background_thread_pause_check(tsdn_t *tsdn, background_thread_info_t *info) {
175 if (unlikely(info->state == background_thread_paused)) {
176 malloc_mutex_unlock(tsdn, &info->mtx);
177 /* Wait on global lock to update status. */
178 malloc_mutex_lock(tsdn, &background_thread_lock);
179 malloc_mutex_unlock(tsdn, &background_thread_lock);
180 malloc_mutex_lock(tsdn, &info->mtx);
181 return true;
182 }
183
184 return false;
185}
186
187static inline void
188background_work_sleep_once(tsdn_t *tsdn, background_thread_info_t *info,
189 unsigned ind) {
190 uint64_t ns_until_deferred = BACKGROUND_THREAD_DEFERRED_MAX;
191 unsigned narenas = narenas_total_get();
192 bool slept_indefinitely = background_thread_indefinite_sleep(info);
193
194 for (unsigned i = ind; i < narenas; i += max_background_threads) {
195 arena_t *arena = arena_get(tsdn, i, false);
196 if (!arena) {
197 continue;
198 }
199 /*
200 * If thread was woken up from the indefinite sleep, don't
201 * do the work instantly, but rather check when the deferred
202 * work that caused this thread to wake up is scheduled for.
203 */
204 if (!slept_indefinitely) {
205 arena_do_deferred_work(tsdn, arena);
206 }
207 if (ns_until_deferred <= BACKGROUND_THREAD_MIN_INTERVAL_NS) {
208 /* Min interval will be used. */
209 continue;
210 }
211 uint64_t ns_arena_deferred = pa_shard_time_until_deferred_work(
212 tsdn, &arena->pa_shard);
213 if (ns_arena_deferred < ns_until_deferred) {
214 ns_until_deferred = ns_arena_deferred;
215 }
216 }
217
218 uint64_t sleep_ns;
219 if (ns_until_deferred == BACKGROUND_THREAD_DEFERRED_MAX) {
220 sleep_ns = BACKGROUND_THREAD_INDEFINITE_SLEEP;
221 } else {
222 sleep_ns =
223 (ns_until_deferred < BACKGROUND_THREAD_MIN_INTERVAL_NS)
224 ? BACKGROUND_THREAD_MIN_INTERVAL_NS
225 : ns_until_deferred;
226
227 }
228
229 background_thread_sleep(tsdn, info, sleep_ns);
230}
231
232static bool
233background_threads_disable_single(tsd_t *tsd, background_thread_info_t *info) {
234 if (info == &background_thread_info[0]) {
235 malloc_mutex_assert_owner(tsd_tsdn(tsd),
236 &background_thread_lock);
237 } else {
238 malloc_mutex_assert_not_owner(tsd_tsdn(tsd),
239 &background_thread_lock);
240 }
241
242 pre_reentrancy(tsd, NULL);
243 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
244 bool has_thread;
245 assert(info->state != background_thread_paused);
246 if (info->state == background_thread_started) {
247 has_thread = true;
248 info->state = background_thread_stopped;
249 pthread_cond_signal(&info->cond);
250 } else {
251 has_thread = false;
252 }
253 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
254
255 if (!has_thread) {
256 post_reentrancy(tsd);
257 return false;
258 }
259 void *ret;
260 if (pthread_join(info->thread, &ret)) {
261 post_reentrancy(tsd);
262 return true;
263 }
264 assert(ret == NULL);
265 n_background_threads--;
266 post_reentrancy(tsd);
267
268 return false;
269}
270
271static void *background_thread_entry(void *ind_arg);
272
273static int
274background_thread_create_signals_masked(pthread_t *thread,
275 const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg) {
276 /*
277 * Mask signals during thread creation so that the thread inherits
278 * an empty signal set.
279 */
280 sigset_t set;
281 sigfillset(&set);
282 sigset_t oldset;
283 int mask_err = pthread_sigmask(SIG_SETMASK, &set, &oldset);
284 if (mask_err != 0) {
285 return mask_err;
286 }
287 int create_err = pthread_create_wrapper(thread, attr, start_routine,
288 arg);
289 /*
290 * Restore the signal mask. Failure to restore the signal mask here
291 * changes program behavior.
292 */
293 int restore_err = pthread_sigmask(SIG_SETMASK, &oldset, NULL);
294 if (restore_err != 0) {
295 malloc_printf("<jemalloc>: background thread creation "
296 "failed (%d), and signal mask restoration failed "
297 "(%d)\n", create_err, restore_err);
298 if (opt_abort) {
299 abort();
300 }
301 }
302 return create_err;
303}
304
305static bool
306check_background_thread_creation(tsd_t *tsd, unsigned *n_created,
307 bool *created_threads) {
308 bool ret = false;
309 if (likely(*n_created == n_background_threads)) {
310 return ret;
311 }
312
313 tsdn_t *tsdn = tsd_tsdn(tsd);
314 malloc_mutex_unlock(tsdn, &background_thread_info[0].mtx);
315 for (unsigned i = 1; i < max_background_threads; i++) {
316 if (created_threads[i]) {
317 continue;
318 }
319 background_thread_info_t *info = &background_thread_info[i];
320 malloc_mutex_lock(tsdn, &info->mtx);
321 /*
322 * In case of the background_thread_paused state because of
323 * arena reset, delay the creation.
324 */
325 bool create = (info->state == background_thread_started);
326 malloc_mutex_unlock(tsdn, &info->mtx);
327 if (!create) {
328 continue;
329 }
330
331 pre_reentrancy(tsd, NULL);
332 int err = background_thread_create_signals_masked(&info->thread,
333 NULL, background_thread_entry, (void *)(uintptr_t)i);
334 post_reentrancy(tsd);
335
336 if (err == 0) {
337 (*n_created)++;
338 created_threads[i] = true;
339 } else {
340 malloc_printf("<jemalloc>: background thread "
341 "creation failed (%d)\n", err);
342 if (opt_abort) {
343 abort();
344 }
345 }
346 /* Return to restart the loop since we unlocked. */
347 ret = true;
348 break;
349 }
350 malloc_mutex_lock(tsdn, &background_thread_info[0].mtx);
351
352 return ret;
353}
354
355static void
356background_thread0_work(tsd_t *tsd) {
357 /* Thread0 is also responsible for launching / terminating threads. */
358 VARIABLE_ARRAY(bool, created_threads, max_background_threads);
359 unsigned i;
360 for (i = 1; i < max_background_threads; i++) {
361 created_threads[i] = false;
362 }
363 /* Start working, and create more threads when asked. */
364 unsigned n_created = 1;
365 while (background_thread_info[0].state != background_thread_stopped) {
366 if (background_thread_pause_check(tsd_tsdn(tsd),
367 &background_thread_info[0])) {
368 continue;
369 }
370 if (check_background_thread_creation(tsd, &n_created,
371 (bool *)&created_threads)) {
372 continue;
373 }
374 background_work_sleep_once(tsd_tsdn(tsd),
375 &background_thread_info[0], 0);
376 }
377
378 /*
379 * Shut down other threads at exit. Note that the ctl thread is holding
380 * the global background_thread mutex (and is waiting) for us.
381 */
382 assert(!background_thread_enabled());
383 for (i = 1; i < max_background_threads; i++) {
384 background_thread_info_t *info = &background_thread_info[i];
385 assert(info->state != background_thread_paused);
386 if (created_threads[i]) {
387 background_threads_disable_single(tsd, info);
388 } else {
389 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
390 if (info->state != background_thread_stopped) {
391 /* The thread was not created. */
392 assert(info->state ==
393 background_thread_started);
394 n_background_threads--;
395 info->state = background_thread_stopped;
396 }
397 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
398 }
399 }
400 background_thread_info[0].state = background_thread_stopped;
401 assert(n_background_threads == 1);
402}
403
404static void
405background_work(tsd_t *tsd, unsigned ind) {
406 background_thread_info_t *info = &background_thread_info[ind];
407
408 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
409 background_thread_wakeup_time_set(tsd_tsdn(tsd), info,
410 BACKGROUND_THREAD_INDEFINITE_SLEEP);
411 if (ind == 0) {
412 background_thread0_work(tsd);
413 } else {
414 while (info->state != background_thread_stopped) {
415 if (background_thread_pause_check(tsd_tsdn(tsd),
416 info)) {
417 continue;
418 }
419 background_work_sleep_once(tsd_tsdn(tsd), info, ind);
420 }
421 }
422 assert(info->state == background_thread_stopped);
423 background_thread_wakeup_time_set(tsd_tsdn(tsd), info, 0);
424 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
425}
426
427static void *
428background_thread_entry(void *ind_arg) {
429 unsigned thread_ind = (unsigned)(uintptr_t)ind_arg;
430 assert(thread_ind < max_background_threads);
431#ifdef JEMALLOC_HAVE_PTHREAD_SETNAME_NP
432 pthread_setname_np(pthread_self(), "jemalloc_bg_thd");
433#elif defined(__FreeBSD__) || defined(__DragonFly__)
434 pthread_set_name_np(pthread_self(), "jemalloc_bg_thd");
435#endif
436 if (opt_percpu_arena != percpu_arena_disabled) {
437 set_current_thread_affinity((int)thread_ind);
438 }
439 /*
440 * Start periodic background work. We use internal tsd which avoids
441 * side effects, for example triggering new arena creation (which in
442 * turn triggers another background thread creation).
443 */
444 background_work(tsd_internal_fetch(), thread_ind);
445 assert(pthread_equal(pthread_self(),
446 background_thread_info[thread_ind].thread));
447
448 return NULL;
449}
450
451static void
452background_thread_init(tsd_t *tsd, background_thread_info_t *info) {
453 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
454 info->state = background_thread_started;
455 background_thread_info_init(tsd_tsdn(tsd), info);
456 n_background_threads++;
457}
458
459static bool
460background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) {
461 assert(have_background_thread);
462 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
463
464 /* We create at most NCPUs threads. */
465 size_t thread_ind = arena_ind % max_background_threads;
466 background_thread_info_t *info = &background_thread_info[thread_ind];
467
468 bool need_new_thread;
469 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
470 need_new_thread = background_thread_enabled() &&
471 (info->state == background_thread_stopped);
472 if (need_new_thread) {
473 background_thread_init(tsd, info);
474 }
475 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
476 if (!need_new_thread) {
477 return false;
478 }
479 if (arena_ind != 0) {
480 /* Threads are created asynchronously by Thread 0. */
481 background_thread_info_t *t0 = &background_thread_info[0];
482 malloc_mutex_lock(tsd_tsdn(tsd), &t0->mtx);
483 assert(t0->state == background_thread_started);
484 pthread_cond_signal(&t0->cond);
485 malloc_mutex_unlock(tsd_tsdn(tsd), &t0->mtx);
486
487 return false;
488 }
489
490 pre_reentrancy(tsd, NULL);
491 /*
492 * To avoid complications (besides reentrancy), create internal
493 * background threads with the underlying pthread_create.
494 */
495 int err = background_thread_create_signals_masked(&info->thread, NULL,
496 background_thread_entry, (void *)thread_ind);
497 post_reentrancy(tsd);
498
499 if (err != 0) {
500 malloc_printf("<jemalloc>: arena 0 background thread creation "
501 "failed (%d)\n", err);
502 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
503 info->state = background_thread_stopped;
504 n_background_threads--;
505 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
506
507 return true;
508 }
509
510 return false;
511}
512
513/* Create a new background thread if needed. */
514bool
515background_thread_create(tsd_t *tsd, unsigned arena_ind) {
516 assert(have_background_thread);
517
518 bool ret;
519 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
520 ret = background_thread_create_locked(tsd, arena_ind);
521 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
522
523 return ret;
524}
525
526bool
527background_threads_enable(tsd_t *tsd) {
528 assert(n_background_threads == 0);
529 assert(background_thread_enabled());
530 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
531
532 VARIABLE_ARRAY(bool, marked, max_background_threads);
533 unsigned nmarked;
534 for (unsigned i = 0; i < max_background_threads; i++) {
535 marked[i] = false;
536 }
537 nmarked = 0;
538 /* Thread 0 is required and created at the end. */
539 marked[0] = true;
540 /* Mark the threads we need to create for thread 0. */
541 unsigned narenas = narenas_total_get();
542 for (unsigned i = 1; i < narenas; i++) {
543 if (marked[i % max_background_threads] ||
544 arena_get(tsd_tsdn(tsd), i, false) == NULL) {
545 continue;
546 }
547 background_thread_info_t *info = &background_thread_info[
548 i % max_background_threads];
549 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
550 assert(info->state == background_thread_stopped);
551 background_thread_init(tsd, info);
552 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
553 marked[i % max_background_threads] = true;
554 if (++nmarked == max_background_threads) {
555 break;
556 }
557 }
558
559 bool err = background_thread_create_locked(tsd, 0);
560 if (err) {
561 return true;
562 }
563 for (unsigned i = 0; i < narenas; i++) {
564 arena_t *arena = arena_get(tsd_tsdn(tsd), i, false);
565 if (arena != NULL) {
566 pa_shard_set_deferral_allowed(tsd_tsdn(tsd),
567 &arena->pa_shard, true);
568 }
569 }
570 return false;
571}
572
573bool
574background_threads_disable(tsd_t *tsd) {
575 assert(!background_thread_enabled());
576 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
577
578 /* Thread 0 will be responsible for terminating other threads. */
579 if (background_threads_disable_single(tsd,
580 &background_thread_info[0])) {
581 return true;
582 }
583 assert(n_background_threads == 0);
584 unsigned narenas = narenas_total_get();
585 for (unsigned i = 0; i < narenas; i++) {
586 arena_t *arena = arena_get(tsd_tsdn(tsd), i, false);
587 if (arena != NULL) {
588 pa_shard_set_deferral_allowed(tsd_tsdn(tsd),
589 &arena->pa_shard, false);
590 }
591 }
592
593 return false;
594}
595
596bool
597background_thread_is_started(background_thread_info_t *info) {
598 return info->state == background_thread_started;
599}
600
601void
602background_thread_wakeup_early(background_thread_info_t *info,
603 nstime_t *remaining_sleep) {
604 /*
605 * This is an optimization to increase batching. At this point
606 * we know that background thread wakes up soon, so the time to cache
607 * the just freed memory is bounded and low.
608 */
609 if (remaining_sleep != NULL && nstime_ns(remaining_sleep) <
610 BACKGROUND_THREAD_MIN_INTERVAL_NS) {
611 return;
612 }
613 pthread_cond_signal(&info->cond);
614}
615
616void
617background_thread_prefork0(tsdn_t *tsdn) {
618 malloc_mutex_prefork(tsdn, &background_thread_lock);
619 background_thread_enabled_at_fork = background_thread_enabled();
620}
621
622void
623background_thread_prefork1(tsdn_t *tsdn) {
624 for (unsigned i = 0; i < max_background_threads; i++) {
625 malloc_mutex_prefork(tsdn, &background_thread_info[i].mtx);
626 }
627}
628
629void
630background_thread_postfork_parent(tsdn_t *tsdn) {
631 for (unsigned i = 0; i < max_background_threads; i++) {
632 malloc_mutex_postfork_parent(tsdn,
633 &background_thread_info[i].mtx);
634 }
635 malloc_mutex_postfork_parent(tsdn, &background_thread_lock);
636}
637
638void
639background_thread_postfork_child(tsdn_t *tsdn) {
640 for (unsigned i = 0; i < max_background_threads; i++) {
641 malloc_mutex_postfork_child(tsdn,
642 &background_thread_info[i].mtx);
643 }
644 malloc_mutex_postfork_child(tsdn, &background_thread_lock);
645 if (!background_thread_enabled_at_fork) {
646 return;
647 }
648
649 /* Clear background_thread state (reset to disabled for child). */
650 malloc_mutex_lock(tsdn, &background_thread_lock);
651 n_background_threads = 0;
652 background_thread_enabled_set(tsdn, false);
653 for (unsigned i = 0; i < max_background_threads; i++) {
654 background_thread_info_t *info = &background_thread_info[i];
655 malloc_mutex_lock(tsdn, &info->mtx);
656 info->state = background_thread_stopped;
657 int ret = pthread_cond_init(&info->cond, NULL);
658 assert(ret == 0);
659 background_thread_info_init(tsdn, info);
660 malloc_mutex_unlock(tsdn, &info->mtx);
661 }
662 malloc_mutex_unlock(tsdn, &background_thread_lock);
663}
664
665bool
666background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) {
667 assert(config_stats);
668 malloc_mutex_lock(tsdn, &background_thread_lock);
669 if (!background_thread_enabled()) {
670 malloc_mutex_unlock(tsdn, &background_thread_lock);
671 return true;
672 }
673
674 nstime_init_zero(&stats->run_interval);
675 memset(&stats->max_counter_per_bg_thd, 0, sizeof(mutex_prof_data_t));
676
677 uint64_t num_runs = 0;
678 stats->num_threads = n_background_threads;
679 for (unsigned i = 0; i < max_background_threads; i++) {
680 background_thread_info_t *info = &background_thread_info[i];
681 if (malloc_mutex_trylock(tsdn, &info->mtx)) {
682 /*
683 * Each background thread run may take a long time;
684 * avoid waiting on the stats if the thread is active.
685 */
686 continue;
687 }
688 if (info->state != background_thread_stopped) {
689 num_runs += info->tot_n_runs;
690 nstime_add(&stats->run_interval, &info->tot_sleep_time);
691 malloc_mutex_prof_max_update(tsdn,
692 &stats->max_counter_per_bg_thd, &info->mtx);
693 }
694 malloc_mutex_unlock(tsdn, &info->mtx);
695 }
696 stats->num_runs = num_runs;
697 if (num_runs > 0) {
698 nstime_idivide(&stats->run_interval, num_runs);
699 }
700 malloc_mutex_unlock(tsdn, &background_thread_lock);
701
702 return false;
703}
704
705#undef BACKGROUND_THREAD_NPAGES_THRESHOLD
706#undef BILLION
707#undef BACKGROUND_THREAD_MIN_INTERVAL_NS
708
709#ifdef JEMALLOC_HAVE_DLSYM
710#include <dlfcn.h>
711#endif
712
713static bool
714pthread_create_fptr_init(void) {
715 if (pthread_create_fptr != NULL) {
716 return false;
717 }
718 /*
719 * Try the next symbol first, because 1) when use lazy_lock we have a
720 * wrapper for pthread_create; and 2) application may define its own
721 * wrapper as well (and can call malloc within the wrapper).
722 */
723#ifdef JEMALLOC_HAVE_DLSYM
724 pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create");
725#else
726 pthread_create_fptr = NULL;
727#endif
728 if (pthread_create_fptr == NULL) {
729 if (config_lazy_lock) {
730 malloc_write("<jemalloc>: Error in dlsym(RTLD_NEXT, "
731 "\"pthread_create\")\n");
732 abort();
733 } else {
734 /* Fall back to the default symbol. */
735 pthread_create_fptr = pthread_create;
736 }
737 }
738
739 return false;
740}
741
742/*
743 * When lazy lock is enabled, we need to make sure setting isthreaded before
744 * taking any background_thread locks. This is called early in ctl (instead of
745 * wait for the pthread_create calls to trigger) because the mutex is required
746 * before creating background threads.
747 */
748void
749background_thread_ctl_init(tsdn_t *tsdn) {
750 malloc_mutex_assert_not_owner(tsdn, &background_thread_lock);
751#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
752 pthread_create_fptr_init();
753 pthread_create_wrapper_init();
754#endif
755}
756
757#endif /* defined(JEMALLOC_BACKGROUND_THREAD) */
758
759bool
760background_thread_boot0(void) {
761 if (!have_background_thread && opt_background_thread) {
762 malloc_printf("<jemalloc>: option background_thread currently "
763 "supports pthread only\n");
764 return true;
765 }
766#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
767 if ((config_lazy_lock || opt_background_thread) &&
768 pthread_create_fptr_init()) {
769 return true;
770 }
771#endif
772 return false;
773}
774
775bool
776background_thread_boot1(tsdn_t *tsdn, base_t *base) {
777#ifdef JEMALLOC_BACKGROUND_THREAD
778 assert(have_background_thread);
779 assert(narenas_total_get() > 0);
780
781 if (opt_max_background_threads > MAX_BACKGROUND_THREAD_LIMIT) {
782 opt_max_background_threads = DEFAULT_NUM_BACKGROUND_THREAD;
783 }
784 max_background_threads = opt_max_background_threads;
785
786 background_thread_enabled_set(tsdn, opt_background_thread);
787 if (malloc_mutex_init(&background_thread_lock,
788 "background_thread_global",
789 WITNESS_RANK_BACKGROUND_THREAD_GLOBAL,
790 malloc_mutex_rank_exclusive)) {
791 return true;
792 }
793
794 background_thread_info = (background_thread_info_t *)base_alloc(tsdn,
795 base, opt_max_background_threads *
796 sizeof(background_thread_info_t), CACHELINE);
797 if (background_thread_info == NULL) {
798 return true;
799 }
800
801 for (unsigned i = 0; i < max_background_threads; i++) {
802 background_thread_info_t *info = &background_thread_info[i];
803 /* Thread mutex is rank_inclusive because of thread0. */
804 if (malloc_mutex_init(&info->mtx, "background_thread",
805 WITNESS_RANK_BACKGROUND_THREAD,
806 malloc_mutex_address_ordered)) {
807 return true;
808 }
809 if (pthread_cond_init(&info->cond, NULL)) {
810 return true;
811 }
812 malloc_mutex_lock(tsdn, &info->mtx);
813 info->state = background_thread_stopped;
814 background_thread_info_init(tsdn, info);
815 malloc_mutex_unlock(tsdn, &info->mtx);
816 }
817#endif
818
819 return false;
820}
generated by cgit v1.2.3 (git 2.46.0) at 2026-04-30 12:20:14 +0000