diff options
Diffstat (limited to 'examples/redis-unstable/deps/jemalloc/src/tsd.c')
| -rw-r--r-- | examples/redis-unstable/deps/jemalloc/src/tsd.c | 549 |
1 files changed, 0 insertions, 549 deletions
diff --git a/examples/redis-unstable/deps/jemalloc/src/tsd.c b/examples/redis-unstable/deps/jemalloc/src/tsd.c deleted file mode 100644 index e8e4f3a..0000000 --- a/examples/redis-unstable/deps/jemalloc/src/tsd.c +++ /dev/null @@ -1,549 +0,0 @@ -#include "jemalloc/internal/jemalloc_preamble.h" -#include "jemalloc/internal/jemalloc_internal_includes.h" - -#include "jemalloc/internal/assert.h" -#include "jemalloc/internal/san.h" -#include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/rtree.h" - -/******************************************************************************/ -/* Data. */ - -/* TSD_INITIALIZER triggers "-Wmissing-field-initializer" */ -JEMALLOC_DIAGNOSTIC_PUSH -JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS - -#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP -JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls = TSD_INITIALIZER; -JEMALLOC_TSD_TYPE_ATTR(bool) JEMALLOC_TLS_MODEL tsd_initialized = false; -bool tsd_booted = false; -#elif (defined(JEMALLOC_TLS)) -JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls = TSD_INITIALIZER; -pthread_key_t tsd_tsd; -bool tsd_booted = false; -#elif (defined(_WIN32)) -DWORD tsd_tsd; -tsd_wrapper_t tsd_boot_wrapper = {false, TSD_INITIALIZER}; -bool tsd_booted = false; -#else - -/* - * This contains a mutex, but it's pretty convenient to allow the mutex code to - * have a dependency on tsd. So we define the struct here, and only refer to it - * by pointer in the header. - */ -struct tsd_init_head_s { - ql_head(tsd_init_block_t) blocks; - malloc_mutex_t lock; -}; - -pthread_key_t tsd_tsd; -tsd_init_head_t tsd_init_head = { - ql_head_initializer(blocks), - MALLOC_MUTEX_INITIALIZER -}; - -tsd_wrapper_t tsd_boot_wrapper = { - false, - TSD_INITIALIZER -}; -bool tsd_booted = false; -#endif - -JEMALLOC_DIAGNOSTIC_POP - -/******************************************************************************/ - -/* A list of all the tsds in the nominal state. */ -typedef ql_head(tsd_t) tsd_list_t; -static tsd_list_t tsd_nominal_tsds = ql_head_initializer(tsd_nominal_tsds); -static malloc_mutex_t tsd_nominal_tsds_lock; - -/* How many slow-path-enabling features are turned on. */ -static atomic_u32_t tsd_global_slow_count = ATOMIC_INIT(0); - -static bool -tsd_in_nominal_list(tsd_t *tsd) { - tsd_t *tsd_list; - bool found = false; - /* - * We don't know that tsd is nominal; it might not be safe to get data - * out of it here. - */ - malloc_mutex_lock(TSDN_NULL, &tsd_nominal_tsds_lock); - ql_foreach(tsd_list, &tsd_nominal_tsds, TSD_MANGLE(tsd_link)) { - if (tsd == tsd_list) { - found = true; - break; - } - } - malloc_mutex_unlock(TSDN_NULL, &tsd_nominal_tsds_lock); - return found; -} - -static void -tsd_add_nominal(tsd_t *tsd) { - assert(!tsd_in_nominal_list(tsd)); - assert(tsd_state_get(tsd) <= tsd_state_nominal_max); - ql_elm_new(tsd, TSD_MANGLE(tsd_link)); - malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); - ql_tail_insert(&tsd_nominal_tsds, tsd, TSD_MANGLE(tsd_link)); - malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); -} - -static void -tsd_remove_nominal(tsd_t *tsd) { - assert(tsd_in_nominal_list(tsd)); - assert(tsd_state_get(tsd) <= tsd_state_nominal_max); - malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); - ql_remove(&tsd_nominal_tsds, tsd, TSD_MANGLE(tsd_link)); - malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); -} - -static void -tsd_force_recompute(tsdn_t *tsdn) { - /* - * The stores to tsd->state here need to synchronize with the exchange - * in tsd_slow_update. - */ - atomic_fence(ATOMIC_RELEASE); - malloc_mutex_lock(tsdn, &tsd_nominal_tsds_lock); - tsd_t *remote_tsd; - ql_foreach(remote_tsd, &tsd_nominal_tsds, TSD_MANGLE(tsd_link)) { - assert(tsd_atomic_load(&remote_tsd->state, ATOMIC_RELAXED) - <= tsd_state_nominal_max); - tsd_atomic_store(&remote_tsd->state, - tsd_state_nominal_recompute, ATOMIC_RELAXED); - /* See comments in te_recompute_fast_threshold(). */ - atomic_fence(ATOMIC_SEQ_CST); - te_next_event_fast_set_non_nominal(remote_tsd); - } - malloc_mutex_unlock(tsdn, &tsd_nominal_tsds_lock); -} - -void -tsd_global_slow_inc(tsdn_t *tsdn) { - atomic_fetch_add_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED); - /* - * We unconditionally force a recompute, even if the global slow count - * was already positive. If we didn't, then it would be possible for us - * to return to the user, have the user synchronize externally with some - * other thread, and then have that other thread not have picked up the - * update yet (since the original incrementing thread might still be - * making its way through the tsd list). - */ - tsd_force_recompute(tsdn); -} - -void tsd_global_slow_dec(tsdn_t *tsdn) { - atomic_fetch_sub_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED); - /* See the note in ..._inc(). */ - tsd_force_recompute(tsdn); -} - -static bool -tsd_local_slow(tsd_t *tsd) { - return !tsd_tcache_enabled_get(tsd) - || tsd_reentrancy_level_get(tsd) > 0; -} - -bool -tsd_global_slow() { - return atomic_load_u32(&tsd_global_slow_count, ATOMIC_RELAXED) > 0; -} - -/******************************************************************************/ - -static uint8_t -tsd_state_compute(tsd_t *tsd) { - if (!tsd_nominal(tsd)) { - return tsd_state_get(tsd); - } - /* We're in *a* nominal state; but which one? */ - if (malloc_slow || tsd_local_slow(tsd) || tsd_global_slow()) { - return tsd_state_nominal_slow; - } else { - return tsd_state_nominal; - } -} - -void -tsd_slow_update(tsd_t *tsd) { - uint8_t old_state; - do { - uint8_t new_state = tsd_state_compute(tsd); - old_state = tsd_atomic_exchange(&tsd->state, new_state, - ATOMIC_ACQUIRE); - } while (old_state == tsd_state_nominal_recompute); - - te_recompute_fast_threshold(tsd); -} - -void -tsd_state_set(tsd_t *tsd, uint8_t new_state) { - /* Only the tsd module can change the state *to* recompute. */ - assert(new_state != tsd_state_nominal_recompute); - uint8_t old_state = tsd_atomic_load(&tsd->state, ATOMIC_RELAXED); - if (old_state > tsd_state_nominal_max) { - /* - * Not currently in the nominal list, but it might need to be - * inserted there. - */ - assert(!tsd_in_nominal_list(tsd)); - tsd_atomic_store(&tsd->state, new_state, ATOMIC_RELAXED); - if (new_state <= tsd_state_nominal_max) { - tsd_add_nominal(tsd); - } - } else { - /* - * We're currently nominal. If the new state is non-nominal, - * great; we take ourselves off the list and just enter the new - * state. - */ - assert(tsd_in_nominal_list(tsd)); - if (new_state > tsd_state_nominal_max) { - tsd_remove_nominal(tsd); - tsd_atomic_store(&tsd->state, new_state, - ATOMIC_RELAXED); - } else { - /* - * This is the tricky case. We're transitioning from - * one nominal state to another. The caller can't know - * about any races that are occurring at the same time, - * so we always have to recompute no matter what. - */ - tsd_slow_update(tsd); - } - } - te_recompute_fast_threshold(tsd); -} - -static void -tsd_prng_state_init(tsd_t *tsd) { - /* - * A nondeterministic seed based on the address of tsd reduces - * the likelihood of lockstep non-uniform cache index - * utilization among identical concurrent processes, but at the - * cost of test repeatability. For debug builds, instead use a - * deterministic seed. - */ - *tsd_prng_statep_get(tsd) = config_debug ? 0 : - (uint64_t)(uintptr_t)tsd; -} - -static bool -tsd_data_init(tsd_t *tsd) { - /* - * We initialize the rtree context first (before the tcache), since the - * tcache initialization depends on it. - */ - rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd)); - tsd_prng_state_init(tsd); - tsd_te_init(tsd); /* event_init may use the prng state above. */ - tsd_san_init(tsd); - return tsd_tcache_enabled_data_init(tsd); -} - -static void -assert_tsd_data_cleanup_done(tsd_t *tsd) { - assert(!tsd_nominal(tsd)); - assert(!tsd_in_nominal_list(tsd)); - assert(*tsd_arenap_get_unsafe(tsd) == NULL); - assert(*tsd_iarenap_get_unsafe(tsd) == NULL); - assert(*tsd_tcache_enabledp_get_unsafe(tsd) == false); - assert(*tsd_prof_tdatap_get_unsafe(tsd) == NULL); -} - -static bool -tsd_data_init_nocleanup(tsd_t *tsd) { - assert(tsd_state_get(tsd) == tsd_state_reincarnated || - tsd_state_get(tsd) == tsd_state_minimal_initialized); - /* - * During reincarnation, there is no guarantee that the cleanup function - * will be called (deallocation may happen after all tsd destructors). - * We set up tsd in a way that no cleanup is needed. - */ - rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd)); - *tsd_tcache_enabledp_get_unsafe(tsd) = false; - *tsd_reentrancy_levelp_get(tsd) = 1; - tsd_prng_state_init(tsd); - tsd_te_init(tsd); /* event_init may use the prng state above. */ - tsd_san_init(tsd); - assert_tsd_data_cleanup_done(tsd); - - return false; -} - -tsd_t * -tsd_fetch_slow(tsd_t *tsd, bool minimal) { - assert(!tsd_fast(tsd)); - - if (tsd_state_get(tsd) == tsd_state_nominal_slow) { - /* - * On slow path but no work needed. Note that we can't - * necessarily *assert* that we're slow, because we might be - * slow because of an asynchronous modification to global state, - * which might be asynchronously modified *back*. - */ - } else if (tsd_state_get(tsd) == tsd_state_nominal_recompute) { - tsd_slow_update(tsd); - } else if (tsd_state_get(tsd) == tsd_state_uninitialized) { - if (!minimal) { - if (tsd_booted) { - tsd_state_set(tsd, tsd_state_nominal); - tsd_slow_update(tsd); - /* Trigger cleanup handler registration. */ - tsd_set(tsd); - tsd_data_init(tsd); - } - } else { - tsd_state_set(tsd, tsd_state_minimal_initialized); - tsd_set(tsd); - tsd_data_init_nocleanup(tsd); - } - } else if (tsd_state_get(tsd) == tsd_state_minimal_initialized) { - if (!minimal) { - /* Switch to fully initialized. */ - tsd_state_set(tsd, tsd_state_nominal); - assert(*tsd_reentrancy_levelp_get(tsd) >= 1); - (*tsd_reentrancy_levelp_get(tsd))--; - tsd_slow_update(tsd); - tsd_data_init(tsd); - } else { - assert_tsd_data_cleanup_done(tsd); - } - } else if (tsd_state_get(tsd) == tsd_state_purgatory) { - tsd_state_set(tsd, tsd_state_reincarnated); - tsd_set(tsd); - tsd_data_init_nocleanup(tsd); - } else { - assert(tsd_state_get(tsd) == tsd_state_reincarnated); - } - - return tsd; -} - -void * -malloc_tsd_malloc(size_t size) { - return a0malloc(CACHELINE_CEILING(size)); -} - -void -malloc_tsd_dalloc(void *wrapper) { - a0dalloc(wrapper); -} - -#if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32) -static unsigned ncleanups; -static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX]; - -#ifndef _WIN32 -JEMALLOC_EXPORT -#endif -void -_malloc_thread_cleanup(void) { - bool pending[MALLOC_TSD_CLEANUPS_MAX], again; - unsigned i; - - for (i = 0; i < ncleanups; i++) { - pending[i] = true; - } - - do { - again = false; - for (i = 0; i < ncleanups; i++) { - if (pending[i]) { - pending[i] = cleanups[i](); - if (pending[i]) { - again = true; - } - } - } - } while (again); -} - -#ifndef _WIN32 -JEMALLOC_EXPORT -#endif -void -_malloc_tsd_cleanup_register(bool (*f)(void)) { - assert(ncleanups < MALLOC_TSD_CLEANUPS_MAX); - cleanups[ncleanups] = f; - ncleanups++; -} - -#endif - -static void -tsd_do_data_cleanup(tsd_t *tsd) { - prof_tdata_cleanup(tsd); - iarena_cleanup(tsd); - arena_cleanup(tsd); - tcache_cleanup(tsd); - witnesses_cleanup(tsd_witness_tsdp_get_unsafe(tsd)); - *tsd_reentrancy_levelp_get(tsd) = 1; -} - -void -tsd_cleanup(void *arg) { - tsd_t *tsd = (tsd_t *)arg; - - switch (tsd_state_get(tsd)) { - case tsd_state_uninitialized: - /* Do nothing. */ - break; - case tsd_state_minimal_initialized: - /* This implies the thread only did free() in its life time. */ - /* Fall through. */ - case tsd_state_reincarnated: - /* - * Reincarnated means another destructor deallocated memory - * after the destructor was called. Cleanup isn't required but - * is still called for testing and completeness. - */ - assert_tsd_data_cleanup_done(tsd); - JEMALLOC_FALLTHROUGH; - case tsd_state_nominal: - case tsd_state_nominal_slow: - tsd_do_data_cleanup(tsd); - tsd_state_set(tsd, tsd_state_purgatory); - tsd_set(tsd); - break; - case tsd_state_purgatory: - /* - * The previous time this destructor was called, we set the - * state to tsd_state_purgatory so that other destructors - * wouldn't cause re-creation of the tsd. This time, do - * nothing, and do not request another callback. - */ - break; - default: - not_reached(); - } -#ifdef JEMALLOC_JET - test_callback_t test_callback = *tsd_test_callbackp_get_unsafe(tsd); - int *data = tsd_test_datap_get_unsafe(tsd); - if (test_callback != NULL) { - test_callback(data); - } -#endif -} - -tsd_t * -malloc_tsd_boot0(void) { - tsd_t *tsd; - -#if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32) - ncleanups = 0; -#endif - if (malloc_mutex_init(&tsd_nominal_tsds_lock, "tsd_nominal_tsds_lock", - WITNESS_RANK_OMIT, malloc_mutex_rank_exclusive)) { - return NULL; - } - if (tsd_boot0()) { - return NULL; - } - tsd = tsd_fetch(); - return tsd; -} - -void -malloc_tsd_boot1(void) { - tsd_boot1(); - tsd_t *tsd = tsd_fetch(); - /* malloc_slow has been set properly. Update tsd_slow. */ - tsd_slow_update(tsd); -} - -#ifdef _WIN32 -static BOOL WINAPI -_tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) { - switch (fdwReason) { -#ifdef JEMALLOC_LAZY_LOCK - case DLL_THREAD_ATTACH: - isthreaded = true; - break; -#endif - case DLL_THREAD_DETACH: - _malloc_thread_cleanup(); - break; - default: - break; - } - return true; -} - -/* - * We need to be able to say "read" here (in the "pragma section"), but have - * hooked "read". We won't read for the rest of the file, so we can get away - * with unhooking. - */ -#ifdef read -# undef read -#endif - -#ifdef _MSC_VER -# ifdef _M_IX86 -# pragma comment(linker, "/INCLUDE:__tls_used") -# pragma comment(linker, "/INCLUDE:_tls_callback") -# else -# pragma comment(linker, "/INCLUDE:_tls_used") -# pragma comment(linker, "/INCLUDE:" STRINGIFY(tls_callback) ) -# endif -# pragma section(".CRT$XLY",long,read) -#endif -JEMALLOC_SECTION(".CRT$XLY") JEMALLOC_ATTR(used) -BOOL (WINAPI *const tls_callback)(HINSTANCE hinstDLL, - DWORD fdwReason, LPVOID lpvReserved) = _tls_callback; -#endif - -#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ - !defined(_WIN32)) -void * -tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block) { - pthread_t self = pthread_self(); - tsd_init_block_t *iter; - - /* Check whether this thread has already inserted into the list. */ - malloc_mutex_lock(TSDN_NULL, &head->lock); - ql_foreach(iter, &head->blocks, link) { - if (iter->thread == self) { - malloc_mutex_unlock(TSDN_NULL, &head->lock); - return iter->data; - } - } - /* Insert block into list. */ - ql_elm_new(block, link); - block->thread = self; - ql_tail_insert(&head->blocks, block, link); - malloc_mutex_unlock(TSDN_NULL, &head->lock); - return NULL; -} - -void -tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block) { - malloc_mutex_lock(TSDN_NULL, &head->lock); - ql_remove(&head->blocks, block, link); - malloc_mutex_unlock(TSDN_NULL, &head->lock); -} -#endif - -void -tsd_prefork(tsd_t *tsd) { - malloc_mutex_prefork(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); -} - -void -tsd_postfork_parent(tsd_t *tsd) { - malloc_mutex_postfork_parent(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); -} - -void -tsd_postfork_child(tsd_t *tsd) { - malloc_mutex_postfork_child(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); - ql_new(&tsd_nominal_tsds); - - if (tsd_state_get(tsd) <= tsd_state_nominal_max) { - tsd_add_nominal(tsd); - } -} |
