diff options
Diffstat (limited to 'examples/redis-unstable/deps/jemalloc/src/large.c')
| -rw-r--r-- | examples/redis-unstable/deps/jemalloc/src/large.c | 322 |
1 files changed, 0 insertions, 322 deletions
diff --git a/examples/redis-unstable/deps/jemalloc/src/large.c b/examples/redis-unstable/deps/jemalloc/src/large.c deleted file mode 100644 index 5fc4bf5..0000000 --- a/examples/redis-unstable/deps/jemalloc/src/large.c +++ /dev/null @@ -1,322 +0,0 @@ -#include "jemalloc/internal/jemalloc_preamble.h" -#include "jemalloc/internal/jemalloc_internal_includes.h" - -#include "jemalloc/internal/assert.h" -#include "jemalloc/internal/emap.h" -#include "jemalloc/internal/extent_mmap.h" -#include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/prof_recent.h" -#include "jemalloc/internal/util.h" - -/******************************************************************************/ - -void * -large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero) { - assert(usize == sz_s2u(usize)); - - return large_palloc(tsdn, arena, usize, CACHELINE, zero); -} - -void * -large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, - bool zero) { - size_t ausize; - edata_t *edata; - UNUSED bool idump JEMALLOC_CC_SILENCE_INIT(false); - - assert(!tsdn_null(tsdn) || arena != NULL); - - ausize = sz_sa2u(usize, alignment); - if (unlikely(ausize == 0 || ausize > SC_LARGE_MAXCLASS)) { - return NULL; - } - - if (likely(!tsdn_null(tsdn))) { - arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, usize); - } - if (unlikely(arena == NULL) || (edata = arena_extent_alloc_large(tsdn, - arena, usize, alignment, zero)) == NULL) { - return NULL; - } - - /* See comments in arena_bin_slabs_full_insert(). */ - if (!arena_is_auto(arena)) { - /* Insert edata into large. */ - malloc_mutex_lock(tsdn, &arena->large_mtx); - edata_list_active_append(&arena->large, edata); - malloc_mutex_unlock(tsdn, &arena->large_mtx); - } - - arena_decay_tick(tsdn, arena); - return edata_addr_get(edata); -} - -static bool -large_ralloc_no_move_shrink(tsdn_t *tsdn, edata_t *edata, size_t usize) { - arena_t *arena = arena_get_from_edata(edata); - ehooks_t *ehooks = arena_get_ehooks(arena); - size_t old_size = edata_size_get(edata); - size_t old_usize = edata_usize_get(edata); - - assert(old_usize > usize); - - if (ehooks_split_will_fail(ehooks)) { - return true; - } - - bool deferred_work_generated = false; - bool err = pa_shrink(tsdn, &arena->pa_shard, edata, old_size, - usize + sz_large_pad, sz_size2index(usize), - &deferred_work_generated); - if (err) { - return true; - } - if (deferred_work_generated) { - arena_handle_deferred_work(tsdn, arena); - } - arena_extent_ralloc_large_shrink(tsdn, arena, edata, old_usize); - - return false; -} - -static bool -large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize, - bool zero) { - arena_t *arena = arena_get_from_edata(edata); - - size_t old_size = edata_size_get(edata); - size_t old_usize = edata_usize_get(edata); - size_t new_size = usize + sz_large_pad; - - szind_t szind = sz_size2index(usize); - - bool deferred_work_generated = false; - bool err = pa_expand(tsdn, &arena->pa_shard, edata, old_size, new_size, - szind, zero, &deferred_work_generated); - - if (deferred_work_generated) { - arena_handle_deferred_work(tsdn, arena); - } - - if (err) { - return true; - } - - if (zero) { - if (opt_cache_oblivious) { - assert(sz_large_pad == PAGE); - /* - * Zero the trailing bytes of the original allocation's - * last page, since they are in an indeterminate state. - * There will always be trailing bytes, because ptr's - * offset from the beginning of the extent is a multiple - * of CACHELINE in [0 .. PAGE). - */ - void *zbase = (void *) - ((uintptr_t)edata_addr_get(edata) + old_usize); - void *zpast = PAGE_ADDR2BASE((void *)((uintptr_t)zbase + - PAGE)); - size_t nzero = (uintptr_t)zpast - (uintptr_t)zbase; - assert(nzero > 0); - memset(zbase, 0, nzero); - } - } - arena_extent_ralloc_large_expand(tsdn, arena, edata, old_usize); - - return false; -} - -bool -large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min, - size_t usize_max, bool zero) { - size_t oldusize = edata_usize_get(edata); - - /* The following should have been caught by callers. */ - assert(usize_min > 0 && usize_max <= SC_LARGE_MAXCLASS); - /* Both allocation sizes must be large to avoid a move. */ - assert(oldusize >= SC_LARGE_MINCLASS - && usize_max >= SC_LARGE_MINCLASS); - - if (usize_max > oldusize) { - /* Attempt to expand the allocation in-place. */ - if (!large_ralloc_no_move_expand(tsdn, edata, usize_max, - zero)) { - arena_decay_tick(tsdn, arena_get_from_edata(edata)); - return false; - } - /* Try again, this time with usize_min. */ - if (usize_min < usize_max && usize_min > oldusize && - large_ralloc_no_move_expand(tsdn, edata, usize_min, zero)) { - arena_decay_tick(tsdn, arena_get_from_edata(edata)); - return false; - } - } - - /* - * Avoid moving the allocation if the existing extent size accommodates - * the new size. - */ - if (oldusize >= usize_min && oldusize <= usize_max) { - arena_decay_tick(tsdn, arena_get_from_edata(edata)); - return false; - } - - /* Attempt to shrink the allocation in-place. */ - if (oldusize > usize_max) { - if (!large_ralloc_no_move_shrink(tsdn, edata, usize_max)) { - arena_decay_tick(tsdn, arena_get_from_edata(edata)); - return false; - } - } - return true; -} - -static void * -large_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, - size_t alignment, bool zero) { - if (alignment <= CACHELINE) { - return large_malloc(tsdn, arena, usize, zero); - } - return large_palloc(tsdn, arena, usize, alignment, zero); -} - -void * -large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, - size_t alignment, bool zero, tcache_t *tcache, - hook_ralloc_args_t *hook_args) { - edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr); - - size_t oldusize = edata_usize_get(edata); - /* The following should have been caught by callers. */ - assert(usize > 0 && usize <= SC_LARGE_MAXCLASS); - /* Both allocation sizes must be large to avoid a move. */ - assert(oldusize >= SC_LARGE_MINCLASS - && usize >= SC_LARGE_MINCLASS); - - /* Try to avoid moving the allocation. */ - if (!large_ralloc_no_move(tsdn, edata, usize, usize, zero)) { - hook_invoke_expand(hook_args->is_realloc - ? hook_expand_realloc : hook_expand_rallocx, ptr, oldusize, - usize, (uintptr_t)ptr, hook_args->args); - return edata_addr_get(edata); - } - - /* - * usize and old size are different enough that we need to use a - * different size class. In that case, fall back to allocating new - * space and copying. - */ - void *ret = large_ralloc_move_helper(tsdn, arena, usize, alignment, - zero); - if (ret == NULL) { - return NULL; - } - - hook_invoke_alloc(hook_args->is_realloc - ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret, - hook_args->args); - hook_invoke_dalloc(hook_args->is_realloc - ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); - - size_t copysize = (usize < oldusize) ? usize : oldusize; - memcpy(ret, edata_addr_get(edata), copysize); - isdalloct(tsdn, edata_addr_get(edata), oldusize, tcache, NULL, true); - return ret; -} - -/* - * locked indicates whether the arena's large_mtx is currently held. - */ -static void -large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata, - bool locked) { - if (!locked) { - /* See comments in arena_bin_slabs_full_insert(). */ - if (!arena_is_auto(arena)) { - malloc_mutex_lock(tsdn, &arena->large_mtx); - edata_list_active_remove(&arena->large, edata); - malloc_mutex_unlock(tsdn, &arena->large_mtx); - } - } else { - /* Only hold the large_mtx if necessary. */ - if (!arena_is_auto(arena)) { - malloc_mutex_assert_owner(tsdn, &arena->large_mtx); - edata_list_active_remove(&arena->large, edata); - } - } - arena_extent_dalloc_large_prep(tsdn, arena, edata); -} - -static void -large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata) { - bool deferred_work_generated = false; - pa_dalloc(tsdn, &arena->pa_shard, edata, &deferred_work_generated); - if (deferred_work_generated) { - arena_handle_deferred_work(tsdn, arena); - } -} - -void -large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata) { - large_dalloc_prep_impl(tsdn, arena_get_from_edata(edata), edata, true); -} - -void -large_dalloc_finish(tsdn_t *tsdn, edata_t *edata) { - large_dalloc_finish_impl(tsdn, arena_get_from_edata(edata), edata); -} - -void -large_dalloc(tsdn_t *tsdn, edata_t *edata) { - arena_t *arena = arena_get_from_edata(edata); - large_dalloc_prep_impl(tsdn, arena, edata, false); - large_dalloc_finish_impl(tsdn, arena, edata); - arena_decay_tick(tsdn, arena); -} - -size_t -large_salloc(tsdn_t *tsdn, const edata_t *edata) { - return edata_usize_get(edata); -} - -void -large_prof_info_get(tsd_t *tsd, edata_t *edata, prof_info_t *prof_info, - bool reset_recent) { - assert(prof_info != NULL); - - prof_tctx_t *alloc_tctx = edata_prof_tctx_get(edata); - prof_info->alloc_tctx = alloc_tctx; - - if ((uintptr_t)alloc_tctx > (uintptr_t)1U) { - nstime_copy(&prof_info->alloc_time, - edata_prof_alloc_time_get(edata)); - prof_info->alloc_size = edata_prof_alloc_size_get(edata); - if (reset_recent) { - /* - * Reset the pointer on the recent allocation record, - * so that this allocation is recorded as released. - */ - prof_recent_alloc_reset(tsd, edata); - } - } -} - -static void -large_prof_tctx_set(edata_t *edata, prof_tctx_t *tctx) { - edata_prof_tctx_set(edata, tctx); -} - -void -large_prof_tctx_reset(edata_t *edata) { - large_prof_tctx_set(edata, (prof_tctx_t *)(uintptr_t)1U); -} - -void -large_prof_info_set(edata_t *edata, prof_tctx_t *tctx, size_t size) { - nstime_t t; - nstime_prof_init_update(&t); - edata_prof_alloc_time_set(edata, &t); - edata_prof_alloc_size_set(edata, size); - edata_prof_recent_alloc_init(edata); - large_prof_tctx_set(edata, tctx); -} |