diff options
| author | Mitja Felicijan <mitja.felicijan@gmail.com> | 2026-01-21 22:52:54 +0100 |
|---|---|---|
| committer | Mitja Felicijan <mitja.felicijan@gmail.com> | 2026-01-21 22:52:54 +0100 |
| commit | dcacc00e3750300617ba6e16eb346713f91a783a (patch) | |
| tree | 38e2d4fb5ed9d119711d4295c6eda4b014af73fd /examples/redis-unstable/deps/jemalloc/src/sec.c | |
| parent | 58dac10aeb8f5a041c46bddbeaf4c7966a99b998 (diff) | |
| download | crep-dcacc00e3750300617ba6e16eb346713f91a783a.tar.gz | |
Remove testing data
Diffstat (limited to 'examples/redis-unstable/deps/jemalloc/src/sec.c')
| -rw-r--r-- | examples/redis-unstable/deps/jemalloc/src/sec.c | 422 |
1 files changed, 0 insertions, 422 deletions
diff --git a/examples/redis-unstable/deps/jemalloc/src/sec.c b/examples/redis-unstable/deps/jemalloc/src/sec.c deleted file mode 100644 index df67559..0000000 --- a/examples/redis-unstable/deps/jemalloc/src/sec.c +++ /dev/null @@ -1,422 +0,0 @@ -#include "jemalloc/internal/jemalloc_preamble.h" -#include "jemalloc/internal/jemalloc_internal_includes.h" - -#include "jemalloc/internal/sec.h" - -static edata_t *sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, - size_t alignment, bool zero, bool guarded, bool frequent_reuse, - bool *deferred_work_generated); -static bool sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, - size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated); -static bool sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, - size_t old_size, size_t new_size, bool *deferred_work_generated); -static void sec_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata, - bool *deferred_work_generated); - -static void -sec_bin_init(sec_bin_t *bin) { - bin->being_batch_filled = false; - bin->bytes_cur = 0; - edata_list_active_init(&bin->freelist); -} - -bool -sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, pai_t *fallback, - const sec_opts_t *opts) { - assert(opts->max_alloc >= PAGE); - - size_t max_alloc = PAGE_FLOOR(opts->max_alloc); - pszind_t npsizes = sz_psz2ind(max_alloc) + 1; - - size_t sz_shards = opts->nshards * sizeof(sec_shard_t); - size_t sz_bins = opts->nshards * (size_t)npsizes * sizeof(sec_bin_t); - size_t sz_alloc = sz_shards + sz_bins; - void *dynalloc = base_alloc(tsdn, base, sz_alloc, CACHELINE); - if (dynalloc == NULL) { - return true; - } - sec_shard_t *shard_cur = (sec_shard_t *)dynalloc; - sec->shards = shard_cur; - sec_bin_t *bin_cur = (sec_bin_t *)&shard_cur[opts->nshards]; - /* Just for asserts, below. */ - sec_bin_t *bin_start = bin_cur; - - for (size_t i = 0; i < opts->nshards; i++) { - sec_shard_t *shard = shard_cur; - shard_cur++; - bool err = malloc_mutex_init(&shard->mtx, "sec_shard", - WITNESS_RANK_SEC_SHARD, malloc_mutex_rank_exclusive); - if (err) { - return true; - } - shard->enabled = true; - shard->bins = bin_cur; - for (pszind_t j = 0; j < npsizes; j++) { - sec_bin_init(&shard->bins[j]); - bin_cur++; - } - shard->bytes_cur = 0; - shard->to_flush_next = 0; - } - /* - * Should have exactly matched the bin_start to the first unused byte - * after the shards. - */ - assert((void *)shard_cur == (void *)bin_start); - /* And the last bin to use up the last bytes of the allocation. */ - assert((char *)bin_cur == ((char *)dynalloc + sz_alloc)); - sec->fallback = fallback; - - - sec->opts = *opts; - sec->npsizes = npsizes; - - /* - * Initialize these last so that an improper use of an SEC whose - * initialization failed will segfault in an easy-to-spot way. - */ - sec->pai.alloc = &sec_alloc; - sec->pai.alloc_batch = &pai_alloc_batch_default; - sec->pai.expand = &sec_expand; - sec->pai.shrink = &sec_shrink; - sec->pai.dalloc = &sec_dalloc; - sec->pai.dalloc_batch = &pai_dalloc_batch_default; - - return false; -} - -static sec_shard_t * -sec_shard_pick(tsdn_t *tsdn, sec_t *sec) { - /* - * Eventually, we should implement affinity, tracking source shard using - * the edata_t's newly freed up fields. For now, just randomly - * distribute across all shards. - */ - if (tsdn_null(tsdn)) { - return &sec->shards[0]; - } - tsd_t *tsd = tsdn_tsd(tsdn); - uint8_t *idxp = tsd_sec_shardp_get(tsd); - if (*idxp == (uint8_t)-1) { - /* - * First use; initialize using the trick from Daniel Lemire's - * "A fast alternative to the modulo reduction. Use a 64 bit - * number to store 32 bits, since we'll deliberately overflow - * when we multiply by the number of shards. - */ - uint64_t rand32 = prng_lg_range_u64(tsd_prng_statep_get(tsd), 32); - uint32_t idx = - (uint32_t)((rand32 * (uint64_t)sec->opts.nshards) >> 32); - assert(idx < (uint32_t)sec->opts.nshards); - *idxp = (uint8_t)idx; - } - return &sec->shards[*idxp]; -} - -/* - * Perhaps surprisingly, this can be called on the alloc pathways; if we hit an - * empty cache, we'll try to fill it, which can push the shard over it's limit. - */ -static void -sec_flush_some_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) { - malloc_mutex_assert_owner(tsdn, &shard->mtx); - edata_list_active_t to_flush; - edata_list_active_init(&to_flush); - while (shard->bytes_cur > sec->opts.bytes_after_flush) { - /* Pick a victim. */ - sec_bin_t *bin = &shard->bins[shard->to_flush_next]; - - /* Update our victim-picking state. */ - shard->to_flush_next++; - if (shard->to_flush_next == sec->npsizes) { - shard->to_flush_next = 0; - } - - assert(shard->bytes_cur >= bin->bytes_cur); - if (bin->bytes_cur != 0) { - shard->bytes_cur -= bin->bytes_cur; - bin->bytes_cur = 0; - edata_list_active_concat(&to_flush, &bin->freelist); - } - /* - * Either bin->bytes_cur was 0, in which case we didn't touch - * the bin list but it should be empty anyways (or else we - * missed a bytes_cur update on a list modification), or it - * *was* 0 and we emptied it ourselves. Either way, it should - * be empty now. - */ - assert(edata_list_active_empty(&bin->freelist)); - } - - malloc_mutex_unlock(tsdn, &shard->mtx); - bool deferred_work_generated = false; - pai_dalloc_batch(tsdn, sec->fallback, &to_flush, - &deferred_work_generated); -} - -static edata_t * -sec_shard_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, - sec_bin_t *bin) { - malloc_mutex_assert_owner(tsdn, &shard->mtx); - if (!shard->enabled) { - return NULL; - } - edata_t *edata = edata_list_active_first(&bin->freelist); - if (edata != NULL) { - edata_list_active_remove(&bin->freelist, edata); - assert(edata_size_get(edata) <= bin->bytes_cur); - bin->bytes_cur -= edata_size_get(edata); - assert(edata_size_get(edata) <= shard->bytes_cur); - shard->bytes_cur -= edata_size_get(edata); - } - return edata; -} - -static edata_t * -sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, - sec_bin_t *bin, size_t size) { - malloc_mutex_assert_not_owner(tsdn, &shard->mtx); - - edata_list_active_t result; - edata_list_active_init(&result); - bool deferred_work_generated = false; - size_t nalloc = pai_alloc_batch(tsdn, sec->fallback, size, - 1 + sec->opts.batch_fill_extra, &result, &deferred_work_generated); - - edata_t *ret = edata_list_active_first(&result); - if (ret != NULL) { - edata_list_active_remove(&result, ret); - } - - malloc_mutex_lock(tsdn, &shard->mtx); - bin->being_batch_filled = false; - /* - * Handle the easy case first: nothing to cache. Note that this can - * only happen in case of OOM, since sec_alloc checks the expected - * number of allocs, and doesn't bother going down the batch_fill - * pathway if there won't be anything left to cache. So to be in this - * code path, we must have asked for > 1 alloc, but only gotten 1 back. - */ - if (nalloc <= 1) { - malloc_mutex_unlock(tsdn, &shard->mtx); - return ret; - } - - size_t new_cached_bytes = (nalloc - 1) * size; - - edata_list_active_concat(&bin->freelist, &result); - bin->bytes_cur += new_cached_bytes; - shard->bytes_cur += new_cached_bytes; - - if (shard->bytes_cur > sec->opts.max_bytes) { - sec_flush_some_and_unlock(tsdn, sec, shard); - } else { - malloc_mutex_unlock(tsdn, &shard->mtx); - } - - return ret; -} - -static edata_t * -sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, - bool guarded, bool frequent_reuse, bool *deferred_work_generated) { - assert((size & PAGE_MASK) == 0); - assert(!guarded); - - sec_t *sec = (sec_t *)self; - - if (zero || alignment > PAGE || sec->opts.nshards == 0 - || size > sec->opts.max_alloc) { - return pai_alloc(tsdn, sec->fallback, size, alignment, zero, - /* guarded */ false, frequent_reuse, - deferred_work_generated); - } - pszind_t pszind = sz_psz2ind(size); - assert(pszind < sec->npsizes); - - sec_shard_t *shard = sec_shard_pick(tsdn, sec); - sec_bin_t *bin = &shard->bins[pszind]; - bool do_batch_fill = false; - - malloc_mutex_lock(tsdn, &shard->mtx); - edata_t *edata = sec_shard_alloc_locked(tsdn, sec, shard, bin); - if (edata == NULL) { - if (!bin->being_batch_filled - && sec->opts.batch_fill_extra > 0) { - bin->being_batch_filled = true; - do_batch_fill = true; - } - } - malloc_mutex_unlock(tsdn, &shard->mtx); - if (edata == NULL) { - if (do_batch_fill) { - edata = sec_batch_fill_and_alloc(tsdn, sec, shard, bin, - size); - } else { - edata = pai_alloc(tsdn, sec->fallback, size, alignment, - zero, /* guarded */ false, frequent_reuse, - deferred_work_generated); - } - } - return edata; -} - -static bool -sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, - size_t new_size, bool zero, bool *deferred_work_generated) { - sec_t *sec = (sec_t *)self; - return pai_expand(tsdn, sec->fallback, edata, old_size, new_size, zero, - deferred_work_generated); -} - -static bool -sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, - size_t new_size, bool *deferred_work_generated) { - sec_t *sec = (sec_t *)self; - return pai_shrink(tsdn, sec->fallback, edata, old_size, new_size, - deferred_work_generated); -} - -static void -sec_flush_all_locked(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) { - malloc_mutex_assert_owner(tsdn, &shard->mtx); - shard->bytes_cur = 0; - edata_list_active_t to_flush; - edata_list_active_init(&to_flush); - for (pszind_t i = 0; i < sec->npsizes; i++) { - sec_bin_t *bin = &shard->bins[i]; - bin->bytes_cur = 0; - edata_list_active_concat(&to_flush, &bin->freelist); - } - - /* - * Ordinarily we would try to avoid doing the batch deallocation while - * holding the shard mutex, but the flush_all pathways only happen when - * we're disabling the HPA or resetting the arena, both of which are - * rare pathways. - */ - bool deferred_work_generated = false; - pai_dalloc_batch(tsdn, sec->fallback, &to_flush, - &deferred_work_generated); -} - -static void -sec_shard_dalloc_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, - edata_t *edata) { - malloc_mutex_assert_owner(tsdn, &shard->mtx); - assert(shard->bytes_cur <= sec->opts.max_bytes); - size_t size = edata_size_get(edata); - pszind_t pszind = sz_psz2ind(size); - assert(pszind < sec->npsizes); - /* - * Prepending here results in LIFO allocation per bin, which seems - * reasonable. - */ - sec_bin_t *bin = &shard->bins[pszind]; - edata_list_active_prepend(&bin->freelist, edata); - bin->bytes_cur += size; - shard->bytes_cur += size; - if (shard->bytes_cur > sec->opts.max_bytes) { - /* - * We've exceeded the shard limit. We make two nods in the - * direction of fragmentation avoidance: we flush everything in - * the shard, rather than one particular bin, and we hold the - * lock while flushing (in case one of the extents we flush is - * highly preferred from a fragmentation-avoidance perspective - * in the backing allocator). This has the extra advantage of - * not requiring advanced cache balancing strategies. - */ - sec_flush_some_and_unlock(tsdn, sec, shard); - malloc_mutex_assert_not_owner(tsdn, &shard->mtx); - } else { - malloc_mutex_unlock(tsdn, &shard->mtx); - } -} - -static void -sec_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata, - bool *deferred_work_generated) { - sec_t *sec = (sec_t *)self; - if (sec->opts.nshards == 0 - || edata_size_get(edata) > sec->opts.max_alloc) { - pai_dalloc(tsdn, sec->fallback, edata, - deferred_work_generated); - return; - } - sec_shard_t *shard = sec_shard_pick(tsdn, sec); - malloc_mutex_lock(tsdn, &shard->mtx); - if (shard->enabled) { - sec_shard_dalloc_and_unlock(tsdn, sec, shard, edata); - } else { - malloc_mutex_unlock(tsdn, &shard->mtx); - pai_dalloc(tsdn, sec->fallback, edata, - deferred_work_generated); - } -} - -void -sec_flush(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->opts.nshards; i++) { - malloc_mutex_lock(tsdn, &sec->shards[i].mtx); - sec_flush_all_locked(tsdn, sec, &sec->shards[i]); - malloc_mutex_unlock(tsdn, &sec->shards[i].mtx); - } -} - -void -sec_disable(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->opts.nshards; i++) { - malloc_mutex_lock(tsdn, &sec->shards[i].mtx); - sec->shards[i].enabled = false; - sec_flush_all_locked(tsdn, sec, &sec->shards[i]); - malloc_mutex_unlock(tsdn, &sec->shards[i].mtx); - } -} - -void -sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) { - size_t sum = 0; - for (size_t i = 0; i < sec->opts.nshards; i++) { - /* - * We could save these lock acquisitions by making bytes_cur - * atomic, but stats collection is rare anyways and we expect - * the number and type of stats to get more interesting. - */ - malloc_mutex_lock(tsdn, &sec->shards[i].mtx); - sum += sec->shards[i].bytes_cur; - malloc_mutex_unlock(tsdn, &sec->shards[i].mtx); - } - stats->bytes += sum; -} - -void -sec_mutex_stats_read(tsdn_t *tsdn, sec_t *sec, - mutex_prof_data_t *mutex_prof_data) { - for (size_t i = 0; i < sec->opts.nshards; i++) { - malloc_mutex_lock(tsdn, &sec->shards[i].mtx); - malloc_mutex_prof_accum(tsdn, mutex_prof_data, - &sec->shards[i].mtx); - malloc_mutex_unlock(tsdn, &sec->shards[i].mtx); - } -} - -void -sec_prefork2(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->opts.nshards; i++) { - malloc_mutex_prefork(tsdn, &sec->shards[i].mtx); - } -} - -void -sec_postfork_parent(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->opts.nshards; i++) { - malloc_mutex_postfork_parent(tsdn, &sec->shards[i].mtx); - } -} - -void -sec_postfork_child(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->opts.nshards; i++) { - malloc_mutex_postfork_child(tsdn, &sec->shards[i].mtx); - } -} |
