diff options
Diffstat (limited to 'examples/redis-unstable/deps/jemalloc/src/pac.c')
| -rw-r--r-- | examples/redis-unstable/deps/jemalloc/src/pac.c | 587 |
1 files changed, 0 insertions, 587 deletions
diff --git a/examples/redis-unstable/deps/jemalloc/src/pac.c b/examples/redis-unstable/deps/jemalloc/src/pac.c deleted file mode 100644 index 53e3d82..0000000 --- a/examples/redis-unstable/deps/jemalloc/src/pac.c +++ /dev/null | |||
| @@ -1,587 +0,0 @@ | |||
| 1 | #include "jemalloc/internal/jemalloc_preamble.h" | ||
| 2 | #include "jemalloc/internal/jemalloc_internal_includes.h" | ||
| 3 | |||
| 4 | #include "jemalloc/internal/pac.h" | ||
| 5 | #include "jemalloc/internal/san.h" | ||
| 6 | |||
| 7 | static edata_t *pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, | ||
| 8 | size_t alignment, bool zero, bool guarded, bool frequent_reuse, | ||
| 9 | bool *deferred_work_generated); | ||
| 10 | static bool pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, | ||
| 11 | size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated); | ||
| 12 | static bool pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, | ||
| 13 | size_t old_size, size_t new_size, bool *deferred_work_generated); | ||
| 14 | static void pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, | ||
| 15 | bool *deferred_work_generated); | ||
| 16 | static uint64_t pac_time_until_deferred_work(tsdn_t *tsdn, pai_t *self); | ||
| 17 | |||
| 18 | static inline void | ||
| 19 | pac_decay_data_get(pac_t *pac, extent_state_t state, | ||
| 20 | decay_t **r_decay, pac_decay_stats_t **r_decay_stats, ecache_t **r_ecache) { | ||
| 21 | switch(state) { | ||
| 22 | case extent_state_dirty: | ||
| 23 | *r_decay = &pac->decay_dirty; | ||
| 24 | *r_decay_stats = &pac->stats->decay_dirty; | ||
| 25 | *r_ecache = &pac->ecache_dirty; | ||
| 26 | return; | ||
| 27 | case extent_state_muzzy: | ||
| 28 | *r_decay = &pac->decay_muzzy; | ||
| 29 | *r_decay_stats = &pac->stats->decay_muzzy; | ||
| 30 | *r_ecache = &pac->ecache_muzzy; | ||
| 31 | return; | ||
| 32 | default: | ||
| 33 | unreachable(); | ||
| 34 | } | ||
| 35 | } | ||
| 36 | |||
| 37 | bool | ||
| 38 | pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, | ||
| 39 | edata_cache_t *edata_cache, nstime_t *cur_time, | ||
| 40 | size_t pac_oversize_threshold, ssize_t dirty_decay_ms, | ||
| 41 | ssize_t muzzy_decay_ms, pac_stats_t *pac_stats, malloc_mutex_t *stats_mtx) { | ||
| 42 | unsigned ind = base_ind_get(base); | ||
| 43 | /* | ||
| 44 | * Delay coalescing for dirty extents despite the disruptive effect on | ||
| 45 | * memory layout for best-fit extent allocation, since cached extents | ||
| 46 | * are likely to be reused soon after deallocation, and the cost of | ||
| 47 | * merging/splitting extents is non-trivial. | ||
| 48 | */ | ||
| 49 | if (ecache_init(tsdn, &pac->ecache_dirty, extent_state_dirty, ind, | ||
| 50 | /* delay_coalesce */ true)) { | ||
| 51 | return true; | ||
| 52 | } | ||
| 53 | /* | ||
| 54 | * Coalesce muzzy extents immediately, because operations on them are in | ||
| 55 | * the critical path much less often than for dirty extents. | ||
| 56 | */ | ||
| 57 | if (ecache_init(tsdn, &pac->ecache_muzzy, extent_state_muzzy, ind, | ||
| 58 | /* delay_coalesce */ false)) { | ||
| 59 | return true; | ||
| 60 | } | ||
| 61 | /* | ||
| 62 | * Coalesce retained extents immediately, in part because they will | ||
| 63 | * never be evicted (and therefore there's no opportunity for delayed | ||
| 64 | * coalescing), but also because operations on retained extents are not | ||
| 65 | * in the critical path. | ||
| 66 | */ | ||
| 67 | if (ecache_init(tsdn, &pac->ecache_retained, extent_state_retained, | ||
| 68 | ind, /* delay_coalesce */ false)) { | ||
| 69 | return true; | ||
| 70 | } | ||
| 71 | exp_grow_init(&pac->exp_grow); | ||
| 72 | if (malloc_mutex_init(&pac->grow_mtx, "extent_grow", | ||
| 73 | WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) { | ||
| 74 | return true; | ||
| 75 | } | ||
| 76 | atomic_store_zu(&pac->oversize_threshold, pac_oversize_threshold, | ||
| 77 | ATOMIC_RELAXED); | ||
| 78 | if (decay_init(&pac->decay_dirty, cur_time, dirty_decay_ms)) { | ||
| 79 | return true; | ||
| 80 | } | ||
| 81 | if (decay_init(&pac->decay_muzzy, cur_time, muzzy_decay_ms)) { | ||
| 82 | return true; | ||
| 83 | } | ||
| 84 | if (san_bump_alloc_init(&pac->sba)) { | ||
| 85 | return true; | ||
| 86 | } | ||
| 87 | |||
| 88 | pac->base = base; | ||
| 89 | pac->emap = emap; | ||
| 90 | pac->edata_cache = edata_cache; | ||
| 91 | pac->stats = pac_stats; | ||
| 92 | pac->stats_mtx = stats_mtx; | ||
| 93 | atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED); | ||
| 94 | |||
| 95 | pac->pai.alloc = &pac_alloc_impl; | ||
| 96 | pac->pai.alloc_batch = &pai_alloc_batch_default; | ||
| 97 | pac->pai.expand = &pac_expand_impl; | ||
| 98 | pac->pai.shrink = &pac_shrink_impl; | ||
| 99 | pac->pai.dalloc = &pac_dalloc_impl; | ||
| 100 | pac->pai.dalloc_batch = &pai_dalloc_batch_default; | ||
| 101 | pac->pai.time_until_deferred_work = &pac_time_until_deferred_work; | ||
| 102 | |||
| 103 | return false; | ||
| 104 | } | ||
| 105 | |||
| 106 | static inline bool | ||
| 107 | pac_may_have_muzzy(pac_t *pac) { | ||
| 108 | return pac_decay_ms_get(pac, extent_state_muzzy) != 0; | ||
| 109 | } | ||
| 110 | |||
| 111 | static edata_t * | ||
| 112 | pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size, | ||
| 113 | size_t alignment, bool zero, bool guarded) { | ||
| 114 | assert(!guarded || alignment <= PAGE); | ||
| 115 | |||
| 116 | edata_t *edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty, | ||
| 117 | NULL, size, alignment, zero, guarded); | ||
| 118 | |||
| 119 | if (edata == NULL && pac_may_have_muzzy(pac)) { | ||
| 120 | edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy, | ||
| 121 | NULL, size, alignment, zero, guarded); | ||
| 122 | } | ||
| 123 | if (edata == NULL) { | ||
| 124 | edata = ecache_alloc_grow(tsdn, pac, ehooks, | ||
| 125 | &pac->ecache_retained, NULL, size, alignment, zero, | ||
| 126 | guarded); | ||
| 127 | if (config_stats && edata != NULL) { | ||
| 128 | atomic_fetch_add_zu(&pac->stats->pac_mapped, size, | ||
| 129 | ATOMIC_RELAXED); | ||
| 130 | } | ||
| 131 | } | ||
| 132 | |||
| 133 | return edata; | ||
| 134 | } | ||
| 135 | |||
| 136 | static edata_t * | ||
| 137 | pac_alloc_new_guarded(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size, | ||
| 138 | size_t alignment, bool zero, bool frequent_reuse) { | ||
| 139 | assert(alignment <= PAGE); | ||
| 140 | |||
| 141 | edata_t *edata; | ||
| 142 | if (san_bump_enabled() && frequent_reuse) { | ||
| 143 | edata = san_bump_alloc(tsdn, &pac->sba, pac, ehooks, size, | ||
| 144 | zero); | ||
| 145 | } else { | ||
| 146 | size_t size_with_guards = san_two_side_guarded_sz(size); | ||
| 147 | /* Alloc a non-guarded extent first.*/ | ||
| 148 | edata = pac_alloc_real(tsdn, pac, ehooks, size_with_guards, | ||
| 149 | /* alignment */ PAGE, zero, /* guarded */ false); | ||
| 150 | if (edata != NULL) { | ||
| 151 | /* Add guards around it. */ | ||
| 152 | assert(edata_size_get(edata) == size_with_guards); | ||
| 153 | san_guard_pages_two_sided(tsdn, ehooks, edata, | ||
| 154 | pac->emap, true); | ||
| 155 | } | ||
| 156 | } | ||
| 157 | assert(edata == NULL || (edata_guarded_get(edata) && | ||
| 158 | edata_size_get(edata) == size)); | ||
| 159 | |||
| 160 | return edata; | ||
| 161 | } | ||
| 162 | |||
| 163 | static edata_t * | ||
| 164 | pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, | ||
| 165 | bool zero, bool guarded, bool frequent_reuse, | ||
| 166 | bool *deferred_work_generated) { | ||
| 167 | pac_t *pac = (pac_t *)self; | ||
| 168 | ehooks_t *ehooks = pac_ehooks_get(pac); | ||
| 169 | |||
| 170 | edata_t *edata = NULL; | ||
| 171 | /* | ||
| 172 | * The condition is an optimization - not frequently reused guarded | ||
| 173 | * allocations are never put in the ecache. pac_alloc_real also | ||
| 174 | * doesn't grow retained for guarded allocations. So pac_alloc_real | ||
| 175 | * for such allocations would always return NULL. | ||
| 176 | * */ | ||
| 177 | if (!guarded || frequent_reuse) { | ||
| 178 | edata = pac_alloc_real(tsdn, pac, ehooks, size, alignment, | ||
| 179 | zero, guarded); | ||
| 180 | } | ||
| 181 | if (edata == NULL && guarded) { | ||
| 182 | /* No cached guarded extents; creating a new one. */ | ||
| 183 | edata = pac_alloc_new_guarded(tsdn, pac, ehooks, size, | ||
| 184 | alignment, zero, frequent_reuse); | ||
| 185 | } | ||
| 186 | |||
| 187 | return edata; | ||
| 188 | } | ||
| 189 | |||
| 190 | static bool | ||
| 191 | pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, | ||
| 192 | size_t new_size, bool zero, bool *deferred_work_generated) { | ||
| 193 | pac_t *pac = (pac_t *)self; | ||
| 194 | ehooks_t *ehooks = pac_ehooks_get(pac); | ||
| 195 | |||
| 196 | size_t mapped_add = 0; | ||
| 197 | size_t expand_amount = new_size - old_size; | ||
| 198 | |||
| 199 | if (ehooks_merge_will_fail(ehooks)) { | ||
| 200 | return true; | ||
| 201 | } | ||
| 202 | edata_t *trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty, | ||
| 203 | edata, expand_amount, PAGE, zero, /* guarded*/ false); | ||
| 204 | if (trail == NULL) { | ||
| 205 | trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy, | ||
| 206 | edata, expand_amount, PAGE, zero, /* guarded*/ false); | ||
| 207 | } | ||
| 208 | if (trail == NULL) { | ||
| 209 | trail = ecache_alloc_grow(tsdn, pac, ehooks, | ||
| 210 | &pac->ecache_retained, edata, expand_amount, PAGE, zero, | ||
| 211 | /* guarded */ false); | ||
| 212 | mapped_add = expand_amount; | ||
| 213 | } | ||
| 214 | if (trail == NULL) { | ||
| 215 | return true; | ||
| 216 | } | ||
| 217 | if (extent_merge_wrapper(tsdn, pac, ehooks, edata, trail)) { | ||
| 218 | extent_dalloc_wrapper(tsdn, pac, ehooks, trail); | ||
| 219 | return true; | ||
| 220 | } | ||
| 221 | if (config_stats && mapped_add > 0) { | ||
| 222 | atomic_fetch_add_zu(&pac->stats->pac_mapped, mapped_add, | ||
| 223 | ATOMIC_RELAXED); | ||
| 224 | } | ||
| 225 | return false; | ||
| 226 | } | ||
| 227 | |||
| 228 | static bool | ||
| 229 | pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, | ||
| 230 | size_t new_size, bool *deferred_work_generated) { | ||
| 231 | pac_t *pac = (pac_t *)self; | ||
| 232 | ehooks_t *ehooks = pac_ehooks_get(pac); | ||
| 233 | |||
| 234 | size_t shrink_amount = old_size - new_size; | ||
| 235 | |||
| 236 | if (ehooks_split_will_fail(ehooks)) { | ||
| 237 | return true; | ||
| 238 | } | ||
| 239 | |||
| 240 | edata_t *trail = extent_split_wrapper(tsdn, pac, ehooks, edata, | ||
| 241 | new_size, shrink_amount, /* holding_core_locks */ false); | ||
| 242 | if (trail == NULL) { | ||
| 243 | return true; | ||
| 244 | } | ||
| 245 | ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, trail); | ||
| 246 | *deferred_work_generated = true; | ||
| 247 | return false; | ||
| 248 | } | ||
| 249 | |||
| 250 | static void | ||
| 251 | pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, | ||
| 252 | bool *deferred_work_generated) { | ||
| 253 | pac_t *pac = (pac_t *)self; | ||
| 254 | ehooks_t *ehooks = pac_ehooks_get(pac); | ||
| 255 | |||
| 256 | if (edata_guarded_get(edata)) { | ||
| 257 | /* | ||
| 258 | * Because cached guarded extents do exact fit only, large | ||
| 259 | * guarded extents are restored on dalloc eagerly (otherwise | ||
| 260 | * they will not be reused efficiently). Slab sizes have a | ||
| 261 | * limited number of size classes, and tend to cycle faster. | ||
| 262 | * | ||
| 263 | * In the case where coalesce is restrained (VirtualFree on | ||
| 264 | * Windows), guarded extents are also not cached -- otherwise | ||
| 265 | * during arena destroy / reset, the retained extents would not | ||
| 266 | * be whole regions (i.e. they are split between regular and | ||
| 267 | * guarded). | ||
| 268 | */ | ||
| 269 | if (!edata_slab_get(edata) || !maps_coalesce) { | ||
| 270 | assert(edata_size_get(edata) >= SC_LARGE_MINCLASS || | ||
| 271 | !maps_coalesce); | ||
| 272 | san_unguard_pages_two_sided(tsdn, ehooks, edata, | ||
| 273 | pac->emap); | ||
| 274 | } | ||
| 275 | } | ||
| 276 | |||
| 277 | ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, edata); | ||
| 278 | /* Purging of deallocated pages is deferred */ | ||
| 279 | *deferred_work_generated = true; | ||
| 280 | } | ||
| 281 | |||
| 282 | static inline uint64_t | ||
| 283 | pac_ns_until_purge(tsdn_t *tsdn, decay_t *decay, size_t npages) { | ||
| 284 | if (malloc_mutex_trylock(tsdn, &decay->mtx)) { | ||
| 285 | /* Use minimal interval if decay is contended. */ | ||
| 286 | return BACKGROUND_THREAD_DEFERRED_MIN; | ||
| 287 | } | ||
| 288 | uint64_t result = decay_ns_until_purge(decay, npages, | ||
| 289 | ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD); | ||
| 290 | |||
| 291 | malloc_mutex_unlock(tsdn, &decay->mtx); | ||
| 292 | return result; | ||
| 293 | } | ||
| 294 | |||
| 295 | static uint64_t | ||
| 296 | pac_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) { | ||
| 297 | uint64_t time; | ||
| 298 | pac_t *pac = (pac_t *)self; | ||
| 299 | |||
| 300 | time = pac_ns_until_purge(tsdn, | ||
| 301 | &pac->decay_dirty, | ||
| 302 | ecache_npages_get(&pac->ecache_dirty)); | ||
| 303 | if (time == BACKGROUND_THREAD_DEFERRED_MIN) { | ||
| 304 | return time; | ||
| 305 | } | ||
| 306 | |||
| 307 | uint64_t muzzy = pac_ns_until_purge(tsdn, | ||
| 308 | &pac->decay_muzzy, | ||
| 309 | ecache_npages_get(&pac->ecache_muzzy)); | ||
| 310 | if (muzzy < time) { | ||
| 311 | time = muzzy; | ||
| 312 | } | ||
| 313 | return time; | ||
| 314 | } | ||
| 315 | |||
| 316 | bool | ||
| 317 | pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit, | ||
| 318 | size_t *new_limit) { | ||
| 319 | pszind_t new_ind JEMALLOC_CC_SILENCE_INIT(0); | ||
| 320 | if (new_limit != NULL) { | ||
| 321 | size_t limit = *new_limit; | ||
| 322 | /* Grow no more than the new limit. */ | ||
| 323 | if ((new_ind = sz_psz2ind(limit + 1) - 1) >= SC_NPSIZES) { | ||
| 324 | return true; | ||
| 325 | } | ||
| 326 | } | ||
| 327 | |||
| 328 | malloc_mutex_lock(tsdn, &pac->grow_mtx); | ||
| 329 | if (old_limit != NULL) { | ||
| 330 | *old_limit = sz_pind2sz(pac->exp_grow.limit); | ||
| 331 | } | ||
| 332 | if (new_limit != NULL) { | ||
| 333 | pac->exp_grow.limit = new_ind; | ||
| 334 | } | ||
| 335 | malloc_mutex_unlock(tsdn, &pac->grow_mtx); | ||
| 336 | |||
| 337 | return false; | ||
| 338 | } | ||
| 339 | |||
| 340 | static size_t | ||
| 341 | pac_stash_decayed(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, | ||
| 342 | size_t npages_limit, size_t npages_decay_max, | ||
| 343 | edata_list_inactive_t *result) { | ||
| 344 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||
| 345 | WITNESS_RANK_CORE, 0); | ||
| 346 | ehooks_t *ehooks = pac_ehooks_get(pac); | ||
| 347 | |||
| 348 | /* Stash extents according to npages_limit. */ | ||
| 349 | size_t nstashed = 0; | ||
| 350 | while (nstashed < npages_decay_max) { | ||
| 351 | edata_t *edata = ecache_evict(tsdn, pac, ehooks, ecache, | ||
| 352 | npages_limit); | ||
| 353 | if (edata == NULL) { | ||
| 354 | break; | ||
| 355 | } | ||
| 356 | edata_list_inactive_append(result, edata); | ||
| 357 | nstashed += edata_size_get(edata) >> LG_PAGE; | ||
| 358 | } | ||
| 359 | return nstashed; | ||
| 360 | } | ||
| 361 | |||
| 362 | static size_t | ||
| 363 | pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay, | ||
| 364 | pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay, | ||
| 365 | edata_list_inactive_t *decay_extents) { | ||
| 366 | bool err; | ||
| 367 | |||
| 368 | size_t nmadvise = 0; | ||
| 369 | size_t nunmapped = 0; | ||
| 370 | size_t npurged = 0; | ||
| 371 | |||
| 372 | ehooks_t *ehooks = pac_ehooks_get(pac); | ||
| 373 | |||
| 374 | bool try_muzzy = !fully_decay | ||
| 375 | && pac_decay_ms_get(pac, extent_state_muzzy) != 0; | ||
| 376 | |||
| 377 | for (edata_t *edata = edata_list_inactive_first(decay_extents); edata != | ||
| 378 | NULL; edata = edata_list_inactive_first(decay_extents)) { | ||
| 379 | edata_list_inactive_remove(decay_extents, edata); | ||
| 380 | |||
| 381 | size_t size = edata_size_get(edata); | ||
| 382 | size_t npages = size >> LG_PAGE; | ||
| 383 | |||
| 384 | nmadvise++; | ||
| 385 | npurged += npages; | ||
| 386 | |||
| 387 | switch (ecache->state) { | ||
| 388 | case extent_state_active: | ||
| 389 | not_reached(); | ||
| 390 | case extent_state_dirty: | ||
| 391 | if (try_muzzy) { | ||
| 392 | err = extent_purge_lazy_wrapper(tsdn, ehooks, | ||
| 393 | edata, /* offset */ 0, size); | ||
| 394 | if (!err) { | ||
| 395 | ecache_dalloc(tsdn, pac, ehooks, | ||
| 396 | &pac->ecache_muzzy, edata); | ||
| 397 | break; | ||
| 398 | } | ||
| 399 | } | ||
| 400 | JEMALLOC_FALLTHROUGH; | ||
| 401 | case extent_state_muzzy: | ||
| 402 | extent_dalloc_wrapper(tsdn, pac, ehooks, edata); | ||
| 403 | nunmapped += npages; | ||
| 404 | break; | ||
| 405 | case extent_state_retained: | ||
| 406 | default: | ||
| 407 | not_reached(); | ||
| 408 | } | ||
| 409 | } | ||
| 410 | |||
| 411 | if (config_stats) { | ||
| 412 | LOCKEDINT_MTX_LOCK(tsdn, *pac->stats_mtx); | ||
| 413 | locked_inc_u64(tsdn, LOCKEDINT_MTX(*pac->stats_mtx), | ||
| 414 | &decay_stats->npurge, 1); | ||
| 415 | locked_inc_u64(tsdn, LOCKEDINT_MTX(*pac->stats_mtx), | ||
| 416 | &decay_stats->nmadvise, nmadvise); | ||
| 417 | locked_inc_u64(tsdn, LOCKEDINT_MTX(*pac->stats_mtx), | ||
| 418 | &decay_stats->purged, npurged); | ||
| 419 | LOCKEDINT_MTX_UNLOCK(tsdn, *pac->stats_mtx); | ||
| 420 | atomic_fetch_sub_zu(&pac->stats->pac_mapped, | ||
| 421 | nunmapped << LG_PAGE, ATOMIC_RELAXED); | ||
| 422 | } | ||
| 423 | |||
| 424 | return npurged; | ||
| 425 | } | ||
| 426 | |||
| 427 | /* | ||
| 428 | * npages_limit: Decay at most npages_decay_max pages without violating the | ||
| 429 | * invariant: (ecache_npages_get(ecache) >= npages_limit). We need an upper | ||
| 430 | * bound on number of pages in order to prevent unbounded growth (namely in | ||
| 431 | * stashed), otherwise unbounded new pages could be added to extents during the | ||
| 432 | * current decay run, so that the purging thread never finishes. | ||
| 433 | */ | ||
| 434 | static void | ||
| 435 | pac_decay_to_limit(tsdn_t *tsdn, pac_t *pac, decay_t *decay, | ||
| 436 | pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay, | ||
| 437 | size_t npages_limit, size_t npages_decay_max) { | ||
| 438 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||
| 439 | WITNESS_RANK_CORE, 1); | ||
| 440 | |||
| 441 | if (decay->purging || npages_decay_max == 0) { | ||
| 442 | return; | ||
| 443 | } | ||
| 444 | decay->purging = true; | ||
| 445 | malloc_mutex_unlock(tsdn, &decay->mtx); | ||
| 446 | |||
| 447 | edata_list_inactive_t decay_extents; | ||
| 448 | edata_list_inactive_init(&decay_extents); | ||
| 449 | size_t npurge = pac_stash_decayed(tsdn, pac, ecache, npages_limit, | ||
| 450 | npages_decay_max, &decay_extents); | ||
| 451 | if (npurge != 0) { | ||
| 452 | size_t npurged = pac_decay_stashed(tsdn, pac, decay, | ||
| 453 | decay_stats, ecache, fully_decay, &decay_extents); | ||
| 454 | assert(npurged == npurge); | ||
| 455 | } | ||
| 456 | |||
| 457 | malloc_mutex_lock(tsdn, &decay->mtx); | ||
| 458 | decay->purging = false; | ||
| 459 | } | ||
| 460 | |||
| 461 | void | ||
| 462 | pac_decay_all(tsdn_t *tsdn, pac_t *pac, decay_t *decay, | ||
| 463 | pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay) { | ||
| 464 | malloc_mutex_assert_owner(tsdn, &decay->mtx); | ||
| 465 | pac_decay_to_limit(tsdn, pac, decay, decay_stats, ecache, fully_decay, | ||
| 466 | /* npages_limit */ 0, ecache_npages_get(ecache)); | ||
| 467 | } | ||
| 468 | |||
| 469 | static void | ||
| 470 | pac_decay_try_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay, | ||
| 471 | pac_decay_stats_t *decay_stats, ecache_t *ecache, | ||
| 472 | size_t current_npages, size_t npages_limit) { | ||
| 473 | if (current_npages > npages_limit) { | ||
| 474 | pac_decay_to_limit(tsdn, pac, decay, decay_stats, ecache, | ||
| 475 | /* fully_decay */ false, npages_limit, | ||
| 476 | current_npages - npages_limit); | ||
| 477 | } | ||
| 478 | } | ||
| 479 | |||
| 480 | bool | ||
| 481 | pac_maybe_decay_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay, | ||
| 482 | pac_decay_stats_t *decay_stats, ecache_t *ecache, | ||
| 483 | pac_purge_eagerness_t eagerness) { | ||
| 484 | malloc_mutex_assert_owner(tsdn, &decay->mtx); | ||
| 485 | |||
| 486 | /* Purge all or nothing if the option is disabled. */ | ||
| 487 | ssize_t decay_ms = decay_ms_read(decay); | ||
| 488 | if (decay_ms <= 0) { | ||
| 489 | if (decay_ms == 0) { | ||
| 490 | pac_decay_to_limit(tsdn, pac, decay, decay_stats, | ||
| 491 | ecache, /* fully_decay */ false, | ||
| 492 | /* npages_limit */ 0, ecache_npages_get(ecache)); | ||
| 493 | } | ||
| 494 | return false; | ||
| 495 | } | ||
| 496 | |||
| 497 | /* | ||
| 498 | * If the deadline has been reached, advance to the current epoch and | ||
| 499 | * purge to the new limit if necessary. Note that dirty pages created | ||
| 500 | * during the current epoch are not subject to purge until a future | ||
| 501 | * epoch, so as a result purging only happens during epoch advances, or | ||
| 502 | * being triggered by background threads (scheduled event). | ||
| 503 | */ | ||
| 504 | nstime_t time; | ||
| 505 | nstime_init_update(&time); | ||
| 506 | size_t npages_current = ecache_npages_get(ecache); | ||
| 507 | bool epoch_advanced = decay_maybe_advance_epoch(decay, &time, | ||
| 508 | npages_current); | ||
| 509 | if (eagerness == PAC_PURGE_ALWAYS | ||
| 510 | || (epoch_advanced && eagerness == PAC_PURGE_ON_EPOCH_ADVANCE)) { | ||
| 511 | size_t npages_limit = decay_npages_limit_get(decay); | ||
| 512 | pac_decay_try_purge(tsdn, pac, decay, decay_stats, ecache, | ||
| 513 | npages_current, npages_limit); | ||
| 514 | } | ||
| 515 | |||
| 516 | return epoch_advanced; | ||
| 517 | } | ||
| 518 | |||
| 519 | bool | ||
| 520 | pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state, | ||
| 521 | ssize_t decay_ms, pac_purge_eagerness_t eagerness) { | ||
| 522 | decay_t *decay; | ||
| 523 | pac_decay_stats_t *decay_stats; | ||
| 524 | ecache_t *ecache; | ||
| 525 | pac_decay_data_get(pac, state, &decay, &decay_stats, &ecache); | ||
| 526 | |||
| 527 | if (!decay_ms_valid(decay_ms)) { | ||
| 528 | return true; | ||
| 529 | } | ||
| 530 | |||
| 531 | malloc_mutex_lock(tsdn, &decay->mtx); | ||
| 532 | /* | ||
| 533 | * Restart decay backlog from scratch, which may cause many dirty pages | ||
| 534 | * to be immediately purged. It would conceptually be possible to map | ||
| 535 | * the old backlog onto the new backlog, but there is no justification | ||
| 536 | * for such complexity since decay_ms changes are intended to be | ||
| 537 | * infrequent, either between the {-1, 0, >0} states, or a one-time | ||
| 538 | * arbitrary change during initial arena configuration. | ||
| 539 | */ | ||
| 540 | nstime_t cur_time; | ||
| 541 | nstime_init_update(&cur_time); | ||
| 542 | decay_reinit(decay, &cur_time, decay_ms); | ||
| 543 | pac_maybe_decay_purge(tsdn, pac, decay, decay_stats, ecache, eagerness); | ||
| 544 | malloc_mutex_unlock(tsdn, &decay->mtx); | ||
| 545 | |||
| 546 | return false; | ||
| 547 | } | ||
| 548 | |||
| 549 | ssize_t | ||
| 550 | pac_decay_ms_get(pac_t *pac, extent_state_t state) { | ||
| 551 | decay_t *decay; | ||
| 552 | pac_decay_stats_t *decay_stats; | ||
| 553 | ecache_t *ecache; | ||
| 554 | pac_decay_data_get(pac, state, &decay, &decay_stats, &ecache); | ||
| 555 | return decay_ms_read(decay); | ||
| 556 | } | ||
| 557 | |||
| 558 | void | ||
| 559 | pac_reset(tsdn_t *tsdn, pac_t *pac) { | ||
| 560 | /* | ||
| 561 | * No-op for now; purging is still done at the arena-level. It should | ||
| 562 | * get moved in here, though. | ||
| 563 | */ | ||
| 564 | (void)tsdn; | ||
| 565 | (void)pac; | ||
| 566 | } | ||
| 567 | |||
| 568 | void | ||
| 569 | pac_destroy(tsdn_t *tsdn, pac_t *pac) { | ||
| 570 | assert(ecache_npages_get(&pac->ecache_dirty) == 0); | ||
| 571 | assert(ecache_npages_get(&pac->ecache_muzzy) == 0); | ||
| 572 | /* | ||
| 573 | * Iterate over the retained extents and destroy them. This gives the | ||
| 574 | * extent allocator underlying the extent hooks an opportunity to unmap | ||
| 575 | * all retained memory without having to keep its own metadata | ||
| 576 | * structures. In practice, virtual memory for dss-allocated extents is | ||
| 577 | * leaked here, so best practice is to avoid dss for arenas to be | ||
| 578 | * destroyed, or provide custom extent hooks that track retained | ||
| 579 | * dss-based extents for later reuse. | ||
| 580 | */ | ||
| 581 | ehooks_t *ehooks = pac_ehooks_get(pac); | ||
| 582 | edata_t *edata; | ||
| 583 | while ((edata = ecache_evict(tsdn, pac, ehooks, | ||
| 584 | &pac->ecache_retained, 0)) != NULL) { | ||
| 585 | extent_destroy_wrapper(tsdn, pac, ehooks, edata); | ||
| 586 | } | ||
| 587 | } | ||
