diff --git a/tcmalloc/cpu_cache.h b/tcmalloc/cpu_cache.h index feb4b7273..0196cf6ce 100644 --- a/tcmalloc/cpu_cache.h +++ b/tcmalloc/cpu_cache.h @@ -158,6 +158,10 @@ class StaticForwarder { return Parameters::per_cpu_caches_dynamic_slab_shrink_threshold(); } + static bool release_drained_slab_metadata() { + return Parameters::release_drained_slab_metadata(); + } + bool reuse_size_classes() const { return state_.size_class_configuration() == SizeClassConfiguration::kReuse || @@ -467,6 +471,14 @@ class CpuCache { // Reports total number of times any CPU has been reclaimed. uint64_t GetNumReclaims() const; + // Reports number of times the has been unpopulated + // (which happens when its metadata gets released, after all per-CPU + // metadata slabs on the same hugepage ave been reclaimed). + uint64_t GetNumUnpopulates(int cpu) const; + + // Reports total number of times any CPU has been unpopulated. + uint64_t GetNumUnpopulates() const; + // Reports number of cpus that have touched set to true. int CountTouchedCpus() const; @@ -675,6 +687,9 @@ class CpuCache { std::atomic reclaim_used_bytes; // Tracks number of times this CPU has been reclaimed. std::atomic num_reclaims; + // Tracks number of times this CPU has been unpopulated + // (see GetNumUnpopulates()). + std::atomic num_unpopulates; }; // Determines how we distribute memory in the per-cpu cache to the various @@ -1415,9 +1430,12 @@ inline void CpuCache::Grow(int cpu, size_t size_class, } template -inline void CpuCache::TryReclaimingCaches() { +inline void CpuCache::TryReclaimingCaches() + ABSL_NO_THREAD_SAFETY_ANALYSIS { const int num_cpus = NumCPUs(); + bool any_drained = false; + for (int cpu = 0; cpu < num_cpus; ++cpu) { // Nothing to reclaim if the cpu is not populated. if (!HasPopulated(cpu)) { @@ -1439,6 +1457,7 @@ inline void CpuCache::TryReclaimingCaches() { // stayed constant since the last interval. if (used_bytes != 0 && used_bytes == prev_used_bytes && misses == 0) { Reclaim(cpu); + any_drained = true; } // Takes a snapshot of used bytes in the cache at the end of this interval @@ -1449,6 +1468,24 @@ inline void CpuCache::TryReclaimingCaches() { resize_[cpu].reclaim_used_bytes.store(used_bytes, std::memory_order_relaxed); } + + if (any_drained && forwarder_.release_drained_slab_metadata()) { + for (int cpu = 0; cpu < num_cpus; ++cpu) resize_[cpu].lock.lock(); + freelist_.ReleaseSlabMetadataForDrainedCpus( + [this](int cpu) { return HasPopulated(cpu); }, + [this](int cpu) { + TC_CHECK_EQ( + resize_[cpu].available, resize_[cpu].capacity, + "CPU %u was not actually drained, or available is out of sync", + cpu); + resize_[cpu].populated.store(false, std::memory_order_release); + resize_[cpu].num_unpopulates.fetch_add(1, std::memory_order_relaxed); + }, + [this](void* slab_addr, size_t slab_size) { + return MadviseAwaySlabs(slab_addr, slab_size); + }); + for (int cpu = 0; cpu < num_cpus; ++cpu) resize_[cpu].lock.unlock(); + } } template @@ -2282,6 +2319,20 @@ inline uint64_t CpuCache::GetNumReclaims() const { return reclaims; } +template +inline uint64_t CpuCache::GetNumUnpopulates(int cpu) const { + return resize_[cpu].num_unpopulates.load(std::memory_order_relaxed); +} + +template +inline uint64_t CpuCache::GetNumUnpopulates() const { + uint64_t reclaims = 0; + const int num_cpus = NumCPUs(); + for (int cpu = 0; cpu < num_cpus; ++cpu) + reclaims += resize_[cpu].num_unpopulates.load(std::memory_order_relaxed); + return reclaims; +} + template inline int CpuCache::CountTouchedCpus() const { if (resize_ == nullptr) return 0; diff --git a/tcmalloc/cpu_cache_test.cc b/tcmalloc/cpu_cache_test.cc index 0a9290026..6f6b15193 100644 --- a/tcmalloc/cpu_cache_test.cc +++ b/tcmalloc/cpu_cache_test.cc @@ -337,6 +337,10 @@ class TestStaticForwarder { return false; } + bool release_drained_slab_metadata() const { + return release_drained_slab_metadata_; + } + size_t arena_reported_nonresident_bytes_ = 0; int64_t arena_reported_impending_bytes_ = 0; size_t shrink_to_usage_limit_calls_ = 0; @@ -345,6 +349,7 @@ class TestStaticForwarder { double dynamic_slab_shrink_threshold_ = -1; DynamicSlab dynamic_slab_ = DynamicSlab::kNoop; std::optional size_map_; + bool release_drained_slab_metadata_ = false; private: NumaTopology numa_topology_; @@ -1267,7 +1272,7 @@ static void ColdCacheOperations(CpuCache& cache, int cpu_id, // Runs multiple allocate and deallocate operation on the cpu cache to collect // misses. Once we collect enough misses on this cache, we can shuffle cpu // caches to steal capacity from colder caches to the hot cache. -static void HotCacheOperations(CpuCache& cache, int cpu_id) { +static void HotCacheOperations(CpuCache& cache, int cpu_id, bool reclaim) { constexpr size_t kPtrs = 4096; std::vector ptrs; ptrs.resize(kPtrs); @@ -1288,10 +1293,13 @@ static void HotCacheOperations(CpuCache& cache, int cpu_id) { } } - // We reclaim the cache to reset it so that we record underflows/overflows the - // next time we allocate and deallocate objects. Without reclaim, the cache - // would stay warmed up and it would take more time to drain the colder cache. - cache.Reclaim(cpu_id); + if (reclaim) { + // We reclaim the cache to reset it so that we record underflows/overflows + // the next time we allocate and deallocate objects. Without reclaim, the + // cache would stay warmed up and it would take more time to drain the + // colder cache. + cache.Reclaim(cpu_id); + } } class DynamicWideSlabTest : public testing::Test {}; @@ -1319,7 +1327,7 @@ TEST_F(DynamicWideSlabTest, DynamicSlabThreshold) { constexpr int kCpuId1 = 1; // Accumulate overflows and underflows for kCpuId0. - HotCacheOperations(cache, kCpuId0); + HotCacheOperations(cache, kCpuId0, /*reclaim=*/true); CpuCache::CpuCacheMissStats interval_misses = cache.GetIntervalCacheMissStats(kCpuId0, MissCount::kSlabResize); // Make sure that overflows/underflows ratio is greater than the threshold @@ -1539,7 +1547,7 @@ TEST(CpuCacheTest, ColdHotCacheShuffleTest) { CpuCache::kCacheCapacityThreshold * max_cpu_cache_size; ++num_tries) { ColdCacheOperations(cache, cold_cpu_id, size_class); - HotCacheOperations(cache, hot_cpu_id); + HotCacheOperations(cache, hot_cpu_id, /*reclaim=*/true); cache.ShuffleCpuCaches(); // Check that the capacity is preserved. @@ -1568,7 +1576,7 @@ TEST(CpuCacheTest, ColdHotCacheShuffleTest) { // change the capacity of either of the caches. for (int i = 0; i < 100; ++i) { ColdCacheOperations(cache, cold_cpu_id, size_class); - HotCacheOperations(cache, hot_cpu_id); + HotCacheOperations(cache, hot_cpu_id, /*reclaim=*/true); cache.ShuffleCpuCaches(); // Check that the capacity is preserved. @@ -1611,6 +1619,7 @@ TEST(CpuCacheTest, ReclaimCpuCache) { // None of the caches should have been reclaimed yet. EXPECT_EQ(cache.GetNumReclaims(cpu), 0); + EXPECT_EQ(cache.GetNumUnpopulates(cpu), 0); // Check that caches are empty. uint64_t used_bytes = cache.UsedBytes(cpu); @@ -1709,6 +1718,110 @@ TEST(CpuCacheTest, ReclaimCpuCache) { cache.Deactivate(); } +TEST(CpuCacheTest, ReclaimCpuCacheAndUnpopulate) { + if (!subtle::percpu::IsFast()) { + return; + } + + absl::BitGen rng; + + for (bool enabled : {false, true}) { + SCOPED_TRACE(absl::StrFormat("Feature enabled: %d", enabled)); + + CpuCache cache; + cache.forwarder().release_drained_slab_metadata_ = enabled; + cache.Activate(); + + const size_t size_class = absl::Uniform(rng, 1, 3); + SCOPED_TRACE(absl::StrFormat("Chosen size class: %zu", size_class)); + + const int num_cpus = NumCPUs(); + + // Verify that we fill at least three hugepages; one (or more) + // to be unpopulated, one not to be, and one account for misalignment + // before or after. + uint8_t per_cpu_shift = CpuCachePeer::GetSlabShift(cache); + const auto shift = subtle::percpu::ToShiftType(per_cpu_shift); + const size_t slabs_size = + subtle::percpu::GetSlabsAllocSize(shift, num_cpus); + if (slabs_size < 3 * kHugePageSize) { + TC_LOG("Not enough CPUs to run test; skipping."); + return; + } + + for (int cpu = 0; cpu < num_cpus; ++cpu) { + SCOPED_TRACE(absl::StrFormat("Failed CPU: %d", cpu)); + ColdCacheOperations(cache, cpu, size_class); + EXPECT_TRUE(cache.HasPopulated(cpu)); + EXPECT_EQ(cache.GetNumUnpopulates(cpu), 0); + } + + // None of the caches are stable, so nothing should be reclaimed + // and nothing should be unpopulated. + cache.TryReclaimingCaches(); + EXPECT_EQ(cache.GetNumReclaims(), 0); + EXPECT_EQ(cache.GetNumUnpopulates(), 0); + + // Do some work on every other CPUs. This should block all unpopulates, + // as no hugepage will contain all-reclaimed caches. The other ones + // should be reclaimed, though. + int num_idle_cpus = 0; + for (int cpu = 0; cpu < num_cpus; ++cpu) { + if (cpu % 2 == 0) { + HotCacheOperations(cache, cpu, /*reclaim=*/false); + } else { + ++num_idle_cpus; + } + } + cache.TryReclaimingCaches(); + EXPECT_EQ(cache.GetNumReclaims(), num_idle_cpus); + EXPECT_EQ(cache.GetNumUnpopulates(), 0); + + // Now do work on only one CPU, to record some misses on that, + // but let the others stay idle. (We do an extra reclaim first, + // or HotCacheOperations() wouldn't actually cause misses. + // This reclaim gets included in GetNumReclaims() below.) + // We should have unpopulates after another round of reclaim, + // but not everything. + // + // The “arbitrary” CPU must already be touched (so even), + // and we'd like it to be so far in that we know that it + // would actually get unpopulated if untouched. + int arbitrary_cpu = (num_cpus / 2) & ~1; // Must already be touched. + if (arbitrary_cpu == 0 || arbitrary_cpu + 1 >= num_cpus) { + TC_LOG("Not enough CPUs to run test; skipping."); + return; + } + HotCacheOperations(cache, arbitrary_cpu, /*reclaim=*/false); + cache.TryReclaimingCaches(); + + EXPECT_EQ(cache.GetNumReclaims(arbitrary_cpu), 0); + EXPECT_EQ(cache.GetNumReclaims(), num_cpus - 1); + + if (enabled) { + // The touched CPU cannot be unpopulated, and since it shares hugepage + // with at least one of its neighbors, at least one of those (probably + // both) must remain, too. + EXPECT_EQ(cache.GetNumUnpopulates(arbitrary_cpu), 0); + EXPECT_LT(cache.GetNumUnpopulates(arbitrary_cpu - 1) + + cache.GetNumUnpopulates(arbitrary_cpu + 1), + 2); + + EXPECT_GT(cache.GetNumUnpopulates(), 0); + EXPECT_LT(cache.GetNumUnpopulates(), num_cpus); + } else { + EXPECT_EQ(cache.GetNumUnpopulates(), 0); + } + + // Flip the flag and run a new reclaim, to test the transition. + cache.forwarder().release_drained_slab_metadata_ = !enabled; + cache.TryReclaimingCaches(); + EXPECT_EQ(cache.GetNumReclaims(), num_cpus); + + cache.Deactivate(); + } +} + TEST(CpuCacheTest, SizeClassCapacityTest) { if (!subtle::percpu::IsFast()) { return; diff --git a/tcmalloc/global_stats.cc b/tcmalloc/global_stats.cc index 965041484..d7e4e63cb 100644 --- a/tcmalloc/global_stats.cc +++ b/tcmalloc/global_stats.cc @@ -689,6 +689,9 @@ void DumpStats(Printer& out, int level) { Parameters::release_stale_pages() == ReleaseStalePages::kEnabled ? 1 : 0); + + out.printf("PARAMETER tcmalloc_release_drained_slab_metadata %d\n", + Parameters::release_drained_slab_metadata()); } } @@ -944,6 +947,9 @@ void DumpStatsInPbtxt(Printer& out, int level) { region.PrintBool( "tcmalloc_release_stale_pages", Parameters::release_stale_pages() == ReleaseStalePages::kEnabled); + + region.PrintBool("tcmalloc_release_drained_slab_metadata", + Parameters::release_drained_slab_metadata()); } bool GetNumericProperty(const char* name_data, size_t name_size, diff --git a/tcmalloc/internal/parameter_accessors.h b/tcmalloc/internal/parameter_accessors.h index 34847329d..65f8ada1e 100644 --- a/tcmalloc/internal/parameter_accessors.h +++ b/tcmalloc/internal/parameter_accessors.h @@ -122,6 +122,9 @@ ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_GetSizeClasses( std::vector* absl_nonnull size_classes); ABSL_ATTRIBUTE_WEAK size_t TCMalloc_Internal_GetPageSize(); + +ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetReleaseDrainedSlabMetadata( + bool v); } #endif // TCMALLOC_INTERNAL_PARAMETER_ACCESSORS_H_ diff --git a/tcmalloc/internal/percpu_tcmalloc.h b/tcmalloc/internal/percpu_tcmalloc.h index 989ef4fa8..26f12d68d 100644 --- a/tcmalloc/internal/percpu_tcmalloc.h +++ b/tcmalloc/internal/percpu_tcmalloc.h @@ -38,6 +38,7 @@ #include "absl/functional/function_ref.h" #include "absl/numeric/bits.h" #include "tcmalloc/internal/delay_injection.h" +#include "tcmalloc/internal/is_aligned_to.h" #include "tcmalloc/internal/logging.h" #include "tcmalloc/internal/mincore.h" #include "tcmalloc/internal/optimization.h" @@ -293,6 +294,18 @@ class TcmallocSlab { // Push/Pop/Grow/Shrink concurrently (even on the same CPU) is safe. void Drain(int cpu, DrainHandler drain_handler); + enum HugePageStatus : uint8_t { kNotTouched = 0, kCannotFree, kShouldFree }; + + // Find whether enough consecutive CPUs are drained so that their metadata + // spans an entire hugepage, and if so, release their metadata. + // + // All CPUs' ResizeInfo must be locked before calling this function. + // The function stops them itself. + void ReleaseSlabMetadataForDrainedCpus( + absl::FunctionRef populated, + absl::FunctionRef unpopulate, + absl::FunctionRef madvise_away_slabs); + PerCPUMetadataState MetadataMemoryUsage() const; // Gets the current shift of the slabs. Intended for use by the thread that @@ -378,6 +391,7 @@ class TcmallocSlab { static Header LoadHeader(AtomicHeader* hdrp); static void StoreHeader(AtomicHeader* hdrp, Header hdr); void DrainCpu(void* slabs, Shift shift, int cpu, DrainHandler drain_handler); + bool CpuIsDrained(void* slabs, Shift shift, int cpu); void DrainOldSlabs(void* slabs, Shift shift, int cpu, const std::array& old_begins, DrainHandler drain_handler); @@ -1262,6 +1276,19 @@ void TcmallocSlab::DrainCpu(void* slabs, Shift shift, int cpu, } } +template +bool TcmallocSlab::CpuIsDrained(void* slabs, Shift shift, int cpu) { + for (size_t size_class = 1; size_class < NumClasses; ++size_class) { + uint16_t begin = begins_[size_class].load(std::memory_order_relaxed); + auto* hdrp = GetHeader(slabs, shift, cpu, size_class); + Header hdr = LoadHeader(hdrp); + if (hdr.end != 0 && hdr.end != begin) { + return false; + } + } + return true; +} + template void TcmallocSlab::DrainOldSlabs( void* slabs, Shift shift, int cpu, @@ -1463,6 +1490,122 @@ void TcmallocSlab::Drain(int cpu, DrainHandler drain_handler) { DrainCpu(slabs, shift, cpu, drain_handler); } +template +void TcmallocSlab::ReleaseSlabMetadataForDrainedCpus( + absl::FunctionRef populated, + absl::FunctionRef unpopulate, + absl::FunctionRef madvise_away_slabs) { + const int n_cpus = num_cpus(); + + // For each hugepage touched by our slabs, track whether there is something + // there that needs to be freed (because all CPUs belonging to that hugepage + // are drained). + // + // // The max per-CPU metadata size is smaller than a hugepage (asserted + // below) and we are aligned to it (also checked below), so it's fine to + // allocate tracking for as many hugepages as we can have CPUs. Still, + // we add + 1 as a buffer. + constexpr int kMaxHugePagesTouched = kMaxCpus + 1; + std::array hugepage_status; + std::fill(hugepage_status.begin(), hugepage_status.end(), kNotTouched); + + // We can't allocate while holding the per-cpu spinlocks. + AllocationGuard enforce_no_alloc; + + // Stop all CPUs. They must also be locked, since we are touching the + // populated bit later. + for (auto& state : state_) { + TC_CHECK(!state.stopped.load(std::memory_order_relaxed)); + state.stopped.store(true, std::memory_order_relaxed); + } + FenceAllCpus(); + + // See which ones are actually drained, and which hugepages we can free. + const auto [slabs, shift] = GetSlabsAndShift(std::memory_order_relaxed); + const size_t slab_size_bytes = 1ULL << static_cast(shift); + TC_CHECK_LT(slab_size_bytes, kHugePageSize); + + // If the slab is not aligned to its own size, freeing any hugepage + // would tear through a CPU's data, and we can do nothing. + if (!IsAlignedTo(slabs, slab_size_bytes)) { + TC_BUG("Slabs are not properly aligned"); + return; + } + + auto address_to_hugepage_number = [](const void* addr) { + return reinterpret_cast(addr) >> kHugePageShift; + }; + const size_t base_hugepage_nr = address_to_hugepage_number(slabs); + + void* slabs_start = CpuMemoryStart(slabs, shift, 0); + if (!IsAlignedTo(slabs_start, kHugePageSize)) { + // If our slab doesn't doesn't start hugepage-aligned, + // we cannot free the first hugepage. + hugepage_status[0] = kCannotFree; + } + + // We cannot free the last page page either, if the slabs doesn't + // end perfectly on a hugepage boundary. (At the very least, + // we'd risk tearing a hugepage.) + void* slabs_end = CpuMemoryStart(slabs, shift, n_cpus); + hugepage_status[address_to_hugepage_number(slabs_end) - base_hugepage_nr] = + kCannotFree; + + // Go through all the CPUs and figure out which hugepage its slab + // lives in. (Because we've already tested that slabs are slab-aligned + // and not larger than a hugepage, and they are also powers of two, + // it can never cross hugepages.) + for (size_t cpu = 0; cpu < n_cpus; ++cpu) { + if (!populated(cpu)) { + continue; + } + + size_t slab_hugepage = + address_to_hugepage_number(CpuMemoryStart(slabs, shift, cpu)); + TC_CHECK_GE(slab_hugepage, base_hugepage_nr); + HugePageStatus& status = hugepage_status[slab_hugepage - base_hugepage_nr]; + + if (status == kCannotFree) { + // No need to check, don't do anything. + } else if (CpuIsDrained(slabs, shift, cpu)) { + status = kShouldFree; + } else { + status = kCannotFree; + } + } + + for (size_t hugepage_idx = 0; hugepage_idx < hugepage_status.size(); + ++hugepage_idx) { + if (hugepage_status[hugepage_idx] != kShouldFree) { + continue; + } + + void* hugepage_start = reinterpret_cast( + (base_hugepage_nr + hugepage_idx) * kHugePageSize); + + // Coalesce neighboring madvises. + size_t bytes_to_free = kHugePageSize; + while (hugepage_idx + 1 < hugepage_status.size() && + hugepage_status[hugepage_idx + 1] == kShouldFree) { + bytes_to_free += kHugePageSize; + ++hugepage_idx; + } + + madvise_away_slabs(hugepage_start, bytes_to_free); + size_t first_cpu = (reinterpret_cast(hugepage_start) - + reinterpret_cast(slabs)) / + slab_size_bytes; + for (unsigned i = 0; i < bytes_to_free / slab_size_bytes; ++i) { + unpopulate(first_cpu + i); + } + } + + // Restart the CPUs again. + for (auto& state : state_) { + state.stopped.store(false, std::memory_order_release); + } +} + template void TcmallocSlab::StopCpu(int cpu) { TC_ASSERT(cpu >= 0 && cpu < num_cpus(), "cpu=%d", cpu); diff --git a/tcmalloc/internal/percpu_tcmalloc_fuzz.cc b/tcmalloc/internal/percpu_tcmalloc_fuzz.cc index c9334efa6..3adf601c7 100644 --- a/tcmalloc/internal/percpu_tcmalloc_fuzz.cc +++ b/tcmalloc/internal/percpu_tcmalloc_fuzz.cc @@ -412,6 +412,45 @@ struct Drain { } }; +struct ReleasePerCPUSlabMetadata { + bool madvise_fail; + + template + friend void AbslStringify(Sink& sink, const ReleasePerCPUSlabMetadata& r) { + absl::Format(&sink, "ReleasePerCPUSlabMetadata{.madvise_fail=%v}", + r.madvise_fail); + } + + void Perform(State& state) const { + // Requires all CPUs to be started (it will stop them itself). + for (int cpu = 0; cpu < state.num_cpus; ++cpu) { + if (state.cpu_stopped[cpu]) { + state.slab.StartCpu(cpu); + state.cpu_stopped[cpu] = false; + } + } + + state.slab.ReleaseSlabMetadataForDrainedCpus( + [&state](int cpu) { return state.cpu_initialized[cpu]; }, + [&state](int cpu) { + state.cpu_initialized[cpu] = false; + for (size_t size_class = 1; size_class < kNumClasses; ++size_class) { + TC_CHECK_EQ(state.slab.Length(cpu, size_class), 0); + TC_CHECK_EQ(state.slab.Capacity(cpu, size_class), 0); + } + }, + [&](void* slab_addr, size_t slab_size) { + if (madvise_fail) { + // Simulate that the madvise failed. + return -1; + } else { + madvise(slab_addr, slab_size, MADV_NOHUGEPAGE); + return madvise(slab_addr, slab_size, MADV_DONTNEED); + } + }); + } +}; + struct SwitchCpu { uint8_t cpu_index; @@ -475,7 +514,7 @@ struct StartCpu { using Instruction = std::variant; + ReleasePerCPUSlabMetadata, SwitchCpu, StopCpu, StartCpu>; template void AbslStringify(Sink& sink, const Instruction& i) { diff --git a/tcmalloc/internal/percpu_tcmalloc_test.cc b/tcmalloc/internal/percpu_tcmalloc_test.cc index 27f896322..e8796536f 100644 --- a/tcmalloc/internal/percpu_tcmalloc_test.cc +++ b/tcmalloc/internal/percpu_tcmalloc_test.cc @@ -587,25 +587,33 @@ struct Context { absl::Span mutexes; std::atomic* capacity; std::atomic* stop; - absl::Span init; + absl::Span init; absl::Span> has_init; std::atomic* max_capacity; GetMaxCapacity GetMaxCapacityFunctor() const { return {max_capacity}; } }; -void InitCpuOnce(Context& ctx, int cpu) { - if (cpu < 0) { - cpu = ctx.slab->CacheCpuSlab().first; - if (cpu < 0) { - return; - } - } - absl::base_internal::LowLevelCallOnce(&ctx.init[cpu], [&]() { - absl::MutexLock lock(ctx.mutexes[cpu]); +// NOTE: Once you release the lock, the CPU may be uninit-ed (to have its +// per-CPU slab metadata released). +void InitCpuOnceLockHeld(Context& ctx, int cpu) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(ctx.mutexes[cpu]) { + if (!ctx.init[cpu]) { ctx.slab->InitCpu(cpu, ctx.GetMaxCapacityFunctor()); ctx.has_init[cpu].store(true, std::memory_order_relaxed); - }); + ctx.init[cpu] = true; + } +} + +// See comment on InitCpuOnceLockHeld(); the CPU is not guaranteed to remain +// initialized after the call. +void InitCurrentCpuOnce(Context& ctx) { + int cpu = ctx.slab->CacheCpuSlab().first; + if (cpu < 0) { + return; + } + absl::MutexLock lock(ctx.mutexes[cpu]); + InitCpuOnceLockHeld(ctx, cpu); } int GetResizedMaxCapacities(Context& ctx, @@ -641,13 +649,13 @@ void StressThread(size_t thread_id, absl::BitGen rnd(absl::SeedSeq({thread_id})); while (!*ctx.stop) { size_t size_class = absl::Uniform(rnd, 1, kStressSlabs); - const int what = absl::Uniform(rnd, 0, 91); + const int what = absl::Uniform(rnd, 0, 101); if (what < 10) { if (!block.empty()) { if (ctx.slab->Push(size_class, block.back())) { block.pop_back(); } else { - InitCpuOnce(ctx, -1); + InitCurrentCpuOnce(ctx); } } } else if (what < 20) { @@ -657,7 +665,7 @@ void StressThread(size_t thread_id, EXPECT_NE(item, nullptr); block.push_back(item); } else { - InitCpuOnce(ctx, -1); + InitCurrentCpuOnce(ctx); } } else if (what < 30) { if (!block.empty()) { @@ -701,7 +709,8 @@ void StressThread(size_t thread_id, if (cpu >= 0) { // Grow mutates the header array and must be operating on // an initialized core. - InitCpuOnce(ctx, cpu); + absl::MutexLock lock(ctx.mutexes[cpu]); + InitCpuOnceLockHeld(ctx, cpu); res = ctx.slab->Grow(cpu, size_class, n, [&](uint8_t shift) { return ctx.GetMaxCapacityFunctor()(size_class); @@ -721,11 +730,12 @@ void StressThread(size_t thread_id, } else if (what < 70) { int cpu = absl::Uniform(rnd, 0, num_cpus); + absl::MutexLock lock(ctx.mutexes[cpu]); + // ShrinkOtherCache mutates the header array and must be operating on an // initialized core. - InitCpuOnce(ctx, cpu); + InitCpuOnceLockHeld(ctx, cpu); - absl::MutexLock lock(ctx.mutexes[cpu]); size_t to_shrink = absl::Uniform(rnd, 0, kStressCapacity) + 1; ctx.slab->StopCpu(cpu); size_t total_shrunk = ctx.slab->ShrinkOtherCache( @@ -758,11 +768,12 @@ void StressThread(size_t thread_id, if (to_grow != 0) { int cpu = absl::Uniform(rnd, 0, num_cpus); + absl::MutexLock lock(ctx.mutexes[cpu]); + // GrowOtherCache mutates the header array and must be operating on an // initialized core. - InitCpuOnce(ctx, cpu); + InitCpuOnceLockHeld(ctx, cpu); - absl::MutexLock lock(ctx.mutexes[cpu]); ctx.slab->StopCpu(cpu); size_t grown = ctx.slab->GrowOtherCache( cpu, size_class, to_grow, @@ -772,17 +783,18 @@ void StressThread(size_t thread_id, EXPECT_GE(grown, 0); ctx.capacity->fetch_add(to_grow - grown); } - } else { + } else if (what < 90) { int cpu = absl::Uniform(rnd, 0, num_cpus); // Flip coin on whether to unregister rseq on this thread. const bool unregister = absl::Bernoulli(rnd, 0.5); - // Drain mutates the header array and must be operating on an initialized - // core. - InitCpuOnce(ctx, cpu); - { absl::MutexLock lock(ctx.mutexes[cpu]); + + // Drain mutates the header array and must be operating on an + // initialized core. + InitCpuOnceLockHeld(ctx, cpu); + std::optional scoped_rseq; if (unregister) { scoped_rseq.emplace(); @@ -806,6 +818,60 @@ void StressThread(size_t thread_id, // Verify we re-registered with rseq as required. TC_ASSERT(IsFastNoInit()); + } else { + // Drain a large subset (~90%) of CPUs, then try to release + // per-CPU slab metadata. + + for (int cpu = 0; cpu < num_cpus; ++cpu) { + ctx.mutexes[cpu].lock(); + } + for (int cpu = 0; cpu < num_cpus; ++cpu) { + if (absl::Bernoulli(rnd, 0.1) || + !ctx.has_init[cpu].load(std::memory_order_relaxed)) { + continue; + } + + ctx.slab->Drain( + cpu, [&block, &ctx, cpu](int cpu_arg, size_t size_class, + void** batch, size_t size, size_t cap) { + EXPECT_EQ(cpu, cpu_arg); + EXPECT_LT(size_class, kStressSlabs); + EXPECT_LE(size, kMaxStressCapacity); + EXPECT_LE(cap, kMaxStressCapacity); + for (size_t i = 0; i < size; ++i) { + EXPECT_NE(batch[i], nullptr); + block.push_back(batch[i]); + } + ctx.capacity->fetch_add(cap); + }); + } + + ctx.slab->ReleaseSlabMetadataForDrainedCpus( + [&ctx](int cpu) { + return ctx.has_init[cpu].load(std::memory_order_relaxed); + }, + [&ctx](int cpu) { + ctx.init[cpu] = false; + ctx.has_init[cpu].store(false, std::memory_order_release); + for (size_t size_class = 1; size_class < kStressSlabs; + ++size_class) { + EXPECT_EQ(ctx.slab->Length(cpu, size_class), 0); + EXPECT_EQ(ctx.slab->Capacity(cpu, size_class), 0); + } + }, + [&rnd](void* slab_addr, size_t slab_size) { + if (absl::Bernoulli(rnd, 0.1)) { + // Simulate that the madvise failed. + return -1; + } else { + madvise(slab_addr, slab_size, MADV_NOHUGEPAGE); + return madvise(slab_addr, slab_size, MADV_DONTNEED); + } + }); + + for (int cpu = 0; cpu < num_cpus; ++cpu) { + ctx.mutexes[cpu].unlock(); + } } } } @@ -1021,8 +1087,10 @@ TEST_P(StressThreadTest, Stress) { max_capacity[size_class].store(kStressCapacity, std::memory_order_relaxed); } - // once_flag's protect InitCpu on a CPU. - std::vector init(num_cpus); + // Protect InitCpu on a CPU. Each flag is protected by that CPU's mutex; + // we cannot use absl::once_flag because we need to be able to unpopulate + // the CPU (i.e., set the flag back to false). + absl::FixedArray init(num_cpus, false); // Tracks whether init has occurred on a CPU for use in ResizeSlabs. std::vector> has_init(num_cpus); diff --git a/tcmalloc/parameters.cc b/tcmalloc/parameters.cc index 85a460ba1..8d6e0edf0 100644 --- a/tcmalloc/parameters.cc +++ b/tcmalloc/parameters.cc @@ -233,6 +233,9 @@ ABSL_CONST_INIT std::atomic Parameters::enable_unfiltered_collapse_( ABSL_CONST_INIT std::atomic Parameters::release_max_cold_pages_(false); ABSL_CONST_INIT std::atomic Parameters::event_trace_memory_limit_( 16 << 20); +ABSL_CONST_INIT +std::atomic Parameters::release_drained_slab_metadata_(false); + static std::atomic& madvise_cold_regions_nohugepage_enabled() { ABSL_CONST_INIT static absl::once_flag flag; @@ -345,7 +348,6 @@ ReleaseStalePages Parameters::release_stale_pages() { return v.load(std::memory_order_relaxed); } - int32_t Parameters::max_per_cpu_cache_size() { return tc_globals.cpu_cache().CacheLimit(); } @@ -606,7 +608,6 @@ void TCMalloc_Internal_SetPerCpuCachesDynamicSlabEnabled(bool v) { Parameters::per_cpu_caches_dynamic_slab_.store(v, std::memory_order_relaxed); } - uint8_t TCMalloc_Internal_GetMinHotAccessHint() { return static_cast(Parameters::min_hot_access_hint()); } @@ -670,6 +671,10 @@ void TCMalloc_Internal_SetEventTraceMemoryLimit(int64_t v) { Parameters::event_trace_memory_limit_.store(v, std::memory_order_relaxed); } +void TCMalloc_Internal_SetReleaseDrainedSlabMetadata(bool v) { + Parameters::release_drained_slab_metadata_.store(v, + std::memory_order_relaxed); +} } // extern "C" GOOGLE_MALLOC_SECTION_END diff --git a/tcmalloc/parameters.h b/tcmalloc/parameters.h index 55e0c672d..91b8cccf8 100644 --- a/tcmalloc/parameters.h +++ b/tcmalloc/parameters.h @@ -209,8 +209,15 @@ class Parameters { std::memory_order_relaxed); } - static HeapPartitioningMode heap_partitioning_mode(); + static bool release_drained_slab_metadata() { + return release_drained_slab_metadata_.load(std::memory_order_relaxed); + } + static void set_release_drained_slab_metadata(bool value) { + TCMalloc_Internal_SetReleaseDrainedSlabMetadata(value); + } + + static HeapPartitioningMode heap_partitioning_mode(); // TODO: b/527473378 - Remove this function once the experiment is cleaned up. static ReleaseStalePages release_stale_pages(); @@ -250,6 +257,7 @@ class Parameters { friend void ::TCMalloc_Internal_SetHugeRegionAdaptiveReleaseEnabled(bool v); friend void ::TCMalloc_Internal_SetReleaseMaxColdPages(bool v); friend void ::TCMalloc_Internal_SetEventTraceMemoryLimit(int64_t v); + friend void ::TCMalloc_Internal_SetReleaseDrainedSlabMetadata(bool v); static std::atomic guarded_sampling_interval_; static std::atomic max_per_cpu_cache_size_; @@ -271,6 +279,7 @@ class Parameters { static std::atomic enable_unfiltered_collapse_; static std::atomic release_max_cold_pages_; static std::atomic event_trace_memory_limit_; + static std::atomic release_drained_slab_metadata_; }; } // namespace tcmalloc_internal diff --git a/tcmalloc/testing/get_stats_test.cc b/tcmalloc/testing/get_stats_test.cc index 26b4a7805..0ed902b87 100644 --- a/tcmalloc/testing/get_stats_test.cc +++ b/tcmalloc/testing/get_stats_test.cc @@ -128,7 +128,6 @@ TEST_F(GetStatsTest, Pbtxt) { EXPECT_THAT(buf, HasSubstr("back_small_allocations: false")); EXPECT_THAT(buf, ContainsRegex("(back_size_threshold_bytes: [1-9][0-9]*)")); - EXPECT_THAT(buf, HasSubstr("tcmalloc_release_pages_from_huge_region: true")); if (IsExperimentActive(Experiment::TCMALLOC_HUGE_REGION_ADAPTIVE_RELEASE)) { EXPECT_THAT(buf, HasSubstr("tcmalloc_huge_region_adaptive_release: true")); @@ -160,6 +159,8 @@ TEST_F(GetStatsTest, Pbtxt) { EXPECT_THAT(buf, HasSubstr("tcmalloc_release_stale_pages: false")); } + EXPECT_THAT(buf, HasSubstr("tcmalloc_release_drained_slab_metadata: false")); + sized_delete(alloc, kSize); } @@ -237,7 +238,6 @@ TEST_F(GetStatsTest, Parameters) { EXPECT_THAT( buf, HasSubstr(R"(PARAMETER tcmalloc_enable_unfiltered_collapse 0)")); - EXPECT_THAT( buf, HasSubstr(R"(PARAMETER tcmalloc_release_pages_from_huge_region 1)")); @@ -290,6 +290,10 @@ TEST_F(GetStatsTest, Parameters) { EXPECT_THAT(buf, HasSubstr(R"(PARAMETER tcmalloc_release_stale_pages 0)")); } + + EXPECT_THAT( + buf, + HasSubstr(R"(PARAMETER tcmalloc_release_drained_slab_metadata 0)")); } Parameters::set_hpaa_subrelease(true);