diff --git a/ddprof-lib/src/main/cpp/callTraceHashTable.h b/ddprof-lib/src/main/cpp/callTraceHashTable.h index bc2e2486c..d36cde4c4 100644 --- a/ddprof-lib/src/main/cpp/callTraceHashTable.h +++ b/ddprof-lib/src/main/cpp/callTraceHashTable.h @@ -104,7 +104,7 @@ class CallTraceHashTable { // - ACQUIRE loads in collect(), put(), and putWithExistingId() // Required for correct visibility on weakly-ordered architectures (aarch64). LongHashTable* _table; - + volatile u64 _overflow; u64 calcHash(int num_frames, ASGCT_CallFrame *frames, bool truncated); diff --git a/ddprof-lib/src/main/cpp/callTraceStorage.cpp b/ddprof-lib/src/main/cpp/callTraceStorage.cpp index e7a9d9f4a..7bde3602e 100644 --- a/ddprof-lib/src/main/cpp/callTraceStorage.cpp +++ b/ddprof-lib/src/main/cpp/callTraceStorage.cpp @@ -4,6 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include #include "callTraceStorage.h" #include "counters.h" #include "log.h" diff --git a/ddprof-lib/src/main/cpp/classTagAllocator.h b/ddprof-lib/src/main/cpp/classTagAllocator.h new file mode 100644 index 000000000..9a25d6cfa --- /dev/null +++ b/ddprof-lib/src/main/cpp/classTagAllocator.h @@ -0,0 +1,76 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _CLASS_TAG_ALLOCATOR_H +#define _CLASS_TAG_ALLOCATOR_H + +#include "arch.h" +#include + +// Process-wide, negative JVMTI class-object tag allocator, shared by +// ReferenceChainTracker (which tags every loaded class's own jclass object +// via SetTag - see resolveLoadedClasses(), referenceChains.cpp) and +// LivenessTracker (which needs a stable per-class identifier independent of +// Profiler::classMap()'s dictionary id - see KlassPopulationEntry:: +// stable_class_tag's own comment, livenessTracker.h, for why: that +// dictionary can be compacted/regenerated, silently reassigning the same +// class a different id at different points in the process's life, breaking +// any attempt to correlate a klass_id LivenessTracker reports as growing +// against ReferenceChainTracker::FrontierEntry::referrer_klass values +// recorded at a different time). +// +// A single shared counter, not one independently owned by each subsystem, +// for two reasons, both load-bearing: +// 1. Two independent counters could otherwise hand out the SAME numeric +// value to TWO DIFFERENT classes (one minted by each subsystem for a +// class the other has not seen yet), making any cross-subsystem +// comparison meaningless. +// 2. Class tags must stay strictly NEGATIVE: +// ReferenceChainTracker::heapReferenceCallback() (referenceChains.cpp) +// uses `*tag_ptr < 0` to distinguish "this heap-walk-visited object is a +// pre-tagged class object" from an ordinary admitted instance (always +// tagged with a positive value via nextTag()). A class tagged by a +// counter that does not preserve this sign convention would be +// misidentified as an ordinary object and incorrectly admitted into the +// frontier table - a real correctness bug, not just a matching +// inconvenience. +// +// Deliberately a plain header-only function (Meyer's-singleton pattern, +// exactly like LivenessTracker::instance()/ReferenceChainTracker:: +// instance()'s own lazy-static singletons) rather than a member of either +// singleton class: ReferenceChainTracker already depends on LivenessTracker +// (referenceChains.cpp includes livenessTracker.h and calls into it), so +// putting this counter inside either one and having the other call into it +// would introduce a circular dependency between the two headers. +namespace ClassTagAllocator { + +inline volatile jlong &magnitude() { + static volatile jlong m = 1; + return m; +} + +// Hands out a fresh negative class tag - see this file's own header comment +// for why negative, and why this must be the only place in the process that +// mints one. +inline jlong next() { return -atomicIncRelaxed(magnitude(), (jlong)1); } + +// Test-only: resets the shared counter back to its starting value. Without +// this, gtest cases that assert on exact tag values (e.g. "the first class +// tagged gets -1") would see values keep climbing across every TEST_F in the +// same gtest binary, since this counter is genuinely process-wide (shared +// with LivenessTracker) rather than per-ReferenceChainTracker-instance. +inline void resetForTest() { + // Atomic exchange, matching next()'s atomicIncRelaxed RMW on the same + // variable: a plain volatile store can tear or be lost against a concurrent + // RMW (e.g. a tracker thread from a prior TEST_F not fully quiesced), which + // would mint duplicate negative tags - the cross-subsystem collision this + // shared allocator exists to prevent. Callers must still ensure no tracker + // thread is live (reset in TearDown after tracker->stop()). + __atomic_exchange_n(&magnitude(), (jlong)1, __ATOMIC_RELAXED); +} + +} // namespace ClassTagAllocator + +#endif diff --git a/ddprof-lib/src/main/cpp/counters.h b/ddprof-lib/src/main/cpp/counters.h index 52db46c09..3293d0773 100644 --- a/ddprof-lib/src/main/cpp/counters.h +++ b/ddprof-lib/src/main/cpp/counters.h @@ -164,6 +164,45 @@ * signal for spotting a recurrence. */ \ X(METADATA_TREE_NULL_CHILD, "metadata_tree_null_child") \ X(METADATA_TREE_DEPTH_EXCEEDED, "metadata_tree_depth_exceeded") \ + /* A resolved datadog.ReferenceChain could not be cached in \ + * ReferenceChainTracker::_resolved_chains (referenceChains.h): a brand-new \ + * leak-candidate klass arrived with the cache already at \ + * MAX_RESOLVED_CHAINS, so its chain is dropped rather than evicting some \ + * other still-live sample's chain. See that constant's own comment. */ \ + X(REFERENCE_CHAIN_EVENTS_DROPPED, "reference_chain_events_dropped") \ + /* ReferenceChainTracker::releaseSearchTags() (referenceChains.cpp) failed \ + * to call GetObjectsWithTags() for at least one batch - the search's tag \ + * release is retried on a later call rather than proceeding, but this \ + * counts how often that retry path is taken. */ \ + X(REFERENCE_CHAIN_TAG_RELEASE_FAILED, "reference_chain_tag_release_failed") \ + /* The profiler-side reference-chain writer could not acquire a \ + * sample-record lock within its bounded retry budget and dropped the \ + * already-dequeued datadog.ReferenceChain event for this dump - not \ + * permanently lost, since ReferenceChainTracker::_resolved_chains (see \ + * REFERENCE_CHAIN_EVENTS_DROPPED above) keeps the resolved chain cached \ + * and re-emits it on a later dump while the leak candidate is still \ + * live. */ \ + X(REFERENCE_CHAIN_WRITE_DROPPED, "reference_chain_write_dropped") \ + /* FrontierTable's own calloc/realloc-backed storage (referenceChains.cpp) - \ + * outside NMT's visibility since it bypasses os::malloc, so this is the only \ + * way to attribute its native RSS contribution. */ \ + X(REFERENCE_CHAIN_FRONTIER_TABLE_BYTES, "reference_chain_frontier_table_bytes") \ + X(REFERENCE_CHAIN_FRONTIER_TABLE_CAPACITY, "reference_chain_frontier_table_capacity") \ + X(REFERENCE_CHAIN_CANDIDATE_COUNT, "reference_chain_candidate_count") \ + X(REFERENCE_CHAIN_CANDIDATES_FOUND, "reference_chain_candidates_found") \ + /* admitStaticFieldRoots() per-class non-static quota: non-STATIC_FIELD \ + * edges (CONSTANT_POOL, INTERFACE, SUPERCLASS, CLASS_LOADER, ...) that \ + * were dropped because the class already hit \ + * STATIC_FIELD_SWEEP_NON_STATIC_CAP_PER_CLASS. Total drops across all \ + * classes/laps — compare against kind_counts (k9 total) to gauge how \ + * much CP pressure the quota is absorbing. */ \ + X(REFERENCE_CHAIN_STATIC_SWEEP_NON_STATIC_DROPPED, "reference_chain_static_sweep_non_static_dropped") \ + /* Incremented once per class that hit the non-static cap at least once \ + * in a lap (on the first drop for that class). Distinguishes "a few fat \ + * outlier classes dropping many edges" from "systematic drops across \ + * almost all classes" — if this tracks the total class count per lap, \ + * the cap is too low; if it stays near zero, the cap is fine. */ \ + X(REFERENCE_CHAIN_STATIC_SWEEP_CLASSES_CAPPED, "reference_chain_static_sweep_classes_capped") \ DD_COUNTER_TABLE_FAULT_INJECTION(X) \ DD_COUNTER_TABLE_FI_DEBUG(X) \ DD_COUNTER_TABLE_SAMPLER_PERF(X) \ diff --git a/ddprof-lib/src/main/cpp/flightRecorder.h b/ddprof-lib/src/main/cpp/flightRecorder.h index 79be64f21..96d473862 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.h +++ b/ddprof-lib/src/main/cpp/flightRecorder.h @@ -581,20 +581,21 @@ class FlightRecorder { // Mirrors recordHeapUsage()'s shape exactly - ReferenceChainAbandonedEvent // is not stack-sample-shaped (no tid/call_trace_id), same as HeapUsage. - // Called from Profiler::writeReferenceChainAbandoned() (profiler.cpp), + // Called from the profiler's dump-time abandoned-event drain, // wired from Profiler::dump() the same way LivenessTracker::flush() is. void recordReferenceChainAbandoned(int lock_index, ReferenceChainAbandonedEvent *event); // Mirrors recordReferenceChainAbandoned() above exactly, for - // ReferenceChainEvent instead. Called from Profiler::writeReferenceChain() - // (profiler.cpp), itself called from Profiler::dump()'s drain loop over + // ReferenceChainEvent instead. Called from the profiler's dump()-time + // writer, itself called from Profiler::dump()'s drain loop over // the engine's resolved-chain cache snapshot: the BFS // scheduling thread only caches resolved chains and each dump re-emits // the cache, so chain events // are written on dump()'s own thread, not from the tracker thread, and // unlike recordReferenceChainAbandoned() (unbounded retry budget per - // event) the batch shares one deadline (writeReferenceChain()'s comment). + // event) the batch shares one deadline (see the writer's contract in + // Profiler - the drain batch, not each event, owns the retry budget). void recordReferenceChain(int lock_index, ReferenceChainEvent *event); }; diff --git a/ddprof-lib/src/main/cpp/livenessTracker.cpp b/ddprof-lib/src/main/cpp/livenessTracker.cpp index 4f7e5127e..5bfdb9aef 100644 --- a/ddprof-lib/src/main/cpp/livenessTracker.cpp +++ b/ddprof-lib/src/main/cpp/livenessTracker.cpp @@ -4,16 +4,24 @@ */ #include +#include +#include #include #include #include "arch.h" +#include "common.h" #include "context.h" #include "context_api.h" #include "hotspot/vmStructs.h" +#include "hotspot/vmStructs.inline.h" #include "incbin.h" #include "jniHelper.h" #include "livenessTracker.h" +#include "rcDebugLevel.h" +#include "objectSampler.h" +#include "vmEntry.h" +#include "referenceChains.h" #include "log.h" #include "nativeMem.h" #include "os.h" @@ -29,28 +37,211 @@ constexpr int LivenessTracker::MAX_TRACKING_TABLE_SIZE; constexpr int LivenessTracker::MIN_SAMPLING_INTERVAL; -void LivenessTracker::cleanup_table(bool forced) { +namespace { + +// Max surviving entries whose per-epoch JNI class resolution (resolveKlassId: +// GetObjectClass + Class.getName() + StringDictionary lookup) a single +// cleanup_table() sweep performs under the exclusive _table_lock. Entries +// beyond the budget fall back to their cached class id (the exact semantics +// the allow_resolve=false path already uses); the next epoch's sweep resolves +// the next tranche, and cached ids accumulated across sweeps keep most +// entries resolved anyway. Bounds the sweep's exclusive-lock window so it +// stays proportional to table bookkeeping rather than to survivor count - +// every shared-lock scanner (tagLeakInstances(), getLiveTraceIds()) is +// blocked for the whole sweep otherwise. +constexpr u32 RESOLVE_BUDGET_PER_SWEEP = 256; + +// Trend statistics of a chronological ring window - the one computation +// hasQualifyingGrowth() (per-klass count_ring) and heapFloorRising() (the +// aggregate _heap_floor_ring) both need, factored out so the window/index +// derivation and the two aggregation loops exist in exactly one place rather +// than three near-identical copies. Templated on the reader rather than the +// ring's element type or storage: the per-klass ring is a plain array read +// under the caller's already-held _table_lock, while the heap-floor ring is +// lock-free and read via loadAcquire() (see _heap_floor_ring's own comment, +// livenessTracker.h) - `read(i)` lets each caller supply its own access +// discipline for physical slot `i` without this shared loop needing to know +// which one applies. +// +// DESPITE THE NAME, this is not thirds statistics: the design doc's original +// "mean of earliest third vs mean of recent third" comparison was replaced by +// a full-window least-squares linear regression (see ringThirdsStats below), +// which uses all samples and is far more robust for oscillating-but-growing +// trends. The field names survive as the regression values consumers treat +// as the window's "earliest"/"recent" levels: +// earliest_mean - regression value at the window's OLDEST sample (x = 0); +// recent_mean - regression value at the window's NEWEST sample (x = fill-1); +// earliest_min - true minimum over the FULL window; +// recent_min - true minimum over the most recent HALF of the window. +// Consumers read earliest_mean/recent_mean as a smoothed start-vs-end delta +// (a regression slope over the window's span) and earliest_min/recent_min as +// floor checks. Renaming the fields would touch every consumer for no +// behavioral change, so the mapping is documented here instead. +struct RingThirdsStats { + double earliest_mean; // regression value at the window's oldest sample + double recent_mean; // regression value at the window's newest sample + double earliest_min; // true min over the full window + double recent_min; // true min over the window's most recent half +}; + +template +bool ringThirdsStats(int head, int fill, int ring_size, int min_fill, + Reader read, RingThirdsStats *out) { + if (fill < min_fill) { + return false; + } + // Chronological (oldest-first) index of the window's first sample. + int start = (head - fill + ring_size) % ring_size; + // Full-window least-squares linear regression: y = a + b*x. + // x = sample position within the window (0 = oldest, fill-1 = newest), + // y = population count. This uses all samples (not just first/last + // halves or thirds) and is far more robust for oscillating-but- + // growing trends than comparing two sub-windows. O(fill) = O(30) per + // klass per scan — negligible. + int n = fill; + if (n < 2) { + return false; + } + double sum_x = 0, sum_y = 0, sum_xx = 0, sum_xy = 0; + double earliest_min = std::numeric_limits::max(); + double recent_min = std::numeric_limits::max(); + for (int i = 0; i < n; i++) { + double v = read((start + i) % ring_size); + sum_x += i; + sum_y += v; + sum_xx += (double)i * i; + sum_xy += (double)i * v; + if (v < earliest_min) { + earliest_min = v; + } + if (i >= n / 2 && v < recent_min) { + recent_min = v; + } + } + double denom = (double)n * sum_xx - sum_x * sum_x; + if (denom == 0) { + return false; + } + double slope = ((double)n * sum_xy - sum_x * sum_y) / denom; + double intercept = (sum_y - slope * sum_x) / n; + out->earliest_mean = intercept; + out->recent_mean = intercept + slope * (n - 1); + out->earliest_min = earliest_min; + out->recent_min = recent_min; + return true; +} + +// Recent-half corroboration for a usage ring (see secondsToOOM()'s own +// comment): a rising full-window trend whose most recent half is flat is a +// plateaued step change, not ongoing growth. The recent half's own +// regression (see ringThirdsStats) must show a strictly positive delta for +// the full-window trend to stand. A too-sparse recent half (below min_fill) +// REJECTS the projection rather than letting it through: false means +// "reject". Deliberately stricter than the single-ring version's +// have_recent_half semantics (which only rejected on a confirmed flat +// recent half) - with the ring-fill floors the caller already enforces, a +// sparse recent half means the recent data does not yet support the trend, +// so the boundary projection waits for more samples. +template +bool corroborateRecentHalf(u8 head, u8 fill, int ring_size, int min_fill, + Reader read) { + int half_fill = fill / 2; + RingThirdsStats recent_half_stats; + bool have_half = ringThirdsStats(head, half_fill, ring_size, min_fill, read, + &recent_half_stats); + double half_delta = have_half + ? recent_half_stats.recent_mean - recent_half_stats.earliest_mean + : 0.0; + return have_half && half_delta > 0; +} + +} // namespace + +void LivenessTracker::cleanup_table(bool forced, bool allow_resolve) { u64 current = load(_last_gc_epoch); u64 target_gc_epoch = load(_gc_epoch); + TEST_LOG_SUMMARY("LivenessTracker::cleanup_table forced=%d gc_generations=%d current_epoch=%llu " + "target_epoch=%llu table_size=%d", + forced, _gc_generations.load(std::memory_order_relaxed), (unsigned long long)current, + (unsigned long long)target_gc_epoch, _table_size); - if ((target_gc_epoch == _last_gc_epoch || - !__atomic_compare_exchange_n(&_last_gc_epoch, ¤t, - target_gc_epoch, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) && - !forced) { - // if the last processed GC epoch hasn't changed, or if we failed to update - // it, there's nothing to do - return; - } + // is_epoch_owner is true iff this call is the one that moves _last_gc_epoch + // to target_gc_epoch - i.e. the first cleanup_table() call (forced or not) + // to observe this particular GC epoch transition. Population accounting + // below is gated on this rather than on !forced, so a forced (table- + // overflow) sweep still folds one sample per genuinely new epoch instead + // of either skipping it entirely or double-counting the same epoch across + // repeated forced sweeps. + // + // The authoritative claim happens BELOW, under the table lock: a forced + // sweep can claim a new epoch on one thread while a GC callback claims a + // still-newer epoch on another, and claiming up front would let the newer + // epoch's fold enter the population history before the older one's (lock + // acquisition order is not claim order), folding the same epoch range + // twice and skewing the trend. The check here stays as an advisory + // early-exit so the common no-op call never takes the lock or the JNIEnv. + u64 advisory_current = current; + if (target_gc_epoch != advisory_current || forced) { + JNIEnv *env = VM::jni(); - JNIEnv *env = VM::jni(); + _table_lock.lock(); - int epoch_diff = (int)(target_gc_epoch - current); + u64 claimed = load(_last_gc_epoch); + // Forward-only claim: a cleanup caller captured target_gc_epoch BEFORE it + // took the lock; by the time it holds the lock another caller may already + // have published a NEWER epoch (target 6 published while this call's + // snapshot still says 5). The old `!=`-gated unconditional CAS would move + // _last_gc_epoch BACKWARD 6->5, making epoch_diff negative and wrapping + // every survivor's unsigned age below - folding epochs out of order and + // manufacturing leak candidates / duplicate population samples. Only the + // caller whose target is strictly newer claims; a stale snapshot skips + // epoch accounting entirely (its survivors are still swept, their ages + // simply do not move for this no-op epoch). + bool is_epoch_owner = target_gc_epoch > claimed && + __atomic_compare_exchange_n(&_last_gc_epoch, &claimed, target_gc_epoch, + false, __ATOMIC_RELAXED, __ATOMIC_RELAXED); + // On a lost CAS race `claimed` holds the actual current epoch (>= our + // stale target), so the raw diff is <= 0 for every non-owner; clamp so a + // survivor's unsigned age can never wrap. + int epoch_diff = (int)(target_gc_epoch - claimed); + if (epoch_diff < 0) { + epoch_diff = 0; + } + + // Detect a class-map reset the same way + // ReferenceChainTracker::resolveLoadedClasses() does (referenceChains.cpp) + // - see _last_class_map_generation's own comment (livenessTracker.h). + // cached_klass_id and _klass_population's klass_id keys are + // StringDictionary ids from whatever generation was current when they were + // resolved; once Profiler::start() clears that dictionary and restarts its + // id namespace, those cached ids can silently collide with a newly + // assigned, unrelated class. Drop every such cache before this pass reads + // or writes any of them. + u64 current_class_map_generation = Profiler::instance()->classMap()->generation(); + if (current_class_map_generation != _last_class_map_generation) { + for (u32 i = 0; i < _table_size; i++) { + _table[i].cached_klass_id = 0; + } + for (int i = 0; i < _klass_population_size; i++) { + for (int r = 0; r < _klass_population[i].representative_count; r++) { + jweak rep = _klass_population[i].representatives[r]; + if (rep != nullptr) { + env->DeleteWeakGlobalRef(rep); + } + } + } + _klass_population_size = 0; + _klass_count_scratch_size = 0; + _last_class_map_generation = current_class_map_generation; + } - _table_lock.lock(); u32 sz = _table_size; if (sz > 0) { u64 start = OS::nanotime(), end; u32 newsz = 0; + // Per-sweep JNI-resolution budget - see the survivor loop's allow_resolve + // comment below for why the exclusive-lock window must stay bounded. + u32 resolve_budget = RESOLVE_BUDGET_PER_SWEEP; std::set kept_classes; for (u32 i = 0; i < sz; i++) { if (_table[i].ref != nullptr && @@ -63,22 +254,1634 @@ void LivenessTracker::cleanup_table(bool forced) { _table[i].call_trace_id = 0; } _table[target].age += epoch_diff; + + if (_gc_generations.load(std::memory_order_relaxed) && is_epoch_owner) { + // Per-klass population tracking (design doc's Open Question 3) - + // gated on _gc_generations so this new cost is paid only when the + // caller actually asked for generation/survival-shaped data + // (Arguments::_gc_generations), not for every liveness-tracking + // session. Gated on is_epoch_owner (not !forced) so a forced + // (table-overflow) sweep still contributes one population sample + // per genuinely new GC epoch instead of silently dropping it. + u32 klass_id = 0; + if (allow_resolve && resolve_budget > 0) { + // GetObjectClass + Class.getName() + StringDictionary lookup per + // surviving entry, previously paid only at JFR-flush time (see + // flush_table() below). Only affordable off the allocation-hot + // path - flush_table()/stop()'s cadence and + // LivenessTracker::maybeForceCleanup()'s background-thread tick + // both pass allow_resolve=true; track()'s hot-path forced sweep + // does not (see cleanup_table()'s own header comment). + // + // Bounded per sweep (RESOLVE_BUDGET_PER_SWEEP): every resolution + // here runs under the EXCLUSIVE _table_lock, so an unbounded + // survivor count would stretch this sweep's critical section + // proportionally to the population (blocking every shared-lock + // scanner for the whole per-survivor JNI sequence). Entries past + // the budget fall through to the cached-id path below - the + // exact semantics the allow_resolve=false path already accepts - + // and the next epoch's sweep resolves the next tranche; the + // cached ids accumulated across sweeps keep most entries + // resolved anyway. + resolve_budget--; + jobject ref = env->NewLocalRef(_table[target].ref); + if (ref != nullptr) { + klass_id = resolveKlassId(env, ref); + if (klass_id != 0) { + // Cache the resolution: flush_table() runs its own + // GetObjectClass+Class.getName()+lookupClass() sequence for + // every surviving entry immediately after cleanup_table() + // returns (flush_table() always calls cleanup_table() first), + // which would otherwise repeat this exact JNI round-trip for + // the same object. An object's class is immutable, so this + // value stays valid for flush_table()'s read below, and for + // a later non-resolving sweep's read right below. + _table[target].cached_klass_id = klass_id; + } + env->DeleteLocalRef(ref); + } + } else { + // track()'s table-overflow branch calls cleanup_table(true, + // false) synchronously from the allocation-sampling call stack + // (JVMTI SampledObjectAlloc callback). resolveKlassId() calls + // Class.getName(), a genuine Java-bytecode upcall (unlike the + // plain native jvmti->GetClassSignature() call + // ObjectSampler::recordAllocation already makes on this same + // callback stack) - too costly, and too re-entrancy-prone via + // the String allocation it can trigger, to run from there. Reuse + // whatever class id an earlier resolving sweep already resolved + // for this entry instead; if it was never resolved, this entry's + // sample for this epoch is dropped rather than resolving now. + klass_id = _table[target].cached_klass_id; + } + if (klass_id != 0) { + accumulateKlassCount(klass_id, _table[target].age, _table[target].ref, + _table[target].tid); + } + } } else { jweak tmpRef = _table[i].ref; _table[i].ref = nullptr; env->DeleteWeakGlobalRef(tmpRef); _table[i].call_trace_id = 0; + if (_table[i].leak_tag != 0) { + releaseLeakTag(_table[i].leak_tag); + _table[i].leak_tag = 0; + } } } _table_size = newsz; + TEST_LOG_SUMMARY("LivenessTracker::cleanup_table survivors=%u klass_count_scratch_size=%d", + newsz, _klass_count_scratch_size); + if (_gc_generations.load(std::memory_order_relaxed) && is_epoch_owner) { + // Runs even when _klass_count_scratch is empty: foldKlassCountsLocked() + // records zero population samples for klasses whose every tracked + // instance died this epoch (they never appear in the scratch, and + // without a zero sample a dead population would stay a leak candidate + // until its entry is evicted). + foldKlassCountsLocked(env, target_gc_epoch, allow_resolve); + } + end = OS::nanotime(); Log::debug("Liveness tracker cleanup took %.2fms (%.2fus/element)", 1.0f * (end - start) / 1000 / 1000, 1.0f * (end - start) / 1000 / sz); } _table_lock.unlock(); + } +} + +u32 LivenessTracker::resolveKlassId(JNIEnv *env, jobject ref) { + // Deliberately NOT flush_table()'s own Class.getName()-based resolution + // below: the ids this returns are the CANDIDATE klass ids that + // ReferenceChainTracker matches discovered instances' classes against, and + // RCT resolves those with the GetClassSignature + + // ObjectSampler::normalizeClassSignature() + lookupClass() sequence + // (resolveClassMap(), referenceChains.cpp). StringDictionary keys its + // entries by the exact string, and getName()'s "com.foo.Bar" (dot + // notation) is a DIFFERENT key from the signature's "com/foo/Bar" (slash + // notation) - so a getName()-based id can never equal the signature-based + // id the same class resolves to on the RCT side, and every candidate vs + // discovered-instance comparison failed (observed live on the pod: every + // auto-mark "resolved but no candidate match", and locally: + // LivenessTracker id 63 vs ReferenceChainTracker id 2 for the same class; + // only array classes accidentally matched, since "[B" is notation- + // identical). Same sequence as ObjectSampler::recordAllocation() + // therefore - the third user of it, after recordAllocation() and + // resolveClassMap(). Also strictly cheaper than the old getName() path: + // a plain JVMTI call instead of a Class.getName() JNI upcall that could + // allocate. + jclass clz = env->GetObjectClass(ref); + u32 id = 0; + jvmtiEnv *jvmti = VM::jvmti(); + if (clz != nullptr && jvmti != nullptr) { + char *class_name = nullptr; + if (jvmti->GetClassSignature(clz, &class_name, nullptr) == + JVMTI_ERROR_NONE && + class_name != nullptr) { + const char *name_slice = nullptr; + size_t name_len = 0; + if (ObjectSampler::normalizeClassSignature(class_name, &name_slice, + &name_len)) { + int lookup_id = Profiler::instance()->lookupClass(name_slice, name_len); + if (lookup_id > 0) { + id = (u32)lookup_id; + } + } + jvmti->Deallocate((unsigned char *)class_name); + } + } + if (clz != nullptr) { + env->DeleteLocalRef(clz); + } + return id; +} + +// Inserts (sample_source, age) into scratch.oldest[], sorted by age +// descending, capped at MAX_OLDEST_SAMPLES. Called from accumulateKlassCount() +// to bias representative selection toward long-lived instances. +void LivenessTracker::insertOldestSample(KlassCountScratch &scratch, + jweak sample_source, u32 age, + jint tid) { + int pos = scratch.oldest_count; + for (int i = 0; i < scratch.oldest_count; i++) { + if (age > scratch.oldest[i].age) { + pos = i; + break; + } + } + if (pos < KlassCountScratch::MAX_OLDEST_SAMPLES) { + if (scratch.oldest_count < KlassCountScratch::MAX_OLDEST_SAMPLES) { + scratch.oldest_count++; + } + for (int i = scratch.oldest_count - 1; i > pos; i--) { + scratch.oldest[i] = scratch.oldest[i - 1]; + } + scratch.oldest[pos].ref = sample_source; + scratch.oldest[pos].age = age; + scratch.oldest[pos].tid = tid; + } +} + +jlong LivenessTracker::acquireLeakTag(u64 call_trace_id, jint tid) { + // Pool mutation is serialized by its own lock, not by _table_lock: this is + // called under the SHARED table lock (tagLeakInstances, BFS poll thread) + // while releaseLeakTag() runs under the EXCLUSIVE table lock (cleanup_table, + // GC-callback thread) - shared vs exclusive excludes those two from each + // other, but any future second shared-lock mutator would corrupt the LIFO + // free list. The dedicated lock keeps the pool correct independent of which + // table lock mode the caller holds. Lock order: _table_lock (any mode) is + // always acquired BEFORE _leak_tag_pool_lock, never the reverse. + _leak_tag_pool_lock.lock(); + if (_leak_tag_free_count <= 0) { + _leak_tag_pool_lock.unlock(); + return 0; // pool exhausted + } + int idx = _leak_tag_free_list[--_leak_tag_free_count]; + _leak_tag_info[idx].call_trace_id = call_trace_id; + _leak_tag_info[idx].tid = tid; + _leak_tag_pool_lock.unlock(); + return LEAK_TAG_BASE + idx; +} + +void LivenessTracker::releaseLeakTag(jlong tag) { + if (tag < LEAK_TAG_BASE || tag >= LEAK_TAG_BASE + LEAK_TAG_POOL_SIZE) { + return; + } + int idx = (int)(tag - LEAK_TAG_BASE); + // See acquireLeakTag()'s comment for the dedicated pool lock (and the + // _table_lock -> _leak_tag_pool_lock ordering). + _leak_tag_pool_lock.lock(); + _leak_tag_info[idx].call_trace_id = 0; + _leak_tag_info[idx].tid = 0; + _leak_tag_free_list[_leak_tag_free_count++] = idx; + _leak_tag_pool_lock.unlock(); +} + +bool LivenessTracker::getLeakTagInfo(jlong tag, u64 *out_call_trace_id, + jint *out_tid) const { + if (tag < LEAK_TAG_BASE || tag >= LEAK_TAG_BASE + LEAK_TAG_POOL_SIZE) { + return false; + } + int idx = (int)(tag - LEAK_TAG_BASE); + // releaseLeakTag() zeroes both fields, so a zero/zero slot means the tag + // was released (or never acquired) - any other state is in use. (A slot + // index comparison against _leak_tag_free_count proves nothing here: the + // free list is a LIFO stack of indices, not an index-bounded region.) + // Read under the pool lock (see acquireLeakTag()'s comment): the intended + // caller (ReferenceChainTracker's BFS poll thread) holds no table lock in + // its polling path, and without this lock the releaseLeakTag() zeroing on + // the GC-callback thread would race these reads. + _leak_tag_pool_lock.lock(); + bool in_use = _leak_tag_info[idx].call_trace_id != 0 || + _leak_tag_info[idx].tid != 0; + if (in_use) { + *out_call_trace_id = _leak_tag_info[idx].call_trace_id; + *out_tid = _leak_tag_info[idx].tid; + } + _leak_tag_pool_lock.unlock(); + return in_use; +} + +int LivenessTracker::tagLeakInstances(jvmtiEnv *jvmti, + const KlassCandidate *candidates, + int candidate_count) { + if (!_enabled || _table == nullptr) { + return 0; + } + JNIEnv *env = VM::jni(); + int tagged = 0; + // Tagging priority: instances from allocation sites (tids) with the + // clearest surviving-age diversity go first, then oldest within a tid. + // A continuously-leaking site keeps instances alive across many distinct + // GC generations (many distinct surviving ages), while a one-time burst + // or noise site survives at few distinct ages regardless of how old its + // oldest instance is. If the pool is contended or GC churn races the + // tagging, this ordering makes sure the strongest leak signal keeps the + // tags rather than whichever entry the table scan happens to reach first. + // The (klass, tid) MATCH below scopes this whole ranking to the leak site + // itself: each candidate only claims tracked instances its qualifying + // tids allocated (selectLeakCandidates()'s per-tid gate, livenessTracker.h), + // so machinery survivors of the same class from other threads never + // enter the ranking at all - observed on hotdog, klass-wide matching spent + // 247 pool tags on flat-retention machinery byte[]s with zero + // interceptions while the real leak-site instances churned out of the + // tracking table untagged. + struct TagCandidate { + u32 table_idx; + jint tid; + u32 age; + int distinct_ages; // age diversity of this entry's tid (computed below) + bool leak_tag_recorded; // record already holds a pool tag (reused below) + }; + // Stack scratch: matching entries are bounded by the tracking table's + // small live population (~hundreds); tag in scan order beyond capacity. + TagCandidate scratch[512]; + int n_candidates = 0; + // Per-poll (klass_id, tid) tag summary - the per-instance logging this + // replaces re-logged every stable pool tag on every poll (256 lines/poll + // once the pool saturates with real leak instances), which flooded the + // pod's container log hard enough to rotate its 10MB cap inside a + // verification window (round 5: ~600k lines in 20 min, costing log-head + // from the same window). One summary line per tagged (klass, tid) group + // carries the same diagnostics - allocation size distinguishing real + // leaked chunks from machinery survivors, the age range showing the + // site's surviving-age diversity - at constant volume per poll. + struct TagSummary { + u32 klass_id; + jint tid; + int tagged; + int need_set; + u64 min_age; + u64 max_age; + u64 max_size; + }; + TagSummary summary[16]; + int summary_count = 0; + int summary_overflow = 0; + _table_lock.lockShared(); + u32 sz = _table_size; + for (u32 i = 0; i < sz; i++) { + // Skip slots a concurrent track() reservation has not published yet + // (see TrackingEntry::ready's own comment) - the lock is shared, so + // the reservation is visible while its payload is still being filled. + if (__atomic_load_n(&_table[i].ready, __ATOMIC_ACQUIRE) != 1) { + continue; + } + if (_table[i].ref == nullptr) { + continue; + } + // Check if this entry's (class, allocating thread) matches any + // candidate: class alone is not enough - only the instances a + // candidate's QUALIFYING tids allocated are in tagging scope. + u32 kid = _table[i].cached_klass_id; + if (kid == 0) { + continue; + } + bool match = false; + for (int k = 0; k < candidate_count; k++) { + if (candidates[k].klass_id != kid || + candidates[k].qualifying_tid_count <= 0) { + continue; + } + for (int q = 0; q < candidates[k].qualifying_tid_count; q++) { + if (candidates[k].qualifying_tids[q] == _table[i].tid) { + match = true; + break; + } + } + if (match) { + break; + } + } + if (!match) { + continue; + } + // Entries whose record already holds a pool tag are still collected: + // their tag survives the JVMTI tag only until the object is admitted + // or the search restarts, and the state machine below must re-act on + // the CURRENT JVMTI tag (re-establish, correlate, or leave alone). + if (n_candidates < (int)(sizeof(scratch) / sizeof(scratch[0]))) { + scratch[n_candidates].table_idx = i; + scratch[n_candidates].tid = _table[i].tid; + scratch[n_candidates].age = _table[i].age; + scratch[n_candidates].distinct_ages = 0; + scratch[n_candidates].leak_tag_recorded = _table[i].leak_tag != 0; + n_candidates++; + } + } + // Compute per-tid distinct surviving ages (matching entries only - the + // same diversity signal the epoch fold uses for clustering, computed here + // directly from the tracked entries so the ranking reflects exactly the + // population being tagged). Distinct-tid count is bounded by + // MAX_THREADS_PER_KLASS logic elsewhere but here just use a small array; + // beyond 32 tids, extras share the lowest priority tier. + struct TidAges { + jint tid; + u32 ages[32]; + int age_count; + } tid_ages[32]; + int tid_count = 0; + for (int c = 0; c < n_candidates; c++) { + TidAges *t = nullptr; + for (int ti = 0; ti < tid_count; ti++) { + if (tid_ages[ti].tid == scratch[c].tid) { + t = &tid_ages[ti]; + break; + } + } + if (t == nullptr && tid_count < (int)(sizeof(tid_ages) / sizeof(tid_ages[0]))) { + t = &tid_ages[tid_count++]; + t->tid = scratch[c].tid; + t->age_count = 0; + } + if (t != nullptr) { + bool seen = false; + for (int a = 0; a < t->age_count; a++) { + if (t->ages[a] == scratch[c].age) { + seen = true; + break; + } + } + if (!seen && t->age_count < (int)(sizeof(t->ages) / sizeof(t->ages[0]))) { + t->ages[t->age_count++] = scratch[c].age; + } + } + } + for (int c = 0; c < n_candidates; c++) { + for (int ti = 0; ti < tid_count; ti++) { + if (tid_ages[ti].tid == scratch[c].tid) { + scratch[c].distinct_ages = tid_ages[ti].age_count; + break; + } + } + } + // Sort: entries whose record already holds a pool tag first (they cost + // no pool resources - their work below is correlate-or-re-establish, not + // acquire), then highest age diversity, then oldest age. Small n, + // insertion sort is fine (n <= 512 but in practice tens). + for (int c = 1; c < n_candidates; c++) { + TagCandidate key = scratch[c]; + int j = c - 1; + while (j >= 0 && + ((!scratch[j].leak_tag_recorded && key.leak_tag_recorded) || + (scratch[j].leak_tag_recorded == key.leak_tag_recorded && + (scratch[j].distinct_ages < key.distinct_ages || + (scratch[j].distinct_ages == key.distinct_ages && + scratch[j].age < key.age))))) { + scratch[j + 1] = scratch[j]; + j--; + } + scratch[j + 1] = key; + } + // Per-candidate state machine on the object's CURRENT JVMTI tag. The + // tag on the object is shared state with ReferenceChainTracker's BFS + // (frontier tags) and must never be blindly overwritten: an object + // admitted by the BFS carries its FRONTIER tag on the object, and a + // SetTag(leak_tag) here would orphan that frontier entry (the BFS could + // never resolve it again) while making correlation depend on a parent + // re-walk ever happening. Instead: + // - leak tag on object: already waiting for interception - just make + // sure the record knows it; + // - frontier tag on object: already admitted - correlate the entry + // (ReferenceChainTracker stores the leak tag ON the entry and marks + // the instance discovered), never retag; + // - no tag: plain SetTag (first tagging, or re-establishing after a + // search restart wiped all tags via releaseSearchTags()). + for (int c = 0; c < n_candidates; c++) { + u32 i = scratch[c].table_idx; + jobject ref = env->NewLocalRef(_table[i].ref); + if (ref == nullptr) { + // Object was collected between the null check and now - its record's + // tag (if any) is released by the GC cleanup path, nothing to do. + continue; + } + jlong existing = 0; + jvmtiError tag_err = jvmti->GetTag(ref, &existing); + jlong leak_tag = 0; + bool need_set = false; + if (tag_err == JVMTI_ERROR_NONE && existing >= LEAK_TAG_BASE) { + // Already carries a leak tag (ours, or one adopted below) - waiting + // for the BFS interception. Make sure the record remembers it. + leak_tag = _table[i].leak_tag != 0 ? _table[i].leak_tag : existing; + } else if (tag_err == JVMTI_ERROR_NONE && existing > 0) { + // Frontier tag: the BFS already admitted this object. Correlate the + // existing entry rather than retagging - see the block comment above. + // + // Lock order (enforced, asymmetric): this call runs under THIS tracker's + // SHARED _table_lock and takes ReferenceChainTracker-internal locks + // (FrontierTable's SpinLock - held only inside individual + // insert/lookup/setLeakTag method bodies - and _resolved_chains_lock + // inside invalidateResolvedChain()). Every direction that could invert + // this is excluded today: RCT entry points into LT + // (resolveCandidateRepresentative, selectLeakCandidates, + // tagLeakInstances from pollWatchedTargets) take the table lock before + // touching any shared tracker state and are called with no RCT-internal + // lock held, so no path acquires _table_lock while already holding an + // RCT-internal lock. Any new RCT -> LT call made while holding an + // RCT-internal lock would invert the order and deadlock against this + // site. (Moving the correlate call outside the shared-lock section was + // rejected: it needs _table[i].ref/cached_klass_id, and the table + // index/weak ref can be compacted or reaped by a concurrent + // cleanup_table() once the shared lock is released - re-touching them + // unlocked would read freed/moved slots.) + leak_tag = _table[i].leak_tag; + if (leak_tag == 0) { + leak_tag = acquireLeakTag(_table[i].call_trace_id, _table[i].tid); + if (leak_tag == 0) { + env->DeleteLocalRef(ref); + continue; // pool exhausted - other candidates may still correlate + } + } + if (!ReferenceChainTracker::instance()->correlateAdmittedLeakTag( + existing, leak_tag, _table[i].cached_klass_id)) { + // Not a live frontier tag after all (search just restarted) - + // fall back to plain tagging. + need_set = true; + } + } else if (tag_err == JVMTI_ERROR_NONE && existing < 0) { + // Negative tag: a stable class tag from the shared class-tag allocator + // (classTagAllocator.h) - the candidate is a java.lang.Class mirror. + // Replacing it with a positive leak tag would break class resolution + // and frontier matching for the represented class. Preserve the + // installed class tag untouched; the entry's pool-tag bookkeeping + // stays as it is. + leak_tag = _table[i].leak_tag; + need_set = false; + } else { + // No tag: first tagging, or re-establishment after a restart wiped + // all tags (releaseSearchTags() clears every JVMTI tag while the + // record keeps its pool tag - reusing it keeps pool accounting + // stable across restarts). + leak_tag = _table[i].leak_tag; + if (leak_tag == 0) { + leak_tag = acquireLeakTag(_table[i].call_trace_id, _table[i].tid); + if (leak_tag == 0) { + env->DeleteLocalRef(ref); + break; // pool exhausted + } + } + need_set = true; + } + if (need_set) { + jvmti->SetTag(ref, leak_tag); + } + // Store under the SHARED table lock: the only other writers are exclusive- + // lock holders (cleanup_table() zeroing a dead entry, start() reclaiming + // owned tags), which a shared holder excludes - and tagLeakInstances() + // itself runs on the single BFS poll thread, so no second shared-lock + // writer exists. Readers under either lock mode therefore always see a + // value from one of those serialized writers (aligned jlong stores are + // atomic on all supported architectures); a shared-mode reader may observe + // a stale 0 and re-tag - the correlate/re-establish state machine above is + // idempotent for that case. If a second shared-lock scanner ever starts + // writing leak_tag, this store (and those reads) must move under the + // exclusive lock or become atomic. + _table[i].leak_tag = leak_tag; + tagged++; + // Accumulate into the per-poll summary above instead of logging per + // instance - one stable-pool poll re-logged all 256 tags' identical + // lines every 1.4s before this. + u32 tagged_kid = _table[i].cached_klass_id; + jint tagged_tid = _table[i].tid; + u64 tagged_age = _table[i].age; + u64 tagged_size = _table[i].alloc._size; + int g = 0; + while (g < summary_count && + (summary[g].klass_id != tagged_kid || summary[g].tid != tagged_tid)) { + g++; + } + if (g == summary_count && summary_count >= + (int)(sizeof(summary) / sizeof(summary[0]))) { + // Bounded groups exceeded (candidates are <= 5, qualifying tids <= 8 + // each - 16 covers every realistic (klass, tid) pair; overflow lumps). + summary_overflow++; + } else if (g == summary_count) { + summary[g].klass_id = tagged_kid; + summary[g].tid = tagged_tid; + summary[g].tagged = 1; + summary[g].need_set = (int)need_set; + summary[g].min_age = tagged_age; + summary[g].max_age = tagged_age; + summary[g].max_size = tagged_size; + summary_count++; + } else { + summary[g].tagged++; + summary[g].need_set += (int)need_set; + if (tagged_age < summary[g].min_age) { + summary[g].min_age = tagged_age; + } + if (tagged_age > summary[g].max_age) { + summary[g].max_age = tagged_age; + } + if (tagged_size > summary[g].max_size) { + summary[g].max_size = tagged_size; + } + } + env->DeleteLocalRef(ref); + } + _table_lock.unlockShared(); + for (int g = 0; g < summary_count; g++) { + TEST_LOG("LivenessTracker::tagLeakInstances summary klass_id=%u " + "tid=%d tagged=%d need_set=%d min_age=%llu max_age=%llu " + "max_size=%llu", + summary[g].klass_id, (int)summary[g].tid, summary[g].tagged, + summary[g].need_set, (unsigned long long)summary[g].min_age, + (unsigned long long)summary[g].max_age, + (unsigned long long)summary[g].max_size); + } + if (summary_overflow > 0) { + TEST_LOG("LivenessTracker::tagLeakInstances summary overflow=%d " + "(groups beyond %d)", + summary_overflow, (int)(sizeof(summary) / sizeof(summary[0]))); + } + return tagged; +} + +void LivenessTracker::insertThreadGen(KlassCountScratch &scratch, + jint tid, u32 age) { + // Find or create the thread entry for this tid + for (int i = 0; i < scratch.thread_count; i++) { + if (scratch.threads[i].tid == tid) { + // Insert age into the thread's sorted distinct-age array + auto &t = scratch.threads[i]; + // Every surviving object counts toward the per-tid retained-count + // bar (TID_RETAINED_COUNT_BAR) BEFORE the age-dedup early return + // below: the count is per-instance, the ages are per-cohort. + t.count++; + for (u32 j = 0; j < t.age_count; j++) { + if (t.ages[j] == age) { + return; // age already counted for this thread + } + } + if (t.age_count < KlassCountScratch::MAX_AGES_PER_THREAD) { + // Insert sorted (small array, linear scan + shift) + u32 pos = t.age_count; + for (u32 j = 0; j < t.age_count; j++) { + if (age < t.ages[j]) { + pos = j; + break; + } + } + for (u32 j = t.age_count; j > pos; j--) { + t.ages[j] = t.ages[j - 1]; + } + t.ages[pos] = age; + t.age_count++; + } + return; + } + } + if (scratch.thread_count < KlassCountScratch::MAX_THREADS_PER_KLASS) { + int idx = scratch.thread_count++; + scratch.threads[idx].tid = tid; + scratch.threads[idx].ages[0] = age; + scratch.threads[idx].age_count = 1; + scratch.threads[idx].count = 1; // this object is the tid's first this epoch + } + // else: thread table full — additional threads are not tracked, but + // the oldest[] array still captures instances from all threads. +} + +void LivenessTracker::accumulateKlassCount(u32 klass_id, jlong age, + jweak sample_source, + jint tid) { + // Count distinct GC ages (generations) per klass: group surviving + // tracked objects by klass, then for each klass count the + // number of unique age values. This is the "generation + // count" — if new instances keep arriving while old ones + // survive, the number of distinct ages grows. + for (int i = 0; i < _klass_count_scratch_size; i++) { + if (_klass_count_scratch[i].klass_id == klass_id) { + auto &entry = _klass_count_scratch[i]; + // Per-class age dedup: only count each age once for the klass' + // generation count. But per-site tracking and oldest[] must see + // EVERY surviving object, not just the first per age — so those + // run unconditionally below, outside this dedup check. + bool age_seen = false; + for (u32 a : entry.ages) { + if (a == (u32)age) { + age_seen = true; + break; + } + } + if (!age_seen) { + entry.ages.push_back((u32)age); + } + // Track top-N oldest instances (Lindy bias): insert this sample + // into the oldest[] array, sorted by age descending, capped at + // MAX_OLDEST_SAMPLES. Runs for every object, not just new ages. + insertOldestSample(entry, sample_source, (u32)age, tid); + // Track per-thread distinct surviving generations (Cork/Swat + // heuristic): add this object's age to its thread's age set. + // Runs for every object — the thread's generation cardinality is + // the leak signal, and it must see all surviving objects to be + // accurate. + insertThreadGen(entry, tid, (u32)age); + return; + } + } + if (_klass_count_scratch_size < MAX_KLASS_POPULATION_ENTRIES) { + KlassCountScratch &slot = _klass_count_scratch[_klass_count_scratch_size++]; + slot.klass_id = klass_id; + slot.ages.clear(); + slot.ages.push_back((u32)age); + slot.oldest_count = 0; + slot.thread_count = 0; + insertOldestSample(slot, sample_source, (u32)age, tid); + insertThreadGen(slot, tid, (u32)age); + } + // else: this epoch's scratch snapshot already holds + // MAX_KLASS_POPULATION_ENTRIES distinct surviving klasses - klass_id's + // count for this epoch is dropped rather than growing the scratch array, + // the same best-effort tradeoff _klass_population's own fixed capacity + // already accepts. +} + +jweak LivenessTracker::recordKlassPopulationSampleLocked( + u32 klass_id, u32 count, u64 epoch, int *out_slot, bool *out_created, + jweak *out_evicted, int *out_evicted_count, int max_evicted) { + // Linear scan is fine: MAX_KLASS_POPULATION_ENTRIES is small enough that a + // full scan is cheap, the same shape NativeSocketSampler's fd LRU + // (NativeSocketSampler's fd cache) and this class's own cleanup_table() + // pass already accept for bounded tables. + int slot = -1; + int evict_slot = -1; + for (int i = 0; i < _klass_population_size; i++) { + if (_klass_population[i].klass_id == klass_id) { + slot = i; + break; + } + if (evict_slot < 0 || + _klass_population[i].last_updated_epoch < + _klass_population[evict_slot].last_updated_epoch) { + evict_slot = i; + } + } + + jweak evicted_ref = nullptr; + bool created = false; + if (slot < 0) { + created = true; + if (_klass_population_size < MAX_KLASS_POPULATION_ENTRIES) { + slot = _klass_population_size++; + } else { + // Table full - evict the least-recently-updated entry (evict_slot is + // guaranteed set here since MAX_KLASS_POPULATION_ENTRIES > 0 implies + // at least one iteration of the loop above ran). + slot = evict_slot; + // Return evicted representatives to caller for DeleteWeakGlobalRef. + // recordKlassPopulationSampleLocked has no JNIEnv*, so it cannot + // delete them itself. The caller (foldKlassCountsLocked) has env. + // At most MAX_REPRESENTATIVES_PER_KLASS refs to return. + if (out_evicted != nullptr) { + for (int r = 0; r < _klass_population[slot].representative_count && + *out_evicted_count < max_evicted; r++) { + out_evicted[(*out_evicted_count)++] = + _klass_population[slot].representatives[r]; + } + } + } + _klass_population[slot].klass_id = klass_id; + _klass_population[slot].representative_count = 0; + memset(_klass_population[slot].representatives, 0, sizeof(_klass_population[slot].representatives)); + memset(_klass_population[slot].rep_tids, 0, sizeof(_klass_population[slot].rep_tids)); + _klass_population[slot].ring_head = 0; + _klass_population[slot].ring_fill = 0; + _klass_population[slot].consecutive_positive = 0; + _klass_population[slot].cached_slope = 0.0; + // A reused (evicted) slot's previous class's per-tid trends must not + // leak onto the new one, same as the fields above. + _klass_population[slot].tid_trend_count = 0; + // A reused (evicted) slot's PREVIOUS class's stable tag must not leak + // onto the new one - see KlassPopulationEntry::stable_class_tag's own + // comment. Minted lazily in foldKlassCountsLocked() once a live + // instance is available to resolve the class from. + _klass_population[slot].stable_class_tag = 0; + } + + KlassPopulationEntry &entry = _klass_population[slot]; + entry.count_ring[entry.ring_head] = count; + entry.ring_head = (u8)((entry.ring_head + 1) % KLASS_POPULATION_RING_SIZE); + if (entry.ring_fill < KLASS_POPULATION_RING_SIZE) { + entry.ring_fill++; + } + entry.last_updated_epoch = epoch; + + // Updated here (not in selectLeakCandidates()) so both the production path + // (foldKlassCountsLocked(), once per genuine GC epoch) and the + // klassPopulationRecordForTest() test seam - which calls this method + // directly - keep consecutive_positive in sync with the ring they just + // pushed, rather than requiring every caller to remember to do it (see + // this class's own header comment on hasQualifyingGrowth()). + if (hasQualifyingGrowth(entry)) { + if (entry.consecutive_positive < UINT8_MAX) { + entry.consecutive_positive++; + } + } else { + entry.consecutive_positive = 0; + } + + *out_slot = slot; + *out_created = created; + return evicted_ref; +} + +void LivenessTracker::mintStableClassTagIfNeeded(JNIEnv *env, int slot, + jobject instance) { + if (slot < 0 || slot >= _klass_population_size || instance == nullptr || + _klass_population[slot].stable_class_tag != 0) { + return; + } + jvmtiEnv *jvmti = VM::jvmti(); + jclass klass = env->GetObjectClass(instance); + if (jvmti != nullptr && klass != nullptr) { + jlong tag = 0; + if (jvmti->GetTag(klass, &tag) == JVMTI_ERROR_NONE) { + if (tag == 0) { + jlong new_tag = ClassTagAllocator::next(); + if (jvmti->SetTag(klass, new_tag) == JVMTI_ERROR_NONE) { + // Adopt the tag actually installed on the class object: the + // reference-chain tracker's resolveLoadedClasses() may have + // installed its own tag between our GetTag and SetTag (two SetTag + // calls on the same untagged class - the last writer wins on the + // class object). Re-read so both trackers keep the ONE tag the + // class carries; publishing our own minted tag otherwise would + // permanently disconnect this entry's leak correlation from the + // class tag the reference-chain side keys off of. + jlong installed = 0; + if (jvmti->GetTag(klass, &installed) == JVMTI_ERROR_NONE && + installed != 0) { + tag = installed; + } else { + tag = new_tag; + } + } else { + // SetTag failed: publish nothing. stable_class_tag stays 0 and + // the next fold's minting retry (the need_mint stale-representative + // probe) calls this method again - a tag that is not on the class + // object must never be cached as the stable tag. + tag = 0; + } + } + _klass_population[slot].stable_class_tag = tag; + } + } + if (klass != nullptr) { + env->DeleteLocalRef(klass); + } +} + +void LivenessTracker::foldKlassCountsLocked(JNIEnv *env, u64 epoch, + bool allow_resolve) { + TEST_LOG_SUMMARY("LivenessTracker::foldKlassCountsLocked epoch=%llu scratch_size=%d", + (unsigned long long)epoch, _klass_count_scratch_size); + for (int i = 0; i < _klass_count_scratch_size; i++) { + KlassCountScratch &s = _klass_count_scratch[i]; + TEST_LOG("LivenessTracker::foldKlassCountsLocked scratch[%d] klass_id=%u gen_count=%zu " + "thread_count=%d oldest_count=%d", + i, s.klass_id, s.ages.size(), s.thread_count, s.oldest_count); + for (int ti = 0; ti < s.thread_count; ti++) { + TEST_LOG(" thread[%d] tid=%d age_count=%u", ti, (int)s.threads[ti].tid, + s.threads[ti].age_count); + } + int slot; + bool created; + jweak evicted[KlassPopulationEntry::MAX_REPRESENTATIVES_PER_KLASS]; + int evicted_count = 0; + recordKlassPopulationSampleLocked(s.klass_id, (u32)s.ages.size(), + epoch, &slot, &created, + evicted, &evicted_count, + KlassPopulationEntry::MAX_REPRESENTATIVES_PER_KLASS); + // Per-(klass,tid) qualification fold - see recordTidTrendSamplesLocked()'s + // own comment: pure table work, no JNIEnv, so unlike the representative + // minting below it runs regardless of allow_resolve. + recordTidTrendSamplesLocked(slot, s); + for (int r = 0; r < evicted_count; r++) { + if (evicted[r] != nullptr) { + env->DeleteWeakGlobalRef(evicted[r]); + } + } + if (!allow_resolve) { + // track()'s table-overflow branch calls cleanup_table(true, false) + // synchronously from the JVMTI SampledObjectAlloc callback stack - the + // same reason resolveKlassId() is skipped there (cleanup_table()'s own + // comment above). The representative-minting NewLocalRef/ + // NewWeakGlobalRef/DeleteLocalRef churn below is no Java-bytecode + // upcall, but it is still avoidable JNI work on that hot path; leaving + // the representative unset here is safe because the retry condition + // right below picks it up again on the next allow_resolve=true sweep. + continue; + } + // Also retry minting when an existing entry's representatives are + // stale: either the count is zero, or all stored jweaks refer to + // collected objects. A jweak's pointer value never becomes nullptr + // just because its referent was collected, so we must probe each one. + // Resolving here every epoch bounds any given gap to "one epoch with + // no representative", not permanent. + // + // Mint up to MAX_REPRESENTATIVES_PER_KLASS representatives from the + // oldest surviving instances (Lindy bias: oldest = most likely to be + // leaks). Fresh independent jweaks are minted rather than reusing + // s.oldest[].ref directly — those are TrackingEntry jweaks that get + // deleted when cleanup_table() reaps the original entry. + bool need_mint = created || + _klass_population[slot].representative_count == 0; + if (!need_mint) { + // Check if all representatives are stale + bool any_live = false; + for (int r = 0; r < _klass_population[slot].representative_count; r++) { + jweak rep = _klass_population[slot].representatives[r]; + if (rep != nullptr) { + jobject probe = env->NewLocalRef(rep); + if (probe != nullptr) { + any_live = true; + env->DeleteLocalRef(probe); + break; + } + env->DeleteLocalRef(probe); + } + } + need_mint = !any_live; + } + // Compute the dominant allocating thread (highest generation + // cardinality — most distinct surviving GC ages). This reuses + // the same generation-count signal that selectLeakCandidates() + // uses per-class, applied at per-thread granularity within a + // class. A thread with 12 distinct surviving ages (continuous + // leak) outscores a thread with 1 age (one-time burst), + // regardless of raw instance count or size (Cork/Swat + // heuristic). Thread ID is used instead of call_trace_id because + // lambdas fragment call_trace_id — synthetic methods produce + // slightly different stack hashes for what is logically one + // allocation site. + jint dominant_tid = 0; + u32 dominant_gens = 0; + for (int ti = 0; ti < s.thread_count; ti++) { + if (s.threads[ti].age_count > dominant_gens) { + dominant_gens = s.threads[ti].age_count; + dominant_tid = s.threads[ti].tid; + } + } + // Re-mint if the dominant thread has >1 generation AND none of + // the current reps were minted from it. This handles the startup- + // cache problem: reps minted from noise threads during startup + // stay live even after the real leak thread becomes dominant, + // blocking re-selection because need_mint=false. By checking + // whether reps match the dominant thread, we replace stale reps + // with instances from the actual leak thread. + if (!need_mint && dominant_gens > 1) { + bool rep_matches_dominant = false; + for (int r = 0; r < _klass_population[slot].representative_count; r++) { + if (_klass_population[slot].rep_tids[r] == dominant_tid) { + rep_matches_dominant = true; + break; + } + } + if (!rep_matches_dominant) { + need_mint = true; + TEST_LOG("LivenessTracker::foldKlassCountsLocked re-minting klass_id=%u: " + "dominant_tid=%d dominant_gens=%u but no rep matches", + s.klass_id, (int)dominant_tid, dominant_gens); + } + } + if (need_mint) { + // Clean up old representatives + for (int r = 0; r < _klass_population[slot].representative_count; r++) { + if (_klass_population[slot].representatives[r] != nullptr) { + env->DeleteWeakGlobalRef(_klass_population[slot].representatives[r]); + _klass_population[slot].representatives[r] = nullptr; + } + _klass_population[slot].rep_tids[r] = 0; + } + _klass_population[slot].representative_count = 0; + // Mint fresh representatives, preferring instances from the + // dominant allocating thread. If the dominant thread has fewer + // than MAX_REPRESENTATIVES_PER_KLASS instances in oldest[], fill + // the remaining slots with other oldest instances. + bool minted_any = false; + int minted = 0; + // First pass: instances from the dominant thread (only if it has + // >1 distinct generation — otherwise all threads are equally + // uninteresting and pure oldest-first is fine) + if (dominant_gens > 1) { + for (int r = 0; r < s.oldest_count && + minted < KlassPopulationEntry::MAX_REPRESENTATIVES_PER_KLASS; r++) { + if (s.oldest[r].tid != dominant_tid) continue; + jobject strong = env->NewLocalRef(s.oldest[r].ref); + if (strong != nullptr) { + jweak rep = env->NewWeakGlobalRef(strong); + if (rep == nullptr) { + // NewWeakGlobalRef failed under memory pressure (an + // OutOfMemoryError may be pending). Store no representative + // and stop minting: continuing JNI calls with a pending + // exception is undefined behavior, and a null rep would + // corrupt representative_count. The next epoch's need_mint + // retry re-attempts minting. + if (env->ExceptionCheck()) { + env->ExceptionClear(); + } + env->DeleteLocalRef(strong); + break; + } + int idx = _klass_population[slot].representative_count++; + _klass_population[slot].representatives[idx] = rep; + _klass_population[slot].rep_tids[idx] = dominant_tid; + if (!minted_any) { + mintStableClassTagIfNeeded(env, slot, strong); + minted_any = true; + } + env->DeleteLocalRef(strong); + minted++; + } + } + } + // Second pass: fill remaining slots with other oldest instances + for (int r = 0; r < s.oldest_count && + minted < KlassPopulationEntry::MAX_REPRESENTATIVES_PER_KLASS; r++) { + if (dominant_gens > 1 && s.oldest[r].tid == dominant_tid) continue; + jobject strong = env->NewLocalRef(s.oldest[r].ref); + if (strong != nullptr) { + jweak rep = env->NewWeakGlobalRef(strong); + if (rep == nullptr) { + // Same OOM handling as the dominant-thread loop above: no null + // representative, pending exception cleared, minting stops. + if (env->ExceptionCheck()) { + env->ExceptionClear(); + } + env->DeleteLocalRef(strong); + break; + } + int idx = _klass_population[slot].representative_count++; + _klass_population[slot].representatives[idx] = rep; + _klass_population[slot].rep_tids[idx] = s.oldest[r].tid; + if (!minted_any) { + mintStableClassTagIfNeeded(env, slot, strong); + minted_any = true; + } + env->DeleteLocalRef(strong); + minted++; + } + } + // else: all surviving instances for this klass died before we + // could mint a representative - left with representative_count=0 + // for this epoch, retried on the next one. + TEST_LOG("LivenessTracker::foldKlassCountsLocked minted=%d for klass_id=%u " + "dominant_tid=%d dominant_gens=%u", + minted, s.klass_id, (int)dominant_tid, dominant_gens); + } + } + _klass_count_scratch_size = 0; + + // Zero-sample pass: a klass whose every tracked instance died this epoch + // never appears in _klass_count_scratch, so the loop above never refreshes + // its population entry - its ring keeps the last positive count and + // consecutive_positive keeps its old trend, keeping a dead population a + // leak candidate until the entry is evicted. Record a zero sample for every + // entry this epoch's fold did not touch (pure table work, no JNI; the + // klass_id is always found, so no eviction can displace a later iteration's + // target - the klass_id snapshot below is belt-and-braces for that + // invariant). + for (int i = 0; i < _klass_population_size; i++) { + u32 klass_id = _klass_population[i].klass_id; + if (_klass_population[i].last_updated_epoch == epoch) { + continue; + } + int zero_slot; + bool zero_created; + jweak zero_evicted[KlassPopulationEntry::MAX_REPRESENTATIVES_PER_KLASS]; + int zero_evicted_count = 0; + recordKlassPopulationSampleLocked(klass_id, 0, epoch, &zero_slot, + &zero_created, zero_evicted, + &zero_evicted_count, + KlassPopulationEntry::MAX_REPRESENTATIVES_PER_KLASS); + for (int r = 0; r < zero_evicted_count; r++) { + if (env != nullptr && zero_evicted[r] != nullptr) { + env->DeleteWeakGlobalRef(zero_evicted[r]); + } + } + } +} + +bool LivenessTracker::hasQualifyingGrowth(const KlassPopulationEntry &entry) const { + RingThirdsStats stats; + if (!ringThirdsStats( + entry.ring_head, entry.ring_fill, KLASS_POPULATION_RING_SIZE, + KLASS_POPULATION_MIN_FILL_FOR_TREND, + [&entry](int i) { return (double)entry.count_ring[i]; }, &stats)) { + TEST_LOG("LivenessTracker::hasQualifyingGrowth klass_id=%u ring_fill=%u " + "INSUFFICIENT_FILL (need %d)", + entry.klass_id, entry.ring_fill, + KLASS_POPULATION_MIN_FILL_FOR_TREND); + return false; + } + + // Cached for selectLeakCandidates()'s ranking (KlassPopulationEntry:: + // cached_slope's own comment, livenessTracker.h) - the ring only changes + // on push, so this is the same value a later re-scan would compute. + entry.cached_slope = stats.recent_mean - stats.earliest_mean; + + double growth_bar = LEAK_GROWTH_REL_MIN * stats.earliest_mean; + if (growth_bar < LEAK_GROWTH_ABS_MIN) { + growth_bar = LEAK_GROWTH_ABS_MIN; + } + if (entry.cached_slope < growth_bar) { + TEST_LOG("LivenessTracker::hasQualifyingGrowth klass_id=%u " + "SLOPE_TOO_SMALL slope=%f growth_bar=%f", + entry.klass_id, entry.cached_slope, growth_bar); + return false; + } + + TEST_LOG("LivenessTracker::hasQualifyingGrowth klass_id=%u " + "SLOPE_OK slope=%f growth_bar=%f", + entry.klass_id, entry.cached_slope, growth_bar); + return true; +} + +bool LivenessTracker::hasQualifyingTidGrowth( + const KlassPopulationEntry::TidTrend &trend) const { + RingThirdsStats stats; + if (!ringThirdsStats( + trend.ring_head, trend.ring_fill, + KlassPopulationEntry::TID_TREND_RING_SIZE, + TID_TREND_MIN_FILL_FOR_TREND, + [&trend](int i) { return (double)trend.ring[i]; }, &stats)) { + return false; + } + // Same growth bar as the klass gate (LEAK_GROWTH_REL_MIN/ABS_MIN): + // per-tid age-cardinality is the same small-integer signal the klass gate + // already applies these thresholds to, so they transfer unchanged. + double growth_bar = LEAK_GROWTH_REL_MIN * stats.earliest_mean; + if (growth_bar < LEAK_GROWTH_ABS_MIN) { + growth_bar = LEAK_GROWTH_ABS_MIN; + } + return (stats.recent_mean - stats.earliest_mean) >= growth_bar; +} + +bool LivenessTracker::tidPushQualifies( + const KlassPopulationEntry::TidTrend &trend, u32 current_count) const { + // Second discriminator before the (usually cheaper) ring re-scan: a + // count over the bar qualifies without any history at all. + if (current_count >= TID_RETAINED_COUNT_BAR) { + return true; + } + return hasQualifyingTidGrowth(trend); +} + +void LivenessTracker::recordTidTrendSamplesLocked( + int slot, const KlassCountScratch &scratch) { + if (slot < 0 || slot >= _klass_population_size) { + return; + } + KlassPopulationEntry &entry = _klass_population[slot]; + // Present tids first: find-or-create their trend and push this epoch's + // distinct-age count. (KlassCountScratch threads always carry >=1 + // surviving instance, so every scratch thread has a real claim on a + // slot; a zero-count tid never reaches this fold.) + for (int ti = 0; ti < scratch.thread_count; ti++) { + jint tid = scratch.threads[ti].tid; + KlassPopulationEntry::TidTrend *trend = nullptr; + for (int i = 0; i < entry.tid_trend_count; i++) { + if (entry.tid_trends[i].tid == tid) { + trend = &entry.tid_trends[i]; + break; + } + } + if (trend != nullptr && trend->synthetic) { + // Seam-owned history: the test maintains this ramp itself + // (seedTidTrendSample0), and a real fold interleaving its own low + // count for the same tid would reset the trend's hysteresis every + // System.gc() the scenario runs between rounds. Production data has + // no synthetic trends, so this exemption is inert there. + continue; + } + if (trend == nullptr) { + if (entry.tid_trend_count < KlassPopulationEntry::MAX_TID_TRENDS) { + trend = &entry.tid_trends[entry.tid_trend_count++]; + } else { + // Full: evict the WEAKEST non-synthetic trend (lowest + // consecutive_positive, then lowest ring_fill) - a genuinely rising + // leak tid accumulates hysteresis fast and resists eviction, while + // machinery tids never build any. Synthetic (test-seeded) trends + // are never evicted by real data. If every slot is synthetic, the + // new tid is simply dropped - production data never marks + // anything synthetic, so this only caps a scenario's own seeding. + int victim = -1; + for (int i = 0; i < entry.tid_trend_count; i++) { + if (entry.tid_trends[i].synthetic) { + continue; + } + if (victim < 0 || + (entry.tid_trends[i].consecutive_positive < + entry.tid_trends[victim].consecutive_positive) || + (entry.tid_trends[i].consecutive_positive == + entry.tid_trends[victim].consecutive_positive && + entry.tid_trends[i].ring_fill < + entry.tid_trends[victim].ring_fill)) { + victim = i; + } + } + if (victim < 0) { + continue; // all slots synthetic - drop this tid's sample + } + trend = &entry.tid_trends[victim]; + } + trend->tid = tid; + trend->ring_head = 0; + trend->ring_fill = 0; + trend->consecutive_positive = 0; + trend->synthetic = false; + } + trend->ring[trend->ring_head] = (u8)scratch.threads[ti].age_count; + trend->count_ring[trend->ring_head] = + (u8)std::min(scratch.threads[ti].count, UINT8_MAX); + trend->ring_head = (u8)((trend->ring_head + 1) % + KlassPopulationEntry::TID_TREND_RING_SIZE); + if (trend->ring_fill < KlassPopulationEntry::TID_TREND_RING_SIZE) { + trend->ring_fill++; + } + if (tidPushQualifies(*trend, scratch.threads[ti].count)) { + if (trend->consecutive_positive < UINT8_MAX) { + trend->consecutive_positive++; + } + } else { + trend->consecutive_positive = 0; + } + } + // Tracked tids ABSENT this epoch: a thread whose instances all died + // must not keep a stale rising ring - push 0 so the next + // hasQualifyingTidGrowth() fails and the trend's hysteresis resets (the + // per-tid analogue of the population simply stopping). Synthetic + // (test-seeded) trends are exempt: scenarios interleave real + // System.gc()-driven folds with their seeded ramps, and without the + // exemption every real fold would wipe the seeded qualification before + // the test could ever use it (see TidTrend::synthetic's own comment). + for (int i = 0; i < entry.tid_trend_count; i++) { + KlassPopulationEntry::TidTrend &trend = entry.tid_trends[i]; + if (trend.synthetic) { + continue; + } + bool present = false; + for (int ti = 0; ti < scratch.thread_count; ti++) { + if (scratch.threads[ti].tid == trend.tid) { + present = true; + break; + } + } + if (present) { + continue; + } + trend.ring[trend.ring_head] = 0; + trend.count_ring[trend.ring_head] = 0; + trend.ring_head = (u8)((trend.ring_head + 1) % + KlassPopulationEntry::TID_TREND_RING_SIZE); + if (trend.ring_fill < KlassPopulationEntry::TID_TREND_RING_SIZE) { + trend.ring_fill++; + } + // A 0-count epoch cannot qualify as a rise - reset now rather than + // deferring to the next push, so a long-absent tid does not keep + // hysteresis from before its population died. + trend.consecutive_positive = 0; + } +} + +void LivenessTracker::recordHeapFloorSample(u64 used, u64 timestamp_ns, u64 container_used) { + TEST_LOG("LivenessTracker::recordHeapFloorSample called used=%llu disabled=%d", + (unsigned long long)used, + (int)_heap_floor_recording_disabled_for_test.load(std::memory_order_acquire)); +#ifdef DEBUG + if (_heap_floor_recording_disabled_for_test.load(std::memory_order_acquire)) { + TEST_LOG("LivenessTracker::recordHeapFloorSample SKIPPED (disabled for test)"); + return; + } +#endif + recordHeapFloorSampleUnchecked(used, timestamp_ns, container_used); +} + +void LivenessTracker::recordHeapFloorSampleUnchecked(u64 used, u64 timestamp_ns, u64 container_used) { + TEST_LOG("LivenessTracker::recordHeapFloorSample used=%llu timestamp_ns=%llu container_used=%llu", + (unsigned long long)used, (unsigned long long)timestamp_ns, + (unsigned long long)container_used); + // Lock-free, single-writer-at-a-time - see _heap_floor_ring's own comment + // (livenessTracker.h) for why onGC() cannot take _table_lock here. + // + // Standard SPSC publish order: the payload is a plain store, and the index + // that gates which slots are valid is what carries the release. A reader + // that loadAcquire()s the index is then guaranteed to see this payload + // write too, since it precedes the index's storeRelease() in program + // order and a release store cannot be reordered before an earlier store. + // (The previous version had this backwards - storeRelease() on the + // payload with a plain store on the index - which does not establish any + // ordering between "the index says this slot is valid" and "the payload + // for that slot is visible".) _heap_floor_time_ring's and + // _container_mem_ring's payload writes below are plain stores for the + // same reason - they precede the same storeRelease() in program order. + u8 head = load(_heap_floor_ring_head); + store(_heap_floor_ring[head], used); + store(_heap_floor_time_ring[head], timestamp_ns); + store(_container_mem_ring[head], container_used); + storeRelease(_heap_floor_ring_head, (u8)((head + 1) % KLASS_POPULATION_RING_SIZE)); + u8 fill = load(_heap_floor_ring_fill); + if (fill < KLASS_POPULATION_RING_SIZE) { + storeRelease(_heap_floor_ring_fill, (u8)(fill + 1)); + } +} + +bool LivenessTracker::heapFloorRising() const { + // Matches recordHeapFloorSample()'s storeRelease() on both index fields - + // loadAcquire() here is what makes the payload writes below visible. + u8 fill = loadAcquire(_heap_floor_ring_fill); + u8 head = loadAcquire(_heap_floor_ring_head); + RingThirdsStats stats; + if (!ringThirdsStats( + head, fill, KLASS_POPULATION_RING_SIZE, + KLASS_POPULATION_MIN_FILL_FOR_TREND, + [this](int i) { return (double)load(_heap_floor_ring[i]); }, + &stats)) { + return false; + } + + double growth_bar = HEAP_FLOOR_GROWTH_REL_MIN * stats.earliest_mean; + if (growth_bar < (double)HEAP_FLOOR_GROWTH_ABS_MIN) { + growth_bar = (double)HEAP_FLOOR_GROWTH_ABS_MIN; + } + bool mean_rising = (stats.recent_mean - stats.earliest_mean) >= growth_bar; + if (!mean_rising) { + TEST_LOG("LivenessTracker::heapFloorRising MEAN_NOT_RISING " + "recent_mean=%.0f earliest_mean=%.0f growth_bar=%.0f", + stats.recent_mean, stats.earliest_mean, growth_bar); + return false; + } + + double floor_bar = HEAP_FLOOR_FLOOR_REL_MIN * stats.earliest_min; + if (floor_bar < (double)HEAP_FLOOR_FLOOR_ABS_MIN) { + floor_bar = (double)HEAP_FLOOR_FLOOR_ABS_MIN; + } + bool floor_rising = (stats.recent_min - stats.earliest_min) >= floor_bar; + TEST_LOG("LivenessTracker::heapFloorRising %s " + "recent_mean=%.0f earliest_mean=%.0f recent_min=%.0f earliest_min=%.0f " + "floor_bar=%.0f floor_rising=%d", + floor_rising ? "FLOOR_RISING" : "FLOOR_NOT_RISING", + stats.recent_mean, stats.earliest_mean, + stats.recent_min, stats.earliest_min, + floor_bar, (int)floor_rising); + return floor_rising; +} + +double LivenessTracker::secondsToOOM() const { +#ifdef DEBUG + jlong max_heap = _max_heap_bytes_for_test.load(std::memory_order_acquire); + if (max_heap <= 0) { + max_heap = _max_heap_bytes; + } + jlong container_limit = _container_memory_limit_for_test.load(std::memory_order_acquire); + if (container_limit <= 0) { + container_limit = _container_memory_limit; + } +#else + jlong max_heap = _max_heap_bytes; + jlong container_limit = _container_memory_limit; +#endif + if (!_gc_generations.load(std::memory_order_relaxed) || max_heap <= 0) { + TEST_LOG("LivenessTracker::secondsToOOM -> -1 (gc_generations=%d max_heap=%lld)", + (int)_gc_generations.load(std::memory_order_relaxed), (long long)max_heap); + return -1; + } + + // Both boundaries are projected independently and the SHORTER time wins: + // picking a boundary by the raw limit comparison misses that container + // usage includes native memory, thread stacks, code cache and sibling + // cgroups, so a container whose limit is numerically LARGER than -Xmx can + // still be much closer to exhaustion than the heap itself (and vice + // versa). An unavailable container limit (bare metal, macOS, cgroups + // disabled) is treated as no boundary rather than unbounded. Both rings + // are filled by the same sampler, so they share one window; where a ring + // was never recorded (older recordings, tests that only pass used bytes) + // its projection simply does not fire. + u8 fill = loadAcquire(_heap_floor_ring_fill); + u8 head = loadAcquire(_heap_floor_ring_head); + if (fill < KLASS_POPULATION_MIN_FILL_FOR_TREND) { + TEST_LOG("LivenessTracker::secondsToOOM -> -1 (INSUFFICIENT_FILL fill=%d need=%d)", + (int)fill, KLASS_POPULATION_MIN_FILL_FOR_TREND); + return -1; + } + RingThirdsStats time_stats; + if (!ringThirdsStats( + head, fill, KLASS_POPULATION_RING_SIZE, + KLASS_POPULATION_MIN_FILL_FOR_TREND, + [this](int i) { return (double)load(_heap_floor_time_ring[i]); }, + &time_stats)) { + return -1; + } + double time_delta_ns = time_stats.recent_mean - time_stats.earliest_mean; + if (time_delta_ns <= 0) { + TEST_LOG("LivenessTracker::secondsToOOM -> -1 (NOT_RISING time_delta_ns=%.0f)", + time_delta_ns); + return -1; + } + + double best_seconds = -1.0; + const char *best_source = "none"; + double best_recent_mean = 0; + + RingThirdsStats heap_bytes; + if (max_heap > 0 && + ringThirdsStats(head, fill, KLASS_POPULATION_RING_SIZE, + KLASS_POPULATION_MIN_FILL_FOR_TREND, + [this](int i) { return (double)load(_heap_floor_ring[i]); }, + &heap_bytes) && + corroborateRecentHalf(head, fill, KLASS_POPULATION_RING_SIZE, + HEAP_FLOOR_RECENT_HALF_MIN_FILL, + [this](int i) { return (double)load(_heap_floor_ring[i]); })) { + double remaining = (double)max_heap - heap_bytes.recent_mean; + double secs = remaining <= 0 + ? 0 + : (remaining * time_delta_ns) / + (heap_bytes.recent_mean - heap_bytes.earliest_mean) / 1e9; + if (best_seconds < 0 || secs < best_seconds) { + best_seconds = secs; + best_source = "heap"; + best_recent_mean = heap_bytes.recent_mean; + } + } + + RingThirdsStats container_bytes; + if (container_limit > 0 && + ringThirdsStats(head, fill, KLASS_POPULATION_RING_SIZE, + KLASS_POPULATION_MIN_FILL_FOR_TREND, + [this](int i) { return (double)load(_container_mem_ring[i]); }, + &container_bytes) && + corroborateRecentHalf(head, fill, KLASS_POPULATION_RING_SIZE, + HEAP_FLOOR_RECENT_HALF_MIN_FILL, + [this](int i) { return (double)load(_container_mem_ring[i]); })) { + double remaining = (double)container_limit - container_bytes.recent_mean; + double secs = remaining <= 0 + ? 0 + : (remaining * time_delta_ns) / + (container_bytes.recent_mean - container_bytes.earliest_mean) / 1e9; + if (best_seconds < 0 || secs < best_seconds) { + best_seconds = secs; + best_source = "container"; + best_recent_mean = container_bytes.recent_mean; + } + } + + if (best_seconds < 0) { + TEST_LOG("LivenessTracker::secondsToOOM -> -1 (no rising boundary " + "fill=%d heap=%lld container=%lld)", + (int)fill, (long long)max_heap, (long long)container_limit); + return -1; + } + TEST_LOG("LivenessTracker::secondsToOOM source=%s limit-projection=%.3fs " + "recent_mean=%.0f fill=%d", + best_source, best_seconds, best_recent_mean, (int)fill); + return best_seconds; +} + +int LivenessTracker::selectLeakCandidates(KlassCandidate *out, int max) { + int cap = max < MAX_LEAK_CANDIDATES ? max : MAX_LEAK_CANDIDATES; + if (cap <= 0) { + return 0; + } + + // Kept sorted descending by slope magnitude, at most `cap` (<= + // MAX_LEAK_CANDIDATES == 5) entries - not one per klass - so an + // insertion-sort-style insert per candidate (O(cap) per insert, O(N*cap) + // overall for N <= MAX_KLASS_POPULATION_ENTRIES == 256 klasses) is cheaper + // and simpler than collecting every qualifying candidate and calling + // std::sort. + // A single call, shared by every candidate this scan considers - see + // LEAK_TREND_HYSTERESIS_BASE/CORROBORATED's own comment (livenessTracker.h) + // for why an aggregate, non-attributed signal can only raise or lower the + // bar uniformly, never reorder candidates against each other. Lock-free + // (heapFloorRising()'s own comment), so no relation to _table_lock below. + // Computed once: the trailing diagnostic TEST_LOG at the end of this + // function reuses this cached result instead of re-evaluating the full + // O(ring_fill) ring scan a second time per BFS-thread wake. + const bool heap_floor_rising = heapFloorRising(); + const int required_hysteresis = heap_floor_rising + ? LEAK_TREND_HYSTERESIS_CORROBORATED + : LEAK_TREND_HYSTERESIS_BASE; + + double best_slopes[MAX_LEAK_CANDIDATES]; + int count = 0; + + // Read-only pass over _klass_population - mirrors getLiveTraceIds()'s own + // shared-lock read pattern above, the same table cleanup_table() writes + // under the exclusive lock this shared lock is taken against. + _table_lock.lockShared(); + // Only log when there is actually something to scan - this runs on every + // BFS-thread wake (once per second), so logging an empty scan turns the + // steady, idle state into per-second noise. + if (_klass_population_size > 0) { + TEST_LOG("LivenessTracker::selectLeakCandidates scanning %d klass_population entries", + _klass_population_size); + } + for (int i = 0; i < _klass_population_size; i++) { + const KlassPopulationEntry &entry = _klass_population[i]; + // cached_slope was computed by hasQualifyingGrowth() the last time this + // entry was pushed (recordKlassPopulationSampleLocked()) - the ring only + // changes on push, so re-scanning it here would just recompute the same + // value a moment later. + bool has_trend = entry.ring_fill >= KLASS_POPULATION_MIN_FILL_FOR_TREND; + double slope = entry.cached_slope; + TEST_LOG("LivenessTracker::selectLeakCandidates entry[%d] klass_id=%u ring_fill=%u " + "has_trend=%d slope=%f consecutive_positive=%u required=%d rep_count=%d", + i, entry.klass_id, entry.ring_fill, has_trend, has_trend ? slope : 0.0, + entry.consecutive_positive, required_hysteresis, entry.representative_count); + if (!has_trend || slope <= 0 || entry.consecutive_positive < required_hysteresis) { + // Not enough history yet, flat/shrinking, or hasn't shown a + // qualifying rise (hasQualifyingGrowth()) for enough consecutive + // epochs yet to trust it over sampling/oscillation noise. + continue; + } + // (klass,tid) qualification: the klass-level rise above is also + // produced by churn spread across many threads each retaining a + // stable handful of instances (observed live on hotdog - see + // KlassPopulationEntry::TidTrend's own comment), so a klass only + // becomes a candidate if at least ONE allocating thread's own per-tid + // trend has held a qualifying rise for the same hysteresis. The + // qualifying tids are handed to the caller so tagLeakInstances() can + // scope its pool tags to the leak-site instances only. + jint qualifying_tids[KlassPopulationEntry::MAX_TID_TRENDS]; + int qualifying_tid_count = 0; + for (int t = 0; t < entry.tid_trend_count && + qualifying_tid_count < + (int)(sizeof(qualifying_tids) / + sizeof(qualifying_tids[0])); t++) { + if (entry.tid_trends[t].consecutive_positive >= + (u8)required_hysteresis) { + qualifying_tids[qualifying_tid_count++] = entry.tid_trends[t].tid; + } + } + if (qualifying_tid_count == 0) { + TEST_LOG("LivenessTracker::selectLeakCandidates entry[%d] klass_id=%u " + "klass trend OK but no qualifying tid - skipped", + i, entry.klass_id); + continue; + } + if (count == cap && slope <= best_slopes[cap - 1]) { + // Already holding `cap` stronger (or equal) candidates - this one + // doesn't make the cut. + continue; + } + + int pos = count < cap ? count++ : cap - 1; + best_slopes[pos] = slope; + KlassCandidate &cand = out[pos]; + cand.klass_id = entry.klass_id; + cand.representative = + entry.representative_count > 0 ? entry.representatives[0] : nullptr; + cand.qualifying_tid_count = qualifying_tid_count; + memcpy(cand.qualifying_tids, qualifying_tids, + sizeof(jint) * (size_t)qualifying_tid_count); + while (pos > 0 && best_slopes[pos - 1] < best_slopes[pos]) { + double tmp_slope = best_slopes[pos - 1]; + best_slopes[pos - 1] = best_slopes[pos]; + best_slopes[pos] = tmp_slope; + KlassCandidate tmp_cand = out[pos - 1]; + out[pos - 1] = out[pos]; + out[pos] = tmp_cand; + pos--; + } + } + _table_lock.unlockShared(); + TEST_LOG("LivenessTracker::selectLeakCandidates returning %d candidates (required_hysteresis=%d, heapFloorRising=%d)", + count, required_hysteresis, + (int)heap_floor_rising); + return count; +} + +int LivenessTracker::topKlassesByGenerationCount(u32 *out, int max) { + int cap = max < MAX_LEAK_CANDIDATES ? max : MAX_LEAK_CANDIDATES; + if (cap <= 0) { + return 0; + } + + // Same insertion-sort-style top-k selection as selectLeakCandidates() + // above (small, fixed cap - cheaper than collecting everything and + // sorting), ranked by most-recent count_ring sample instead of slope, and + // with no hysteresis/trend gate at all - see this method's own header + // comment (livenessTracker.h). + u32 best_counts[MAX_LEAK_CANDIDATES]; + int count = 0; + + _table_lock.lockShared(); + for (int i = 0; i < _klass_population_size; i++) { + const KlassPopulationEntry &entry = _klass_population[i]; + if (entry.ring_fill == 0 || entry.stable_class_tag == 0) { + // Never sampled, or a live instance has not been resolved yet to mint + // its stable_class_tag from (foldKlassCountsLocked()'s own comment) - + // nothing usable to rank or return in either case. + continue; + } + // ring_head is "next slot to write" (recordKlassPopulationSampleLocked(), + // livenessTracker.cpp) - the most recently written slot is one behind it, + // wrapping. + u32 latest = entry.count_ring[(entry.ring_head + KLASS_POPULATION_RING_SIZE - 1) % + KLASS_POPULATION_RING_SIZE]; + if (count == cap && latest <= best_counts[cap - 1]) { + continue; + } + int pos = count < cap ? count++ : cap - 1; + best_counts[pos] = latest; + out[pos] = (u32)entry.stable_class_tag; + while (pos > 0 && best_counts[pos - 1] < best_counts[pos]) { + u32 tmp_count = best_counts[pos - 1]; + best_counts[pos - 1] = best_counts[pos]; + best_counts[pos] = tmp_count; + u32 tmp_id = out[pos - 1]; + out[pos - 1] = out[pos]; + out[pos] = tmp_id; + pos--; + } + } + _table_lock.unlockShared(); + TEST_LOG("LivenessTracker::topKlassesByGenerationCount returning %d klass_ids", count); + return count; +} + +jobject LivenessTracker::resolveCandidateRepresentative(JNIEnv *env, u32 klass_id) { + // Shared lock excludes cleanup_table()'s exclusive lock (the only writer, + // and the only place that can DeleteWeakGlobalRef() an entry's + // representatives via foldKlassCountsLocked()'s eviction path above) for + // the whole lookup+resolve, so the value NewLocalRef() runs on here is + // always the table's current one for klass_id, never a snapshot that + // eviction could have invalidated in the meantime - see + // selectLeakCandidates()'s own comment for the race this closes. + // + // Returns the first live representative (oldest first, per the Lindy + // bias in KlassCountScratch::oldest[]). Callers that need all live + // representatives (e.g. pollWatchedTargets() tagging all of them) + // use resolveCandidateRepresentatives() instead. + _table_lock.lockShared(); + jobject obj = nullptr; + for (int i = 0; i < _klass_population_size; i++) { + if (_klass_population[i].klass_id == klass_id) { + for (int r = 0; r < _klass_population[i].representative_count; r++) { + jweak rep = _klass_population[i].representatives[r]; + if (rep != nullptr) { + obj = env->NewLocalRef(rep); + if (obj != nullptr) { + break; + } + } + } + break; + } + } + _table_lock.unlockShared(); + return obj; +} + +int LivenessTracker::resolveCandidateRepresentatives( + JNIEnv *env, u32 klass_id, jobject *out, int max_out) { + // Returns all live representatives for klass_id, oldest first. + // Used by pollWatchedTargets() to tag all representatives with marker + // tags so the canary mechanism has multiple chances to find a + // long-lived instance. See KlassCountScratch::oldest's comment for + // why multiple representatives matter. + _table_lock.lockShared(); + int count = 0; + for (int i = 0; i < _klass_population_size; i++) { + if (_klass_population[i].klass_id == klass_id) { + for (int r = 0; r < _klass_population[i].representative_count && + count < max_out; r++) { + jweak rep = _klass_population[i].representatives[r]; + if (rep != nullptr) { + jobject obj = env->NewLocalRef(rep); + if (obj != nullptr) { + out[count++] = obj; + } + } + } + break; + } + } + _table_lock.unlockShared(); + return count; } void LivenessTracker::flush(std::set &tracked_thread_ids) { @@ -113,16 +1916,33 @@ void LivenessTracker::flush_table(std::set *tracked_thread_ids) { event._alloc = _table[i].alloc; event._skipped = _table[i].skipped; event._ctx = _table[i].ctx; + event.leak_tag = _table[i].leak_tag; - jclass clz = env->GetObjectClass(ref); - jstring name_str = (jstring)env->CallObjectMethod(clz, _Class_getName); - env->DeleteLocalRef(clz); - jniExceptionCheck(env); - const char *name = env->GetStringUTFChars(name_str, nullptr); - int class_id = name != nullptr - ? Profiler::instance()->lookupClass(name, strlen(name)) - : 0; - env->ReleaseStringUTFChars(name_str, name); + int class_id = 0; + if (_table[i].cached_klass_id != 0) { + // Already resolved by cleanup_table()'s survivor loop this epoch + // (resolveKlassId(), only when _gc_generations is enabled) - reuse + // it instead of repeating the GetObjectClass+Class.getName()+ + // lookupClass() JNI round-trip for the same object. + class_id = _table[i].cached_klass_id; + } else { + jclass clz = env->GetObjectClass(ref); + jstring name_str = (jstring)env->CallObjectMethod(clz, _Class_getName); + env->DeleteLocalRef(clz); + jniExceptionCheck(env); + // name_str can be null if the call above threw and + // jniExceptionCheck() cleared the pending exception rather than + // propagating it - GetStringUTFChars()/ReleaseStringUTFChars() + // require a non-null jstring (mirrors resolveKlassId()'s own guard). + if (name_str != nullptr) { + const char *name = env->GetStringUTFChars(name_str, nullptr); + if (name != nullptr) { + class_id = Profiler::instance()->lookupClass(name, strlen(name)); + env->ReleaseStringUTFChars(name_str, name); + } + env->DeleteLocalRef(name_str); + } + } // lookupClass() returns -1 when the class map is at capacity; do not // assign it to the u32 event id (it would wrap to 0xFFFFFFFF and @@ -140,13 +1960,8 @@ void LivenessTracker::flush_table(std::set *tracked_thread_ids) { _table_lock.unlock(); if (_record_heap_usage) { - bool isLastGc = HeapUsage::isLastGCUsageSupported(); - size_t used = isLastGc ? HeapUsage::get()._used_at_last_gc - : loadAcquire(_used_after_last_gc); - if (used == 0) { - used = HeapUsage::get()._used; - isLastGc = false; - } + bool isLastGc; + size_t used = resolvePostGcHeapUsage(&isLastGc); Profiler::instance()->writeHeapUsage(used, isLastGc); } @@ -165,6 +1980,15 @@ Error LivenessTracker::initialize_table(JNIEnv *jni, int sampling_interval) { return Error("Can not track liveness for allocation samples without heap " "size information."); } + // Cached for secondsToOOM() - see _max_heap_bytes' own comment + // (livenessTracker.h) for why this is resolved once here rather than + // re-querying HeapUsage::getMaxHeap() on every projection. + _max_heap_bytes = max_heap; + // Cached the same way and for the same reason - see _container_memory_limit's + // own comment (livenessTracker.h). -1 (unavailable) is a valid outcome + // here, unlike max_heap above: not every JVM runs under a memory-limited + // cgroup. + _container_memory_limit = OS::getContainerMemoryLimit(); int required_table_capacity = sampling_interval > 0 ? max_heap / sampling_interval : max_heap; @@ -189,6 +2013,37 @@ Error LivenessTracker::start(Arguments &args) { if (err) { return err; } + // Initialize leak tag free list. The tracking table survives stop()/ + // start() (see stop()'s own comment), and a preserved entry may still own + // a leak tag - so reclaim those first and build the free list from the + // remainder. Blindly marking every tag free would let a new object + // receive a tag another live object still owns (corrupting leak-tag + // correlation for both), and the eventual second releaseLeakTag() of the + // duplicate would push the same index twice and write past + // _leak_tag_free_list. Owned tags also keep their _leak_tag_info entries + // (erasing them would break getLeakTagInfo() correlation for the + // preserved, still-live owners). + bool tag_owned[LEAK_TAG_POOL_SIZE]; + memset(tag_owned, 0, sizeof(tag_owned)); + _table_lock.lock(); + for (u32 i = 0; i < _table_size; i++) { + if (_table[i].leak_tag >= LEAK_TAG_BASE && + _table[i].leak_tag < LEAK_TAG_BASE + LEAK_TAG_POOL_SIZE) { + tag_owned[_table[i].leak_tag - LEAK_TAG_BASE] = true; + } + } + _table_lock.unlock(); + int free_w = 0; + for (int i = 0; i < LEAK_TAG_POOL_SIZE; i++) { + if (tag_owned[i]) { + continue; + } + _leak_tag_free_list[free_w] = i; + _leak_tag_info[i].call_trace_id = 0; + _leak_tag_info[i].tid = 0; + free_w++; + } + _leak_tag_free_count = free_w; if (!_enabled) { // disabled return Error::OK; @@ -222,6 +2077,14 @@ void LivenessTracker::stop() { Error LivenessTracker::initialize(Arguments &args) { _enabled = args._gc_generations || args._record_liveness; + // Gates per-klass population tracking (see the _gc_generations member's + // own comment in livenessTracker.h). Updated unconditionally alongside + // _record_heap_usage below, ahead of the _initialized guard, for the same + // reason: each profiler start should observe the flag it was actually + // started with, even though the tracking table itself persists across + // recordings. + _gc_generations.store(args._gc_generations, std::memory_order_relaxed); + if (!_enabled) { return Error::OK; } @@ -231,6 +2094,15 @@ Error LivenessTracker::initialize(Arguments &args) { // start gets the correct setting even when the table persists across recordings. _record_heap_usage = args._record_heap_usage; + // Fresh recording: no chase is open, so no watched tids and no urgency + // boost may leak in from a previous recording's lifecycle. This MUST run on + // EVERY start - not only the first initialization: the `_initialized` + // early return below would otherwise leave a previous recording's watched + // thread set and urgent-tracking state active, admitting the new + // recording's unrelated allocations at 100 percent. + __atomic_store_n(&_watched_tid_count, 0, __ATOMIC_RELEASE); + __atomic_store_n(&_urgent_tracking, false, __ATOMIC_RELEASE); + if (_initialized) { // if the tracker was previously initialized return the stored result for // consistency this hack also means that if the profiler is started with @@ -241,6 +2113,26 @@ Error LivenessTracker::initialize(Arguments &args) { } _initialized = true; + // Sync the class-map-generation baseline to what it already is by this + // point, rather than leaving it at the constructor's 0 sentinel (see + // _last_class_map_generation's own comment, livenessTracker.h). By the time + // this runs, ObjectSampler::start() -> LivenessTracker::start() has already + // happened strictly after Profiler::start()'s own _class_map.clearAll() + // (referenceChains.cpp's own comment on this same ordering) - so + // classMap()->generation() here already reflects this process's first + // recording, not the pre-clearAll() baseline the 0 sentinel implies. + // Without this, cleanup_table()'s class-map-reset branch always sees a + // spurious mismatch (0 vs. whatever generation() has already reached) the + // very first time it runs after ANY start() - regardless of how much + // genuinely post-reset, still-valid population/leak-tracking history has + // already accumulated in _klass_population by then - and wipes it all, + // found the hard way via StaticFieldGrowingCollectionScenario silently + // losing its seeded candidate the moment the first post-start GC finished. + // A real subsequent generation bump (a later recording's own clearAll()) + // still trips the mismatch correctly, since by then this field holds + // whatever value cleanup_table() last actually observed, not this sentinel. + _last_class_map_generation = Profiler::instance()->classMap()->generation(); + if (VM::hotspot_version() < 11) { Log::warn("Liveness tracking requires Java 11+"); // disable liveness tracking @@ -283,6 +2175,11 @@ Error LivenessTracker::initialize(Arguments &args) { _table = (TrackingEntry *)malloc(sizeof(TrackingEntry) * _table_cap); if (_table != NULL) { NativeMem::record(NM_LIVENESS, (long long)sizeof(TrackingEntry) * _table_cap); + // Uninitialized malloc storage must never look published to a + // shared-mode scanner (see TrackingEntry::ready's own comment). + for (int i = 0; i < _table_cap; i++) { + _table[i].ready = 0; + } } _gc_epoch = 0; @@ -306,6 +2203,120 @@ Error LivenessTracker::initialize(Arguments &args) { static ThreadLocal rng; static ThreadLocal skipped; +// track()'s admission gate: chase-phase admission boost (see +// admitForTracking()'s declaration comment in livenessTracker.h). Both boost +// paths precede the configured-ratio draw, so a boosted allocation never +// consumes an RNG draw and the non-boosted path keeps the exact +// reject-on-draw > ratio semantics (and per-thread RNG stream position drift +// only when boosts actually happen - harmless, the streams are independent +// per thread and probabilistic by design). +bool LivenessTracker::admitForTracking(jint tid) { + if (__atomic_load_n(&_urgent_tracking, __ATOMIC_ACQUIRE)) { + return true; + } + // Count+array two-phase publish (noteSelectedCandidates() writes the + // slots before release-storing the count): the acquire load pairs with + // that release store so every slot read below is at least as fresh as the + // count observed here - see the project's atomic memory ordering rule + // for why RELAXED is not an option on arm64. + int n = __atomic_load_n(&_watched_tid_count, __ATOMIC_ACQUIRE); + for (int i = 0; i < n; i++) { + if (__atomic_load_n(&_watched_tids[i], __ATOMIC_RELAXED) == tid) { + return true; + } + } + if (_subsample.ratio >= 1.0) { + return true; + } + u64 state = rng.get(); + if (state == 0) { + // Seeded on a thread's first tracked allocation and kept until its TLS is + // released at thread end or JNI detach: the tick keeps two threads that + // share a tid across that boundary on different streams, and the tid + // separates threads alive at the same time. A thread that outlives a + // recording keeps its stream rather than restarting it -- there is no + // per-recording reseed, because stop/start does not clear the slot. + state = xorshift::seed(TSC::ticks(), (u64)tid); + } + u64 draw = xorshift::next(state); + rng.set(state); + return draw < _subsample.threshold; +} + +// Publishes the current poll's qualifying tids as track()'s watched-admission +// set. Called from ReferenceChainTracker's pollWatchedTargets() with the FULL +// candidate selection (never the hasLeakSignal() max=1 probe - that one's +// partial view could drop tids of other still-active candidates), including +// the zero-candidate case, which clears the set: a tid left watched after the +// chase ends would keep admitting that thread at 100% forever (tids get +// recycled by the OS into unrelated threads), so the set must track the +// candidate selection exactly, poll by poll. +void LivenessTracker::noteSelectedCandidates(const KlassCandidate *candidates, + int count) { + jint tids[KlassCandidate::MAX_QUALIFYING_TIDS]; + int n = 0; + if (candidates != nullptr && count > 0) { + for (int i = 0; i < count; i++) { + const KlassCandidate &kc = candidates[i]; + for (int j = 0; j < kc.qualifying_tid_count; j++) { + jint tid = kc.qualifying_tids[j]; + bool dup = false; + for (int k = 0; k < n && !dup; k++) { + dup = (tids[k] == tid); + } + if (!dup) { + tids[n++] = tid; + if (n == KlassCandidate::MAX_QUALIFYING_TIDS) { + goto full; + } + } + } + } + } +full: + // Copy into the live array before publishing the count (two-phase + // publish mirrored by admitForTracking()'s acquire load). Slot accesses + // are ATOMIC (relaxed): the poll thread rewrites slots while allocation + // threads can still read them under an acquire count load that observed + // the OLD count - plain loads/stores there are a C++ data race (UB). + // A reader mid-scan may transiently mix old and new slot values below the + // OLD count - harmless: admission is advisory, and the worst case is one + // allocation admitted per the previous poll's set. + for (int i = 0; i < n; i++) { + __atomic_store_n(&_watched_tids[i], tids[i], __ATOMIC_RELAXED); + } + __atomic_store_n(&_watched_tid_count, n, __ATOMIC_RELEASE); + if (n > 0) { + // TEST_LOG, not Log::debug: one line per poll on the reference-chain + // thread (never the allocation hot path), and pod-side verification of + // the boost engaging needs the tid list in the extracted lib's logs. + TEST_LOG("LivenessTracker::noteSelectedCandidates watched tids[%d]:", + n); + for (int i = 0; i < n; i++) { + TEST_LOG(" watched tid=%d", + __atomic_load_n(&_watched_tids[i], __ATOMIC_RELAXED)); + } + } +} + +void LivenessTracker::setUrgentTracking(bool urgent) { + // Transition-only TEST_LOG (same pattern as ReferenceChainTracker::isUrgent()'s + // latch/release logging): the setter runs every threadLoop() iteration, so + // per-call logging would repeat the steady-state value line indefinitely. + bool prev = __atomic_exchange_n(&_urgent_tracking, urgent, __ATOMIC_ACQ_REL); + if (prev != urgent) { + TEST_LOG_SUMMARY("LivenessTracker::setUrgentTracking urgent=%d", (int)urgent); + } +} + +void LivenessTracker::admissionResetForTest() { + __atomic_store_n(&_urgent_tracking, false, __ATOMIC_RELEASE); + __atomic_store_n(&_watched_tid_count, 0, __ATOMIC_RELEASE); + _subsample = SubsampleRate(0.0); + rng.clear(); + skipped.set(0); +} + void LivenessTracker::releaseThreadLocalState() { rng.clear(); skipped.clear(); @@ -327,23 +2338,9 @@ void LivenessTracker::track(JNIEnv *env, AllocEvent &event, jint tid, // called from ObjectSampler::recordAllocation(). SAMPLER_PERF_PROBE(SP_LIVENESS); - if (_subsample.ratio < 1.0) { - u64 state = rng.get(); - if (state == 0) { - // Seeded on a thread's first tracked allocation and kept until its TLS is - // released at thread end or JNI detach: the tick keeps two threads that - // share a tid across that boundary on different streams, and the tid - // separates threads alive at the same time. A thread that outlives a - // recording keeps its stream rather than restarting it -- there is no - // per-recording reseed, because stop/start does not clear the slot. - state = xorshift::seed(TSC::ticks(), (u64)tid); - } - u64 draw = xorshift::next(state); - rng.set(state); - if (draw >= _subsample.threshold) { - skipped.set(skipped.get() + static_cast(event._weight) * event._size); - return; - } + if (!admitForTracking(tid)) { + skipped.set(skipped.get() + static_cast(event._weight) * event._size); + return; } jweak ref = env->NewWeakGlobalRef(object); @@ -368,6 +2365,10 @@ void LivenessTracker::track(JNIEnv *env, AllocEvent &event, jint tid, !__sync_bool_compare_and_swap(&_table_size, idx, idx + 1)); if (idx < _table_cap) { + // Unpublish first: a previous entry at this index may still be visible + // to shared-mode scanners (their acquire load below then skips it + // instead of racing the re-fill). + __atomic_store_n(&_table[idx].ready, 0, __ATOMIC_RELEASE); _table[idx].tid = tid; _table[idx].time = TSC::ticks(); _table[idx].ref = ref; @@ -376,7 +2377,12 @@ void LivenessTracker::track(JNIEnv *env, AllocEvent &event, jint tid, skipped.set(0); _table[idx].age = 0; _table[idx].call_trace_id = call_trace_id; + _table[idx].leak_tag = 0; _table[idx].ctx = ContextApi::snapshot(); + _table[idx].cached_klass_id = 0; + // Publish: the payload is complete - release pairs with the scanners' + // acquire loads. + __atomic_store_n(&_table[idx].ready, 1, __ATOMIC_RELEASE); } _table_lock.unlockShared(); @@ -387,8 +2393,10 @@ void LivenessTracker::track(JNIEnv *env, AllocEvent &event, jint tid, retried = true; // try cleanup before resizing - there is a good chance it will free some - // space - cleanup_table(true); + // space. allow_resolve=false: this runs synchronously on the + // allocation-sampling callback stack (see cleanup_table()'s own header + // comment for why resolveKlassId() is unsafe here). + cleanup_table(true, false); if (_table_cap < _table_max_cap) { @@ -406,6 +2414,13 @@ void LivenessTracker::track(JNIEnv *env, AllocEvent &event, jint tid, if (tmp != nullptr) { NativeMem::record(NM_LIVENESS, (long long)sizeof(TrackingEntry) * (newcap - _table_cap)); + // Unpublish the uninitialized growth region (see + // TrackingEntry::ready's own comment); the realloc happens + // under the exclusive table lock, so no scanner can observe + // the interim. + for (int i = _table_cap; i < newcap; i++) { + tmp[i].ready = 0; + } _table = tmp; _table_cap = newcap; Log::debug( @@ -432,11 +2447,51 @@ void LivenessTracker::track(JNIEnv *env, AllocEvent &event, jint tid, } } +void LivenessTracker::maybeForceCleanup(u64 now_ns) { + if (!_enabled || !_gc_generations.load(std::memory_order_relaxed)) { + return; + } + constexpr u64 FORCE_CLEANUP_INTERVAL_NS = 30ULL * 1000 * 1000 * 1000; + u64 last_cleanup_ns = load(_last_cleanup_ns); + if (now_ns - last_cleanup_ns < FORCE_CLEANUP_INTERVAL_NS) { + return; + } + if (load(_gc_epoch) == load(_last_gc_epoch)) { + // Nothing happened since the last sweep (organic, forced, or a prior + // call to this method) - re-walking an unchanged table would just + // re-fold the same survivor counts into this epoch's scratch, skewing + // the slope computed from it. Leave _last_cleanup_ns alone so the next + // wake keeps checking at the same ~1s cadence rather than restarting a + // fresh 30s wait with nothing to show for it. + return; + } + store(_last_cleanup_ns, now_ns); + cleanup_table(true, true); +} + void JNICALL LivenessTracker::GarbageCollectionFinish(jvmtiEnv *jvmti_env) { ProfiledThread::initCurrentThreadSignalSafe(); LivenessTracker::instance()->onGC(); } +size_t LivenessTracker::resolvePostGcHeapUsage(bool *out_is_last_gc) { + bool isLastGc = HeapUsage::isLastGCUsageSupported(); + size_t used = isLastGc ? HeapUsage::get()._used_at_last_gc + : loadAcquire(_used_after_last_gc); + TEST_LOG("LivenessTracker::resolvePostGcHeapUsage isLastGc=%d used_at_last_gc=%zu", + (int)isLastGc, used); + if (used == 0) { + used = HeapUsage::get(false)._used; + isLastGc = false; + TEST_LOG("LivenessTracker::resolvePostGcHeapUsage used==0, falling back to HeapUsage::get(false)._used=%zu", + used); + } + if (out_is_last_gc != nullptr) { + *out_is_last_gc = isLastGc; + } + return used; +} + void LivenessTracker::onGC() { if (!_initialized) { return; @@ -448,6 +2503,32 @@ void LivenessTracker::onGC() { if (!HeapUsage::isLastGCUsageSupported()) { store(_used_after_last_gc, HeapUsage::get(false)._used); } + + if (_gc_generations.load(std::memory_order_relaxed)) { + // Feeds heapFloorRising()'s corroboration check (selectLeakCandidates()) + // - gated on _gc_generations, same as the per-klass population table + // itself, since this ring exists purely to support that feature. + // recordHeapFloorSample() itself checks _heap_floor_recording_disabled_for_test + // (debug-only) so a test can seed the ring exclusively. + size_t used = resolvePostGcHeapUsage(nullptr); + TEST_LOG_SUMMARY("LivenessTracker::onGC recording heap floor used=%zu gc_epoch=%llu", + used, (unsigned long long)load(_gc_epoch)); + if (used > 0) { + // A failed read (-1, e.g. transient /sys/fs/cgroup access error) is + // recorded as 0 rather than skipping the sample outright - see + // _container_mem_ring's own comment (livenessTracker.h) for why this + // ring must stay index-aligned with _heap_floor_ring/ + // _heap_floor_time_ring. Harmless when _container_memory_limit is + // itself unavailable (secondsToOOM() never selects this ring then), + // and a rare, self-correcting blip otherwise (the next successful + // read re-establishes the real growth rate). + long container_usage = OS::getContainerMemoryUsage(); + recordHeapFloorSample((u64)used, OS::nanotime(), + container_usage >= 0 ? (u64)container_usage : 0); + } else { + TEST_LOG_SUMMARY("LivenessTracker::onGC used<=0, skipping heap floor record"); + } + } } void LivenessTracker::getLiveTraceIds(CallTraceIdSet& out_buffer) { @@ -467,6 +2548,10 @@ void LivenessTracker::getLiveTraceIds(CallTraceIdSet& out_buffer) { // Collect call_trace_id values from all live tracking entries for (int i = 0; i < _table_size; i++) { TrackingEntry* entry = &_table[i]; + // Skip unpublished slots (shared lock - see TrackingEntry::ready). + if (__atomic_load_n(&entry->ready, __ATOMIC_ACQUIRE) != 1) { + continue; + } if (entry->ref != nullptr) { out_buffer.insert(entry->call_trace_id); } diff --git a/ddprof-lib/src/main/cpp/livenessTracker.h b/ddprof-lib/src/main/cpp/livenessTracker.h index a10012683..a942d6647 100644 --- a/ddprof-lib/src/main/cpp/livenessTracker.h +++ b/ddprof-lib/src/main/cpp/livenessTracker.h @@ -8,11 +8,13 @@ #include "arch.h" #include "callTraceHashTable.h" +#include "classTagAllocator.h" #include "context.h" #include "engine.h" #include "event.h" #include "spinLock.h" #include "xorshift.h" +#include #include #include #include @@ -28,18 +30,29 @@ typedef struct TrackingEntry { jint tid; jlong time; jlong age; + jlong leak_tag; // 0 = untagged; otherwise a tag from the leak tag pool Context ctx; + // Set by cleanup_table()'s survivor loop via resolveKlassId() when + // _gc_generations is enabled (0 otherwise, or if resolution failed - 0 is + // StringDictionary's own "no entry" sentinel, so a real id is never 0). + // flush_table() reuses this instead of re-resolving the same object's + // class via a second GetObjectClass+Class.getName()+lookupClass() JNI + // round-trip - an object's class never changes, so a value resolved here + // stays valid for flush_table()'s later read of the same entry. track() + // resets this to 0 for every newly tracked entry. + u32 cached_klass_id; + // Publication flag for the slot payload. track() holds only the shared + // table lock while it reserves a slot via a _table_size CAS and then fills + // it in, so a shared-mode scanner (tagLeakInstances(), getLiveTraceIds()) + // can observe the reserved index before the payload is written - the + // table's malloc/realloc storage is uninitialized. track() stores 0 here + // (release) before re-filling and 1 (release) after the payload is + // complete; scanners load-acquire it and skip anything != 1. Freshly + // malloc'd/realloc'd regions are zeroed at allocation time so a garbage + // non-zero flag can never publish an uninitialized payload. + volatile int ready; } TrackingEntry; -// The liveness subsampling rate, held as the ratio and the xorshift draw -// threshold derived from it. -// -// The two have to move together: the threshold decides which allocations are -// kept, while the ratio is what Recording reports into the JFR chunk, so a -// threshold left behind by a ratio change samples at one rate and claims -// another -- silently, and in a direction no assertion would notice. The only -// constructor derives the threshold from the ratio, so a caller cannot set one -// without the other; assigning a new SubsampleRate replaces both. struct SubsampleRate { double ratio; u64 threshold; @@ -48,6 +61,143 @@ struct SubsampleRate { : ratio(subsample_ratio), threshold(xorshift::threshold(subsample_ratio)) {} }; +// Fixed-capacity, LRU-evicted per-klass population history, keyed by klass +// StringDictionary id (Profiler::classMap(), the same id TrackingEntry/ +// AllocEvent resolves lazily today only at flush time, flush_table() below). +// This is the data the design doc's Open Question 3 "positive +// population-slope ranking" proposal needs: a rolling +// window of how many tracked instances of a klass are alive at each GC +// epoch, plus one representative instance to chase a chain for if the trend +// looks leak-shaped (see selectLeakCandidates() below for the ranking, and +// referenceChains.cpp's pollWatchedTargets() for how a ranked candidate gets +// consumed - this struct only stores the raw history). +typedef struct KlassPopulationEntry { + u32 klass_id; // StringDictionary id; 0 means "unused slot" (0 is + // also StringDictionary's own "no entry" sentinel, + // so a real id is never 0 - see resolveKlassId()). + // Up to MAX_REPRESENTATIVES_PER_KLASS representative instances of this + // klass, biased toward the oldest surviving instances (highest GC age). + // The first live one is used by resolveCandidateRepresentative(); all live + // ones are tagged by pollWatchedTargets(). See KlassCountScratch::oldest's + // comment for why multiple representatives matter. + static constexpr int MAX_REPRESENTATIVES_PER_KLASS = 3; + jweak representatives[MAX_REPRESENTATIVES_PER_KLASS]; + jint rep_tids[MAX_REPRESENTATIVES_PER_KLASS]; // tid each rep was minted from + int representative_count; + u32 count_ring[30]; // ring buffer of per-epoch generation counts: the + // number of distinct GC ages among a klass's + // surviving tracked instances that epoch (see + // accumulateKlassCount(), livenessTracker.cpp). + u8 ring_head; // next slot to write + u8 ring_fill; // samples written so far, caps at 30 + // Number of consecutive epochs (most recent first) for which + // LivenessTracker::hasQualifyingGrowth() found this entry's ring to show a + // leak-shaped rise, updated every time a new sample is pushed + // (recordKlassPopulationSampleLocked()) - reset to 0 the moment a single + // epoch fails the test. selectLeakCandidates() requires this to reach a + // hysteresis threshold before trusting the klass, rather than acting on + // one qualifying epoch alone: a population merely oscillating (no net + // growth) satisfies a single-epoch test on roughly half of all epochs, so + // without this counter it gets reported as a leak candidate almost as + // often as a real leak does. + u8 consecutive_positive; + // Slope (regression value at the ring's newest sample minus the value at + // its oldest sample - see ringThirdsStats, livenessTracker.cpp) as of the + // last push, computed and cached by hasQualifyingGrowth() alongside + // consecutive_positive above - selectLeakCandidates() reads this directly + // for ranking instead of re-scanning the ring: the ring only changes on + // push, so a second scan at scan time would just recompute the same + // value. Meaningless (left at its previous value, or 0 for a newly + // created entry) whenever ring_fill < KLASS_POPULATION_MIN_FILL_FOR_TREND + // - callers must check ring_fill first, exactly as before this field + // existed. + mutable double cached_slope; + u64 last_updated_epoch; // _gc_epoch value as of the last write, for LRU + // eviction when the table is full + // Stable per-class identifier, from the process-wide, negative-tag + // allocator shared with ReferenceChainTracker (classTagAllocator.h) - NOT + // the same value as klass_id above. klass_id (Profiler::classMap()'s + // dictionary id) can end up different for the exact same class depending + // on when/which subsystem resolves it, because that dictionary can be + // compacted/regenerated independently of this table - found the hard way + // correlating ReferenceChainTracker::FrontierEntry::referrer_klass values + // (also a classMap id, resolved at a different time by a different + // subsystem) against a LivenessTracker-reported growing klass_id: the + // exact same class ("[B" in the reproducing case) resolved to two + // different classMap ids depending on which subsystem asked. This field + // exists specifically so cross-subsystem klass matching (referenceChains.h's + // own _watched_leak_klass_ids) has something both sides can agree on + // regardless of classMap's own housekeeping. 0 until minted (see + // foldKlassCountsLocked()'s own comment for when that happens) - always + // negative once minted (ClassTagAllocator::next()'s own convention). + jlong stable_class_tag; + // Per-(klass,tid) trend qualification (the disjoint-tagged-vs-frontier + // pod finding: a whole-klass rising generation count can come from churn + // spread across MANY allocating threads, each of which retains a STABLE + // handful of instances - observed live on the hotdog pod where the only + // qualifying [B candidate's tagged instances were 24-16KB machinery + // byte[]s whose per-site retention never rose). Each TidTrend holds one + // allocating thread's per-epoch count of distinct surviving GC ages (the + // same generation-cardinality signal the klass ring above uses, at + // per-thread granularity), and selectLeakCandidates() only reports the + // klass as a candidate if at least one tid here shows a sustained rise of + // its own - continuous retention concentrated in one thread is the leak + // shape; retention spread thinly and stably across threads is machinery. + // A real leak filled from many threads still qualifies: every thread that + // keeps adding surviving instances grows its own per-tid age span. The + // age trend alone cannot see one-cohort-per-thread accumulation (each + // one-shot thread's instances share one age, so its distinct-age count + // stays 1 forever), which is why qualification is that trend OR the + // retained-count bar (TID_RETAINED_COUNT_BAR): machinery threads retain + // neither rising age spans nor a large surviving count, and fail both. + static constexpr int MAX_TID_TRENDS = 8; + static constexpr int TID_TREND_RING_SIZE = 16; + struct TidTrend { + jint tid; // allocating thread id (TrackingEntry::tid's space) + u8 ring[TID_TREND_RING_SIZE]; // per-epoch distinct surviving age counts + u8 count_ring[TID_TREND_RING_SIZE]; // per-epoch surviving tracked- + // instance counts (same head/fill as ring - + // always pushed together; feeds the + // TID_RETAINED_COUNT_BAR discriminator) + u8 ring_head; + u8 ring_fill; + u8 consecutive_positive; // sustained-qualification hysteresis, same + // push-time update pattern as the klass + // consecutive_positive above + // Seeded by tidTrendRecordForTest() only: the real fold + // (recordTidTrendSamplesLocked()) never marks entries synthetic, and + // exempts synthetic entries from both its zero-pushes-for-absent-tids + // decay and its slot-eviction search - a scenario's seeded ramp would + // otherwise be wiped/reset by the very next real GC's fold. Production + // data has no synthetic entries, so the exemption is inert there. + bool synthetic; + }; + TidTrend tid_trends[MAX_TID_TRENDS]; + int tid_trend_count; +} KlassPopulationEntry; + +// One leak-candidate result from selectLeakCandidates() below: the klass to +// chase, a currently-live representative instance of it, and the threads +// whose per-tid trends qualified it - ready to hand to +// referenceChains.cpp's pollWatchedTargets() (the design doc's Open Question +// 3 bridging step). Deliberately excludes the slope/rank that produced the +// ranking - the caller only needs identity, matching the design doc's own +// "KlassCandidate { u32 klass_id; jweak representative; }" sketch, extended +// with the qualifying tids selectLeakCandidates() now requires. +typedef struct KlassCandidate { + u32 klass_id; + jweak representative; + // The tids whose per-tid trends cleared the same hysteresis gate that + // qualified this klass (selectLeakCandidates() fills this; empty never + // happens for a returned candidate). tagLeakInstances() only tags + // tracked instances of klass_id allocated by one of these threads - + // the per-(klass,tid) tagging scope that keeps machinery churn of the + // same class from consuming the leak-tag pool. + static constexpr int MAX_QUALIFYING_TIDS = KlassPopulationEntry::MAX_TID_TRENDS; + jint qualifying_tids[MAX_QUALIFYING_TIDS]; + int qualifying_tid_count; +} KlassCandidate; + // Aligned to satisfy SpinLock member alignment requirement (64 bytes) // Required because this class contains SpinLock _table_lock member class alignas(alignof(SpinLock)) LivenessTracker { @@ -58,6 +208,112 @@ class alignas(alignof(SpinLock)) LivenessTracker { constexpr static int MAX_TRACKING_TABLE_SIZE = 262144; constexpr static int MIN_SAMPLING_INTERVAL = 524288; // 512kiB + // _klass_population/_klass_count_scratch below are both scanned linearly + // (lookup and LRU-eviction search) - this size keeps every such scan cheap + // enough that a plain linear scan is fine, rather than requiring an index. + constexpr static int MAX_KLASS_POPULATION_ENTRIES = 256; + // Design doc's Open Question 3 proposal: "ring buffer of up to 30 recent + // population counts". + constexpr static int KLASS_POPULATION_RING_SIZE = 30; + // Design doc's Open Question 3 proposal: "Only trust the trend once the + // window has a minimum fill (e.g. ≥10 samples) to avoid noise right after + // a klass starts being tracked." + constexpr static int KLASS_POPULATION_MIN_FILL_FOR_TREND = 10; + // Per-tid trend gate's own minimum fill (see KlassPopulationEntry:: + // TidTrend): 6 samples of the 16-slot ring leaves a 2-3 sample + // regression, small enough that a per-tid qualification (6 pushes + + // the 3-5 hysteresis epochs, ~9-11) completes no later than the klass + // gate's own ~13-15-epoch latency, keeping the klass ring the sole + // latency driver for candidate emergence. + constexpr static int TID_TREND_MIN_FILL_FOR_TREND = 6; + // Per-tid qualification's second discriminator, OR-ed with the age + // trend: a thread whose surviving tracked-instance count of THIS ONE + // klass clears this bar qualifies without waiting out the trend/hysteresis + // latency. Covers the shape the age-trend gate structurally cannot see - + // one-cohort-per-thread accumulation (each one-shot worker thread's + // instances all share one age, so its distinct-age count stays 1 forever, + // but it retains hundreds of instances), and shortens qualification for + // any big per-thread retained set. + // 8 tracked survivors of a single class from one thread is roughly an + // order of magnitude above the machinery shapes this gate exists to + // exclude (buffer/thread pools retain 1-5 tracked per class/thread, + // observed on the hotdog pod's tagged machinery byte[]s); the live-heap + // subsample makes real instance counts ~10x the tracked count, so the bar + // corresponds to ~80+ retained same-class instances from one thread. + constexpr static u32 TID_RETAINED_COUNT_BAR = 8; + // secondsToOOM() below corroborates the full-window heap-floor regression + // with a second regression over just the most recent half of the same + // ring, and requires both to show a positive slope. Guards against a + // one-time step change that immediately plateaus (e.g. a cache warming up + // once at startup) - the full-window fit alone stays "rising" for as long + // as the step's samples remain in the window, but the recent half's own + // slope collapses back to ~0 as soon as growth actually stops, well + // before the full window ages the step out. fill/2 is always >= + // HEAP_FLOOR_RECENT_HALF_MIN_FILL once the full-window fit itself has + // cleared KLASS_POPULATION_MIN_FILL_FOR_TREND (10), so this never blocks + // a reading the full-window check wouldn't already have blocked. + constexpr static int HEAP_FLOOR_RECENT_HALF_MIN_FILL = 5; + // Design doc's Open Question 3 proposal: "seed only the top 3-5 by trend + // magnitude" - this is the upper end of that range. selectLeakCandidates() + // also honors the caller-supplied `max`, so the effective cutoff is + // min(max, MAX_LEAK_CANDIDATES, ); + // "no separate budget constant is needed" per the design doc, this top-N + // cutoff doubles as the per-pass seeding cap. + // (Moved to public section for ReferenceChainTracker access.) + + // --- Sustained-trend gate (hasQualifyingGrowth() below) --- + // count_ring holds, per epoch, the number of distinct GC ages + // (generations) among a klass's surviving tracked instances + // (accumulateKlassCount(), livenessTracker.cpp) rather than the raw + // surviving instance count: a klass whose survivors keep spanning more + // distinct allocation cohorts over time is one where old instances are + // not dying as new ones arrive, which is the leak shape this gate looks + // for. The gate itself is a single condition - the ring's regression end + // value must exceed its start value by a meaningful + // margin (LEAK_GROWTH_REL_MIN/LEAK_GROWTH_ABS_MIN, whichever is larger). + // An earlier revision of this gate also required the recent half's + // *minimum* to exceed the full window's minimum (a floor-rise check, + // to reject oscillations whose peak alone passes the growth test) - that + // check was tuned for raw population counts (which can run into the + // thousands) and does not transfer to generation counts, which are small + // integers bounded by how many distinct cohorts a klass can realistically + // accumulate; it was dropped rather than re-tuned. + constexpr static double LEAK_GROWTH_REL_MIN = 0.15; + // Absolute floor for the growth bar: the slope must exceed + // max(LEAK_GROWTH_REL_MIN * earliest_mean, LEAK_GROWTH_ABS_MIN). + constexpr static int LEAK_GROWTH_ABS_MIN = 1; + + // Required number of consecutive qualifying epochs + // (KlassPopulationEntry::consecutive_positive) before selectLeakCandidates() + // trusts a klass as a leak candidate. Lower (CORROBORATED) when the + // aggregate post-GC live heap (heapFloorRising() below) is independently + // showing a sustained rise of its own over the same horizon - that is + // whole-heap evidence this klass's growth isn't an isolated artifact + // (redistribution/churn that nets out heap-wide, or per-klass sampling + // noise), so fewer of this klass's own epochs are needed to trust it. + // heapFloorRising() is a single call per selectLeakCandidates() scan, not + // per candidate: the aggregate heap has no per-klass attribution, so it + // cannot single out which klass (if any) is responsible for its rise - + // it can only raise or lower the bar for every candidate in that scan + // uniformly, never reorder them against each other. + constexpr static int LEAK_TREND_HYSTERESIS_BASE = 5; + constexpr static int LEAK_TREND_HYSTERESIS_CORROBORATED = 3; + + // --- Aggregate post-GC heap floor (heapFloorRising() below) --- + // Same "regression growth + floor rise" shape as the per-klass test + // above, applied to a single global ring of post-GC live heap size + // instead of one klass's sampled population - see this class's own ring + // (_heap_floor_ring below). Its own thresholds are deliberately looser + // (fraction-of-heap, not fraction-of-one-klass): this signal is diluted by + // every other klass's allocation activity (a leak far smaller than these + // thresholds is invisible against the rest of the heap), so it is not + // sensitive enough to gate on directly - it is used only as the + // LEAK_TREND_HYSTERESIS_BASE/CORROBORATED selector above. + constexpr static double HEAP_FLOOR_GROWTH_REL_MIN = 0.02; + constexpr static u64 HEAP_FLOOR_GROWTH_ABS_MIN = 1ULL << 20; // 1MiB + constexpr static double HEAP_FLOOR_FLOOR_REL_MIN = 0.01; + constexpr static u64 HEAP_FLOOR_FLOOR_ABS_MIN = 1ULL << 19; // 512KiB + bool _initialized; bool _enabled; Error _stored_error; @@ -70,6 +326,18 @@ class alignas(alignof(SpinLock)) LivenessTracker { SubsampleRate _subsample; + // Chase-phase admission-boost state (admitForTracking()'s declaration + // comment below). _watched_tids is two-phase-published: slots first, then + // _watched_tid_count with RELEASE (admitForTracking()'s ACQUIRE load pairs + // with it) - a reader never trusts a slot beyond the count it observed. + // Every slot access is ATOMIC (__atomic_* builtins, relaxed): the poll + // thread rewrites slots while allocation threads may still read them under + // an acquire count load that observed the OLD count - plain accesses there + // are a C++ data race (UB), not just a staleness artifact. + jint _watched_tids[KlassCandidate::MAX_QUALIFYING_TIDS]; + volatile int _watched_tid_count; + volatile bool _urgent_tracking; + bool _record_heap_usage; jclass _Class; @@ -78,12 +346,261 @@ class alignas(alignof(SpinLock)) LivenessTracker { volatile u64 _gc_epoch; volatile u64 _last_gc_epoch; + // Timestamp (OS::nanotime()) of the last cleanup_table() sweep that + // actually ran, whether organic (flush_table()'s JFR cadence) or forced + // (track()'s table-overflow branch). Read/written only by + // maybeForceCleanup() below - see that method's own comment for why a + // third, time-based trigger is needed on top of those two. + volatile u64 _last_cleanup_ns; + size_t _used_after_last_gc; + // Ring of post-GC live heap sizes, one sample per GC epoch, feeding + // heapFloorRising() below - same shape as KlassPopulationEntry::count_ring + // but a single global instance rather than one per klass, and lock-free + // rather than _table_lock-guarded: onGC() runs from the JVMTI + // GarbageCollectionFinish callback, which can fire synchronously mid-way + // through a JNI upcall this class itself is making while already holding + // _table_lock (e.g. cleanup_table()'s Class.getName() call, if that + // allocation triggers a GC) - taking the same lock here would risk a + // self-deadlock on a non-reentrant SpinLock. GC completions are never + // concurrent with each other (HotSpot never runs two GCs at once), so + // onGC() is always a single writer at a time, matching the existing + // lock-free _gc_epoch/_used_after_last_gc fields' own assumption - see + // recordHeapFloorSample()/heapFloorRising() (livenessTracker.cpp) for the + // load/store ordering this relies on. + u64 _heap_floor_ring[KLASS_POPULATION_RING_SIZE]; + // OS::nanotime() paired index-for-index with _heap_floor_ring above (same + // head/fill, always pushed together by recordHeapFloorSample() - see that + // method's comment) - heapFloorRising() itself has no use for elapsed + // time, but secondsToOOM() below needs it to turn the ring's byte growth + // into a rate rather than just a magnitude. + u64 _heap_floor_time_ring[KLASS_POPULATION_RING_SIZE]; + // Container (cgroup) memory usage, one sample per GC epoch, pushed to the + // same head/fill index as _heap_floor_ring/_heap_floor_time_ring above by + // the same recordHeapFloorSample() call - so this ring shares + // _heap_floor_time_ring's timestamps rather than needing its own. Exists + // because container memory can grow from causes _heap_floor_ring never + // sees at all (native/off-heap allocations - direct buffers, JNI/native + // libraries) - see secondsToOOM()'s own comment for how this ring and + // _heap_floor_ring are used as two independent OOM boundaries. + u64 _container_mem_ring[KLASS_POPULATION_RING_SIZE]; + volatile u8 _heap_floor_ring_head; + volatile u8 _heap_floor_ring_fill; + + // Debug-only test seam: when true, onGC() skips recordHeapFloorSample() + // so a test can seed the ring exclusively via heapFloorRecordForTest() + // without a real GC interleaving a sample with a real OS::nanotime() + // timestamp and real heap usage, corrupting secondsToOOM()'s projection. + // This closes the race where a real GC fires between + // resetKlassPopulationForTest0() and shouldRunPassForTest0() and a + // concurrent recordHeapFloorSample() corrupts the projection. +#ifdef DEBUG + // Atomic (not plain bool) so the GC thread sees the test thread's store + // on arm64's weak memory model. Checked inside recordHeapFloorSample() + // at the actual write point, not just in onGC(), so a GC already past the + // onGC() gate when the test sets this flag still cannot corrupt the ring. + std::atomic _heap_floor_recording_disabled_for_test{false}; +#endif + + // Runtime.maxMemory(), resolved once by initialize_table() (the same call + // that already requires it to enable liveness tracking at all - see that + // method's own Error path) and cached here so secondsToOOM() - polled once + // per ReferenceChainTracker::threadLoop wake, ~1s - does not repeat + // HeapUsage::getMaxHeap()'s handful of JNI calls on every poll. A JVM's max + // heap does not change at runtime, so a value resolved once stays valid. + // -1 if never resolved (mirrors getMaxHeap()'s own sentinel). + jlong _max_heap_bytes; + +#ifdef DEBUG + // Atomic mirror used only by the setMaxHeapBytesForTest() seam so the + // test thread's store is published to the BFS thread's secondsToOOM() + // read on arm64's weak memory model. _max_heap_bytes above stays plain + // for the production path (written once from the control thread during + // initialize_table(), before the BFS thread starts, so no cross-thread + // visibility issue there). + std::atomic _max_heap_bytes_for_test{-1}; +#endif + + // OS::getContainerMemoryLimit(), resolved once by initialize_table() next + // to _max_heap_bytes above (same call site, same "resolved once, does not + // change at runtime" reasoning) and cached here so secondsToOOM() does not + // re-walk the cgroup hierarchy on every poll. -1 if never resolved, or if + // this process is not running under a memory-limited cgroup (bare metal, + // macOS, cgroups disabled) - secondsToOOM() treats -1 as "unbounded" so + // this boundary never wins over the heap-based one. + jlong _container_memory_limit; + +#ifdef DEBUG + // Mirrors _max_heap_bytes_for_test above for the same reason: lets a test + // exercise secondsToOOM()'s container boundary without a real cgroup. + std::atomic _container_memory_limit_for_test{-1}; +#endif + + // Gates the per-klass population tracking below. Set from + // args._gc_generations in initialize() - deliberately not folded into + // _enabled (which also covers plain _record_liveness): this doesn't + // resolve the design doc's own "still undecided" bullet under Open + // Question 3 by itself, but the plan built on top of this table requires + // liveness tracking *and* _gc_generations, matching the doc's stated + // fallback of "no target-seeding" when generations tracking isn't on + // (Arguments::_gc_generations). std::atomic (relaxed) since initialize() + // writes it from the control thread while the BFS thread + // (maybeForceCleanup()) and the GC-callback thread (cleanup_table()) can + // still be reading it from a session that persists across a restart. + std::atomic _gc_generations; + + // Per-klass population history table (see KlassPopulationEntry above). + // Populated only from cleanup_table()'s GC-epoch-advance pass, never from + // track() (the allocation sampling hot path) - see + // accumulateKlassCount()/foldKlassCountsLocked() below. Guarded by + // _table_lock, the same lock cleanup_table() already holds for the + // duration of its epoch-advance pass, rather than adding a second lock. + KlassPopulationEntry _klass_population[MAX_KLASS_POPULATION_ENTRIES]; + int _klass_population_size; + + // Scratch space reused across cleanup_table() calls (a member field, not a + // per-call stack/heap allocation - cleanup_table() runs on a GC-signal + // cadence, not the allocation hot path, but this codebase's + // allocation-free preference still applies wherever avoiding an + // allocation is cheap) to accumulate this epoch's per-klass surviving + // counts before folding them into _klass_population's ring buffers at the + // end of the pass. + typedef struct KlassCountScratch { + u32 klass_id; + // Distinct GC ages (generations) of surviving tracked instances + // of this klass at this epoch. The size of this vector + // is the klass' generation count. + std::vector ages; + // Top-N oldest surviving instances of this klass seen this epoch, + // sorted by age descending. Used by foldKlassCountsLocked() to mint + // representatives biased toward long-lived instances (Lindy effect: + // the oldest surviving instances are the most likely to be leaks). + // Fixed-size to avoid heap allocation in the GC callback path. + static constexpr int MAX_OLDEST_SAMPLES = 3; + struct OldestSample { + jweak ref; + u32 age; + jint tid; // allocating thread of this instance + }; + OldestSample oldest[MAX_OLDEST_SAMPLES]; + int oldest_count; + // Per-thread generation tracking (Cork/Swat heuristic adapted): + // track distinct surviving GC ages per allocating thread (tid) + // within this klass. The thread with the most distinct surviving + // generations is the strongest leak signal — it reuses the same + // generation-count signal that selectLeakCandidates() uses + // per-class, applied at per-thread granularity within a class. + // A thread with 12 distinct surviving ages (continuous leak) + // outscores a thread with 1 age (one-time burst). + // + // Thread ID is used instead of call_trace_id because lambdas + // fragment call_trace_id — synthetic methods produce slightly + // different stack hashes for what is logically one allocation + // site, yielding N sites × 1 generation instead of 1 site × N + // generations. Thread ID is stable and naturally separates leak + // threads from noise threads. + static constexpr int MAX_THREADS_PER_KLASS = 16; + static constexpr int MAX_AGES_PER_THREAD = 32; + struct ThreadGens { + jint tid; + u32 ages[MAX_AGES_PER_THREAD]; // sorted distinct ages + u32 age_count; + u32 count; // surviving tracked instances of this klass this epoch + // (increments per object, before the age-dedup early + // return below) - feeds the per-tid retained-count bar + // (TID_RETAINED_COUNT_BAR) + }; + ThreadGens threads[MAX_THREADS_PER_KLASS]; + int thread_count; + } KlassCountScratch; + KlassCountScratch _klass_count_scratch[MAX_KLASS_POPULATION_ENTRIES]; + int _klass_count_scratch_size; + + // Profiler::classMap()'s generation as of the last cleanup_table() call + // that checked it, mirroring ReferenceChainTracker::_last_class_map_generation + // (referenceChains.h). Profiler::start() calls _class_map.clearAll() + // (profiler.cpp) whenever `reset || _start_time == 0`, restarting that + // StringDictionary's id namespace at 1 - but TrackingEntry::cached_klass_id + // and _klass_population's klass_id keys are ids resolved from that + // dictionary, and both survive stop()/start() cycles (this class's table is + // designed to persist across recordings). Left unguarded, an id cached + // before a reset would silently collide with whatever unrelated class the + // new generation reassigns that same id to. cleanup_table() compares this + // against Profiler::instance()->classMap()->generation() and, on a + // mismatch, drops every such cached id before resuming. Constructor- + // initialized to 0 (StringDictionary's own initial generation) but + // immediately re-synced in initialize() to whatever generation() already is + // by then - NOT left at 0, despite 0 also being correct for a + // never-yet-reset classMap in isolation: ObjectSampler::start() -> + // LivenessTracker::start() always runs strictly after Profiler::start()'s + // own _class_map.clearAll() (referenceChains.cpp's comment on this same + // ordering), so by the time initialize() runs, generation() has *already* + // bumped once for this process's first recording. Leaving the 0 sentinel + // in place here would make cleanup_table()'s very first call always see a + // spurious mismatch against that already-happened bump, discarding + // whatever genuinely post-reset population history had already + // accumulated by then - found the hard way (see initialize()'s own + // comment). + u64 _last_class_map_generation; + + // --- Leak tag pool --- + // Reusable pool of JVMTI tags for directly tagging tracked leaking + // objects. Tags are in range [LEAK_TAG_BASE, LEAK_TAG_BASE+POOL_SIZE). + // When a tracked object is GC'd, its tag is returned to the pool. + // This lets the BFS find the exact leaking objects (not just any + // instance of the same class) and correlate chains with HeapLiveObject. + static constexpr int LEAK_TAG_POOL_SIZE = 256; + static constexpr jlong LEAK_TAG_BASE = 0x40000000LL; + // Serializes the pool's free list and _leak_tag_info entries. NOT _table_lock: + // acquireLeakTag() runs under the SHARED table lock (tagLeakInstances) while + // releaseLeakTag() runs under the EXCLUSIVE one (cleanup_table) - a shared + // holder excludes the exclusive one, but only this dedicated lock makes the + // pool safe for any future second shared-lock mutator, and getLeakTagInfo() + // takes NO table lock at all (BFS poll thread). Lock order: _table_lock + // (any mode) is always acquired BEFORE _leak_tag_pool_lock, never the + // reverse; no path acquires _table_lock while holding the pool lock. + mutable SpinLock _leak_tag_pool_lock; + int _leak_tag_free_list[LEAK_TAG_POOL_SIZE]; + int _leak_tag_free_count; + // Side table: for each tag in the pool, the (call_trace_id, tid) of + // the tracked object it was assigned to. Used by ReferenceChainTracker + // for coverage tracking (adaptive CPU budget). + struct LeakTagInfo { + u64 call_trace_id; + jint tid; + }; + LeakTagInfo _leak_tag_info[LEAK_TAG_POOL_SIZE]; + + jlong acquireLeakTag(u64 call_trace_id, jint tid); + void releaseLeakTag(jlong tag); + Error initialize(Arguments &args); Error initialize_table(JNIEnv *jni, int sampling_interval); - void cleanup_table(bool force = false); + // force=true is used by track()'s table-overflow branch to run a cleanup + // synchronously from the allocation-sampling call stack, bypassing the + // GC-epoch-changed check below. The per-klass population tracking below + // (_gc_generations) runs on both paths, once per genuinely new GC epoch + // (see "is_epoch_owner" in livenessTracker.cpp). + // + // allow_resolve gates resolveKlassId() - a real Class.getName() + // Java-bytecode upcall, unlike the plain native JVMTI calls already made + // elsewhere on track()'s callback stack - independently of force: force + // only says "bypass the epoch-unchanged early-exit", it says nothing about + // which call stack this is running on. track()'s hot-path call passes + // force=true, allow_resolve=false (too costly/re-entrancy-prone to resolve + // from the SampledObjectAlloc callback stack - reuses whatever + // cached_klass_id an entry already picked up from an earlier resolving + // sweep, or skips accounting for that entry this epoch if it was never + // resolved). flush_table()/stop() pass the defaults (force=false, + // allow_resolve=true) - the original organic, GC-cadence path. LivenessTracker::maybeForceCleanup() passes force=true, + // allow_resolve=true: it runs on ReferenceChainTracker's own background + // thread (referenceChains.cpp), not the allocation hot path, so the same + // upcalls flush_table() already makes safely are just as safe there - see + // that method's own comment for why a third caller needs both bypassing + // the early-exit *and* resolution. + void cleanup_table(bool force = false, bool allow_resolve = true); void flush_table(std::set *tracked_thread_ids); @@ -92,11 +609,206 @@ class alignas(alignof(SpinLock)) LivenessTracker { jlong getMaxMemory(JNIEnv *env); + // Resolves the best available post-GC heap usage sample, mirroring + // flush_table()'s own resolution order (JDK17+ exact + // CollectedHeap::_used_at_last_gc when supported, otherwise onGC()'s own + // _used_after_last_gc snapshot, falling back to a live usage read if + // neither has produced anything yet, e.g. before the first GC). Shared by + // flush_table()'s JFR event and onGC()'s heap-floor ring sample so both + // read the same value the same way. Returns 0 only if HeapUsage itself has + // nothing to offer. *out_is_last_gc (if non-null) reports which case was + // used, for callers (flush_table()) that need to say so in the JFR event. + size_t resolvePostGcHeapUsage(bool *out_is_last_gc); + + // --- Per-klass population tracking (cleanup_table()'s epoch-advance pass only) --- + + // Resolves the StringDictionary id for `ref`'s class, mirroring + // flush_table()'s existing class-name resolution above (GetObjectClass + + // Class.getName() + Profiler::lookupClass()) - this is the "genuinely new + // cost on an existing pass" the design doc flags, previously paid only at + // JFR-flush time. Returns 0 (StringDictionary's own "no entry" sentinel) + // if the name could not be resolved or interned. + u32 resolveKlassId(JNIEnv *env, jobject ref); + + // Increments klass_id's running sample count in _klass_count_scratch for + // the epoch currently being processed, creating a new scratch slot (with + // `sample_source` remembered for a possible new KlassPopulationEntry) if + // this is the first surviving instance of this klass seen so far this + // epoch. No-op if the scratch table is already full and klass_id is not + // present - the same fixed-capacity/best-effort tradeoff + // _klass_population's own table already accepts, one level up. + void accumulateKlassCount(u32 klass_id, jlong age, jweak sample_source, + jint tid); + void insertOldestSample(KlassCountScratch &scratch, jweak sample_source, + u32 age, jint tid); + void insertThreadGen(KlassCountScratch &scratch, jint tid, u32 age); + + // Pushes `count` into klass_id's ring buffer, creating the entry (evicting + // the least-recently-updated entry first if the table is already at + // MAX_KLASS_POPULATION_ENTRIES capacity - the same evict-LRU-on-insert- + // when-full shape NativeSocketSampler's fd cache already solves, + // NativeSocketSampler's insertFdAddrLocked(), and the same + // "single agent-owned pass, lock already held by caller" shape + // cleanup_table() itself already uses) if klass_id has never been seen. + // A newly-created entry's `representative` is left null - it is the + // caller's job (foldKlassCountsLocked(), which owns the JNIEnv this + // method deliberately does not touch) to fill it in, which keeps this + // method free of any JNI call and therefore directly exercisable by gtest + // without a live JVM. On return, *out_slot is the table slot used for + // klass_id and *out_created is true iff a new entry was created (an + // evicted-and-reused slot counts as "created", since the old klass_id's + // data was fully replaced). Returns the evicted entry's representative + // jweak (nullptr if nothing was evicted, or the evicted entry had none) + // so the caller can DeleteWeakGlobalRef() it. + // Precondition: _table_lock is held (by cleanup_table(), the only + // production caller). + jweak recordKlassPopulationSampleLocked(u32 klass_id, u32 count, u64 epoch, + int *out_slot, bool *out_created, + jweak *out_evicted = nullptr, + int *out_evicted_count = nullptr, + int max_evicted = 0); + + // Drains _klass_count_scratch into _klass_population for the epoch that + // just finished, minting a fresh representative jweak (from each entry's + // KlassCountScratch::sample_source) for klasses not already present, and + // retrying the mint for existing entries whose representative is still + // null (a previous epoch's mint attempt can fail if sample_source died in + // the window between cleanup_table()'s survival check and the mint - see + // this method's own retry-condition comment, livenessTracker.cpp) - see + // recordKlassPopulationSampleLocked()'s comment for why that JNI work + // happens here rather than inside it. A fresh weak global ref + // is used instead of aliasing sample_source directly because + // sample_source is the corresponding TrackingEntry's own jweak: that + // entry's slot in _table is reused (and its jweak deleted via + // DeleteWeakGlobalRef) the moment the tracked object dies and + // cleanup_table() reaps it, which would leave _klass_population holding a + // dangling handle if it aliased the same jweak. Resets + // _klass_count_scratch_size to 0 once drained. Called with _table_lock + // held, at the end of cleanup_table()'s epoch-advance pass. + // allow_resolve mirrors resolveKlassId()'s own parameter (cleanup_table()'s + // header comment): when false, this runs synchronously on the JVMTI + // SampledObjectAlloc callback stack (track()'s table-overflow branch), so + // the representative-minting NewLocalRef/NewWeakGlobalRef/DeleteLocalRef + // churn below is skipped - the ring/count bookkeeping still happens, and a + // missing representative is retried on the next allow_resolve=true sweep + // (see the retry-condition comment in livenessTracker.cpp). + void foldKlassCountsLocked(JNIEnv *env, u64 epoch, bool allow_resolve); + + // Get-or-mints slot's stable_class_tag (see that field's own comment) from + // `instance`'s class, if not already minted for this slot's current + // occupant. Shared by foldKlassCountsLocked() (the production allocation- + // sampling path) and klassPopulationSetRepresentativeForTest() (the + // debug-only test seam StaticFieldGrowingCollectionScenario-style + // external-process tests drive instead of real sampling) - both hand this + // a live representative instance to resolve the class from, so neither + // needs its own copy of the GetObjectClass/GetTag/SetTag sequence. No-op + // if slot is out of range, instance is null, or a tag is already minted. + void mintStableClassTagIfNeeded(JNIEnv *env, int slot, jobject instance); + + // --- Slope computation and candidate ranking (selectLeakCandidates() below) --- + + // Per-tid sustained-trend gate half #1: the same regression growth + // test hasQualifyingGrowth() below applies to a klass's ring, at + // KlassPopulationEntry::TidTrend granularity (TID_TREND_MIN_FILL_FOR_TREND + // samples of that smaller ring, same LEAK_GROWTH_REL_MIN/ABS_MIN growth + // bar - per-tid age-cardinality is the same small-integer signal the + // klass gate already uses, so the same bar transfers). Pure read over the + // trend's ring; no cached slope (nothing ranks tids against each other by + // slope - tagLeakInstances ranks its candidates by the live tracking + // table's own age diversity, not by this history). + bool hasQualifyingTidGrowth(const KlassPopulationEntry::TidTrend &trend) const; + + // Per-tid qualification for ONE epoch push: the age-trend test above OR + // the just-pushed surviving-instance count clearing + // TID_RETAINED_COUNT_BAR (the one-cohort-per-thread accumulation shape + // the age trend structurally cannot see - see that constant's own + // comment). The caller applies the result to the trend's own + // consecutive_positive, the same push-time pattern + // recordKlassPopulationSampleLocked() uses for the klass-level counter; + // the required hysteresis then makes both discriminators SUSTAINED + // (a thread over the bar must stay over it for required_hysteresis + // consecutive epochs - a transient burst that GCs away resets). + bool tidPushQualifies(const KlassPopulationEntry::TidTrend &trend, + u32 current_count) const; + + // Folds one epoch's per-thread scratch (KlassCountScratch::threads - + // per-tid distinct surviving GC ages, already maintained by + // accumulateKlassCount()/insertThreadGen()) into slot's per-tid trend + // rings: present tids push their epoch count; tracked tids ABSENT this + // epoch push 0 (a thread whose instances all died must not keep a stale + // rising ring - the 0 push fails hasQualifyingTidGrowth() and resets that + // trend's consecutive_positive, the per-tid analogue of the population + // simply stopping); synthetic (test-seeded) trends are exempt from that + // decay and from eviction. New tids beyond MAX_TID_TRENDS evict the + // non-synthetic trend with the lowest consecutive_positive (then the + // lowest ring_fill) - a genuinely rising leak tid accumulates hysteresis + // fast and resists eviction, machinery tids never do. Best-effort caveat: + // KlassCountScratch caps per-klass threads at 16 > MAX_TID_TRENDS, so a + // klass with more than 16 allocating threads in one epoch can miss a + // tracked tid from the scratch and push it a spurious 0 - the same + // fixed-capacity best-effort the scratch itself already accepts. + // _table_lock held exclusively by the caller (foldKlassCountsLocked()). + void recordTidTrendSamplesLocked(int slot, const KlassCountScratch &scratch); + + // The sustained-trend gate (this class's own header comment above, + // "Sustained-trend gate") - both-required growth-magnitude and floor-rise + // tests, design doc's original "mean of thirds" choice since replaced by + // full-window least-squares regression (see ringThirdsStats, + // livenessTracker.cpp - cheap, allocation-free, one pass over the + // ring, no sorting or extra storage). A single scan + // (ringThirdsStats(), livenessTracker.cpp) both derives the pass/fail + // result below AND updates entry.cached_slope (regression end value minus + // start value) for selectLeakCandidates()'s ranking, rather than + // that method re-scanning the same unchanged ring a moment later. Returns + // false (leaving entry.cached_slope untouched) if entry.ring_fill is below + // KLASS_POPULATION_MIN_FILL_FOR_TREND - not enough history yet to trust a + // trend; callers must check ring_fill themselves before trusting + // cached_slope, exactly as they checked this method's own return value + // before cached_slope existed. + // + // Called from recordKlassPopulationSampleLocked() every time a new sample + // is pushed (both the production path, + // foldKlassCountsLocked()->recordKlassPopulationSampleLocked(), and the + // klassPopulationRecordForTest() test seam that calls the same method + // directly), so KlassPopulationEntry::consecutive_positive/cached_slope + // are always kept in sync with the ring they summarize, regardless of + // caller. + bool hasQualifyingGrowth(const KlassPopulationEntry &entry) const; + + // Pushes `used`/`timestamp_ns`/`container_used` into _heap_floor_ring/ + // _heap_floor_time_ring/_container_mem_ring - see those members' own + // comments for why this is lock-free rather than _table_lock-guarded. + // Called only from onGC() (single-writer-at-a-time, same comment), which + // supplies OS::nanotime() and OS::getContainerMemoryUsage() explicitly + // rather than this method calling them internally - keeps this method + // itself deterministic for the heapFloorRecordForTest() test seam below. + void recordHeapFloorSample(u64 used, u64 timestamp_ns, u64 container_used); + + // Internal: pushes to the rings without checking + // _heap_floor_recording_disabled_for_test. Called by + // recordHeapFloorSample() (after the debug-only flag check) and by + // heapFloorRecordForTest() (which bypasses the check so a test can + // seed the rings even while real GC recording is disabled). + void recordHeapFloorSampleUnchecked(u64 used, u64 timestamp_ns, u64 container_used); + + // Reads whether the aggregate post-GC live heap has itself shown a + // sustained rise over _heap_floor_ring's horizon - see + // LEAK_TREND_HYSTERESIS_BASE/CORROBORATED's own comment above for how + // selectLeakCandidates() uses this (a uniform hysteresis-threshold + // selector for the whole scan, never a per-candidate veto or boost). + bool heapFloorRising() const; + public: static LivenessTracker *instance() { static LivenessTracker instance; return &instance; } + + // Public read accessor for the auto-tuner (ReferenceChainTracker::autoTuneDefaults) + // and secondsToOOM(). See _max_heap_bytes's own comment above. + jlong maxHeapBytes() const { return _max_heap_bytes; } + + constexpr static int MAX_LEAK_CANDIDATES = 5; // Delete copy constructor and assignment operator to prevent copies LivenessTracker(const LivenessTracker&) = delete; LivenessTracker& operator=(const LivenessTracker&) = delete; @@ -104,24 +816,710 @@ class alignas(alignof(SpinLock)) LivenessTracker { LivenessTracker() : _initialized(false), _enabled(false), _stored_error(Error::OK), _table_size(0), _table_cap(0), _table_max_cap(0), _table(NULL), - _subsample(0.1), _record_heap_usage(false), _Class(NULL), + _subsample(0.1), _watched_tid_count(0), _urgent_tracking(false), + _record_heap_usage(false), _Class(NULL), _Class_getName(0), _gc_epoch(0), _last_gc_epoch(0), - _used_after_last_gc(0) {} + _last_cleanup_ns(0), _used_after_last_gc(0), + _heap_floor_ring_head(0), _heap_floor_ring_fill(0), + _max_heap_bytes(-1), _container_memory_limit(-1), + _gc_generations(false), + _klass_population_size(0), _klass_count_scratch_size(0), + _last_class_map_generation(0), + _leak_tag_free_count(LEAK_TAG_POOL_SIZE) {} Error start(Arguments &args); void stop(); void track(JNIEnv *env, AllocEvent &event, jint tid, jobject object, u64 call_trace_id); + + // track()'s admission gate: a chase-phase raise of the live-samples + // tracking probability over the configured _subsample ratio (default 10% - + // a 90% probabilistic drop whose thinning of small per-(klass, tid) + // populations was observed live as the intermittent zero-tag runs in + // LeakTagCorrelationReferenceChainTest - see that test's own comment for + // the full lottery analysis). Two independent raises, both advisory-only + // (a missed raise just falls back to the configured ratio's behavior): + // - watched tids: noteSelectedCandidates() publishes + // selectLeakCandidates()'s qualifying tids - exactly the (klass, tid) + // scope tagLeakInstances() tags and the reference-chain chase intercepts + // - and admitForTracking() admits them at 100%. Bounded by the candidate + // threads' own allocation rate (at most MAX_QUALIFYING_TIDS threads), so + // the tracking table's volume/cleanup cost scales with the leak's own + // threads, not the process's whole allocation rate. + // - urgency: setUrgentTracking() from ReferenceChainTracker::threadLoop()'s + // seconds_to_oom ramp admits everything. Under the OOM ramp the process + // is expected to die soon; maximizing what the last chapter captures + // outweighs the tracking table's transient volume. + // Fail-open by construction: the boost only ever adds admissions on top of + // the configured ratio - a stale or missed boost cannot drop an allocation + // that the ratio would have admitted. + bool admitForTracking(jint tid); + + // Publishes the current candidate poll's qualifying tids as the watched + // set above. Called from ReferenceChainTracker's full poll + // (pollWatchedTargets()) every wake - including the zero-candidate case, + // which clears the set (a tid left watched after the chase ends would keep + // admitting that thread at 100% across OS tid reuse). NOT called from + // hasLeakSignal()'s max=1 probe: that partial view could silently drop + // other still-active candidates' tids. + void noteSelectedCandidates(const KlassCandidate *candidates, int count); + + // Urgency raise toggle - see admitForTracking()'s comment above. Set every + // threadLoop() iteration with that iteration's urgency computation, so the + // boost tracks the ramp exactly (both engaging and releasing). + void setUrgentTracking(bool urgent); + void flush(std::set &tracked_thread_ids); - // Frees this thread's subsampling RNG state (track()'s gen/dis/skipped + // Frees this thread's subsampling RNG state (track()'s rng/skipped // ThreadLocals, livenessTracker.cpp). Must be called from a thread that is // about to detach/terminate - see those ThreadLocal's own comment for why // their pthread-key destructors alone cannot be relied on for JNI-attached // threads. Safe to call even if this thread never called track(). static void releaseThreadLocalState(); + // Reads the per-klass population histories (_klass_population) and + // writes up to `max` leak candidates into `out`: klasses whose recent + // population trend is positive (growing), ranked by trend magnitude + // descending, capped at MAX_LEAK_CANDIDATES regardless of `max` (design + // doc's Open Question 3 "top 3-5" cutoff). Returns the number of + // candidates written (0 if _gc_generations was never enabled - + // _klass_population stays empty in that case, since population tracking + // is gated on it, so no separate guard is needed here). Called on demand + // by the BFS-pass poll, not on any timer of its own; does no JNI work, so it is safe + // to call from any thread that can take _table_lock (mirrors + // getLiveTraceIds()'s own shared-lock read pattern, livenessTracker.cpp). + // + // The `representative` jweak copied into KlassCandidate here is a snapshot + // only - callers MUST NOT resolve it directly (e.g. via NewLocalRef()) + // after this method has returned and _table_lock released. This table's + // LRU eviction (recordKlassPopulationSampleLocked(), livenessTracker.cpp) + // can DeleteWeakGlobalRef() that exact handle at any point afterwards + // (from cleanup_table()'s epoch-advance pass, running on a different + // thread), which invalidates the handle - a later NewLocalRef() on it is + // undefined behavior per the JNI spec, not merely "returns null". Use + // resolveCandidateRepresentative() below instead, which re-reads the + // table's current value for klass_id atomically with the resolve. + int selectLeakCandidates(KlassCandidate *out, int max); + + // Tag the tracked instances of the given leak candidates - only the + // instances allocated by a candidate's QUALIFYING tids + // (KlassCandidate::qualifying_tids, filled by selectLeakCandidates() + // from the per-tid trends that cleared the same hysteresis gate) - with + // JVMTI tags from the leak tag pool. Called by pollWatchedTargets() + // with this poll's candidate list. The tid scope is the fix for the + // disjoint-tagged-vs-frontier pod finding's pool-economy half: a leak + // klass's tracked population also contains machinery instances of the + // same class from other threads, and klass-wide tagging spent pool tags + // on those (observed on hotdog: 247 tagged, all machinery byte[]s with + // flat per-site retention, zero ever intercepted) while the real + // leak-site instances churned out of the pool. The BFS recognizes these + // tags by range check (isLeakTag) and admits the specific objects into + // the frontier, storing the leak tag for correlation with HeapLiveObject + // events. Returns the number of tags assigned. + int tagLeakInstances(jvmtiEnv *jvmti, const KlassCandidate *candidates, + int candidate_count); + + // Look up the (call_trace_id, tid) recorded for a leak tag. Returns + // false if the tag is not a valid leak tag or has been returned to the + // pool. Used by ReferenceChainTracker for coverage tracking. + bool getLeakTagInfo(jlong tag, u64 *out_call_trace_id, + jint *out_tid) const; + + // Reads _klass_population and writes up to `max` STABLE CLASS TAGS + // (KlassPopulationEntry::stable_class_tag - NOT the classMap dictionary + // klass_id selectLeakCandidates() above deals in; see that field's own + // comment for why the distinction matters) into `out`, ranked by MOST + // RECENT count_ring sample (the "generation count" - accumulateKlassCount()'s + // own comment: the number of distinct GC ages among a klass's surviving + // tracked instances - livenessTracker.cpp) descending, capped at + // MAX_LEAK_CANDIDATES. Unlike selectLeakCandidates() above, this applies + // NO trend/hysteresis gate at all (no ring_fill minimum beyond "at least + // one sample", no consecutive_positive requirement, no positive-slope + // requirement) - it is meant to be called only once + // ReferenceChainTracker::hasLeakSignal() has ALREADY fired via the + // slower, hysteresis-gated selectLeakCandidates() path, as a faster, + // broader follow-up ranking that does not itself need to wait out that + // same hysteresis a second time for ReferenceChainTracker's own rotation- + // priority use (see referenceChains.h's own comment on + // _watched_leak_klass_ids for why). Same shared-lock read pattern as + // selectLeakCandidates(); a klass whose ring is entirely empty (never + // sampled) or whose stable_class_tag has not been minted yet (no live + // instance resolved so far - foldKlassCountsLocked()'s own comment) is + // skipped, since neither has anything usable to rank or return. Returns + // the number of tags written. + int topKlassesByGenerationCount(u32 *out, int max); + + // Re-reads klass_id's current representative from _klass_population and + // resolves it to a fresh JNI local ref, both under the same _table_lock + // critical section - closes the race selectLeakCandidates()'s own comment + // above describes: a KlassCandidate snapshot returned by that method can + // go stale (LRU-evicted and DeleteWeakGlobalRef()'d) at any point before a + // caller gets around to resolving it. Looking the entry up again by + // klass_id here, under lock, guarantees NewLocalRef() only ever runs on a + // representative jweak this table still actually owns at the moment of the + // call: if klass_id has since been evicted (or was never assigned a + // representative), the lookup simply fails to find it and this returns + // nullptr without ever touching the stale handle. Returns nullptr if + // klass_id is no longer present, has no representative yet, or the + // representative's referent has since been collected (NewLocalRef() on a + // jweak returns null in that case, JNI spec). Mirrors the shared-lock read + // pattern selectLeakCandidates()/getLiveTraceIds() already use. + jobject resolveCandidateRepresentative(JNIEnv *env, u32 klass_id); + int resolveCandidateRepresentatives(JNIEnv *env, u32 klass_id, + jobject *out, int max_out); + + // Exposes the _gc_generations gate (see that member's own comment) so a + // caller outside this class - ReferenceChainTracker::pollWatchedTargets() + // (referenceChains.cpp), the LivenessTracker-to-ReferenceChainTracker + // bridging step - can skip calling selectLeakCandidates() entirely when + // the feature isn't in use, + // rather than relying on that method's own "returns 0" fallback to make + // the no-op cheap. Read-only; this accessor never toggles the flag. + bool gcGenerationsEnabled() const { + return _gc_generations.load(std::memory_order_relaxed); + } + + // Time-to-OOM projection against whichever of two independent boundaries + // is tighter: the JVM heap (_heap_floor_ring vs _max_heap_bytes) or the + // container/cgroup memory limit (_container_mem_ring vs + // _container_memory_limit). The two rings are pushed together (same + // head/fill index, shared _heap_floor_time_ring timestamps - see + // _container_mem_ring's own comment) so this compares _max_heap_bytes + // against _container_memory_limit up front (the latter treated as + // unbounded when unavailable) and runs the regression-based rate + // extrapolation (allocation-free, one ring scan) only once, against + // whichever limit is smaller - not once per boundary. This matters + // because container memory can grow from causes the heap-floor ring never + // sees at all (native/off-heap allocations), so a heap-only projection can + // under-warn right up until an OOM-kill from container memory pressure + // that had nothing to do with heap occupancy. + // + // Exists because selectLeakCandidates()'s per-klass gate + // (KLASS_POPULATION_MIN_FILL_FOR_TREND ring samples plus + // LEAK_TREND_HYSTERESIS_BASE/CORROBORATED consecutive qualifying epochs) + // can take longer to trust a candidate than a fast leak has left before + // OOM - this gives ReferenceChainTracker::hasLeakSignal() an independent, + // rate-based signal to start a search immediately instead of waiting on + // that gate. Not a claim that growth stays linear, only that a + // short-horizon linear extrapolation is a reasonable urgency signal. + // Returns a negative value if the chosen ring is not filled enough yet + // (gcGenerationsEnabled() is off, or too few GC epochs have happened), not + // rising, or neither _max_heap_bytes nor _container_memory_limit was ever + // resolved - callers must treat any non-positive return as "no projection + // available", not "zero seconds". Returns 0 if the chosen ring's recent + // mean has already reached its limit. + // + // Known limitation, inherited from heapFloorRising() rather than + // introduced here: cleanup_table()'s class-map-reset branch clears + // _klass_population but never _heap_floor_ring/_heap_floor_time_ring/ + // _container_mem_ring, so a stop()/start() gap with a real wall-clock + // pause in between can still mix pre-gap and post-gap samples into the + // same window. heapFloorRising() only risked a magnitude error from this; + // this method additionally divides by elapsed time, so the same gap + // understates the growth rate (overstates the projected time-to-OOM) + // rather than the reverse - not solved here. + double secondsToOOM() const; + + // Third trigger for cleanup_table(), alongside track()'s table-overflow + // branch (forced) and flush_table()'s JFR-flush cadence (organic): those + // two both depend on ObjectSampler's allocation-sampling callback firing + // often enough. ObjectSampler::updateConfiguration()'s PID controller + // throttles the JVMTI heap sampling interval toward a fixed target *event + // rate*, not a fixed *byte* rate - under sustained, fast heap growth this + // can push the interval high enough that SampledObjectAlloc (and therefore + // track()) stops firing in practice, starving cleanup_table() of both its + // forced trigger and the per-klass population samples + // selectLeakCandidates()'s slope computation needs. If that happens, the + // history cleanup_table() would otherwise have advanced goes stale and + // ReferenceChainTracker::hasLeakSignal() can never see a positive trend + // again, no matter how much the leaking population actually grows. + // + // Called once per ReferenceChainTracker::threadLoop wake (~1s cadence, see + // referenceChains.cpp) with a live JNIEnv already in hand - a convenient, + // already-existing periodic tick, not a new thread. No-ops unless both: + // (a) at least 30s have passed since the last cleanup_table() sweep + // (organic, forced, or one run by this method), and (b) at least one GC + // has happened since then (gcEpoch() != _last_gc_epoch) - so this never + // does a pointless sweep of an unchanged table. + void maybeForceCleanup(u64 now_ns); + static void JNICALL GarbageCollectionFinish(jvmtiEnv *jvmti_env); + // Test seams - not part of the production API. Mirrors + // NativeSocketSampler's own "for testing only" accessors + // (nativeSocketSampler.h's fdAddrCacheSizeForTest()/ + // fdAddrCacheInsertForTest()) rather than befriending the test binary. + // These only exercise the JNI-free ring/eviction mechanics + // (recordKlassPopulationSampleLocked() takes no JNIEnv), never + // foldKlassCountsLocked()'s representative-minting step, which needs a + // live JVM and is therefore out of gtest's reach. + int klassPopulationSizeForTest() const { return _klass_population_size; } + + // Runs foldKlassCountsLocked()'s JNI-free table work only (the zero-sample + // pass over _klass_population): with an empty _klass_count_scratch and + // env == nullptr no representative-minting JNI can run, so the zero-sample + // pass for klasses absent from this epoch's fold is what gets exercised. + void foldKlassCountsZeroSampleForTest(u64 epoch) { + _table_lock.lock(); + foldKlassCountsLocked(nullptr, epoch, /*allow_resolve=*/false); + _table_lock.unlock(); + } + + // Leak-tag pool test seams - same "for testing only" rationale as the + // klass-population seams above: the pool acquire/release/info mechanics + // are JNI-free pure logic, so they are directly testable; only + // tagLeakInstances() itself needs a live JVM (SetTag/NewLocalRef) and stays + // out of gtest's reach. + void leakTagPoolResetForTest() { + for (int i = 0; i < LEAK_TAG_POOL_SIZE; i++) { + _leak_tag_free_list[i] = i; + _leak_tag_info[i].call_trace_id = 0; + _leak_tag_info[i].tid = 0; + } + _leak_tag_free_count = LEAK_TAG_POOL_SIZE; + } + + jlong acquireLeakTagForTest(u64 call_trace_id, jint tid) { + return acquireLeakTag(call_trace_id, tid); + } + + void releaseLeakTagForTest(jlong tag) { releaseLeakTag(tag); } + + int leakTagFreeCountForTest() const { return _leak_tag_free_count; } + + // Admission-boost test seams - same "for testing only" rationale as the + // leak-tag pool seams above: admitForTracking()/noteSelectedCandidates() + // are JNI-free pure logic (atomic reads + the per-thread RNG draw), so they + // are directly testable; only track() beyond the gate needs a live JNIEnv + // (NewWeakGlobalRef) and stays out of gtest's reach. + bool admitForTrackingForTest(jint tid) { return admitForTracking(tid); } + + void setSubsampleRatioForTest(double ratio) { _subsample = SubsampleRate(ratio); } + + // Reset for tests: clears both boost paths, forces ratio=0 (deterministic + // reject for unboosted tids - xorshift::threshold(0) is 0, which no draw + // compares <, so a ratio of 0 can never admit), and returns this thread's + // rng ThreadLocal to the unseeded sentinel so the next admitForTracking() + // draw comes from a freshly-seeded stream. + void admissionResetForTest(); + + int watchedTidCountForTest() const { + return __atomic_load_n(&_watched_tid_count, __ATOMIC_ACQUIRE); + } + + jint watchedTidForTest(int i) const { + return __atomic_load_n(&_watched_tids[i], __ATOMIC_RELAXED); + } + + static jlong leakTagBaseForTest() { return LEAK_TAG_BASE; } + + static int leakTagPoolSizeForTest() { return LEAK_TAG_POOL_SIZE; } + bool klassPopulationLookupForTest(u32 klass_id, KlassPopulationEntry *out) const { + for (int i = 0; i < _klass_population_size; i++) { + if (_klass_population[i].klass_id == klass_id) { + *out = _klass_population[i]; + return true; + } + } + return false; + } + // recordKlassPopulationSampleLocked()'s own precondition is "_table_lock is + // held (by cleanup_table(), the only production caller)" - this seam is + // called from a Java/test thread while the BFS thread + // (ReferenceChainTracker::threadLoop()) may concurrently be inside + // cleanup_table()'s epoch-advance pass, which holds _table_lock while + // mutating the very same _klass_population/_klass_population_size fields + // (including its class-map-generation-reset branch, which can wipe the + // whole table). Without taking the lock here too, a seeded sample could + // race that wipe and silently vanish moments after being recorded. Mirrors + // klassPopulationSetRepresentativeForTest() below, which already does this + // correctly. + // TEST-ONLY synthetic-to-real klass-id aliasing. Since the leak-tag pool + // redesign, every consumer of a candidate klass id keys on the REAL id + // space of Profiler::lookupClass()/ReferenceChainTracker's class tags: + // tagLeakInstances() scans the live-heap tracking table's cached_klass_id, + // and discovered-instance recording resolves an admitted object's class + // to the same space. A purely synthetic seeded id matches none of those + // (observed live: tagged=0 for every poll, "resolved but no candidate + // match" for every auto-mark, no chains ever built). The scenarios' debug + // seams therefore alias each synthetic id to the representative's real + // class id, established by klassPopulationSetRepresentativeForTest() + // below (the only seam holding an actual instance) and applied by + // klassPopulationRecordForTest() above. + struct TestKlassAlias { + u32 synthetic; + u32 real; + }; + static constexpr int MAX_TEST_KLASS_ALIASES = 8; + TestKlassAlias _test_klass_aliases[MAX_TEST_KLASS_ALIASES]; + int _test_klass_alias_count = 0; + + // All three below require _table_lock held (same as every other + // _klass_population mutator). + u32 resolveTestKlassAliasLocked(u32 klass_id) const { + for (int i = 0; i < _test_klass_alias_count; i++) { + if (_test_klass_aliases[i].synthetic == klass_id) { + return _test_klass_aliases[i].real; + } + } + return klass_id; + } + + void registerTestKlassAliasLocked(u32 synthetic, u32 real) { + for (int i = 0; i < _test_klass_alias_count; i++) { + if (_test_klass_aliases[i].synthetic == synthetic) { + _test_klass_aliases[i].real = real; + return; + } + } + if (_test_klass_alias_count < MAX_TEST_KLASS_ALIASES) { + _test_klass_aliases[_test_klass_alias_count++] = {synthetic, real}; + } + } + + void removeKlassPopulationEntryLocked(int slot) { + if (slot < 0 || slot >= _klass_population_size) { + return; + } + memmove(&_klass_population[slot], &_klass_population[slot + 1], + sizeof(KlassPopulationEntry) * + (size_t)(_klass_population_size - slot - 1)); + _klass_population_size--; + } + + jweak klassPopulationRecordForTest(u32 klass_id, u32 count, u64 epoch, + int *out_slot, bool *out_created, + jweak *out_evicted = nullptr, + int *out_evicted_count = nullptr, + int max_evicted = 0) { + _table_lock.lock(); + klass_id = resolveTestKlassAliasLocked(klass_id); + jweak evicted = recordKlassPopulationSampleLocked(klass_id, count, epoch, + out_slot, out_created, + out_evicted, + out_evicted_count, + max_evicted); + _table_lock.unlock(); + return evicted; + } + + // Seeds one per-tid trend sample for tests/scenarios: pushes `count` as + // tid's epoch sample on klass_id's KlassPopulationEntry, marking the + // trend SYNTHETIC (exempt from the real fold's absent-tid decay and slot + // eviction - see TidTrend::synthetic's own comment) so a scenario's ramp + // survives the interleaved real GC folds that the same test's + // System.gc() churn triggers. The tid MUST be the allocating thread's real + // profiler tid (ProfiledThread::currentTid()'s space - + // JavaProfiler.getTid() on the leaking thread) whenever the test also + // relies on tagLeakInstances() tagging real tracked instances: the + // production tagging scope is exactly the qualifying-tid set, and a fake + // tid matches no tracked instance. Creates the klass entry if absent + // (count-0/epoch-0 first sample, same creation branch + // klassPopulationSetRepresentativeForTest() relies on) and resolves the + // synthetic-id aliasing exactly like klassPopulationRecordForTest() + // above. Out of gtest's reach in one respect: gtest call sites have no + // live tracked instances to tag anyway (they exercise the qualification + // gate only), so a distinct gtest-chosen tid is fine there. + void tidTrendRecordForTest(u32 klass_id, jint tid, u32 count, u64 epoch) { + _table_lock.lock(); + klass_id = resolveTestKlassAliasLocked(klass_id); + int slot = -1; + for (int i = 0; i < _klass_population_size; i++) { + if (_klass_population[i].klass_id == klass_id) { + slot = i; + break; + } + } + if (slot < 0) { + int out_slot; + bool created; + recordKlassPopulationSampleLocked(klass_id, 0, 0, &out_slot, &created); + slot = out_slot; + } + KlassPopulationEntry &entry = _klass_population[slot]; + KlassPopulationEntry::TidTrend *trend = nullptr; + for (int i = 0; i < entry.tid_trend_count; i++) { + if (entry.tid_trends[i].tid == tid) { + trend = &entry.tid_trends[i]; + break; + } + } + if (trend == nullptr && entry.tid_trend_count < + KlassPopulationEntry::MAX_TID_TRENDS) { + trend = &entry.tid_trends[entry.tid_trend_count++]; + trend->tid = tid; + trend->ring_head = 0; + trend->ring_fill = 0; + trend->consecutive_positive = 0; + } + if (trend != nullptr) { + trend->synthetic = true; + // The one seeded value lands in BOTH the age ring and the retained- + // count ring, so a test can qualify a tid either way the production + // gate does: a rising small-value ramp exercises the age-trend + // discriminator, while one flat value over TID_RETAINED_COUNT_BAR + // exercises the retained-count bar (the one-cohort-per-thread + // accumulation shape - see that constant's own comment). + trend->ring[trend->ring_head] = (u8)count; + trend->count_ring[trend->ring_head] = (u8)count; + trend->ring_head = (u8)((trend->ring_head + 1) % + KlassPopulationEntry::TID_TREND_RING_SIZE); + if (trend->ring_fill < KlassPopulationEntry::TID_TREND_RING_SIZE) { + trend->ring_fill++; + } + if (tidPushQualifies(*trend, count)) { + if (trend->consecutive_positive < UINT8_MAX) { + trend->consecutive_positive++; + } + } else { + trend->consecutive_positive = 0; + } + (void)epoch; // the per-tid ring is push-ordered like the klass ring; + // recordKlassPopulationSampleLocked()'s own callers + // likewise only use epoch for last_updated_epoch LRU + } + _table_lock.unlock(); + } + // Sets an entry's representative directly - production code only ever + // does this via foldKlassCountsLocked()'s JNI-dependent minting step + // (out of gtest's reach, see the class comment above), so tests use this + // seam instead to set up a fake representative and assert it comes back + // out of recordKlassPopulationSampleLocked() as the evicted jweak when + // that entry is later LRU-evicted. No-op if klass_id is not present. + // Also called from a live-JVM test (not just gtest) while the BFS thread + // (ReferenceChainTracker::threadLoop()) may concurrently be inside + // cleanup_table()'s epoch-advance pass, which holds _table_lock while + // mutating _klass_population/_klass_population_size - so this seam takes + // the same lock rather than writing the field unguarded (mirrors + // klassPopulationResetForTest() immediately below). Deletes any previous + // representative via DeleteWeakGlobalRef() before overwriting, the same + // way foldKlassCountsLocked() handles a stale representative on eviction - + // otherwise repeated calls for the same klass_id leak a JNI weak global + // ref per call. + void klassPopulationSetRepresentativeForTest(JNIEnv *env, u32 klass_id, jweak rep) { + // ALIAS RESOLUTION (see _test_klass_aliases' own comment above): resolve + // the representative's real klass id BEFORE taking _table_lock, so the + // Class.getName() JNI upcall never runs under the lock. Needs a live JVM + // (a valid Class.getName methodID and a resolvable representative); in + // gtest (env == nullptr or _Class_getName == 0) the alias is skipped and + // the old synthetic-only behavior applies. + u32 real_id = klass_id; + jobject strong = nullptr; + if (env != nullptr && rep != nullptr && _Class_getName != nullptr) { + strong = env->NewLocalRef(rep); + if (strong != nullptr) { + u32 resolved = resolveKlassId(env, strong); + if (resolved != 0 && resolved != klass_id) { + real_id = resolved; + } + } + } + _table_lock.lock(); + if (real_id != klass_id) { + registerTestKlassAliasLocked(klass_id, real_id); + // Re-key any already-seeded synthetic entry to the real id so its + // ring history (hysteresis ramp included) survives the alias. If a + // real-keyed entry already exists too (real allocation sampling + // already folded genuine samples for this same class), the real + // entry wins and the synthetic one is dropped: genuine history for + // the same class outranks seeded history. + int syn_slot = -1; + int real_slot = -1; + for (int i = 0; i < _klass_population_size; i++) { + if (_klass_population[i].klass_id == klass_id) { + syn_slot = i; + } else if (_klass_population[i].klass_id == real_id) { + real_slot = i; + } + } + if (syn_slot >= 0 && real_slot < 0) { + _klass_population[syn_slot].klass_id = real_id; + } else if (syn_slot >= 0 && real_slot >= 0) { + removeKlassPopulationEntryLocked(syn_slot); + } + } + // Find-or-create the entry under real_id (previously a silent no-op + // when absent - but with aliasing, set-representative-first is the + // load-bearing order: scenarios must establish the alias BEFORE any + // seeding, and with no entry yet there is nothing to store the + // representative into. Creating via a count-0, epoch-0 ring sample + // reuses recordKlassPopulationSampleLocked()'s own creation branch + // exactly; the seeded rising ramp that follows still clears + // hasQualifyingGrowth() (a single 0 at the ring's start only lowers the + // regression start value, which RAISES the slope). + int slot = -1; + for (int i = 0; i < _klass_population_size; i++) { + if (_klass_population[i].klass_id == real_id) { + slot = i; + break; + } + } + if (slot < 0) { + jweak evicted[KlassPopulationEntry::MAX_REPRESENTATIVES_PER_KLASS]; + int evicted_count = 0; + bool created = false; + recordKlassPopulationSampleLocked(real_id, 0, 0, &slot, &created, + evicted, &evicted_count, + KlassPopulationEntry::MAX_REPRESENTATIVES_PER_KLASS); + if (env != nullptr) { + for (int r = 0; r < evicted_count; r++) { + if (evicted[r] != nullptr) { + env->DeleteWeakGlobalRef(evicted[r]); + } + } + } + } + if (slot >= 0) { + // Clean up old representatives + for (int r = 0; r < _klass_population[slot].representative_count; r++) { + jweak prev = _klass_population[slot].representatives[r]; + if (prev != nullptr && env != nullptr) { + env->DeleteWeakGlobalRef(prev); + } + } + _klass_population[slot].representative_count = 0; + if (rep != nullptr) { + _klass_population[slot].representatives[0] = rep; + _klass_population[slot].representative_count = 1; + } + // Mint stable_class_tag from this same representative if this slot + // has not gotten one yet - this seam is how live-JVM, + // StaticFieldGrowingCollectionScenario-style tests seed a candidate + // instead of real allocation sampling (foldKlassCountsLocked()), + // which would otherwise never run for them, leaving + // stable_class_tag permanently unminted and + // topKlassesByGenerationCount() unable to report this klass at all - + // found the hard way, exactly the failure this comment is warning + // about. + // env is nullptr in some gtest call sites that only exercise the + // representative-swap bookkeeping above and don't care about + // stable_class_tag - guard against it rather than crash. + if (strong != nullptr) { + mintStableClassTagIfNeeded(env, slot, strong); + } + } + _table_lock.unlock(); + if (strong != nullptr) { + env->DeleteLocalRef(strong); + } + } + // Sets an entry's stable_class_tag directly - production code only ever + // mints this via foldKlassCountsLocked()'s JVMTI-dependent get-or-assign + // step (out of gtest's reach, same rationale as + // klassPopulationSetRepresentativeForTest() above), so tests that exercise + // topKlassesByGenerationCount() via klassPopulationRecordForTest() (which + // bypasses foldKlassCountsLocked() entirely) need this seam - without it, + // stable_class_tag would stay 0 (never minted) and + // topKlassesByGenerationCount() would skip every entry. No-op if klass_id + // is not present. + void klassPopulationSetStableClassTagForTest(u32 klass_id, jlong tag) { + _table_lock.lock(); + for (int i = 0; i < _klass_population_size; i++) { + if (_klass_population[i].klass_id == klass_id) { + _klass_population[i].stable_class_tag = tag; + break; + } + } + _table_lock.unlock(); + } + // Unlike the other klassPopulation*ForTest() seams above, this one is + // also called from a live-JVM test (not just gtest) while the BFS thread + // (ReferenceChainTracker::threadLoop()) may concurrently be inside + // cleanup_table()'s epoch-advance pass, which holds _table_lock while + // mutating _klass_population_size/_klass_population - so this seam must + // take the same lock rather than writing the field unguarded. + // Also clears _klass_count_scratch_size: a stale scratch holding a full + // epoch's worth of real klass entries survives this reset otherwise, and + // the first real GC after it folds all of them into the just-cleared + // table, filling it to MAX_KLASS_POPULATION_ENTRIES in one fold. Every + // seeded test entry then carries a synthetic last_updated_epoch (1..20) + // far below the real GC epochs of the folded entries, so it is the + // permanent LRU-eviction victim: a fold landing mid-seeding (real GCs + // fire every few ms on slow runners) evicts the entry and each later + // seed push re-creates it with a reset ring, leaving ring_fill below + // KLASS_POPULATION_MIN_FILL_FOR_TREND at select time - the observed + // shouldSelectSeededKlassAsLeakCandidateOnPositiveSlope flake on + // musl-aarch64. With the scratch cleared at reset, the table can only + // fill from allocations made during the test's own microsecond-scale + // body, which never reaches the eviction threshold. + void klassPopulationResetForTest() { + _table_lock.lock(); + _klass_population_size = 0; + _klass_count_scratch_size = 0; + _test_klass_alias_count = 0; + _table_lock.unlock(); + // Also reset the heap-floor ring: it is a sibling piece of the same + // _gc_generations-gated feature, read by every selectLeakCandidates() + // scan (heapFloorRising()), so leaving it populated across tests in the + // same gtest binary would leak one test's heap-usage history into the + // next test's hysteresis threshold. storeRelease() (not plain store()) + // so the reset is published to the BFS/GC thread on arm64's weak memory + // model - a relaxed store may never be visible to a reader doing + // loadAcquire() on these same indices. + storeRelease(_heap_floor_ring_head, (u8)0); + storeRelease(_heap_floor_ring_fill, (u8)0); + } + + // Test seams for the heap-floor ring (mirrors klassPopulation*ForTest()'s + // own seams immediately above) - lock-free, see _heap_floor_ring's own + // comment, so no locking wrapper is needed here either. timestamp_ns + // defaults to 0 for existing callers that only exercise + // heapFloorRising()/heapFloorRisingForTest() (which never reads the time + // ring) - a test exercising secondsToOOM() must pass real, increasing + // values explicitly. + // container_used defaults to 0 for existing callers that only exercise + // heapFloorRising()/heapFloorRisingForTest() (which never reads + // _container_mem_ring) - a test exercising secondsToOOM()'s container + // boundary must pass real, increasing values explicitly. + void heapFloorRecordForTest(u64 used, u64 timestamp_ns = 0, u64 container_used = 0) { + recordHeapFloorSampleUnchecked(used, timestamp_ns, container_used); + } + bool heapFloorRisingForTest() const { return heapFloorRising(); } + // Bypasses initialize_table()'s JNI-dependent HeapUsage::getMaxHeap() call + // (out of gtest's reach, same reason setGcGenerationsForTest() exists) so + // secondsToOOM() can be exercised directly against a fake max heap size. + void setMaxHeapBytesForTest(jlong v) { +#ifdef DEBUG + _max_heap_bytes_for_test.store(v, std::memory_order_release); +#else + _max_heap_bytes = v; +#endif + } + + // Mirrors setMaxHeapBytesForTest() above, for secondsToOOM()'s container + // boundary - bypasses initialize_table()'s real OS::getContainerMemoryLimit() + // call so a test can exercise the container-vs-heap comparison with a + // fake, deterministic limit. + void setContainerMemoryLimitForTest(jlong v) { +#ifdef DEBUG + _container_memory_limit_for_test.store(v, std::memory_order_release); +#else + _container_memory_limit = v; +#endif + } + +#ifdef DEBUG + // Temporarily disables onGC()'s own recordHeapFloorSample() call so a + // test can seed the ring exclusively via heapFloorRecordForTest() without + // a real GC interleaving a sample. See _heap_floor_recording_disabled_for_test's + // own comment above. + void setHeapFloorRecordingForTest(bool enabled) { + _heap_floor_recording_disabled_for_test.store(!enabled, std::memory_order_release); + } +#endif + + // Sets _gc_generations directly, bypassing initialize() (which requires a + // live JVM - VM::hotspot_version()/VM::jni(), see that method's own code - + // out of gtest's reach the same way foldKlassCountsLocked()'s + // representative-minting step is, per this seam block's own comment + // above). Callers outside this class that only need to exercise + // gcGenerationsEnabled()'s gate (e.g. referenceChains_ut.cpp's + // pollWatchedTargets() tests) use this instead of standing up a full + // initialize()/start() call. + void setGcGenerationsForTest(bool v) { + _gc_generations.store(v, std::memory_order_relaxed); + } + private: void getLiveTraceIds(CallTraceIdSet& out_buffer); }; diff --git a/ddprof-lib/src/main/cpp/os.h b/ddprof-lib/src/main/cpp/os.h index a3d028381..ee3a74b86 100644 --- a/ddprof-lib/src/main/cpp/os.h +++ b/ddprof-lib/src/main/cpp/os.h @@ -212,6 +212,7 @@ class OS { static int getCpuCount(); static int getCgroupCpuMillicores(); static long getContainerMemoryLimit(); + static long getContainerMemoryUsage(); static u64 getProcessCpuTime(u64* utime, u64* stime); static u64 getTotalCpuTime(u64* utime, u64* stime); diff --git a/ddprof-lib/src/main/cpp/os_linux.cpp b/ddprof-lib/src/main/cpp/os_linux.cpp index 874085c7a..daabc8f5b 100644 --- a/ddprof-lib/src/main/cpp/os_linux.cpp +++ b/ddprof-lib/src/main/cpp/os_linux.cpp @@ -910,7 +910,23 @@ int OS::getCgroupCpuMillicores() { // Applies the smallest (most restrictive) memory.max found across this // process's cgroup v2 group and all of its ancestors up to the mount root. -static long walkCgroupV2MemoryLimit(char* path) { +// When a limit wins, the cgroup it came from is remembered in +// *winner_path_out: getContainerMemoryUsage() must read memory.current from +// that same cgroup, because an ancestor-level limit also covers sibling +// cgroups whose usage the leaf's memory.current excludes - pairing the +// ancestor limit with leaf usage would overstate the available memory and +// delay the OOM projection. +static char g_memory_limit_cgroup_path[PATH_MAX] = {0}; + +// Which cgroup hierarchy (v2 or v1) supplied the winning limit recorded in +// g_memory_limit_cgroup_path: the usage file must be read from the SAME +// hierarchy (v2 memory.current vs v1 memory.usage_in_bytes), not probed by +// filename order - on a hybrid system both controller files can be visible +// for the same cgroup dir, and reading the wrong one pairs the limit with an +// unrelated usage number. +static bool g_memory_limit_cgroup_v2 = true; + +static long walkCgroupV2MemoryLimit(char* path, char* winner_path_out) { size_t base_len = strlen("/sys/fs/cgroup"); long best = -1; for (;;) { @@ -925,6 +941,9 @@ static long walkCgroupV2MemoryLimit(char* path) { long limit = atol(buf); if (limit > 0 && (best < 0 || limit < best)) { best = limit; + if (winner_path_out != nullptr) { + snprintf(winner_path_out, PATH_MAX, "%s", path); + } } } } @@ -937,8 +956,9 @@ static long walkCgroupV2MemoryLimit(char* path) { } // Walks ancestors the same way as walkCgroupV2MemoryLimit(), but reads the -// cgroup v1 memory controller's limit file instead. -static long walkCgroupV1MemoryLimit(char* path) { +// cgroup v1 memory controller's limit file instead. See the v2 walk's +// comment for the winner-path bookkeeping. +static long walkCgroupV1MemoryLimit(char* path, char* winner_path_out) { size_t base_len = strlen("/sys/fs/cgroup/memory"); long best = -1; for (;;) { @@ -954,6 +974,9 @@ static long walkCgroupV1MemoryLimit(char* path) { // A limit of 9223372036854771712 (LLONG_MAX rounded) means unconstrained. if (limit > 0 && limit < 0x7ffffffffffff000L && (best < 0 || limit < best)) { best = limit; + if (winner_path_out != nullptr) { + snprintf(winner_path_out, PATH_MAX, "%s", path); + } } } } @@ -969,6 +992,11 @@ long OS::getContainerMemoryLimit() { char subpath[PATH_MAX]; char path[PATH_MAX]; + // Recomputed on every call; getContainerMemoryUsage() pairs its usage + // read with whatever path won here (see the winner-path comment on + // walkCgroupV2MemoryLimit()). + g_memory_limit_cgroup_path[0] = '\0'; + // Try cgroup v2 first, resolved from this process's own cgroup path. if (getOwnCgroupPath("", subpath, sizeof(subpath))) { size_t base_len = strlen("/sys/fs/cgroup"); @@ -982,7 +1010,8 @@ long OS::getContainerMemoryLimit() { int fd = open(leaf, O_RDONLY); if (fd != -1) { close(fd); - return walkCgroupV2MemoryLimit(path); + g_memory_limit_cgroup_v2 = true; + return walkCgroupV2MemoryLimit(path, g_memory_limit_cgroup_path); } } } @@ -1002,7 +1031,101 @@ long OS::getContainerMemoryLimit() { int fd = open(leaf, O_RDONLY); if (fd != -1) { close(fd); - return walkCgroupV1MemoryLimit(path); + g_memory_limit_cgroup_v2 = false; + return walkCgroupV1MemoryLimit(path, g_memory_limit_cgroup_path); + } + } + } + } + + return -1; +} + +// Reads the current usage from the same cgroup level that supplied +// getContainerMemoryLimit()'s winning limit when one was recorded - an +// ancestor-level limit also covers sibling cgroups, whose usage the leaf's +// memory.current excludes, so pairing an ancestor limit with leaf usage +// would overstate the available memory and delay the OOM projection. The +// leaf's own memory.current DOES count everything charged to it (including +// its descendants), so the ancestor read only matters for sibling-inclusive +// totals. +long OS::getContainerMemoryUsage() { + char subpath[PATH_MAX]; + char path[PATH_MAX]; + + // Same cgroup the winning limit came from, if the limit walk recorded + // one - read its usage first, falling back to the process's own leaf. + // The usage filename follows the hierarchy that supplied the limit (the + // recorded winner's hierarchy is authoritative, not filename order). + if (g_memory_limit_cgroup_path[0] != '\0') { + char file[PATH_MAX]; + const char *fmt = g_memory_limit_cgroup_v2 ? "%s/memory.current" + : "%s/memory.usage_in_bytes"; + if ((size_t)snprintf(file, sizeof(file), fmt, + g_memory_limit_cgroup_path) < sizeof(file)) { + int fd = open(file, O_RDONLY); + if (fd != -1) { + char buf[32] = {0}; + ssize_t r = read(fd, buf, sizeof(buf) - 1); + close(fd); + if (r > 0) { + long usage = atol(buf); + if (usage >= 0) { + return usage; + } + } + } + } + } + + // Try cgroup v2 first, resolved from this process's own cgroup path. + if (getOwnCgroupPath("", subpath, sizeof(subpath))) { + size_t base_len = strlen("/sys/fs/cgroup"); + size_t sub_len = strlen(subpath); + if (base_len + sub_len < sizeof(path)) { + memcpy(path, "/sys/fs/cgroup", base_len); + memcpy(path + base_len, subpath, sub_len + 1); + + char leaf[PATH_MAX]; + if ((size_t)snprintf(leaf, sizeof(leaf), "%s/memory.current", path) < sizeof(leaf)) { + int fd = open(leaf, O_RDONLY); + if (fd != -1) { + char buf[32] = {0}; + ssize_t r = read(fd, buf, sizeof(buf) - 1); + close(fd); + if (r > 0) { + long usage = atol(buf); + if (usage >= 0) { + return usage; + } + } + } + } + } + } + + // Fall back to cgroup v1, likewise resolved from the process's own path. + if (getOwnCgroupPath("memory", subpath, sizeof(subpath))) { + const char* base = "/sys/fs/cgroup/memory"; + size_t base_len = strlen(base); + size_t sub_len = strlen(subpath); + if (base_len + sub_len < sizeof(path)) { + memcpy(path, base, base_len); + memcpy(path + base_len, subpath, sub_len + 1); + + char leaf[PATH_MAX]; + if ((size_t)snprintf(leaf, sizeof(leaf), "%s/memory.usage_in_bytes", path) < sizeof(leaf)) { + int fd = open(leaf, O_RDONLY); + if (fd != -1) { + char buf[32] = {0}; + ssize_t r = read(fd, buf, sizeof(buf) - 1); + close(fd); + if (r > 0) { + long usage = atol(buf); + if (usage >= 0) { + return usage; + } + } } } } @@ -1046,9 +1169,12 @@ int OS::createMemoryFile(const char* name) { void OS::copyFile(int src_fd, int dst_fd, off_t offset, size_t size) { // copy_file_range() is probably better, but not supported on all kernels + size_t requested = size; while (size > 0) { ssize_t bytes = sendfile(dst_fd, src_fd, &offset, size); if (bytes <= 0) { + TEST_LOG("OS::copyFile sendfile returned %zd, errno=%d, remaining=%zu of requested=%zu", + bytes, errno, size, requested); break; } size -= (size_t)bytes; diff --git a/ddprof-lib/src/main/cpp/os_macos.cpp b/ddprof-lib/src/main/cpp/os_macos.cpp index 805a11b84..3d3aa8ef6 100644 --- a/ddprof-lib/src/main/cpp/os_macos.cpp +++ b/ddprof-lib/src/main/cpp/os_macos.cpp @@ -384,6 +384,10 @@ long OS::getContainerMemoryLimit() { return -1; // macOS has no cgroup support. } +long OS::getContainerMemoryUsage() { + return -1; // macOS has no cgroup support. +} + u64 OS::getProcessCpuTime(u64* utime, u64* stime) { struct tms buf; clock_t real = times(&buf); diff --git a/ddprof-lib/src/main/cpp/painBudget.h b/ddprof-lib/src/main/cpp/painBudget.h new file mode 100644 index 000000000..426ec4c1d --- /dev/null +++ b/ddprof-lib/src/main/cpp/painBudget.h @@ -0,0 +1,101 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _PAINBUDGET_H +#define _PAINBUDGET_H + +#include "arch.h" + +/* + * A leaky bucket over *cost* (milliseconds of expensive work already spent), + * not over an event *rate* - unlike PidController/RateLimiter (which target + * a steady events-per-second throughput), this answers "have we spent too + * much recently to justify doing more expensive work right now?" + * + * Typical use: a subsystem that occasionally does one genuinely expensive, + * bounded operation (here: a full-heap BFS pass) wants to avoid doing that + * operation back-to-back if it keeps being expensive, while still allowing + * it immediately again if the last one was cheap. spend() records how much + * an operation cost; canStartNow() drains the balance by however much + * wall-clock time has passed (at _refill_rate) and reports whether the + * debt has cleared. + * + * _refill_rate is the one tunable: the fraction of wall-clock time this + * budget is willing to let its owner spend on the expensive operation, on + * average (e.g. 0.01 = "at most ~1% of wall-clock time, averaged over + * time"). Unlike PidController's gain triples (P/I/D), this single ratio + * has a direct, human-interpretable meaning and needs no derivation beyond + * picking that target fraction. + */ +class PainBudget { +private: + double _balance_ms; // accumulated debt in ms; 0 means "clear to spend" + double _refill_rate; // fraction of wall-clock time allowed, e.g. 0.01 + u64 _last_update_ns; // OS::nanotime() as of the last drain(); 0 = never drained yet + + void drain(u64 now_ns) { + if (_last_update_ns == 0) { + // First call ever - nothing to drain yet, just establish the baseline. + _last_update_ns = now_ns; + return; + } + if (now_ns <= _last_update_ns) { + // Clock stepped backward (or a test passed a literal smaller than the + // OS::nanotime()-seeded baseline): the unsigned subtraction would wrap + // to ~2^64 ns and drain the entire debt in one call. The failure + // direction is permissive (a blocked restart becomes allowed), so clamp + // to zero elapsed and re-baseline instead. + _last_update_ns = now_ns; + return; + } + u64 elapsed_ns = now_ns - _last_update_ns; + double elapsed_ms = (double)elapsed_ns / 1000000.0; + // _refill_rate == 0.0 (the default constructor argument) makes this a + // no-op forever: the balance never drains, so once spend() has pushed it + // above 0 canStartNow() stays false permanently. Callers that want the + // budget to actually refill must pass a positive _refill_rate. + _balance_ms -= elapsed_ms * _refill_rate; + if (_balance_ms < 0) { + _balance_ms = 0; + } + _last_update_ns = now_ns; + } + +public: + explicit PainBudget(double refill_rate = 0.0) + : _balance_ms(0), _refill_rate(refill_rate), _last_update_ns(0) {} + + // Records that an operation just cost `pain_ms` milliseconds of + // wall-clock time. Does not drain first - the cost is added on top of + // whatever debt (already correctly drained as of the last canStartNow() + // call) currently exists. + void spend(u64 pain_ms) { _balance_ms += (double)pain_ms; } + + // True once the debt has drained back to zero at _refill_rate - i.e. it + // is now affordable, on average, to spend more pain. Drains the balance + // as a side effect, so repeated calls correctly reflect elapsed time + // even if spend() is never called again. + bool canStartNow(u64 now_ns) { + drain(now_ns); + return _balance_ms <= 0; + } + + // Test/introspection only - current debt after draining as of now_ns. + double balanceMs(u64 now_ns) { + drain(now_ns); + return _balance_ms; + } + + // Changes the refill rate without resetting accumulated debt - unlike + // assigning a freshly-constructed PainBudget(rate), which would zero + // _balance_ms. Drains at the *old* rate up to now_ns first, so the rate + // change only affects time elapsed after this call. + void setRefillRate(double refill_rate, u64 now_ns) { + drain(now_ns); + _refill_rate = refill_rate; + } +}; + +#endif // _PAINBUDGET_H diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index a3cf0aaf4..3d1472a64 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -31,6 +31,7 @@ #include "objectSampler.h" #include "os.h" #include "perfEvents.h" +#include "referenceChains.h" #include "safeAccess.h" #include "samplerPerf.h" #include "stackFrame.h" @@ -1702,6 +1703,26 @@ Error Profiler::start(Arguments &args, bool reset) { } } } + if ((activated & EM_ALLOC) && args._reference_chains) { + // Reference-chain tracking chases LivenessTracker's leak-tagged + // candidates, so it only runs when allocation sampling actually + // activated. Must run AFTER ObjectSampler::start() -> + // LivenessTracker::start() (ordering noted in referenceChains.cpp) - + // the block above is that ordering point. + error = ReferenceChainTracker::instance()->start(args); + if (error) { + Log::warn("%s", error.message()); + error = Error::OK; // recoverable - recording continues without chains + } else { + ReferenceChainTracker::instance()->startThread(); + // Pre-existing threads must be registered from the profiler lifecycle + // (registerExistingThreads()'s own comment): Profiler::onThreadStart() + // only sees threads started after the recording began. + ReferenceChainTracker::instance()->registerExistingThreads(VM::jvmti(), + VM::jni()); + _reference_chains_active = true; + } + } if (_event_mask & EM_NATIVEMEM) { error = malloc_tracer.start(args); if (error) { @@ -1791,6 +1812,15 @@ Error Profiler::stop() { if (_event_mask & EM_ALLOC) _alloc_engine->stop(); + if (_reference_chains_active) { + // Join the BFS thread and clear the recording-boundary state before the + // rest of the teardown (stopThread() wakes and joins; stop() resets the + // per-recording caches). Matches the Profiler::stop() order documented + // in referenceChains.cpp's stopThread()/stop() comments. + ReferenceChainTracker::instance()->stopThread(); + ReferenceChainTracker::instance()->stop(); + _reference_chains_active = false; + } if (_event_mask & EM_NATIVEMEM) malloc_tracer.stop(); // Stop the refresher BEFORE socket unpatch: the refresher calls @@ -1952,6 +1982,31 @@ Error Profiler::dump(const char *path, const int length) { // by the live objects LivenessTracker::instance()->flush(thread_ids); + // Emit the reference-chain tracker's pending events into this dumping + // chunk: chain events are snapshot-and-kept (re-emitted into every chunk + // while the sample stays live), abandonment events are a true drain. + // Runs before rotateDictsAndRun() so the events land inside the chunk + // being written, and under a profiler lock like every other + // recording-buffer writer (dump runs on a normal thread holding only + // _state_lock; the _state_lock -> _locks order is the codebase's). + { + int dump_tid = ProfiledThread::currentTid(); + u32 lock_index = getLockIndex(dump_tid >= 0 ? dump_tid : 0); + _locks[lock_index].lock(); + std::vector chain_events; + ReferenceChainTracker::instance()->drainPendingChainEvents(&chain_events); + for (auto &event : chain_events) { + _jfr.recordReferenceChain(lock_index, &event); + } + std::vector abandoned_events; + ReferenceChainTracker::instance()->drainPendingAbandonedEvents( + &abandoned_events); + for (auto &event : abandoned_events) { + _jfr.recordReferenceChainAbandoned(lock_index, &event); + } + _locks[lock_index].unlock(); + } + Libraries::instance()->refresh(); updateJavaThreadNames(); updateNativeThreadNames(); diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index 722796035..c679009e7 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -133,6 +133,12 @@ class alignas(alignof(SpinLock)) Profiler { alignas(DEFAULT_CACHE_LINE_SIZE) volatile u64 _sample_seq; alignas(DEFAULT_CACHE_LINE_SIZE) u64 _failures[ASGCT_FAILURE_TYPES]; bool _wall_precheck = false; + // True between a Profiler::start() that activated the reference-chain + // tracker and the matching stop(): gates stopThread()/stop() in + // Profiler::stop() so a recording started without the tracker never tears + // it down (the tracker is only started when allocation sampling activated + // AND referencechains=true - see Profiler::start()). + bool _reference_chains_active = false; SpinLock _class_map_lock; SpinLock _locks[CONCURRENCY_LEVEL]; diff --git a/ddprof-lib/src/main/cpp/rcDebugLevel.h b/ddprof-lib/src/main/cpp/rcDebugLevel.h new file mode 100644 index 000000000..204259062 --- /dev/null +++ b/ddprof-lib/src/main/cpp/rcDebugLevel.h @@ -0,0 +1,65 @@ +#ifndef _RC_DEBUG_LEVEL_H +#define _RC_DEBUG_LEVEL_H + +// Runtime level gate for the reference-chains subsystem's TEST_LOG +// diagnostics (referenceChains.cpp + livenessTracker.cpp). Included +// AFTER common.h, this re-points THIS translation unit's TEST_LOG at a +// level check and adds TEST_LOG_SUMMARY: +// +// level 0 silent (the default - keeps DEBUG builds pod-safe) +// level 1 lifecycle/summary: state-machine transitions, per-pass and +// per-poll outcomes (candidates, canary, rotation counters, +// drain/re-emit, leak-tag correlation) +// level 2 full diagnostics: per-object/per-klass/per-entry lines +// (heap admits, auto-marks, sweep/fold internals) +// +// Sources, in order: the env var below at first use, overridden at +// runtime by the file below (re-checked about once per second from the +// reference-chains thread loop - refresh does open/read, so it never +// runs in heap callbacks; heap callbacks only read the cached atomic). +// +// env: DD_PROFILING_REFERENCE_CHAINS_DEBUG=0|1|2 +// file: /tmp/ddprof_root/refchains_debug_level (single digit 0/1/2; +// remove the file to fall back to the env value) +// +// All machinery is DEBUG-build-only: in non-debug builds TEST_LOG is +// already a no-op (common.h) and TEST_LOG_SUMMARY matches it, so this +// header costs nothing. + +#include "common.h" + +// The level machinery is compiled in ALL builds (the gtest binary is a +// non-DEBUG build and tests it directly; in non-DEBUG builds nothing calls +// it because the TEST_LOG macros are no-ops), while the macros below stay +// DEBUG-only like TEST_LOG itself. +int rcDebugLevel(); // cached; lazy env init on first use +void rcDebugLevelRefresh(bool force = false); // file override check, ~1s TTL +int parseRcDebugLevel(const char *value); // pure: NULL/invalid -> -1, else 0/1/2 +int readRcDebugLevelFile(const char *path); // pure: -1 missing/invalid + +#ifdef DEBUG + +#undef TEST_LOG +#define TEST_LOG(fmt, ...) \ + do { \ + if (rcDebugLevel() >= 2) { \ + fprintf(stdout, "[TEST::INFO] " fmt "\n", ##__VA_ARGS__); \ + fflush(stdout); \ + } \ + } while (0) + +#define TEST_LOG_SUMMARY(fmt, ...) \ + do { \ + if (rcDebugLevel() >= 1) { \ + fprintf(stdout, "[TEST::INFO] " fmt "\n", ##__VA_ARGS__); \ + fflush(stdout); \ + } \ + } while (0) + +#else // DEBUG + +#define TEST_LOG_SUMMARY(fmt, ...) // No-op in non-debug mode + +#endif // DEBUG + +#endif // _RC_DEBUG_LEVEL_H diff --git a/ddprof-lib/src/main/cpp/referenceChainAnchors.cpp b/ddprof-lib/src/main/cpp/referenceChainAnchors.cpp new file mode 100644 index 000000000..16dd391bf --- /dev/null +++ b/ddprof-lib/src/main/cpp/referenceChainAnchors.cpp @@ -0,0 +1,887 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "referenceChains.h" +#include "referenceChainInternal.h" +#include "common.h" +#include "counters.h" +#include "jniHelper.h" +#include "jvmThread.h" +#include "livenessTracker.h" +#include "log.h" +#include "objectSampler.h" +#include "os.h" +#include "profiler.h" +#include "rcDebugLevel.h" +#include "tsc.h" +#include "vmEntry.h" +#include +#include +#include +#include +#include +#include +#include +#include + +// Candidate-scoped reach: bounded descend walks from anchor objects (see +namespace { + +// Class tags to neither admit nor descend into during a descend walk (see +// ReferenceChainPassContext::_no_descend_class_tags' own comment). +const char *const kNoDescendClassNames[] = { + "java/lang/ClassLoader", "java/lang/ThreadGroup", + "java/security/ProtectionDomain", +}; + +int resolveNoDescendClassTags(jvmtiEnv *jvmti, JNIEnv *jni, + jlong *out, int cap) { + int count = 0; + for (const char *name : kNoDescendClassNames) { + if (count >= cap) { + break; + } + jclass cls = jni->FindClass(name); + if (cls == nullptr) { + // Not loadable in this JVM (e.g. java.security classes stripped by a minimal runtime) - skip; + // the gate simply does not cover it. + jni->ExceptionClear(); + continue; + } + jlong tag = 0; + if (jvmti->GetTag(cls, &tag) == JVMTI_ERROR_NONE && tag != 0) { + out[count++] = tag; + } + jni->DeleteLocalRef(cls); + } + return count; +} + +// java.lang.ThreadLocal$ThreadLocalMap's class tag for walkCandidateThreadLocals()'s anchor gate +// (see ReferenceChainPassContext:: _anchor_descend_class_tag's own comment): the value type of BOTH +// of Thread's threadLocals and inheritableThreadLocals fields, and its exact class tag is what the +// anchor gate compares against. +jlong resolveThreadLocalMapClassTag(jvmtiEnv *jvmti, JNIEnv *jni) { + jlong tag = 0; + jclass cls = jni->FindClass("java/lang/ThreadLocal$ThreadLocalMap"); + if (cls == nullptr) { + jni->ExceptionClear(); + return 0; + } + jvmti->GetTag(cls, &tag); + jni->DeleteLocalRef(cls); + return tag; +} + +} // namespace + +void ReferenceChainTracker::descendFromAnchor( + jvmtiEnv *jvmti, JNIEnv *jni, jobject anchor, jlong anchor_tag, + u32 anchor_depth, jlong anchor_descend_class_tag, int budget, + int *edges_admitted, bool *truncated, bool *frontier_cap_hit, + u64 *safepoint_ticks) { + ReferenceChainPassContext ctx; + ctx.tracker = this; + ctx.frontier = _frontier; + // Bound admission to DESCENT_HOPS below the anchor, still subject to the global hop cap. + int descent_cap = (int)anchor_depth + DESCENT_HOPS; + ctx.hop_cap = descent_cap < _hop_cap ? descent_cap : _hop_cap; + ctx.budget = budget; + ctx.edges_admitted = 0; + ctx.truncated = false; + ctx.frontier_cap_hit = false; + + ctx._no_descend_class_tag_count = + resolveNoDescendClassTags(jvmti, jni, ctx._no_descend_class_tags, + ReferenceChainPassContext::NO_DESCEND_CLASS_CAP); + if (anchor_descend_class_tag != 0) { + ctx._descent_anchor_tag = anchor_tag; + ctx._anchor_descend_class_tag = anchor_descend_class_tag; + } + + jvmtiHeapCallbacks callbacks; + memset(&callbacks, 0, sizeof(callbacks)); + callbacks.heap_reference_callback = heapReferenceCallback; + u64 follow_start_ticks = TSC::ticks(); + jvmti->FollowReferences(0, nullptr, anchor, &callbacks, &ctx); + *safepoint_ticks += TSC::ticks() - follow_start_ticks; + *edges_admitted += ctx.edges_admitted; + *truncated = *truncated || ctx.truncated; + *frontier_cap_hit = *frontier_cap_hit || ctx.frontier_cap_hit; +} + +std::vector +ReferenceChainTracker::collectStaticFieldAnchorsForRotation(int max_count) { + std::vector selected; + if (max_count <= 0 || _static_anchor_index.empty()) { + return selected; + } + // Tiered selection over _static_anchor_index (O(anchors) per pass, under ONE shared lock - the + // lookups below are lookupLocked()). + size_t idx_size = _static_anchor_index.size(); + if (_anchor_container_cursor >= idx_size) { + _anchor_container_cursor = 0; + } + if (_anchor_other_cursor >= idx_size) { + _anchor_other_cursor = 0; + } + struct TierPick { + size_t pos; + jlong tag; + }; + std::vector leak_picks; + std::vector fresh_picks; + std::vector container_picks; + std::vector other_picks; + leak_picks.reserve(16); + // Fresh picks kept by the queue drain (bounded by max_count) - used to keep the fair-tier + // consumption below from double-selecting them. + std::unordered_set fresh_kept_tags; + const size_t fresh_queue_len = _static_anchor_fresh_queue.size(); + _frontier->withSharedLock([&](const FrontierTable *frontier) { + // Index scan: partition every eligible anchor into the leak tier or one of the two fair tiers + // (the fresh lane is decided by the queue drain below - a fresh-kept anchor also lands in a + // fair pick vector here and is skipped at consumption time via fresh_kept_tags). + for (size_t i = 0; i < idx_size; i++) { + jlong tag = _static_anchor_index[i]; + FrontierEntry entry{}; + if (!frontier->lookupLocked(tag, &entry) || + entry.parent_tag != 0 || + (entry.root_kind != (u8)JVMTI_HEAP_REFERENCE_STATIC_FIELD && + entry.root_kind != (u8)JVMTI_HEAP_REFERENCE_JNI_GLOBAL) || + (entry.state != FrontierEntryState::FRONTIER && + entry.state != FrontierEntryState::EXPANDED) || + isQueuedForRotation(tag)) { + continue; + } + if (entry.leak_tag != 0) { + leak_picks.push_back(TierPick{i, tag}); + } else if (i < _static_anchor_own_class_tags.size()) { + auto shape_it = + _class_shape_cache.find(_static_anchor_own_class_tags[i]); + if (shape_it != _class_shape_cache.end() && + shape_it->second == (u8)AnchorClassShape::CONTAINER) { + container_picks.push_back(TierPick{i, tag}); + } else { + other_picks.push_back(TierPick{i, tag}); + } + } else { + other_picks.push_back(TierPick{i, tag}); + } + } + // Fresh-lane drain. Every queue entry is popped (its ONE first look is spent either way): kept + // if eligible AND (container-shaped OR not-yet-classified) AND room remains in the budget; + // dropped otherwise. + int fresh_room = max_count - (int)leak_picks.size(); + size_t drain_pos = fresh_queue_len <= idx_size ? idx_size - fresh_queue_len : 0; + while (!_static_anchor_fresh_queue.empty()) { + if (fresh_room <= 0) { + // Budget exhausted before the queue drained: everything remaining spends its first look now + // and falls back to the fair tiers at its index position (covered, not urgent). + _static_anchor_fresh_queue.clear(); + break; + } + jlong tag = _static_anchor_fresh_queue.front(); + _static_anchor_fresh_queue.pop_front(); + size_t pos = drain_pos; + drain_pos++; + if (pos >= idx_size || _static_anchor_index[pos] != tag) { + // The suffix-window invariant broke (cannot happen today; defensive): fall back to a search + // rather than mis-shape the entry - the queue is small, this is not a hot path once + // healthy. + auto it = + std::find(_static_anchor_index.begin(), + _static_anchor_index.end(), tag); + if (it == _static_anchor_index.end()) { + continue; + } + pos = (size_t)(it - _static_anchor_index.begin()); + } + FrontierEntry entry{}; + if (!frontier->lookupLocked(tag, &entry) || + entry.parent_tag != 0 || + (entry.root_kind != (u8)JVMTI_HEAP_REFERENCE_STATIC_FIELD && + entry.root_kind != (u8)JVMTI_HEAP_REFERENCE_JNI_GLOBAL) || + (entry.state != FrontierEntryState::FRONTIER && + entry.state != FrontierEntryState::EXPANDED) || + isQueuedForRotation(tag) || entry.leak_tag != 0) { + continue; // dead/demoted/queued/leak-tier: first look spent, not + // fresh-kept (the leak tier selects it via the scan if it is leak-tagged) + } + bool keep = false; // container or not-yet-classified rides the + // lane; the wrapper admits one pass before reconcile can classify its + // class + if (pos < _static_anchor_own_class_tags.size()) { + auto shape_it = + _class_shape_cache.find(_static_anchor_own_class_tags[pos]); + keep = shape_it == _class_shape_cache.end() || + shape_it->second == (u8)AnchorClassShape::CONTAINER; + } else { + keep = true; // no own-class tag recorded - treat as unknown + } + if (!keep) { + continue; // classified non-container: the other tier owns it + } + fresh_picks.push_back(TierPick{pos, tag}); + fresh_kept_tags.insert(tag); + fresh_room--; + } + }); + // Cursor-fair consumption of one tier: scan picks (sorted by pos by construction) starting at + // entries with pos >= cursor, stop at `want` OR at the lap end (NO within-call wrap: re-walking + // anchors this same call already covered would waste walk budget - the leftover budget flows to + // the next tier instead, and the cursor resets to 0 so the NEXT call starts a fresh lap). + auto consume_tier_fair = [&](const std::vector &picks, + size_t &cursor, int want) { + int took = 0; + if (want <= 0 || picks.empty()) { + return took; + } + size_t consumed_pos = 0; + for (size_t k = 0; k < picks.size() && took < want; k++) { + const TierPick &p = picks[k]; + if (p.pos < cursor) { + continue; + } + if (fresh_kept_tags.count(p.tag) > 0) { + continue; + } + selected.push_back(p.tag); + consumed_pos = p.pos; + took++; + } + if (took > 0) { + cursor = consumed_pos + 1 >= idx_size ? 0 : consumed_pos + 1; + } else { + // Took nothing AND no pick sits at or ahead of the cursor: this lap + // already passed every current member of the tier (members selected in + // earlier calls and since demoted out of eligibility). Without a reset + // the cursor never wraps again - every later call skips them all + // (p.pos < cursor) and the tier starves until a NEW anchor is appended + // at a higher pos. Treat the lap as completed-but-unproductive and + // restart it, matching the wrap semantics applied above. + bool any_ahead = false; + for (const TierPick &p : picks) { + if (p.pos >= cursor) { + any_ahead = true; + break; + } + } + if (!any_ahead) { + cursor = 0; + } + } + return took; + }; + int budget_left = max_count; + for (const TierPick &p : leak_picks) { + if (budget_left <= 0) { + break; + } + selected.push_back(p.tag); + budget_left--; + } + // Fresh lane: queue order (admission order) so a burst larger than the budget spends the oldest + // first looks first and nothing jumps the queue; outranked fresh anchors fall back to the fair + // tiers at their positions (the drain already dropped them from the queue). + for (const TierPick &p : fresh_picks) { + if (budget_left <= 0) { + break; + } + selected.push_back(p.tag); + budget_left--; + } + budget_left -= consume_tier_fair(container_picks, _anchor_container_cursor, + budget_left); + // The other tier is the last consumer of the budget - its leftover has no further reader, so + // don't accumulate it back into budget_left (a dead store clang scan-build flags). + consume_tier_fair(other_picks, _anchor_other_cursor, budget_left); + return selected; +} + +void ReferenceChainTracker::pushAtRiskStaticAnchor(jlong tag, u32 klass_id) { + if (_static_anchor_fifo_set.contains(tag)) { + return; + } + if (_static_anchor_fifo.size() >= STATIC_ANCHOR_FIFO_CAP) { + // The per-class quota prevents a single class from saturating this queue. + return; + } + auto count_it = _static_anchor_fifo_klass_counts.find(klass_id); + if (count_it != _static_anchor_fifo_klass_counts.end() && + count_it->second >= STATIC_ANCHOR_ATRISK_PER_KLASS_CAP) { + // Per-class quota drop: this class already holds its share of the lane, and its oldest entry + // drains within a few passes (STATIC_ANCHOR_FIFO_DRAIN=16/pass). + return; + } + if (count_it == _static_anchor_fifo_klass_counts.end()) { + count_it = _static_anchor_fifo_klass_counts.emplace(klass_id, 0U).first; + } + count_it->second++; + _static_anchor_fifo.push_back(AtRiskAnchor{tag, klass_id}); + _static_anchor_fifo_set.insert(tag); +} + +void ReferenceChainTracker::addToStaticAnchorIndex(jlong tag, + jlong own_class_tag, + u8 root_kind) { + if (root_kind != (u8)JVMTI_HEAP_REFERENCE_STATIC_FIELD && + root_kind != (u8)JVMTI_HEAP_REFERENCE_JNI_GLOBAL) { + return; + } + // Dedup: a push at first admission and another at upgrade would double-add. + if (!_static_anchor_index_tags.insert(tag).second) { + return; + } + _static_anchor_index.push_back(tag); + _static_anchor_own_class_tags.push_back(own_class_tag); + // Walk newly admitted anchors before the fair cursors. + _static_anchor_fresh_queue.push_back(tag); + if (_static_anchor_fresh_queue.size() > STATIC_ANCHOR_FRESH_CAP) { + _static_anchor_fresh_queue.pop_front(); + } +} + +bool ReferenceChainTracker::resolveContainerInterfaceTags( + jvmtiEnv *jvmti, JNIEnv *jni) { + if (_collection_iface_class_tag != 0 && _map_iface_class_tag != 0) { + return true; + } + // resolveLoadedClasses() tags every loaded class (including these bootstrap interfaces) with its + // class-tag-allocator tag (a NEGATIVE value - see nextClassTag()'s own comment) before any anchor + // can be admitted, but classify defensively: if an interface object somehow carries no tag yet, + // mint one via the shared allocator (same sequence resolveLoadedClasses() itself uses) so the + // comparison below is well-defined. + struct Iface { + const char *name; + jlong *tag_out; + }; + Iface ifaces[2] = {{"java/util/Collection", &_collection_iface_class_tag}, + {"java/util/Map", &_map_iface_class_tag}}; + for (const Iface &iface : ifaces) { + if (*iface.tag_out != 0) { + continue; + } + // Each interface is resolved independently: a transient JVMTI error on + // one of them (GetTag/SetTag/FindClass failing for exactly one) must not + // abort the other's resolution - the already-resolved tag stays cached in + // its slot either way, and a per-interface failure only leaves THAT tag + // at 0 for this call (the caller treats a false return as "shapes + // unknown this pass" and retries on the next reconcile). Failing the + // whole call on the first error would keep both anchors unclassified for + // as long as one interface keeps erroring, even though the other resolved + // fine. + jclass local = jni->FindClass(iface.name); + if (jniExceptionCheck(jni) || local == nullptr) { + jni->ExceptionClear(); + continue; + } + jlong tag = 0; + bool ok = jvmti->GetTag(local, &tag) == JVMTI_ERROR_NONE; + if (ok && tag == 0) { + jlong new_tag = nextClassTag(); + if (jvmti->SetTag(local, new_tag) == JVMTI_ERROR_NONE) { + // Adopt-on-reread, same cross-tracker race as + // resolveLoadedClasses()/mintStableClassTagIfNeeded(): another tracker may have installed + // its own tag between our GetTag and SetTag - keep the one tag the interface object + // carries. + jlong installed = 0; + if (jvmti->GetTag(local, &installed) == JVMTI_ERROR_NONE && + installed != 0) { + tag = installed; + } else { + tag = new_tag; + } + } else { + ok = false; + } + } + if (ok && tag != 0) { + *iface.tag_out = tag; + } + jni->DeleteLocalRef(local); + // A per-interface GetTag/SetTag failure leaves that tag at 0 - the final + // check below reports false only if an interface genuinely has no tag, + // never as an early abort that skips the other interface. + } + return _collection_iface_class_tag != 0 && _map_iface_class_tag != 0; +} + +bool ReferenceChainTracker::classImplementsContainerOrMap(jvmtiEnv *jvmti, + JNIEnv *jni, + jclass klass) { + // BFS over the superclass chain + every visited class's interfaces, comparing GetTag() against + // the two cached interface class tags. + std::vector work; + std::unordered_set visited; + work.push_back(klass); + bool found = false; + int hops = 0; + while (!found && !work.empty() && hops++ < 64) { + jclass cur = work.back(); + work.pop_back(); + jlong cur_tag = 0; + if (jvmti->GetTag(cur, &cur_tag) != JVMTI_ERROR_NONE || cur_tag == 0) { + // Early exit: the popped ref is this walk's own (GetSuperclass/ GetImplementedInterfaces + // local, or the caller's klass) - delete it like the loop bottom does, or the long-lived + // engine thread leaks a JNI local ref per untagged hop. + if (cur != klass) { + jni->DeleteLocalRef(cur); + } + continue; + } + if (visited.count(cur_tag) > 0) { + // Early exit: same local-ref ownership as above - delete before continuing (a hierarchy + // diamond revisits interfaces here). + if (cur != klass) { + jni->DeleteLocalRef(cur); + } + continue; + } + visited.insert(cur_tag); + if (cur_tag == _collection_iface_class_tag || + cur_tag == _map_iface_class_tag) { + found = true; + // The popped `cur` ref never reaches the loop's bottom delete. + if (cur != klass) { + jni->DeleteLocalRef(cur); + } + break; + } + jclass super = jni->GetSuperclass(cur); + if (!jniExceptionCheck(jni) && super != nullptr) { + work.push_back(super); + } else { + jni->ExceptionClear(); + } + jint iface_count = 0; + jclass *ifaces = nullptr; + if (jvmti->GetImplementedInterfaces(cur, &iface_count, &ifaces) == + JVMTI_ERROR_NONE && + ifaces != nullptr) { + for (jint i = 0; i < iface_count; i++) { + if (ifaces[i] != nullptr) { + work.push_back(ifaces[i]); + } + } + jvmti->Deallocate((unsigned char *)ifaces); + } + // `cur` is either the caller-provided klass (caller-managed ref - NOT deleted here) or a ref + // this walk minted (GetSuperclass/ GetImplementedInterfaces locals, deleted immediately after + // use). + if (cur != klass) { + jni->DeleteLocalRef(cur); + } + } + // Single exit: every remaining ref minted into `work` (early hop-bound exit or the found-break) + // is deleted here rather than leaking locals for the process lifetime (the engine thread never + // detaches). + for (jclass r : work) { + if (r != nullptr && r != klass) { + jni->DeleteLocalRef(r); + } + } + return found; +} + +void ReferenceChainTracker::reconcileAnchorClassShapes(jvmtiEnv *jvmti, + JNIEnv *jni) { + if (jni == nullptr) { + return; + } + if (_static_anchor_own_class_tags.empty()) { + return; + } + // Collect up to ANCHOR_SHAPE_RECONCILE_BUDGET distinct class tags that appear in the anchor index + // but are not yet classified. + std::vector unknown; + unknown.reserve(8); + std::unordered_set seen; + for (jlong class_tag : _static_anchor_own_class_tags) { + if (class_tag == 0 || seen.count(class_tag) > 0 || + _class_shape_cache.count(class_tag) > 0) { + continue; + } + seen.insert(class_tag); + unknown.push_back(class_tag); + if ((int)unknown.size() >= ANCHOR_SHAPE_RECONCILE_BUDGET) { + break; + } + } + if (unknown.empty()) { + return; + } + if (!resolveContainerInterfaceTags(jvmti, jni)) { + return; + } + // The per-class interface walk below mints local refs (GetSuperclass, GetImplementedInterfaces) + // that are only deleted as the BFS pops them; bound the outstanding count explicitly rather than + // relying on the JVM to grow the local-ref table. + if (jni->EnsureLocalCapacity(512) < 0 || jniExceptionCheck(jni)) { + jni->ExceptionClear(); + return; + } + // One GetObjectsWithTags call resolves the class objects for the whole batch (class objects are + // tagged with their class tags). + jint obj_count = 0; + jobject *objs = nullptr; + jlong *obj_tags = nullptr; + if (jvmti->GetObjectsWithTags((jint)unknown.size(), unknown.data(), + &obj_count, &objs, &obj_tags) != + JVMTI_ERROR_NONE || + obj_count <= 0) { + if (objs != nullptr) { + jvmti->Deallocate((unsigned char *)objs); + } + if (obj_tags != nullptr) { + jvmti->Deallocate((unsigned char *)obj_tags); + } + return; + } + for (jint i = 0; i < obj_count; i++) { + jclass klass = (jclass)objs[i]; + jlong class_tag = obj_tags[i]; + // class tags are NEGATIVE (a namespace disjoint from positive frontier tags); 0 means the + // object was never tagged - skip only that. + if (class_tag == 0 || klass == nullptr) { + if (klass != nullptr) { + jni->DeleteLocalRef(klass); + } + continue; + } + AnchorClassShape shape = classImplementsContainerOrMap(jvmti, jni, klass) + ? AnchorClassShape::CONTAINER + : AnchorClassShape::NON_CONTAINER; + _class_shape_cache[class_tag] = (u8)shape; + // GetObjectsWithTags() returned a local ref for every resolved class - this runs on the + // long-lived BFS thread, where undeleted locals accumulate until detach and pin their classes + // against unload. + jni->DeleteLocalRef(klass); + } + jvmti->Deallocate((unsigned char *)objs); + jvmti->Deallocate((unsigned char *)obj_tags); +} + +int ReferenceChainTracker::drainStaticAnchorFifo(int max_count, + std::vector &out) { + if (max_count <= 0 || _static_anchor_fifo.empty()) { + return 0; + } + int drained = 0; + while (drained < max_count && !_static_anchor_fifo.empty()) { + AtRiskAnchor entry = _static_anchor_fifo.front(); + _static_anchor_fifo.pop_front(); + auto count_it = _static_anchor_fifo_klass_counts.find(entry.klass_id); + if (count_it != _static_anchor_fifo_klass_counts.end() && + --count_it->second == 0) { + // Erased at zero so the map is bounded by the FIFO's live contents (<= 1024 distinct + // classes), not by the search lifetime. + _static_anchor_fifo_klass_counts.erase(count_it); + } + out.push_back(entry); + drained++; + } + _static_anchor_fifo_set.rebuildFrom(_static_anchor_fifo); + return drained; +} + +void ReferenceChainTracker::requeueStaticAnchorFifoFront( + const std::vector &entries) { + if (entries.empty()) { + return; + } + // Reverse order onto the front preserves the tags' relative FIFO order (push_front of the LAST + // entry first leaves the FIRST entry at the deque's front). + for (size_t i = entries.size(); i-- > 0;) { + _static_anchor_fifo_klass_counts[entries[i].klass_id]++; + _static_anchor_fifo.push_front(entries[i]); + } + _static_anchor_fifo_set.rebuildFrom(_static_anchor_fifo); +} + +void ReferenceChainTracker::walkStaticFieldAnchors( + jvmtiEnv *jvmti, JNIEnv *jni, const std::vector &anchor_tags, + int budget, int *edges_admitted, bool *truncated, bool *frontier_cap_hit, + u64 *safepoint_ticks, std::vector *unwalked) { + if (anchor_tags.empty()) { + return; + } + // Resolve all anchors in one O(tag_map) call. + jint resolved_count = 0; + jobject *objects = nullptr; + jlong *resolved_tags = nullptr; + if (jvmti->GetObjectsWithTags((jint)anchor_tags.size(), anchor_tags.data(), + &resolved_count, &objects, + &resolved_tags) != JVMTI_ERROR_NONE) { + return; + } + int walked = 0; + // First index the walk did NOT consume (breaks before an anchor's walk report i, breaks after + // report i+1; a completed loop keeps the resolved_count sentinel). + jint first_unwalked = resolved_count; + // First index whose local ref has not been deleted yet. Every break path deletes objects[i] + // before breaking, so anything at or after i+1 still holds a live local ref and must be cleaned + // up below - this runs on the long-lived BFS thread, where undeleted locals accumulate until + // detach and pin their objects against collection. + jint first_undeleted = resolved_count; + for (jint i = 0; i < resolved_count; i++) { + FrontierEntry entry{}; + if (!_frontier->lookup(resolved_tags[i], &entry)) { + // Dead-or-stale between selection and here - skip; release machinery owns dead-entry cleanup, + // never here. + jni->DeleteLocalRef(objects[i]); + continue; + } + int remaining = budget - *edges_admitted; + if (remaining <= 0) { + jni->DeleteLocalRef(objects[i]); + first_unwalked = i; + first_undeleted = i + 1; + break; + } + int edges_before = *edges_admitted; + descendFromAnchor(jvmti, jni, objects[i], resolved_tags[i], entry.depth, + /*anchor_descend_class_tag=*/0, remaining, edges_admitted, + truncated, frontier_cap_hit, safepoint_ticks); + TEST_LOG("ReferenceChainTracker::walkStaticFieldAnchors anchor walk " + "outcome tag=%lld edges=%d truncated=%d cap_hit=%d", + (long long)resolved_tags[i], *edges_admitted - edges_before, + (int)*truncated, (int)*frontier_cap_hit); + walked++; + jni->DeleteLocalRef(objects[i]); + if (*truncated && !*frontier_cap_hit) { + // Budget/deadline exhausted mid-set - remaining anchors keep their rotation turn via the + // cursor next pass (the wrapping cursor already tolerates a short selection). + first_unwalked = i + 1; + first_undeleted = i + 1; + break; + } + if (*frontier_cap_hit) { + first_unwalked = i + 1; + first_undeleted = i + 1; + break; + } + } + // Release the local refs of anchors the early exits above skipped - each break only deleted its + // own objects[i]. + for (jint i = first_undeleted; i < resolved_count; i++) { + jni->DeleteLocalRef(objects[i]); + } + if (unwalked != nullptr && first_unwalked < resolved_count) { + unwalked->insert(unwalked->end(), resolved_tags + first_unwalked, + resolved_tags + resolved_count); + } + jvmti->Deallocate((unsigned char *)objects); + jvmti->Deallocate((unsigned char *)resolved_tags); + TEST_LOG_SUMMARY("ReferenceChainTracker::walkStaticFieldAnchors selected=%zu " + "walked=%d edges_admitted=%d truncated=%d frontier_cap_hit=%d", + anchor_tags.size(), walked, *edges_admitted, (int)*truncated, + (int)*frontier_cap_hit); +} + +void ReferenceChainTracker::walkCandidateThreadLocals( + jvmtiEnv *jvmti, JNIEnv *jni, int budget, int *edges_admitted, + bool *truncated, bool *frontier_cap_hit, u64 *safepoint_ticks) { + if (_candidate_count <= 0) { + return; + } + // Flatten the per-slot qualifying-tid snapshot into (slot, tid) pairs, then walk up to + // THREAD_WALK_MAX_ANCHORS of them per pass, rotating via _thread_walk_anchor_cursor so every + // qualifying tid gets a turn within ceil(total / THREAD_WALK_MAX_ANCHORS) passes instead of + // always walking the first candidates' tids. + int slot[MAX_CANDIDATE_QUALIFYING_TIDS * MAX_LEAK_CANDIDATES_FROM_LT]; + jint tid[sizeof(slot) / sizeof(slot[0])]; + int total = 0; + for (int s = 0; s < _candidate_count; s++) { + for (int q = 0; q < _candidate_qualifying_tid_count[s]; q++) { + if (total >= (int)(sizeof(slot) / sizeof(slot[0]))) { + break; + } + slot[total] = s; + tid[total] = _candidate_qualifying_tids[s][q]; + total++; + } + } + if (total == 0) { + return; + } + jlong descend_class_tag = resolveThreadLocalMapClassTag(jvmti, jni); + if (_thread_walk_anchor_cursor < 0 || + _thread_walk_anchor_cursor >= total) { + _thread_walk_anchor_cursor = 0; + } + int walked = 0; + const int start = _thread_walk_anchor_cursor; + int i = start; + do { + jobject thread_obj; + { + MutexLocker ml(_thread_objects_lock); + auto it = _thread_objects.find(tid[i]); + if (it == _thread_objects.end()) { + thread_obj = nullptr; // Thread died/never registered - skip + } else { + thread_obj = it->second; + } + } + if (thread_obj != nullptr) { + // Anchor admission, idempotent across passes: a tag that still maps to a live entry is reused + // as-is (the Thread object is commonly root-attached by root enumeration already); a stale + // positive tag (search restart reissued tags from 1, releaseSearchTags() did not clear this + // object because release only touches FrontierTable entries) must be re-minted, otherwise the + // walk would parent new children onto a dead table slot or, worse, onto the entry a reissued + // tag now belongs to. + jlong anchor_tag = getTag(jvmti, thread_obj); + u32 anchor_depth = 0; + FrontierEntry anchor_entry{}; + if (anchor_tag > 0 && _frontier->lookup(anchor_tag, &anchor_entry)) { + anchor_depth = anchor_entry.depth; + } else { + jclass thread_class = jni->GetObjectClass(thread_obj); + jlong class_tag = 0; + jvmti->GetTag(thread_class, &class_tag); + u32 referrer_klass = classTags()->resolve(class_tag); + jlong fresh_tag = tagObject(jvmti, thread_obj); + if (fresh_tag != 0 && + _frontier->insert(fresh_tag, 0, referrer_klass, 0, + FrontierEntryState::FRONTIER, + (u8)JVMTI_HEAP_REFERENCE_THREAD, class_tag)) { + anchor_tag = fresh_tag; + } else { + if (fresh_tag != 0) { + // Frontier insert failed (table full): the tag-release scan only touches inserted + // frontier entries, so an installed-but-unowned tag would survive the search and + // collide with a reused tag number after a restart (fresh _next_tag from 1). + clearTag(jvmti, thread_obj); + } + anchor_tag = 0; + } + jni->DeleteLocalRef(thread_class); + } + if (anchor_tag != 0) { + int remaining = budget - *edges_admitted; + if (remaining > 0) { + descendFromAnchor(jvmti, jni, thread_obj, anchor_tag, anchor_depth, + descend_class_tag, remaining, edges_admitted, + truncated, frontier_cap_hit, safepoint_ticks); + walked++; + } + } + } + i = (i + 1) % total; + if (*frontier_cap_hit || walked >= THREAD_WALK_MAX_ANCHORS || + budget - *edges_admitted <= 0) { + break; + } + } while (i != start); + _thread_walk_anchor_cursor = i; + TEST_LOG_SUMMARY("ReferenceChainTracker::walkCandidateThreadLocals candidates=%d " + "tids=%d walked=%d edges_admitted=%d truncated=%d " + "frontier_cap_hit=%d", + _candidate_count, total, walked, *edges_admitted, (int)*truncated, + (int)*frontier_cap_hit); +} + +void ReferenceChainTracker::registerExistingThreads(jvmtiEnv *jvmti, + JNIEnv *jni) { + if (!_enabled || jvmti == nullptr || jni == nullptr) { + return; + } + // onThreadStart() cannot register threads that predate profiler attachment. + jint thread_count = 0; + jthread *thread_objects = nullptr; + if (jvmti->GetAllThreads(&thread_count, &thread_objects) != JVMTI_ERROR_NONE) { + return; + } + for (jint i = 0; i < thread_count; i++) { + jthread thread = thread_objects[i]; + if (thread == nullptr) { + continue; + } + int tid = JVMThread::nativeThreadId(jni, thread); + if (jni->ExceptionCheck()) { + jni->ExceptionClear(); + continue; + } + if (tid >= 0) { + registerThreadObject(jni, tid, thread); + } + jni->DeleteLocalRef(thread); + } + jvmti->Deallocate((unsigned char *)thread_objects); +} + +void ReferenceChainTracker::registerThreadObject(JNIEnv *jni, int tid, + jthread thread) { + if (!_enabled || jni == nullptr || thread == nullptr) { + return; + } + jobject ref = jni->NewGlobalRef(thread); + if (ref == nullptr) { + return; + } + MutexLocker ml(_thread_objects_lock); + auto it = _thread_objects.find(tid); + if (it != _thread_objects.end()) { + // Same deferred-deletion rule as unregisterThreadObject(): a walk may still hold a copy of the + // replaced ref. + _thread_refs_pending_delete.push_back(it->second); + } + _thread_objects[tid] = ref; +} + +void ReferenceChainTracker::unregisterThreadObject(JNIEnv *jni, int tid) { + if (jni == nullptr) { + return; + } + MutexLocker ml(_thread_objects_lock); + auto it = _thread_objects.find(tid); + if (it != _thread_objects.end()) { + // NOT DeleteGlobalRef() here: walkCandidateThreadLocals() may have already copied this jobject + // out of the map (lock released) and still be using it as a FollowReferences anchor - deleting + // a global ref invalidates it for every other JNI call (JNI spec), so deletion is deferred to + // releaseEndedThreadRefs() on the BFS thread (see _thread_refs_pending_delete's comment). + _thread_refs_pending_delete.push_back(it->second); + _thread_objects.erase(it); + } +} + +void ReferenceChainTracker::releaseEndedThreadRefs(JNIEnv *jni) { + if (jni == nullptr) { + return; + } + std::vector pending; + { + MutexLocker ml(_thread_objects_lock); + pending.swap(_thread_refs_pending_delete); + } + for (size_t i = 0; i < pending.size(); i++) { + jni->DeleteGlobalRef(pending[i]); + } +} + +void ReferenceChainTracker::releaseAllThreadObjects(JNIEnv *jni) { + if (jni == nullptr) { + return; + } + // Recording stop: the BFS thread is joined (Profiler::stop() order), so no walk phase can hold a + // copied ref - the deferred-deletion indirection of unregisterThreadObject() is unnecessary here + // and every ref can go now. + std::vector pending; + { + MutexLocker ml(_thread_objects_lock); + for (auto &kv : _thread_objects) { + pending.push_back(kv.second); + } + _thread_objects.clear(); + pending.insert(pending.end(), _thread_refs_pending_delete.begin(), + _thread_refs_pending_delete.end()); + _thread_refs_pending_delete.clear(); + } + for (size_t i = 0; i < pending.size(); i++) { + jni->DeleteGlobalRef(pending[i]); + } +} + diff --git a/ddprof-lib/src/main/cpp/referenceChainEvents.cpp b/ddprof-lib/src/main/cpp/referenceChainEvents.cpp new file mode 100644 index 000000000..648cc360b --- /dev/null +++ b/ddprof-lib/src/main/cpp/referenceChainEvents.cpp @@ -0,0 +1,820 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "referenceChains.h" +#include "referenceChainInternal.h" +#include "common.h" +#include "counters.h" +#include "jniHelper.h" +#include "jvmThread.h" +#include "livenessTracker.h" +#include "log.h" +#include "objectSampler.h" +#include "os.h" +#include "profiler.h" +#include "rcDebugLevel.h" +#include "tsc.h" +#include "vmEntry.h" +#include +#include +#include +#include +#include +#include +#include +#include + +// Target-selection bridging step - LivenessTracker's leak-candidate ranking feeds + +void ReferenceChainTracker::requeueChainRootForRotation(jlong tag) { + if (_frontier == nullptr || tag <= 0) { + return; + } + // Walk the parent chain up to the root-attached entry - the same links reconstructChain() walks, + // but we only need the tag, not the class ids. + jlong root_tag = tag; + FrontierEntry entry{}; + int hops = 0; + while (hops++ < _hop_cap) { + if (!_frontier->lookup(root_tag, &entry) || entry.parent_tag == 0) { + break; + } + root_tag = entry.parent_tag; + } + if (root_tag == tag) { + return; // tag IS the root - nothing above it to requeue + } + if (!_frontier->lookup(root_tag, &entry) || + entry.state != FrontierEntryState::EXPANDED) { + return; // root pruned or still pending expansion - nothing to re-walk + } + if (isQueuedForRotation(root_tag) || + _priority_expand.size() >= PRIORITY_EXPAND_CAP) { + return; + } + TEST_LOG("ReferenceChainTracker::requeueChainRootForRotation root_tag=%lld " + "target_tag=%lld", + (long long)root_tag, (long long)tag); + _priority_expand.push_back(root_tag); + _priority_expand_set.insert(root_tag); +} + +namespace { + +// The discovered-chain gate's suppression predicate, shared by EVERY site that caches a resolved +// chain - the poll's discovered-instances loop AND both representative build paths (the +// canary/marker path and the normal-tag path). +bool suppressChainEvent(const ReferenceChainEvent &event) { + return event._depth < 2 && isTransientRootKind(event._root_kind); +} + +} // namespace + +// Chain-event reconstruction for a discovered/correlated instance (out of line from +// referenceChains.h so these sites share the TU's level-gated TEST_LOG; per-instance outcomes are +// level-2 diagnostics). +bool ReferenceChainTracker::buildChainEvent(jvmtiEnv *jvmti, JNIEnv *jni, + jlong target_tag, + ReferenceChainEvent *out) { + if (_frontier == nullptr || out == nullptr) { + TEST_LOG("ReferenceChainTracker::buildChainEvent false: " + "frontier=%p out=%p", (void *)_frontier, (void *)out); + return false; + } + FrontierEntry entry{}; + if (!_frontier->lookup(target_tag, &entry)) { + TEST_LOG("ReferenceChainTracker::buildChainEvent false: " + "target_tag=%lld not in frontier", (long long)target_tag); + return false; + } + std::vector chain; + std::vector edges; + u8 root_kind = 0; + FrontierEntry terminal{}; + if (!_frontier->reconstructChain(target_tag, &chain, &root_kind, &edges, + &terminal)) { + TEST_LOG("ReferenceChainTracker::buildChainEvent false: " + "reconstructChain failed for target_tag=%lld", + (long long)target_tag); + return false; + } + appendStaticFieldRootType(terminal, &chain, &edges); + TEST_LOG("ReferenceChainTracker::buildChainEvent target_tag=%lld chain_size=%zu " + "chain[0]=%u depth=%u root_kind=%u leak_tag=%lld", + (long long)target_tag, chain.size(), chain.empty() ? 0u : chain[0], + entry.depth, (unsigned)root_kind, (long long)entry.leak_tag); + out->_target_tag = entry.leak_tag != 0 ? (u64)entry.leak_tag : (u64)target_tag; + out->_depth = entry.depth; + out->_root_kind = root_kind; + out->_hops.resize(chain.size()); + for (size_t i = 0; i < chain.size(); i++) { + out->_hops[i].klass_id = chain[i]; + } + // Retention-edge labels, aligned with the hops (see fillHopEdgeLabels()). + fillHopEdgeLabels(jvmti, jni, edges, &out->_hops); + return true; +} + +// Appends the root TYPE as a chain element for a static-field-rooted chain: the frontier path's +// root-side end is the static field's HOLDER instance (the object stored in the field), but the +// chain's root is the DECLARING CLASS - the holder is "the field instance referenced by the root +// type", one hop below it. +void ReferenceChainTracker::appendStaticFieldRootType( + const FrontierEntry &terminal, std::vector *chain, + std::vector *edges) { + if (chain == nullptr || + terminal.root_kind != (u8)JVMTI_HEAP_REFERENCE_STATIC_FIELD || + terminal.referrer_class_tag == 0) { + return; + } + u32 root_klass = classTags()->resolve(terminal.referrer_class_tag); + if (root_klass == 0) { + return; + } + chain->push_back(root_klass); + if (edges != nullptr) { + ChainHopEdge root_edge{}; + root_edge.field_index = -1; + root_edge.edge_kind = terminal.root_kind; + root_edge.referrer_class_tag = 0; + edges->push_back(root_edge); + } +} + +// Canary chain reconstruction (out of line for the same reason). +bool ReferenceChainTracker::buildCanaryChainEvent(int candidate_idx, + ReferenceChainEvent *out) { + if (_frontier == nullptr || out == nullptr || candidate_idx < 0 || + candidate_idx >= _candidate_count) { + TEST_LOG_SUMMARY("ReferenceChainTracker::buildCanaryChainEvent false: " + "frontier=%p out=%p idx=%d count=%d", + (void *)_frontier, (void *)out, candidate_idx, + _candidate_count); + return false; + } + jlong parent_tag = _candidate_parent_tags[candidate_idx]; + u32 candidate_klass = _candidate_referrer_klasses[candidate_idx]; + jlong frontier_tag = _candidate_frontier_tags[candidate_idx]; + std::vector chain; + u8 root_kind = 0; + // The root-attached entry the walk ends at - both branches below leave `entry` holding it (the + // walk's last lookup, or the candidate's own entry for a root-referenced candidate). + FrontierEntry terminal{}; + if (parent_tag > 0) { + // Walk parent_tag back to root through the frontier table. + FrontierEntry entry{}; + if (!_frontier->lookup(parent_tag, &entry)) { + TEST_LOG_SUMMARY("ReferenceChainTracker::buildCanaryChainEvent false: " + "parent_tag=%lld not in frontier (candidate=%d)", + (long long)parent_tag, candidate_idx); + return false; + } + // Bounded like every sibling walk over this same parent chain: + // FrontierTable::reconstructChain() bounds at maxCapacity() hops and + // returns false on a "cyclic or corrupt parent chain", + // requeueChainRootForRotation() bounds at _hop_cap, and improveChain()'s + // cycle guard bounds at 4096. improveChain() structurally prevents cycles + // (it refuses a parent whose chain routes through the entry), but the + // table's contents are also written by insert() with no such validation, + // so a corrupt chain must fail safe instead of spinning this poll-thread + // walk forever: every tag maps to a distinct slot (tags are never + // reused), so a well-formed chain can visit at most maxCapacity() entries + // before reaching parent_tag == 0 or repeating a slot. + const int hop_bound = _frontier->maxCapacity(); + int hops = 0; + for (jlong tag = parent_tag; tag > 0; hops++) { + if (hops > hop_bound) { + TEST_LOG_SUMMARY("ReferenceChainTracker::buildCanaryChainEvent false: " + "chain walk exceeded hop bound (%d) - cyclic or " + "corrupt parent chain (candidate=%d)", + hops, candidate_idx); + return false; + } + if (!_frontier->lookup(tag, &entry)) { + TEST_LOG_SUMMARY("ReferenceChainTracker::buildCanaryChainEvent false: " + "chain walk: tag=%lld not in frontier (candidate=%d)", + (long long)tag, candidate_idx); + return false; + } + chain.push_back(entry.referrer_klass); + tag = entry.parent_tag; + } + // The root kind describes the chain's ROOT, not the candidate-side parent: + // the walk's last iteration is always the root-attached entry (parent_tag + // == 0, root_kind != 0), so `entry` holds it here - same terminal-root + // semantics reconstructChain() uses for *out_root_kind. The parent-side + // entry read before the loop would almost always yield 0 (interior entries + // carry root_kind == 0), misreporting every walk-reconstructed chain and + // defeating suppressChainEvent()'s transient-root gate. + root_kind = entry.root_kind; + terminal = entry; + } else if (parent_tag == 0 && frontier_tag > 0) { + // Root-referenced candidate: chain is just [candidate_klass]. + FrontierEntry entry{}; + if (!_frontier->lookup(frontier_tag, &entry)) { + TEST_LOG_SUMMARY("ReferenceChainTracker::buildCanaryChainEvent false: " + "frontier_tag=%lld not in frontier (candidate=%d)", + (long long)frontier_tag, candidate_idx); + return false; + } + root_kind = entry.root_kind; + terminal = entry; + } else { + TEST_LOG_SUMMARY("ReferenceChainTracker::buildCanaryChainEvent false: " + "never pruned (candidate=%d parent_tag=%lld frontier_tag=%lld)", + candidate_idx, (long long)parent_tag, + (long long)frontier_tag); + return false; // never pruned (candidate not reached) + } + // Prepend the candidate's own referrer_klass. + chain.push_back(candidate_klass); + // The chain was built root-to-parent; reverse to get candidate-to-root. + std::reverse(chain.begin(), chain.end()); + // Same root-type element buildChainEvent() appends: the canary walk's terminal entry is the + // root-attached entry, and for a static-field root the declaring class belongs at the chain's + // root-side end (after the reverse). + appendStaticFieldRootType(terminal, &chain, nullptr); + out->_target_tag = (u64)frontier_tag; + out->_depth = _candidate_depths[candidate_idx]; + out->_root_kind = root_kind; + const size_t chain_size = chain.size(); + out->_hops.resize(chain_size); + for (size_t i = 0; i < chain.size(); i++) { + out->_hops[i].klass_id = chain[i]; + } + TEST_LOG_SUMMARY("ReferenceChainTracker::buildCanaryChainEvent candidate=%d " + "parent_tag=%lld chain_size=%zu", + candidate_idx, (long long)parent_tag, chain_size); + return true; +} + +void ReferenceChainTracker::pollWatchedTargets(jvmtiEnv *jvmti, JNIEnv *jni) { + if (!_enabled || jvmti == nullptr || jni == nullptr || + !LivenessTracker::instance()->gcGenerationsEnabled()) { + // Avoid candidate-table work when generation tracking is disabled. + return; + } + + // Stamp every entry this poll refreshes with the current search generation. + const u64 current_search_ns = load(_search_start_ns); + + // klass_ids resolved (and therefore already pruned-if-dead) by the candidate loop below, so the + // prune pass afterwards skips re-resolving them - it only needs to cover cached klasses that are + // no longer flagged. + + // selectLeakCandidates() clamps this to its private candidate limit. + constexpr int kMaxWatchedCandidates = 8; + KlassCandidate candidates[kMaxWatchedCandidates]; + int candidate_count = LivenessTracker::instance()->selectLeakCandidates( + candidates, kMaxWatchedCandidates); + + // Publish this poll's qualifying tids as LivenessTracker's watched-admission set (see + // noteSelectedCandidates()'s own comment, livenessTracker.h): exactly the (klass, tid) scope + // tagLeakInstances() tags and this chase intercepts gets its allocations admitted at 100% instead + // of the default 10% ratio lottery. + LivenessTracker::instance()->noteSelectedCandidates(candidates, + candidate_count); + + // Refresh the faster, un-hysteresis-gated klass_id ranking rotation priority uses (see + // _watched_leak_klass_ids' own comment) - but only once selectLeakCandidates() above has ALREADY + // found at least one qualifying candidate via its own slower hysteresis gate: this mechanism is + // meant to crank once the trend detector has triggered, not to run the ranking independently + // before that gate has ever fired. + if (candidate_count > 0) { + // Snapshot the OLD watched set before overwriting it, so any klass_id that's newly appearing + // this refresh can get its one-time retroactive catch-up + // (seedLeakAccumulationForNewlyWatchedKlass() - see _watched_leak_klass_ids' own comment for + // why admission-time tracking alone cannot see objects admitted before watching started). + u32 previously_watched[MAX_WATCHED_LEAK_KLASSES]; + int previously_watched_count = _watched_leak_klass_count; + for (int i = 0; i < previously_watched_count; i++) { + previously_watched[i] = _watched_leak_klass_ids[i]; + } + _watched_leak_klass_count = LivenessTracker::instance()->topKlassesByGenerationCount( + _watched_leak_klass_ids, MAX_WATCHED_LEAK_KLASSES); + for (int i = 0; i < _watched_leak_klass_count; i++) { + u32 klass_id = _watched_leak_klass_ids[i]; + bool already_watched = false; + for (int j = 0; j < previously_watched_count; j++) { + if (previously_watched[j] == klass_id) { + already_watched = true; + break; + } + } + if (!already_watched) { + seedLeakAccumulationForNewlyWatchedKlass(klass_id); + } + } + } + // Only log when there are candidates to act on - this poll runs on every BFS-thread wake (once + // per second), so logging a zero count is per-second noise for the common idle case. + if (candidate_count > 0) { + TEST_LOG("ReferenceChainTracker::pollWatchedTargets candidate_count=%d", candidate_count); + // Admit any candidate selectLeakCandidates() returns this poll that doesn't already occupy a + // slot, into the next free slot. + for (int i = 0; i < candidate_count; i++) { + u32 klass_id = candidates[i].klass_id; + bool already_tracked = false; + for (int s = 0; s < _candidate_count; s++) { + if (_candidate_klass_ids[s] == klass_id) { + already_tracked = true; + break; + } + } + if (already_tracked) { + continue; + } + if (_candidate_count >= MAX_LEAK_CANDIDATES_FROM_LT) { + TEST_LOG_SUMMARY("ReferenceChainTracker::pollWatchedTargets canary: klass_id=%u " + "qualifies but all %d slots are occupied - not tracked this search", + klass_id, MAX_LEAK_CANDIDATES_FROM_LT); + continue; + } + // Candidate admission: the marker->leak-tag migration retired + // pre-tagging the representative object (the retired marker-tag decode + // branches are gone); this candidate is discovered when the walk or the + // poll intercepts one of its leak-tagged instances. + int slot = _candidate_count; + _candidate_klass_ids[slot] = klass_id; + _candidate_count = slot + 1; + TEST_LOG_SUMMARY("ReferenceChainTracker::pollWatchedTargets canary: admitted klass_id=%u " + "into slot=%d (candidate_count now %d)", + klass_id, slot, _candidate_count); + Counters::increment(REFERENCE_CHAIN_CANDIDATE_COUNT, 1); + } + // Refresh the per-slot qualifying-tid snapshot the walk phases read + // (walkCandidateThreadLocals()): zero every slot first, then fill from THIS poll's candidates - + // a klass whose per-tid trend stopped qualifying must stop having its tids walked, exactly like + // it stops consuming pool tags (tagLeakInstances() below keeps the same per-poll-candidates + // scope for the same reason). + memset(_candidate_qualifying_tid_count, 0, + sizeof(_candidate_qualifying_tid_count)); + for (int i = 0; i < candidate_count; i++) { + for (int s = 0; s < _candidate_count; s++) { + if (_candidate_klass_ids[s] != candidates[i].klass_id) { + continue; + } + int n = candidates[i].qualifying_tid_count; + if (n > MAX_CANDIDATE_QUALIFYING_TIDS) { + n = MAX_CANDIDATE_QUALIFYING_TIDS; + } + for (int q = 0; q < n; q++) { + _candidate_qualifying_tids[s][q] = candidates[i].qualifying_tids[q]; + } + _candidate_qualifying_tid_count[s] = n; + break; + } + } + // Tag the tracked instances of THIS poll's candidates with leak tags. + int tagged = LivenessTracker::instance()->tagLeakInstances( + jvmti, candidates, candidate_count); + _leak_tags_assigned = tagged; + _leak_tags_resolved = 0; // reset on each tagging round + TEST_LOG("ReferenceChainTracker::pollWatchedTargets tagLeakInstances tagged=%d", + tagged); + } + + for (int i = 0; i < candidate_count; i++) { + TEST_LOG("ReferenceChainTracker::pollWatchedTargets candidate[%d] klass_id=%u", i, + candidates[i].klass_id); + // Deliberately does NOT resolve candidates[i].representative directly: that field is a snapshot + // taken under selectLeakCandidates()'s own shared-lock scan, which can go stale (LRU-evicted + // and DeleteWeakGlobalRef()'d by LivenessTracker's cleanup_table(), running concurrently on a + // different thread) at any point between that call and this one - see selectLeakCandidates()'s + // comment (livenessTracker.h) for why resolving it here would be undefined behavior, not just a + // null result. + const u32 klass_id = candidates[i].klass_id; + jobject obj = LivenessTracker::instance()->resolveCandidateRepresentative( + jni, klass_id); + if (obj == nullptr) { + TEST_LOG("ReferenceChainTracker::pollWatchedTargets candidate[%d] klass_id=%u " + "representative could not be resolved (died/evicted)", + i, klass_id); + // The representative died, but the canary chain (if the candidate was pruned by BFS before + // the representative died) only needs the frontier table — not the live representative. + bool built_from_canary = false; + for (int s = 0; s < _candidate_count; s++) { + if (_candidate_klass_ids[s] != klass_id) continue; + if ((_candidate_found_bits & (1ULL << s)) && + _candidate_frontier_tags[s] != 0) { + jlong canary_ftag = _candidate_frontier_tags[s]; + _resolved_chains_lock.lock(); + bool need = (_resolved_chains.find(canary_ftag) == _resolved_chains.end()); + _resolved_chains_lock.unlock(); + if (need) { + ReferenceChainEvent event; + built_from_canary = buildCanaryChainEvent(s, &event); + TEST_LOG("ReferenceChainTracker::pollWatchedTargets " + "buildCanaryChainEvent(dead rep, slot=%d) -> %d", + s, (int)built_from_canary); + if (built_from_canary && suppressChainEvent(event)) { + TEST_LOG("ReferenceChainTracker::pollWatchedTargets " + "filtered depth=%u root_kind=%d canary_ftag=%lld " + "klass_id=%u (dead-rep path)", + event._depth, (int)event._root_kind, + (long long)canary_ftag, klass_id); + built_from_canary = false; + invalidateResolvedChain(canary_ftag); + } else if (built_from_canary) { + event._start_time = TSC::ticks(); + cacheResolvedChain(canary_ftag, std::move(event), + canary_ftag, current_search_ns); + } + } + } + break; + } + if (!built_from_canary) { + // The representative died. Per-instance caching means we don't erase by klass_id — chains + // for other instances of this class may still be valid. + } + continue; // candidate died, or was evicted, since LivenessTracker flagged it + } + + { + jclass obj_klass = jni->GetObjectClass(obj); + char *obj_class_name = nullptr; + if (obj_klass != nullptr && + jvmti->GetClassSignature(obj_klass, &obj_class_name, nullptr) == + JVMTI_ERROR_NONE && + obj_class_name != nullptr) { + TEST_LOG("ReferenceChainTracker::pollWatchedTargets candidate[%d] " + "klass_id=%u class_name=%s", + i, klass_id, obj_class_name); + jvmti->Deallocate((unsigned char *)obj_class_name); + } + if (obj_klass != nullptr) { + jni->DeleteLocalRef(obj_klass); + } + } + + // Read the existing tag; seeding here would bypass the forward walk. + jlong tag = getTag(jvmti, obj); + + // NOTE: the retired canary marker-tag decode used to live here (an object + // pre-tagged with MARKER_TAG_BASE - slot took the legacy canary chain + // reconstruction). The marker->leak-tag migration stopped pre-tagging + // candidate representatives entirely - no JVMTI tag in the process can + // ever be <= MARKER_TAG_BASE (-2^62): leak tags are positive + // (LEAK_TAG_BASE), frontier tags positive, class tags small negative + // magnitudes - so the branch was unreachable, and a stale negative tag + // (a class tag) could never silently take the legacy canary path. + // Candidate chain reconstruction now runs through the leak-tag discovery + // block below and the dead-representative canary path earlier in this + // loop. + + // Normal path: tag > 0 means the walk visited this object and assigned it a + // frontier tag. + + // Keep the holder chain's root warm in the rotation queue: a growing container's current + // internals are only reachable via the holder's re-walk (requeueChainRootForRotation()'s own + // comment). + if (tag > 0) { + requeueChainRootForRotation(tag); + } + + // Reconstruct only when this klass has no current chain cached: either nothing cached yet, or + // what is cached was built from a different tag or an earlier search generation (see + // current_search_ns above). + bool need_refresh = false; + jlong rep_chain_key = 0; + for (int s = 0; s < _candidate_count; s++) { + if (_candidate_klass_ids[s] == klass_id) { + rep_chain_key = _candidate_frontier_tags[s]; + break; + } + } + if (rep_chain_key != 0) { + _resolved_chains_lock.lock(); + auto it = _resolved_chains.find(rep_chain_key); + need_refresh = (it == _resolved_chains.end() || + it->second.source_search_ns != current_search_ns); + _resolved_chains_lock.unlock(); + } + TEST_LOG("ReferenceChainTracker::pollWatchedTargets candidate[%d] klass_id=%u tag=%lld " + "needRefresh=%d", + i, klass_id, (long long)tag, need_refresh); + if (need_refresh) { + ReferenceChainEvent event; + bool built = buildChainEvent(jvmti, jni, tag, &event); + TEST_LOG("ReferenceChainTracker::pollWatchedTargets buildChainEvent(tag=%lld) -> %d", + (long long)tag, built); + if (built && suppressChainEvent(event)) { + TEST_LOG("ReferenceChainTracker::pollWatchedTargets " + "filtered depth=%u root_kind=%d rep_tag=%lld klass_id=%u " + "(representative path)", + event._depth, (int)event._root_kind, (long long)tag, + klass_id); + built = false; + invalidateResolvedChain(tag); + } + if (built) { + // Provisional stamp; drainPendingChainEvents() re-stamps each copy at dump time so the + // event lands in that chunk's window. + event._start_time = TSC::ticks(); + cacheResolvedChain(tag, std::move(event), tag, current_search_ns); + } + } + // tag == 0: The representative object has no tag — the BFS walk hasn't reached it yet AND it is + // not yet leak-tagged. + + // Build chain events for auto-marked discovered instances of this class. + buildDiscoveredInstanceChains(jvmti, jni, klass_id, current_search_ns); + + jni->DeleteLocalRef(obj); + } + + // Orphan fix: slots whose klass is NOT among this poll's candidates. + for (int s = 0; s < _candidate_count; s++) { + u32 slot_klass = _candidate_klass_ids[s]; + bool in_poll = false; + for (int i = 0; i < candidate_count; i++) { + if (candidates[i].klass_id == slot_klass) { + in_poll = true; + break; + } + } + if (!in_poll && slot_klass != 0) { + TEST_LOG("ReferenceChainTracker::pollWatchedTargets orphan slot " + "sweep: slot=%d klass_id=%u not in poll candidates - building " + "its discovered chains", + s, slot_klass); + buildDiscoveredInstanceChains(jvmti, jni, slot_klass, current_search_ns); + } + } + + // Per-instance caching: chains are keyed by frontier tag, not klass_id. +} + +// Inserts or refreshes klass_id's resolved chain - see _resolved_chains' comment +// (referenceChains.h) for why a resolved chain is cached and re-emitted rather than emitted once. +bool ReferenceChainTracker::cacheResolvedChain(jlong source_tag, + ReferenceChainEvent &&event, + jlong source_tag_val, + u64 source_search_ns) { + _resolved_chains_lock.lock(); + auto it = _resolved_chains.find(source_tag); + if (it == _resolved_chains.end() && + (int)_resolved_chains.size() >= MAX_RESOLVED_CHAINS) { + _resolved_chains_lock.unlock(); + Counters::increment(REFERENCE_CHAIN_EVENTS_DROPPED); + TEST_LOG("ReferenceChainTracker::cacheResolvedChain dropped new source_tag=%lld, " + "cache full (at MAX_RESOLVED_CHAINS=%d)", + (long long)source_tag, MAX_RESOLVED_CHAINS); + return false; + } + CachedChain &slot = _resolved_chains[source_tag]; + slot.event = std::move(event); + slot.source_tag = source_tag_val; + slot.source_search_ns = source_search_ns; + TEST_LOG("ReferenceChainTracker::cacheResolvedChain source_tag=%lld cache_size=%d", + (long long)source_tag, (int)_resolved_chains.size()); + _resolved_chains_lock.unlock(); + return true; +} + +void ReferenceChainTracker::invalidateResolvedChain(jlong source_tag) { + _resolved_chains_lock.lock(); + auto it = _resolved_chains.find(source_tag); + if (it != _resolved_chains.end()) { + _resolved_chains.erase(it); + TEST_LOG("ReferenceChainTracker::invalidateResolvedChain source_tag=%lld", + (long long)source_tag); + } + _resolved_chains_lock.unlock(); +} + +// Builds and caches chain events for every auto-marked discovered instance recorded against a slot +// holding klass_id (see the auto-mark block in heapReferenceCallback() for how instances get +// recorded). +void ReferenceChainTracker::buildDiscoveredInstanceChains(jvmtiEnv *jvmti, + JNIEnv *jni, + u32 klass_id, + u64 current_search_ns) { +for (int s = 0; s < _candidate_count; s++) { + if (_candidate_klass_ids[s] != klass_id) continue; + TEST_LOG("ReferenceChainTracker::pollWatchedTargets " + "discovered loop: klass_id=%u slot=%d discovered_count=%d", + klass_id, s, _candidate_discovered_count[s]); + if (_candidate_discovered_count[s] == 0) { + TEST_LOG("ReferenceChainTracker::pollWatchedTargets " + "no discovered instances for klass_id=%u slot=%d", + klass_id, s); + } + for (int d = 0; d < _candidate_discovered_count[s]; d++) { + jlong disc_tag = _candidate_discovered_tags[s][d]; + if (disc_tag == 0) { + TEST_LOG("ReferenceChainTracker::pollWatchedTargets " + "disc_tag=0 at idx=%d for klass_id=%u slot=%d", + d, klass_id, s); + continue; + } + // Skip if already cached for this instance - but only for the CURRENT search generation: + // restartSearch() resets the frontier tag namespace, so a cached entry under the same numeric + // tag from an earlier search describes a different object and must not suppress the rebuild + // (the generation check mirrors the rep-refresh paths in pollWatchedTargets()). + const u64 current_search_ns = load(_search_start_ns); + _resolved_chains_lock.lock(); + auto cached_it = _resolved_chains.find(disc_tag); + bool already_cached = (cached_it != _resolved_chains.end() && + cached_it->second.source_search_ns == + current_search_ns); + _resolved_chains_lock.unlock(); + if (already_cached) { + TEST_LOG("ReferenceChainTracker::pollWatchedTargets " + "already_cached disc_tag=%lld klass_id=%u slot=%d idx=%d", + (long long)disc_tag, klass_id, s, d); + continue; + } + ReferenceChainEvent event; + bool built = buildChainEvent(jvmti, jni, disc_tag, &event); + // Retention-explanation filter. Only applies to discovered instances, not canary. + if (built && suppressChainEvent(event)) { + TEST_LOG("ReferenceChainTracker::pollWatchedTargets " + "filtered depth=%u root_kind=%d disc_tag=%lld klass_id=%u", + event._depth, (int)event._root_kind, + (long long)disc_tag, klass_id); + built = false; + // Also drop any chain cached for this tag before the filter existed (or before an + // improveChain/reparent upgraded it) - drainPendingChainEvents() re-emits cached chains + // unconditionally, so suppressing only the build would leave the noise chains re-emitting + // forever. + invalidateResolvedChain(disc_tag); + } + if (built) { + event._start_time = TSC::ticks(); + // Coverage accounting below must only advance for a chain that was actually stored - a + // cache-full drop would let the search report the candidate as found without ever emitting + // its chain. + if (cacheResolvedChain(disc_tag, std::move(event), disc_tag, + current_search_ns)) { + // Track coverage for adaptive CPU budget + if (event._target_tag >= (u64)LEAK_TAG_BASE) { + _leak_tags_resolved++; + // A qualifying thread must discover at least one candidate instance. + if (!(_candidate_found_bits & (1ULL << s))) { + _candidate_found_bits |= (1ULL << s); + _candidate_frontier_tags[s] = disc_tag; + _candidate_parent_tags[s] = 0; + _candidate_depths[s] = event._depth; + _candidate_referrer_klasses[s] = klass_id; + TEST_LOG_SUMMARY("ReferenceChainTracker::pollWatchedTargets canary " + "found: klass_id=%u slot=%d leak chain target_tag=%llu " + "via disc_tag=%lld (%d/%d candidates found)", + klass_id, s, (unsigned long long)event._target_tag, + (long long)disc_tag, + __builtin_popcountll(_candidate_found_bits), + _candidate_count); + } + } + TEST_LOG("ReferenceChainTracker::pollWatchedTargets " + "auto-marked chain for klass_id=%u tag=%lld target_tag=%llu", + klass_id, (long long)disc_tag, + (unsigned long long)event._target_tag); + } + } else { + TEST_LOG("ReferenceChainTracker::pollWatchedTargets " + "buildChainEvent failed for discovered tag=%lld " + "klass_id=%u slot=%d disc_idx=%d", + (long long)disc_tag, klass_id, s, d); + } + } + break; +} +} + +void ReferenceChainTracker::recordDiscoveredInstance(u32 klass_id, + jlong frontier_tag, + bool leak_correlated) { + // See the declaration's own comment (referenceChains.h) for the noise-eviction rationale. + for (int s = 0; s < _candidate_count; s++) { + if (_candidate_klass_ids[s] != klass_id) { + continue; + } + if (_candidate_discovered_count[s] < MAX_DISCOVERED_INSTANCES_PER_CLASS) { + _candidate_discovered_tags[s][_candidate_discovered_count[s]++] = + frontier_tag; + TEST_LOG("ReferenceChainTracker::recordDiscoveredInstance slot=%d " + "klass_id=%u tag=%lld leak_correlated=%d count=%d", + s, klass_id, (long long)frontier_tag, (int)leak_correlated, + _candidate_discovered_count[s]); + return; + } + if (!leak_correlated) { + return; // full - noise never displaces anything + } + // All slots full and this instance is leak-correlated: evict the first slot held by an entry + // with no leak tag (a noise instance). + for (int d = 0; d < _candidate_discovered_count[s]; d++) { + jlong victim = _candidate_discovered_tags[s][d]; + FrontierEntry victim_entry{}; + if (_frontier == nullptr || + !_frontier->lookup(victim, &victim_entry) || + victim_entry.leak_tag == 0) { + _candidate_discovered_tags[s][d] = frontier_tag; + invalidateResolvedChain(victim); + TEST_LOG("ReferenceChainTracker::recordDiscoveredInstance evicted " + "noise slot=%d idx=%d victim_tag=%lld for leak tag=%lld", + s, d, (long long)victim, (long long)frontier_tag); + return; + } + } + TEST_LOG("ReferenceChainTracker::recordDiscoveredInstance all slots " + "leak-correlated, dropping tag=%lld klass_id=%u", + (long long)frontier_tag, klass_id); + return; + } +} + +bool ReferenceChainTracker::correlateAdmittedLeakTag(jlong frontier_tag, + jlong leak_tag, + u32 klass_id) { + // See the declaration's own comment (referenceChains.h). Called from + // LivenessTracker::tagLeakInstances() on this same thread (pollWatchedTargets -> + // tagLeakInstances), so _candidate_* slot access here never races heapReferenceCallback's + // auto-mark path. + if (_frontier == nullptr) { + return false; + } + FrontierEntry entry{}; + if (!_frontier->lookup(frontier_tag, &entry)) { + return false; // not a live frontier tag (or the search restarted) + } + if (entry.leak_tag != 0) { + // Already correlated (idempotent) - e.g. a second tagLeakInstances round after a post-restart + // re-admission. + return true; + } + _frontier->setLeakTag(frontier_tag, leak_tag); + TEST_LOG_SUMMARY("ReferenceChainTracker::correlateAdmittedLeakTag " + "frontier_tag=%lld leak_tag=%lld depth=%u parent_tag=%lld", + (long long)frontier_tag, (long long)leak_tag, entry.depth, + (long long)entry.parent_tag); + recordDiscoveredInstance(klass_id, frontier_tag, true); + return true; +} + +void ReferenceChainTracker::drainPendingChainEvents( + std::vector *out) { + if (out == nullptr) { + return; + } + // Snapshot-and-keep, not a drain: every cached chain is copied out (and re-stamped so it lands in + // the dumping chunk's window) while the cache itself is left intact, so the same live sample's + // chain re-emits into every chunk it survives into (see _resolved_chains' comment). + u64 now = TSC::ticks(); + _resolved_chains_lock.lock(); + for (const auto &kv : _resolved_chains) { + out->push_back(kv.second.event); + out->back()._start_time = now; + } + _resolved_chains_lock.unlock(); + TEST_LOG_SUMMARY("ReferenceChainTracker::drainPendingChainEvents re-emitted=%d", + (int)out->size()); +} + +void ReferenceChainTracker::enqueuePendingAbandonedEvent() { + // Called right after runPass() (referenceChains.cpp) writes SearchState::ABANDONED, on the same + // thread, before shouldRunPass() gets a chance to call restartSearch() - so + // buildAbandonedEvent()'s live read of _search_state/_abandon_reason/etc. + ReferenceChainAbandonedEvent event; + if (!buildAbandonedEvent(&event)) { + return; + } + // Stamp when the search actually stopped, not when a later dump writes the queued event - an + // abandon is a point-in-time occurrence and dump() can lag it by a whole chunk rotation. + event._start_time = TSC::ticks(); + _pending_abandoned_events_lock.lock(); + if ((int)_pending_abandoned_events.size() >= MAX_PENDING_ABANDONED_EVENTS) { + _pending_abandoned_events_lock.unlock(); + Counters::increment(REFERENCE_CHAIN_EVENTS_DROPPED); + TEST_LOG_SUMMARY("ReferenceChainTracker::enqueuePendingAbandonedEvent dropped, " + "queue full (at MAX_PENDING_ABANDONED_EVENTS=%d)", + MAX_PENDING_ABANDONED_EVENTS); + return; + } + _pending_abandoned_events.push_back(event); + TEST_LOG_SUMMARY("ReferenceChainTracker::enqueuePendingAbandonedEvent reason=%d " + "queue_size=%d", + (int)event._reason, (int)_pending_abandoned_events.size()); + _pending_abandoned_events_lock.unlock(); +} + +void ReferenceChainTracker::drainPendingAbandonedEvents( + std::vector *out) { + if (out == nullptr) { + return; + } + // True drain, unlike drainPendingChainEvents() above: each queued event describes a discrete past + // occurrence, not an ongoing live sample, so once Profiler::dump() (profiler.cpp) has emitted it + // there is nothing left to re-report on the next dump. + _pending_abandoned_events_lock.lock(); + out->insert(out->end(), _pending_abandoned_events.begin(), + _pending_abandoned_events.end()); + _pending_abandoned_events.clear(); + _pending_abandoned_events_lock.unlock(); + TEST_LOG_SUMMARY("ReferenceChainTracker::drainPendingAbandonedEvents drained=%d", + (int)out->size()); +} diff --git a/ddprof-lib/src/main/cpp/referenceChainFrontier.cpp b/ddprof-lib/src/main/cpp/referenceChainFrontier.cpp new file mode 100644 index 000000000..a8d709c6f --- /dev/null +++ b/ddprof-lib/src/main/cpp/referenceChainFrontier.cpp @@ -0,0 +1,426 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "referenceChainFrontier.h" +#include "counters.h" +#include "log.h" +#include "rcDebugLevel.h" +#include +#include +#include +#include +#include + +FrontierTable::FrontierTable(int max_cap) + : _table_size(0), _table_cap(0), _table_max_cap(std::max(max_cap, 0)), + _table(nullptr) { + _table_cap = std::min(INITIAL_TABLE_CAPACITY, _table_max_cap); + if (_table_cap > 0) { + _table = (FrontierEntry *)calloc(_table_cap, sizeof(FrontierEntry)); + if (_table == nullptr) { + _table_cap = 0; + } + } + Counters::increment(REFERENCE_CHAIN_FRONTIER_TABLE_BYTES, + (jlong)_table_cap * sizeof(FrontierEntry)); + Counters::increment(REFERENCE_CHAIN_FRONTIER_TABLE_CAPACITY, _table_cap); +} + +FrontierTable::~FrontierTable() { + Counters::increment(REFERENCE_CHAIN_FRONTIER_TABLE_BYTES, + -(jlong)_table_cap * sizeof(FrontierEntry)); + Counters::increment(REFERENCE_CHAIN_FRONTIER_TABLE_CAPACITY, -_table_cap); + free(_table); +} + +void FrontierTable::resetCapacityForTest(int max_cap) { + _table_lock.lock(); + Counters::increment(REFERENCE_CHAIN_FRONTIER_TABLE_BYTES, + -(jlong)_table_cap * sizeof(FrontierEntry)); + Counters::increment(REFERENCE_CHAIN_FRONTIER_TABLE_CAPACITY, -_table_cap); + free(_table); + _table = nullptr; + _table_max_cap = std::max(max_cap, 0); + _table_cap = std::min(INITIAL_TABLE_CAPACITY, _table_max_cap); + if (_table_cap > 0) { + _table = (FrontierEntry *)calloc(_table_cap, sizeof(FrontierEntry)); + if (_table == nullptr) { + _table_cap = 0; + } + } + Counters::increment(REFERENCE_CHAIN_FRONTIER_TABLE_BYTES, + (jlong)_table_cap * sizeof(FrontierEntry)); + Counters::increment(REFERENCE_CHAIN_FRONTIER_TABLE_CAPACITY, _table_cap); + _table_size.store(0, std::memory_order_relaxed); + _table_lock.unlock(); +} + +bool FrontierTable::growLocked(int required_cap) { + if (required_cap <= _table_cap) { + return true; + } + if (_table_cap >= _table_max_cap) { + return false; + } + + int newcap = _table_cap; + while (newcap < required_cap && newcap < _table_max_cap) { + newcap = newcap == 0 ? std::min(INITIAL_TABLE_CAPACITY, _table_max_cap) + : std::min(newcap * 2, _table_max_cap); + } + if (newcap <= _table_cap) { + return false; + } + + FrontierEntry *tmp = + (FrontierEntry *)realloc(_table, sizeof(FrontierEntry) * newcap); + if (tmp == nullptr) { + Log::debug( + "ReferenceChains: frontier table resize to %d entries failed", newcap); + return false; + } + // realloc() does not zero the newly grown region - clear it so lookup() never returns garbage + // state for a slot that hasn't been inserted yet. + memset(tmp + _table_cap, 0, sizeof(FrontierEntry) * (newcap - _table_cap)); + Counters::increment(REFERENCE_CHAIN_FRONTIER_TABLE_BYTES, + (jlong)(newcap - _table_cap) * sizeof(FrontierEntry)); + Counters::increment(REFERENCE_CHAIN_FRONTIER_TABLE_CAPACITY, + newcap - _table_cap); + _table = tmp; + _table_cap = newcap; + return _table_cap >= required_cap; +} + +bool FrontierTable::insert(jlong tag, jlong parent_tag, u32 referrer_klass, + u32 depth, u8 state, u8 root_kind, + jlong class_tag, jint referrer_field_index, + u8 edge_kind, jlong referrer_class_tag) { + if (tag <= 0 || tag - 1 >= (jlong)INT_MAX) { + return false; + } + int idx = (int)(tag - 1); + + // Exclusive lock for the whole write (growLocked() already requires it) - a shared lock here + // would not exclude lookup()'s own shared-mode read of the same slot, letting a concurrent reader + // observe a torn entry. + _table_lock.lock(); + if (idx >= _table_cap && !growLocked(idx + 1)) { + _table_lock.unlock(); + Log::debug("ReferenceChains: frontier table capacity exhausted " + "(cap=%d, max=%d, tag=%lld)", + _table_cap, _table_max_cap, (long long)tag); + return false; + } + _table[idx].parent_tag = parent_tag; + _table[idx].referrer_klass = referrer_klass; + _table[idx].depth = depth; + _table[idx].state = state; + _table[idx].root_kind = root_kind; + _table[idx].class_tag = class_tag; + _table[idx].leak_tag = 0; + _table[idx].referrer_field_index = referrer_field_index; + _table[idx].edge_kind = edge_kind; + _table[idx].referrer_class_tag = referrer_class_tag; + + // Published under the same exclusive lock as the slot write: advancing + // _table_size only after unlock lets a concurrent insert for a higher index + // CAS the size past this entry's idx first, and a shared-lock reader then + // passes its `idx < _table_size` check on a value published outside the + // lock - reading this slot before its write is guaranteed visible. Keeping + // the write+publish pair inside the lock makes the size an exact bound on + // fully-written slots for every lock-ordered reader. + int sz = _table_size.load(std::memory_order_relaxed); + while (sz < idx + 1 && + !_table_size.compare_exchange_weak(sz, idx + 1, + std::memory_order_relaxed)) { + // sz reloaded with the current value by compare_exchange_weak on failure; retry until either + // this thread wins or another thread already advanced _table_size past idx + 1. + } + _table_lock.unlock(); + return true; +} + +bool FrontierTable::lookup(jlong tag, FrontierEntry *out) { + if (tag <= 0 || tag - 1 >= (jlong)INT_MAX) { + return false; + } + int idx = (int)(tag - 1); + + bool found = false; + _table_lock.lockShared(); + if (idx < _table_size) { + *out = _table[idx]; + found = true; + } + _table_lock.unlockShared(); + return found; +} + +bool FrontierTable::lookupLocked(jlong tag, FrontierEntry *out) const { + if (tag <= 0 || tag - 1 >= (jlong)INT_MAX) { + return false; + } + int idx = (int)(tag - 1); + if (idx < _table_size) { + *out = _table[idx]; + return true; + } + return false; +} + +void FrontierTable::clear(jlong tag) { + if (tag <= 0 || tag - 1 >= (jlong)INT_MAX) { + return; + } + int idx = (int)(tag - 1); + + // Exclusive lock: this mutates a slot lookup() may be reading concurrently under its own shared + // lock (see insert()'s own comment above). + _table_lock.lock(); + if (idx < _table_size) { + _table[idx].state = FrontierEntryState::ABANDONED; + } + _table_lock.unlock(); +} + +void FrontierTable::markEdge(jlong tag) { + if (tag <= 0 || tag - 1 >= (jlong)INT_MAX) { + return; + } + int idx = (int)(tag - 1); + + _table_lock.lock(); + if (idx < _table_size) { + _table[idx].state = FrontierEntryState::EDGE; + } + _table_lock.unlock(); +} + +void FrontierTable::markExpanded(jlong tag) { + if (tag <= 0 || tag - 1 >= (jlong)INT_MAX) { + return; + } + int idx = (int)(tag - 1); + + _table_lock.lock(); + if (idx < _table_size) { + _table[idx].state = FrontierEntryState::EXPANDED; + } + _table_lock.unlock(); +} + +void FrontierTable::updateRootKind(jlong tag, u8 root_kind) { + if (tag <= 0 || tag - 1 >= (jlong)INT_MAX) { + return; + } + int idx = (int)(tag - 1); + + _table_lock.lock(); + if (idx < _table_size) { + _table[idx].root_kind = root_kind; + } + _table_lock.unlock(); +} + +bool FrontierTable::improveChain(jlong tag, jlong parent_tag, + u32 referrer_klass, u32 depth, + u8 root_kind, jint referrer_field_index, + u8 edge_kind, jlong referrer_class_tag) { + // Replace a shallow root-attached entry (parent_tag == 0, depth == 0) with a deeper + // chain-attached entry when the object is reached via a longer path. + if (tag <= 0 || tag - 1 >= (jlong)INT_MAX) { + return false; + } + // a "chain" whose parent is the entry itself is never an improvement - it is the self-edge a + // this-field produces, and it is REAL in the heap: every java.util.Collections$Synchronized* + // holder carries mutex == this, so walking such a holder's own subtree (the rotation anchor walk + // or a BFS descent) re-reports the holder as its own child through that field. + if (parent_tag == tag) { + return false; + } + // Ancestor-walk bound for the cycle guard below - see its comment. + static constexpr int IMPROVE_CHAIN_GUARD_MAX_HOPS = 4096; + { + int guard_hops = 0; + jlong cur = parent_tag; + while (cur > 0 && guard_hops <= IMPROVE_CHAIN_GUARD_MAX_HOPS) { + if (cur == tag) { + TEST_LOG_SUMMARY("FrontierTable::improveChain refused: new parent " + "chain routes through the entry (cycle) tag=%lld " + "parent_tag=%lld depth=%u", + (long long)tag, (long long)parent_tag, depth); + return false; + } + FrontierEntry guard_entry{}; + if (!lookup(cur, &guard_entry)) { + break; + } + cur = guard_entry.parent_tag; + guard_hops++; + } + if (cur != 0) { + // The parent chain neither reached a root nor was fully verified within the guard bound - + // applying this improve could embed an unresolvable (cyclic or dangling) chain. + TEST_LOG_SUMMARY("FrontierTable::improveChain refused: unverifiable " + "parent chain tag=%lld parent_tag=%lld depth=%u " + "walk_stopped_at=%lld", + (long long)tag, (long long)parent_tag, depth, + (long long)cur); + return false; + } + } + + int idx = (int)(tag - 1); + + _table_lock.lock(); + bool improved = false; + if (idx < _table_size && depth > _table[idx].depth) { + _table[idx].parent_tag = parent_tag; + _table[idx].referrer_klass = referrer_klass; + _table[idx].depth = depth; + _table[idx].root_kind = root_kind; + _table[idx].referrer_field_index = referrer_field_index; + _table[idx].edge_kind = edge_kind; + _table[idx].referrer_class_tag = referrer_class_tag; + improved = true; + } + _table_lock.unlock(); + return improved; +} + +bool FrontierTable::reparentToDurableRoot(jlong tag, jlong new_parent_tag, + u32 referrer_klass, + jint referrer_field_index, + u8 edge_kind) { + // See the declaration's own comment (referenceChains.h) for why this exists as a sibling of + // improveChain(): equal-depth depth-1 noise->real re-parenting. + if (tag <= 0 || tag - 1 >= (jlong)INT_MAX || new_parent_tag <= 0 || + new_parent_tag - 1 >= (jlong)INT_MAX || new_parent_tag == tag) { + return false; + } + int idx = (int)(tag - 1); + int new_par_idx = (int)(new_parent_tag - 1); + + _table_lock.lock(); + bool swapped = false; + if (idx < _table_size && _table[idx].depth == 1 && + _table[idx].parent_tag > 0 && _table[idx].parent_tag != new_parent_tag) { + int old_par_idx = (int)(_table[idx].parent_tag - 1); + if (old_par_idx >= 0 && old_par_idx < _table_size && + new_par_idx < _table_size && + _table[new_par_idx].parent_tag == 0 && + _table[new_par_idx].root_kind != 0 && + !isTransientRootKind(_table[new_par_idx].root_kind) && + _table[old_par_idx].parent_tag == 0 && + isTransientRootKind(_table[old_par_idx].root_kind)) { + // New parent is a root-attached DURABLE root (static field, JNI global, thread) and the + // current parent is a root-attached TRANSIENT one - same depth, strictly better retention + // explanation. + _table[idx].parent_tag = new_parent_tag; + _table[idx].referrer_klass = referrer_klass; + _table[idx].referrer_field_index = referrer_field_index; + _table[idx].edge_kind = edge_kind; + swapped = true; + } + } + _table_lock.unlock(); + return swapped; +} + +bool FrontierTable::reconstructChain(jlong target_tag, + std::vector *out_chain, + u8 *out_root_kind, + std::vector *out_edges, + FrontierEntry *out_terminal) { + FrontierEntry entry{}; + if (!lookup(target_tag, &entry)) { + return false; + } + + std::vector chain; + std::vector edges; + jlong tag = target_tag; + u8 root_kind = 0; + int hops = 0; + // Bounded by maxCapacity(): every tag maps to a distinct slot (this table's "tags/slots are never + // reused" invariant, see the class comment above), so a well-formed parent_tag chain can visit at + // most maxCapacity() slots before either reaching parent_tag == 0 or repeating a slot. + for (; hops <= maxCapacity() && tag != 0; hops++) { + if (!lookup(tag, &entry)) { + // parent_tag pointed at a tag that was never inserted - should not happen for a chain built + // entirely within one BFS pass, but do not fabricate a partial chain silently. + TEST_LOG_SUMMARY("FrontierTable::reconstructChain broken chain: " + "target=%lld failed at hop=%d tag=%lld (parent tag never " + "inserted)", + (long long)target_tag, hops, (long long)tag); + return false; + } + chain.push_back(entry.referrer_klass); + if (out_edges != nullptr) { + // edges[i] describes the edge INTO chain[i]: the entry's own recorded edge identity, plus the + // referrer's class tag - the parent entry's own class for interior hops, the declaring class + // for root-attached static edges (FrontierEntry::referrer_class_tag, filled only there, since + // a class-object referrer has no parent entry to read from). + ChainHopEdge hop{}; + hop.field_index = entry.referrer_field_index; + if (entry.parent_tag == 0) { + hop.edge_kind = entry.root_kind; + hop.referrer_class_tag = entry.referrer_class_tag; + } else { + hop.edge_kind = entry.edge_kind; + FrontierEntry parent_entry{}; + hop.referrer_class_tag = + lookup(entry.parent_tag, &parent_entry) ? parent_entry.class_tag : 0; + } + edges.push_back(hop); + } + // Deliberately NOT marking the walked entries EDGE: the EDGE state was write-only "degenerate + // EdgeStore" bookkeeping (nothing ever reads it), while the rotation collectors select EXPANDED + // entries - demoting a resolved path's holders to EDGE made them permanently invisible to + // rotation, so later leak instances behind a changed holder were never re-discovered. + root_kind = entry.root_kind; + tag = entry.parent_tag; + } + if (tag != 0) { + // Ran past the defensive hop bound without reaching a root-attached entry (parent_tag == 0) - a + // corrupted/cyclic chain. + { + jlong dbg = target_tag; + FrontierEntry dbg_e{}; + char pairs[256]; + size_t off = 0; + for (int d = 0; d < 12 && dbg != 0 && off < sizeof(pairs) - 24; d++) { + if (!lookup(dbg, &dbg_e)) { + break; + } + off += (size_t)snprintf(pairs + off, sizeof(pairs) - off, "%lld->%lld ", + (long long)dbg, (long long)dbg_e.parent_tag); + dbg = dbg_e.parent_tag; + } + TEST_LOG_SUMMARY("FrontierTable::reconstructChain hop bound: " + "target=%lld stuck at tag=%lld after %d hops - cyclic or " + "corrupt parent chain; hops: %.*s", + (long long)target_tag, (long long)tag, hops, (int)off, pairs); + } + return false; + } + + *out_chain = std::move(chain); + if (out_edges != nullptr) { + *out_edges = std::move(edges); + } + if (out_root_kind != nullptr) { + // The loop's last iteration is always the root-attached entry (the one whose parent_tag == 0 + // that just ended the loop), so root_kind here is that entry's own FrontierEntry::root_kind. + *out_root_kind = root_kind; + } + if (out_terminal != nullptr) { + // `entry` still holds the loop's last successful lookup - the root-attached entry that ended + // the walk. + *out_terminal = entry; + } + return true; +} + diff --git a/ddprof-lib/src/main/cpp/referenceChainFrontier.h b/ddprof-lib/src/main/cpp/referenceChainFrontier.h new file mode 100644 index 000000000..e744e840e --- /dev/null +++ b/ddprof-lib/src/main/cpp/referenceChainFrontier.h @@ -0,0 +1,288 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + + +#ifndef _REFERENCECHAINFRONTIER_H +#define _REFERENCECHAINFRONTIER_H + +#include "arch.h" +#include "common.h" +#include "spinLock.h" +#include +#include +#include +#include +#include +#include +#include + +namespace FrontierEntryState { +constexpr u8 FRONTIER = 0; // discovered, not yet expanded by FollowReferences +constexpr u8 EXPANDED = 1; // expanded; children (if any) are in the table +constexpr u8 EDGE = 2; // on a path toward a target sample (EdgeStore) +constexpr u8 ABANDONED = 3; // tag released; entry kept only to avoid reuse +} // namespace FrontierEntryState + +// Search outcome; per-pass truncation does not imply abandonment. +namespace SearchState { +constexpr u8 RUNNING = 0; // at least one more pass may still make progress +constexpr u8 COMPLETED = 1; // reachable graph fully explored within caps +constexpr u8 ABANDONED = 2; // TTL or frontier-size cap forced an incomplete stop +} // namespace SearchState + +// Reason reported when a search is abandoned. +namespace SearchAbandonReason { +constexpr u8 NONE = 0; // not (yet) abandoned +constexpr u8 FRONTIER_CAP = 1; // frontier-size cap hit +constexpr u8 TTL = 2; // wall-clock TTL exceeded with work still pending +// Canary candidate-discovery has made no progress for NO_PROGRESS_PASS_LIMIT consecutive passes. +constexpr u8 CANARY_STUCK = 3; +} // namespace SearchAbandonReason + +// Metadata for one tagged frontier object. +typedef struct FrontierEntry { + jlong parent_tag; // links back to the record that discovered this one + u32 referrer_klass; // StringDictionary id, 0 = unresolved/none + u32 depth; // hop count from the frontier's seed, for the hop cap + u8 state; // one of FrontierEntryState's constants + // The leak tag assigned by LivenessTracker to this specific tracked object, copied from the JVMTI + // tag at admission time. + jlong leak_tag; + // jvmtiHeapReferenceKind of the edge that admitted this entry, but only meaningful when + // parent_tag == 0 (this entry is root-attached) - 0 (no JVMTI_HEAP_REFERENCE_* value is 0) for + // every other entry, since a non-root entry's own referrer edge kind is not what + // reconstructChain()'s callers want to report (they want to label the chain's root, not every + // hop). + u8 root_kind; + // Raw JVMTI class tag of THIS entry's own object, from the shared, process-wide allocator + // (classTagAllocator.h) - NOT referrer_klass above (a classMap dictionary id, which can differ + // for the same class at different times if that dictionary gets compacted/regenerated - see + // LivenessTracker::KlassPopulationEntry::stable_class_tag's own comment for the bug this was + // found fixing). + jlong class_tag; + + // Retention-edge identity of the edge that admitted THIS entry, captured at admission time for + // the same cannot-replay-the-callback reason as class_tag above: it lets the emitted + // datadog.ReferenceChain name the field each hop is retained through, turning the bare class list + // into a readable path ("LeakHolder.SINK -> HashMap.table -> Entry.value") - which is the + // diagnostic point of the whole feature. + jint referrer_field_index; + + // jvmtiHeapReferenceKind of the admitting edge for an INTERIOR hop (an entry with parent_tag != 0 + // - "this object was reached from its parent via this kind of edge"). + u8 edge_kind; + + // Referrer's class tag when the referrer is a CLASS OBJECT rather than a frontier entry (a + // root-attached static-field admission - the referrer is the declaring class, parent_tag == 0 so + // there is no parent entry to read a class from). + jlong referrer_class_tag; +} FrontierEntry; + +// Per-hop retention-edge identity collected by reconstructChain() alongside the class chain: +// everything needed at emission to label HOW chain[i] is retained (the field of chain[i+1] pointing +// at it, or the root edge for the last hop). +typedef struct ChainHopEdge { + // FrontierEntry::referrer_field_index/edge_kind of the entry for chain[i] + jint field_index; // -1 = not a field/static-field edge + u8 edge_kind; // admitting edge kind (root hops: root_kind) + // The referrer's raw class tag: the PARENT entry's class_tag for interior hops, + // FrontierEntry::referrer_class_tag for root-attached hops. + jlong referrer_class_tag; +} ChainHopEdge; + +// Higher values identify longer-lived root kinds. +inline int rootKindDurability(u8 root_kind) { + switch (root_kind) { + case JVMTI_HEAP_REFERENCE_STATIC_FIELD: + case JVMTI_HEAP_REFERENCE_SYSTEM_CLASS: + return 3; + case JVMTI_HEAP_REFERENCE_JNI_GLOBAL: + return 2; + case JVMTI_HEAP_REFERENCE_MONITOR: + case JVMTI_HEAP_REFERENCE_STACK_LOCAL: + case JVMTI_HEAP_REFERENCE_JNI_LOCAL: + case JVMTI_HEAP_REFERENCE_THREAD: + case JVMTI_HEAP_REFERENCE_OTHER: + return 1; + default: + return 0; // root_kind's own "not set"/non-root-attached value + } +} + +// Stack and JNI locals are transient roots. +inline bool isTransientRootKind(u8 root_kind) { + return root_kind == JVMTI_HEAP_REFERENCE_STACK_LOCAL || + root_kind == JVMTI_HEAP_REFERENCE_JNI_LOCAL; +} + +// Tag-indexed slot table storing FrontierEntry metadata, modeled on LivenessTracker's TrackingEntry +// table (livenessTracker.h:21-30): CAS-safe doubling resize under a signal-safe SpinLock +// (spinLock.h), reusing its shared/exclusive split so reads (lookup) never race a resize. +class alignas(alignof(SpinLock)) FrontierTable { +private: + // Provisional default pending empirical tuning - not benchmark-derived. + static constexpr int INITIAL_TABLE_CAPACITY = 1024; + + // mutable: capacity()/maxCapacity() below are const accessors that still need to take this lock + // to read _table_cap/_table_max_cap safely. + mutable SpinLock _table_lock; + // 1 + highest index ever inserted (informational upper bound for lookup(); never shrinks, since + // tags/slots are never reused). + std::atomic _table_size; + int _table_cap; + int _table_max_cap; + FrontierEntry *_table; + + // Grows _table (doubling) until it holds at least `required_cap` slots or _table_max_cap is + // reached. + bool growLocked(int required_cap); + +public: + // `max_cap` <= 0 disables the table (capacity() stays 0, every insert() reports exhaustion) - + // callers are expected to guard on the config flag before constructing one, but this makes a + // misconfigured cap fail safe rather than crash. + explicit FrontierTable(int max_cap); + ~FrontierTable(); + + FrontierTable(const FrontierTable &) = delete; + FrontierTable &operator=(const FrontierTable &) = delete; + + // Writes (parent_tag, referrer_klass, depth, state) into the slot for `tag` (index = tag - 1), + // growing the table if needed. + bool insert(jlong tag, jlong parent_tag, u32 referrer_klass, u32 depth, + u8 state = FrontierEntryState::FRONTIER, u8 root_kind = 0, + jlong class_tag = 0, + jint referrer_field_index = -1, u8 edge_kind = 0, + jlong referrer_class_tag = 0); + + // Reads the slot for `tag` into *out. Returns false (leaving *out untouched) if `tag` is not + // positive or has never been inserted. + bool lookup(jlong tag, FrontierEntry *out); + + // Runs `fn(this)` with the shared lock held for the whole call, for a caller that needs to look + // up many tags back to back (e.g. the rotation collectors' O(size()) sweeps in + // referenceChains.cpp) under ONE lock acquisition, instead of paying SpinLock's lock/unlock cost + // on every single lookup() call. + template void withSharedLock(Fn &&fn) const { + SharedLockGuard guard(&_table_lock); + fn(this); + } + + // Same as lookup() above, but assumes the caller already holds the shared lock via + // withSharedLock() below. + bool lookupLocked(jlong tag, FrontierEntry *out) const; + + // Marks metadata abandoned; the caller must clear the JVMTI tag. + void clear(jlong tag); + + // Marks the slot as part of a resolved path. + void markEdge(jlong tag); + + // Marks the slot after all outgoing edges have been visited. + void markExpanded(jlong tag); + + // Updates only the recorded root kind. + void updateRootKind(jlong tag, u8 root_kind); + + // Set the leak tag on a frontier entry (the JVMTI tag assigned by LivenessTracker to this + // specific tracked leaking object). + void setLeakTag(jlong tag, jlong leak_tag) { + if (tag <= 0 || tag - 1 >= (jlong)INT_MAX) { + return; + } + int idx = (int)(tag - 1); + _table_lock.lock(); + if (idx < _table_size) { + _table[idx].leak_tag = leak_tag; + } + _table_lock.unlock(); + } + + // Replace a shallow root-attached entry (parent_tag == 0, depth == 0) with a deeper + // chain-attached entry when the object is reached via a longer path. + bool improveChain(jlong tag, jlong parent_tag, u32 referrer_klass, + u32 depth, u8 root_kind, jint referrer_field_index = -1, + u8 edge_kind = 0, jlong referrer_class_tag = 0); + + // Equal-depth re-parenting, the one case improveChain() above cannot express: a depth-1 entry + // whose current parent is a TRANSIENT root (stack local / JNI local - a momentarily-live frame) + // is re-parented to a DURABLE root-attached parent (static field, JNI global, thread) when one is + // seen admitting the same object at the same depth. + bool reparentToDurableRoot(jlong tag, jlong new_parent_tag, + u32 referrer_klass, + jint referrer_field_index = -1, u8 edge_kind = 0); + + // Walks parent_tag links starting at `target_tag` back to a root-attached entry (parent_tag == + // 0), appending each visited entry's referrer_klass to *out_chain in leaf-to-root order. + bool reconstructChain(jlong target_tag, std::vector *out_chain, + u8 *out_root_kind = nullptr, + std::vector *out_edges = nullptr, + FrontierEntry *out_terminal = nullptr); + + // Search restart (ReferenceChainTracker::restartSearch(), this class's own header comment): marks + // every slot unoccupied again without releasing _table's allocation - a new search's nextTag() + // sequence restarts at 1, reusing these same slot indices, so lookup()/insert() must not read + // back the previous search's now-irrelevant entries for them. + void resetForRestart() { + _table_lock.lock(); + _table_size.store(0, std::memory_order_relaxed); + _table_lock.unlock(); + } + + // Debug-only test seam (ReferenceChainTracker::resetSearchStateForTest()). + void resetCapacityForTest(int max_cap); + + // _table_cap/_table_max_cap are plain ints, not atomics like _table_size, and + // resetCapacityForTest() (debug-only test seam, see its own comment) rewrites both under + // _table_lock after freeing/reallocating _table. + int capacity() const { + _table_lock.lock(); + int cap = _table_cap; + _table_lock.unlock(); + return cap; + } + int maxCapacity() const { + _table_lock.lock(); + int max_cap = _table_max_cap; + _table_lock.unlock(); + return max_cap; + } + + // Current upper bound on assigned slots (mirrors _table_size's own comment: "1 + highest index + // ever inserted"). + int size() const { return _table_size.load(std::memory_order_relaxed); } +}; + +// Tag-indexed table mapping a *class* tag (see ReferenceChainTracker::nextClassTag() - always +// negative, a namespace disjoint from the positive FrontierTable object tags above so a raw tag +// value alone always tells the heap-walk callback which table it belongs to) to the +// StringDictionary id of that class's resolved name (Profiler::classMap(), the same interning table +// LivenessTracker uses via Profiler::lookupClass(), livenessTracker.cpp). +class ClassTagTable { +private: + std::unordered_map _table; + +public: + void insert(jlong class_tag, u32 dict_id) { _table[class_tag] = dict_id; } + + // Returns the StringDictionary id for `class_tag`, or 0 if it was never inserted (0 is + // StringDictionary's own "no entry" sentinel too, so this composes with + // FrontierEntry::referrer_klass's documented 0 = unresolved/none convention without a separate + // "found" out-parameter). + u32 resolve(jlong class_tag) const { + auto it = _table.find(class_tag); + return it != _table.end() ? it->second : 0; + } + + size_t size() const { return _table.size(); } + + // Drops every cached class_tag -> dict_id mapping - used when the underlying StringDictionary + // itself was reset (see ReferenceChainTracker::_last_class_map_generation's comment) and every id + // here now points at a namespace that no longer exists. + void clear() { _table.clear(); } +}; + + +#endif // _REFERENCECHAINFRONTIER_H diff --git a/ddprof-lib/src/main/cpp/referenceChainInternal.h b/ddprof-lib/src/main/cpp/referenceChainInternal.h new file mode 100644 index 000000000..721b5683d --- /dev/null +++ b/ddprof-lib/src/main/cpp/referenceChainInternal.h @@ -0,0 +1,84 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + + +#ifndef _REFERENCECHAININTERNAL_H +#define _REFERENCECHAININTERNAL_H + +#include "arch.h" +#include +#include + +class FrontierTable; +class ReferenceChainTracker; + +extern thread_local bool t_inGCCallback; + +struct ReferenceChainPassContext { + ReferenceChainTracker *tracker; + FrontierTable *frontier; + int hop_cap; + int budget; + int edges_admitted; + bool truncated; + + // Set only when `truncated` became true because frontier->insert() itself reported capacity + // exhaustion, as opposed to edges_admitted reaching budget. + bool frontier_cap_hit; + + // ARRAY-HOLDER BATCHING: when non-null, expandFrontier() is driving a one-hop expansion of a + // batch of boundary objects passed to a single FollowReferences(initial_object=holder_array) + // call. + std::unordered_set *batch_tags = nullptr; + + // Rolling resume cursor for expandFrontier(): tracks the tag of the last batch entry that + // FollowReferences visited (the callback at the batch_tags descent-gate updates this). + jlong _last_visited_batch_tag = 0; + + // Batch entries the callback finished visiting before the truncation (set only by + // heapReferenceCallback()'s batch_tags descent gate). + std::unordered_set *_completed_batch_tags = nullptr; + + // Set only by admitStaticFieldRoots(): the seed holder array for that sweep holds loaded-class + // objects (negative-tagged by resolveLoadedClasses(), see the *tag_ptr < 0 branch below), and the + // whole point of the sweep is to walk past that holder->class edge into each class's own outgoing + // references - chiefly STATIC_FIELD - which the *tag_ptr < 0 check would otherwise stop cold + // before FollowReferences ever gets to report them. + bool static_field_seed = false; + + // PER-CLASS NON-STATIC QUOTA (admitStaticFieldRoots() only). + jlong _seed_class_tag = 0; // negative tag of the class currently + // being descended (0 before the first class edge is seen) + int _class_other_admitted = 0; // non-STATIC_FIELD edges admitted for + // the current class this lap + int _class_other_cap = 0; // per-class cap; 0 disables the quota + // (admit all) when not in seed sweep + // Number of distinct classes entered so far in this chunk's descent (incremented on each + // class-boundary tag change). + int _classes_in_chunk_visited = 0; + + // Amortizes tracker->_pass_deadline_ns's OS::nanotime() check (heapReference + // Callback()/heapRootCallback() run once per visited edge/root - checking wall-clock on literally + // every call would add real overhead on a large heap) - checked only every 4096th call, local to + // this ctx so each of runPassManualWalk()'s several sub-calls (root enum, static-field sweep, + // expandFrontier(), rotation) starts its own count. + int deadline_check_counter = 0; + + // True while expandFrontier() is walking a batch drawn from _priority_expand (a + // rotation-selected, already-EXPANDED parent) rather than the ordinary _pending_expand backlog - + // see _priority_expand's own comment. + bool admit_priority = false; + + // DESCEND-WALK controls (descendFromAnchor()'s calls only; null/0 everywhere else, so every gate + // below is a no-op for the ordinary walk phases): _no_descend_class_tags: exact class tags to + // neither admit nor descend into for the duration of this walk. + static constexpr int NO_DESCEND_CLASS_CAP = 8; + jlong _no_descend_class_tags[NO_DESCEND_CLASS_CAP] = {0}; + int _no_descend_class_tag_count = 0; + jlong _descent_anchor_tag = 0; + jlong _anchor_descend_class_tag = 0; +}; + +#endif // _REFERENCECHAININTERNAL_H diff --git a/ddprof-lib/src/main/cpp/referenceChainLabels.cpp b/ddprof-lib/src/main/cpp/referenceChainLabels.cpp new file mode 100644 index 000000000..2bf17d977 --- /dev/null +++ b/ddprof-lib/src/main/cpp/referenceChainLabels.cpp @@ -0,0 +1,328 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "referenceChains.h" +#include "referenceChainInternal.h" +#include "common.h" +#include "counters.h" +#include "jniHelper.h" +#include "jvmThread.h" +#include "livenessTracker.h" +#include "log.h" +#include "objectSampler.h" +#include "os.h" +#include "profiler.h" +#include "rcDebugLevel.h" +#include "tsc.h" +#include "vmEntry.h" +#include +#include +#include +#include +#include +#include +#include +#include + +// Retention-edge labels: naming the field each chain hop is retained +namespace { + +// Fallback label for a hop whose edge is not a field reference (or whose field ordinal could not be +// decoded) - the edge KIND, never a fabricated name. +const char *hopEdgeKindLabel(u8 kind) { + switch (kind) { + case JVMTI_HEAP_REFERENCE_CLASS: + return "class"; + case JVMTI_HEAP_REFERENCE_FIELD: + return "field"; + case JVMTI_HEAP_REFERENCE_ARRAY_ELEMENT: + return "element"; + case JVMTI_HEAP_REFERENCE_CLASS_LOADER: + return "class_loader"; + case JVMTI_HEAP_REFERENCE_SIGNERS: + return "signers"; + case JVMTI_HEAP_REFERENCE_PROTECTION_DOMAIN: + return "protection_domain"; + case JVMTI_HEAP_REFERENCE_INTERFACE: + return "interface"; + case JVMTI_HEAP_REFERENCE_STATIC_FIELD: + return "static_field"; + case JVMTI_HEAP_REFERENCE_CONSTANT_POOL: + return "constant_pool"; + case JVMTI_HEAP_REFERENCE_SUPERCLASS: + return "superclass"; + case JVMTI_HEAP_REFERENCE_JNI_GLOBAL: + return "jni_global"; + case JVMTI_HEAP_REFERENCE_SYSTEM_CLASS: + return "system_class"; + case JVMTI_HEAP_REFERENCE_MONITOR: + return "monitor"; + case JVMTI_HEAP_REFERENCE_STACK_LOCAL: + return "stack_local"; + case JVMTI_HEAP_REFERENCE_JNI_LOCAL: + return "jni_local"; + case JVMTI_HEAP_REFERENCE_THREAD: + return "thread"; + case JVMTI_HEAP_REFERENCE_OTHER: + return "other"; + default: + return "unknown"; + } +} + +// Appends the own-declared field names of `cls` to *out, in GetClassFields() order. +bool appendClassFieldNames(jvmtiEnv *jvmti, JNIEnv *jni, jclass cls, + std::vector *out) { + jint count = 0; + jfieldID *fields = nullptr; + if (jvmti->GetClassFields(cls, &count, &fields) != JVMTI_ERROR_NONE) { + return false; + } + bool ok = true; + for (jint i = 0; i < count; i++) { + char *name = nullptr; + if (jvmti->GetFieldName(cls, fields[i], &name, nullptr, nullptr) != + JVMTI_ERROR_NONE || + name == nullptr) { + ok = false; + break; + } + out->emplace_back(name); + jvmti->Deallocate((unsigned char *)name); + } + jvmti->Deallocate((unsigned char *)fields); + return ok; +} + +// Counts `iface`'s own declared fields plus every transitive superinterface's, each interface +// counted exactly once (keyed on the shared `seen` set insert - an interface's own fields go in +// only when its tag is newly seen, so a diamond's shared parent contributes once no matter how many +// branches reach it). +jlong interfaceSubtreeFieldCount(jvmtiEnv *jvmti, JNIEnv *jni, jclass iface, + std::unordered_set *seen) { + jlong tag = 0; + if (jvmti->GetTag(iface, &tag) != JVMTI_ERROR_NONE || tag == 0) { + // Untagged interface: cannot dedupe reliably - fail the whole decode rather than risk double + // counting. + return -1; + } + if (!seen->insert(tag).second) { + return 0; // already counted this interface (a shared subinterface) + } + jlong total = 0; + jint field_count = 0; + jfieldID *fields = nullptr; + if (jvmti->GetClassFields(iface, &field_count, &fields) == + JVMTI_ERROR_NONE) { + // This interface's own fields count toward any implementor's ordinal base - matching the spec's + // "count of the fields in all the interfaces implemented by C" (jvmtiHeapReferenceInfoField). + total += field_count; + jvmti->Deallocate((unsigned char *)fields); + } + jint iface_count = 0; + jclass *supers = nullptr; + if (jvmti->GetImplementedInterfaces(iface, &iface_count, &supers) != + JVMTI_ERROR_NONE) { + return -1; + } + bool ok = true; + for (jint i = 0; i < iface_count; i++) { + if (supers[i] == nullptr) { + continue; + } + jlong sub = interfaceSubtreeFieldCount(jvmti, jni, supers[i], seen); + jni->DeleteLocalRef(supers[i]); + if (sub < 0) { + ok = false; + } else if (ok) { + total += sub; + } + } + jvmti->Deallocate((unsigned char *)supers); + return ok ? total : -1; +} + +// Sums the field counts of every interface transitively implemented/extended by `cls`, each +// interface counted exactly once (see interfaceSubtreeFieldCount() above - the own-field add lives +// behind the shared seen-set insert, so an interface diamond no longer double-counts its shared +// parent). +jlong interfaceFieldCount(jvmtiEnv *jvmti, JNIEnv *jni, jclass cls, + std::unordered_set *seen) { + jint iface_count = 0; + jclass *ifaces = nullptr; + if (jvmti->GetImplementedInterfaces(cls, &iface_count, &ifaces) != + JVMTI_ERROR_NONE) { + return -1; + } + jlong total = 0; + bool ok = true; + for (jint i = 0; i < iface_count; i++) { + if (ifaces[i] == nullptr) { + continue; + } + jlong sub = interfaceSubtreeFieldCount(jvmti, jni, ifaces[i], seen); + jni->DeleteLocalRef(ifaces[i]); + if (sub < 0) { + ok = false; + } else if (ok) { + total += sub; + } + } + jvmti->Deallocate((unsigned char *)ifaces); + return ok ? total : -1; +} + +} // namespace + +const ReferenceChainTracker::HopLabelClass * +ReferenceChainTracker::hopLabelClassFor(jvmtiEnv *jvmti, JNIEnv *jni, + jlong class_tag) { + auto it = _hop_label_cache.find(class_tag); + if (it != _hop_label_cache.end()) { + return &it->second; + } + // Bounded: chains reference few distinct referrer classes; a wholesale clear at the cap (rather + // than LRU eviction) keeps this O(1) and is correct because the cache is purely derived state - + // any cleared entry is transparently rebuilt on its next hop. + if (_hop_label_cache.size() >= HOP_LABEL_CLASS_CACHE_CAP) { + _hop_label_cache.clear(); + } + HopLabelClass entry{}; + entry.class_tag = class_tag; + entry.decode_failed = true; // until proven otherwise + do { + // GetSuperclass is a JNI (not JVMTI) function - modern JVMTI dropped it (the spec delivers + // superclass references via heap callbacks, jvmti.xml's JVMTI_HEAP_REFERENCE_SUPERCLASS note); + // the rest are JVMTI slots. + if (jvmti->functions->GetObjectsWithTags == nullptr || + jvmti->functions->IsInterface == nullptr || + jvmti->functions->GetImplementedInterfaces == nullptr || + jvmti->functions->GetClassFields == nullptr || + jvmti->functions->GetFieldName == nullptr || + jni->functions->GetSuperclass == nullptr) { + break; + } + // Resolve the class object from its raw tag (negative - the shared allocator's class tags; + // GetObjectsWithTags accepts any tag value). + jint count = 0; + jobject *objects = nullptr; + jlong *tags = nullptr; + if (jvmti->GetObjectsWithTags(1, &class_tag, &count, &objects, &tags) != + JVMTI_ERROR_NONE || + count != 1 || objects == nullptr || objects[0] == nullptr) { + // Both result arrays are JVMTI-allocated on success and must be Deallocate()d by the caller - + // same contract as every other GetObjectsWithTags() call site in this file. + if (objects != nullptr) { + jvmti->Deallocate((unsigned char *)objects); + } + if (tags != nullptr) { + jvmti->Deallocate((unsigned char *)tags); + } + break; + } + jclass cls = (jclass)objects[0]; + // The arrays were only needed to obtain the class object - the jobject handle stays valid on + // its own - so release them before the (multiple, break-exited) decode branches below, which + // otherwise all leak them. + jvmti->Deallocate((unsigned char *)objects); + jvmti->Deallocate((unsigned char *)tags); + jboolean is_interface = JNI_FALSE; + std::vector names; + bool ok = false; + if (jvmti->IsInterface(cls, &is_interface) == JVMTI_ERROR_NONE) { + if (is_interface) { + // The spec's INTERFACE branch: base = fields of all superinterfaces of I, then I's own + // fields (jvmtiHeapReferenceInfoField). + std::unordered_set seen; + jlong base = interfaceFieldCount(jvmti, jni, cls, &seen); + if (base >= 0) { + names.resize((size_t)base); // positioned but unnamed: ordinal [0, + // base) is interface fields, only reachable through an + // interface branch decode of a superinterface + ok = appendClassFieldNames(jvmti, jni, cls, &names); + } + } else { + // The spec's CLASS branch: base = fields of all interfaces implemented by C, then the + // superclass chain root-first (java.lang.Object's fields first, C's own last), each class's + // fields in GetClassFields() order. + std::unordered_set seen; + jlong base = interfaceFieldCount(jvmti, jni, cls, &seen); + if (base >= 0) { + names.resize((size_t)base); + // GetSuperclass walks UP, so gather then append in reverse (root first). + jclass supers[128]; + int depth = 0; + jclass k = cls; + while (k != nullptr && + depth < (int)(sizeof(supers) / sizeof(supers[0]))) { + supers[depth++] = k; + k = jni->GetSuperclass(k); + } + ok = (k == nullptr); // deeper than 128 classes: fail rather than + // misname + for (int i = depth - 1; ok && i >= 0; i--) { + ok = appendClassFieldNames(jvmti, jni, supers[i], &names); + } + for (int i = 0; i < depth; i++) { + jni->DeleteLocalRef(supers[i]); + } + // supers[0] IS cls - the loop above already deleted it. Null it so the shared cleanup + // below does not delete the same local ref a second time (checked JNI reports an invalid + // local ref and aborts). + cls = nullptr; + } + } + } + if (cls != nullptr) { + jni->DeleteLocalRef(cls); + } + if (!ok) { + break; + } + entry.field_names = std::move(names); + entry.decode_failed = false; + } while (false); + auto inserted = _hop_label_cache.emplace(class_tag, std::move(entry)); + return &inserted.first->second; +} + +void ReferenceChainTracker::resolveHopEdgeLabel(jvmtiEnv *jvmti, JNIEnv *jni, + ChainHopEdge edge, char *out, + size_t out_cap) { + if (jvmti != nullptr && jni != nullptr && + (edge.edge_kind == JVMTI_HEAP_REFERENCE_FIELD || + edge.edge_kind == JVMTI_HEAP_REFERENCE_STATIC_FIELD) && + edge.field_index >= 0 && edge.referrer_class_tag != 0 && out_cap > 0) { + const HopLabelClass *labels = + hopLabelClassFor(jvmti, jni, edge.referrer_class_tag); + if (labels != nullptr && !labels->decode_failed && + (size_t)edge.field_index < labels->field_names.size()) { + const std::string &name = + labels->field_names[(size_t)edge.field_index]; + if (!name.empty()) { + size_t n = name.size() < out_cap - 1 ? name.size() : out_cap - 1; + memcpy(out, name.data(), n); + out[n] = '\0'; + return; + } + // Empty positioned slot (an interface-field ordinal below the class's own base) - fall + // through to the kind label. + } + } + if (out_cap > 0) { + snprintf(out, out_cap, "%s", hopEdgeKindLabel(edge.edge_kind)); + } +} + +void ReferenceChainTracker::fillHopEdgeLabels( + jvmtiEnv *jvmti, JNIEnv *jni, const std::vector &edges, + std::vector *out) { + char label[MAX_HOP_EDGE_LABEL + 1]; + for (size_t i = 0; i < edges.size() && i < out->size(); i++) { + resolveHopEdgeLabel(jvmti, jni, edges[i], label, sizeof(label)); + (*out)[i].edge_label = label; + } +} + diff --git a/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp b/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp new file mode 100644 index 000000000..6c142029a --- /dev/null +++ b/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp @@ -0,0 +1,1301 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "referenceChains.h" +#include "referenceChainInternal.h" +#include "common.h" +#include "counters.h" +#include "jniHelper.h" +#include "jvmThread.h" +#include "livenessTracker.h" +#include "log.h" +#include "objectSampler.h" +#include "os.h" +#include "profiler.h" +#include "rcDebugLevel.h" +#include "tsc.h" +#include "vmEntry.h" +#include +#include +#include +#include +#include +#include +#include +#include + +// Manual walk driver - IterateOverReachableObjects root/stack-ref enumeration + +namespace { +// jvmtiHeapRootKind (IterateOverReachableObjects's root/stack-ref callbacks, ordinals 1-7) and +// jvmtiHeapReferenceKind (FrontierEntry::root_kind's own type, FollowReferences' callback, ordinals +// 8/21-27) are different, disjoint enums per the real jvmti.h - storing a raw jvmtiHeapRootKind +// value into root_kind unmodified would make flightRecorder.cpp's rootKindName() report "unknown" +// for every root-callback-attributed chain. +u8 translateHeapRootKind(jvmtiHeapRootKind root_kind) { + switch (root_kind) { + case JVMTI_HEAP_ROOT_JNI_GLOBAL: + return (u8)JVMTI_HEAP_REFERENCE_JNI_GLOBAL; + case JVMTI_HEAP_ROOT_SYSTEM_CLASS: + return (u8)JVMTI_HEAP_REFERENCE_SYSTEM_CLASS; + case JVMTI_HEAP_ROOT_MONITOR: + return (u8)JVMTI_HEAP_REFERENCE_MONITOR; + case JVMTI_HEAP_ROOT_STACK_LOCAL: + return (u8)JVMTI_HEAP_REFERENCE_STACK_LOCAL; + case JVMTI_HEAP_ROOT_JNI_LOCAL: + return (u8)JVMTI_HEAP_REFERENCE_JNI_LOCAL; + case JVMTI_HEAP_ROOT_THREAD: + return (u8)JVMTI_HEAP_REFERENCE_THREAD; + case JVMTI_HEAP_ROOT_OTHER: + default: + return (u8)JVMTI_HEAP_REFERENCE_OTHER; + } +} + +} // namespace + +jvmtiIterationControl JNICALL ReferenceChainTracker::heapRootCallback( + jvmtiHeapRootKind root_kind, jlong class_tag, jlong size, jlong *tag_ptr, + void *user_data) { + ReferenceChainPassContext *ctx = (ReferenceChainPassContext *)user_data; + if (ctx->tracker->_abort_pass_requested.load(std::memory_order_relaxed)) { + ctx->truncated = true; + return JVMTI_ITERATION_ABORT; + } + if (ctx->truncated) { + return JVMTI_ITERATION_ABORT; + } + + u32 referrer_klass = ctx->tracker->classTags()->resolve(class_tag); + u8 translated_root_kind = translateHeapRootKind(root_kind); + AdmitResult result = ctx->tracker->admitObject( + ctx->frontier, ctx->hop_cap, ctx->budget, &ctx->edges_admitted, tag_ptr, + /*parent_tag=*/0, referrer_klass, /*depth=*/0, translated_root_kind, + class_tag); + switch (result) { + case AdmitResult::BUDGET_EXHAUSTED: + ctx->truncated = true; + return JVMTI_ITERATION_ABORT; + case AdmitResult::FRONTIER_CAP_HIT: + ctx->truncated = true; + ctx->frontier_cap_hit = true; + return JVMTI_ITERATION_ABORT; + case AdmitResult::ALREADY_ADMITTED: + if (isLeakTag(*tag_ptr)) { + // Leak-tagged object met as a DIRECT heap root (JNI global, stack local, ...): convert the + // leak tag exactly like heapReferenceCallback()'s interception branch so a chain can be built + // when the direct root is the only retention path. + jlong leak_tag = *tag_ptr; + jlong frontier_tag = ctx->tracker->nextTag(); + if (ctx->frontier->insert(frontier_tag, /*parent_tag=*/0, referrer_klass, + /*depth=*/0, FrontierEntryState::FRONTIER, + translated_root_kind, class_tag)) { + ctx->frontier->setLeakTag(frontier_tag, leak_tag); + *tag_ptr = frontier_tag; + ctx->edges_admitted++; + ctx->tracker->trackLeakAccumulation(ctx->frontier, class_tag, 0, + frontier_tag); + // Index maintenance, mirroring the edge path's interception branch: a leak-tagged root + // attached by a durable root edge is the highest-priority anchor tier. + if (translated_root_kind == (u8)JVMTI_HEAP_REFERENCE_STATIC_FIELD || + translated_root_kind == (u8)JVMTI_HEAP_REFERENCE_JNI_GLOBAL) { + ctx->tracker->addToStaticAnchorIndex(frontier_tag, class_tag, + translated_root_kind); + } + if (ctx->tracker->_candidate_count > 0) { + u32 klass_id = ctx->tracker->classTags()->resolve(class_tag); + ctx->tracker->recordDiscoveredInstance(klass_id, frontier_tag, true); + } + // Queue for expandFrontier() - the plain backlog lane, mirroring admitObject()'s + // non-priority push tail. + ctx->tracker->_pending_expand.push_back(frontier_tag); + } else { + // Frontier cap hit - same outcome as admitObject()'s own failure. + ctx->truncated = true; + ctx->frontier_cap_hit = true; + return JVMTI_ITERATION_ABORT; + } + break; + } + // Prefer the more durable root when an object is rediscovered. + ctx->tracker->maybeUpgradeRootAttachedRootKind(ctx->frontier, *tag_ptr, + translated_root_kind); + break; + default: + break; + } + return JVMTI_ITERATION_CONTINUE; +} + +jvmtiIterationControl JNICALL ReferenceChainTracker::stackRefCallback( + jvmtiHeapRootKind root_kind, jlong class_tag, jlong size, jlong *tag_ptr, + jlong thread_tag, jint depth, jmethodID method, jint slot, + void *user_data) { + // Stack-local/JNI-local roots carry thread/frame/slot detail JVMTI reports via this callback's + // richer shape, but FrontierEntry has nowhere to record it (depth/method/slot are not part of the + // record) - admission is otherwise identical to heapRootCallback() above, so this just forwards. + return heapRootCallback(root_kind, class_tag, size, tag_ptr, user_data); +} + +void ReferenceChainTracker::runPassManualWalk(jvmtiEnv *jvmti, JNIEnv *jni, + bool run_root_enum, + int root_enum_budget, + int expand_budget, + int *edges_admitted, + bool *truncated, + bool *frontier_cap_hit, + u64 *safepoint_ticks) { + assert(!t_inGCCallback && + "IterateOverReachableObjects/FollowReferences are JVMTI " + "Heap-category calls and must not be made from " + "GarbageCollectionStart/Finish"); + + // Safe point to delete the global refs of threads that ended since the last drain: this runs on + // the BFS thread before any walk phase, and refs erased from _thread_objects + // (unregisterThreadObject()) can no longer be copied out by walkCandidateThreadLocals(), so no + // walk holds them. + releaseEndedThreadRefs(jni); + + *safepoint_ticks = 0; + + // Shared wall-clock ceiling for this whole call's static-field sweep, expandFrontier(), and + // rotation sub-calls below (see _pass_deadline_ns's own comment) - deliberately NOT applied to + // root/stack-ref enumeration itself, which is instead cadence-gated by run_root_enum/ + // ROOT_ENUM_MIN_INTERVAL_NS. + _pass_deadline_ns = _effective_pause_target_ms > 0 + ? OS::nanotime() + (u64)_effective_pause_target_ms * 1000000ULL + : 0; + + *edges_admitted = 0; + *truncated = false; + *frontier_cap_hit = false; + + // Reserve a slice for rotation up front, across all three tiers (see + // ROOT_KIND_ROTATION_BUDGET/LEAK_ACCUMULATION_ROTATION_BUDGET/ STALE_EXPANDED_ROTATION_BUDGET's + // own comments) so rotation still gets to run this pass even when ordinary work below spends + // everything else and truncates. + int rotation_reserved_budget = std::min( + expand_budget / 2, ROOT_KIND_ROTATION_BUDGET + + LEAK_ACCUMULATION_ROTATION_BUDGET + + STALE_EXPANDED_ROTATION_BUDGET); + int budget = expand_budget - rotation_reserved_budget; + + // Root/stack-ref enumeration alone (unlike a root-seeded FollowReferences call on the fallback + // path) never discovers a root's own transitive children - IterateOverReachableObjects's + // root/stack-ref callbacks are given no oop, only a tag_ptr (see heapRootCallback()'s own + // comment) - so even when it runs this pass, the expandFrontier() call below is still needed to + // make any further progress. + if (run_root_enum) { + ReferenceChainPassContext ctx; + ctx.tracker = this; + ctx.frontier = _frontier; + ctx.hop_cap = _hop_cap; + ctx.budget = root_enum_budget; + ctx.edges_admitted = 0; + ctx.truncated = false; + ctx.frontier_cap_hit = false; + + u64 root_enum_start_ticks = TSC::ticks(); + jvmtiError root_err = jvmti->IterateOverReachableObjects( + heapRootCallback, stackRefCallback, /*object_ref_callback=*/nullptr, + &ctx); + *safepoint_ticks += TSC::ticks() - root_enum_start_ticks; + + // expand_budget is spent independently of root_enum_budget below (see + // ROOT_ENUM_MIN_INTERVAL_NS's own comment) - ctx.edges_admitted is written straight into + // *edges_admitted so the static-field/expand/ rotation budget math below is never shrunk by + // whatever root enumeration admitted. + *edges_admitted = ctx.edges_admitted; + _last_root_enum_ns = OS::nanotime(); + + if (root_err != JVMTI_ERROR_NONE) { + *truncated = true; + *frontier_cap_hit = false; + _root_enum_truncated_last_time = false; + return; + } + if (ctx.truncated) { + *truncated = true; + *frontier_cap_hit = ctx.frontier_cap_hit; + // Only a budget-exhausted truncation (not a frontier-cap-hit, which abandons the search + // outright) is grounds to retry root enumeration on the very next pass - see + // _root_enum_truncated_last_time's own comment. + _root_enum_truncated_last_time = !ctx.frontier_cap_hit; + return; + } + _root_enum_truncated_last_time = false; + } + + int expand_phase_edges_admitted = 0; + + // Candidate-scoped reach, prong 1: descend-walk the current candidates' qualifying threads' + // ThreadLocalMap subgraphs BEFORE any breadth-first work this pass - reaching the tagged + // instances under a thread-retained holder must not queue behind the ordinary backlog (see + // walkCandidateThreadLocals()'s own comment). + if (_candidate_count > 0) { + int thread_walk_edges_admitted = 0; + bool thread_walk_truncated = false; + bool thread_walk_frontier_cap_hit = false; + // Give the thread walk its own fresh deadline so the root-enum walk above never eats its slice + // (per-sub-op reset rationale, see expand below). + _pass_deadline_ns = _effective_pause_target_ms > 0 + ? OS::nanotime() + + (u64)_effective_pause_target_ms * 1000000ULL + : 0; + walkCandidateThreadLocals(jvmti, jni, budget, &thread_walk_edges_admitted, + &thread_walk_truncated, + &thread_walk_frontier_cap_hit, safepoint_ticks); + expand_phase_edges_admitted += thread_walk_edges_admitted; + *edges_admitted += thread_walk_edges_admitted; + if (thread_walk_frontier_cap_hit) { + // Frontier-cap mid-thread-walk is the same search-abandonment grounds as anywhere else - do + // not spend more of this pass's budget. + *truncated = true; + *frontier_cap_hit = true; + return; + } + if (thread_walk_truncated) { + *truncated = true; + } + } + + // Static-field roots (SomeClass.staticField -> obj) are not reachable via + // IterateOverReachableObjects' root/stack-ref callbacks above - see admitStaticFieldRoots()'s own + // comment - so this pass would otherwise never discover an object retained only that way. + TEST_LOG_SUMMARY("ReferenceChainTracker::runPassManualWalk static_sweep_gate " + "resolved=%d swept=%d cursor=%d", + _last_resolved_class_count, _last_static_field_class_count, + _static_field_sweep_cursor); + if (_last_resolved_class_count != _last_static_field_class_count) { + int static_field_edges_admitted = 0; + bool static_field_truncated = false; + bool static_field_frontier_cap_hit = false; + bool static_field_cycle_complete = false; + int static_field_budget = std::max(budget - expand_phase_edges_admitted, 0); + admitStaticFieldRoots(jvmti, jni, _hop_cap, static_field_budget, + &static_field_edges_admitted, &static_field_truncated, + &static_field_frontier_cap_hit, + &static_field_cycle_complete, safepoint_ticks); + expand_phase_edges_admitted += static_field_edges_admitted; + *edges_admitted += static_field_edges_admitted; + if (static_field_truncated) { + *truncated = true; + *frontier_cap_hit = static_field_frontier_cap_hit; + if (static_field_frontier_cap_hit) { + // Frontier-size cap hit while admitting static-field roots is the same "grounds to ABANDON + // the whole search" outcome BUDGET_EXHAUSTED/FRONTIER_CAP_HIT handling above gives root + // enumeration - do not spend any more of this pass's budget on the ordinary expansion + // below. + return; + } + } + if (static_field_cycle_complete) { + // The chunk cursor completed a full lap over the loaded-class list with no chunk truncating + // along the way (possibly discovering nothing, if every static field seen was already + // ALREADY_ADMITTED) - remember the class count it covered so a later pass with no new classes + // can skip re-running the sweep entirely. + _last_static_field_class_count = _last_resolved_class_count; + } + } + + int expand_edges_admitted = 0; + bool expand_truncated = false; + bool expand_frontier_cap_hit = false; + int remaining_budget = std::max(budget - expand_phase_edges_admitted, 0); + // Give expand its own fresh deadline so the static-field sweep's FollowReferences calls don't eat + // expand's time. + _pass_deadline_ns = _effective_pause_target_ms > 0 + ? OS::nanotime() + (u64)_effective_pause_target_ms * 1000000ULL + : 0; + expandFrontier(jvmti, jni, _hop_cap, remaining_budget, + &expand_edges_admitted, &expand_truncated, + &expand_frontier_cap_hit, safepoint_ticks); + expand_phase_edges_admitted += expand_edges_admitted; + *edges_admitted += expand_edges_admitted; + *truncated = *truncated || expand_truncated; + *frontier_cap_hit = expand_frontier_cap_hit; + TEST_LOG_SUMMARY("ReferenceChainTracker::runPassManualWalk expand_phase " + "edges_admitted=%d truncated=%d frontier_cap_hit=%d " + "remaining_budget=%d", + expand_edges_admitted, (int)expand_truncated, + (int)expand_frontier_cap_hit, remaining_budget); + + // Note: unlike a hard truncation during root/stack-ref enumeration or the static-field sweep + // above (which return early - the pass never even reached ordinary expansion), a truncated + // ordinary expansion does NOT skip rotation below: rotation runs on its own reserved slice of + // budget (see rotation_reserved_budget's own comment above) precisely because ordinary expansion + // truncates on nearly every pass under a sustained fast-growing backlog, and that is exactly the + // situation - a mutable field reassigned out from under an already-EXPANDED entry - rotation + // exists to correct. + + // Revisit a bounded subset of expanded entries to observe changed references. + std::vector rotation_tags = + collectStaleRootKindEntriesForRotation(ROOT_KIND_ROTATION_BUDGET); + std::vector leak_accumulation_tags = + collectLeakAccumulationCandidatesForRotation( + LEAK_ACCUMULATION_ROTATION_BUDGET); + // Also re-walk a bounded, rotating subset of EXPANDED entries regardless of root attribution: a + // mutable field reassigned since an object's one-time expansion - e.g. HashMap.table on resize - + // is otherwise never observed again, silently orphaning everything only reachable through the + // field's current value. + std::vector stale_expanded_tags = + collectStaleExpandedEntriesForRotation(STALE_EXPANDED_ROTATION_BUDGET); + // Candidate-scoped reach, prong 2: root-attached static holders are descend-walked directly (see + // collectStaticFieldAnchorsForRotation()/ walkStaticFieldAnchors()'s own comments) - not pushed + // onto the priority lane, so they are independent of the queue tiers above. + reconcileAnchorClassShapes(jvmti, jni); + std::vector static_anchor_tags = + collectStaticFieldAnchorsForRotation(STATIC_ANCHOR_ROTATION_BUDGET); + std::vector static_anchor_fifo_drained; + drainStaticAnchorFifo(STATIC_ANCHOR_FIFO_DRAIN, static_anchor_fifo_drained); + for (const AtRiskAnchor &at_risk : static_anchor_fifo_drained) { + static_anchor_tags.push_back(at_risk.tag); + } + if (rotation_tags.empty() && leak_accumulation_tags.empty() && + stale_expanded_tags.empty() && static_anchor_tags.empty()) { + return; + } + // rotation_reserved_budget + max(budget - expand_phase_edges_admitted, 0) is exactly + // expand_budget - expand_phase_edges_admitted: budget already IS expand_budget - + // rotation_reserved_budget (above), and expand_phase_edges_ admitted can never exceed budget (the + // static-field sweep and ordinary expandFrontier() calls above are both capped to budget-derived + // slices), so the max() is never actually needed to avoid going negative. + int rotation_budget = expand_budget - expand_phase_edges_admitted; + int rotation_edges_admitted = 0; + bool rotation_truncated = false; + // Give rotation its own fresh deadline, same as expand above. + _pass_deadline_ns = _effective_pause_target_ms > 0 + ? OS::nanotime() + (u64)_effective_pause_target_ms * 1000000ULL + : 0; + bool rotation_frontier_cap_hit = false; + // Prong 2 static-anchor descend walks run FIRST inside rotation's slice: they are the + // highest-value rotation work (bounded, targeted, and the only rotation tier that can reach a + // collection-shaped static holder's internals in one pass), and their edges draw down the same + // rotation budget the queue-tier batch below uses - a pass whose anchor walks admit the holder's + // whole internal structure needs less one-hop rotation work, not more. + if (!static_anchor_tags.empty() && rotation_budget > 0) { + int static_anchor_edges_admitted = 0; + bool static_anchor_truncated = false; + bool static_anchor_frontier_cap_hit = false; + std::vector static_anchor_unwalked; + walkStaticFieldAnchors(jvmti, jni, static_anchor_tags, rotation_budget, + &static_anchor_edges_admitted, + &static_anchor_truncated, + &static_anchor_frontier_cap_hit, safepoint_ticks, + &static_anchor_unwalked); + rotation_edges_admitted += static_anchor_edges_admitted; + rotation_budget -= static_anchor_edges_admitted; + *truncated = *truncated || static_anchor_truncated; + // B' requeue: resolved-but-unwalked anchors that came from this pass's FIFO drain go back to + // the FIFO front, order-preserving, so a pass whose budget died mid-batch walks them first next + // pass instead of waiting for the next sweep lap's re-push. + if (!static_anchor_unwalked.empty() && + !static_anchor_fifo_drained.empty()) { + std::vector static_anchor_requeue; + for (jlong tag : static_anchor_unwalked) { + FrontierEntry entry{}; + if (!_frontier->lookup(tag, &entry)) { + continue; + } + for (const AtRiskAnchor &at_risk : static_anchor_fifo_drained) { + if (tag == at_risk.tag) { + static_anchor_requeue.push_back(at_risk); + break; + } + } + } + if (!static_anchor_requeue.empty()) { + requeueStaticAnchorFifoFront(static_anchor_requeue); + } + } + if (static_anchor_frontier_cap_hit) { + *frontier_cap_hit = true; + return; + } + } + // expandFrontier() SETS (does not add into) its edges output - see its entry - so the anchor + // walks' edges are kept in a separate counter and summed here. + int queue_tier_edges_admitted = 0; + expandFrontier(jvmti, jni, _hop_cap, rotation_budget, + &queue_tier_edges_admitted, &rotation_truncated, + &rotation_frontier_cap_hit, safepoint_ticks); + rotation_edges_admitted += queue_tier_edges_admitted; + *edges_admitted += rotation_edges_admitted; + // OR, not overwrite: the ordinary expand phase above may have already set these to true (real + // truncation/cap-hit left in _pending_expand), and a rotation batch that happens to finish + // cleanly must not erase that - has_pending_frontier (runPass()) and the FRONTIER_CAP abandon + // check both read these as "did any of this pass's sub-phases truncate/cap-hit", not just the + // last one that ran. + *truncated = *truncated || rotation_truncated; + *frontier_cap_hit = *frontier_cap_hit || rotation_frontier_cap_hit; +} + +// Incremental resumption across passes. + +void ReferenceChainTracker::markAllFrontierExpanded() { + while (!_priority_expand.empty()) { + _frontier->markExpanded(_priority_expand.front()); + _priority_expand.pop_front(); + } + _priority_expand_set.clear(); + while (!_pending_expand.empty()) { + _frontier->markExpanded(_pending_expand.front()); + _pending_expand.pop_front(); + } +} + +void ReferenceChainTracker::expandFrontier(jvmtiEnv *jvmti, JNIEnv *jni, + int hop_cap, int budget, + int *edges_admitted, + bool *truncated, + bool *frontier_cap_hit, + u64 *safepoint_ticks) { + assert(!t_inGCCallback && + "GetObjectsWithTags/FollowReferences are JVMTI Heap-category calls " + "and must not be made from GarbageCollectionStart/Finish"); + + ReferenceChainPassContext ctx; + ctx.tracker = this; + ctx.frontier = _frontier; + ctx.hop_cap = hop_cap; + ctx.budget = budget; + ctx.edges_admitted = 0; + ctx.truncated = false; + ctx.frontier_cap_hit = false; + + // ARRAY-HOLDER BATCHING: expand a whole batch of boundary objects with ONE + // FollowReferences(initial_object=holder_array) call per BFS level, instead of one + // FollowReferences PER frontier entry. + std::unordered_set batch_tags; + ctx.batch_tags = &batch_tags; + // Completed-batch-entry tracking for the order-independent truncated-batch resume (see + // ReferenceChainPassContext::_completed_batch_tags) - reset per batch along with the rolling + // cursor. + std::unordered_set completed_batch_tags; + ctx._completed_batch_tags = &completed_batch_tags; + + jvmtiHeapCallbacks callbacks; + memset(&callbacks, 0, sizeof(callbacks)); + callbacks.heap_reference_callback = heapReferenceCallback; + + // java/lang/Object element type for the transient frontier-holder array. + if (jni != nullptr && _cached_object_class == nullptr) { + jclass local = jni->FindClass("java/lang/Object"); + if (!jniExceptionCheck(jni) && local != nullptr) { + _cached_object_class = (jclass)jni->NewGlobalRef(local); + } + if (local != nullptr) { + jni->DeleteLocalRef(local); + } + } + jclass object_class = _cached_object_class; + + bool progress = true; + // FAIR-SHARE DRAIN: alternate batches between _priority_expand and _pending_expand whenever both + // are non-empty (priority still takes the first batch of each call). + while (!ctx.truncated && progress && object_class != nullptr) { + // Wall-clock deadline check per iteration: GetObjectsWithTags runs OUTSIDE any FollowReferences + // callback, so heapReferenceCallback()'s amortized deadline check never sees its cost. + if (_pass_deadline_ns != 0 && OS::nanotime() >= _pass_deadline_ns) { + ctx.truncated = true; + break; + } + progress = false; + + // Alternate lanes (see FAIR-SHARE DRAIN above); priority still goes first so a + // rotation-selected parent's re-discovery keeps its head-of-queue property, but no lane can + // monopolize the drain. + bool from_priority; + if (_priority_expand.empty()) { + from_priority = false; + } else if (_pending_expand.empty()) { + from_priority = true; + } else { + from_priority = _expand_lane_prefer_priority; + _expand_lane_prefer_priority = !_expand_lane_prefer_priority; + } + std::deque &source = + from_priority ? _priority_expand : _pending_expand; + ctx.admit_priority = from_priority; + if (source.empty()) { + break; // nothing pending in either lane + } + + // SELF-CALIBRATING ADAPTIVE BATCH SIZE for GetObjectsWithTags. + size_t gotw_batch_size = + _gotw_batch_size != 0 ? _gotw_batch_size : GOTW_INITIAL_BATCH_SIZE; + size_t batch_size = std::min( + source.size(), + std::min((size_t)std::max(std::min(budget, _budget), 1), + gotw_batch_size)); + std::vector candidate_tags(source.begin(), + source.begin() + batch_size); + + // Resolve this batch's live boundary objects. GetObjectsWithTags iterates the whole tag map, + // but does so under a no-safepoint mutex on this (Java) thread - it is NOT a stop-the-world VM + // operation, unlike the FollowReferences below (jvmtiTagMap.cpp: get_objects_with_tags takes + // Mutex::_no_safepoint_check_flag and calls entry_iterate directly, whereas follow_references + // does VMThread::execute()). + jint resolved_count = 0; + jobject *resolved_objects = nullptr; + jlong *resolved_tags = nullptr; + u64 gotw_start_ns = OS::nanotime(); + jvmtiError resolve_err = jvmti->GetObjectsWithTags( + (jint)candidate_tags.size(), candidate_tags.data(), &resolved_count, + &resolved_objects, &resolved_tags); + u64 gotw_elapsed_ns = OS::nanotime() - gotw_start_ns; + // Self-calibrate (PROPORTIONAL batch control): update the EMA of PER-CALL elapsed time, then + // scale the batch so ONE call fills the remaining wall-clock window. + if (batch_size > 0 && gotw_elapsed_ns > 0) { + if (_gotw_ema_call_ns == 0) { + _gotw_ema_call_ns = gotw_elapsed_ns; + } else { + _gotw_ema_call_ns = _gotw_ema_call_ns * 4 / 5 + gotw_elapsed_ns / 5; + } + u64 now_ns = OS::nanotime(); + u64 window_ns = + gotwWindowNs( + _pass_deadline_ns != 0 && _pass_deadline_ns > now_ns + ? _pass_deadline_ns - now_ns + : 0, + source.size()); + // window_ns / ema_call_ns == how many such calls fit the window; scaling the CURRENT + // calibration batch by that ratio sizes the next call to consume the whole window in one go. + size_t calib_batch = + _gotw_batch_size != 0 ? _gotw_batch_size : GOTW_INITIAL_BATCH_SIZE; + size_t next_batch = (size_t)((u64)calib_batch * window_ns / + std::max(_gotw_ema_call_ns, 1ULL)); + _gotw_batch_size = std::min(std::max(next_batch, GOTW_MIN_BATCH), + GOTW_MAX_BATCH); + } + if (resolve_err != JVMTI_ERROR_NONE) { + ctx.truncated = true; + break; + } + + std::unordered_map live; + for (jint i = 0; i < resolved_count; i++) { + live[resolved_tags[i]] = resolved_objects[i]; + } + + // Build the frontier-holder array from the live boundary objects and record their tags so + // heapReferenceCallback() descends into exactly these (one hop). + batch_tags.clear(); + jobjectArray holder = nullptr; + if (resolved_count > 0) { + jint capacity_err = jni->EnsureLocalCapacity(resolved_count + 16); + if (capacity_err < 0 || jniExceptionCheck(jni)) { + // Could not guarantee local-ref headroom for this batch - treat like any other batch-level + // failure below (JVMTI error / OOM building the holder array): retry this batch on a later + // pass rather than proceeding into NewObjectArray with no capacity guarantee. + ctx.truncated = true; + } else { + holder = jni->NewObjectArray(resolved_count, object_class, nullptr); + if (jniExceptionCheck(jni)) { + // OutOfMemoryError building the holder array (or any other exception NewObjectArray + // raised) left `holder` null; make sure the pending exception does not survive into the + // next JNI call below or the next expandFrontier() invocation on this same long-lived + // BFS-thread JNIEnv (JNI spec: undefined behavior with a pending exception across + // ordinary JNI calls). + holder = nullptr; + } + if (holder != nullptr) { + for (jint i = 0; i < resolved_count; i++) { + jni->SetObjectArrayElement(holder, i, resolved_objects[i]); + if (jniExceptionCheck(jni)) { + // e.g. an array-store-class failure. Abort building this batch's holder rather than + // handing a partially-populated array (with a just-cleared pending exception) to + // FollowReferences. + ctx.truncated = true; + break; + } + batch_tags.insert(resolved_tags[i]); + } + } + if (holder == nullptr) { + // NewObjectArray failed (OOM/local-ref exhaustion) - the FollowReferences call below + // (which would have discovered this batch's children) never runs. + ctx.truncated = true; + } else if (!ctx.truncated) { + // A single FollowReferences over the holder array expands this whole BFS level in one + // stop-the-world HeapWalkOperation (instead of one per frontier entry). + ctx._last_visited_batch_tag = 0; // reset rolling cursor + completed_batch_tags.clear(); + u64 follow_start_ticks = TSC::ticks(); + jvmtiError follow_err = + jvmti->FollowReferences(0, nullptr, holder, &callbacks, &ctx); + *safepoint_ticks += TSC::ticks() - follow_start_ticks; + if (follow_err != JVMTI_ERROR_NONE) { + ctx.truncated = true; + } + } + } + } + + if (!ctx.truncated) { + // The whole batch had all its direct children admitted this level: dead entries are pruned, + // live ones are marked EXPANDED, and all are popped off the front. + for (jlong tag : candidate_tags) { + if (live.find(tag) == live.end()) { + _frontier->clear(tag); + } else { + _frontier->markExpanded(tag); + } + source.pop_front(); + } + progress = true; + } else if (!completed_batch_tags.empty()) { + // ROLLING RESUME (order-independent): FollowReferences truncated mid-batch, but the callback + // recorded exactly which batch entries it finished visiting (see + // ReferenceChainPassContext::_completed_batch_tags). + std::vector keep; + keep.reserve(candidate_tags.size()); + for (jlong tag : candidate_tags) { + if (completed_batch_tags.count(tag) != 0) { + if (live.find(tag) == live.end()) { + _frontier->clear(tag); + } else { + _frontier->markExpanded(tag); + } + } else { + keep.push_back(tag); + } + source.pop_front(); + } + // Re-queue the unvisited remainder at the front, preserving input order (push_front in + // reverse). + for (size_t i = keep.size(); i-- > 0;) { + source.push_front(keep[i]); + } + } + // else truncated with no batch entry visited (e.g. GetObjectsWithTags error, holder allocation + // failure, or truncation before the first batch entry was reached): leave the entire batch at + // the front of the source queue for a later pass to retry, same as before. + + if (from_priority) { + // This batch popped entries off _priority_expand's front (or, on truncation, was left + // untouched) - re-derive the membership index from the deque's current contents either way so + // isQueuedForRotation() stays exact for the rotation collectors that run later in this same + // pass. + _priority_expand_set.rebuildFrom(_priority_expand); + } + + if (holder != nullptr) { + jni->DeleteLocalRef(holder); + } + if (jni != nullptr) { + for (jint i = 0; i < resolved_count; i++) { + jni->DeleteLocalRef(resolved_objects[i]); + } + } + if (resolved_objects != nullptr) { + jvmti->Deallocate((unsigned char *)resolved_objects); + } + if (resolved_tags != nullptr) { + jvmti->Deallocate((unsigned char *)resolved_tags); + } + } + + // object_class is NOT deleted here - it is now cached in _cached_object_class and reused across + // calls on this same JNIEnv (see above), not a per-call local ref. + + if (!ctx.truncated && jni != nullptr && object_class == nullptr && + (!_pending_expand.empty() || !_priority_expand.empty())) { + // FindClass("java/lang/Object") failed for this (attached) JNIEnv, so the batching loop above + // never ran even though pending frontier work remains. + ctx.truncated = true; + } + + *edges_admitted = ctx.edges_admitted; + *truncated = ctx.truncated; + *frontier_cap_hit = ctx.frontier_cap_hit; +} + +void ReferenceChainTracker::admitStaticFieldRoots(jvmtiEnv *jvmti, JNIEnv *jni, + int hop_cap, int budget, + int *edges_admitted, + bool *truncated, + bool *frontier_cap_hit, + bool *cycle_complete, + u64 *safepoint_ticks) { + assert(!t_inGCCallback && + "GetLoadedClasses/FollowReferences are JVMTI Heap-category calls " + "and must not be made from GarbageCollectionStart/Finish"); + *edges_admitted = 0; + *truncated = false; + *frontier_cap_hit = false; + *cycle_complete = false; + + if (jni == nullptr) { + // No JNIEnv to build the holder array on (some test seams) - see expandFrontier()'s own + // identical guard. + return; + } + + jint class_count = 0; + jclass *classes = nullptr; + jvmtiError classes_err = jvmti->GetLoadedClasses(&class_count, &classes); + if (classes_err != JVMTI_ERROR_NONE) { + return; + } + if (class_count <= 0) { + if (classes != nullptr) { + jvmti->Deallocate((unsigned char *)classes); + } + return; + } + + // GetLoadedClasses() gives no ordering guarantee across separate calls, so the cursor below is + // only meaningful as an index into THIS call's array - reprioritize it every call rather than + // trying to cache an ordering. + jint app_boundary = 0; + for (jint i = 0; i < class_count; i++) { + jobject loader = nullptr; + jvmtiError loader_err = jvmti->GetClassLoader(classes[i], &loader); + bool is_app_class = (loader_err == JVMTI_ERROR_NONE) && (loader != nullptr); + if (loader != nullptr) { + jni->DeleteLocalRef(loader); + } + if (is_app_class) { + if (i != app_boundary) { + std::swap(classes[i], classes[app_boundary]); + } + app_boundary++; + } + } + + // Stable chunk order: GetLoadedClasses() returns an arbitrary order per call, so an index cursor + // over raw call order can MISS classes entirely within a lap (each chunk would cover a different + // random subset). + { + std::vector tags((size_t)class_count, 0); + for (jint i = 0; i < class_count; i++) { + jvmti->GetTag(classes[i], &tags[i]); + } + std::vector order((size_t)class_count); + for (jint i = 0; i < class_count; i++) { + order[i] = i; + } + std::sort(order.begin(), order.begin() + app_boundary, + [&tags](jint a, jint b) { return tags[a] < tags[b]; }); + std::sort(order.begin() + app_boundary, order.end(), + [&tags](jint a, jint b) { return tags[a] < tags[b]; }); + std::vector sorted((size_t)class_count); + for (jint i = 0; i < class_count; i++) { + sorted[i] = classes[order[i]]; + } + memcpy(classes, sorted.data(), (size_t)class_count * sizeof(jclass)); + } + + if (_static_field_sweep_cursor >= class_count) { + // Loaded-class count shrank since the last chunk (classes unloaded) - restart the lap rather + // than reading out of range. + _static_field_sweep_cursor = 0; + _static_field_sweep_cycle_truncated = false; + } + jint chunk_start = _static_field_sweep_cursor; + jint chunk_end = + std::min(chunk_start + STATIC_FIELD_SWEEP_CHUNK_CLASSES, class_count); + jint chunk_count = chunk_end - chunk_start; + + // Same java/lang/Object element-type cache expandFrontier() uses for its own frontier-holder + // array - shared across both call sites on this same attached JNIEnv rather than a second + // FindClass() per pass. + if (_cached_object_class == nullptr) { + jclass local = jni->FindClass("java/lang/Object"); + if (!jniExceptionCheck(jni) && local != nullptr) { + _cached_object_class = (jclass)jni->NewGlobalRef(local); + } + if (local != nullptr) { + jni->DeleteLocalRef(local); + } + } + jclass object_class = _cached_object_class; + + if (object_class == nullptr || + jni->EnsureLocalCapacity(class_count + 16) < 0 || + jniExceptionCheck(jni)) { + for (jint i = 0; i < class_count; i++) { + jni->DeleteLocalRef(classes[i]); + } + jvmti->Deallocate((unsigned char *)classes); + return; + } + + jobjectArray holder = jni->NewObjectArray(chunk_count, object_class, nullptr); + if (jniExceptionCheck(jni)) { + // OutOfMemoryError (or any other exception) building the holder - clear it rather than let it + // survive into the DeleteLocalRef() calls below (JNI spec: undefined behavior with a pending + // exception across ordinary JNI calls), same as expandFrontier()'s identical case. + holder = nullptr; + } + if (holder != nullptr) { + // Fill in REVERSE chunk order: holder[0] = classes[chunk_end-1], ..., holder[chunk_count-1] = + // classes[chunk_start]. + for (jint i = 0; i < chunk_count; i++) { + jni->SetObjectArrayElement(holder, i, classes[chunk_end - 1 - i]); + if (jniExceptionCheck(jni)) { + holder = nullptr; + break; + } + } + } + + // GetLoadedClasses() returned a local ref for every class regardless of chunk selection - free + // all of them here, not just the chunk. + for (jint i = 0; i < class_count; i++) { + jni->DeleteLocalRef(classes[i]); + } + jvmti->Deallocate((unsigned char *)classes); + + if (holder == nullptr) { + // OOM/local-ref exhaustion/array-store failure - skip this pass's sweep rather than treating it + // like the manual walk's own truncation (see this method's own header comment). + return; + } + + ReferenceChainPassContext ctx; + ctx.tracker = this; + ctx.frontier = _frontier; + ctx.hop_cap = hop_cap; + ctx.budget = budget; + ctx.edges_admitted = 0; + ctx.truncated = false; + ctx.frontier_cap_hit = false; + // Empty (not null) batch_tags forces heapReferenceCallback() to stop at exactly one hop past each + // class - see this method's own header comment for why a deeper descent here would reintroduce + // the whole-graph FollowReferences cost the array-holder batching design otherwise avoids. + std::unordered_set empty_batch_tags; + ctx.batch_tags = &empty_batch_tags; + // Lets heapReferenceCallback() walk past the holder->class seed edge (see + // ReferenceChainPassContext::static_field_seed's own comment) so this sweep actually reaches each + // class's static fields instead of stopping at the negative-tagged class object itself. + ctx.static_field_seed = true; + // Per-class non-STATIC_FIELD admission cap (see ReferenceChainPassContext::_class_other_cap's own + // comment). + ctx._class_other_cap = STATIC_FIELD_SWEEP_NON_STATIC_CAP_PER_CLASS; + + jvmtiHeapCallbacks callbacks; + memset(&callbacks, 0, sizeof(callbacks)); + callbacks.heap_reference_callback = heapReferenceCallback; + u64 follow_start_ticks = TSC::ticks(); + jvmtiError follow_err = + jvmti->FollowReferences(0, nullptr, holder, &callbacks, &ctx); + *safepoint_ticks += TSC::ticks() - follow_start_ticks; + jni->DeleteLocalRef(holder); + if (follow_err != JVMTI_ERROR_NONE) { + return; + } + + *edges_admitted = ctx.edges_admitted; + *truncated = ctx.truncated; + *frontier_cap_hit = ctx.frontier_cap_hit; + + if (ctx.truncated) { + _static_field_sweep_cycle_truncated = true; + // Resumable cursor: instead of skipping to chunk_end (losing every class after the interruption + // point for the rest of this lap), redo the chunk on the next pass. + _static_field_sweep_cursor = chunk_start; + } else { + // Full advance: every class in the chunk was processed. + _static_field_sweep_cursor = chunk_end; + } + if (_static_field_sweep_cursor >= class_count) { + *cycle_complete = !_static_field_sweep_cycle_truncated; + _static_field_sweep_cursor = 0; + _static_field_sweep_cycle_truncated = false; + } +} + +bool ReferenceChainTracker::releaseSearchTags(jvmtiEnv *jvmti, JNIEnv *jni) { + assert(!t_inGCCallback && + "GetObjectsWithTags is a JVMTI Heap-category call and must not be " + "made from GarbageCollectionStart/Finish"); + if (jvmti == nullptr || _frontier == nullptr) { + return true; // nothing to release + } + + jlong scan_limit = _frontier->size(); + std::vector live_tags; + for (jlong tag = 1; tag <= scan_limit; tag++) { + FrontierEntry entry{}; + if (_frontier->lookup(tag, &entry) && + entry.state != FrontierEntryState::ABANDONED) { + live_tags.push_back(tag); + } + } + if (live_tags.empty()) { + return true; + } + + jint resolved_count = 0; + jobject *resolved_objects = nullptr; + jlong *resolved_tags = nullptr; + if (jvmti->GetObjectsWithTags((jint)live_tags.size(), live_tags.data(), + &resolved_count, &resolved_objects, + &resolved_tags) != JVMTI_ERROR_NONE) { + // GetObjectsWithTags() itself failed (e.g. JVMTI_ERROR_OUT_OF_MEMORY): we do NOT know which, if + // any, of live_tags are still live objects, so do not mark any of them ABANDONED here - doing + // so while their JVMTI tag might still be set would let a restarted search's nextTag() sequence + // eventually reissue the same numeric tag to a brand-new object, corrupting FrontierTable's + // tag-uniqueness invariant (see this method's own header comment). + Counters::increment(REFERENCE_CHAIN_TAG_RELEASE_FAILED); + Log::warn("ReferenceChains: GetObjectsWithTags failed while releasing " + "%zu search tag(s); will retry before allowing a search " + "restart", + live_tags.size()); + return false; + } + + for (jint i = 0; i < resolved_count; i++) { + // clearTag() rather than a raw SetTag() call - reuses the same helper (and its GC-callback + // self-consistency assert) tagObject/ getTag already go through. + clearTag(jvmti, resolved_objects[i]); + if (jni != nullptr) { + jni->DeleteLocalRef(resolved_objects[i]); + } + } + if (resolved_objects != nullptr) { + jvmti->Deallocate((unsigned char *)resolved_objects); + } + if (resolved_tags != nullptr) { + jvmti->Deallocate((unsigned char *)resolved_tags); + } + // Tags that failed to resolve above are already dead (JVMTI forgot them with their object) - + // nothing to release, just mark the record ABANDONED below like every other entry this search + // owned. + for (jlong tag : live_tags) { + _frontier->clear(tag); + } + return true; +} + +bool ReferenceChainTracker::runPass(jvmtiEnv *jvmti, JNIEnv *jni, + bool *out_truncated) { + if (!_enabled || jvmti == nullptr || _frontier == nullptr) { + TEST_LOG_SUMMARY("ReferenceChainTracker::runPass early-exit: enabled=%d jvmti=%p frontier=%p", + _enabled, (void *)jvmti, (void *)_frontier); + return false; + } + + if (_search_state != SearchState::RUNNING) { + // The search already reached a terminal outcome - nothing left for another pass to do until + // shouldRunPass() decides to restartSearch() (this class's header comment), which flips + // _search_started back to false before this method is called again. + if (!_tags_released) { + _tags_released = releaseSearchTags(jvmti, jni); + } + TEST_LOG_SUMMARY("ReferenceChainTracker::runPass no-op: searchState=%d already terminal " + "tagsReleased=%d", + (int)_search_state, _tags_released); + if (out_truncated != nullptr) { + *out_truncated = false; + } + return true; + } + + resolveLoadedClasses(jvmti, jni); + + TEST_LOG_SUMMARY("ReferenceChainTracker::runPass starting JVMTI walk: " + "search_started=%d frontierSize=%zu", + _search_started, _frontier != nullptr ? _frontier->size() : (size_t)0); + + int edges_admitted = 0; + bool truncated = false; + bool frontier_cap_hit = false; + jvmtiError err; + // Whole-call wall-clock duration of runPassManualWalk() below - includes root/stack-ref + // enumeration dispatch, frontier-table bookkeeping, and rotation-candidate collection, in + // addition to the actual in-safepoint JVMTI calls. + u64 pass_wall_ticks = 0; + // Genuine in-safepoint cost of this pass, accumulated by runPassManualWalk() across every + // IterateOverReachableObjects/ FollowReferences call it makes (root enum, static-field sweep, + // ordinary expansion, rotation re-expansion) - explicitly excluding GetObjectsWithTags (not a + // safepoint call) and every bookkeeping line in between. + u64 safepoint_ticks = 0; + + // Every pass is driven by the manual walk (runPassManualWalk() - IterateOverReachableObjects for + // roots, then a batched array-holder FollowReferences per BFS level in expandFrontier()), on + // every collector. + bool manual_first_pass = !_search_started; + if (manual_first_pass) { + _search_started = true; + store(_search_start_ns, OS::nanotime()); + } + + // Root/stack-ref enumeration alone never discovers a root's transitive children + // (runPassManualWalk()'s own comment) - there is no "first pass walks the whole graph inline" + // shortcut here, so every pass (first or resumed) takes the same expand-frontier shape. + u64 now_ns = OS::nanotime(); + bool run_root_enum = manual_first_pass || _root_enum_truncated_last_time || + (now_ns - _last_root_enum_ns >= ROOT_ENUM_MIN_INTERVAL_NS); + + int frontier_size_before_pass = _frontier != nullptr ? _frontier->size() : 0; + + u64 call_start_ticks = TSC::ticks(); + runPassManualWalk(jvmti, jni, run_root_enum, _first_pass_budget, + _effective_budget, &edges_admitted, &truncated, + &frontier_cap_hit, &safepoint_ticks); + pass_wall_ticks = TSC::ticks() - call_start_ticks; + // TSC::ticks() is monotonic but not necessarily free of measurement noise between the outer + // call_start_ticks snapshot and the several inner TSC::ticks() snapshots safepoint_ticks is built + // from - clamp rather than underflow if the accumulated safepoint portion ever reads back larger + // than the whole-call wall time it's a subset of. + u64 non_safepoint_ticks = + pass_wall_ticks > safepoint_ticks ? pass_wall_ticks - safepoint_ticks : 0; + err = JVMTI_ERROR_NONE; + + store(_passes_run, load(_passes_run) + 1); + _last_pass_gc_finish_epoch = gcFinishEpoch(); + store(_last_pass_ns, OS::nanotime()); + if (!run_root_enum) { + // A pass that ran root/stack-ref enumeration spends _first_pass_budget, not _effective_budget - + // its duration is not a signal about the per-pass cost updatePacing() is trying to regulate + // (expandFrontier()'s cheap, per-node expansion calls), so feeding it in here would throttle + // _effective_budget down for every one of those unrelated later passes based on a single, + // deliberately oversized outlier. + updatePacing(safepoint_ticks); + } else { + // Excluded from the budget/cadence controller above, but not from the borrow ceiling's + // revocation check (see maybeRevokeBorrowForRootEnumPass()'s own comment) - a root-enum pass's + // in-safepoint cost is real pause time and must still be able to revoke a borrowed-budget grant + // the pacing controller would otherwise keep believing is safe. + maybeRevokeBorrowForRootEnumPass(safepoint_ticks); + } + // accumulate this pass's own in-safepoint cost toward the running total restartSearch() will + // spend into _safepoint_pain_budget once the search reaches a terminal state - same + // TSC::ticks_to_millis() conversion updatePacing() already uses for its own pass-duration signal. + _search_pain_ms += TSC::ticks_to_millis(safepoint_ticks); + // Independent leaky bucket for the non-safepoint remainder of this pass (root/stack-ref + // enumeration dispatch, frontier-table admission, rotation-candidate collection) - see + // _cpu_pain_budget's own comment (referenceChains.h) for why this needs to be tracked separately + // from both _safepoint_pain_budget above and _pause_pid's safepoint_ticks signal. + _cpu_pain_budget.spend(TSC::ticks_to_millis(non_safepoint_ticks)); + + // Apply terminal conditions in priority order. + bool has_pending_frontier = truncated; + int frontier_size_after = _frontier->size(); + if (frontier_cap_hit) { + // Frontier table is full -- no new entries can ever be admitted, so frontier_size_after can + // never exceed frontier_size_before_pass again. + store(_abandon_reason, (u8)SearchAbandonReason::FRONTIER_CAP); + storeRelease(_search_state, (u8)SearchState::ABANDONED); + enqueuePendingAbandonedEvent(); + TEST_LOG_SUMMARY("ReferenceChainTracker::runPass frontier cap hit -- " + "abandoning search (size=%d)", + frontier_size_after); + } else if (!has_pending_frontier && _watched_leak_klass_count == 0) { + storeRelease(_search_state, (u8)SearchState::COMPLETED); + } else if (_ttl_ms > 0 && + TSC::ticks_to_millis(OS::nanotime() - load(_search_start_ns)) >= + (u64)_ttl_ms) { + // TTL bounds stop-the-world work independently of frontier progress. + store(_abandon_reason, (u8)SearchAbandonReason::TTL); + storeRelease(_search_state, (u8)SearchState::ABANDONED); + enqueuePendingAbandonedEvent(); + TEST_LOG_SUMMARY("ReferenceChainTracker::runPass ttl expired -- abandoning search " + "(ttl=%ldms elapsed_ms=%llu)", + _ttl_ms, + (unsigned long long)TSC::ticks_to_millis(OS::nanotime() - + load(_search_start_ns))); + } else if (_passes_since_last_progress >= NO_PROGRESS_PASS_LIMIT && + !isUrgent()) { + // The frontier hasn't grown for NO_PROGRESS_PASS_LIMIT consecutive passes — the search is + // genuinely stuck (not just slow), so abandon. + store(_abandon_reason, (u8)SearchAbandonReason::TTL); + storeRelease(_search_state, (u8)SearchState::ABANDONED); + enqueuePendingAbandonedEvent(); + } else if (_candidate_count > 0 && + __builtin_popcountll(_candidate_found_bits) == + (u64)_candidate_count) { + // Canary early termination: all leaked candidates have been found -- the search is complete. + storeRelease(_search_state, (u8)SearchState::COMPLETED); + Counters::increment(REFERENCE_CHAIN_CANDIDATES_FOUND, + __builtin_popcountll(_candidate_found_bits)); + } else if (_candidate_count > 0 && + _passes_since_last_progress >= NO_PROGRESS_PASS_LIMIT && + _passes_since_last_candidate_progress >= + canaryStuckPassLimit()) { + // Canary-specific stuck detector - deliberately NOT suppressed by isUrgent() (contrast the + // ordinary TTL check above). + store(_abandon_reason, (u8)SearchAbandonReason::CANARY_STUCK); + storeRelease(_search_state, (u8)SearchState::ABANDONED); + enqueuePendingAbandonedEvent(); + if (_canary_stuck_restart_count < MAX_CANARY_STUCK_BACKOFF_SHIFT) { + _canary_stuck_restart_count++; + } + } + + // Track progress: if the frontier grew this pass, reset the no-progress counter. + if (frontier_size_after > frontier_size_before_pass) { + _passes_since_last_progress = 0; + } else { + _passes_since_last_progress++; + } + + // Track canary-specific progress separately - see _passes_since_last_candidate_progress's own + // comment for why frontier growth above does not substitute for this. + { + u64 pass_wall_ms = (u64)TSC::ticks_to_millis(pass_wall_ticks); + _canary_pass_ema_ms = _canary_pass_ema_ms == 0 + ? pass_wall_ms + : _canary_pass_ema_ms * 4 / 5 + pass_wall_ms / 5; + } + int candidate_progress_mark = + _candidate_count + (int)__builtin_popcountll(_candidate_found_bits); + if (candidate_progress_mark > _last_candidate_progress_mark) { + _last_candidate_progress_mark = candidate_progress_mark; + _passes_since_last_candidate_progress = 0; + // Real chase progress (a candidate found or a new one admitted) - the canary lane gets its + // back-to-back spacing back (multiplier 1, see _canary_backoff_mult's own comment). + _canary_backoff_mult = 1; + } else { + _passes_since_last_candidate_progress++; + // No chase progress: double the canary lane's work-scaled spacing multiplier, capped. + if (_candidate_count > 0 && + __builtin_popcountll(_candidate_found_bits) < (u64)_candidate_count) { + _canary_backoff_mult = + std::min(_canary_backoff_mult * 2, CANARY_BACKOFF_MULT_MAX); + _last_canary_pass_ns = OS::nanotime(); + } + } + + if (load(_search_state) != SearchState::RUNNING) { + _tags_released = releaseSearchTags(jvmti, jni); + if (_candidate_count > 0) { + // No marker-tag release pass: the marker->leak-tag migration retired + // pre-tagged candidate representatives (nothing sets _candidate_tags + // anymore), so there are no per-candidate marker JVMTI tags to clear - + // releaseSearchTags() above owns every live tag this search minted. + // (The old GetObjectsWithTags(1, &_candidate_tags[i]) loop here was + // worse than dead: with every _candidate_tags[i] left at 0 it asked + // JVMTI to enumerate ALL UNTAGGED objects - potentially the whole + // heap - once per candidate slot on every search stop.) + _candidate_count = 0; + _candidate_found_bits = 0; + memset(_candidate_discovered_count, 0, sizeof(_candidate_discovered_count)); + memset(_candidate_qualifying_tid_count, 0, + sizeof(_candidate_qualifying_tid_count)); + _passes_since_last_candidate_progress = 0; + } + // Only CANARY_STUCK should keep escalating canaryStuckPassLimit() - any other terminal reason + // (natural completion, all candidates found, frontier cap, TTL) is an unrelated outcome for + // this chase sequence, so a fresh restart afterward should start back at the base limit. + if (load(_abandon_reason) != SearchAbandonReason::CANARY_STUCK) { + _canary_stuck_restart_count = 0; + } + } + + if (out_truncated != nullptr) { + *out_truncated = truncated; + } + + TEST_LOG_SUMMARY("ReferenceChainTracker::runPass done: err=%d edges_admitted=%d truncated=%d " + "frontier_cap_hit=%d searchState=%d abandonReason=%d frontierSize=%d " + "effectiveBudget=%d effectiveCadenceNs=%llu pendingExpand=%zu priorityExpand=%zu " + "candidateFound=%d/%d discoveredCounts=[%d,%d,%d,%d,%d]", + (int)err, edges_admitted, truncated, frontier_cap_hit, (int)load(_search_state), + (int)_abandon_reason, _frontier->size(), _effective_budget, + (unsigned long long)_effective_cadence_ns, + _pending_expand.size(), _priority_expand.size(), + (int)__builtin_popcountll(_candidate_found_bits), _candidate_count, + _candidate_count > 0 ? _candidate_discovered_count[0] : 0, + _candidate_count > 1 ? _candidate_discovered_count[1] : 0, + _candidate_count > 2 ? _candidate_discovered_count[2] : 0, + _candidate_count > 3 ? _candidate_discovered_count[3] : 0, + _candidate_count > 4 ? _candidate_discovered_count[4] : 0); + + return err == JVMTI_ERROR_NONE; +} + +// Pause-time-SLO feedback loop (see this method's declaration in + +void ReferenceChainTracker::updatePacing(u64 pass_wall_ticks) { + // Truncating to whole milliseconds matches every other PidController usage in this codebase + // (ObjectSampler/MallocTracer/NativeSocketSampler all feed it integer counts, pidController.h's + // `compute(u64 input, ...)`) - sub-ms precision is not meaningful against a millisecond-scale + // target anyway. + u64 pass_ms = TSC::ticks_to_millis(pass_wall_ticks); + // time_delta_coefficient is deliberately 1.0, not a real-elapsed-time ratio - unlike + // ObjectSampler's usage (objectSampler.cpp), which rescales an event count accumulated over a + // variable-length real-time window against a fixed-real-time target, _pause_pid was constructed + // with sampling_window=1 (its own constructor comment above, in start()): one compute() call *is* + // one pass, and pass_ms already IS the per-call quantity being compared against the per-call + // ceiling _target encodes. + double signal = _pause_pid.compute(pass_ms, 1.0); + + // Budget-borrowing (referenceChains.h's _borrowed_budget comment): only a sustained run of + // comfortably-under-target passes earns extra headroom above _budget, and any pass that is not + // comfortably under target revokes it immediately - _budget itself must stay the ceiling the + // instant this search stops proving it has pause-time room to spare. + bool comfortably_under_target = + _effective_pause_target_ms > 0 && + (double)pass_ms <= (double)_effective_pause_target_ms * BORROW_UNDER_TARGET_FRACTION; + if (comfortably_under_target) { + if (_consecutive_under_target_passes < BORROW_WARMUP_PASSES) { + _consecutive_under_target_passes++; + } + if (_consecutive_under_target_passes >= BORROW_WARMUP_PASSES) { + int64_t max_borrow = (int64_t)_budget * (BORROW_CEILING_MULTIPLIER - 1); + int64_t grown = _borrowed_budget + + (int64_t)std::llround((double)_budget * BORROW_GROWTH_FRACTION); + _borrowed_budget = std::min(grown, max_borrow); + } + } else { + _consecutive_under_target_passes = 0; + _borrowed_budget = 0; + } + + int64_t ceiling = (int64_t)_budget + _borrowed_budget; + int64_t floor = ceiling > 0 ? std::min((int64_t)MIN_EFFECTIVE_BUDGET, ceiling) + : 0; + int64_t desired = (int64_t)_effective_budget + (int64_t)std::lround(signal); + int64_t clamped = std::max(floor, std::min(ceiling, desired)); + // Whatever part of `desired` the clamp above could not absorb - positive when there was more + // headroom than the ceiling allows, negative when the pass is still over the pause-time target + // even at the floor. + int64_t overflow = desired - clamped; + _effective_budget = (int)clamped; + + if (overflow < 0) { + // Still over the pause-time ceiling even at the minimum budget - widen the fallback interval + // instead of shrinking the budget further. + u64 step = (u64)(-overflow) * CADENCE_NS_PER_EDGE_OVERFLOW; + _effective_cadence_ns = + std::min(_effective_cadence_ns + step, MAX_EFFECTIVE_CADENCE_NS); + } else if (overflow > 0) { + // Comfortably under the ceiling even at the maximum (config) budget - relax the fallback + // interval. + u64 step = (u64)overflow * CADENCE_NS_PER_EDGE_OVERFLOW; + _effective_cadence_ns = + step >= _effective_cadence_ns + ? MIN_EFFECTIVE_CADENCE_NS + : std::max(_effective_cadence_ns - step, MIN_EFFECTIVE_CADENCE_NS); + } + // overflow == 0: the budget clamp alone fully absorbed this pass's correction - leave the cadence + // at its current value. +} + +// A root/stack-ref enumeration pass never reaches updatePacing() above (see runPass()'s own comment +// on why its wall-clock cost is excluded from the per-pass PID/effective-budget signal), but it +// still spends real pause-time-SLO time. +void ReferenceChainTracker::maybeRevokeBorrowForRootEnumPass( + u64 pass_wall_ticks) { + if (_effective_pause_target_ms <= 0) { + return; + } + u64 pass_ms = TSC::ticks_to_millis(pass_wall_ticks); + bool comfortably_under_target = + (double)pass_ms <= (double)_effective_pause_target_ms * BORROW_UNDER_TARGET_FRACTION; + if (!comfortably_under_target) { + _consecutive_under_target_passes = 0; + _borrowed_budget = 0; + // The ceiling updatePacing() would compute right now collapses to _budget alone (no + // _borrowed_budget term above) - re-clamp _effective_budget immediately instead of leaving the + // borrow-inflated value in place until the next ordinary pass's updatePacing() call. + _effective_budget = std::min(_effective_budget, (int)_budget); + } +} + diff --git a/ddprof-lib/src/main/cpp/referenceChainWalk.cpp b/ddprof-lib/src/main/cpp/referenceChainWalk.cpp new file mode 100644 index 000000000..a74ecfbc3 --- /dev/null +++ b/ddprof-lib/src/main/cpp/referenceChainWalk.cpp @@ -0,0 +1,888 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "referenceChains.h" +#include "referenceChainInternal.h" +#include "common.h" +#include "counters.h" +#include "jniHelper.h" +#include "jvmThread.h" +#include "livenessTracker.h" +#include "log.h" +#include "objectSampler.h" +#include "os.h" +#include "profiler.h" +#include "rcDebugLevel.h" +#include "tsc.h" +#include "vmEntry.h" +#include +#include +#include +#include +#include +#include +#include +#include + +// Heap-walk engine + +void ReferenceChainTracker::resolveLoadedClasses(jvmtiEnv *jvmti, + JNIEnv *jni) { + // Profiler::start() resets the class-name StringDictionary (_class_map.clearAll(), profiler.cpp) + // whenever `reset || _start_time == 0` - which restarts its id namespace at 1, but does NOT touch + // any class's JVMTI-level class-object tag (JVM-level state, unrelated to our dictionary). + u64 current_generation = Profiler::instance()->classMap()->generation(); + bool class_map_reset = current_generation != _last_class_map_generation; + if (class_map_reset) { + TEST_LOG_SUMMARY("ReferenceChainTracker::resolveLoadedClasses class_map generation " + "changed: old=%llu new=%llu - clearing _class_tags and " + "candidate klass_ids may be stale", + (unsigned long long)_last_class_map_generation, + (unsigned long long)current_generation); + _class_tags.clear(); + // Force the scan below to run even if GetLoadedClasses()'s count happens to match the last-seen + // count - -1 can never equal `class_count` (always >= 0), unlike 0 which is a legitimate "no + // classes loaded yet" starting value. + _last_resolved_class_count = -1; + _last_class_map_generation = current_generation; + } + + jclass *classes = nullptr; + jint class_count = 0; + if (jvmti->GetLoadedClasses(&class_count, &classes) != JVMTI_ERROR_NONE || + classes == nullptr) { + return; + } + + // Skip the per-class GetTag()/GetClassSignature() scan entirely once the loaded-class count has + // not CHANGED since the last time this ran it: every already-tagged class stays tagged forever + // (tags are never cleared once assigned - see _class_tags' own comment), so a resumed pass with + // no newly-loaded classes has nothing left to resolve. + if (class_count != _last_resolved_class_count) { + for (jint i = 0; i < class_count; i++) { + jclass klass = classes[i]; + jlong tag = 0; + // Resolve if not yet tagged (ordinary case: a newly-loaded class), or unconditionally on a + // class-map reset (class_map_reset above) - a class already tagged from a prior generation + // still carries that same JVMTI tag (untouched by clearAll()), but the dictionary id it used + // to map to is gone, so its name must be re-resolved into the new generation too. + if (jvmti->GetTag(klass, &tag) == JVMTI_ERROR_NONE && + (tag == 0 || class_map_reset)) { + // Resolve its name now, via the same GetClassSignature + normalizeClassSignature + + // Profiler::lookupClass sequence ObjectSampler::recordAllocation() already uses + // (objectSampler.cpp:76-90), reused rather than re-derived. + char *class_name = nullptr; + if (jvmti->GetClassSignature(klass, &class_name, nullptr) == + JVMTI_ERROR_NONE && + class_name != nullptr) { + const char *name_slice = nullptr; + size_t name_len = 0; + if (ObjectSampler::normalizeClassSignature(class_name, &name_slice, + &name_len)) { + int id = Profiler::instance()->lookupClass(name_slice, name_len); + if (id != -1) { + TEST_LOG("ReferenceChainTracker::resolveClassMap id=%d name=%.*s", + id, (int)name_len, name_slice); + // Reuse the existing tag if this class was already tagged by a prior generation - + // only the resolved id needs refreshing, not the tag identity heapReferenceCallback() + // keys off of. + jlong class_tag = tag != 0 ? tag : nextClassTag(); + if (tag != 0 || + jvmti->SetTag(klass, class_tag) == JVMTI_ERROR_NONE) { + if (tag == 0) { + // Adopt the tag actually installed on the class object: LivenessTracker's + // mintStableClassTagIfNeeded() may have installed its own tag between our GetTag + // and SetTag (two SetTag calls on the same untagged class - the last writer wins + // on the class). + jlong installed = 0; + if (jvmti->GetTag(klass, &installed) == JVMTI_ERROR_NONE && + installed != 0) { + class_tag = installed; + } + } + _class_tags.insert(class_tag, (u32)id); + } + } + } + jvmti->Deallocate((unsigned char *)class_name); + } + } + // GetLoadedClasses() hands back class_count fresh JNI local refs - delete each immediately + // rather than holding all of them alive at once, since class_count can run into the + // thousands. + if (jni != nullptr) { + jni->DeleteLocalRef(klass); + } + } + _last_resolved_class_count = class_count; + } else if (jni != nullptr) { + // Still owe DeleteLocalRef for every fresh local ref GetLoadedClasses() just handed back, even + // though the scan above was skipped. + for (jint i = 0; i < class_count; i++) { + jni->DeleteLocalRef(classes[i]); + } + } + jvmti->Deallocate((unsigned char *)classes); +} + +jint JNICALL ReferenceChainTracker::heapReferenceCallback( + jvmtiHeapReferenceKind reference_kind, + const jvmtiHeapReferenceInfo *reference_info, jlong class_tag, + jlong referrer_class_tag, jlong size, jlong *tag_ptr, + jlong *referrer_tag_ptr, jint length, void *user_data) { + ReferenceChainPassContext *ctx = (ReferenceChainPassContext *)user_data; + + if (ctx->tracker->_abort_pass_requested.load(std::memory_order_relaxed)) { + // stopThread() has set this right before pthread_kill()/pthread_join() - see that method's own + // comment. + ctx->truncated = true; + return JVMTI_VISIT_ABORT; + } + + if (ctx->tracker->_pass_deadline_ns != 0 && + (++ctx->deadline_check_counter & 0xFFF) == 0 && + OS::nanotime() >= ctx->tracker->_pass_deadline_ns) { + // This pass has run past its wall-clock share (see _pass_deadline_ns's own comment) - treat it + // exactly like ordinary budget exhaustion so it ends early without abandoning the search; a + // later pass re-enumerates whatever roots/edges this one didn't get to. + ctx->truncated = true; + return JVMTI_VISIT_ABORT; + } + + // Retention-edge identity for every admission site below: the JVMTI heap callback's field ordinal + // (the JVMTI-SPECIFICATION numbering over the referrer's flattened field space - see + // FrontierEntry:: referrer_field_index's own comment) for FIELD/STATIC_FIELD edges, -1 otherwise; + // and the referrer's class tag when the referrer is a CLASS OBJECT (root-attached static edges - + // interior hops get their referrer class from the parent entry at chain-reconstruction time, so + // only the parent_tag==0 case needs it recorded here). + jint edge_field_index = -1; + if (reference_info != nullptr && + (reference_kind == JVMTI_HEAP_REFERENCE_FIELD || + reference_kind == JVMTI_HEAP_REFERENCE_STATIC_FIELD)) { + edge_field_index = reference_info->field.index; + } + jlong edge_referrer_class_tag = 0; + if (referrer_tag_ptr != nullptr && *referrer_tag_ptr < 0) { + edge_referrer_class_tag = *referrer_tag_ptr; + } + + // NOTE: the retired canary marker-tag decode used to live here (objects + // pre-tagged with MARKER_TAG_BASE - i were pruned as leaves and recorded as + // chain roots). The marker->leak-tag migration stopped pre-tagging candidate + // representatives entirely - no JVMTI tag in the process can ever be + // <= MARKER_TAG_BASE (-2^62): leak tags are positive (LEAK_TAG_BASE), + // frontier tags positive, class tags small negative magnitudes - so the + // branch was unreachable. Candidate discovery now runs exclusively through + // the leak-tag interception in pollWatchedTargets() (which also records + // _candidate_found_bits/_candidate_frontier_tags). + + if (*tag_ptr < 0) { + if (ctx->static_field_seed && + reference_kind == JVMTI_HEAP_REFERENCE_ARRAY_ELEMENT && + referrer_tag_ptr != nullptr && *referrer_tag_ptr == 0) { + // admitStaticFieldRoots()'s own holder[i] -> class edge: referrer_tag_ptr points at the + // transient, never-tagged seed array itself (tag 0), not at a frontier-admitted parent. + return JVMTI_VISIT_OBJECTS; + } + // Referee is a class object already tagged negative by resolveLoadedClasses() (that pre-pass + // runs before FollowReferences in runPass(), so every loaded class already carries a negative + // tag by this point). + return 0; + } + if (reference_kind == JVMTI_HEAP_REFERENCE_CLASS || + reference_kind == JVMTI_HEAP_REFERENCE_SYSTEM_CLASS) { + // Definitionally a class by reference_kind (CLASS: "reference from an object to its class"; + // SYSTEM_CLASS: a root reference to a class) even if resolveLoadedClasses() failed to + // resolve/tag this particular one (e.g. a transient StringDictionary contention failure) and + // its tag is therefore not yet negative. + return 0; + } + + if (ctx->truncated) { + // Defensive: FollowReferences should already have stopped delivering callbacks after a + // JVMTI_VISIT_ABORT return below; this just avoids doing further work if one more callback + // arrives anyway. + return JVMTI_VISIT_ABORT; + } + + jlong parent_tag = 0; + u32 depth = 0; + if (referrer_tag_ptr != nullptr) { + jlong rtag = *referrer_tag_ptr; + if (rtag > 0) { + FrontierEntry parent{}; + if (ctx->frontier->lookup(rtag, &parent)) { + parent_tag = rtag; + depth = parent.depth + 1; + } + // lookup() failing for a positive rtag should not happen - a referrer must already be one of + // our tagged frontier objects for its own outgoing edges to be traversed at all + // (FollowReferences only explores past an object this callback returned JVMTI_VISIT_OBJECTS + // for) - but fall back to root-like (parent_tag=0/depth=0) rather than corrupt the chain if + // it ever does. + } + // rtag < 0: referrer is a pre-tagged class object (e.g. a static field holding this reference) + // - treated as root-like rather than attributed to a parent hop, since class objects are never + // admitted as frontier entries and so have no depth/parent_tag of their own (see the *tag_ptr < + // 0 check above). + } + // referrer_tag_ptr == nullptr: a heap-root reference (JNI global, thread stack local/JNI local, + // monitor, thread, system class, ...) - parent_tag and depth stay 0. + + if (depth >= (u32)ctx->hop_cap) { + // Hop cap: do not admit this object into the frontier, and do not expand further from it - + // enforced here rather than discovering-then-discarding. + return 0; + } + + if (ctx->static_field_seed && referrer_tag_ptr != nullptr && + *referrer_tag_ptr < 0) { + // Referrer is the class object opened by the static_field_seed branch above. + if (*referrer_tag_ptr != ctx->_seed_class_tag) { + ctx->_seed_class_tag = *referrer_tag_ptr; + ctx->_class_other_admitted = 0; + ctx->_classes_in_chunk_visited++; + } + if (reference_kind != JVMTI_HEAP_REFERENCE_STATIC_FIELD) { + if (ctx->_class_other_cap > 0 && + ctx->_class_other_admitted >= ctx->_class_other_cap) { + // Quota exhausted for this class - drop the edge. Count every drop, and count the first + // drop for this class separately so the two counters together distinguish "a few fat + // outlier classes dropping many edges" from "systematic drops across almost all classes" + // (cap too low). + Counters::increment(REFERENCE_CHAIN_STATIC_SWEEP_NON_STATIC_DROPPED); + if (ctx->_class_other_admitted == ctx->_class_other_cap) { + Counters::increment(REFERENCE_CHAIN_STATIC_SWEEP_CLASSES_CAPPED); + } + return 0; + } + ctx->_class_other_admitted++; + } + } + + // Leak tag: this object was directly tagged by LivenessTracker's tagLeakInstances() because it's + // a tracked leaking object. + if (isLeakTag(*tag_ptr)) { + jlong leak_tag = *tag_ptr; + // Allocate a frontier tag for this object + jlong frontier_tag = ctx->tracker->nextTag(); + u32 referrer_klass = ctx->tracker->classTags()->resolve(class_tag); + u8 root_kind = parent_tag == 0 ? (u8)reference_kind : 0; + if (ctx->frontier->insert(frontier_tag, parent_tag, referrer_klass, + depth, FrontierEntryState::FRONTIER, + root_kind, class_tag, edge_field_index, + (u8)reference_kind, + parent_tag == 0 ? edge_referrer_class_tag : 0)) { + // Store the leak tag in the frontier entry + ctx->frontier->setLeakTag(frontier_tag, leak_tag); + *tag_ptr = frontier_tag; + ctx->edges_admitted++; + TEST_LOG("ReferenceChainTracker::heapReferenceCallback leak-tag " + "intercepted: leak_tag=%lld -> frontier_tag=%lld depth=%u " + "parent_tag=%lld", + (long long)leak_tag, (long long)frontier_tag, depth, + (long long)parent_tag); + ctx->tracker->trackLeakAccumulation(ctx->frontier, class_tag, + parent_tag, frontier_tag); + // Index maintenance: a leak-tagged object admitted root-attached by a durable root edge (e.g. + // a static field directly holding a tagged chunk) is the highest-priority anchor tier + // (leak_tag != 0). + if (parent_tag == 0 && + (root_kind == (u8)JVMTI_HEAP_REFERENCE_STATIC_FIELD || + root_kind == (u8)JVMTI_HEAP_REFERENCE_JNI_GLOBAL)) { + ctx->tracker->addToStaticAnchorIndex(frontier_tag, class_tag, + root_kind); + } + // Auto-mark: record this as a discovered instance, with eviction rights over uncorrelated + // noise slots (see recordDiscoveredInstance). + if (ctx->tracker->_candidate_count > 0) { + u32 klass_id = ctx->tracker->classTags()->resolve(class_tag); + ctx->tracker->recordDiscoveredInstance(klass_id, frontier_tag, true); + } + } else { + // Frontier cap hit + ctx->truncated = true; + ctx->frontier_cap_hit = true; + return JVMTI_VISIT_ABORT; + } + return JVMTI_VISIT_OBJECTS; + } + + // DESCEND-WALK GATES (no-ops on every ordinary walk - the ReferenceChainPassContext fields below + // are zero-initialized and only descendFromAnchor() sets them). + if (ctx->_no_descend_class_tag_count > 0) { + for (int i = 0; i < ctx->_no_descend_class_tag_count; i++) { + if (ctx->_no_descend_class_tags[i] == class_tag) { + return 0; + } + } + } + if (ctx->_descent_anchor_tag != 0 && ctx->_anchor_descend_class_tag != 0 && + referrer_tag_ptr != nullptr && + *referrer_tag_ptr == ctx->_descent_anchor_tag && + class_tag != ctx->_anchor_descend_class_tag) { + // This descend walk's ANCHOR object's own edge, and the referee is not the gate class (see + // ReferenceChainPassContext::_anchor_descend_class_tag's own comment - e.g. + // walkCandidateThreadLocals() walks ONLY the Thread's ThreadLocalMap edges, never enumerating + // the Thread's other fields). + return 0; + } + + if (*tag_ptr == 0) { + // First time this object is visited in this pass. + u32 referrer_klass = ctx->tracker->classTags()->resolve(class_tag); + // reference_kind describes this admitting edge; only meaningful for a root-attached entry + // (parent_tag == 0) - see FrontierEntry::root_kind's own comment for why a non-root entry's + // edge kind is not recorded. + u8 root_kind = parent_tag == 0 ? (u8)reference_kind : 0; + ReferenceChainTracker::AdmitResult result = ctx->tracker->admitObject( + ctx->frontier, ctx->hop_cap, ctx->budget, &ctx->edges_admitted, + tag_ptr, parent_tag, referrer_klass, depth, root_kind, class_tag, + ctx->admit_priority, edge_field_index, (u8)reference_kind, + parent_tag == 0 ? edge_referrer_class_tag : 0); + switch (result) { + case ReferenceChainTracker::AdmitResult::BUDGET_EXHAUSTED: + ctx->truncated = true; + return JVMTI_VISIT_ABORT; + case ReferenceChainTracker::AdmitResult::FRONTIER_CAP_HIT: + // Stop this pass when the frontier table is full. + ctx->truncated = true; + ctx->frontier_cap_hit = true; + return JVMTI_VISIT_ABORT; + default: + // ADMITTED, or HOP_CAP/ALREADY_ADMITTED (neither reachable here: the hop-cap check above + // already returned before this branch, and *tag_ptr == 0 rules out ALREADY_ADMITTED) - + // nothing further to do. + break; + } + // Index maintenance: track root-attached durable anchors for O(anchors) collector iteration + // instead of O(frontier_size) table scan. + if (result == ReferenceChainTracker::AdmitResult::ADMITTED && + parent_tag == 0) { + ctx->tracker->addToStaticAnchorIndex(*tag_ptr, class_tag, root_kind); + } + // Auto-mark: if this object's class matches a watched leak class, record its frontier tag so + // pollWatchedTargets() can build a chain event for it. + if (result == ReferenceChainTracker::AdmitResult::ADMITTED && + ctx->tracker->_candidate_count > 0) { + u32 klass_id = ctx->tracker->classTags()->resolve(class_tag); + if (klass_id == 0) { + // class_tag not in _class_tags - either class map rotated (resolveLoadedClasses hasn't + // re-resolved yet) or this class was never tagged. + TEST_LOG("ReferenceChainTracker::auto-mark class_tag=%lld " + "unresolved (not in _class_tags)", + (long long)class_tag); + } else { + bool matched = false; + for (int s = 0; s < ctx->tracker->_candidate_count; s++) { + if (ctx->tracker->_candidate_klass_ids[s] == klass_id) { + matched = true; + ctx->tracker->recordDiscoveredInstance(klass_id, *tag_ptr, + false); + break; + } + } + if (!matched && klass_id != 0) { + // klass_id resolved but doesn't match any candidate - likely class map rotation made + // candidate klass_ids stale + TEST_LOG("ReferenceChainTracker::auto-mark klass_id=%u " + "resolved but no candidate match (candidates=[%u,%u,%u,%u,%u])", + klass_id, + ctx->tracker->_candidate_count > 0 ? ctx->tracker->_candidate_klass_ids[0] : 0, + ctx->tracker->_candidate_count > 1 ? ctx->tracker->_candidate_klass_ids[1] : 0, + ctx->tracker->_candidate_count > 2 ? ctx->tracker->_candidate_klass_ids[2] : 0, + ctx->tracker->_candidate_count > 3 ? ctx->tracker->_candidate_klass_ids[3] : 0, + ctx->tracker->_candidate_count > 4 ? ctx->tracker->_candidate_klass_ids[4] : 0); + } + } + } + } else if (*tag_ptr > 0) { + // Already-tagged object reached via a new edge. This arm - NOT the first-admission block above + // - is where an already-admitted entry's shape can be corrected: + // improveChain/reparentToDurableRoot for a deeper/equal-durable path, + // maybeUpgradeRootAttachedRootKind for a new root-like edge. + if (parent_tag != 0) { + // This new path is deeper - replace the shallow root-attached entry with the deeper + // chain-attached entry. + u32 referrer_klass = ctx->tracker->classTags()->resolve(class_tag); + // Pre-read the CURRENT shape: if improveChain() below succeeds, this root-attached durable + // entry is about to be replaced with a deeper chain-attached one - i.e. it is leaving the + // population collectStaticFieldAnchorsForRotation() can select, at exactly this moment. + FrontierEntry pre_improve_entry{}; + bool was_root_attached_durable = + ctx->frontier->lookup(*tag_ptr, &pre_improve_entry) && + pre_improve_entry.parent_tag == 0 && + rootKindDurability(pre_improve_entry.root_kind) >= 2; + if (ctx->frontier->improveChain(*tag_ptr, parent_tag, referrer_klass, + depth, 0, edge_field_index, + (u8)reference_kind)) { + // Chain was improved — invalidate any cached chain for this tag so pollWatchedTargets + // rebuilds it with the deeper path. + ctx->tracker->invalidateResolvedChain(*tag_ptr); + if (was_root_attached_durable) { + // Demotion push (B'): the replaced entry's static/JNI-global attribution was its only + // anchor-tier eligibility, and it is gone now. + ctx->tracker->pushAtRiskStaticAnchor( + *tag_ptr, ctx->tracker->classTags()->resolve(class_tag)); + } + } else if (ctx->frontier->reparentToDurableRoot( + *tag_ptr, parent_tag, referrer_klass, edge_field_index, + (u8)reference_kind)) { + // Equal-depth re-parent from a transient root to a durable one (improveChain() cannot + // express it - see its declaration) - same cache invalidation so the rebuilt chain uses the + // durable root. + ctx->tracker->invalidateResolvedChain(*tag_ptr); + } + } else { + // Already-admitted entry reached via a NEW root-like edge (parent_tag == 0): the static-field + // sweep's class -> field edge reports the class as the referrer with a negative tag, which + // the rtag < 0 branch above treats as root-like (class objects are never frontier entries), + // and heap-root references arrive here with referrer_tag_ptr == nullptr. + if (ctx->tracker->maybeUpgradeRootAttachedRootKind(ctx->frontier, + *tag_ptr, + (u8)reference_kind)) { + ctx->tracker->invalidateResolvedChain(*tag_ptr); + } else if (reference_kind == JVMTI_HEAP_REFERENCE_STATIC_FIELD) { + // The upgrade refused (maybeUpgradeRootAttachedRootKind returns false for parent_tag != 0 + // by design), so this STATIC_FIELD edge just proved an at-risk static attachment the anchor + // tier's parent_tag == 0 filter can never see: a holder already admitted as a non-root + // child (find-anchor-holder-eviction). + FrontierEntry entry{}; + if (ctx->frontier->lookup(*tag_ptr, &entry) && + entry.parent_tag != 0) { + ctx->tracker->pushAtRiskStaticAnchor( + *tag_ptr, ctx->tracker->classTags()->resolve(class_tag)); + } + } + } + } + + if (ctx->batch_tags != nullptr) { + // ARRAY-HOLDER BATCHING one-hop descent control (see ReferenceChainPassContext:: batch_tags). + jlong my_tag = *tag_ptr; + if (my_tag > 0 && ctx->batch_tags->count(my_tag) != 0) { + // The previously visited batch entry (if any) is now fully processed - record it for the + // order-independent truncated-batch resume (see + // ReferenceChainPassContext::_completed_batch_tags' own comment). + if (ctx->_last_visited_batch_tag != 0 && + ctx->_last_visited_batch_tag != my_tag && + ctx->_completed_batch_tags != nullptr) { + ctx->_completed_batch_tags->insert(ctx->_last_visited_batch_tag); + } + // Track this batch entry as visited for the rolling resume cursor (see + // _last_visited_batch_tag's own comment). + ctx->_last_visited_batch_tag = my_tag; + return JVMTI_VISIT_OBJECTS; + } + return 0; + } + + return JVMTI_VISIT_OBJECTS; +} + +ReferenceChainTracker::AdmitResult ReferenceChainTracker::admitObject( + FrontierTable *frontier, int hop_cap, int budget, int *edges_admitted, + jlong *tag_ptr, jlong parent_tag, u32 referrer_klass, u32 depth, + u8 root_kind, jlong class_tag, bool priority, + jint edge_field_index, u8 edge_kind, jlong edge_referrer_class_tag) { + // edge_* default-declared in the header; heapRootCallback() passes the defaults (a root reference + // is not a field edge) unchanged. + if (*tag_ptr != 0) { + return AdmitResult::ALREADY_ADMITTED; + } + if (depth >= (u32)hop_cap) { + return AdmitResult::HOP_CAP; + } + if (*edges_admitted >= budget) { + return AdmitResult::BUDGET_EXHAUSTED; + } + jlong tag = nextTag(); + if (!frontier->insert(tag, parent_tag, referrer_klass, depth, + FrontierEntryState::FRONTIER, root_kind, class_tag, + edge_field_index, edge_kind, edge_referrer_class_tag)) { + return AdmitResult::FRONTIER_CAP_HIT; + } + *tag_ptr = tag; + (*edges_admitted)++; + // Queue for expandFrontier()/markAllFrontierExpanded() - see _pending_expand's/_priority_expand's + // own declaration comments for why this replaces a scan over the admitted range, and for why a + // rotation-discovered child (priority=true) skips the ordinary backlog. + if (priority && _priority_expand.size() < PRIORITY_EXPAND_CAP) { + _priority_expand.push_back(tag); + _priority_expand_set.insert(tag); + } else { + // Priority lane full: the rotation backpressure falls back to the ordinary backlog rather than + // silently dropping the re-discovered subtree (see PRIORITY_EXPAND_CAP's own comment). + _pending_expand.push_back(tag); + } + trackLeakAccumulation(frontier, class_tag, parent_tag, tag); + return AdmitResult::ADMITTED; +} + +void ReferenceChainTracker::trackLeakAccumulation(FrontierTable *frontier, + jlong class_tag, + jlong parent_tag, + jlong tag) { + // Cheapest checks first: no klass_id is currently watched (the common case before hasLeakSignal() + // has ever fired - see _watched_leak_klass_ids' own comment), or this admission has no real + // parent to attribute to (a root-attached entry - nothing to aggregate by, since the "container" + // concept this tracks is specifically about a PARENT object's field holding the leaf, not the + // leaf itself being root-attached). + if (_watched_leak_klass_count <= 0 || parent_tag == 0 || class_tag == 0) { + return; + } + // (u32) truncation matches _watched_leak_klass_ids' own storage (see that field's comment) - + // class tags are small, negative, sequentially-minted values in practice + // (ClassTagAllocator::next()), so this never actually loses distinguishing information; it just + // keeps the comparison and the signature-key packing below in the same 32-bit space both already + // used for the (superseded) classMap-id scheme. + u32 truncated_class_tag = (u32)class_tag; + bool watched = false; + for (int i = 0; i < _watched_leak_klass_count; i++) { + if (_watched_leak_klass_ids[i] == truncated_class_tag) { + watched = true; + break; + } + } + if (!watched) { + return; + } + FrontierEntry parent_entry{}; + if (!frontier->lookup(parent_tag, &parent_entry) || + parent_entry.class_tag == 0) { + // Parent since pruned/dead between its own admission and this child's, or admitted before this + // field existed on it (should not happen in practice - class_tag is set at every admission - + // but a stale/unknown parent identity is not something to attribute this observation to either + // way. + return; + } + u64 key = leakSignatureKey(truncated_class_tag, (u32)parent_entry.class_tag); + _leak_signature_totals[key]++; + auto it = _leak_parent_fanout.find(parent_tag); + if (it == _leak_parent_fanout.end()) { + TEST_LOG("ReferenceChainTracker::trackLeakAccumulation fanout-insert " + "parent_tag=%lld parent_class_tag=%lld child_class_tag=%lld", + (long long)parent_tag, (long long)parent_entry.class_tag, + (long long)class_tag); + _leak_parent_fanout.emplace(parent_tag, LeakParentFanoutEntry{key, 1}); + } else { + // The signature key for a given parent_tag is fixed once recorded (parent_entry.class_tag never + // changes once admitted; the LEAF side of the key is fixed by which klass_id is currently + // watched at the time of THIS call, which could in principle differ between two children of the + // same parent if _watched_leak_klass_ids itself changed between them - overwrite rather than + // accumulate under a stale key in that case, since the stored signature_key should always + // reflect the most recently observed watched klass_id for this parent). + it->second.signature_key = key; + it->second.fanout++; + } + // ANCESTOR FANOUT: the direct parent is not necessarily the part of the holder chain that STAYS + // LIVE. + jlong ancestor = parent_entry.parent_tag; + int hops = 0; + while (ancestor != 0 && hops++ < _hop_cap) { + FrontierEntry ancestor_entry{}; + if (!_frontier->lookup(ancestor, &ancestor_entry)) { + break; + } + if (_leak_parent_fanout.find(ancestor) == _leak_parent_fanout.end()) { + _leak_parent_fanout.emplace(ancestor, LeakParentFanoutEntry{key, 1}); + } + if (ancestor_entry.parent_tag == 0) { + break; // root-attached: the holder chain ends here + } + ancestor = ancestor_entry.parent_tag; + } +} + +void ReferenceChainTracker::seedLeakAccumulationForNewlyWatchedKlass( + u32 klass_id) { + if (_frontier == nullptr) { + // pollWatchedTargets() can run before the first pass has ever created the frontier table - + // nothing to seed from yet. + return; + } + int table_size = _frontier->size(); + if (table_size <= 0) { + return; + } + // Inlines trackLeakAccumulation()'s own signature/fanout update logic (rather than calling it per + // matching entry) deliberately: this whole scan already holds _frontier's shared lock for its + // duration (matching collectStaleExpandedEntriesForRotation()'s own lockShared() rationale - a + // per-tag SpinLock acquisition would double the cost of this O(table_size) sweep), and + // trackLeakAccumulation() takes that same lock itself via frontier->lookup() - calling it from + // inside an already-held shared section would risk a reentrant-lock deadlock if a writer is ever + // concurrently pending, so this uses lookupLocked() throughout instead. + _frontier->withSharedLock([&](const FrontierTable *frontier) { + for (jlong tag = 1; tag <= table_size; tag++) { + FrontierEntry entry{}; + if (!frontier->lookupLocked(tag, &entry) || + entry.state != FrontierEntryState::EXPANDED || + entry.parent_tag == 0 || (u32)entry.class_tag != klass_id) { + continue; + } + FrontierEntry parent_entry{}; + if (!frontier->lookupLocked(entry.parent_tag, &parent_entry) || + parent_entry.class_tag == 0) { + continue; + } + u64 key = leakSignatureKey(klass_id, (u32)parent_entry.class_tag); + _leak_signature_totals[key]++; + auto it = _leak_parent_fanout.find(entry.parent_tag); + if (it == _leak_parent_fanout.end()) { + _leak_parent_fanout.emplace(entry.parent_tag, + LeakParentFanoutEntry{key, 1}); + } else { + it->second.signature_key = key; + it->second.fanout++; + } + } + }); +} + +bool ReferenceChainTracker::maybeUpgradeRootAttachedRootKind( + FrontierTable *frontier, jlong tag, u8 new_root_kind) { + FrontierEntry entry{}; + if (!frontier->lookup(tag, &entry)) { + return false; + } + if (entry.parent_tag != 0) { + // Not root-attached - per this phase's option (a) resolution of the parent_tag==0/root_kind + // invariant conflict (referenceChains.h's FrontierEntry::root_kind comment), only a + // root-context update may ever write a non-zero root_kind, and only onto an entry that is + // already root-attached. + return false; + } + if (rootKindDurability(new_root_kind) <= rootKindDurability(entry.root_kind)) { + return false; + } + frontier->updateRootKind(tag, new_root_kind); + addToStaticAnchorIndex(tag, entry.class_tag, new_root_kind); + return true; +} + +std::vector +ReferenceChainTracker::collectStaleRootKindEntriesForRotation( + int max_count) { + std::vector selected; + int table_size = _frontier->size(); + if (max_count <= 0 || table_size <= 0) { + return selected; + } + if (_root_kind_rotation_cursor <= 0 || + _root_kind_rotation_cursor > table_size) { + _root_kind_rotation_cursor = 1; + } + + // Held for the whole sweep below (potentially wrapping all the way around table_size) rather than + // once per tag via lookup() - the same rationale as collectStaleExpandedEntriesForRotation()'s + // own lockShared() use: a per-tag SpinLock acquisition would double this scan's cost under a + // large frontier table. + jlong start_tag = _root_kind_rotation_cursor; + jlong tag = start_tag; + _frontier->withSharedLock([&](const FrontierTable *frontier) { + do { + FrontierEntry entry{}; + if (frontier->lookupLocked(tag, &entry) && + entry.state == FrontierEntryState::EXPANDED && + entry.parent_tag == 0 && isTransientRootKind(entry.root_kind) && + !isQueuedForRotation(tag) && + _priority_expand.size() < PRIORITY_EXPAND_CAP) { + selected.push_back(tag); + _priority_expand.push_back(tag); + _priority_expand_set.insert(tag); + if ((int)selected.size() >= max_count) { + tag = tag % table_size + 1; + break; + } + } + tag = tag % table_size + 1; + } while (tag != start_tag); + }); + + _root_kind_rotation_cursor = tag; + return selected; +} + +std::vector +ReferenceChainTracker::collectStaleExpandedEntriesForRotation( + int max_count) { + std::vector selected; + int table_size = _frontier->size(); + if (max_count <= 0 || table_size <= 0) { + return selected; + } + // LEAK-PARENT PRIORITY, FAIR-SHARED WITH THE BLIND LAP: _leak_parent_fanout knows the EXPANDED + // parents that actually lead to watched leak-klass children - re-walking one of those re-sees its + // current children (improveChain() upgrades children first admitted via a shallower path, + // leak-tag interception for the tagged ones) and catches elements added since its expansion, + // which is exactly the mutation this rotation exists to observe. + if (!_leak_parent_fanout.empty() && + _priority_expand.size() < PRIORITY_EXPAND_CAP) { + int fanout_budget = (max_count + 1) / 2; + size_t fanout_size = _leak_parent_fanout.size(); + u64 skip = _leak_parent_rotation_cursor % fanout_size; + auto it = _leak_parent_fanout.begin(); + while (it != _leak_parent_fanout.end()) { + if ((int)selected.size() >= fanout_budget || + _priority_expand.size() >= PRIORITY_EXPAND_CAP) { + break; + } + if (skip > 0) { + skip--; + ++it; + continue; + } + jlong parent_tag = it->first; + if (isQueuedForRotation(parent_tag)) { + ++it; + continue; + } + FrontierEntry entry{}; + // Dead parent: either the frontier slot is gone entirely, or it was clear()'d (dead object / + // restart wipe) - clear() marks the slot ABANDONED rather than removing it, so both + // conditions must erase (tags are never reused within a search and the fanout is wiped on + // restart, so an ABANDONED parent can never come back to life). + if (!_frontier->lookup(parent_tag, &entry) || + entry.state == FrontierEntryState::ABANDONED) { + it = _leak_parent_fanout.erase(it); + continue; + } + if (entry.state != FrontierEntryState::EXPANDED) { + ++it; + continue; + } + selected.push_back(parent_tag); + _priority_expand.push_back(parent_tag); + _priority_expand_set.insert(parent_tag); + ++it; + } + _leak_parent_rotation_cursor += selected.size() + 1; + if ((int)selected.size() >= max_count) { + // Budget exhausted by the fanout alone (only possible for max_count == 1, where the fanout's + // ceil-half share is the whole budget) - fanout-priority preserved, and the lap below has + // nothing left to do this pass. + return selected; + } + } + if (_stale_expanded_rotation_cursor <= 0 || + _stale_expanded_rotation_cursor > table_size) { + _stale_expanded_rotation_cursor = 1; + } + // Resume scanning from _stale_expanded_rotation_cursor rather than always restarting at tag 1: a + // frontier table can accumulate far more than max_count entries that are EXPANDED and stay that + // way forever (long-lived infrastructure objects - caches, maps, bootstrap classes). + int deadline_check_counter = 0; + jlong start_tag = _stale_expanded_rotation_cursor; + jlong tag = start_tag; + _frontier->withSharedLock([&](const FrontierTable *frontier) { + do { + if (_pass_deadline_ns != 0 && + (++deadline_check_counter & 0xFFF) == 0 && + OS::nanotime() >= _pass_deadline_ns) { + // Ran past this pass's wall-clock share - stop scanning with whatever was already selected + // (possibly none) and resume from here next call. + break; + } + FrontierEntry entry{}; + if (frontier->lookupLocked(tag, &entry) && + entry.state == FrontierEntryState::EXPANDED && + !isQueuedForRotation(tag) && + _priority_expand.size() < PRIORITY_EXPAND_CAP) { + selected.push_back(tag); + _priority_expand.push_back(tag); + _priority_expand_set.insert(tag); + if ((int)selected.size() >= max_count) { + tag = tag % table_size + 1; + break; + } + } + tag = tag % table_size + 1; + } while (tag != start_tag); + }); + _stale_expanded_rotation_cursor = tag; + return selected; +} + +// Select high-fanout parents of classes reported as growing. +std::vector +ReferenceChainTracker::collectLeakAccumulationCandidatesForRotation( + int max_count) { + std::vector selected; + if (max_count <= 0 || _leak_signature_totals.empty()) { + return selected; + } + + // Tier 1: rank signatures by growth since the last pass's snapshot. + u64 winning_key = 0; + bool have_winner = false; + u32 best_delta = 0; + for (const auto &kv : _leak_signature_totals) { + u32 prev = 0; + auto prev_it = _leak_signature_prev_totals.find(kv.first); + if (prev_it != _leak_signature_prev_totals.end()) { + prev = prev_it->second; + } + u32 delta = kv.second > prev ? kv.second - prev : 0; + if (delta > 0 && (!have_winner || delta > best_delta)) { + have_winner = true; + best_delta = delta; + winning_key = kv.first; + } + } + // Roll the snapshot forward for the NEXT pass's comparison regardless of whether this pass found + // a winner - a signature that didn't grow this pass still needs its current total remembered so a + // future pass's delta is computed against the right baseline, not against however many passes ago + // it was last checked. + _leak_signature_prev_totals = _leak_signature_totals; + if (!have_winner) { + // Nothing grew since last pass - nothing to prioritize this tier this time + // (collectStaleExpandedEntriesForRotation()'s unprioritized fallback still covers this + // population eventually). + return selected; + } + + // Tier 2: within the winning signature only, rank concrete parent objects by their own fanout - + // collected first, then partially sorted, since _leak_parent_fanout's total size is what bounds + // this method's cost (not table_size), and is expected to be small (see that map's own comment). + std::vector> candidates; // (parent_tag, fanout) + for (const auto &kv : _leak_parent_fanout) { + if (kv.second.signature_key == winning_key && !isQueuedForRotation(kv.first)) { + FrontierEntry entry{}; + if (_frontier->lookup(kv.first, &entry) && + (entry.state == FrontierEntryState::EXPANDED || + entry.state == FrontierEntryState::FRONTIER)) { + candidates.emplace_back(kv.first, kv.second.fanout); + } + } + } + std::sort(candidates.begin(), candidates.end(), + [](const std::pair &a, const std::pair &b) { + return a.second > b.second; + }); + for (const auto &c : candidates) { + if ((int)selected.size() >= max_count || + _priority_expand.size() >= PRIORITY_EXPAND_CAP) { + break; + } + selected.push_back(c.first); + _priority_expand_set.insert(c.first); + FrontierEntry state_entry{}; + bool is_expanded = _frontier->lookup(c.first, &state_entry) && + state_entry.state == FrontierEntryState::EXPANDED; + TEST_LOG("ReferenceChainTracker::" + "collectLeakAccumulationCandidatesForRotation selected " + "parent_tag=%lld state=%s fanout=%u", + (long long)c.first, is_expanded ? "EXPANDED" : "FRONTIER", + c.second); + } + // Place the whole selection at the head of the priority lane, keeping the fanout ranking order + // (see the FRONTIER-state case in the Tier 2 comment above for why the head and not the tail): + // push_front reverses, so insert back-to-front. + for (auto it = selected.rbegin(); it != selected.rend(); ++it) { + _priority_expand.push_front(*it); + } + return selected; +} + diff --git a/ddprof-lib/src/main/cpp/referenceChains.cpp b/ddprof-lib/src/main/cpp/referenceChains.cpp new file mode 100644 index 000000000..719a40136 --- /dev/null +++ b/ddprof-lib/src/main/cpp/referenceChains.cpp @@ -0,0 +1,1062 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "referenceChains.h" +#include "referenceChainInternal.h" +#include "common.h" +#include "counters.h" +#include "jniHelper.h" +#include "jvmThread.h" +#include "livenessTracker.h" +#include "log.h" +#include "objectSampler.h" +#include "os.h" +#include "profiler.h" +#include "rcDebugLevel.h" +#include "tsc.h" +#include "vmEntry.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Reference-chains debug-log level (see rcDebugLevel.h). Level 0 silent +namespace { +constexpr const char *kRcDebugLevelEnv = "DD_PROFILING_REFERENCE_CHAINS_DEBUG"; +constexpr const char *kRcDebugLevelFile = "/tmp/ddprof_root/refchains_debug_level"; +constexpr u64 kRcDebugLevelRefreshTtlNs = 1000000000ULL; // 1s +std::atomic g_rc_debug_level{-1}; // -1 = not yet resolved from env +std::atomic g_rc_debug_level_last_refresh_ns{0}; + +int envRcDebugLevel() { + int lvl = parseRcDebugLevel(getenv(kRcDebugLevelEnv)); + return lvl < 0 ? 0 : lvl; // invalid/unset env means silent +} +} // namespace + +int rcDebugLevel() { + int lvl = g_rc_debug_level.load(std::memory_order_relaxed); + if (lvl >= 0) { + return lvl; + } + // Lazy one-time env resolve; may fire from a heap callback on the very first log line, which is + // still strictly cheaper than the fprintf the same line performs in a DEBUG build. + lvl = envRcDebugLevel(); + g_rc_debug_level.store(lvl, std::memory_order_relaxed); + return lvl; +} + +int parseRcDebugLevel(const char *value) { + if (value == nullptr || *value == '\0') { + return -1; + } + // Trim surrounding whitespace (files written via `echo N >` end with \n). + while (*value == ' ' || *value == '\t' || *value == '\n' || *value == '\r') { + ++value; + } + const char *end = value + strlen(value); + while (end > value && (end[-1] == ' ' || end[-1] == '\t' || + end[-1] == '\n' || end[-1] == '\r')) { + --end; + } + if (end == value || end - value != 1) { + return -1; // exactly one digit + } + if (*value < '0' || *value > '2') { + return -1; + } + return *value - '0'; +} + +int readRcDebugLevelFile(const char *path) { + if (path == nullptr) { + return -1; + } + // The knob file lives under the world-writable /tmp (see kRcDebugLevelFile's + // comment): refuse anything that is not a regular file owned by root or the + // current user, so a local user cannot plant a symlink or a pre-created + // file of their own and force the DEBUG-build diagnostics on. The worst + // impact of a forged file is log-volume/CPU from enabled TEST_LOG in a + // DEBUG build, but the check is cheap and keeps the knob owner-scoped. + struct stat st; + if (lstat(path, &st) != 0 || !S_ISREG(st.st_mode)) { + return -1; // missing, symlink, fifo, dir - treat as "no override" + } + if (st.st_uid != 0 && st.st_uid != geteuid()) { + return -1; + } + FILE *f = fopen(path, "r"); + if (f == nullptr) { + return -1; + } + char buf[16]; + size_t n = fread(buf, 1, sizeof(buf) - 1, f); + fclose(f); + buf[n] = '\0'; + return parseRcDebugLevel(buf); +} + +void rcDebugLevelRefresh(bool force) { + u64 now = OS::nanotime(); + u64 last = g_rc_debug_level_last_refresh_ns.load(std::memory_order_relaxed); + if (!force && last != 0 && now >= last && + now - last < kRcDebugLevelRefreshTtlNs) { + return; + } + g_rc_debug_level_last_refresh_ns.store(now, std::memory_order_relaxed); + int lvl = readRcDebugLevelFile(kRcDebugLevelFile); + if (lvl < 0) { + lvl = envRcDebugLevel(); // file absent/invalid -> fall back to env + } + g_rc_debug_level.store(lvl, std::memory_order_relaxed); +} + +// ReferenceChainTracker + +// Marks the calling thread as executing inside the GarbageCollectionStart/ Finish JVMTI callback +// for the duration of the guard's lifetime. +thread_local bool t_inGCCallback = false; + +namespace { +class GCCallbackGuard { +public: + GCCallbackGuard() { t_inGCCallback = true; } + ~GCCallbackGuard() { t_inGCCallback = false; } +}; +} // namespace + +void ReferenceChainTracker::autoTuneDefaults(Arguments &args) { + // Only tune defaults the operator did not set explicitly. + const u8 tuned = args._reference_chains_tuned_mask; + + // Max heap is resolved by LivenessTracker::initialize_table() at this point + // (ObjectSampler::start() -> LivenessTracker::start() runs before ReferenceChainTracker::start() + // in Profiler::start()). + jlong max_heap = LivenessTracker::instance()->maxHeapBytes(); + if (max_heap <= 0) { + return; // can't tune without heap size + } + + // Available processors from JVMTI (cached by FlightRecorder, but we can query JVMTI directly + // here). + jint nprocs = 1; + jvmtiEnv *jvmti = VM::jvmti(); + if (jvmti != nullptr) { + jvmti->GetAvailableProcessors(&nprocs); + } + if (nprocs < 1) nprocs = 1; + + // Heap size in MiB. + double heap_mib = (double)max_heap / (1024.0 * 1024.0); + + // --- Budget (edges per BFS pass) --- Scale with sqrt(heap_mib): a 4 GiB heap gets 2x, a 16 GiB + // heap gets 4x, a 64 GiB heap gets 8x the default 1000. + if (!(tuned & REF_CHAINS_TUNED_BUDGET)) { + int scaled = (int)(DEFAULT_REFERENCE_CHAINS_BUDGET * std::sqrt(heap_mib / 512.0)); + args._reference_chains_budget = std::max(DEFAULT_REFERENCE_CHAINS_BUDGET, + std::min(scaled, MAX_REFERENCE_CHAINS_BUDGET)); + } + + // --- First-pass budget --- The root enumeration pass is one-shot per search and can afford a + // much larger budget. + if (!(tuned & REF_CHAINS_TUNED_FIRST_PASS_BUDGET)) { + int fpb = args._reference_chains_budget * 10; + args._reference_chains_first_pass_budget = std::min(fpb, + MAX_REFERENCE_CHAINS_FIRST_PASS_BUDGET); + } + + // --- TTL (per-search wall-clock lifetime) --- The search needs enough time to cover the heap at + // the tuned budget. + if (!(tuned & REF_CHAINS_TUNED_TTL)) { + long scaled_ttl = (long)(DEFAULT_REFERENCE_CHAINS_TTL_MS * (heap_mib / 512.0)); + scaled_ttl = std::max(DEFAULT_REFERENCE_CHAINS_TTL_MS, std::min(scaled_ttl, + (long)(30 * 60 * 1000))); // 30 min max + args._reference_chains_ttl_ms = scaled_ttl; + } + + // --- Frontier cap --- The frontier grows with the number of edges admitted per pass. + if (!(tuned & REF_CHAINS_TUNED_FRONTIER_CAP)) { + int scaled_cap = (int)(DEFAULT_REFERENCE_CHAINS_FRONTIER_CAP * + ((double)args._reference_chains_budget / DEFAULT_REFERENCE_CHAINS_BUDGET)); + args._reference_chains_frontier_cap = std::max( + DEFAULT_REFERENCE_CHAINS_FRONTIER_CAP, + std::min(scaled_cap, MAX_REFERENCE_CHAINS_FRONTIER_CAP)); + } + + // --- Pause target --- More available processors = the JVM can afford a slightly longer per-pass + // safepoint without impacting application throughput. + if (!(tuned & REF_CHAINS_TUNED_PAUSE_TARGET)) { + long scaled_pause = DEFAULT_REFERENCE_CHAINS_PAUSE_TARGET_MS * + (1 + (nprocs - 1) / 3); + args._reference_chains_pause_target_ms = std::min(scaled_pause, (long)50); + } + + // --- Pain budget percent --- More cores = more spare capacity for background work. + if (!(tuned & REF_CHAINS_TUNED_PAIN_BUDGET)) { + int scaled_pain = DEFAULT_REFERENCE_CHAINS_PAIN_BUDGET_PERCENT * + (1 + (nprocs - 1) / 4); + args._reference_chains_pain_budget_percent = std::min(scaled_pain, 5); + } + + Log::info("Reference chain auto-tuner: heap=%.0f MiB nprocs=%d -> " + "budget=%d ttl=%ldms framecap=%d pausetarget=%ldms painbudget=%d%% firstpassbudget=%d", + heap_mib, (int)nprocs, + args._reference_chains_budget, args._reference_chains_ttl_ms, + args._reference_chains_frontier_cap, + args._reference_chains_pause_target_ms, + args._reference_chains_pain_budget_percent, + args._reference_chains_first_pass_budget); +} + +Error ReferenceChainTracker::start(Arguments &args) { + _enabled = args._reference_chains; + + if (!_enabled) { + Log::info("Reference chain tracking is disabled"); + return Error::OK; + } + + // Recording-boundary hygiene: Profiler::start() clears the class dictionary (restart its id + // namespace) right before this runs, so cached chain events and queued abandonment events from a + // prior recording carry StringDictionary ids from a wiped generation - re-emitting them into the + // new recording would write missing or newly-reassigned class ids for chains that describe the + // previous recording's objects. + _resolved_chains_lock.lock(); + _resolved_chains.clear(); + _resolved_chains_lock.unlock(); + _pending_abandoned_events_lock.lock(); + _pending_abandoned_events.clear(); + _pending_abandoned_events_lock.unlock(); + _urgency_budget_boosted = false; + + // Auto-tune defaults that the operator did not set explicitly, based on max heap size and + // available processors. + autoTuneDefaults(args); + + Log::info("Reference chain tracking is enabled (hops=%d, budget=%d, " + "ttl=%ldms, framecap=%d, pausetarget=%ldms, painbudget=%d%%)", + args._reference_chains_hop_cap, args._reference_chains_budget, + args._reference_chains_ttl_ms, args._reference_chains_frontier_cap, + args._reference_chains_pause_target_ms, + args._reference_chains_pain_budget_percent); + + // Like LivenessTracker's own table, construct the frontier table once and keep it across repeated + // start()/stop() cycles - do not reallocate on a second start() with a possibly different cap, + // for the same reason LivenessTracker keeps its first-initialize() result. + _configured_frontier_cap = args._reference_chains_frontier_cap; + if (_frontier == nullptr) { + _frontier = new FrontierTable(_configured_frontier_cap); + } + // The configured budget is what the urgency ramp restores when urgency clears (see the urgency + // block in threadLoop()) - the live _budget must not be snapshotted for that, it may already be + // boosted. + _configured_budget = args._reference_chains_budget; + + _hop_cap = args._reference_chains_hop_cap; + _budget = args._reference_chains_budget; + // 0 (unset) auto-scales from _budget instead of falling back to it plainly - see this field's own + // comment (referenceChains.h) for why a steady-state per-pass budget is the wrong size for the + // first pass. + _first_pass_budget = args._reference_chains_first_pass_budget > 0 + ? args._reference_chains_first_pass_budget + : std::min(_budget * AUTO_FIRST_PASS_BUDGET_MULTIPLIER, + AUTO_FIRST_PASS_BUDGET_CAP); + _ttl_ms = args._reference_chains_ttl_ms; + + // Pause-time pacing controller: (re)seed the controller's ceiling and the adaptive values it + // drives. + _pause_target_ms = args._reference_chains_pause_target_ms; + _effective_pause_target_ms = _pause_target_ms; + _effective_budget = _budget; + _effective_cadence_ns = PASS_CADENCE_NS; + _candidate_count = 0; + _candidate_found_bits = 0; + memset(_candidate_discovered_count, 0, sizeof(_candidate_discovered_count)); + memset(_candidate_qualifying_tid_count, 0, + sizeof(_candidate_qualifying_tid_count)); + _passes_since_last_candidate_progress = 0; + _last_candidate_progress_mark = 0; + // Fresh chase gets a fresh back-to-back spacing allowance (see _canary_backoff_mult's own + // comment). + _canary_backoff_mult = 1; + _canary_pass_ema_ms = 0; + _last_canary_pass_ns = 0; + _canary_stuck_restart_count = 0; + // Budget-borrowing (referenceChains.h's _borrowed_budget comment): reset alongside the rest of + // the pacing controller's state, so a restarted search never inherits headroom earned by a + // previous one. + _borrowed_budget = 0; + _consecutive_under_target_passes = 0; + _pause_pid = PidController((u64)std::max(_pause_target_ms, 0L), + 10, // proportional gain: reacts to a single + // pass's over/under-ceiling error without needing many passes to + // notice - a duration-ms error is typically single/ + // low-double-digit in magnitude (unlike the shared triple's + // event-count scale), so a smaller P keeps a one-pass overshoot + // from. + 1, // integral gain: small and round - + // pidController.cpp's `_integral_value` has no built-in clamp, + // and this controller is invoked once per BFS pass rather than + // on the other three usages' roughly-periodic + // one-call-per-second cadence, so windup accumulates faster per + // wall-clock. + 2, // derivative gain: small, matching the + // shared triple's own "the derivational gain is rather small" + // rationale (objectSampler.cpp) - a single slow/ fast pass + // should not itself trigger a large swing + 1, // sampling_window=1: one compute() call + // *is* one pass, not a fixed real-time window like the other + // three usages assume (see _pause_pid's own comment) + 5.0 // cutoff_secs: a round value, halved from + // the shared triple's own "15" since a pass-scoped signal is + // naturally noisier per-call than a roughly-1s- cadence one + ); + + // (re)seed _safepoint_pain_budget from the configured refill rate, mirroring _pause_pid's own + // reconstruct-in-start() pattern above. + _safepoint_pain_budget = PainBudget( + std::max(args._reference_chains_pain_budget_percent, 0) / 100.0); + _pain_budget_refill_rate = std::max(args._reference_chains_pain_budget_percent, 0) / 100.0; + // Same refill rate as _safepoint_pain_budget above - one operator-facing "how much background + // cost is acceptable" percentage covers both leaky buckets (see _cpu_pain_budget's own comment, + // referenceChains.h). + _cpu_pain_budget = PainBudget(_pain_budget_refill_rate); + + // Lazy-enable, matching LivenessTracker::start(): the GC callbacks are wired unconditionally in + // vmEntry.cpp, but the events themselves are only turned on for this JVMTI env when the flag is + // on. + jvmtiEnv *jvmti = VM::jvmti(); + jvmti->SetEventNotificationMode( + JVMTI_ENABLE, JVMTI_EVENT_GARBAGE_COLLECTION_START, nullptr); + jvmti->SetEventNotificationMode( + JVMTI_ENABLE, JVMTI_EVENT_GARBAGE_COLLECTION_FINISH, nullptr); + + // Deliberately does NOT create the BFS thread (threadEntry()/threadLoop() below) here - + // threadLoop()'s VM::attachThread() call dereferences VM::_vm unconditionally (vmEntry.h:191-195) + // and crashes if the VM is not yet attached, which is exactly the case in this file's own gtest + // binary (referenceChains_ut.cpp calls start() directly with no live JVM). + + return Error::OK; +} + +void ReferenceChainTracker::stop() { + if (!_enabled) { + return; + } + Log::info("Reference chain tracking stopped"); + + // Do not disable GC notifications here - LivenessTracker follows the same rule since the JVMTI + // env and its tracker singletons are expected to survive across multiple start/stop recording + // cycles. +} + +void ReferenceChainTracker::startThread() { + if (!_enabled || _running.load(std::memory_order_acquire)) { + return; + } + // Reset from any previous stopThread() call - a dynamic-attach profiler can go through multiple + // start()/stop() cycles in one JVM lifetime (this class's own start()/stop() header comments), + // and a stale abort request left set from the prior cycle would make heapReferenceCallback() + // abort this new cycle's very first pass instantly. + _abort_pass_requested.store(false, std::memory_order_relaxed); + + // Publish _running=true *before* creating the thread, not after. + _running.store(true, std::memory_order_release); + pthread_t thread; + if (pthread_create(&thread, NULL, threadEntry, this) != 0) { + Log::warn("Unable to create ReferenceChains BFS thread"); + _running.store(false, std::memory_order_release); + return; + } + _thread = thread; +} + +void ReferenceChainTracker::stopThread() { + if (!_running.load(std::memory_order_acquire)) { + return; + } + _running.store(false, std::memory_order_release); + // Ask any in-flight JVMTI FollowReferences walk (heapReferenceCallback()) to abort at its next + // callback invocation - set before pthread_kill() below, since that signal alone cannot interrupt + // a call already inside the JVM/JVMTI implementation. + _abort_pass_requested.store(true, std::memory_order_relaxed); + // Same wake-then-join shape as BaseWallClock::stop() (wallClock.cpp:324-333): + // pthread_kill(WAKEUP_SIGNAL) interrupts threadLoop()'s OS::sleep() early (WAKEUP_SIGNAL/SIGIO is + // installed with a no-op handler unconditionally in vmEntry.cpp, so this signal never terminates + // the thread) so it re-checks _running and exits promptly rather than waiting out the rest of the + // current sleep interval. + pthread_kill(_thread, WAKEUP_SIGNAL); + int res = pthread_join(_thread, NULL); + if (res != 0) { + Log::warn("Unable to join ReferenceChains BFS thread on stop %d", res); + } +} + +// Runs scheduled passes on an attached agent thread. +void ReferenceChainTracker::threadLoop() { + struct Cleanup { + ReferenceChainTracker *tracker; + ~Cleanup() { + // No cached-class cleanup needed before detaching: _cached_object_class is a global ref, + // deliberately valid across attach/detach cycles (see its own comment in referenceChains.h) - + // unlike the per-attach local ref it replaced, which this destructor used to have to clear + // here. + VM::detachThread(); + } + } cleanup{this}; + JNIEnv *jni = VM::attachThread("java-profiler ReferenceChains"); + jvmtiEnv *jvmti = VM::jvmti(); + if (jni == nullptr) { + // AttachCurrentThreadAsDaemon() failed - mirror pollWatchedTargets()'s own jni==nullptr early + // return rather than letting a null JNIEnv flow into + // runPass()/resolveLoadedClasses()/expandFrontier()/ releaseSearchTags() below: those only + // guard their DeleteLocalRef() calls on `jni != nullptr`, so without this check every + // GetLoadedClasses()/GetObjectsWithTags() local ref returned on this (permanently un-attached) + // thread would leak for the rest of the process's lifetime. + Log::warn("ReferenceChains: VM::attachThread failed; BFS thread exiting"); + return; + } + DEBUG_ONLY(rcDebugLevelRefresh(true)); // apply the override file before the first log line + TEST_LOG_SUMMARY("ReferenceChainTracker::threadLoop started, cadence=%lluns rc_debug_level=%d", (unsigned long long)_effective_cadence_ns, rcDebugLevel()); + + int iteration = 0; + while (_running.load(std::memory_order_acquire)) { + // Fixed ~1s cadence, no early wake on GC (see onGCFinish()'s own comment) - stopThread() still + // interrupts this via its own pthread_kill so shutdown stays prompt. + double seconds_to_oom = LivenessTracker::instance()->secondsToOOM(); + bool urgent = seconds_to_oom >= 0 && seconds_to_oom < OOM_RAMP_START_S; + long target_ms = _pause_target_ms; + u64 cadence_ns = _effective_cadence_ns; + if (urgent) { + double x = 1.0 - seconds_to_oom / OOM_RAMP_START_S; // 0 at 30min out, 1 at OOM + target_ms = std::lround(_pause_target_ms * + std::pow((double)URGENT_PAUSE_TARGET_MS / std::max(_pause_target_ms, 1L), x)); + // Ramp from the fixed configured cadence, not the currently-adaptive _effective_cadence_ns - + // using the live value as the ramp's own moving anchor would compound the exponent across + // iterations instead of tracking urgency directly from a stable baseline. + cadence_ns = (u64)std::llround((double)PASS_CADENCE_NS * + std::pow((double)URGENT_CADENCE_NS / (double)PASS_CADENCE_NS, x)); + // While urgent, the ramp owns _effective_cadence_ns outright so shouldRunPass()'s cadence + // gate and the per-pass log actually reflect it. + _effective_cadence_ns = cadence_ns; + } + if (target_ms != _effective_pause_target_ms) { + _effective_pause_target_ms = target_ms; + _pause_pid = PidController((u64)std::max(_effective_pause_target_ms, 0L), + 10, 1, 2, 1, 5.0); + } + // Once in the ramp window, hold the budget ceiling raised for the urgency episode's entire + // duration rather than only right before OOM: the process is likely to die anyway, so it's + // worth spending whatever budget it takes to collect good diagnostic data for as long as we + // have. + if (urgent && !_urgency_budget_boosted) { + _urgency_budget_boosted = true; + _budget = std::min(_budget * 4, MAX_REFERENCE_CHAINS_BUDGET); + TEST_LOG_SUMMARY("ReferenceChainTracker::threadLoop urgency budget boost " + "budget=%d configured=%d", + _budget, _configured_budget); + } else if (!urgent && _urgency_budget_boosted) { + _urgency_budget_boosted = false; + _budget = _configured_budget; + TEST_LOG_SUMMARY("ReferenceChainTracker::threadLoop urgency budget restore " + "budget=%d", _budget); + } + // Third trigger for LivenessTracker::cleanup_table() (see + // LivenessTracker::maybeForceCleanup()'s own comment): track()'s table-overflow branch and + // flush_table()'s JFR cadence can both starve under ObjectSampler's PID-controlled sampling + // interval, leaving hasLeakSignal() below stuck on a stale population history no matter how + // long a real leak keeps growing. + u64 wake_now_ns = OS::nanotime(); + LivenessTracker::instance()->maybeForceCleanup(wake_now_ns); + + // No fast-path skip here: shouldRunPass() below already returns false cheaply (a couple of + // atomic loads/comparisons, no JVMTI call) for a RUNNING search with no new GC and cadence not + // yet elapsed. + u64 now_ns = OS::nanotime(); + + // Re-check the runtime debug-level override file (~1s TTL; see rcDebugLevel.h) - never in heap + // callbacks, which only read the cached atomic. + DEBUG_ONLY(rcDebugLevelRefresh()); + + // Hand this iteration's ramp state to shouldRunPass() before it decides - the canary-backoff + // gate is bypassed while the OOM urgency ramp is active (see _oom_ramp_active's own comment) - + // and raise LivenessTracker's tracking admission to 100% for the same ramp (see + // setUrgentTracking()'s own comment, livenessTracker.h): same state, same iteration, so the + // boost tracks the ramp exactly, engaging and releasing together. + _oom_ramp_active = urgent; + LivenessTracker::instance()->setUrgentTracking(urgent); + + bool should_run = shouldRunPass(now_ns); + // Only sleep when idle (no pass will run). When a canary search is active or a pass is about to + // run, skip the sleep to run passes back-to-back. + if (!should_run && cadence_ns > 0) { + OS::sleep(cadence_ns); + if (!_running.load(std::memory_order_acquire)) { + break; + } + now_ns = OS::nanotime(); + } + // Log the loop state only when a pass is actually going to run - the idle wakes (should_run == + // false) are the common steady state and logging them every second is pure noise. + if (should_run) { + TEST_LOG_SUMMARY("ReferenceChainTracker::threadLoop iteration=%d shouldRunPass=%d searchState=%d " + "passesRun=%d effectiveCadenceNs=%llu effectiveBudget=%d gcFinishEpoch=%llu " + "lastPassGcFinishEpoch=%llu nowMinusLastPassNs=%llu", + ++iteration, should_run, (int)_search_state, _passes_run, + (unsigned long long)_effective_cadence_ns, _effective_budget, + (unsigned long long)gcFinishEpoch(), (unsigned long long)_last_pass_gc_finish_epoch, + (unsigned long long)(now_ns - _last_pass_ns)); + runPassSerialized(jvmti, jni); + } + // Target-selection bridging step: poll once per scheduling cycle, after runPass() - so this + // poll always sees the most recent pass's tagging (see pollWatchedTargets()'s own comment). + pollWatchedTargetsSerialized(jvmti, jni); + } +} + +void JNICALL ReferenceChainTracker::GarbageCollectionStart(jvmtiEnv *jvmti_env) { + ReferenceChainTracker::instance()->onGCStart(); +} + +void JNICALL ReferenceChainTracker::GarbageCollectionFinish(jvmtiEnv *jvmti_env) { + ReferenceChainTracker::instance()->onGCFinish(); +} + +void ReferenceChainTracker::onGCStart() { + if (!_enabled) { + return; + } + // JVMTI spec: only Memory Management category calls (Allocate/Deallocate) are allowed from inside + // this callback - nothing else may run here. + GCCallbackGuard guard; + atomicIncRelaxed(_gc_start_epoch, (u64)1); +} + +void ReferenceChainTracker::onGCFinish() { + if (!_enabled) { + return; + } + GCCallbackGuard guard; + // Heap-category JVMTI calls are forbidden from GC callbacks. + atomicIncRelaxed(_gc_finish_epoch, (u64)1); +} + +bool ReferenceChainTracker::shouldRunPass(u64 now_ns) { + if (!_search_started) { + // Same gate as a restart (canAffordNewSearch() below) - a brand-new tracker must not pay for + // the first whole-heap walk/tagging pass either when there is no leak candidate to justify it. + bool afford = canAffordNewSearch(now_ns); + TEST_LOG_SUMMARY("ReferenceChainTracker::shouldRunPass search_not_started " + "canAffordNewSearch=%d", (int)afford); + if (!afford) { + return false; + } + // This episode's one urgency-authorized search (_urgent_search_spent's own comment, + // referenceChains.h) is the one about to start. + _urgent_search_spent = _urgent_latched; + TEST_LOG_SUMMARY("ReferenceChainTracker::shouldRunPass -> true (search not started yet)"); + return true; // nothing has run yet - always worth taking the first pass + } + if (_search_state != SearchState::RUNNING) { + // Terminal outcome already reached (runPass()'s Termination section). + if (!_tags_released) { + // releaseSearchTags() failed to confirm every live tag this search owned was actually cleared + // - restartSearch() must never run until that is confirmed (see _tags_released's own + // comment), so return true unconditionally here: that drives threadLoop() to call runPass() + // again, whose terminal-state branch retries the release, rather than letting + // canAffordNewSearch()/restartSearch() below run ahead of it. + TEST_LOG_SUMMARY("ReferenceChainTracker::shouldRunPass -> true (retrying tag " + "release before restart is allowed)"); + return true; + } + // Charge the finished search's accumulated safepoint cost BEFORE the restart gate - + // canAffordNewSearch() must see the cost of the search that just ended, otherwise an expensive + // search earns one free immediate successor (the accumulator is spent here, once per search; + // repeated terminal visits spend a zeroed accumulator). + _safepoint_pain_budget.spend(_search_pain_ms); + _search_pain_ms = 0; + // Restart (this class's own header comment) if the pain budget has drained and there is still + // (or again) a leak indication to chase - canAffordNewSearch() is always true when + // LivenessTracker's population trends are not in use at all, so this only ever changes behavior + // for a search that already ran once. + if (canAffordNewSearch(now_ns)) { + // Same entitlement bookkeeping as the first-search branch above. + _urgent_search_spent = _urgent_latched; + restartSearch(); + TEST_LOG_SUMMARY("ReferenceChainTracker::shouldRunPass -> true (restarting search)"); + return true; + } + // No log here: a terminal search waiting for a restart to become warranted is the common idle + // state, re-evaluated every second, so logging it is pure per-second noise (see threadLoop()). + TEST_LOG_SUMMARY("ReferenceChainTracker::shouldRunPass terminal_blocked " + "tags_released=%d safepoint_pain=%d search_state=%d", + (int)_tags_released, + (int)_safepoint_pain_budget.canStartNow(now_ns), + (int)_search_state); + return false; + } + // Canary search active with candidates still to find - computed ahead of the pain-budget check + // below so the refill-rate raise and the backoff gate further down agree on the same snapshot of + // _candidate_found_bits. + bool canary_active = _candidate_count > 0 && + __builtin_popcountll(_candidate_found_bits) < (u64)_candidate_count; + // Adaptive CPU budget: 100x refill while a canary chase is open - NOT a rate control (the canary + // lane's rate is bounded by _canary_backoff_ns's progress-driven exponential backoff, see its own + // comment) but a double-throttle guard: the base refill rate is tuned for the ordinary ~1 pass/s + // whole-graph cadence and would otherwise starve a chase the backoff has already paced. + double multiplier = + canary_active ? CANARY_PAIN_BUDGET_REFILL_MULTIPLIER : 1.0; + _cpu_pain_budget.setRefillRate( + std::min(_pain_budget_refill_rate * multiplier, 1.0), + now_ns); + if (!_cpu_pain_budget.canStartNow(now_ns)) { + TEST_LOG_SUMMARY("ReferenceChainTracker::shouldRunPass blocked by cpu_pain_budget " + "balance=%.1fms refill_rate=%.4f canary_active=%d " + "multiplier=%.1f", + _cpu_pain_budget.balanceMs(now_ns), + std::min(_pain_budget_refill_rate * multiplier, 1.0), + (int)canary_active, multiplier); + return false; + } + if (canary_active) { + // Canary-lane pacing: the chase's rate bound - work-scaled spacing (_canary_backoff_mult's own + // comment for the law and the live burn it bounds). + u64 spacing_ns = + (u64)_canary_backoff_mult * _canary_pass_ema_ms * 1000000ULL; + // mult == 1 (fresh chase, or last pass made progress) means the gate is OFF - the chase runs at + // its natural pass rate, one pass starting as soon as the last ended. + if (!_oom_ramp_active && _canary_backoff_mult > 1 && + now_ns - _last_canary_pass_ns < spacing_ns) { + TEST_LOG("ReferenceChainTracker::shouldRunPass held off by canary " + "backoff mult=%d ema_ms=%llu since_last_pass=%llums", + _canary_backoff_mult, + (unsigned long long)_canary_pass_ema_ms, + (unsigned long long)((now_ns - _last_canary_pass_ns) / + 1000000ULL)); + return false; + } + TEST_LOG_SUMMARY("ReferenceChainTracker::shouldRunPass -> true (canary search, " + "%d/%d candidates found, backoff_mult=%d ema_ms=%llu)", + (int)__builtin_popcountll(_candidate_found_bits), + (int)_candidate_count, _canary_backoff_mult, + (unsigned long long)_canary_pass_ema_ms); + return true; + } + u64 gc_finish_epoch = gcFinishEpoch(); + if (gc_finish_epoch != _last_pass_gc_finish_epoch) { + // Triggering section: "a GC just happened, a pass may be worth running soon". + TEST_LOG_SUMMARY("ReferenceChainTracker::shouldRunPass -> true (gcFinishEpoch=%llu != " + "lastPassGcFinishEpoch=%llu)", + (unsigned long long)gc_finish_epoch, + (unsigned long long)_last_pass_gc_finish_epoch); + return true; + } + // Pause-time pacing controller: compares against _effective_cadence_ns, not the fixed + // PASS_CADENCE_NS - see that field's own comment (referenceChains.h) for how updatePacing() + // widens or relaxes it from the measured pause-time signal. + bool cadence_elapsed = now_ns - _last_pass_ns >= _effective_cadence_ns; + // Only log when the cadence actually elapsed (a pass will run). + if (cadence_elapsed) { + TEST_LOG_SUMMARY("ReferenceChainTracker::shouldRunPass -> true (now_ns=%llu last_pass_ns=%llu " + "delta=%llu effectiveCadenceNs=%llu)", + (unsigned long long)now_ns, (unsigned long long)_last_pass_ns, + (unsigned long long)(now_ns - _last_pass_ns), + (unsigned long long)_effective_cadence_ns); + } + return cadence_elapsed; +} + +// Search restart gate (this class's own header comment). Deliberately a probe (max=1) rather than +// reusing pollWatchedTargets()'s own selectLeakCandidates() call - that one runs after runPass() in +// threadLoop()'s own iteration and needs the *list* to poll each candidate's tag; this only needs +// to know whether at least one exists. +bool ReferenceChainTracker::isUrgent() const { + double seconds_to_oom = LivenessTracker::instance()->secondsToOOM(); + if (seconds_to_oom >= 0 && seconds_to_oom < OOM_URGENT_THRESHOLD_S) { + _urgent_release_ticks = 0; + if (!_urgent_latched) { + _urgent_latched = true; + // A fresh episode gets a fresh entitlement to one search. + _urgent_search_spent = false; + TEST_LOG_SUMMARY("ReferenceChainTracker::isUrgent latching urgency " + "(secondsToOOM=%.1f < OOM_URGENT_THRESHOLD_S=%.1f)", + seconds_to_oom, OOM_URGENT_THRESHOLD_S); + } + return true; + } + if (_urgent_latched) { + // Negative means "no rising trend to project from" (secondsToOOM()'s own unknown/NOT_RISING + // encoding), which counts toward release just like a comfortably distant projection does. + if (seconds_to_oom < 0 || seconds_to_oom >= OOM_URGENT_RELEASE_S) { + if (++_urgent_release_ticks >= URGENT_RELEASE_CONSECUTIVE) { + _urgent_latched = false; + _urgent_release_ticks = 0; + _urgent_search_spent = false; + TEST_LOG_SUMMARY("ReferenceChainTracker::isUrgent releasing urgency " + "(secondsToOOM=%.1f clear of OOM_URGENT_RELEASE_S=%.1f for " + "%d consecutive observations)", + seconds_to_oom, OOM_URGENT_RELEASE_S, + URGENT_RELEASE_CONSECUTIVE); + return false; + } + } else { + // Between the two bars, or a single noisy reading past the release bar followed by one that + // is not - neither releases the latch. + _urgent_release_ticks = 0; + } + return true; + } + return false; +} + +bool ReferenceChainTracker::hasLeakSignal() { + if (!LivenessTracker::instance()->gcGenerationsEnabled()) { + // No population-trend signal to gate on at all - see this method's own header comment for why + // that means "always true" here. + return true; + } + double seconds_to_oom = LivenessTracker::instance()->secondsToOOM(); + // isUrgent() is called unconditionally, not short-circuited behind _urgent_search_spent: it is + // what maintains the latch/release counter, so skipping it would freeze the episode state (see + // _urgent_latched). + bool urgent = isUrgent(); + if (urgent && !_urgent_search_spent) { + // Heap-wide floor is rising fast enough that OOM is imminent - don't wait for a specific klass + // to clear selectLeakCandidates()'s own per-klass ring-fill/hysteresis gate; see + // OOM_URGENT_THRESHOLD_S's own comment (referenceChains.h) for why that gate alone is too slow + // here. + TEST_LOG_SUMMARY("ReferenceChainTracker::hasLeakSignal -> true (urgent, " + "secondsToOOM=%.1f)", + seconds_to_oom); + return true; + } + KlassCandidate probe[1]; + int n = LivenessTracker::instance()->selectLeakCandidates(probe, 1); + TEST_LOG_SUMMARY("ReferenceChainTracker::hasLeakSignal -> %s (secondsToOOM=%.1f, " + "candidates=%d, urgent=%d, urgentSearchSpent=%d)", + n > 0 ? "true" : "false", seconds_to_oom, n, urgent, + _urgent_search_spent); + return n > 0; +} + +bool ReferenceChainTracker::canAffordNewSearch(u64 now_ns) { + if (!_safepoint_pain_budget.canStartNow(now_ns)) { + TEST_LOG_SUMMARY("ReferenceChainTracker::canAffordNewSearch blocked by " + "safepoint_pain_budget balance=%.1fms refill_rate=%.4f", + _safepoint_pain_budget.balanceMs(now_ns), + _pain_budget_refill_rate); + return false; // still cooling down from the last search's own cost + } + return hasLeakSignal(); +} + +// Search restart (this class's own header comment). Called only from shouldRunPass() once +// canAffordNewSearch() has approved it, immediately before returning true for this same iteration - +// runPass() then sees _search_started == false and takes the first-pass branch, exactly like a +// brand-new tracker. +void ReferenceChainTracker::restartSearch() { + // Only called once shouldRunPass() has confirmed _tags_released - never while a prior search's + // release might still be pending (see _tags_released's own comment): resetting _next_tag to 1 / + // the frontier table below while some object could still hold this search's now- ambiguous tag + // would let the restarted search's fresh tags collide with it. + assert(_tags_released && + "restartSearch() must not run before releaseSearchTags() has " + "confirmed every live tag was cleared"); + + // The finishing search's accumulated safepoint cost is spent by the terminal restart gate in + // shouldRunPass() BEFORE it calls this (see the gate's comment: the gate must see the finished + // search's cost), so no spend happens here. + + if (_frontier != nullptr) { + _frontier->resetForRestart(); + } + _next_tag = 1; + // Hop-edge label cache: keyed by raw class tags, which survive a restart (the shared class-tag + // allocator is deliberately not reset - see this method's own declaration comment) - but the + // frontier entries referencing them do not, and a restart is the natural bounded clear point for + // a cache capped by HOP_LABEL_CLASS_CACHE_CAP wholesale. + _hop_label_cache.clear(); + // The shared class-tag counter (classTagAllocator.h)/_class_tags intentionally untouched - see + // this method's own declaration comment (referenceChains.h). + + _search_started = false; + store(_search_state, (u8)SearchState::RUNNING); + store(_abandon_reason, (u8)SearchAbandonReason::NONE); + store(_search_start_ns, (u64)0); + _pending_expand.clear(); + _priority_expand.clear(); + _priority_expand_set.clear(); + _static_anchor_fifo.clear(); + _static_anchor_fifo_set.clear(); + _static_anchor_fifo_klass_counts.clear(); + _static_anchor_index.clear(); + _static_anchor_own_class_tags.clear(); + _static_anchor_index_tags.clear(); + _anchor_container_cursor = 0; + _anchor_other_cursor = 0; + // Fresh lane: nothing admits before the search does, so nothing can have a pending first look + // either. + _static_anchor_fresh_queue.clear(); + // Discovered-instance tags are FRONTIER tags - the reset above just invalidated every one of them + // (fresh tags restart from 1). + memset(_candidate_discovered_tags, 0, sizeof(_candidate_discovered_tags)); + memset(_candidate_discovered_count, 0, sizeof(_candidate_discovered_count)); + // Both keyed by frontier tags this restart is about to invalidate (fresh tags start again from 1) + // - a stale entry surviving past a restart would be compared against whatever unrelated object + // the new search has since reassigned that tag to. + _leak_signature_totals.clear(); + _leak_signature_prev_totals.clear(); + _leak_parent_fanout.clear(); + _leak_tags_assigned = 0; + _leak_tags_resolved = 0; + _last_pass_gc_finish_epoch = 0; + store(_last_pass_ns, (u64)0); + store(_passes_run, 0); + // Reset back to their just-constructed values (0 / -1) like every other per-search field this + // method touches: resolveLoadedClasses() and admitStaticFieldRoots() must both run + // unconditionally on the restarted search's first pass, exactly as they do for a brand-new + // tracker. + _last_resolved_class_count = 0; + _last_static_field_class_count = -1; + // _resolved_chains is intentionally left intact: a chain resolved by the finishing search stays + // cached (and keeps being re-emitted on every dump) across the restart, since it describes a + // sample that is still live. +} + +void ReferenceChainTracker::resetSearchStateForTest(jvmtiEnv *jvmti, + JNIEnv *jni) { + // Every field touched below is otherwise only ever mutated by the BFS thread itself + // (threadLoop()/runPass()/pollWatchedTargets()) - without stopping it first, a pass already in + // flight on that thread can observe this reset only partially, or overwrite it right back (e.g. + // finish a pass that was already headed for SearchState::ABANDONED after this method has just + // forced SearchState::RUNNING below), a race found in practice, not just in theory. + stopThread(); + + // Clear every live tag this search still holds before resetting - the same ordering + // restartSearch() itself requires (its own assert), so a stale tag from whatever search a + // previous test left running cannot collide with the fresh search's own tags once _next_tag is + // rewound below. + if (jvmti != nullptr && jni != nullptr) { + releaseSearchTags(jvmti, jni); + } + _tags_released = true; + + _safepoint_pain_budget.spend(_search_pain_ms); + _search_pain_ms = 0; + // Reset the pain budget entirely so a fresh test starts from zero debt, independent of how much + // wall-clock time has elapsed since the last test's spend(). + _safepoint_pain_budget = PainBudget(_pain_budget_refill_rate); + // Mirror the reset for the non-safepoint budget - same test-isolation rationale as + // _safepoint_pain_budget above. + _cpu_pain_budget = PainBudget(_pain_budget_refill_rate); + // Same test-isolation rationale: a latched urgency episode left behind by an earlier test would + // otherwise deny this one its own urgency-authorized search (see _urgent_search_spent). + _urgent_latched = false; + _urgent_release_ticks = 0; + _urgent_search_spent = false; + + if (_frontier != nullptr) { + // Rebuilds the table at this test's own _configured_frontier_cap, undoing any smaller framecap= + // an earlier test left it permanently sized at (this class's own header comment on + // @TestMethodOrder) - restartSearch()'s production path only calls the cheaper + // resetForRestart() since it never needs to change the cap mid-JVM. + _frontier->resetCapacityForTest(_configured_frontier_cap); + } + _next_tag = 1; + + _search_started = false; + store(_search_state, (u8)SearchState::RUNNING); + store(_abandon_reason, (u8)SearchAbandonReason::NONE); + store(_search_start_ns, (u64)0); + _pending_expand.clear(); + _priority_expand.clear(); + _priority_expand_set.clear(); + _static_anchor_fifo.clear(); + _static_anchor_fifo_set.clear(); + _static_anchor_fifo_klass_counts.clear(); + _static_anchor_index.clear(); + _static_anchor_own_class_tags.clear(); + _static_anchor_index_tags.clear(); + _anchor_container_cursor = 0; + _anchor_other_cursor = 0; + _static_anchor_fresh_queue.clear(); + // Same stale-frontier-tag hygiene as restartSearch() (see its comment): discovered tags are + // frontier tags, invalid across the test reset just as across a restart. + memset(_candidate_discovered_tags, 0, sizeof(_candidate_discovered_tags)); + memset(_candidate_discovered_count, 0, sizeof(_candidate_discovered_count)); + // Test-only extra: production restartSearch() keeps _class_shape_cache (class tags are + // JVM-lifetime-stable there), but test scenarios script class-tag values directly and a later + // test can reuse an earlier one for a different mock class - clear the cache between tests. + _class_shape_cache.clear(); + // Same reset rationale as restartSearch()'s own comment. + _leak_signature_totals.clear(); + _leak_signature_prev_totals.clear(); + _leak_parent_fanout.clear(); + _leak_tags_assigned = 0; + _leak_tags_resolved = 0; + _last_pass_gc_finish_epoch = 0; + store(_last_pass_ns, (u64)0); + store(_passes_run, 0); + _passes_since_last_progress = 0; + _passes_since_last_candidate_progress = 0; + _last_candidate_progress_mark = 0; + _canary_backoff_mult = 1; + _canary_pass_ema_ms = 0; + _last_canary_pass_ns = 0; + _canary_stuck_restart_count = 0; + // Same "just-constructed values" contract resetForRestart() already documents for these two + // fields - without it, a prior test's fully-swept (or partially-swept) state survives in this + // process-wide singleton (ReferenceChainTracker::instance()) and can wrongly skip + // admitStaticFieldRoots() entirely on this test's first pass if its resolved class count happens + // to match whatever an earlier test last left behind (found via a real gtest-suite-order failure, + // not hypothetical). + _last_resolved_class_count = 0; + _last_static_field_class_count = -1; + _static_field_sweep_cursor = 0; + _static_field_sweep_cycle_truncated = false; + _candidate_count = 0; + _candidate_found_bits = 0; + memset(_candidate_discovered_count, 0, sizeof(_candidate_discovered_count)); + memset(_candidate_qualifying_tid_count, 0, + sizeof(_candidate_qualifying_tid_count)); + // _candidate_parent_tags/_candidate_referrer_klasses/_candidate_depths will be filled at pruning + // time. + _resolved_chains_lock.lock(); + _resolved_chains.clear(); + _resolved_chains_lock.unlock(); + _pending_abandoned_events_lock.lock(); + _pending_abandoned_events.clear(); + _pending_abandoned_events_lock.unlock(); + + // Restart the BFS thread against this freshly reset state - startThread() itself clears + // _abort_pass_requested, so the new thread's very first pass is not instantly aborted by the flag + // stopThread() just set above. + startThread(); +} + +long ReferenceChainTracker::pendingExpandPositionForTest(jlong tag) const { + if (tag == 0) { + return -2; + } + // _priority_expand drains first (expandFrontier()'s own comment), so its entries are reported as + // coming before _pending_expand's. + long pos = 0; + for (jlong queued : _priority_expand) { + if (queued == tag) { + return pos; + } + pos++; + } + for (jlong queued : _pending_expand) { + if (queued == tag) { + return pos; + } + pos++; + } + return -1; +} + +size_t ReferenceChainTracker::pendingExpandSizeForTest() const { + return _pending_expand.size() + _priority_expand.size(); +} + +jlong ReferenceChainTracker::tagObject(jvmtiEnv *jvmti, jobject obj) { + assert(!t_inGCCallback && + "SetTag is a JVMTI Heap-category call and must not be made from " + "GarbageCollectionStart/Finish"); + jlong tag = nextTag(); + jvmtiError err = jvmti->SetTag(obj, tag); + if (err != JVMTI_ERROR_NONE) { + return 0; + } + return tag; +} + +jlong ReferenceChainTracker::getTag(jvmtiEnv *jvmti, jobject obj) { + assert(!t_inGCCallback && + "GetTag is a JVMTI Heap-category call and must not be made from " + "GarbageCollectionStart/Finish"); + jlong tag = 0; + jvmtiError err = jvmti->GetTag(obj, &tag); + if (err != JVMTI_ERROR_NONE) { + return 0; + } + return tag; +} + +void ReferenceChainTracker::clearTag(jvmtiEnv *jvmti, jobject obj) { + assert(!t_inGCCallback && + "SetTag is a JVMTI Heap-category call and must not be made from " + "GarbageCollectionStart/Finish"); + jvmti->SetTag(obj, 0); +} + +jlong ReferenceChainTracker::tagAsRootForTest(jvmtiEnv *jvmti, JNIEnv *jni, + jobject obj) { + if (_frontier == nullptr || jvmti == nullptr || jni == nullptr || + obj == nullptr) { + TEST_LOG_SUMMARY("ReferenceChainTracker::tagAsRootForTest refused: " + "frontier=%p jvmti=%p jni=%p obj=%p", + (void *)_frontier, (void *)jvmti, (void *)jni, (void *)obj); + return 0; + } + // Resolves the klass_id via the same GetClassSignature + normalizeClassSignature + + // Profiler::lookupClass sequence every consumer in this subsystem uses + // (ObjectSampler::recordAllocation(), LivenessTracker::resolveKlassId(), resolveClassMap() above) + // - the id space is load-bearing here: pollWatchedTargets() matches frontier entries against leak + // candidates by klass_id, and the candidate ids come from that signature-notation space + // (Class.getName()'s dot form is a DIFFERENT StringDictionary key - see + // find-klass-id-notation-mismatch). + u32 klass_id = 0; + jclass klass = jni->GetObjectClass(obj); + char *class_name = nullptr; + if (jvmti->GetClassSignature(klass, &class_name, nullptr) == + JVMTI_ERROR_NONE && + class_name != nullptr) { + const char *name_slice = nullptr; + size_t name_len = 0; + if (ObjectSampler::normalizeClassSignature(class_name, &name_slice, + &name_len)) { + int id = Profiler::instance()->lookupClass(name_slice, name_len); + if (id != -1) { + klass_id = (u32)id; + } + } + jvmti->Deallocate((unsigned char *)class_name); + } + jni->DeleteLocalRef(klass); + + // Tags `obj` and inserts it as a frontier root (parent_tag=0, depth=0), exactly the convention + // runPass()'s heap-root callback path already uses (referenceChains.cpp's + // heapReferenceCallback(), referrer_tag_ptr == nullptr branch) - this lets a test drive the real + // BFS/chain- reconstruction logic (runPass()/pollWatchedTargets()/buildChainEvent()) against a + // known, caller-chosen live object, decoupled from whether the real root-seeded walk or + // LivenessTracker's probabilistic sampler happens to reach/select it on its own. + jlong tag = tagObject(jvmti, obj); + if (tag == 0) { + TEST_LOG_SUMMARY("ReferenceChainTracker::tagAsRootForTest refused: " + "tagObject (SetTag) failed"); + return 0; + } + if (!_frontier->insert(tag, 0, klass_id, 0)) { + TEST_LOG_SUMMARY("ReferenceChainTracker::tagAsRootForTest refused: " + "frontier insert failed tag=%lld klass_id=%u", + (long long)tag, klass_id); + clearTag(jvmti, obj); + return 0; + } + // Discovery recording for this seam's decoupling contract. + if (_candidate_count > 0) { + recordDiscoveredInstance(klass_id, tag, false); + } + return tag; +} diff --git a/ddprof-lib/src/main/cpp/referenceChains.h b/ddprof-lib/src/main/cpp/referenceChains.h new file mode 100644 index 000000000..fb483dd81 --- /dev/null +++ b/ddprof-lib/src/main/cpp/referenceChains.h @@ -0,0 +1,1299 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _REFERENCECHAINS_H +#define _REFERENCECHAINS_H + +#include "arch.h" +#include "arguments.h" +#include "classTagAllocator.h" +#include "common.h" +#include "event.h" +#include "painBudget.h" +#include "pidController.h" +#include "spinLock.h" +#include "mutex.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "referenceChainFrontier.h" + +// Incremental reference-chain search driven by a dedicated JVMTI agent thread. +class ReferenceChainTracker { + // Test-only accessor (referenceChains_ut.cpp), mirroring vmEntry.h's VMTestAccessor pattern: + // since instance() is a process-wide singleton, the search-lifecycle fields + // (_search_state/_search_started/etc.) would otherwise leak across separate TEST_F cases in the + // same gtest binary. + friend class ReferenceChainsTestAccessor; + +private: + bool _enabled; + + // Frontier metadata table. Constructed lazily on the first start() with the flag enabled, sized + // from args._reference_chains_frontier_cap; like LivenessTracker's table (LivenessTracker's own + // table does the same) it survives stop() so it persists across multiple start/stop recording + // cycles. + FrontierTable *_frontier; + + // args._reference_chains_frontier_cap as of the most recent start() call - recorded + // unconditionally (even once _frontier already exists and start() itself skips reconstructing + // it), so resetSearchStateForTest() has something to rebuild the table at other than whatever cap + // the first start() in this JVM happened to use (see _frontier's own comment). + int _configured_frontier_cap; + + // The args-configured per-pass budget (Arguments::_reference_chains_budget), recorded at start(). + int _configured_budget; + + // Class-tag -> StringDictionary id table. Populated by resolveLoadedClasses(), read by + // heapReferenceCallback(). + ClassTagTable _class_tags; + + // Profiler::classMap()'s generation as of the last resolveLoadedClasses() call. + u64 _last_class_map_generation; + + // GetLoadedClasses() count as of the last resolveLoadedClasses() call that actually ran its + // per-class GetTag()/GetClassSignature() scan - lets that method skip the scan entirely on a + // resumed pass where the loaded-class count has not CHANGED (see resolveLoadedClasses()'s own + // comment for why this must be an equality check, not just a "grew" check: the count is not + // monotonic once class unloading is in play). + int _last_resolved_class_count; + + // GetLoadedClasses() count as of the last runPassManualWalk() call whose admitStaticFieldRoots() + // sweep actually ran (i.e. was not skipped by the guard below) AND completed without being + // truncated. + int _last_static_field_class_count; + + // Index into the (per-call, app-classes-first-partitioned) loaded-class list that + // admitStaticFieldRoots() resumes from on its next call - see that method's own comment for why a + // single FollowReferences over every loaded class at once (no cursor) could never finish within + // one pass's safepoint deadline on a JVM with tens of thousands of loaded classes. + int _static_field_sweep_cursor; + + // Set when any chunk within the current lap (the cursor's walk from 0 back to 0) truncates. + bool _static_field_sweep_cycle_truncated; + + // Per-call cap on how many classes admitStaticFieldRoots() includes in one FollowReferences call + // - see that method's own comment. + static constexpr int STATIC_FIELD_SWEEP_CHUNK_CLASSES = 512; + + // Per-class cap on non-STATIC_FIELD edges admitted during one admitStaticFieldRoots() lap. + static constexpr int STATIC_FIELD_SWEEP_NON_STATIC_CAP_PER_CLASS = 32; + + // GC epoch counters, bumped from the JVMTI GC callbacks and read by shouldRunPass(). + volatile u64 _gc_start_epoch; + volatile u64 _gc_finish_epoch; + + // Monotonically increasing tag source for frontier objects. + volatile jlong _next_tag; + + // Per-pass tunables, copied from Arguments in start(). + int _hop_cap; + int _budget; + + // Larger one-shot edge budget for the search's root-seeded first pass. + int _first_pass_budget; + + // Wall-clock TTL for one search; <= 0 disables the cutoff. + long _ttl_ms; + + // Pause-time pacing controller: pause-time-SLO ceiling copied from Arguments in start() + // (Arguments::_reference_chains_pause_target_ms) - the single pause-time-SLO target, in place of + // guessing _budget/PASS_CADENCE_NS directly. + long _pause_target_ms; + // Runtime-adjusted pause target: when isUrgent(), this is bumped to URGENT_PAUSE_TARGET_MS so + // each pass can explore more edges. + long _effective_pause_target_ms; + // Passes since the frontier last grew. Reset to 0 whenever _frontier->size() increases (new + // entries admitted). + int _passes_since_last_progress; + // Passes since _candidate_found_bits last changed (a candidate was newly found, or a new + // candidate was admitted into a slot). + int _passes_since_last_candidate_progress; + // candidate_count + popcount(found_bits) as of the last pass this was updated. + int _last_candidate_progress_mark; + + // Canary-lane pass pacing, work-scaled: the chase's inter-pass spacing is _canary_backoff_mult x + // _canary_pass_ema_ms - a multiple of what a pass actually COSTS, not a fixed wall-clock + // constant. + int _canary_backoff_mult; + // 0.8/0.2 EMA of each pass's whole-call wall duration in ms + // (TSC::ticks_to_millis(pass_wall_ticks), runPass()), updated every pass - kept warm regardless + // of canary state so a chase that opens on a known-cost crawl sizes correctly from its first + // held-off decision. + u64 _canary_pass_ema_ms; + // End-of-pass timestamp of the last pass that ran with a canary chase still open (the reference + // point for the spacing above). + u64 _last_canary_pass_ns; + // Whether threadLoop()'s OOM urgency ramp is currently active, set each loop iteration BEFORE + // shouldRunPass() (same thread, no atomics needed). + bool _oom_ramp_active; + + // How many consecutive times in a row runPass()'s canary-stuck check + // (CANARY_NO_PROGRESS_PASS_LIMIT below) has abandoned this same candidate-chase sequence. + int _canary_stuck_restart_count; + + // Canary-search candidate set: LivenessTracker-flagged leak klasses this + // tracker chases, one slot per klass. The retired marker-tag pre-tagging + // (each candidate's representative pre-tagged MARKER_TAG_BASE - i for an + // identity match in the walk) was replaced by leak-tag interception - + // LivenessTracker tags specific tracked instances, and discovery records + // _candidate_found_bits/_candidate_frontier_tags when one is resolved. + static constexpr int MAX_LEAK_CANDIDATES_FROM_LT = 5; + + // How many klass_ids _watched_leak_klass_ids tracks at once - matches + // LivenessTracker::MAX_LEAK_CANDIDATES (livenessTracker.h), the cap + // LivenessTracker::topKlassesByGenerationCount() itself already enforces; duplicated here for the + // same reason MAX_LEAK_CANDIDATES_FROM_LT above already duplicates it, rather than depending on a + // private LivenessTracker constant. + static constexpr int MAX_WATCHED_LEAK_KLASSES = 5; + int _candidate_count; + u64 _candidate_found_bits; + // klass_id occupying each slot, so pollWatchedTargets() can tell whether a klass_id + // selectLeakCandidates() returns this poll already has a slot (and must not be + // re-tagged/re-admitted) or is new (and should be admitted into the next free slot). + u32 _candidate_klass_ids[MAX_LEAK_CANDIDATES_FROM_LT]; + jlong _candidate_frontier_tags[MAX_LEAK_CANDIDATES_FROM_LT]; + // Per-candidate chain link recorded at pruning time: parent_tag (referrer's frontier tag, + // positive) and referrer_klass. + jlong _candidate_parent_tags[MAX_LEAK_CANDIDATES_FROM_LT]; + u32 _candidate_referrer_klasses[MAX_LEAK_CANDIDATES_FROM_LT]; + u32 _candidate_depths[MAX_LEAK_CANDIDATES_FROM_LT]; + + // Per-SLOT snapshot of the qualifying allocating-thread tids + // LivenessTracker::selectLeakCandidates() reported for that slot's klass this poll, refreshed by + // pollWatchedTargets() (zeroed first, then filled from the poll's candidates so a klass that + // stops qualifying stops walking its tids too). + static constexpr int MAX_CANDIDATE_QUALIFYING_TIDS = 16; + jint _candidate_qualifying_tids[MAX_LEAK_CANDIDATES_FROM_LT] + [MAX_CANDIDATE_QUALIFYING_TIDS]; + int _candidate_qualifying_tid_count[MAX_LEAK_CANDIDATES_FROM_LT]; + + // tid -> JNI global reference to the live java.lang.Thread object, fed from + // Profiler::onThreadStart/onThreadEnd (registerThreadObject()/unregisterThreadObject()). + Mutex _thread_objects_lock; + std::unordered_map _thread_objects; + // Global refs of ended threads awaiting deletion. unregisterThreadObject() must NOT + // DeleteGlobalRef() directly: walkCandidateThreadLocals() copies the jobject out of + // _thread_objects under _thread_objects_lock, releases the lock, and can still be using it as a + // FollowReferences anchor when a concurrent ThreadEnd erases the entry - deleting there would be + // JNI use-after-free (once deleted, a global ref is invalid for every other JNI call). + std::vector _thread_refs_pending_delete; + + // Auto-marked instances: when the BFS walk discovers ANY object whose class matches a watched + // leak class (not just the pre-tagged representative), its frontier tag is recorded here so + // pollWatchedTargets() can build chain events for all of them. + static constexpr int MAX_DISCOVERED_INSTANCES_PER_CLASS = 8; + jlong _candidate_discovered_tags[MAX_LEAK_CANDIDATES_FROM_LT] + [MAX_DISCOVERED_INSTANCES_PER_CLASS]; + int _candidate_discovered_count[MAX_LEAK_CANDIDATES_FROM_LT]; + + // klass_ids from LivenessTracker::topKlassesByGenerationCount() (a faster, un-hysteresis-gated + // ranking than the canary candidate set above - see that method's own comment), refreshed once + // per BFS-thread tick but only once hasLeakSignal() has already fired via the slower, + // hysteresis-gated selectLeakCandidates() path (per design discussion: this whole mechanism only + // cranks once the trend detector has already triggered, so it never needs to wait out that same + // hysteresis a second time on its own). + u32 _watched_leak_klass_ids[MAX_WATCHED_LEAK_KLASSES]; + int _watched_leak_klass_count = 0; + + // Packs a (leaf_klass_id, parent_class_id) pair into one map key for + // _leak_signature_totals/_leak_signature_prev_totals below - both are StringDictionary ids (u32), + // so this never loses information and avoids defining a custom hash/equality functor for a + // 2-field struct key. + static u64 leakSignatureKey(u32 leaf_klass_id, u32 parent_class_id) { + return ((u64)leaf_klass_id << 32) | (u64)parent_class_id; + } + + // Tier 1 of the leak-accumulation rotation design (see + // collectLeakAccumulationCandidatesForRotation()'s own comment for the full design): aggregate, + // per (leaf_klass_id, parent_class_id) signature - not per object - how many admitted children of + // that leaf klass_id have been observed under a parent of that class. + std::unordered_map _leak_signature_totals; + + // Snapshot of _leak_signature_totals as of the END of the previous pass - runPassManualWalk() + // computes each signature's delta (totals - this) to rank signatures by growth before rolling + // this forward to the current totals for the next pass's comparison. + std::unordered_map _leak_signature_prev_totals; + + // Tier 2 of the leak-accumulation rotation design: per PARENT TAG (not per class), how many + // admitted children of a watched leaf klass_id this specific parent object holds, plus which + // signature it belongs to (so rotation-selection can filter to the pass's winning signature + // without a second lookup). + struct LeakParentFanoutEntry { + u64 signature_key; + u32 fanout; + }; + std::unordered_map _leak_parent_fanout; + + // Rotating skip-count over _leak_parent_fanout's iteration order for + // collectStaleExpandedEntriesForRotation()'s leak-parent-priority tier - advances by the number + // of parents selected each pass so, across passes, every fanout parent gets re-walked within + // ceil(fanout_size/budget) passes instead of only whichever entries the hash iteration happens to + // yield first. + u64 _leak_parent_rotation_cursor = 0; + + // Pause-time pacing controller: the actual per-pass budget runPass() passes to + // FollowReferences/expandFrontier(), replacing _budget's old role as a literal per-pass value - + // _budget above becomes this controller's ceiling instead (never exceeded, see updatePacing()), + // while this field is what updatePacing() actually raises/lowers pass to pass. + int _effective_budget; + + // Pause-time pacing controller: the actual fallback cadence shouldRunPass()/threadLoop() compare + // against, replacing the fixed PASS_CADENCE_NS constant below in that role once updatePacing() + // starts adjusting it - see PASS_CADENCE_NS's own comment for why that constant survives as this + // field's starting value rather than being deleted outright. + u64 _effective_cadence_ns; + + // Budget-borrowing: extra headroom updatePacing() has temporarily granted above _budget's own + // ceiling, earned by a sustained run of comfortably- under-target passes (see + // BORROW_WARMUP_PASSES's own comment). + int64_t _borrowed_budget; + + // Budget-borrowing: number of consecutive passes (since the last reset) that came in comfortably + // under _pause_target_ms (see BORROW_UNDER_TARGET_FRACTION). + int _consecutive_under_target_passes; + + // Pause-time pacing controller: this tracker's own PidController instance - see updatePacing() + // below for the full mechanism, and PASS_CADENCE_NS's neighboring constants for why its gains are + // not copied from ObjectSampler/ MallocTracer/NativeSocketSampler's shared triple. + PidController _pause_pid; + + // Self-calibrating adaptive batch sizing for GetObjectsWithTags (see expandFrontier()'s own + // comment). + size_t _gotw_batch_size = 0; // 0 = unset, use GOTW_INITIAL_BATCH_SIZE + u64 _gotw_ema_call_ns = 0; // EMA of per-call elapsed, 0 = unset + + // Nominal per-call window for the proportional batch control above when no phase deadline is set + // (expandFrontier()'s window is the REMAINING deadline, which the phases refresh per invocation). + static constexpr u64 GOTW_CPU_BUDGET_NS = 25000000; // 25ms + + // Effective window for the proportional batch control (expandFrontier() calls this with the + // remaining pass deadline and the depth of the lane the next GetObjectsWithTags call will drain). + u64 gotwWindowNs(u64 remaining_ns, size_t lane_depth) const { + u64 window_ns = remaining_ns != 0 ? remaining_ns : GOTW_CPU_BUDGET_NS; + if (lane_depth >= GOTW_BACKLOG_MIN_DEPTH && + _gotw_ema_call_ns > window_ns) { + window_ns = std::max(window_ns, + _gotw_ema_call_ns * GOTW_BACKLOG_WINDOW_MULT); + } + return window_ns; + } + + // Lane depth beyond which gotwWindowNs()'s backlog widening applies. + static constexpr size_t GOTW_BACKLOG_MIN_DEPTH = 4096; + + // How many measured per-call floors one widened window may cost - see gotwWindowNs() above. + static constexpr u64 GOTW_BACKLOG_WINDOW_MULT = 3; + + // Conservative initial batch_size before the first GetObjectsWithTags measurement. + static constexpr int GOTW_INITIAL_BATCH_SIZE = 64; + // AIMD bounds for the adaptive batch: MAX bounds JNI local refs per call, MIN keeps each call + // from resolving a single object. + static constexpr size_t GOTW_MAX_BATCH = 512; + static constexpr size_t GOTW_MIN_BATCH = 8; + + // Search lifecycle state. _search_started distinguishes a search's first pass (seed + // FollowReferences from the heap roots) from a resumed pass (expand the persisted frontier, see + // expandFrontier()) - runPass() below. + bool _search_started; + volatile u8 _search_state; + + // True once releaseSearchTags() has confirmed every live tag this search owned was actually + // cleared (or there were none) - see that method's own comment for why a GetObjectsWithTags() + // failure must NOT be treated as "released". + bool _tags_released; + + // Whether threadLoop()'s urgency ramp currently holds the multiplied _budget (see the urgency + // block there and _configured_budget's own comment). + bool _urgency_budget_boosted; + + // Hysteresis state behind isUrgent(), which used to be a bare `secondsToOOM() < + // OOM_URGENT_THRESHOLD_S` comparison. + mutable bool _urgent_latched; + mutable int _urgent_release_ticks; + mutable bool _urgent_search_spent; + + // Set (once) at the same point runPass() moves _search_state to ABANDONED - see + // SearchAbandonReason's own comment for why this exists and buildAbandonedEvent()/abandonReason() + // below for how it is read. + volatile u8 _abandon_reason; + + // Wall-clock timestamp (OS::nanotime()) of the search's first pass - baseline for the TTL cutoff + // above. + volatile u64 _search_start_ns; + + // Tags currently in FrontierEntryState::FRONTIER (admitted but not yet expanded), in admission + // order. + std::deque _pending_expand; + + // Fast-lane counterpart to _pending_expand above: entries admitted while re-walking a + // rotation-selected (already-EXPANDED) parent go here instead, and expandFrontier() drains this + // queue ahead of the ordinary one. + std::deque _priority_expand; + + // Which lane the NEXT expandFrontier() batch comes from when both lanes are non-empty (the + // alternation toggle). + bool _expand_lane_prefer_priority = true; + + // Upper bound on _priority_expand above. 1024 holds a few passes' worth of rotation selection + // (budgets sum to ~272/pass) so a truncating rotation phase still has work waiting next pass. + static constexpr size_t PRIORITY_EXPAND_CAP = 1024; + + // O(1) membership index over _priority_expand, backing isQueuedForRotation(): the rotation + // collectors run that check for EVERY FrontierTable slot they visit (~199k EXPANDED entries on a + // large heap), and the original linear scan over the deque cost up to ~200M comparisons per + // rotation pass at the cap - observed prominently in profiles. + class PriorityExpandSet { + private: + // 2^11 == 2 * PRIORITY_EXPAND_CAP == 2048 slots. The shift below derives from it; keep both in + // sync. + static constexpr u64 SLOT_SHIFT = 11; + static constexpr u64 SLOT_MASK = (1ULL << SLOT_SHIFT) - 1; + jlong _keys[1ULL << SLOT_SHIFT]; + u8 _used[1ULL << SLOT_SHIFT]; // 0 = empty, 1 = occupied + + static u64 mix(jlong tag) { + // Fibonacci hashing: spreads near-sequential integer tags evenly across the table's + // power-of-two slot space. + return (u64)tag * 0x9E3779B97F4A7C15ULL; + } + + public: + bool contains(jlong tag) const { + u64 i = mix(tag) >> (64 - SLOT_SHIFT); + while (_used[i]) { + if (_keys[i] == tag) { + return true; + } + i = (i + 1) & SLOT_MASK; + } + return false; + } + + // Idempotent: returns false if `tag` is already indexed. + bool insert(jlong tag) { + u64 i = mix(tag) >> (64 - SLOT_SHIFT); + while (_used[i]) { + if (_keys[i] == tag) { + return false; + } + i = (i + 1) & SLOT_MASK; + } + _used[i] = 1; + _keys[i] = tag; + return true; + } + + void clear() { + memset(_used, 0, sizeof(_used)); + } + + // Re-derives the index from the deque's CURRENT contents. + template void rebuildFrom(const Deque &queue) { + clear(); + for (jlong tag : queue) { + insert(tag); + } + } + } _priority_expand_set; + + // B' (find-anchor-holder-eviction / find-anchor-live-feed-design): a static holder richly + // referenced from the running graph is EXCLUDED from the static-anchor tier forever once its + // frontier entry is chain-attached (first admitted via a non-root path, or demoted by + // improveChain) - maybeUpgradeRootAttachedRootKind() refuses entries with parent_tag != 0 by + // design, so collectStaticFieldAnchorsForRotation() (parent_tag == 0 filter) can never select it. + struct AtRiskAnchor { + jlong tag; + u32 klass_id; + // PriorityExpandSet::rebuildFrom() iterates `jlong tag : queue` - the implicit conversion keeps + // that template generic over both the plain-jlong _priority_expand deque and this pair deque. + operator jlong() const { return tag; } + }; + std::deque _static_anchor_fifo; + + // Membership index over _static_anchor_fifo (push-side dedupe, so one lap's repeated static edges + // onto the same chain-attached holder push it once) - a second instance of PriorityExpandSet, + // whose fixed 2048 slot table keeps the cap at PRIORITY_EXPAND_CAP (1024). + PriorityExpandSet _static_anchor_fifo_set; + static constexpr size_t STATIC_ANCHOR_FIFO_CAP = PRIORITY_EXPAND_CAP; + + // Per-class admission counts backing the per-klass cap on at-risk anchor pushes. + std::unordered_map _static_anchor_fifo_klass_counts; + static constexpr u32 STATIC_ANCHOR_ATRISK_PER_KLASS_CAP = 64; + + // Index of root-attached STATIC_FIELD/JNI_GLOBAL frontier entries, so + // collectStaticFieldAnchorsForRotation() iterates O(anchors) instead of scanning the full + // frontier table O(frontier_size). + std::vector _static_anchor_index; + + // Parallel to _static_anchor_index: the OWN class tag of each anchor object (the class of the + // static field's VALUE, not the holder class). + std::vector _static_anchor_own_class_tags; + + // Dedup companion for _static_anchor_index (O(1) membership; the population is ~28k on a real JVM + // - see addToStaticAnchorIndex()'s own comment). + std::unordered_set _static_anchor_index_tags; + + // Shape of a class as an anchor candidate: does the class implement java/util/Collection or + // java/util/Map (directly or via superclasses/ interfaces)? + enum class AnchorClassShape : u8 { UNKNOWN = 0, CONTAINER = 1, NON_CONTAINER = 2 }; + + // class tag -> AnchorClassShape, process-lifetime (class tags are never reused - the shared + // class-tag allocator is deliberately not reset by restartSearch()). + std::unordered_map _class_shape_cache; + + // java/util/Collection and java/util/Map class tags, resolved once lazily by + // resolveContainerInterfaceTags() (0 = not yet resolved; a resolved value is NEGATIVE - class + // tags are a negative namespace, see nextClassTag()'s own comment). + jlong _collection_iface_class_tag = 0; + jlong _map_iface_class_tag = 0; + + // Fair-rotation cursors (index POSITIONS, not tags) for the two cursor-fair tiers of + // collectStaticFieldAnchorsForRotation(): _anchor_container_cursor for container-shaped anchors, + // _anchor_other_cursor for everything else. + size_t _anchor_container_cursor = 0; + size_t _anchor_other_cursor = 0; + + // FIFO of newly admitted anchors awaiting their first walk. + std::deque _static_anchor_fresh_queue; + + // Cap for _static_anchor_fresh_queue. The queue normally holds only ~one pass of sweep admits + // (admits happen in passes, the collector drains every pass, and the drain DROPS everything it + // does not keep, so the queue empties each call); the cap only guards a pathological burst (e.g. + // a pass admitting thousands) from growing it unbounded in native memory. + static constexpr size_t STATIC_ANCHOR_FRESH_CAP = 1024; + + // java/lang/Object jclass cache for expandFrontier()'s and admitStaticFieldRoots()'s holder-array + // element type (referenceChains.cpp) - resolved once via FindClass()+NewGlobalRef() and reused + // for the tracker's lifetime. + jclass _cached_object_class = nullptr; + + // Rotation cursor for collectStaleRootKindEntriesForRotation(): 1-based tag to resume scanning + // from on the next call, so consecutive calls sweep forward through the table instead of always + // re-examining the same low-tag entries first. + jlong _root_kind_rotation_cursor; + + // Same role as _root_kind_rotation_cursor above, but for + // collectStaleExpandedEntriesForRotation(): without its own persistent cursor, that sweep always + // restarted from tag 1 on every call, so a frontier table holding >= + // STALE_EXPANDED_ROTATION_BUDGET low-tag entries that stay EXPANDED forever (long-lived + // infrastructure objects) filled its entire per-pass cap from that population alone, every pass, + // permanently starving any EXPANDED entry with a higher tag (e.g. a static field's collection, + // admitted only once its class loads well after startup) of ever being re-queued. + jlong _stale_expanded_rotation_cursor; + + // Per-pass cap on how many transient-root_kind entries collectStaleRootKindEntriesForRotation() + // selects - round, provisional like this subsystem's other unbenchmarked constants (see e.g. + // MIN_EFFECTIVE_BUDGET's own comment): small enough that a pass dominated by rotation work never + // meaningfully competes with genuinely new discoveries for the same pass's budget, large enough + // that a search with a modest number of transient roots converges to durable attribution within a + // handful of passes rather than needing hundreds. + static constexpr int ROOT_KIND_ROTATION_BUDGET = 16; + + // Per-pass cap on how many EXPANDED entries collectStaleExpandedEntriesForRotation() re-queues + // for expansion, uniformly across the WHOLE frontier table regardless of lineage. + static constexpr int STALE_EXPANDED_ROTATION_BUDGET = 256; + + // Candidate-scoped reach (see descendFromAnchor()/walkCandidateThreadLocals()/ + // walkStaticFieldAnchors()'s own comments): how many hops BELOW a descend walk's anchor the walk + // may admit. + static constexpr int DESCENT_HOPS = 16; + + // Per-pass cap on how many candidate threads walkCandidateThreadLocals() descend-walks. + static constexpr int THREAD_WALK_MAX_ANCHORS = 4; + + // Per-pass cap on how many root-attached static holders walkStaticFieldAnchors() resolve + + // descend-walk. + static constexpr int STATIC_ANCHOR_ROTATION_BUDGET = 32; + + // Per-pass cap on how many DISTINCT anchor classes reconcileAnchorClassShapes() classifies (one + // GetObjectsWithTags call for the batch + a depth-bounded interface walk per class). + static constexpr int ANCHOR_SHAPE_RECONCILE_BUDGET = 128; + + // Per-pass cap on how many AT-RISK static holders (frontier entries with parent_tag != 0 - the + // find-anchor-holder-eviction population) drainStaticAnchorFifo() pops for the same + // walkStaticFieldAnchors() batch. + static constexpr int STATIC_ANCHOR_FIFO_DRAIN = 16; + + // (Removed: the old single wrapping-index-cursor selection was replaced by tiered selection with + // per-tier cursors — see collectStaticFieldAnchorsForRotation()'s own comment and + // _anchor_container_cursor/_anchor_other_cursor.) + + // Cursor over the flattened (slot, tid) enumeration of _candidate_qualifying_tids above, so + // walkCandidateThreadLocals()'s THREAD_WALK_MAX_ANCHORS-per-pass cap rotates fairly instead of + // always walking the same first candidates' tids. + int _thread_walk_anchor_cursor; + + // Per-pass cap on how many EXPANDED entries collectLeakAccumulationCandidatesForRotation() + // re-queues - see that method's own comment. + static constexpr int LEAK_ACCUMULATION_ROTATION_BUDGET = 16; + + // Snapshot of gcFinishEpoch() as of the end of the last pass. + u64 _last_pass_gc_finish_epoch; + + // OS::nanotime() as of the end of the last pass. Written by runPass() on the BFS thread; also + // read cross-thread by buildAbandonedEvent()'s elapsed-time calculation, so + // volatile/load()-accessed like _search_state above. + volatile u64 _last_pass_ns; + + // Total passes run this search. Written by runPass() on the BFS thread; read cross-thread by + // passesRun()/buildAbandonedEvent(), so volatile/ load()-accessed like _search_state above. + volatile int _passes_run; + + // Resolved reference chains, keyed by the leak-candidate klass_id pollWatchedTargets() + // reconstructed each one for. + struct CachedChain { + ReferenceChainEvent event; + jlong source_tag; + u64 source_search_ns; + }; + // Bounded by the number of discovered instances across all candidate slots: + // MAX_LEAK_CANDIDATES_FROM_LT * MAX_DISCOVERED_INSTANCES_PER_CLASS = 5 * 8 = 40, plus up to 5 + // canary chains. + static constexpr int MAX_RESOLVED_CHAINS = 128; + // Keyed by frontier tag (individual instance identity), NOT by klass_id. + std::unordered_map _resolved_chains; + SpinLock _resolved_chains_lock; + + // Abandoned-search events awaiting Profiler::dump() (profiler.cpp). + static constexpr int MAX_PENDING_ABANDONED_EVENTS = 16; + std::vector _pending_abandoned_events; + SpinLock _pending_abandoned_events_lock; + + // leaky bucket over the wall-clock cost of past searches, gating how soon a *restarted* search + // may take its first pass - see PainBudget's own comment (painBudget.h) and canAffordNewSearch() + // below. + PainBudget _safepoint_pain_budget; + // Cached refill rate from start(), reused by resetSearchStateForTest() so a test reset rebuilds + // the budget with the same rate. + double _pain_budget_refill_rate = 0.0; + u64 _search_pain_ms; + + // Non-safepoint CPU-time pain budget: gates shouldRunPass() independently of both + // _safepoint_pain_budget above (which only cools down *restarts*, spent once per finished search) + // and _pause_pid's per-pass signal (updatePacing(), now fed only the genuine in-safepoint portion + // of each pass - see runPass()'s own comment). + PainBudget _cpu_pain_budget; + + // The cache above is mutated on this tracker's own BFS scheduling thread (pollWatchedTargets()) + // and read on whatever thread calls Profiler::dump() (drainPendingChainEvents()); + // _resolved_chains_lock (declared with the cache) is the only synchronization between them. + + // Fallback cadence between passes. + static constexpr u64 PASS_CADENCE_NS = 1000000000ULL; // 1s + + // Heap-wide time-to-OOM urgency threshold (LivenessTracker::secondsToOOM()) - hasLeakSignal() + // below forces a search to start immediately once the projection drops under this, rather than + // waiting for a klass to clear selectLeakCandidates()'s own per-klass ring-fill/hysteresis gate + // (KLASS_POPULATION_MIN_FILL_FOR_TREND plus LEAK_TREND_HYSTERESIS_BASE/ CORROBORATED epochs, + // livenessTracker.h). + static constexpr double OOM_URGENT_THRESHOLD_S = 300.0; // 5 minutes + + // Release side of OOM_URGENT_THRESHOLD_S's hysteresis (see _urgent_latched). + static constexpr double OOM_URGENT_RELEASE_S = 2 * OOM_URGENT_THRESHOLD_S; + static constexpr int URGENT_RELEASE_CONSECUTIVE = 5; + + // Horizon over which threadLoop() ramps the pause target and cadence toward their urgent ceilings + // as secondsToOOM() falls, once it reports a confirmed rising trend (see secondsToOOM()'s own + // NOT_RISING gate — a non-negative value already means real growth, not noise). + static constexpr double OOM_RAMP_START_S = 1800.0; // 30 minutes + + // Ceilings the pause target and cadence ramp toward as secondsToOOM() approaches zero within + // OOM_RAMP_START_S: the ramp is exponential (slow near OOM_RAMP_START_S out, aggressive near OOM) + // since the process is likely to die anyway and diagnostic data collected right before that is + // worth spending STW time and CPU on. + static constexpr long URGENT_PAUSE_TARGET_MS = 100; // ceiling STW ms per pass + static constexpr u64 URGENT_CADENCE_NS = 10000000ULL; // 10ms floor between passes + + // True when LivenessTracker::secondsToOOM() projects exhaustion sooner + + // Auto-scaled default for _first_pass_budget when Arguments::_reference_chains_first_pass_budget + // is unset (0) - see _first_pass_budget's own comment for why plain _budget is the wrong + // fallback. + static constexpr int AUTO_FIRST_PASS_BUDGET_MULTIPLIER = 50; + static constexpr int AUTO_FIRST_PASS_BUDGET_CAP = 200000; + + // Minimum wall-clock gap between root/stack-ref enumeration attempts (runPassManualWalk()'s + // IterateOverReachableObjects call) after the search's own first pass. + static constexpr u64 ROOT_ENUM_MIN_INTERVAL_NS = 2000000000ULL; // 2s + + // Pause-time pacing controller: bounds and conversion constants for updatePacing()'s + // budget/cadence adjustment - see that method's own comment for the full mechanism. + static constexpr int MIN_EFFECTIVE_BUDGET = 2000; + + // Bounds for _effective_cadence_ns. The lower bound is not 0: threadLoop() sleeps for exactly + // this many nanoseconds each loop iteration (below), so a true 0 would busy-loop the BFS thread. + static constexpr u64 MIN_EFFECTIVE_CADENCE_NS = 10000000ULL; // 10ms + static constexpr u64 MAX_EFFECTIVE_CADENCE_NS = PASS_CADENCE_NS * 4; // 4s + + // Conversion factor from "edges of budget signal updatePacing()'s clamp could not absorb" to a + // cadence adjustment in nanoseconds - the two are different units (edge count vs. + static constexpr u64 CADENCE_NS_PER_EDGE_OVERFLOW = 1000000ULL; // 1ms/edge + + // Budget-borrowing (see _borrowed_budget's own comment): how many consecutive + // comfortably-under-target passes (BORROW_UNDER_TARGET_FRACTION) must be observed before + // updatePacing() starts growing _borrowed_budget at all. + static constexpr int BORROW_WARMUP_PASSES = 5; + + // Budget-borrowing: a pass counts toward BORROW_WARMUP_PASSES/keeps _borrowed_budget only when + // pass_ms is at most this fraction of _pause_target_ms - deliberately stricter than merely "under + // the ceiling" (which the ordinary _effective_budget clamp already guarantees), so growth is + // gated on *comfortable* headroom, not on shaving the pass in just under the wire. + static constexpr double BORROW_UNDER_TARGET_FRACTION = 0.5; + + // Budget-borrowing: hard cap on how far updatePacing() may grow (_budget + _borrowed_budget) + // above _budget alone - _borrowed_budget itself is clamped so the resulting ceiling never exceeds + // _budget * BORROW_CEILING_MULTIPLIER. + static constexpr int BORROW_CEILING_MULTIPLIER = 4; + + // Budget-borrowing: fraction of _budget by which _borrowed_budget grows on each pass once + // BORROW_WARMUP_PASSES has been reached - a fraction of the configured budget rather than of the + // current borrowed amount, so growth stays linear (predictable, boundable within a known number + // of passes) rather than compounding. + static constexpr double BORROW_GROWTH_FRACTION = 0.25; + + // Agent-owned BFS thread; startThread()/stopThread() own its lifecycle. + pthread_t _thread; + // std::atomic rather than plain volatile bool - volatile alone gives no C++ memory-model + // acquire/release guarantees (it only prevents the compiler from eliding/reordering that one + // variable's own accesses), so a weakly-ordered CPU (e.g. arm64) could let the BFS thread's + // stopThread()- side write (see stopThread()'s own comment) become visible to threadLoop() later + // than intended, missing the shutdown request on one wakeup and sleeping/looping an extra cycle + // before pthread_join() unblocks it. + std::atomic _running; + + // Cooperative-cancellation flag for an in-flight JVMTI FollowReferences walk: stopThread() sets + // this before pthread_kill()/pthread_join() (that signal alone cannot interrupt a call already + // inside the JVM/JVMTI implementation), and heapReferenceCallback() checks it on every + // invocation, aborting the walk within one callback rather than letting pthread_join() block + // until the walk finishes on its own - see both methods' own comments. + std::atomic _abort_pass_requested; + + // Wall-clock deadline for the pass currently in flight (OS::nanotime() ticks; 0 = no deadline). + u64 _pass_deadline_ns = 0; + + // Last time root/stack-ref enumeration actually ran (OS::nanotime() ticks; 0 before the search's + // first pass). + u64 _last_root_enum_ns = 0; + + // Set true when the most recent root/stack-ref enumeration attempt ended via BUDGET_EXHAUSTED + // (not FRONTIER_CAP_HIT, which stops admitting new frontier entries but leaves the search RUNNING + // - see runPass()'s frontier_cap_hit handling) - runPass() treats this as grounds to retry root + // enumeration on the very next pass regardless of ROOT_ENUM_MIN_INTERVAL_NS, so a + // still-incomplete attempt is not left waiting out the full interval before continuing. + bool _root_enum_truncated_last_time = false; + + ReferenceChainTracker() + : _enabled(false), + _frontier(nullptr), _configured_frontier_cap(0), + _last_class_map_generation(0), + _last_resolved_class_count(0), + _last_static_field_class_count(-1), + _static_field_sweep_cursor(0), + _static_field_sweep_cycle_truncated(false), + _gc_start_epoch(0), + _gc_finish_epoch(0), _next_tag(1), + _hop_cap(0), _budget(0), _first_pass_budget(0), _ttl_ms(0), _pause_target_ms(0), + _effective_pause_target_ms(0), _passes_since_last_progress(0), + _passes_since_last_candidate_progress(0), _last_candidate_progress_mark(0), + _canary_backoff_mult(1), _canary_pass_ema_ms(0), + _last_canary_pass_ns(0), _oom_ramp_active(false), + _canary_stuck_restart_count(0), + _effective_budget(0), _effective_cadence_ns(PASS_CADENCE_NS), + _pause_pid(1, 1.0, 1.0, 1.0, 1, 1.0), _search_started(false), + _tags_released(true), _urgent_latched(false), + _urgent_release_ticks(0), _urgent_search_spent(false), + _urgency_budget_boosted(false), _configured_budget(0), + _search_state(SearchState::RUNNING), + _abandon_reason(SearchAbandonReason::NONE), _search_start_ns(0), + _last_pass_gc_finish_epoch(0), _last_pass_ns(0), + _passes_run(0), + _root_kind_rotation_cursor(1), + _stale_expanded_rotation_cursor(1), + _thread_walk_anchor_cursor(0), + _safepoint_pain_budget(0.0), _search_pain_ms(0), _cpu_pain_budget(0.0), + _thread(), _running(false), _abort_pass_requested(false) {} + + void onGCStart(); + void onGCFinish(); + + static void *threadEntry(void *self) { + ((ReferenceChainTracker *)self)->threadLoop(); + return nullptr; + } + void threadLoop(); + + // Runs a pass when the GC-finish epoch advanced or the cadence elapsed. + bool shouldRunPass(u64 now_ns); + + // Cheap probe (max=1, not the real poll pollWatchedTargets() makes) into LivenessTracker's + // population-trend table: true if at least one klass shows a positive population slope worth + // chasing. + bool hasLeakSignal(); + + // Latched, hysteretic view of LivenessTracker::secondsToOOM() crossing OOM_URGENT_THRESHOLD_S - + // see _urgent_latched for the latch/release rules and why the raw comparison flaps. + bool isUrgent() const; + + // Search restart gate (this class's own header comment): true once _safepoint_pain_budget has + // drained back to zero (canStartNow()) *and* hasLeakSignal() above reports at least one leak + // candidate. + bool canAffordNewSearch(u64 now_ns); + + // Resets every per-search field back to its just-constructed value so the next runPass() call + // takes the "first pass of a search" branch again, exactly like a fresh ReferenceChainTracker + // would. + void restartSearch(); + + // Marks every entry still queued in _pending_expand EXPANDED and drains the queue. + void markAllFrontierExpanded(); + + // Resolves pending frontier entries via GetObjectsWithTags (dead objects prune for free), then + // expands each with FollowReferences. + void expandFrontier(jvmtiEnv *jvmti, JNIEnv *jni, int hop_cap, int budget, + int *edges_admitted, bool *truncated, + bool *frontier_cap_hit, u64 *safepoint_ticks); + + // Static-field counterpart to heapRootCallback()'s GC-root enumeration: + // IterateOverReachableObjects's root/stack-ref callbacks never report a class's static fields + // (there is no jvmtiHeapRootKind for STATIC_FIELD - translateHeapRootKind()'s own comment), so + // without this call an object retained only via `SomeClass.staticField` is never discovered by + // either root enumeration or expandFrontier() (which only descends from already-admitted, + // non-class frontier entries - class objects are never admitted, see heapReferenceCallback()'s + // own comment). + void admitStaticFieldRoots(jvmtiEnv *jvmti, JNIEnv *jni, int hop_cap, + int budget, int *edges_admitted, + bool *truncated, bool *frontier_cap_hit, + bool *cycle_complete, u64 *safepoint_ticks); + + // Clears every live JVMTI tag this search still owns; frontier metadata is kept so + // reconstructChain() keeps working. + bool releaseSearchTags(jvmtiEnv *jvmti, JNIEnv *jni); + + // Feeds the measured pass duration into _pause_pid and rescales the effective budget and + // cadence; _budget remains the hard ceiling. + void updatePacing(u64 pass_wall_ticks); + + // Root/stack-ref enumeration passes never reach updatePacing() (runPass()'s own comment: their + // fixed dispatch cost would wrongly throttle _effective_budget for every unrelated later pass), + // but a slow one still spends real pause-time-SLO budget the borrow ceiling promised was safe to + // hand out. + void maybeRevokeBorrowForRootEnumPass(u64 pass_wall_ticks); + + // Tags every not-yet-tagged loaded class (GetLoadedClasses()) with a fresh nextClassTag() and + // resolves its name into _class_tags, via the same GetClassSignature + normalizeClassSignature + + // Profiler::lookupClass sequence ObjectSampler::recordAllocation() already uses + // (objectSampler.cpp:76-90) - reusing that normalization helper rather than re-deriving it. + void resolveLoadedClasses(jvmtiEnv *jvmti, JNIEnv *jni); + + // jvmtiHeapReferenceCallback for runPass()'s FollowReferences call (see runPass() below for the + // full walk). + static jint JNICALL heapReferenceCallback( + jvmtiHeapReferenceKind reference_kind, + const jvmtiHeapReferenceInfo *reference_info, jlong class_tag, + jlong referrer_class_tag, jlong size, jlong *tag_ptr, + jlong *referrer_tag_ptr, jint length, void *user_data); + + // Outcome of admitObject() below - lets each of its two call sites (heapReferenceCallback() above + // and the IterateOverReachableObjects root/ stack-ref callbacks, both in referenceChains.cpp) + // translate the same admission decision into its own callback-shape-appropriate return + // value/truncation flag, instead of duplicating the decision twice. + enum class AdmitResult { + ALREADY_ADMITTED, // *tag_ptr != 0: nothing to do, not a truncation + HOP_CAP, // depth >= hop_cap: not admitted, not a truncation + BUDGET_EXHAUSTED, // edges_admitted >= budget: this pass's cap + FRONTIER_CAP_HIT, // FrontierTable::insert() itself is full: stops + // admitting new entries but does not itself abandon the search (see + // runPass()'s frontier_cap_hit handling - the no-progress detector abandons + // only if the frontier then stops growing) + ADMITTED, + }; + + // First-discovery admission core: factored out of heapReferenceCallback()'s inline admission + // branch so the manual-walk driver's root/stack-ref callbacks stay in sync with FollowReferences' + // own admission by construction, not by copy-paste. + AdmitResult admitObject(FrontierTable *frontier, int hop_cap, int budget, + int *edges_admitted, jlong *tag_ptr, + jlong parent_tag, u32 referrer_klass, u32 depth, + u8 root_kind, jlong class_tag, + bool priority = false, + jint edge_field_index = -1, u8 edge_kind = 0, + jlong edge_referrer_class_tag = 0); + + // Called by admitObject() on every successful ADMITTED result (root or non-root, ordinary or + // priority) - the single shared admission path, so this needs no duplicate call site at + // heapReferenceCallback()/ heapRootCallback()/stackRefCallback(). + void trackLeakAccumulation(FrontierTable *frontier, jlong class_tag, + jlong parent_tag, jlong tag); + + // Record a discovered instance for a watched candidate class: store its frontier tag in the + // class's discovery slots so pollWatchedTargets() can build its chain event. + void buildDiscoveredInstanceChains(jvmtiEnv *jvmti, JNIEnv *jni, + u32 klass_id, u64 current_search_ns); + + void recordDiscoveredInstance(u32 klass_id, jlong frontier_tag, + bool leak_correlated); + + // Correlate a leak tag with an instance the BFS admitted BEFORE tagLeakInstances() tagged it (its + // JVMTI tag is a frontier tag, its frontier entry has leak_tag == 0). + + // One-time retroactive catch-up for a klass_id the moment it FIRST enters _watched_leak_klass_ids + // (pollWatchedTargets() calls this only for the newly-added ids in each refresh, never for ones + // already being watched). + void seedLeakAccumulationForNewlyWatchedKlass(u32 klass_id); + + // Upgrades root_kind when a rediscovered heap root is more durable; testable without a JVMTI mock. + bool maybeUpgradeRootAttachedRootKind(FrontierTable *frontier, jlong tag, + u8 new_root_kind); + + // True if `tag` is already sitting in _priority_expand - either queued earlier this same pass by + // the other rotation collector, or left over from a prior pass's truncated batch + // (expandFrontier() leaves those at the front of the queue for a later retry rather than popping + // them). + bool isQueuedForRotation(jlong tag) const { + return _priority_expand_set.contains(tag); + } + + // Re-queues transient-rooted expanded entries so a durable root can supersede a stale + // attribution. + std::vector collectStaleRootKindEntriesForRotation(int max_count); + + // Bounded rotating re-expansion for stale mutable fields: expandFrontier() observes an object's + // outgoing references exactly once (on the FollowReferences call that marks it EXPANDED) and + // never revisits it, so a field that is later reassigned to point at a different object - e.g. + // HashMap.table on resize - has its new value permanently unobserved once the map itself is + // EXPANDED; the old table array's own frontier entry eventually resolves to a dead object via + // GetObjectsWithTags and gets silently cleared with zero children, orphaning everything only + // reachable through the *current* table. + std::vector collectStaleExpandedEntriesForRotation(int max_count); + + // Bounded rotating re-expansion targeting the accumulation point of a klass LivenessTracker has + // flagged as growing (LivenessTracker:: topKlassesByGenerationCount(), _watched_leak_klass_ids) - + // the design's actual targeted tier. + std::vector collectLeakAccumulationCandidatesForRotation( + int max_count); + + // CANDIDATE-SCOPED REACH: bounded descend walk from an anchor object. + void descendFromAnchor(jvmtiEnv *jvmti, JNIEnv *jni, jobject anchor, + jlong anchor_tag, u32 anchor_depth, + jlong anchor_descend_class_tag, int budget, + int *edges_admitted, bool *truncated, + bool *frontier_cap_hit, u64 *safepoint_ticks); + + // Prong 1 of the candidate-scoped reach design (thread-retained taxonomy: ThreadLocal-held caches + // and thread-owned collections): per pass, walk up to THREAD_WALK_MAX_ANCHORS of the current + // candidates' qualifying tids' live Thread objects (registerThreadObject()'s map above) with + // descendFromAnchor() (anchor-gated to ThreadLocalMap, see above). + void walkCandidateThreadLocals(jvmtiEnv *jvmti, JNIEnv *jni, int budget, + int *edges_admitted, bool *truncated, + bool *frontier_cap_hit, u64 *safepoint_ticks); + + // Selects root-attached durable entries with a tiered cursor: leak-tagged, fresh, container-shaped, + // then everything else fairly. + std::vector collectStaticFieldAnchorsForRotation(int max_count); + // At-risk anchor push: deduped, capped, and quota-limited per klass. + void pushAtRiskStaticAnchor(jlong tag, u32 klass_id); + // Add `tag` to _static_anchor_index if its root_kind is a durable anchor-tier kind (STATIC_FIELD + // or JNI_GLOBAL). + void addToStaticAnchorIndex(jlong tag, jlong own_class_tag, u8 root_kind); + + // True iff `klass` implements java/util/Collection or java/util/Map, directly or transitively + // (superclass chain + interfaces of every visited class, depth-bounded, visited set to survive + // interface diamonds). + bool classImplementsContainerOrMap(jvmtiEnv *jvmti, JNIEnv *jni, + jclass klass); + + // Resolve _collection_iface_class_tag/_map_iface_class_tag once; returns false if the interfaces + // cannot be resolved yet (leaves them at -1 so the next call retries). + bool resolveContainerInterfaceTags(jvmtiEnv *jvmti, JNIEnv *jni); + + // Lazy shape reconciliation for the anchor index: scans _static_anchor_own_class_tags for class + // tags not yet in _class_shape_cache, resolves up to ANCHOR_SHAPE_RECONCILE_BUDGET of them per + // pass via one GetObjectsWithTags call (class objects are tagged with their class tags) and + // classifies each. + void reconcileAnchorClassShapes(jvmtiEnv *jvmti, JNIEnv *jni); + // Pops up to max_count AtRiskAnchor entries off _static_anchor_fifo's front into `out` + // (appending), decrementing each popped entry's class occupancy in + // _static_anchor_fifo_klass_counts (erased at zero, so the map tracks the FIFO's live contents), + // and re-derives the set from the deque's remaining contents (PriorityExpandSet's tombstone-free + // rebuildFrom contract). + int drainStaticAnchorFifo(int max_count, std::vector &out); + // Pushes `entries` back to _static_anchor_fifo's FRONT in reverse order (preserving FIFO order), + // re-incrementing each entry's class occupancy, and rebuilds the set - the truncated-walk requeue + // path. + void requeueStaticAnchorFifoFront(const std::vector &entries); + // When non-null, receives the tags of RESOLVED-but-unwalked anchors at the truncation break point + // - GetObjectsWithTags may return fewer anchors than requested (dead tags drop out) in its own + // order, so the caller cannot recover the un-walked set from a consumed index; the walk hands the + // exact tags back instead. + void walkStaticFieldAnchors(jvmtiEnv *jvmti, JNIEnv *jni, + const std::vector &anchor_tags, + int budget, int *edges_admitted, bool *truncated, + bool *frontier_cap_hit, u64 *safepoint_ticks, + std::vector *unwalked = nullptr); + + // jvmtiHeapRootCallback/jvmtiStackReferenceCallback for runPassManualWalk()'s + // IterateOverReachableObjects call (referenceChains.cpp). + static jvmtiIterationControl JNICALL + heapRootCallback(jvmtiHeapRootKind root_kind, jlong class_tag, jlong size, + jlong *tag_ptr, void *user_data); + static jvmtiIterationControl JNICALL stackRefCallback( + jvmtiHeapRootKind root_kind, jlong class_tag, jlong size, + jlong *tag_ptr, jlong thread_tag, jint depth, jmethodID method, + jint slot, void *user_data); + + // Manual-walk pass driver: when `run_root_enum` is true, seeds/refreshes root-attached frontier + // entries via IterateOverReachableObjects (heapRootCallback()/stackRefCallback() above) using + // `root_enum_budget`; then, regardless of `run_root_enum`, drains _pending_expand via + // admitStaticFieldRoots()/expandFrontier() up to `expand_budget`. + void runPassManualWalk(jvmtiEnv *jvmti, JNIEnv *jni, bool run_root_enum, + int root_enum_budget, int expand_budget, + int *edges_admitted, bool *truncated, + bool *frontier_cap_hit, u64 *safepoint_ticks); + + // Inserts (or refreshes) klass_id's resolved chain in _resolved_chains, recording the + // source_tag/source_search_ns it was reconstructed from so a later poll can tell a stale entry + // from a current one. + bool cacheResolvedChain(jlong source_tag, ReferenceChainEvent &&event, + jlong source_tag_val, u64 source_search_ns); + + // Remove a cached chain so pollWatchedTargets rebuilds it on the next poll. + void invalidateResolvedChain(jlong source_tag); + + // Snapshots the just-abandoned search into _pending_abandoned_events - called from runPass() + // (referenceChains.cpp) immediately after it writes SearchState::ABANDONED, while + // buildAbandonedEvent()'s source fields are still valid (see _pending_abandoned_events' own + // comment for why this cannot be deferred to dump()-time). + void enqueuePendingAbandonedEvent(); + +public: + static ReferenceChainTracker *instance() { + static ReferenceChainTracker instance; + return &instance; + } + + // tid -> java.lang.Thread global-ref registry (see _thread_objects). + void registerThreadObject(JNIEnv *jni, int tid, jthread thread); + void unregisterThreadObject(JNIEnv *jni, int tid); + + // Delete the global refs unregisterThreadObject() queued in _thread_refs_pending_delete (see that + // member's comment for why the deletion is deferred). + void releaseEndedThreadRefs(JNIEnv *jni); + + // Recording-stop cleanup: delete EVERY remaining registered Thread global ref (dead threads' + // queued refs first, then the live registry) and empty the registry. + void releaseAllThreadObjects(JNIEnv *jni); + + // One-time sweep over the JVM's CURRENTLY LIVE threads at recording start, registering each into + // the same tid -> Thread-object registry via JVMThread::nativeThreadId(). + void registerExistingThreads(jvmtiEnv *jvmti, JNIEnv *jni); + + // Correlate a leak tag with an instance the BFS admitted BEFORE tagLeakInstances() tagged it (its + // JVMTI tag is a frontier tag, its frontier entry has leak_tag == 0). + bool correlateAdmittedLeakTag(jlong frontier_tag, jlong leak_tag, + u32 klass_id); + + // Abandon the search after this many consecutive passes with zero new frontier entries admitted + // (genuinely stuck, not just slow). + static constexpr int NO_PROGRESS_PASS_LIMIT = 30; + + // Base limit for the canary-specific stuck detector: candidate-discovery must show no progress + // (no change to _candidate_found_bits, no new candidate admitted into a slot) for this many + // consecutive passes AND the whole-graph frontier must also have stalled for + // NO_PROGRESS_PASS_LIMIT passes (see runPass()'s CANARY_STUCK branch) before a canary search is + // abandoned. + static constexpr int CANARY_NO_PROGRESS_PASS_LIMIT = 30; + + // Upper bound on how many times canaryStuckPassLimit() doubles the base limit (2^8 = 256x -> 7680 + // passes at the default base of 30) - bounds the escalation so a search that is ACTUALLY stuck + // forever (as opposed to merely deep) still gets abandoned in finite time rather than growing its + // patience without limit. + static constexpr int MAX_CANARY_STUCK_BACKOFF_SHIFT = 8; + + // The canary-stuck pass limit for the *current* restart attempt: CANARY_NO_PROGRESS_PASS_LIMIT + // doubled once per consecutive CANARY_STUCK restart of this candidate-chase sequence, capped at + // MAX_CANARY_STUCK_BACKOFF_SHIFT doublings. + int canaryStuckPassLimit() const { + return CANARY_NO_PROGRESS_PASS_LIMIT + << std::min(_canary_stuck_restart_count, + MAX_CANARY_STUCK_BACKOFF_SHIFT); + } + + // Multiplier cap for the canary lane's work-scaled backoff (see _canary_backoff_mult's own + // comment). + static constexpr int CANARY_BACKOFF_MULT_MAX = 16; + + // While a canary search has candidates still unresolved, shouldRunPass() raises + // _cpu_pain_budget's refill rate by this factor (capped at 100%/wall-clock). + static constexpr double CANARY_PAIN_BUDGET_REFILL_MULTIPLIER = 100.0; + + // Coverage tracking: how many leak tags have been assigned vs resolved. + int _leak_tags_assigned = 0; + int _leak_tags_resolved = 0; + + // Leak tags are positive JVMTI tags in a dedicated range, assigned by LivenessTracker's tag pool + // to specific tracked leaking objects. + static constexpr jlong LEAK_TAG_BASE = 0x40000000LL; + static constexpr int LEAK_TAG_POOL_SIZE = 256; + + // Check whether a JVMTI tag is a leak tag (from LivenessTracker's pool). + static bool isLeakTag(jlong tag) { + return tag >= LEAK_TAG_BASE && tag < LEAK_TAG_BASE + LEAK_TAG_POOL_SIZE; + } + + // Max candidates LivenessTracker::selectLeakCandidates() can return. + + // Test accessor for _passes_since_last_progress. + int passesSinceLastProgressForTest() const { return _passes_since_last_progress; } + // Canary-lane backoff state - see _canary_backoff_mult's own comment. + int canaryBackoffMultForTest() const { return _canary_backoff_mult; } + u64 canaryPassEmaMsForTest() const { return _canary_pass_ema_ms; } + u64 lastCanaryPassNsForTest() const { return _last_canary_pass_ns; } + void setCanaryBackoffForTest(int mult, u64 ema_ms, u64 last_pass_ns) { + _canary_backoff_mult = mult; + _canary_pass_ema_ms = ema_ms; + _last_canary_pass_ns = last_pass_ns; + } + void setOomRampActiveForTest(bool active) { _oom_ramp_active = active; } + int candidateCountForTest() const { return _candidate_count; } + void setCandidateCountForTest(int n) { _candidate_count = n; } + void setCandidateKlassIdForTest(int idx, u32 klass_id) { + _candidate_klass_ids[idx] = klass_id; + } + jlong candidateDiscoveredTagForTest(int slot, int idx) const { + return _candidate_discovered_tags[slot][idx]; + } + int candidateDiscoveredCountForTest(int slot) const { + return _candidate_discovered_count[slot]; + } + u64 candidateFoundBitsForTest() const { return _candidate_found_bits; } + void setCandidateFrontierTagForTest(int idx, jlong tag) { _candidate_frontier_tags[idx] = tag; } + void setCandidateParentTagForTest(int idx, jlong tag) { _candidate_parent_tags[idx] = tag; } + void setCandidateReferrerKlassForTest(int idx, u32 klass_id) { + _candidate_referrer_klasses[idx] = klass_id; + } + void setCandidateDepthForTest(int idx, u32 depth) { _candidate_depths[idx] = depth; } + int passesSinceLastCandidateProgressForTest() const { return _passes_since_last_candidate_progress; } + int canaryStuckRestartCountForTest() const { return _canary_stuck_restart_count; } + + ReferenceChainTracker(const ReferenceChainTracker &) = delete; + ReferenceChainTracker &operator=(const ReferenceChainTracker &) = delete; + + Error start(Arguments &args); + + // Scales unset referencechains defaults (budget, ttl, framecap, pausetarget, painbudget, + // firstpassbudget) from the process's max heap size and available processor count, so a large + // heap doesn't starve the BFS (the defaults are tuned for a small heap and abandon via TTL before + // making meaningful progress). + void autoTuneDefaults(Arguments &args); + void stop(); + + // Spawns the BFS thread (threadEntry()/threadLoop()) if reference chain tracking is enabled and + // no thread is already running. + void startThread(); + + // Stops and joins the BFS thread started by startThread(), mirroring BaseWallClock::stop()'s + // pthread_kill(WAKEUP_SIGNAL) + pthread_join() shape (wallClock.cpp) - WAKEUP_SIGNAL is already + // installed unconditionally in vmEntry.cpp, so no extra signal setup is needed here. + void stopThread(); + + bool enabled() const { return _enabled; } + + u64 gcStartEpoch() { return load(_gc_start_epoch); } + u64 gcFinishEpoch() { return load(_gc_finish_epoch); } + + // JVMTI tag helpers used by the heap-walk callbacks. + jlong nextTag() { return atomicIncRelaxed(_next_tag, (jlong)1); } + + // Serializes runPass()+pollWatchedTargets() between threadLoop() and the test seams - see + // runPassForTest()'s comment. + Mutex _engine_lock; + jlong tagObject(jvmtiEnv *jvmti, jobject obj); + jlong getTag(jvmtiEnv *jvmti, jobject obj); + void clearTag(jvmtiEnv *jvmti, jobject obj); + + // Hands out a fresh negative class tag, from the shared, process-wide counter both this class and + // LivenessTracker mint from - see classTagAllocator.h's own header comment for why this must be + // shared rather than a private counter here. + jlong nextClassTag() { return ClassTagAllocator::next(); } + + // Returns the frontier metadata table, or nullptr if the subsystem was never started with the + // flag enabled. + FrontierTable *frontierTable() { return _frontier; } + + // Returns the class-tag resolution table. Exposed for testing in isolation, matching + // frontierTable()'s existing rationale. + ClassTagTable *classTags() { return &_class_tags; } + + // Runs exactly one bounded BFS pass and returns. The first call for a search seeds + // FollowReferences from the heap roots (heap_filter=0, klass=NULL, initial_object=NULL - see this + // method's own comment in referenceChains.cpp for why FollowReferences rather than + // IterateThroughHeap); every later call resumes from the persisted frontier via expandFrontier() + // instead of re-walking from the roots (see expandFrontier()'s comment for why - re-walking from + // the roots each call would re-traverse the entire already-discovered subgraph every pass, + // defeating the point of a per-pass budget). + bool runPass(jvmtiEnv *jvmti, JNIEnv *jni, bool *out_truncated = nullptr); + + // Serialized entry points for the two engine drivers: the real BFS thread (threadLoop(), below) + // and the debug seams (javaApi.cpp's runReferenceChainPass0()/pollReferenceChainTargets0()). + bool runPassSerialized(jvmtiEnv *jvmti, JNIEnv *jni) { + MutexLocker engine_guard(_engine_lock); + return runPass(jvmti, jni); + } + + void pollWatchedTargetsSerialized(jvmtiEnv *jvmti, JNIEnv *jni) { + MutexLocker engine_guard(_engine_lock); + pollWatchedTargets(jvmti, jni); + } + + // Search-level outcome (SearchState's constants) - see runPass()'s comment for exactly when this + // leaves RUNNING. + u8 searchState() { return loadAcquire(_search_state); } + + // Total passes run for the current/most recent search. Exposed for tests to confirm multi-pass + // resumption actually happened. + int passesRun() { return load(_passes_run); } + + // Which SearchAbandonReason cutoff moved the search out of RUNNING, or SearchAbandonReason::NONE + // if it never left RUNNING or left via SearchState::COMPLETED instead. + u8 abandonReason() { return load(_abandon_reason); } + + // Reference-chain JFR event surface: fills *out from frontierTable()-> + // reconstructChain(target_tag, ...) (see that method's own comment for the leaf-to-root ordering + // and the parent_tag walk it performs). + void resolveHopEdgeLabel(jvmtiEnv *jvmti, JNIEnv *jni, ChainHopEdge edge, + char *out, size_t out_cap); + + // Fills *out with one label per chain hop, aligned with the chain's leaf-to-root order (edges[i] + // = the retention edge INTO chain[i]), via resolveHopEdgeLabel() above. + static constexpr size_t MAX_HOP_EDGE_LABEL = + MAX_REFERENCE_CHAIN_EDGE_LABEL; + // Per-referrer-class ordinal->name list cache behind resolveHopEdgeLabel(): decoding the spec + // ordinal requires walking the class's whole interface closure + superclass chain (GetClassFields + // + GetFieldName per field), and chains re-emit on every dump, so the decoded ordinal space of + // each chain-relevant class is built once here. + static constexpr size_t HOP_LABEL_CLASS_CACHE_CAP = 1024; + struct HopLabelClass { + jlong class_tag; + // One entry per ordinal in the class's flattened field space - the i-th element is the name of + // ordinal i. + std::vector field_names; + bool decode_failed; + }; + std::unordered_map _hop_label_cache; + + // Cache lookup/decode behind resolveHopEdgeLabel() - see HopLabelClass's own comment. + const HopLabelClass *hopLabelClassFor(jvmtiEnv *jvmti, JNIEnv *jni, + jlong class_tag); + + void fillHopEdgeLabels(jvmtiEnv *jvmti, JNIEnv *jni, + const std::vector &edges, + std::vector *out); + + bool buildChainEvent(jvmtiEnv *jvmti, JNIEnv *jni, jlong target_tag, + ReferenceChainEvent *out); + + // Appends the root TYPE element (the declaring class, resolved from + // FrontierEntry::referrer_class_tag) to a static-field-rooted chain - see the definition's + // comment in referenceChains.cpp for the full rationale and the skip conditions. + void appendStaticFieldRootType(const FrontierEntry &terminal, + std::vector *chain, + std::vector *edges); + + // Canary-search chain reconstruction: builds the chain for a canary candidate from the + // per-candidate chain link recorded at pruning time (_candidate_parent_tags[] etc.), walking + // parent_tag through the frontier table (positive tags, so lookup() works). + bool buildCanaryChainEvent(int candidate_idx, ReferenceChainEvent *out); + + // Reports a search's termination state without needing a target tag. + bool buildAbandonedEvent(ReferenceChainAbandonedEvent *out) { + // Acquire-load, not a plain relaxed load - see searchState()'s own comment for why: this is the + // same guard-then-read-details pattern. + if (out == nullptr || loadAcquire(_search_state) != SearchState::ABANDONED) { + return false; + } + out->_reason = load(_abandon_reason); + out->_passes_run = (u32)load(_passes_run); + out->_frontier_size = _frontier != nullptr ? (u32)_frontier->size() : 0; + out->_hop_cap = _hop_cap; + out->_budget = _budget; + out->_ttl_ms = _ttl_ms; + out->_elapsed_ns = load(_last_pass_ns) - load(_search_start_ns); + return true; + } + + // Bridges LivenessTracker leak candidates into cached ReferenceChain events; reads the existing + // tag, never seeds one. + void pollWatchedTargets(jvmtiEnv *jvmti, JNIEnv *jni); + + // Targeted holder re-walk: enqueues `tag`'s chain-root entry (the root-attached ancestor of its + // frontier chain) onto _priority_expand so the next rotation/expand pass re-walks the holder that + // retains everything below `tag`. + void requeueChainRootForRotation(jlong tag); + + // Appends a copy of every currently-cached resolved chain to *out, re-stamped with a fresh + // _start_time so it lands in the dumping chunk's time window, WITHOUT clearing the cache - a + // repeatable snapshot, not a drain, so the same live sample's chain is re-emitted into every JFR + // chunk it survives into (see _resolved_chains' own comment). + void drainPendingChainEvents(std::vector *out); + + // Appends every abandoned-search event queued since the last call and clears the queue - a true + // drain, unlike drainPendingChainEvents() above: an abandoned search is a discrete past + // occurrence, not an ongoing live sample, so there is nothing left to re-report once + // Profiler::dump() (profiler.cpp) has emitted it. + void drainPendingAbandonedEvents(std::vector *out); + + static void JNICALL GarbageCollectionStart(jvmtiEnv *jvmti_env); + static void JNICALL GarbageCollectionFinish(jvmtiEnv *jvmti_env); + + // Test seam - not part of the production API. Mirrors LivenessTracker's own "Test seams" block + // (livenessTracker.h). + jlong tagAsRootForTest(jvmtiEnv *jvmti, JNIEnv *jni, jobject obj); + + // Test seam - not part of the production API. Since ReferenceChainTracker is a process-wide + // singleton (ExternalProcessReferenceChainTest's own class javadoc explains why that matters: + // only the *first* test to ever call runPass() in a shared JVM gets a real root-seeded walk, + // since runPass() only re-walks from the roots once per search's whole lifetime), an in-process + // test that needs its own genuine first-ever root walk calls this at the start of its test body + // to force exactly that - releasing any tags a previous test's search still held, then resetting + // search/frontier state to the same "brand-new tracker" state restartSearch() + // (referenceChains.cpp) produces, plus the target- dedup/pending-event state restartSearch() + // itself intentionally leaves for pollWatchedTargets()/drainPendingChainEvents() to self-clear + // (this is an immediate, out-of-band reset - there is no next real pass here to observe the + // change and clear them the ordinary way). + void resetSearchStateForTest(jvmtiEnv *jvmti, JNIEnv *jni); + + // Test seam - not part of the production API. Diagnostic-only: reports how far a given + // (already-tagged) object sits from the front of _pending_expand's FIFO queue, to distinguish + // "not yet expanded because its own FIFO position hasn't come up yet" from "already expanded" or + // "never admitted at all" without needing a debugger. + long pendingExpandPositionForTest(jlong tag) const; + + // Test seam - not part of the production API. Companion to pendingExpandPositionForTest() above, + // for computing a position's fraction of the current backlog. + size_t pendingExpandSizeForTest() const; + + // Test seam - not part of the production API. Exposes the private shouldRunPass() gate directly, + // so a test can assert whether a fresh/terminal search would be allowed to start right now - in + // particular, whether LivenessTracker::secondsToOOM()'s urgent-OOM bypass (hasLeakSignal(), see + // OOM_URGENT_THRESHOLD_S's own comment above) opens this gate even with zero per-klass leak + // candidate (confirmable in the same test via + // LivenessTracker::selectLeakCandidates()/JavaProfiler's selectLeakCandidateKlassIds0() seam) - + // something runReferenceChainPass0() (javaApi.cpp) cannot show, since it calls runPass() directly + // and never consults this gate at all. + bool shouldRunPassForTest(u64 now_ns) { return shouldRunPass(now_ns); } +}; + +#endif // _REFERENCECHAINS_H diff --git a/ddprof-lib/src/main/cpp/safeAccess.h b/ddprof-lib/src/main/cpp/safeAccess.h index 63deb6c6f..a237975f8 100644 --- a/ddprof-lib/src/main/cpp/safeAccess.h +++ b/ddprof-lib/src/main/cpp/safeAccess.h @@ -1,6 +1,6 @@ /* * Copyright 2021 Andrei Pangin -* Copyright 2026 Datadog, Inc + * Copyright 2026 Datadog, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ddprof-lib/src/main/cpp/stringDictionary.h b/ddprof-lib/src/main/cpp/stringDictionary.h index b5572b623..f16f802a7 100644 --- a/ddprof-lib/src/main/cpp/stringDictionary.h +++ b/ddprof-lib/src/main/cpp/stringDictionary.h @@ -459,6 +459,11 @@ class StringDictionaryBuffer { class StringDictionary { std::atomic _next_id{1}; // starts at 1; id=0 reserved as "no entry" std::atomic _accepting{true}; // false while clearAll() is resetting buffers + // Bumped by clearAll() only. Lets a cache keyed by ids from this + // dictionary (e.g. ReferenceChainTracker::_class_tags, referenceChains.h) + // detect "the id namespace was wiped out from under me" and invalidate + // itself, rather than assuming ids stay valid across a clearAll(). + std::atomic _generation{0}; StringDictionaryBuffer _a, _b, _c; TripleBufferRotator _rot; int _counter_offset; // offset into DICTIONARY_KEYS / DICTIONARY_KEYS_BYTES counter rows @@ -489,6 +494,9 @@ class StringDictionary { } } + // Current id-namespace generation; see _generation's own comment. + u64 generation() const { return _generation.load(std::memory_order_acquire); } + // Insert into active buffer; returns globally stable id. NOT signal-safe. u32 lookup(const char* key, size_t len) { if (!_accepting.load(std::memory_order_acquire)) return 0; @@ -635,6 +643,7 @@ class StringDictionary { _next_id.store(1, std::memory_order_relaxed); Counters::set(DICTIONARY_KEYS, 0, _counter_offset); Counters::set(DICTIONARY_KEYS_BYTES, 0, _counter_offset); + _generation.fetch_add(1, std::memory_order_release); _accepting.store(true, std::memory_order_release); } }; diff --git a/ddprof-lib/src/main/cpp/vmEntry.cpp b/ddprof-lib/src/main/cpp/vmEntry.cpp index 17bde30d9..acf838232 100644 --- a/ddprof-lib/src/main/cpp/vmEntry.cpp +++ b/ddprof-lib/src/main/cpp/vmEntry.cpp @@ -17,6 +17,7 @@ #include "log.h" #include "os.h" #include "profiler.h" +#include "referenceChains.h" #include "safeAccess.h" #include "samplerPerf.h" #include "threadLocalData.h" @@ -444,6 +445,17 @@ bool VM::initializeRequestStackTrace() { return false; } +// JVMTI delivers ONE callback per event slot; both trackers need the +// GarbageCollectionFinish signal (liveness GC epochs + reference-chain pass +// scheduling), so this vmEntry-level forwarder fans the single slot out to +// both static callbacks. Order is irrelevant (each only does lock-free +// bookkeeping); LivenessTracker's callback keeps its own +// initCurrentThreadSignalSafe() behavior. +void JNICALL ForwardedGarbageCollectionFinish(jvmtiEnv *jvmti_env) { + LivenessTracker::GarbageCollectionFinish(jvmti_env); + ReferenceChainTracker::GarbageCollectionFinish(jvmti_env); +} + bool VM::initProfilerBridge(JavaVM *vm, bool attach) { TEST_LOG("VM::initProfilerBridge"); if (!initShared(vm)) { @@ -518,7 +530,8 @@ bool VM::initProfilerBridge(JavaVM *vm, bool attach) { callbacks.ThreadStart = Profiler::ThreadStart; callbacks.ThreadEnd = Profiler::ThreadEnd; callbacks.SampledObjectAlloc = ObjectSampler::SampledObjectAlloc; - callbacks.GarbageCollectionFinish = LivenessTracker::GarbageCollectionFinish; + callbacks.GarbageCollectionStart = ReferenceChainTracker::GarbageCollectionStart; + callbacks.GarbageCollectionFinish = ForwardedGarbageCollectionFinish; callbacks.NativeMethodBind = VMStructs::NativeMethodBind; _jvmti->SetEventCallbacks(&callbacks, sizeof(callbacks)); diff --git a/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp b/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp index e0117af6a..7d8ee7263 100644 --- a/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp +++ b/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp @@ -15,6 +15,7 @@ */ #include +#include "livenessTracker.h" #include "../../main/cpp/gtest_crash_handler.h" #include #include @@ -289,3 +290,1045 @@ TEST_F(LivenessTrackerTest, CapacityDoesNotExceedMaxCap) { // In the actual code, this would trigger: if (_table_cap != newcap) { ... } // which would be false, so no resize would be attempted } + +// --------------------------------------------------------------------------- +// Per-klass population tracking. These exercise LivenessTracker::instance() +// directly rather than +// a mock: recordKlassPopulationSampleLocked() deliberately makes no JNI call +// (see its header comment), so it is safe to call on the real singleton +// without a live JVM attached, unlike start()/track()/flush() elsewhere in +// this class. Fake jweak values below are opaque pointers the method under +// test never dereferences - only stored and handed back to the caller. +class KlassPopulationTest : public ::testing::Test { +protected: + void SetUp() override { + installGtestCrashHandler(); + // The table persists across recordings by design (see + // LivenessTracker::initialize()'s own comment on why _initialized + // survives multiple start() calls) - reset it explicitly here so + // tests don't observe leftover state from a previous test case + // sharing the same process-wide singleton. + LivenessTracker::instance()->klassPopulationResetForTest(); + } + + void TearDown() override { + LivenessTracker::instance()->klassPopulationResetForTest(); + restoreDefaultSignalHandlers(); + } + + static jweak fakeRef(uintptr_t tag) { + return reinterpret_cast(tag); + } +}; + +// A brand new klass_id creates a new entry: out_created is true, the table +// grows by one, and the single pushed sample is the ring's only member. +TEST_F(KlassPopulationTest, InsertCreatesNewEntry) { + LivenessTracker *tracker = LivenessTracker::instance(); + + int slot = -1; + bool created = false; + jweak evicted = tracker->klassPopulationRecordForTest(/*klass_id=*/1, + /*count=*/5, + /*epoch=*/1, + &slot, &created); + + EXPECT_TRUE(created); + EXPECT_EQ(evicted, nullptr); + EXPECT_EQ(tracker->klassPopulationSizeForTest(), 1); + + KlassPopulationEntry entry; + ASSERT_TRUE(tracker->klassPopulationLookupForTest(1, &entry)); + EXPECT_EQ(entry.klass_id, 1u); + EXPECT_EQ(entry.ring_fill, 1); + EXPECT_EQ(entry.ring_head, 1); + EXPECT_EQ(entry.count_ring[0], 5); + EXPECT_EQ(entry.last_updated_epoch, 1u); + EXPECT_EQ(entry.representative_count, 0); +} + +// A second sample for an already-known klass_id updates the same slot in +// place (out_created is false, table size unchanged) rather than creating a +// second entry. +TEST_F(KlassPopulationTest, InsertExistingUpdatesSameSlotInPlace) { + LivenessTracker *tracker = LivenessTracker::instance(); + + int slot1 = -1, slot2 = -1; + bool created1 = false, created2 = false; + tracker->klassPopulationRecordForTest(7, 3, 1, &slot1, &created1); + jweak evicted = tracker->klassPopulationRecordForTest(7, 4, 2, &slot2, + &created2); + + EXPECT_TRUE(created1); + EXPECT_FALSE(created2); + EXPECT_EQ(slot1, slot2); + EXPECT_EQ(evicted, nullptr); + EXPECT_EQ(tracker->klassPopulationSizeForTest(), 1); + + KlassPopulationEntry entry; + ASSERT_TRUE(tracker->klassPopulationLookupForTest(7, &entry)); + EXPECT_EQ(entry.ring_fill, 2); + EXPECT_EQ(entry.count_ring[0], 3); + EXPECT_EQ(entry.count_ring[1], 4); + EXPECT_EQ(entry.last_updated_epoch, 2u); +} + +// Ring buffer wraparound: pushing more than KLASS_POPULATION_RING_SIZE (30) +// samples must not grow ring_fill past 30, and the ring must overwrite the +// oldest slots in order rather than corrupting adjacent entries. +TEST_F(KlassPopulationTest, RingBufferWrapsAroundAtThirtySamples) { + LivenessTracker *tracker = LivenessTracker::instance(); + + const int RING_SIZE = 30; + for (int i = 0; i < RING_SIZE + 5; i++) { + int slot; + bool created; + tracker->klassPopulationRecordForTest(42, (u16)(i + 1), i + 1, &slot, + &created); + EXPECT_EQ(created, i == 0); + } + + KlassPopulationEntry entry; + ASSERT_TRUE(tracker->klassPopulationLookupForTest(42, &entry)); + // Still capped at 30 even though 35 samples were pushed. + EXPECT_EQ(entry.ring_fill, RING_SIZE); + // ring_head wrapped: 35 writes into a 30-slot ring lands back at index 5. + EXPECT_EQ(entry.ring_head, 5); + // 35 pushes write ring indices 0..29 with values 1..30, then wrap and + // overwrite indices 0..4 with values 31..35 - leaving indices 5..29 + // still holding values 6..30 (never overwritten) and indices 0..4 + // holding the wrapped-around values 31..35. + EXPECT_EQ(entry.count_ring[5], 6); + EXPECT_EQ(entry.count_ring[29], 30); + EXPECT_EQ(entry.count_ring[0], 31); + EXPECT_EQ(entry.count_ring[4], 35); + EXPECT_EQ(entry.last_updated_epoch, RING_SIZE + 5u); +} + +// A klass whose every tracked instance died never appears in +// _klass_count_scratch, so the per-scratch fold skips it - without the +// zero-sample pass its ring would keep the last positive count and its +// consecutive_positive trend, keeping a dead population a leak candidate +// until the entry is evicted. Running a fold for a later epoch with an +// empty scratch must push a zero sample into the absent entry's ring, +// refresh its last_updated_epoch, and reset consecutive_positive. +TEST_F(KlassPopulationTest, FoldRecordsZeroSampleForClassesThatDisappear) { + LivenessTracker *tracker = LivenessTracker::instance(); + + // Seed a positive population history for klass 42 through epoch 3. + for (u64 epoch = 1; epoch <= 3; epoch++) { + int slot; + bool created; + tracker->klassPopulationRecordForTest(42, /*count=*/3 + (int)epoch, + epoch, &slot, &created); + } + KlassPopulationEntry entry; + ASSERT_TRUE(tracker->klassPopulationLookupForTest(42, &entry)); + ASSERT_EQ(entry.last_updated_epoch, 3u); + + // Every instance of klass 42 died: epoch 4's fold runs with an EMPTY + // scratch (no survivors at all). + tracker->foldKlassCountsZeroSampleForTest(/*epoch=*/4); + + ASSERT_TRUE(tracker->klassPopulationLookupForTest(42, &entry)); + // Zero sample recorded for the fold's epoch... + EXPECT_EQ(entry.last_updated_epoch, 4u); + // ...visible as the most recently pushed ring value... + u8 head = (u8)((entry.ring_head + 30 - 1) % + 30); + EXPECT_EQ(entry.count_ring[head], 0u); + // ...and the stale positive trend is cleared. + EXPECT_EQ(entry.consecutive_positive, 0); + + // A klass that DID receive a survivor sample this epoch is not touched + // again by the zero-sample pass (its just-pushed count stays intact). + int slot; + bool created; + tracker->klassPopulationRecordForTest(43, /*count=*/2, /*epoch=*/4, &slot, + &created); + EXPECT_TRUE(created); + tracker->foldKlassCountsZeroSampleForTest(/*epoch=*/4); + ASSERT_TRUE(tracker->klassPopulationLookupForTest(43, &entry)); + // The survivor sample (2) is still the entry's most recent value - not + // overwritten by a zero sample. + u8 head43 = (u8)((entry.ring_head + 30 - 1) % + 30); + EXPECT_EQ(entry.count_ring[head43], 2u); +} + +// Filling the table to MAX_KLASS_POPULATION_ENTRIES and then inserting one +// more distinct klass_id must evict the least-recently-updated entry (the +// smallest last_updated_epoch) and return its representative jweak so the +// caller can release it. +TEST_F(KlassPopulationTest, EvictsLeastRecentlyUpdatedEntryWhenFull) { + LivenessTracker *tracker = LivenessTracker::instance(); + + const int CAP = 256; // MAX_KLASS_POPULATION_ENTRIES + for (u32 klass_id = 1; klass_id <= (u32)CAP; klass_id++) { + int slot; + bool created; + // epoch == klass_id, so klass_id 1 is the least-recently-updated + // entry once the table is full. + tracker->klassPopulationRecordForTest(klass_id, 1, klass_id, &slot, + &created); + ASSERT_TRUE(created); + } + EXPECT_EQ(tracker->klassPopulationSizeForTest(), CAP); + + jweak victim_ref = fakeRef(0xdead); + tracker->klassPopulationSetRepresentativeForTest(nullptr, 1, victim_ref); + + int slot; + bool created; + // Eviction now returns evicted refs via output array, not return value. + jweak evicted[KlassPopulationEntry::MAX_REPRESENTATIVES_PER_KLASS]; + int evicted_count = 0; + tracker->klassPopulationRecordForTest( + /*klass_id=*/CAP + 1, /*count=*/1, /*epoch=*/CAP + 1, &slot, &created, + evicted, &evicted_count, KlassPopulationEntry::MAX_REPRESENTATIVES_PER_KLASS); + + EXPECT_TRUE(created); + ASSERT_EQ(evicted_count, 1); + EXPECT_EQ(evicted[0], victim_ref); + // Table stays at capacity - the evicted slot was reused, not appended. + EXPECT_EQ(tracker->klassPopulationSizeForTest(), CAP); + + KlassPopulationEntry evicted_klass_entry; + EXPECT_FALSE(tracker->klassPopulationLookupForTest(1, &evicted_klass_entry)) + << "klass_id 1 should have been fully replaced by the eviction"; + + KlassPopulationEntry new_entry; + ASSERT_TRUE(tracker->klassPopulationLookupForTest(CAP + 1, &new_entry)); + EXPECT_EQ(new_entry.representative_count, 0); + EXPECT_EQ(new_entry.ring_fill, 1); +} + +// --------------------------------------------------------------------------- +// Slope computation and candidate ranking. Same rationale as KlassPopulationTest above +// for exercising LivenessTracker::instance() directly: selectLeakCandidates() +// makes no JNI call (it only copies the opaque jweak field, never +// dereferences it), so it is safe to call on the real singleton without a +// live JVM, and the *ForTest seams already in place are enough to seed +// arbitrary ring-buffer states without going through cleanup_table(). +class SelectLeakCandidatesTest : public ::testing::Test { +protected: + void SetUp() override { + installGtestCrashHandler(); + LivenessTracker::instance()->klassPopulationResetForTest(); + } + + void TearDown() override { + LivenessTracker::instance()->klassPopulationResetForTest(); + restoreDefaultSignalHandlers(); + } + + static jweak fakeRef(uintptr_t tag) { + return reinterpret_cast(tag); + } + + // Pushes `n` samples (count values `counts[0..n)`, one per epoch starting + // at `start_epoch`) into klass_id's ring buffer via the same + // recordKlassPopulationSampleLocked() path production code drives from + // cleanup_table()'s epoch-advance pass (klassPopulationRecordForTest() is + // a direct pass-through to it, see its header comment). ALSO seeds a + // qualifying per-tid trend for the same epochs (a linear 1..n ramp on a + // fixed synthetic tid) - selectLeakCandidates() now requires a qualifying + // allocating thread on top of the klass-level ramp, so a series seeded + // through this helper represents a genuinely thread-concentrated leak. + // Tests that specifically exercise the per-tid gate itself seed the + // tid trends (or their absence) directly via tidTrendRecordForTest(). + static void seedSeries(LivenessTracker *tracker, u32 klass_id, + const u16 *counts, int n, u64 start_epoch) { + for (int i = 0; i < n; i++) { + int slot; + bool created; + tracker->klassPopulationRecordForTest(klass_id, counts[i], + start_epoch + i, &slot, + &created); + tracker->tidTrendRecordForTest(klass_id, /*tid=*/42, + (u32)(i + 1), start_epoch + i); + } + } +}; + +// A klass whose population is monotonically increasing for long enough has a +// positive slope, clears the growth/floor magnitude bars +// (hasQualifyingGrowth()) for enough consecutive epochs to satisfy the +// sustained-trend hysteresis requirement, and is returned, carrying its +// representative jweak through unchanged. 20 samples (not just the 10-sample +// minimum fill) - see MinimumFillAloneDoesNotClearHysteresis/ +// SustainedGrowthClearsHysteresis below for the boundary this margin avoids. +TEST_F(SelectLeakCandidatesTest, GrowingPopulationIsSelected) { + LivenessTracker *tracker = LivenessTracker::instance(); + + u16 growing[20]; + for (int i = 0; i < 20; i++) { + growing[i] = (u16)(i + 1); + } + seedSeries(tracker, /*klass_id=*/1, growing, 20, /*start_epoch=*/1); + jweak rep = fakeRef(0x1); + tracker->klassPopulationSetRepresentativeForTest(nullptr, 1, rep); + + KlassCandidate out[5]; + int count = tracker->selectLeakCandidates(out, 5); + + ASSERT_EQ(count, 1); + EXPECT_EQ(out[0].klass_id, 1u); + EXPECT_EQ(out[0].representative, rep); +} + +// A klass with a flat population (zero slope) is not a growth candidate - +// the design doc requires strictly positive slope, not "non-negative". +TEST_F(SelectLeakCandidatesTest, FlatPopulationIsNotSelected) { + LivenessTracker *tracker = LivenessTracker::instance(); + + const u16 flat[10] = {5, 5, 5, 5, 5, 5, 5, 5, 5, 5}; + seedSeries(tracker, /*klass_id=*/1, flat, 10, /*start_epoch=*/1); + + KlassCandidate out[5]; + int count = tracker->selectLeakCandidates(out, 5); + + EXPECT_EQ(count, 0); +} + +// A klass whose population is shrinking has a negative slope and must not be +// reported as a leak candidate. +TEST_F(SelectLeakCandidatesTest, ShrinkingPopulationIsNotSelected) { + LivenessTracker *tracker = LivenessTracker::instance(); + + const u16 shrinking[10] = {10, 9, 8, 7, 6, 5, 4, 3, 2, 1}; + seedSeries(tracker, /*klass_id=*/1, shrinking, 10, /*start_epoch=*/1); + + KlassCandidate out[5]; + int count = tracker->selectLeakCandidates(out, 5); + + EXPECT_EQ(count, 0); +} + +// A klass with fewer than KLASS_POPULATION_MIN_FILL_FOR_TREND (10) samples +// is skipped regardless of how strong its apparent trend looks - not enough +// history yet to trust it (design doc's explicit minimum-fill requirement). +TEST_F(SelectLeakCandidatesTest, JustBelowMinimumFillIsNotSelected) { + LivenessTracker *tracker = LivenessTracker::instance(); + + const u16 growing_but_short[9] = {1, 2, 3, 4, 5, 6, 7, 8, 9}; + seedSeries(tracker, /*klass_id=*/1, growing_but_short, 9, + /*start_epoch=*/1); + + KlassCandidate out[5]; + int count = tracker->selectLeakCandidates(out, 5); + + EXPECT_EQ(count, 0); +} + +// Exactly KLASS_POPULATION_MIN_FILL_FOR_TREND (10) samples clears +// hasQualifyingGrowth() on only its very last push - every earlier push saw +// ring_fill below the minimum and was rejected outright, so +// consecutive_positive is only 1 by the time fill reaches 10. One qualifying +// epoch does not clear the sustained-trend hysteresis requirement +// (LEAK_TREND_HYSTERESIS_BASE, 5 consecutive qualifying epochs) on its own - +// this used to be enough before that gate existed (hence this test's name), +// but is not anymore; see SustainedGrowthClearsHysteresis below for the new +// equivalent boundary test. +TEST_F(SelectLeakCandidatesTest, MinimumFillAloneDoesNotClearHysteresis) { + LivenessTracker *tracker = LivenessTracker::instance(); + + const u16 growing[10] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + seedSeries(tracker, /*klass_id=*/1, growing, 10, /*start_epoch=*/1); + + KlassCandidate out[5]; + int count = tracker->selectLeakCandidates(out, 5); + + EXPECT_EQ(count, 0); +} + +// Once growth/floor keeps qualifying for enough additional epochs past +// min-fill to reach LEAK_TREND_HYSTERESIS_BASE (5 consecutive qualifying +// epochs: fill 10 through 14), the klass is trusted. +TEST_F(SelectLeakCandidatesTest, SustainedGrowthClearsHysteresis) { + LivenessTracker *tracker = LivenessTracker::instance(); + + u16 growing[14]; + for (int i = 0; i < 14; i++) { + growing[i] = (u16)(i + 1); + } + seedSeries(tracker, /*klass_id=*/1, growing, 14, /*start_epoch=*/1); + + KlassCandidate out[5]; + int count = tracker->selectLeakCandidates(out, 5); + + EXPECT_EQ(count, 1); +} + +// The aggregate post-GC heap floor (heapFloorRising()) lowers the number of +// consecutive qualifying epochs required from LEAK_TREND_HYSTERESIS_BASE (5) +// to LEAK_TREND_HYSTERESIS_CORROBORATED (3) for every candidate in the same +// scan - it cannot single out which klass is responsible for its own rise, +// so it can only raise or lower this bar uniformly, never reorder candidates +// against each other (see that pair's own comment, livenessTracker.h). +TEST_F(SelectLeakCandidatesTest, HeapFloorCorroborationLowersRequiredHysteresis) { + LivenessTracker *tracker = LivenessTracker::instance(); + + // 12 samples: 3 consecutive qualifying epochs past min-fill (fill = 10, + // 11, 12) - enough for LEAK_TREND_HYSTERESIS_CORROBORATED (3) but not + // LEAK_TREND_HYSTERESIS_BASE (5). + u16 growing[12]; + for (int i = 0; i < 12; i++) { + growing[i] = (u16)(i + 1); + } + seedSeries(tracker, /*klass_id=*/1, growing, 12, /*start_epoch=*/1); + + KlassCandidate out[5]; + EXPECT_EQ(tracker->selectLeakCandidates(out, 5), 0) + << "3 qualifying epochs clear the corroborated (3) but not the base " + "(5) hysteresis bar - without heap-floor corroboration this klass " + "must not be selected yet"; + + // A rising aggregate heap floor (10 samples, clearly growing) makes + // heapFloorRising() report true, lowering the bar for this same scan. + constexpr u64 GiB = 1ULL << 30; + constexpr u64 MiB = 1ULL << 20; + for (int i = 0; i < 10; i++) { + tracker->heapFloorRecordForTest(2 * GiB + (u64)i * 50 * MiB); + } + ASSERT_TRUE(tracker->heapFloorRisingForTest()); + + EXPECT_EQ(tracker->selectLeakCandidates(out, 5), 1); +} + +// --- Per-(klass, tid) qualification gate (TidTrend, livenessTracker.h) --- +// The disjoint-tagged-vs-frontier pod finding: a whole-klass rising +// generation count can come from churn spread across MANY allocating +// threads, each retaining a STABLE handful of instances. A klass with a +// qualifying klass-level trend but NO thread whose own trend qualifies is +// NOT a leak candidate. +TEST_F(SelectLeakCandidatesTest, KlassTrendWithoutQualifyingTidIsNotSelected) { + LivenessTracker *tracker = LivenessTracker::instance(); + + // Klass-level ramp only - no per-tid trend seeded at all (the raw seam + // loop, not seedSeries(), which would seed a qualifying tid too). + u16 growing[20]; + for (int i = 0; i < 20; i++) { + growing[i] = (u16)(i + 1); + int slot; + bool created; + tracker->klassPopulationRecordForTest(/*klass_id=*/1, growing[i], + /*epoch=*/i + 1, &slot, &created); + } + + KlassCandidate out[5]; + EXPECT_EQ(tracker->selectLeakCandidates(out, 5), 0) + << "a klass-level rise with no qualifying allocating thread is the " + "machinery-churn shape observed on the hotdog pod - it must not " + "become a candidate"; +} + +// A klass trend plus a tid trend that is FLAT (machinery: stable small +// retained set, no rising age span, below the retained-count bar) does +// not qualify either - each discriminator is necessary, not just one of +// them being absent. +TEST_F(SelectLeakCandidatesTest, FlatTidTrendDoesNotQualify) { + LivenessTracker *tracker = LivenessTracker::instance(); + + u16 growing[20]; + for (int i = 0; i < 20; i++) { + growing[i] = (u16)(i + 1); + int slot; + bool created; + tracker->klassPopulationRecordForTest(/*klass_id=*/1, growing[i], + /*epoch=*/i + 1, &slot, &created); + // Constant 2 surviving tracked instances, every epoch: below the + // retained-count bar and no rising age-cardinality trend. + tracker->tidTrendRecordForTest(/*klass_id=*/1, /*tid=*/7, /*count=*/2, + /*epoch=*/i + 1); + } + + KlassCandidate out[5]; + EXPECT_EQ(tracker->selectLeakCandidates(out, 5), 0); +} + +// The retained-count bar (TID_RETAINED_COUNT_BAR) qualifies a tid whose +// instances all share one age (one-cohort-per-thread accumulation - each +// one-shot worker thread's distinct-age count stays 1 forever) as long as +// it retains enough tracked instances - the discriminator that covers +// one-cohort-per-thread allocator shapes. +TEST_F(SelectLeakCandidatesTest, RetainedCountBarQualifiesOneCohortShape) { + LivenessTracker *tracker = LivenessTracker::instance(); + + u16 growing[20]; + for (int i = 0; i < 20; i++) { + growing[i] = (u16)(i + 1); + int slot; + bool created; + tracker->klassPopulationRecordForTest(/*klass_id=*/1, growing[i], + /*epoch=*/i + 1, &slot, &created); + // 12 > TID_RETAINED_COUNT_BAR (8), flat every epoch: the age trend + // alone would never qualify (no rise), the bar does. + tracker->tidTrendRecordForTest(/*klass_id=*/1, /*tid=*/7, /*count=*/12, + /*epoch=*/i + 1); + } + + KlassCandidate out[5]; + ASSERT_EQ(tracker->selectLeakCandidates(out, 5), 1); + ASSERT_GE(out[0].qualifying_tid_count, 1); + EXPECT_EQ(out[0].qualifying_tids[0], 7); +} + +// A rising per-tid trend alone (below the retained-count bar) qualifies: +// small leaks grow their age span long before their count clears the bar - +// the hotdog pod's simulated-memory-leak thread (12 tracked [B instances, +// age_count rising 2->3) is exactly this shape. +TEST_F(SelectLeakCandidatesTest, RisingTidTrendQualifiesBelowCountBar) { + LivenessTracker *tracker = LivenessTracker::instance(); + + u16 growing[20]; + for (int i = 0; i < 20; i++) { + growing[i] = (u16)(i + 1); + int slot; + bool created; + tracker->klassPopulationRecordForTest(/*klass_id=*/1, growing[i], + /*epoch=*/i + 1, &slot, &created); + // Rising, but capped at 6 tracked instances (below the bar of 8). + u32 tid_count = (u32)((i / 3) + 1) > 6 ? 6 : (u32)((i / 3) + 1); + tracker->tidTrendRecordForTest(/*klass_id=*/1, /*tid=*/9, tid_count, + /*epoch=*/i + 1); + } + + KlassCandidate out[5]; + ASSERT_EQ(tracker->selectLeakCandidates(out, 5), 1); + ASSERT_GE(out[0].qualifying_tid_count, 1); + EXPECT_EQ(out[0].qualifying_tids[0], 9); +} + +// Multiple positive-slope klasses must come back sorted by slope magnitude +// descending, not insertion order. 20-sample linear series (not the original +// 10 - see GrowingPopulationIsSelected's own note) at three distinct growth +// rates so every klass clears the growth/floor magnitude bars and the +// sustained-trend hysteresis requirement, while still ranking distinctly. +TEST_F(SelectLeakCandidatesTest, OrdersByMagnitudeDescending) { + LivenessTracker *tracker = LivenessTracker::instance(); + + u16 strong[20], weak[20], medium[20]; + for (int i = 0; i < 20; i++) { + strong[i] = (u16)(1 + i * 3); // steepest -> strongest + weak[i] = (u16)(1 + i * 1); // shallowest -> weakest + medium[i] = (u16)(1 + i * 2); + } + + seedSeries(tracker, /*klass_id=*/1, strong, 20, /*start_epoch=*/1); + seedSeries(tracker, /*klass_id=*/2, weak, 20, /*start_epoch=*/1); + seedSeries(tracker, /*klass_id=*/3, medium, 20, /*start_epoch=*/1); + + KlassCandidate out[5]; + int count = tracker->selectLeakCandidates(out, 5); + + ASSERT_EQ(count, 3); + EXPECT_EQ(out[0].klass_id, 1u); // strongest + EXPECT_EQ(out[1].klass_id, 3u); // middle + EXPECT_EQ(out[2].klass_id, 2u); // weakest +} + +// More than MAX_LEAK_CANDIDATES (5) positive-slope klasses exist: only the +// top 5 by magnitude are returned, even though the caller asked for more - +// design doc's "top 3-5" cutoff is an upper bound the method itself enforces, +// not just a suggestion to the caller. +TEST_F(SelectLeakCandidatesTest, CapsAtMaxLeakCandidatesRegardlessOfRequestedMax) { + LivenessTracker *tracker = LivenessTracker::instance(); + + // 7 klasses, each growing by a distinct amount per sample so every one + // has a distinct, positive slope: klass_id N grows by N per sample. 20 + // samples (not 10 - see GrowingPopulationIsSelected's own note) so every + // klass also clears the sustained-trend hysteresis requirement. + for (u32 klass_id = 1; klass_id <= 7; klass_id++) { + u16 series[20]; + for (int i = 0; i < 20; i++) { + series[i] = (u16)(1 + i * klass_id); + } + seedSeries(tracker, klass_id, series, 20, /*start_epoch=*/1); + } + + KlassCandidate out[10]; + int count = tracker->selectLeakCandidates(out, 10); + + ASSERT_EQ(count, 5); // MAX_LEAK_CANDIDATES, not the requested 10 + // Steeper growth (larger klass_id) means larger slope - the 5 returned + // must be the 5 largest klass_ids, strongest first. + EXPECT_EQ(out[0].klass_id, 7u); + EXPECT_EQ(out[1].klass_id, 6u); + EXPECT_EQ(out[2].klass_id, 5u); + EXPECT_EQ(out[3].klass_id, 4u); + EXPECT_EQ(out[4].klass_id, 3u); +} + +// The caller's own buffer capacity (`max`) is honored when it is smaller +// than MAX_LEAK_CANDIDATES - the method must never write past `max` slots. +TEST_F(SelectLeakCandidatesTest, HonorsCallerSuppliedMaxBelowCap) { + LivenessTracker *tracker = LivenessTracker::instance(); + + for (u32 klass_id = 1; klass_id <= 3; klass_id++) { + u16 series[20]; + for (int i = 0; i < 20; i++) { + series[i] = (u16)(1 + i * klass_id); + } + seedSeries(tracker, klass_id, series, 20, /*start_epoch=*/1); + } + + KlassCandidate out[2]; + int count = tracker->selectLeakCandidates(out, 2); + + ASSERT_EQ(count, 2); + EXPECT_EQ(out[0].klass_id, 3u); // strongest + EXPECT_EQ(out[1].klass_id, 2u); // second-strongest; klass 1 dropped +} + +// An empty population table (nothing tracked yet, or _gc_generations was +// never enabled so population tracking's own gate left the table empty) yields no +// candidates regardless of `max` - no separate guard is needed inside +// selectLeakCandidates() beyond the table being empty. +TEST_F(SelectLeakCandidatesTest, EmptyTableReturnsZero) { + LivenessTracker *tracker = LivenessTracker::instance(); + + KlassCandidate out[5]; + int count = tracker->selectLeakCandidates(out, 5); + + EXPECT_EQ(count, 0); +} + +// --------------------------------------------------------------------------- +// topKlassesByGenerationCount() - ranks by most-recent count_ring sample, +// with NO trend/hysteresis gate at all (unlike selectLeakCandidates() above) +// - see its own header comment (livenessTracker.h) for why: it exists to run +// AFTER hasLeakSignal() has already fired via the slower, hysteresis-gated +// path, as a faster follow-up ranking for ReferenceChainTracker's rotation +// priority. Reuses SelectLeakCandidatesTest's fixture/seedSeries() seam - +// same table, same seeding mechanism, different read method under test. +// +// Returns stable_class_tag, NOT klass_id (the classMap dictionary id) - see +// that field's own comment (livenessTracker.h) for why the two are +// deliberately different values. klassPopulationRecordForTest() (seedSeries()'s +// own underlying seam) bypasses foldKlassCountsLocked() entirely, so it never +// mints a stable_class_tag - tests seed it explicitly via +// klassPopulationSetStableClassTagForTest(), using a value distinct from +// klass_id in each test below specifically so a test that accidentally +// asserted against klass_id instead would fail loudly, not silently pass by +// coincidence. +// --------------------------------------------------------------------------- + +// A single sample (ring_fill == 1, far below selectLeakCandidates()'s +// KLASS_POPULATION_MIN_FILL_FOR_TREND) is enough to rank - the whole point +// of skipping the hysteresis gate. +TEST_F(SelectLeakCandidatesTest, TopKlassesByGenerationCountNeedsOnlyOneSample) { + LivenessTracker *tracker = LivenessTracker::instance(); + + const u16 single[1] = {42}; + seedSeries(tracker, /*klass_id=*/7, single, 1, /*start_epoch=*/1); + tracker->klassPopulationSetStableClassTagForTest(7, -700); + + u32 out[5]; + int count = tracker->topKlassesByGenerationCount(out, 5); + + ASSERT_EQ(count, 1); + EXPECT_EQ(out[0], (u32)-700); +} + +// A klass with samples but no minted stable_class_tag yet (no live instance +// resolved so far - foldKlassCountsLocked()'s own comment) has nothing +// usable to return and must be skipped, not reported with a bogus 0 tag. +TEST_F(SelectLeakCandidatesTest, TopKlassesByGenerationCountSkipsUnmintedStableClassTag) { + LivenessTracker *tracker = LivenessTracker::instance(); + + const u16 single[1] = {42}; + seedSeries(tracker, /*klass_id=*/7, single, 1, /*start_epoch=*/1); + // Deliberately no klassPopulationSetStableClassTagForTest() call. + + u32 out[5]; + int count = tracker->topKlassesByGenerationCount(out, 5); + + EXPECT_EQ(count, 0); +} + +// Ranking is by the MOST RECENT sample, not the peak or the mean - a klass +// whose count has since fallen still ranks by where it is NOW. +TEST_F(SelectLeakCandidatesTest, TopKlassesByGenerationCountUsesMostRecentSampleNotPeak) { + LivenessTracker *tracker = LivenessTracker::instance(); + + const u16 peakedThenFell[4] = {100, 5, 5, 5}; // peak 100, now 5 + const u16 steady[4] = {10, 10, 10, 10}; // never peaked, now 10 + seedSeries(tracker, /*klass_id=*/1, peakedThenFell, 4, /*start_epoch=*/1); + seedSeries(tracker, /*klass_id=*/2, steady, 4, /*start_epoch=*/1); + tracker->klassPopulationSetStableClassTagForTest(1, -100); + tracker->klassPopulationSetStableClassTagForTest(2, -200); + + u32 out[5]; + int count = tracker->topKlassesByGenerationCount(out, 5); + + ASSERT_EQ(count, 2); + EXPECT_EQ(out[0], (u32)-200) << "klass 2 (currently 10) should outrank " + "klass 1 (currently 5, despite an " + "earlier peak of 100)"; + EXPECT_EQ(out[1], (u32)-100); +} + +// A flat or shrinking population - which selectLeakCandidates() would +// exclude entirely (zero/negative slope) - still ranks here: this method +// applies no growth-direction requirement, only magnitude. +TEST_F(SelectLeakCandidatesTest, TopKlassesByGenerationCountIncludesFlatAndShrinkingPopulations) { + LivenessTracker *tracker = LivenessTracker::instance(); + + const u16 flat[3] = {50, 50, 50}; + const u16 shrinking[3] = {30, 20, 10}; + seedSeries(tracker, /*klass_id=*/1, flat, 3, /*start_epoch=*/1); + seedSeries(tracker, /*klass_id=*/2, shrinking, 3, /*start_epoch=*/1); + tracker->klassPopulationSetStableClassTagForTest(1, -100); + tracker->klassPopulationSetStableClassTagForTest(2, -200); + + u32 out[5]; + int count = tracker->topKlassesByGenerationCount(out, 5); + + ASSERT_EQ(count, 2); + EXPECT_EQ(out[0], (u32)-100) << "flat-at-50 outranks shrinking-to-10"; + EXPECT_EQ(out[1], (u32)-200); +} + +TEST_F(SelectLeakCandidatesTest, TopKlassesByGenerationCountCapsAtMaxLeakCandidates) { + LivenessTracker *tracker = LivenessTracker::instance(); + + for (u32 klass_id = 1; klass_id <= 8; klass_id++) { + const u16 sample[1] = {(u16)(klass_id * 10)}; + seedSeries(tracker, klass_id, sample, 1, /*start_epoch=*/1); + tracker->klassPopulationSetStableClassTagForTest(klass_id, -(jlong)(klass_id * 100)); + } + + u32 out[10]; + int count = tracker->topKlassesByGenerationCount(out, 10); + + EXPECT_EQ(count, 5) << "capped at MAX_LEAK_CANDIDATES regardless of the " + "caller-supplied max"; + // Descending by most-recent count: klass 8 (count 80, tag -800) first. + EXPECT_EQ(out[0], (u32)-800); + EXPECT_EQ(out[4], (u32)-400); +} + +TEST_F(SelectLeakCandidatesTest, TopKlassesByGenerationCountEmptyTableReturnsZero) { + LivenessTracker *tracker = LivenessTracker::instance(); + + u32 out[5]; + int count = tracker->topKlassesByGenerationCount(out, 5); + + EXPECT_EQ(count, 0); +} + +// --------------------------------------------------------------------------- +// Heap-wide time-to-OOM projection (secondsToOOM()) - the aggressive-leak gap +// selectLeakCandidates()'s per-klass ring-fill/hysteresis gate leaves open: +// that gate can take longer to trust a candidate than a fast, heap-wide leak +// has left before OOM (see ReferenceChainTracker::hasLeakSignal()'s +// OOM_URGENT_THRESHOLD_S fast path, referenceChains.h/.cpp). Exercises the +// heap-floor ring/time-ring pair directly via the same heapFloorRecordForTest() +// seam SelectLeakCandidatesTest's HeapFloorCorroboration test above already +// uses for heapFloorRising(), plus setMaxHeapBytesForTest() to avoid the +// JNI-dependent HeapUsage::getMaxHeap() call this suite has no live JVM for. +// --------------------------------------------------------------------------- +class SecondsToOOMTest : public ::testing::Test { +protected: + static constexpr u64 SEC_NS = 1000000000ULL; + static constexpr u64 MiB = 1ULL << 20; + + void SetUp() override { + installGtestCrashHandler(); + LivenessTracker::instance()->klassPopulationResetForTest(); + LivenessTracker::instance()->setGcGenerationsForTest(true); + } + + void TearDown() override { + LivenessTracker::instance()->klassPopulationResetForTest(); + LivenessTracker::instance()->setGcGenerationsForTest(false); + LivenessTracker::instance()->setMaxHeapBytesForTest(-1); + restoreDefaultSignalHandlers(); + } + + // Ten samples, one second apart, growing by 100MiB each: earliest third + // (indices 0-2) means to 1100MiB at t=1s, recent third (indices 7-9) + // means to 1800MiB at t=8s - a 700MiB rise over 7s, i.e. exactly + // 100MiB/s, chosen so the projected time-to-exhaustion below comes out + // to a clean value rather than a value only checked against itself. + static void seedRisingFloor(LivenessTracker *tracker) { + for (int i = 0; i < 10; i++) { + tracker->heapFloorRecordForTest(1000 * MiB + (u64)i * 100 * MiB, + (u64)i * SEC_NS); + } + } +}; + +// Fewer than KLASS_POPULATION_MIN_FILL_FOR_TREND (10) heap-floor samples - +// same "not enough history yet" gate ringThirdsStats() applies to every +// other trend check in this class. +TEST_F(SecondsToOOMTest, NotEnoughSamplesReturnsNegative) { + LivenessTracker *tracker = LivenessTracker::instance(); + tracker->setMaxHeapBytesForTest((jlong)(2800 * MiB)); + for (int i = 0; i < 9; i++) { + tracker->heapFloorRecordForTest(1000 * MiB + (u64)i * 100 * MiB, + (u64)i * SEC_NS); + } + EXPECT_LT(tracker->secondsToOOM(), 0.0); +} + +// A flat floor (zero byte delta between the earliest and recent thirds) is +// not rising - no projection is offered, mirroring hasQualifyingGrowth()'s +// own "strictly positive slope" requirement. +TEST_F(SecondsToOOMTest, FlatFloorReturnsNegative) { + LivenessTracker *tracker = LivenessTracker::instance(); + tracker->setMaxHeapBytesForTest((jlong)(2800 * MiB)); + for (int i = 0; i < 10; i++) { + tracker->heapFloorRecordForTest(1000 * MiB, (u64)i * SEC_NS); + } + EXPECT_LT(tracker->secondsToOOM(), 0.0); +} + +// No heap-floor history is ever recorded outside _gc_generations (onGC()'s +// own gate) - secondsToOOM() must not fabricate a projection from whatever +// ring contents happen to be left over from a previous _gc_generations +// session. +TEST_F(SecondsToOOMTest, GcGenerationsDisabledReturnsNegative) { + LivenessTracker *tracker = LivenessTracker::instance(); + tracker->setMaxHeapBytesForTest((jlong)(2800 * MiB)); + seedRisingFloor(tracker); + tracker->setGcGenerationsForTest(false); + + EXPECT_LT(tracker->secondsToOOM(), 0.0); +} + +// A rising floor is meaningless without a resolved max heap size to project +// against - initialize_table()'s own Error path (livenessTracker.cpp) never +// lets liveness tracking start without one, but secondsToOOM() must still +// guard the case explicitly rather than dividing/comparing against -1. +TEST_F(SecondsToOOMTest, UnresolvedMaxHeapReturnsNegative) { + LivenessTracker *tracker = LivenessTracker::instance(); + tracker->setMaxHeapBytesForTest(-1); + seedRisingFloor(tracker); + + EXPECT_LT(tracker->secondsToOOM(), 0.0); +} + +// The worked example seedRisingFloor() documents: 700MiB rise over 7s +// (100MiB/s) with 1000MiB of headroom (2800MiB max heap - 1800MiB recent +// floor mean) projects to exactly 10 seconds. +TEST_F(SecondsToOOMTest, RisingFloorProjectsExpectedSeconds) { + LivenessTracker *tracker = LivenessTracker::instance(); + tracker->setMaxHeapBytesForTest((jlong)(2800 * MiB)); + seedRisingFloor(tracker); + + EXPECT_NEAR(tracker->secondsToOOM(), 9.0, 1e-6); +} + +// The floor's own recent-third mean has already reached the max heap size - +// exhaustion is "now", not some positive number of seconds out. +TEST_F(SecondsToOOMTest, FloorAtMaxHeapReturnsZero) { + LivenessTracker *tracker = LivenessTracker::instance(); + tracker->setMaxHeapBytesForTest((jlong)(1800 * MiB)); // == recent third's mean + seedRisingFloor(tracker); + + EXPECT_EQ(tracker->secondsToOOM(), 0.0); +} + +// --------------------------------------------------------------------------- +// Leak tag pool (design A: direct tagging of leaking objects) +// --------------------------------------------------------------------------- +// The pool hands out JVMTI tags in [LEAK_TAG_BASE, LEAK_TAG_BASE + 256) and +// recycles them when the tracked object dies. These tests exercise the pure +// pool mechanics (acquire/release/info); tagLeakInstances() itself needs a +// live JVM (SetTag) and is verified on-pod. + +class LeakTagPoolTest : public ::testing::Test { +protected: + void SetUp() override { + LivenessTracker::instance()->leakTagPoolResetForTest(); + } +}; + +TEST_F(LeakTagPoolTest, AcquireReturnsTagsInLeakRange) { + LivenessTracker *tracker = LivenessTracker::instance(); + jlong base = tracker->leakTagBaseForTest(); + for (int i = 0; i < tracker->leakTagPoolSizeForTest(); i++) { + jlong tag = tracker->acquireLeakTagForTest(/*call_trace_id=*/100 + i, + /*tid=*/7 + i); + EXPECT_GE(tag, base) << "tag below pool range at acquire " << i; + EXPECT_LT(tag, base + tracker->leakTagPoolSizeForTest()) + << "tag above pool range at acquire " << i; + } + // Pool exhausted: further acquires fail with 0. + EXPECT_EQ(0, tracker->acquireLeakTagForTest(1, 1)); + EXPECT_EQ(0, tracker->leakTagFreeCountForTest()); +} + +TEST_F(LeakTagPoolTest, ReleaseReturnsTagToPoolAndInfoIsInvalidated) { + LivenessTracker *tracker = LivenessTracker::instance(); + int pool_size = tracker->leakTagPoolSizeForTest(); + + jlong tag = tracker->acquireLeakTagForTest(42, 99); + ASSERT_GE(tag, tracker->leakTagBaseForTest()); + + u64 call_trace_id = 0; + jint tid = 0; + EXPECT_TRUE(tracker->getLeakTagInfo(tag, &call_trace_id, &tid)); + EXPECT_EQ(42u, call_trace_id); + EXPECT_EQ(99, tid); + + tracker->releaseLeakTagForTest(tag); + // Released tags must not report stale info. + u64 stale_ctid = 12345; + jint stale_tid = 12345; + EXPECT_FALSE(tracker->getLeakTagInfo(tag, &stale_ctid, &stale_tid)) + << "released tag still reports info"; + + // The released tag can be acquired again (reusable pool), and the free + // count was restored: pool_size-1 after the acquire, back to pool_size + // after the release, pool_size-1 again after the re-acquire. + EXPECT_EQ(pool_size, tracker->leakTagFreeCountForTest()); + jlong re_tag = tracker->acquireLeakTagForTest(43, 100); + EXPECT_EQ(tag, re_tag) << "released tag should be recycled first (LIFO)"; + EXPECT_EQ(pool_size - 1, tracker->leakTagFreeCountForTest()); +} + +TEST_F(LeakTagPoolTest, ReleaseOutsidePoolRangeIsIgnored) { + LivenessTracker *tracker = LivenessTracker::instance(); + jlong base = tracker->leakTagBaseForTest(); + int free_before = tracker->leakTagFreeCountForTest(); + + // Tags outside [base, base+pool): frontier tags (small positive), class + // tags (negative), and one-past-the-end must all be rejected. + tracker->releaseLeakTagForTest(1); + tracker->releaseLeakTagForTest(-1); + tracker->releaseLeakTagForTest(0); + tracker->releaseLeakTagForTest(base + tracker->leakTagPoolSizeForTest()); + tracker->releaseLeakTagForTest(base - 1); + + EXPECT_EQ(free_before, tracker->leakTagFreeCountForTest()) + << "out-of-range releases must not corrupt the free list"; +} + +// --------------------------------------------------------------------------- +// Chase-phase admission boost (admitForTracking()/noteSelectedCandidates()/ +// setUrgentTracking() - see livenessTracker.h). Same "exercise the singleton +// directly" rationale as KlassPopulationTest above: the admission gate is +// JNI-free pure logic (atomic reads + the per-thread RNG draw), so it is +// testable without a live JVM; only track() beyond the gate needs a JNIEnv. +// +// Determinism: admissionResetForTest() forces _subsample_ratio to 0 and resets +// this thread's RNG ThreadLocal, so an unboosted tid's draw always comes from +// a freshly default-seeded mt19937 whose first draw is strictly in (0,1) - +// ratio 0 can never admit. That pins the fall-through case without asserting +// on RNG internals. +class AdmissionBoostTest : public ::testing::Test { +protected: + void SetUp() override { + installGtestCrashHandler(); + LivenessTracker::instance()->admissionResetForTest(); + } + + void TearDown() override { + // Leave the singleton clean for any test that follows in the same + // process (watched tids / urgency would 100%-admit unrelated tids). + LivenessTracker::instance()->admissionResetForTest(); + LivenessTracker::instance()->setSubsampleRatioForTest(0.1); + restoreDefaultSignalHandlers(); + } + + static void fillCandidate(KlassCandidate *kc, jint tid) { + kc->klass_id = 1; + kc->representative = reinterpret_cast(0x1); + kc->qualifying_tids[0] = tid; + kc->qualifying_tid_count = 1; + } +}; + +// A watched tid is admitted even though the configured ratio (0) would +// deterministically reject it - the boost precedes the ratio draw. +TEST_F(AdmissionBoostTest, WatchedTidAdmittedDespiteRejectingRatio) { + LivenessTracker *tracker = LivenessTracker::instance(); + KlassCandidate kc; + fillCandidate(&kc, /*tid=*/42); + tracker->noteSelectedCandidates(&kc, 1); + + EXPECT_TRUE(tracker->admitForTrackingForTest(42)); + // An unwatched tid on the same thread falls through to the ratio draw and + // is rejected (ratio 0). + EXPECT_FALSE(tracker->admitForTrackingForTest(43)); +} + +// Urgency admits everything - any tid, watched or not. +TEST_F(AdmissionBoostTest, UrgencyAdmitsAllTids) { + LivenessTracker *tracker = LivenessTracker::instance(); + tracker->setUrgentTracking(true); + EXPECT_TRUE(tracker->admitForTrackingForTest(1234)); + EXPECT_TRUE(tracker->admitForTrackingForTest(5678)); + + // Releasing urgency restores the ratio gate: unwatched tids reject again, + // watched tids stay boosted. + tracker->setUrgentTracking(false); + EXPECT_FALSE(tracker->admitForTrackingForTest(1234)); + KlassCandidate kc; + fillCandidate(&kc, 1234); + tracker->noteSelectedCandidates(&kc, 1); + EXPECT_TRUE(tracker->admitForTrackingForTest(1234)); +} + +// noteSelectedCandidates() dedupes tids shared across candidates and caps +// the watched set at MAX_QUALIFYING_TIDS. +TEST_F(AdmissionBoostTest, WatchedTidsAreDedupedAndCapped) { + LivenessTracker *tracker = LivenessTracker::instance(); + constexpr int kMax = KlassCandidate::MAX_QUALIFYING_TIDS; // 8 + + // Two candidates: 5 tids each, tids 3 and 4 shared -> union is 1..7, in + // candidate order (candidates are rank-ordered, so a later candidate's + // tids only append what the earlier ones did not cover). + KlassCandidate kc[2]; + for (int t = 0; t < 5; t++) { + kc[0].qualifying_tids[t] = 1 + t; // 1..5 + kc[1].qualifying_tids[t] = 3 + t; // 3..7 -> adds 6,7 + } + for (int c = 0; c < 2; c++) { + kc[c].klass_id = 1 + c; + kc[c].representative = reinterpret_cast(0x1 + c); + kc[c].qualifying_tid_count = 5; + } + tracker->noteSelectedCandidates(kc, 2); + ASSERT_EQ(7, tracker->watchedTidCountForTest()); + for (int i = 0; i < 7; i++) { + EXPECT_EQ(i + 1, tracker->watchedTidForTest(i)) + << "shared tids must not duplicate; union stays in order"; + } + + // A poll whose union exceeds the cap (1..8 from candidate 1, 9 from + // candidate 2) keeps the earlier candidates' tids and drops the overflow + // tid - and that overflow tid is not admitted. + KlassCandidate over[2]; + for (int t = 0; t < kMax; t++) { + over[0].qualifying_tids[t] = 1 + t; // 1..8 + } + over[0].qualifying_tid_count = kMax; + over[0].klass_id = 1; + over[0].representative = reinterpret_cast(0x1); + fillCandidate(&over[1], /*tid=*/9); + tracker->noteSelectedCandidates(over, 2); + ASSERT_EQ(kMax, tracker->watchedTidCountForTest()); + for (int i = 0; i < kMax; i++) { + EXPECT_EQ(i + 1, tracker->watchedTidForTest(i)); + } + EXPECT_FALSE(tracker->admitForTrackingForTest(9)); + EXPECT_TRUE(tracker->admitForTrackingForTest(8)); +} + +// A zero-candidate poll clears the watched set - a tid left watched after the +// chase ends would keep admitting that thread at 100% across OS tid reuse. +TEST_F(AdmissionBoostTest, ZeroCandidatePollClearsWatchedSet) { + LivenessTracker *tracker = LivenessTracker::instance(); + KlassCandidate kc; + fillCandidate(&kc, 77); + tracker->noteSelectedCandidates(&kc, 1); + EXPECT_TRUE(tracker->admitForTrackingForTest(77)); + + tracker->noteSelectedCandidates(nullptr, 0); + EXPECT_EQ(tracker->watchedTidCountForTest(), 0); + EXPECT_FALSE(tracker->admitForTrackingForTest(77)); +} diff --git a/ddprof-lib/src/test/cpp/referenceChainsAnchorTests.inc b/ddprof-lib/src/test/cpp/referenceChainsAnchorTests.inc new file mode 100644 index 000000000..af0dbdf43 --- /dev/null +++ b/ddprof-lib/src/test/cpp/referenceChainsAnchorTests.inc @@ -0,0 +1,820 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +TEST_F(ReferenceChainsBfsTest, StaticAnchorRotationWalksRootAttachedStaticHolders) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + void *holderCls = (void *)0x4001, *chunkCls = (void *)0x4002; + addClass(holderCls, "Lcom/rc/descendwalk/StaticHolder;"); + int chunk = addClass(chunkCls, "Lcom/rc/descendwalk/StaticChunk;"); + + int holderNode = addNode(); + int tableNode = addNode(); + int entryNode = addNode(); + int leakChunk = addNode(); + const jlong leak_tag = ReferenceChainsTestAccessor::leakTagBase(); + node_tags[leakChunk] = leak_tag; + + // Static Map -> table -> Entry -> chunk: the collection-shaped static holder's internals, + // deeper than one hop. + script = { + {JVMTI_HEAP_REFERENCE_FIELD, holderNode, tableNode, -1}, + {JVMTI_HEAP_REFERENCE_ARRAY_ELEMENT, tableNode, entryNode, -1}, + {JVMTI_HEAP_REFERENCE_FIELD, entryNode, leakChunk, chunk}, + }; + + // Seed the frontier exactly as the static sweep admits a static field's value: root-attached, + // STATIC_FIELD root kind, FRONTIER state. + FrontierTable *frontier = tracker->frontierTable(); + node_tags[holderNode] = 101; // mock_GetObjectsWithTags' resolvable tag + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 101, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + ReferenceChainsTestAccessor::addToStaticAnchorIndexForTest( + 101, 9001, JVMTI_HEAP_REFERENCE_STATIC_FIELD); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 102, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_STACK_LOCAL)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 103, 101, 1, FrontierEntryState::FRONTIER, 0)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 104, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_JNI_GLOBAL)); + ReferenceChainsTestAccessor::addToStaticAnchorIndexForTest( + 104, 9004, JVMTI_HEAP_REFERENCE_JNI_GLOBAL); + + std::vector selected = + ReferenceChainsTestAccessor::collectStaticFieldAnchorsForRotationForTest( + 4); + // Both DURABLE root kinds are selected (STATIC_FIELD 101, JNI_GLOBAL 104, in cursor/tag order); + // the transient-root decoy and the child are not. + ASSERT_EQ(2u, selected.size()); + EXPECT_EQ(101, selected[0]); + EXPECT_EQ(104, selected[1]); + std::vector walk_selected = {selected[0]}; + + // The static anchor's whole internal structure is admitted by one bounded walk, intercepting + // the leak tag at depth 3 below the holder. + int edges = 0; + ReferenceChainsTestAccessor::walkStaticFieldAnchorsForTest( + &mock_jvmti, &mock_jni, walk_selected, 1000, &edges); + jlong table_ftag = tags_ever_assigned[tableNode]; + jlong entry_ftag = tags_ever_assigned[entryNode]; + jlong chunk_ftag = tags_ever_assigned[leakChunk]; + ASSERT_GT(table_ftag, 0) << "table array was not reached by the anchor walk"; + ASSERT_GT(entry_ftag, 0) << "Entry was not reached one hop below table"; + ASSERT_NE(chunk_ftag, leak_tag) + << "leak-tagged chunk inside the static holder was never intercepted"; + EXPECT_EQ(leak_tag, ReferenceChainsTestAccessor::frontierLeakTag(chunk_ftag)); + FrontierEntry chunk_entry{}; + ASSERT_TRUE(frontier->lookup(chunk_ftag, &chunk_entry)); + EXPECT_EQ(entry_ftag, chunk_entry.parent_tag); + EXPECT_EQ(3u, chunk_entry.depth); + + tracker->stop(); +} + +// Round-14 tiered selection: a container-shaped anchor (its own class implements Collection/Map - +// the LEAK_BUFFER wrapper shape) admitted at a LATE index position must leap the queue of ~28k +// other-tier anchors (the round-13 hotdog measurement: admission-order selection put the leak +// holder at position ~12-21k against ~4k walk coverage per search - deterministically unreachable). +TEST_F(ReferenceChainsBfsTest, ContainerAnchorLeapsQueueAcrossLargeIndex) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + constexpr int kAnchorCount = 28000; + for (int i = 0; i < kAnchorCount; i++) { + jlong tag = 100 + i; + jlong class_tag = 500000 + i; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, tag, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + ReferenceChainsTestAccessor::addToStaticAnchorIndexForTest( + tag, class_tag, JVMTI_HEAP_REFERENCE_STATIC_FIELD); + // Containers only at late positions 24000..24003 - buried behind 24k other-tier anchors + // under any admission-order cursor. + bool container = (i >= 24000 && i <= 24003); + ReferenceChainsTestAccessor::primeClassShapeForTest(class_tag, + container); + } + // One leak-tagged anchor at the very tail - tier 0, must lead. + jlong leak_anchor = 100 + kAnchorCount; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, leak_anchor, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + frontier->setLeakTag(leak_anchor, 777); + ReferenceChainsTestAccessor::addToStaticAnchorIndexForTest( + leak_anchor, 599999, JVMTI_HEAP_REFERENCE_STATIC_FIELD); + ReferenceChainsTestAccessor::primeClassShapeForTest( + 599999, false /* its tier comes from leak_tag, not shape */); + + std::vector selected = + ReferenceChainsTestAccessor::collectStaticFieldAnchorsForRotationForTest( + 16); + ASSERT_EQ(16u, selected.size()) + << "28k+5 eligible anchors at a 16 budget must fill the selection"; + EXPECT_EQ(leak_anchor, selected[0]) + << "the leak-tagged anchor (tier 0) must lead the walk order"; + // The four container anchors are selected in this FIRST call despite positions 24000+ (tags + // 24100-24103). + for (jlong t : {24100, 24101, 24102, 24103}) { + EXPECT_NE(std::find(selected.begin(), selected.end(), t), selected.end()) + << "container anchor " << t + << " did not leap the other-tier queue"; + } + // Sanity: an early other-tier anchor also made the cut (cursor-fair fill from position 0). + EXPECT_NE(std::find(selected.begin(), selected.end(), (jlong)100), + selected.end()); + + tracker->stop(); +} + +// Round-15 fresh-admission priority: the hotdog wrapper is admitted LATE in a search (its holder +// class sits at sweep index 24627 of 33270, so admission lands at the anchor-index TAIL) - behind +// the whole fair container backlog measured at 1633 entries against 44-75-pass search lifetimes +// (the fair container cursor would reach it at pass ~102+, after the search is dead). +TEST_F(ReferenceChainsBfsTest, FreshContainerWalkedBeforeFairBacklog) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + // A fair container backlog, "admitted long ago": 200 containers at positions 0..199. + constexpr int kBacklog = 200; + for (int i = 0; i < kBacklog; i++) { + jlong tag = 500 + i; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, tag, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + ReferenceChainsTestAccessor::addToStaticAnchorIndexForTest( + tag, 700000 + i, JVMTI_HEAP_REFERENCE_STATIC_FIELD); + ReferenceChainsTestAccessor::primeClassShapeForTest(700000 + i, true); + } + // First collector call: all 200 anchors are fresh (nothing has had a first look yet), the lane + // keeps 16 and the rest spend their first look (they fall back to the fair container tier). + std::vector first = + ReferenceChainsTestAccessor::collectStaticFieldAnchorsForRotationForTest( + 16); + ASSERT_EQ(16u, first.size()) << "200 eligible containers at a 16 budget"; + + // The late admission wave: a wrapper-class container at the index tail, a fresh classified + // NON-container (a fresh String static - it must NOT ride the fresh lane), and a fresh + // unclassified anchor (a genuinely new class - it must ride the lane, so the classification lag + // cannot lose the wrapper's fresh window). + const jlong wrapper_tag = 500 + kBacklog; + const jlong fresh_string_tag = 500 + kBacklog + 1; + const jlong fresh_unknown_tag = 500 + kBacklog + 2; + for (auto [tag, class_tag, container, prime] : + {std::make_tuple(wrapper_tag, (jlong)799001, true, true), + std::make_tuple(fresh_string_tag, (jlong)799002, false, true), + std::make_tuple(fresh_unknown_tag, (jlong)799003, false, + false /* deliberately unclassified */)}) { + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, tag, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + ReferenceChainsTestAccessor::addToStaticAnchorIndexForTest( + tag, class_tag, JVMTI_HEAP_REFERENCE_STATIC_FIELD); + if (prime) { + ReferenceChainsTestAccessor::primeClassShapeForTest(class_tag, + container); + } + } + + std::vector second = + ReferenceChainsTestAccessor::collectStaticFieldAnchorsForRotationForTest( + 16); + ASSERT_EQ(16u, second.size()); + EXPECT_EQ(wrapper_tag, second[0]) + << "the freshly admitted container must lead the walk order, not " + "wait behind the 184-container fair backlog"; + EXPECT_NE(std::find(second.begin(), second.end(), fresh_unknown_tag), + second.end()) + << "an unclassified fresh anchor must ride the fresh lane (the " + "wrapper admits a pass before reconcile classifies its class)"; + EXPECT_EQ(std::find(second.begin(), second.end(), fresh_string_tag), + second.end()) + << "a fresh NON-container stays in the other tier - the fresh lane " + "is the container lane, or fresh Strings would flood it"; + // The fair container lap still gets the leftover budget. Call 1's fresh picks (500-515) spent + // the whole budget, so the fair cursor never advanced - call 2's fair picks start at tag 500 + // again (a benign one-call overlap: walks are idempotent, and it only happens when fresh and + // fair coincide at a lap boundary). + EXPECT_NE(std::find(second.begin(), second.end(), (jlong)500), + second.end()) + << "the fair container lap must still advance with the leftover " + "budget"; + EXPECT_EQ(14, (int)std::count_if(second.begin(), second.end(), + [](jlong t) { + return t >= 500 && t < 500 + kBacklog; + })) + << "16 budget - 2 fresh picks = 14 fair-container picks"; + + // And the dropped fresh entries from call 1 (516-699 spent their first look) are still covered + // by the fair tier: a third call with no new admits must keep advancing the fair container + // cursor from wherever call 2 left it, not re-drain anything. + std::vector third = + ReferenceChainsTestAccessor::collectStaticFieldAnchorsForRotationForTest( + 16); + ASSERT_EQ(16u, third.size()); + EXPECT_EQ(std::find(third.begin(), third.end(), wrapper_tag), + third.end()) + << "the wrapper already had its first look - it must not be " + "re-selected while the fair cursor has 184 uncovered peers"; + + tracker->stop(); +} + +// Round-14 tier fairness: the other tier (everything not leak-tagged, not container-shaped) still +// reaches full coverage across wraps - a 40-anchor tier at a 16 budget covers all 40 in exactly 3 +// calls with no duplicates within a call. +TEST_F(ReferenceChainsBfsTest, AnchorOtherTierFairCoverageAcrossWraps) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + constexpr int kOtherCount = 40; + for (int i = 0; i < kOtherCount; i++) { + jlong tag = 200 + i; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, tag, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + ReferenceChainsTestAccessor::addToStaticAnchorIndexForTest( + tag, 300000 + i, JVMTI_HEAP_REFERENCE_STATIC_FIELD); + ReferenceChainsTestAccessor::primeClassShapeForTest(300000 + i, false); + } + + std::vector all_selected; + for (int call = 0; call < 3; call++) { + std::vector selected = + ReferenceChainsTestAccessor:: + collectStaticFieldAnchorsForRotationForTest(16); + int expected = (call == 2) ? 8 : 16; + ASSERT_EQ(expected, (int)selected.size()) + << "call " << call << " should select " << expected; + std::set dedup(selected.begin(), selected.end()); + ASSERT_EQ(selected.size(), dedup.size()) + << "no anchor may be selected twice within a call"; + all_selected.insert(all_selected.end(), selected.begin(), + selected.end()); + } + std::set covered(all_selected.begin(), all_selected.end()); + ASSERT_EQ(40u, covered.size()) << "full other-tier coverage expected"; + for (int i = 0; i < kOtherCount; i++) { + EXPECT_NE(covered.find(200 + i), covered.end()) + << "other-tier anchor " << 200 + i << " never selected"; + } + + tracker->stop(); +} + +// Round-13/14 restart hygiene: discovered-instance tags are FRONTIER tags; restartSearch() resets +// the frontier table and _next_tag=1, so any surviving discovered slot either fails +// reconstructChain (observed on-pod: 'buildChainEvent failed ... +TEST_F(ReferenceChainsBfsTest, RestartSearchClearsDiscoveredInstanceTags) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + // A watched candidate with one discovered instance recorded. + ReferenceChainsTestAccessor::setCandidateCountForTest(1); + ReferenceChainsTestAccessor::setCandidateKlassIdForTest(0, 7); + ReferenceChainsTestAccessor::recordDiscoveredInstanceForTest(7, 4242, + false); + ASSERT_EQ(4242, ReferenceChainsTestAccessor::discoveredTagForTest(0, 0)); + ASSERT_EQ(1, ReferenceChainsTestAccessor::discoveredCountForTest(0)); + + // An anchor index entry (tag + parallel class tag) to confirm the index reset covers the + // parallel structures too. + ReferenceChainsTestAccessor::addToStaticAnchorIndexForTest( + 4242, 555, JVMTI_HEAP_REFERENCE_STATIC_FIELD); + (void)frontier; + + ReferenceChainsTestAccessor::restartSearchForTest(); + + EXPECT_EQ(0, ReferenceChainsTestAccessor::discoveredTagForTest(0, 0)) + << "stale discovered frontier tag survived restartSearch()"; + EXPECT_EQ(0, ReferenceChainsTestAccessor::discoveredCountForTest(0)); + EXPECT_TRUE(ReferenceChainsTestAccessor::anchorIndexIsEmptyForTest()) + << "anchor index (or its parallel arrays) survived restartSearch()"; + + tracker->stop(); +} + +// Round-19 (pod 289f8 — three JVMs of "canary search, 0/1 candidates found" while leak-tagged +// instances WERE intercepted and chains re-emitted): the marker->leak-tag design migration never +// updated the found criterion. +TEST_F(ReferenceChainsBfsTest, LeakTagChainMarksCanaryFound) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + // Slot 0: candidate klass 7 whose discovered instance carries a leak tag (entry.leak_tag set — + // the shape a walk + tagLeakInstances correlation produces; target_tag becomes the leak tag). + ReferenceChainsTestAccessor::setCandidateCountForTest(2); + ReferenceChainsTestAccessor::setCandidateKlassIdForTest(0, 7); + ReferenceChainsTestAccessor::setCandidateKlassIdForTest(1, 9); + // depth=1 at a durable STATIC_FIELD root (root_kind 8) so the retention filter + // (suppressChainEvent: depth==0, or depth==1 at a transient root) keeps both chains. + ASSERT_TRUE(frontier->insert(4242, 0, 7, 1, FrontierEntryState::FRONTIER, + 8, /*class_tag=*/0, -1, /*kind=*/0)); + frontier->setLeakTag(4242, 1073742079); + ReferenceChainsTestAccessor::recordDiscoveredInstanceForTest(7, 4242, true); + ASSERT_EQ(4242, ReferenceChainsTestAccessor::candidateDiscoveredTagForTest(0, 0)); + ASSERT_TRUE(frontier->insert(5353, 0, 9, 1, FrontierEntryState::FRONTIER, + 8, 0, -1, 0)); + ReferenceChainsTestAccessor::recordDiscoveredInstanceForTest(9, 5353, false); + ASSERT_EQ(5353, ReferenceChainsTestAccessor::candidateDiscoveredTagForTest(1, 0)); + + ReferenceChainsTestAccessor::buildDiscoveredInstanceChainsForTest(7, 1); + ReferenceChainsTestAccessor::buildDiscoveredInstanceChainsForTest(9, 1); + + // Slot 0 found via its leak-tag chain (bit 0 set, link recorded); slot 1's noise chain did not + // mark anything found. + EXPECT_EQ(1ULL, ReferenceChainsTestAccessor::candidateFoundBitsForTest()); + EXPECT_EQ(4242, ReferenceChainsTestAccessor::candidateFrontierTagForTest(0)); + EXPECT_EQ(0, ReferenceChainsTestAccessor::candidateFrontierTagForTest(1)) + << "noise-target chain must not mark the canary found"; + + tracker->stop(); +} + +// B' push site 1 - DEMOTION TIME (find-anchor-holder-eviction / _static_anchor_fifo): when +// improveChain() replaces a root-attached durable (STATIC_FIELD/ JNI_GLOBAL) entry with a deeper +// chain-attached path, the entry is leaving the anchor tier's eligible population at exactly that +// moment - the push must fire right there. +TEST_F(ReferenceChainsBfsTest, DemotionPushFiresWhenImproveChainEvictsRootAttachedStatic) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int parentNode = addNode(); + int holderNode = addNode(); + + // The chain edge whose delivery demotes the holder. + script = { + {JVMTI_HEAP_REFERENCE_FIELD, parentNode, holderNode, -1}, + }; + + // Seed exactly the pre-demotion shape: holder root-attached STATIC (anchor-eligible), parent a + // root-attached frontier object whose expansion delivers the deeper chain edge. + FrontierTable *frontier = tracker->frontierTable(); + node_tags[holderNode] = 105; + node_tags[parentNode] = 104; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 105, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 104, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_STACK_LOCAL)); + ReferenceChainsTestAccessor::pushPendingExpandForTest(104); + + int edges = 0; + ReferenceChainsTestAccessor::expandFrontierForTest(&mock_jvmti, &mock_jni, + &edges); + // The chain edge was delivered: the holder's entry is now chain-attached (improveChain replaced + // the depth-0 root-attached admission), and the demotion pushed its tag into the at-risk FIFO. + FrontierEntry entry{}; + ASSERT_TRUE(frontier->lookup(105, &entry)); + EXPECT_EQ(104, entry.parent_tag); + EXPECT_EQ(1u, entry.depth); + ASSERT_EQ(1u, ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); + EXPECT_TRUE(ReferenceChainsTestAccessor::staticAnchorFifoContainsForTest(105)); + + // Re-walking the same edge must NOT push twice: improveChain refuses (new depth 1 is not > + // current 1), and the set dedupes regardless. + int edges2 = 0; + ReferenceChainsTestAccessor::pushPendingExpandForTest(104); + ReferenceChainsTestAccessor::expandFrontierForTest(&mock_jvmti, &mock_jni, + &edges2); + EXPECT_EQ(1u, ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); + + tracker->stop(); +} + +// B' push site 2 - SWEEP TIME: the static sweep's class->field edge onto an already-admitted +// CHAIN-ATTACHED entry (the admission-order eviction shape: born as a non-root child, never +// root-attached at all) must feed the FIFO. +TEST_F(ReferenceChainsBfsTest, SweepPushFiresOnStaticEdgeOntoChainAttachedHolder) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=100")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int classNode = addNode(); + int holderNode = addNode(); + // A child reachable ONLY from the holder: nothing in the scripted graph walks holderNode except + // the FIFO-drained anchor walk, so the child's admission after runPass is the end-to-end + // evidence that the sweep pushed the holder, the rotation phase drained it, and + // walkStaticFieldAnchors walked it. + int holderChildNode = addNode(); + addClass((void *)&node_tags[classNode], "Lcom/rc/statics/ChainBornHolder;"); + + script = { + // Only the sweep's static edge onto the holder, plus the holder's own child edge for the + // anchor walk to admit: nothing else reaches holderNode or holderChildNode, so the only + // possible push is the static-edge-onto-chain-attached site. + {JVMTI_HEAP_REFERENCE_STATIC_FIELD, classNode, holderNode, -1}, + {JVMTI_HEAP_REFERENCE_FIELD, holderNode, holderChildNode, -1}, + }; + + // Seed the born-chain-attached shape the eviction leaves: holder already a non-root child + // (parent 104, depth 1). + FrontierTable *frontier = tracker->frontierTable(); + node_tags[holderNode] = 105; + // The child is untagged (0): the anchor walk's admission assigns it a fresh frontier tag, + // observable via node_tags after the pass. + ASSERT_EQ(0, node_tags[holderChildNode]); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 105, 104, 1, FrontierEntryState::FRONTIER, 0)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 104, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_STACK_LOCAL)); + + bool truncated = true; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + + // The rotation phase of the same pass drained the pushed tag into the anchor walk, and the walk + // admitted the holder's child - the push itself left no residue in the FIFO (drained empty) and + // never re-attributed the holder's entry (re-rooting is the documented refusal that motivated + // the FIFO in the first place). + EXPECT_EQ(0u, ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); + jlong childTag = tags_ever_assigned[holderChildNode]; + ASSERT_GT(childTag, 0) << "the FIFO-drained anchor walk never admitted " + "the holder's child"; + FrontierEntry childEntry{}; + ASSERT_TRUE(frontier->lookup(childTag, &childEntry)); + EXPECT_EQ(105, childEntry.parent_tag); + EXPECT_EQ(2u, childEntry.depth); + // The holder's entry is untouched by the push - the B' feed records the at-risk shape, it never + // re-attributes the entry. + FrontierEntry entry{}; + ASSERT_TRUE(frontier->lookup(105, &entry)); + EXPECT_EQ(104, entry.parent_tag); + EXPECT_EQ(1u, entry.depth); + + tracker->stop(); +} + +// B' mechanics: a chain-attached holder drained from the at-risk FIFO is descend-walked and +// intercepts a leak chunk 3 hops below it - the repair for the population the root-attached +// collector demonstrably cannot select (the negative control below). +TEST_F(ReferenceChainsBfsTest, AtRiskAnchorFifoDrainAndWalkIntercept) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + void *holderCls = (void *)0x6001, *chunkCls = (void *)0x6002; + addClass(holderCls, "Lcom/rc/descendwalk/ChainAttachedHolder;"); + int chunk = addClass(chunkCls, "Lcom/rc/descendwalk/ChainChunk;"); + + int parentNode = addNode(); + int holderNode = addNode(); + int tableNode = addNode(); + int entryNode = addNode(); + int leakChunk = addNode(); + const jlong leak_tag = ReferenceChainsTestAccessor::leakTagBase(); + node_tags[leakChunk] = leak_tag; + + // Chain: parent -> holder -> table -> entry -> leak chunk. + script = { + {JVMTI_HEAP_REFERENCE_FIELD, holderNode, tableNode, -1}, + {JVMTI_HEAP_REFERENCE_ARRAY_ELEMENT, tableNode, entryNode, -1}, + {JVMTI_HEAP_REFERENCE_FIELD, entryNode, leakChunk, chunk}, + }; + + // Seed the holder exactly as the eviction leaves it: chain-attached (parent_tag = 104, depth 1, + // no root_kind). + FrontierTable *frontier = tracker->frontierTable(); + node_tags[holderNode] = 105; + // root_kind = 0: the entry is chain-attached, and a non-root entry's edge kind is not recorded + // (FrontierEntry::root_kind's own comment). + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 105, 104, 1, FrontierEntryState::FRONTIER, 0)); + // The chain's root: TRANSIENT (stack local), so the collector's durable root-kind filter skips + // it too - the whole table is un-selectable. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 104, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_STACK_LOCAL)); + + // Negative control: the root-attached collector selects NOTHING from a table holding only a + // chain-attached holder and a transient root - the pre-B' behavior that stranded the + // dual-reachable population. + std::vector selected = + ReferenceChainsTestAccessor::collectStaticFieldAnchorsForRotationForTest(4); + ASSERT_TRUE(selected.empty()); + + // B': push + drain + walk reaches what the collector cannot. + ReferenceChainsTestAccessor::pushStaticAnchorFifoForTest(105, 28366); + std::vector drained; + // 16 = ReferenceChainTracker::STATIC_ANCHOR_FIFO_DRAIN (private), the same per-pass drain cap + // runPassManualWalk() uses. + ASSERT_EQ(1, ReferenceChainsTestAccessor::drainStaticAnchorFifoForTest( + 16, drained)); + ASSERT_EQ(1u, drained.size()); + EXPECT_EQ(105, drained[0].tag); + EXPECT_EQ(28366u, drained[0].klass_id); + EXPECT_EQ(0u, ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); + + int edges = 0; + std::vector drained_tags; + for (const auto &at_risk : drained) { + drained_tags.push_back(at_risk.tag); + } + ReferenceChainsTestAccessor::walkStaticAnchorFifoForTest( + &mock_jvmti, &mock_jni, drained_tags, 1000, &edges, nullptr); + jlong table_ftag = tags_ever_assigned[tableNode]; + jlong entry_ftag = tags_ever_assigned[entryNode]; + jlong chunk_ftag = tags_ever_assigned[leakChunk]; + ASSERT_GT(table_ftag, 0) << "table array was not reached by the anchor walk"; + ASSERT_GT(entry_ftag, 0) << "Entry was not reached one hop below table"; + ASSERT_NE(chunk_ftag, leak_tag) + << "leak-tagged chunk inside the chain-attached holder was never " + "intercepted"; + EXPECT_EQ(leak_tag, + ReferenceChainsTestAccessor::frontierLeakTag(chunk_ftag)); + FrontierEntry chunk_entry{}; + ASSERT_TRUE(frontier->lookup(chunk_ftag, &chunk_entry)); + EXPECT_EQ(entry_ftag, chunk_entry.parent_tag); + EXPECT_EQ(4u, chunk_entry.depth); + + tracker->stop(); +} + +// B' requeue mechanics: a truncated anchor walk reports exactly the RESOLVED-but-unwalked tags, and +// requeueStaticAnchorFifoFront() restores them to the FIFO front in order with a consistent +// membership set - so an at-risk holder that lost its budget turn keeps it for the next pass +// instead of waiting for the next sweep lap. +TEST_F(ReferenceChainsBfsTest, TruncatedAnchorWalkRequeuesUnwalkedFifoTags) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + void *holderCls = (void *)0x7001; + addClass(holderCls, "Lcom/rc/descendwalk/RequeueHolder;"); + + int holderANode = addNode(); + int holderBNode = addNode(); + int tableNode = addNode(); + int entryNode = addNode(); + + // Holder A's subtree is deep enough that a budget of 2 truncates the walk after A (two edges + // admitted, budget exhausted on the descend); holder B then must come back unwalked. + script = { + {JVMTI_HEAP_REFERENCE_FIELD, holderANode, tableNode, -1}, + {JVMTI_HEAP_REFERENCE_ARRAY_ELEMENT, tableNode, entryNode, -1}, + }; + + FrontierTable *frontier = tracker->frontierTable(); + node_tags[holderANode] = 105; + node_tags[holderBNode] = 106; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 105, 104, 1, FrontierEntryState::FRONTIER, 0)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 106, 104, 1, FrontierEntryState::FRONTIER, 0)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 104, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_STACK_LOCAL)); + + ReferenceChainsTestAccessor::pushStaticAnchorFifoForTest(105, 2001); + ReferenceChainsTestAccessor::pushStaticAnchorFifoForTest(106, 2002); + std::vector drained; + // 16 = STATIC_ANCHOR_FIFO_DRAIN (private), the per-pass drain cap. + ASSERT_EQ(2, ReferenceChainsTestAccessor::drainStaticAnchorFifoForTest( + 16, drained)); + ASSERT_EQ(2u, drained.size()); + EXPECT_EQ(105, drained[0].tag); + EXPECT_EQ(106, drained[1].tag); + + int edges = 0; + std::vector drained_tags; + for (const auto &at_risk : drained) { + drained_tags.push_back(at_risk.tag); + } + std::vector unwalked; + ReferenceChainsTestAccessor::walkStaticAnchorFifoForTest( + &mock_jvmti, &mock_jni, drained_tags, 2, &edges, &unwalked); + ASSERT_EQ(1u, unwalked.size()); + EXPECT_EQ(106, unwalked[0]); + EXPECT_NE(0, tags_ever_assigned[tableNode]) + << "holder A's walk never ran - the truncation happened too early"; + + // Requeue exactly what the caller-side filter in runPassManualWalk() would requeue (here: + // everything unwalked, both FIFO-sourced), keeping the drained entries' klass so the per-class + // occupancy is restored. + std::vector requeue; + for (jlong unwalked_tag : unwalked) { + for (const auto &at_risk : drained) { + if (unwalked_tag == at_risk.tag) { + requeue.push_back(at_risk); + break; + } + } + } + ReferenceChainsTestAccessor::requeueStaticAnchorFifoFrontForTest(requeue); + EXPECT_EQ(1u, ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); + EXPECT_TRUE(ReferenceChainsTestAccessor::staticAnchorFifoContainsForTest(106)); + EXPECT_FALSE(ReferenceChainsTestAccessor::staticAnchorFifoContainsForTest(105)); + std::vector redrained; + ASSERT_EQ(1, ReferenceChainsTestAccessor::drainStaticAnchorFifoForTest( + 16, redrained)); + ASSERT_EQ(1u, redrained.size()); + EXPECT_EQ(106, redrained[0].tag); + + tracker->stop(); +} + +// a this-field self-edge is REAL in the heap - every java.util.Collections$Synchronized* holder +// carries mutex == this - so walking such a holder's own subtree (rotation anchor walk or BFS +// descent) re-reports the holder as its own child through that field. +TEST_F(ReferenceChainsBfsTest, + SelfEdgeFieldDoesNotDemoteRootAttachedStaticHolder) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + void *holderCls = (void *)0x7101; + int holderClsIdx = + addClass(holderCls, "Ljava/util/Collections$SynchronizedRandomAccessList;"); + int holderNode = addNode(); + + // The holder's own self-edge: referrer == referee == holder (the mutex == this field), exactly + // as its rotation walk re-reports it. + script = { + {JVMTI_HEAP_REFERENCE_FIELD, holderNode, holderNode, holderClsIdx}, + }; + + // Seed the pre-demotion shape: holder root-attached STATIC_FIELD (anchor-eligible), admitted at + // depth 0. + FrontierTable *frontier = tracker->frontierTable(); + node_tags[holderNode] = 105; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 105, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + ReferenceChainsTestAccessor::pushPendingExpandForTest(105); + + // A delivered self-edge trips BOTH sibling guards: improveChain refuses, and the + // already-admitted block's else-if then offers the same self-parent to reparentToDurableRoot, + // which refuses too. + int edges = 0; + ReferenceChainsTestAccessor::expandFrontierForTest(&mock_jvmti, &mock_jni, + &edges); + + // The self-edge was delivered and refused: the entry keeps its root-attached shape (the + // collector's parent_tag == 0 eligibility) and no demotion push fired. + FrontierEntry entry{}; + ASSERT_TRUE(frontier->lookup(105, &entry)); + EXPECT_EQ(0, entry.parent_tag); + EXPECT_EQ((u8)JVMTI_HEAP_REFERENCE_STATIC_FIELD, entry.root_kind); + EXPECT_EQ(0u, entry.depth); + EXPECT_EQ(0u, ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); + + // The sibling guards and the direct table calls agree: a self-parent is refused by both + // improvement paths, unconditionally. + EXPECT_FALSE(frontier->improveChain(105, 105, 0, 5, 0, -1, 0, 0)); + EXPECT_FALSE(frontier->reparentToDurableRoot(105, 105, 0, -1, 0)); + + tracker->stop(); +} + +// the B' at-risk FIFO sat cap-pinned at 1024 because three classes flooded it (klass 1: 1396 +// pushes, klass 215: 1063, klass 1733: 988+), so the LEAK_BUFFER wrapper's pushes (klass 28366) +// were dropped at the cap check and the lane designed to repair exactly its demotion never carried +// it. +TEST_F(ReferenceChainsBfsTest, AtRiskFifoPerClassQuotaKeepsFloodOut) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + const u32 quota = + ReferenceChainsTestAccessor::kAtRiskPerKlassCap; + ASSERT_EQ(64u, quota); + + // The flood: only the first `quota` pushes of one class land. + for (int i = 0; i < 70; i++) { + ReferenceChainsTestAccessor::pushStaticAnchorFifoForTest(1000 + i, + 1733); + } + EXPECT_EQ(quota, ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); + // The flood's first `quota` tags hold their slots and the excess is dropped at the quota check + // - absent from the FIFO, not queued. + EXPECT_TRUE(ReferenceChainsTestAccessor::staticAnchorFifoContainsForTest( + 1000 + (int)quota - 1)); + EXPECT_FALSE(ReferenceChainsTestAccessor::staticAnchorFifoContainsForTest( + 1000 + (int)quota)); + + // The wrapper's push (a different class) lands despite the flood - exactly the push the pod + // dropped. + ReferenceChainsTestAccessor::pushStaticAnchorFifoForTest(2000, 28366); + EXPECT_EQ(quota + 1, + ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); + EXPECT_TRUE( + ReferenceChainsTestAccessor::staticAnchorFifoContainsForTest(2000)); + + // Tag dedupe is unchanged: the same tag never enters twice. + ReferenceChainsTestAccessor::pushStaticAnchorFifoForTest(2000, 28366); + EXPECT_EQ(quota + 1, + ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); + + // Full drain: order preserved (flood first, newcomer last), occupancy erased with the entries - + // the next flood can land again (it never owns MORE than its quota, but it is not permanently + // locked out either). + std::vector drained; + ASSERT_EQ((int)quota + 1, + ReferenceChainsTestAccessor::drainStaticAnchorFifoForTest( + 1024, drained)); + ASSERT_EQ(quota + 1, drained.size()); + EXPECT_EQ(1000, drained.front().tag); + EXPECT_EQ(1733u, drained.front().klass_id); + EXPECT_EQ(2000, drained.back().tag); + EXPECT_EQ(28366u, drained.back().klass_id); + for (int i = 0; i < 70; i++) { + ReferenceChainsTestAccessor::pushStaticAnchorFifoForTest(1000 + i, + 1733); + } + EXPECT_EQ(quota, ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); + + // Partial drain at the real per-pass rate (16/pass): the flood's occupancy is 64 - 16 = 48 + // after the drain, so its next push lands (refilling its share as it drains - the flood + // self-throttles, it never starves the lane). + std::vector partial; + ASSERT_EQ(16, ReferenceChainsTestAccessor::drainStaticAnchorFifoForTest( + 16, partial)); + ReferenceChainsTestAccessor::pushStaticAnchorFifoForTest(3000, 1733); + EXPECT_EQ(quota - 16 + 1, + ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); + EXPECT_TRUE( + ReferenceChainsTestAccessor::staticAnchorFifoContainsForTest(3000)); + + // Requeue restores occupancy exactly: the requeued entries occupy their slots again and drain + // in FIFO order. + std::vector requeue(partial.begin(), + partial.end()); + ReferenceChainsTestAccessor::requeueStaticAnchorFifoFrontForTest(requeue); + EXPECT_EQ(quota + 1, + ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); + std::vector after_requeue; + ASSERT_EQ(16, + ReferenceChainsTestAccessor::drainStaticAnchorFifoForTest( + 16, after_requeue)); + EXPECT_EQ(1000, after_requeue.front().tag); + + // Saturated-but-diverse: 16 distinct classes at exactly their quota fill the 1024 cap, and the + // newcomer is dropped at the CAP (legitimate saturation - no eviction), not because of any + // flood. + std::vector rest; + ASSERT_EQ((int)quota - 16 + 1, + ReferenceChainsTestAccessor::drainStaticAnchorFifoForTest( + 1024, rest)); + for (u32 klass = 1; klass <= 16; klass++) { + for (u32 i = 0; i < quota; i++) { + ReferenceChainsTestAccessor::pushStaticAnchorFifoForTest( + 10000 + klass * 100 + i, 4000 + klass); + } + } + EXPECT_EQ(1024u, + ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); + // The 16 saturated classes sit exactly at their per-class quota (no quota drop is even possible + // at exactly `quota` pushes), so the newcomer's absence below is the CAP's doing, not the + // quota's. + ReferenceChainsTestAccessor::pushStaticAnchorFifoForTest(20000, 28366); + EXPECT_EQ(1024u, + ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); + EXPECT_FALSE( + ReferenceChainsTestAccessor::staticAnchorFifoContainsForTest(20000)); + + // Leave the FIFO drained: this test is the one that saturates it, and the reset() seam above + // now clears it for the next test regardless - but a drained ending also keeps this test + // order-independent even if that seam ever regresses again. + std::vector final_drain; + ASSERT_EQ(1024, + ReferenceChainsTestAccessor::drainStaticAnchorFifoForTest( + 1024, final_drain)); + + tracker->stop(); +} + +// Retention-edge labels (fillHopEdgeLabels()/hopLabelClassFor()): the emitted chain's per-hop +// labels must decode the JVMTI-SPECIFICATION field ordinal captured at admission - the interface +// offset, the superclass-chain order, the interface-referrer branch - and degrade to the edge KIND +// on any undecodable hop, never a fabricated name (the fail-safe contract: a wrong numbering on an +// unverified JVM degrades, it does not lie). diff --git a/ddprof-lib/src/test/cpp/referenceChainsBfsTests.inc b/ddprof-lib/src/test/cpp/referenceChainsBfsTests.inc new file mode 100644 index 000000000..0a0b98ea4 --- /dev/null +++ b/ddprof-lib/src/test/cpp/referenceChainsBfsTests.inc @@ -0,0 +1,1275 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +namespace { + +struct ScriptedEdge { + jvmtiHeapReferenceKind kind; + int referrer_idx; // -1 = heap root (no referrer) + int referee_idx; // index into ReferenceChainsBfsTest::node_tags + int class_idx; // index into ReferenceChainsBfsTest::classes, or -1 +}; + +struct ScriptedClass { + void *klass; + const char *signature; // JVMTI class signature, e.g. "Lcom/example/Foo;" +}; + +// Retention-edge label decode fixtures (ReferenceChainsBfsTest's field_decode_hierarchy + the +// hierarchy-introspection mock slots): a fake class hierarchy the slots read, mirroring just enough +// JVMTI class shape for the spec-ordinal decoder (own-declared fields in GetClassFields order, +// direct superclass, directly implemented/extended interfaces). +struct FakeField { + void *id; // fake jfieldID + const char *name; +}; +struct FakeClass { + bool is_interface; + void *super; // fake jclass, or nullptr + std::vector interfaces; // directly implemented/extended + std::vector fields; // own-declared, GetClassFields order +}; + +} // namespace + +class ReferenceChainsBfsTest : public ::testing::Test { +protected: + jvmtiInterface_1_ jvmti_tbl{}; + _jvmtiEnv mock_jvmti{}; + JNINativeInterface_ jni_tbl{}; + JNIEnv_ mock_jni{}; + + std::unordered_map tags; + std::vector classes; + std::vector script; + std::vector node_tags; + + // node_tags[idx] mirrors "the object's *current* live JVMTI tag" (0 once releaseSearchTags() + // clears it, exactly like a real GetTag() would report after SetTag(obj, 0)). + std::vector tags_ever_assigned; + + // Tags that GetObjectsWithTags() below reports as unresolvable, simulating the referenced + // object having died (GC'd) between passes - see the resolve-or-drop tests. + std::unordered_set dead_tags; + + // When true, mock_GetObjectsWithTags() below fails outright (as if the real JVMTI call had hit + // e.g. JVMTI_ERROR_OUT_OF_MEMORY), for ReleaseSearchTagsFailureTest - simulates + // releaseSearchTags()'s own GetObjectsWithTags() call failing rather than an individual tag + // failing to resolve (dead_tags above). + bool fail_get_objects_with_tags = false; + + // When non-zero, mock_GetObjectsWithTags() below busy-waits this many nanoseconds. + u64 gotw_delay_ns = 0; + + // Synthetic frontier-holder arrays for expandFrontier()'s array-holder walk: + // mock_NewObjectArray() hands back an opaque handle, mock_SetObjectArrayElement() records its + // elements here, and mock_FollowReferences() treats every recorded element as an expansion seed + // (one hop, gated by the production callback's batch_tags) when the holder is passed as + // initial_object. + std::unordered_map> holders; + uintptr_t next_holder = 0xF00D0000; + + // FindClass(name) -> registered fake class (see mock_FindClass' own comment): names + // descendFromAnchor()'s resolutions look up ("java/lang/ClassLoader", "java/lang/ThreadGroup", + // "java/security/ProtectionDomain", "java/lang/ThreadLocal$ThreadLocalMap", + // "java/lang/Thread"). + std::unordered_map find_classes; + // Fake class returned by mock_GetObjectClass() for unregistered objects + // (walkCandidateThreadLocals()'s fresh-anchor admission path). + void *thread_class = nullptr; + + // DeleteGlobalRef call count (see mock_DeleteGlobalRef). + int global_refs_deleted_ = 0; + + jvmtiEnv *orig_jvmti = nullptr; + + static ReferenceChainsBfsTest *active_fixture; + + void SetUp() override { + active_fixture = this; + // See ReferenceChainsTestAccessor's own comment - without this, a prior test in this suite + // that drove the search to SearchState::COMPLETED/ABANDONED would make every runPass() call + // below a permanent no-op. + ReferenceChainsTestAccessor::reset(); + jvmti_tbl = jvmtiInterface_1_{}; + // start() calls VM::jvmti()->SetEventNotificationMode() - stub it and swap VM::_jvmti + // (VMTestAccessor, declared above) the same way ReferenceChainsTest's fixture does, so + // start() does not dereference the real (null, no live JVM) jvmtiEnv. + jvmti_tbl.SetEventNotificationMode = &mock_SetEventNotificationMode; + jvmti_tbl.SetTag = &mock_SetTag; + jvmti_tbl.GetTag = &mock_GetTag; + jvmti_tbl.GetLoadedClasses = &mock_GetLoadedClasses; + jvmti_tbl.GetClassLoader = &mock_GetClassLoader; + jvmti_tbl.GetClassSignature = &mock_GetClassSignature; + jvmti_tbl.Deallocate = &mock_Deallocate; + jvmti_tbl.FollowReferences = &mock_FollowReferences; + jvmti_tbl.IterateOverReachableObjects = &mock_IterateOverReachableObjects; + jvmti_tbl.GetObjectsWithTags = &mock_GetObjectsWithTags; + // Retention-edge label decode path (hopLabelClassFor()). + jvmti_tbl.IsInterface = &mock_IsInterface; + jvmti_tbl.GetImplementedInterfaces = &mock_GetImplementedInterfaces; + jvmti_tbl.GetClassFields = &mock_GetClassFields; + jvmti_tbl.GetFieldName = &mock_GetFieldName; + mock_jvmti.functions = &jvmti_tbl; + orig_jvmti = VMTestAccessor::getJvmti(); + VMTestAccessor::setJvmti(&mock_jvmti); + + jni_tbl = JNINativeInterface_{}; + jni_tbl.DeleteLocalRef = &mock_DeleteLocalRef; + jni_tbl.FindClass = &mock_FindClass; + jni_tbl.GetObjectClass = &mock_GetObjectClass; + jni_tbl.GetSuperclass = &mock_JniGetSuperclass; + jni_tbl.NewGlobalRef = &mock_NewGlobalRef; + jni_tbl.DeleteGlobalRef = &mock_DeleteGlobalRef; + jni_tbl.EnsureLocalCapacity = &mock_EnsureLocalCapacity; + jni_tbl.NewObjectArray = &mock_NewObjectArray; + jni_tbl.SetObjectArrayElement = &mock_SetObjectArrayElement; + jni_tbl.ExceptionCheck = &mock_ExceptionCheck; + jni_tbl.ExceptionClear = &mock_ExceptionClear; + mock_jni.functions = &jni_tbl; + } + + void TearDown() override { + VMTestAccessor::setJvmti(orig_jvmti); + active_fixture = nullptr; + } + + // Registers a fake class (matched by identity, not by any real JNI semantics) that + // resolveLoadedClasses() will discover via the mocked GetLoadedClasses(). + int addClass(void *klass, const char *signature) { + classes.push_back({klass, signature}); + return (int)classes.size() - 1; + } + + // addClass() + a mock_FindClass(name) registry entry in one step, for the classes + // descendFromAnchor()'s resolution helpers look up by name (see find_classes' own comment). + int registerClassForFindClass(void *klass, const char *name, + const char *signature) { + int idx = addClass(klass, signature); + find_classes[name] = klass; + return idx; + } + + // Adds an as-yet-untagged frontier node, returning its index into node_tags for use as a + // ScriptedEdge referrer_idx/referee_idx. + int addNode() { + node_tags.push_back(0); + tags_ever_assigned.push_back(0); + return (int)node_tags.size() - 1; + } + + // Reverse lookup from a node's synthetic identity (&node_tags[idx], see mock_FollowReferences' + // initial_object handling below) back to its index. + int indexOfNode(jobject obj) const { + for (size_t i = 0; i < node_tags.size(); i++) { + if (obj == (jobject)&node_tags[i]) { + return (int)i; + } + } + return -1; + } + + static jvmtiError JNICALL mock_SetTag(jvmtiEnv *, jobject object, jlong tag) { + // releaseSearchTags() calls SetTag(obj, 0) on the resolved objects GetObjectsWithTags() + // (below) hands back for a frontier node - route that through node_tags[idx] directly (the + // same storage GetObjectsWithTags's resolution and the production callback's tag_ptr writes + // both key off of), so the release is actually observable, not just recorded in a side map + // nothing else reads. + int idx = active_fixture->indexOfNode(object); + if (idx >= 0) { + active_fixture->node_tags[idx] = tag; + return JVMTI_ERROR_NONE; + } + if (tag == 0) { + active_fixture->tags.erase(object); + } else { + active_fixture->tags[object] = tag; + } + return JVMTI_ERROR_NONE; + } + + static jvmtiError JNICALL mock_GetTag(jvmtiEnv *, jobject object, jlong *tag_ptr) { + int idx = active_fixture->indexOfNode(object); + if (idx >= 0) { + *tag_ptr = active_fixture->node_tags[idx]; + return JVMTI_ERROR_NONE; + } + auto it = active_fixture->tags.find(object); + *tag_ptr = it != active_fixture->tags.end() ? it->second : 0; + return JVMTI_ERROR_NONE; + } + + static jvmtiError JNICALL mock_GetLoadedClasses(jvmtiEnv *, jint *count_ptr, + jclass **classes_ptr) { + auto &classes = active_fixture->classes; *count_ptr = (jint)classes.size(); + *classes_ptr = classes.empty() + ? nullptr + : (jclass *)malloc(sizeof(jclass) * classes.size()); + for (size_t i = 0; i < classes.size(); i++) { + (*classes_ptr)[i] = (jclass)classes[i].klass; + } + return JVMTI_ERROR_NONE; + } + + // admitStaticFieldRoots()'s app-classes-first partition (referenceChains.cpp) calls this for + // every loaded class. + static jvmtiError JNICALL mock_GetClassLoader(jvmtiEnv *, jclass, + jobject *classloader_ptr) { + *classloader_ptr = nullptr; + return JVMTI_ERROR_NONE; + } + + static jvmtiError JNICALL mock_GetClassSignature(jvmtiEnv *, jclass klass, + char **signature_ptr, + char **generic_ptr) { + for (auto &c : active_fixture->classes) { + if (c.klass == (void *)klass) { + *signature_ptr = strdup(c.signature); + if (generic_ptr != nullptr) { + *generic_ptr = nullptr; + } + return JVMTI_ERROR_NONE; + } + } + return JVMTI_ERROR_INVALID_CLASS; + } + + static jvmtiError JNICALL mock_Deallocate(jvmtiEnv *, unsigned char *mem) { + free(mem); + return JVMTI_ERROR_NONE; + } + + static void JNICALL mock_DeleteLocalRef(JNIEnv *, jobject) { + // no-op: this fixture's fake jobject/jclass values are not real JNI local refs. + } + + // Retention-edge label decode fixtures (fillHopEdgeLabels()/ hopLabelClassFor()): a fake class + // hierarchy the hierarchy-introspection slots below read, plus a tag -> fake jclass map (the + // decoder resolves the referrer class from its raw tag via GetObjectsWithTags - the test + // populates field_decode_classes from resolveLoadedClasses()-minted tags, and + // mock_GetObjectsWithTags consults it first). + std::unordered_map field_decode_classes; + std::unordered_map field_decode_hierarchy; + + // The hierarchy-introspection slots the decoder needs (IsInterface/ + // GetImplementedInterfaces/GetClassFields/GetFieldName on the JVMTI table, GetSuperclass on the + // JNI table - modern JVMTI dropped its own GetSuperclass). + static jvmtiError JNICALL mock_IsInterface(jvmtiEnv *, jclass cls, + jboolean *is_interface_ptr) { + auto it = active_fixture->field_decode_hierarchy.find(cls); + if (it == active_fixture->field_decode_hierarchy.end()) { + return JVMTI_ERROR_INVALID_CLASS; + } + *is_interface_ptr = it->second.is_interface ? JNI_TRUE : JNI_FALSE; + return JVMTI_ERROR_NONE; + } + static jclass JNICALL mock_JniGetSuperclass(JNIEnv *, jclass cls) { + auto it = active_fixture->field_decode_hierarchy.find(cls); + return it == active_fixture->field_decode_hierarchy.end() + ? nullptr + : (jclass)it->second.super; + } + static jvmtiError JNICALL mock_GetImplementedInterfaces( + jvmtiEnv *, jclass cls, jint *count_ptr, jclass **ifaces_ptr) { + auto it = active_fixture->field_decode_hierarchy.find(cls); + if (it == active_fixture->field_decode_hierarchy.end()) { + return JVMTI_ERROR_INVALID_CLASS; + } + const std::vector &ifaces = it->second.interfaces; + *count_ptr = (jint)ifaces.size(); + *ifaces_ptr = ifaces.empty() + ? nullptr + : (jclass *)malloc(sizeof(jclass) * ifaces.size()); + for (size_t i = 0; i < ifaces.size(); i++) { + (*ifaces_ptr)[i] = (jclass)ifaces[i]; + } + return JVMTI_ERROR_NONE; + } + static jvmtiError JNICALL mock_GetClassFields( + jvmtiEnv *, jclass cls, jint *count_ptr, jfieldID **fields_ptr) { + auto it = active_fixture->field_decode_hierarchy.find(cls); + if (it == active_fixture->field_decode_hierarchy.end()) { + return JVMTI_ERROR_INVALID_CLASS; + } + const std::vector &fields = it->second.fields; + *count_ptr = (jint)fields.size(); + *fields_ptr = fields.empty() + ? nullptr + : (jfieldID *)malloc(sizeof(jfieldID) * fields.size()); + for (size_t i = 0; i < fields.size(); i++) { + (*fields_ptr)[i] = (jfieldID)fields[i].id; + } + return JVMTI_ERROR_NONE; + } + static jvmtiError JNICALL mock_GetFieldName( + jvmtiEnv *, jclass, jfieldID field, char **name_ptr, + char ** /*signature_ptr*/, char ** /*generic_ptr*/) { + for (const auto &kv : active_fixture->field_decode_hierarchy) { + for (const FakeField &f : kv.second.fields) { + if (f.id == (void *)field) { + size_t len = strlen(f.name) + 1; + char *name = (char *)malloc(len); + memcpy(name, f.name, len); + *name_ptr = name; + return JVMTI_ERROR_NONE; + } + } + } + return JVMTI_ERROR_INVALID_FIELDID; + } + + // expandFrontier()/admitStaticFieldRoots() resolve java/lang/Object once as the holder array's + // element type - a non-null fake jclass is all it needs (the type is never introspected, only + // passed to NewObjectArray()). + static jclass JNICALL mock_FindClass(JNIEnv *, const char *name) { + auto it = active_fixture->find_classes.find(name); + if (it != active_fixture->find_classes.end()) { + return (jclass)it->second; + } + return (jclass)0xC1A55; + } + + // walkCandidateThreadLocals()'s fresh-anchor admission calls GetObjectClass(thread) - + // unregistered classes return the fixture's fake Thread class (set_thread_class) so the anchor + // entry's class tag resolves through the same mocked GetTag/tagging path. + static jclass JNICALL mock_GetObjectClass(JNIEnv *, jobject) { + return (jclass)active_fixture->thread_class; + } + + + // The production code wraps that fake jclass in a global ref (a real local ref would dangle + // across JNI-entered test seams - see _cached_object_class's own comment). + static jobject JNICALL mock_NewGlobalRef(JNIEnv *, jobject obj) { + return obj; + } + + // Counts DeleteGlobalRef calls - the deferred thread-ref teardown test + // (ThreadRefUnregisterDefersGlobalRefDeletion) asserts on the count. + static void JNICALL mock_DeleteGlobalRef(JNIEnv *, jobject) { + active_fixture->global_refs_deleted_++; + } + + static jint JNICALL mock_EnsureLocalCapacity(JNIEnv *, jint) { + return JNI_OK; + } + + // expandFrontier() calls jniExceptionCheck() after every upcall that can legally throw + // (NewObjectArray/SetObjectArrayElement/EnsureLocalCapacity failures) - this fixture's mocks + // never throw, so there is never a pending exception to report or clear. + static jboolean JNICALL mock_ExceptionCheck(JNIEnv *) { + return JNI_FALSE; + } + + static void JNICALL mock_ExceptionClear(JNIEnv *) { + // no-op: mock_ExceptionCheck() never reports a pending exception. + } + + // Hands back a fresh opaque holder handle and registers it in `holders` so + // mock_SetObjectArrayElement()/mock_FollowReferences() can find its elements. + static jobjectArray JNICALL mock_NewObjectArray(JNIEnv *, jsize, jclass, + jobject) { + jobject handle = (jobject)(active_fixture->next_holder++); + active_fixture->holders[handle] = {}; + return (jobjectArray)handle; + } + + static void JNICALL mock_SetObjectArrayElement(JNIEnv *, jobjectArray array, + jsize idx, jobject value) { active_fixture->holders[(jobject)array].push_back(value); + } + + // runPassManualWalk()'s root enumeration (the default, non-fallback path): reports each + // scripted root edge's referee to heapRootCallback() exactly as a real + // IterateOverReachableObjects() reports a root-held object - tag_ptr only, no oop, no + // transitive children (see runPassManualWalk()'s own comment). + static jvmtiError JNICALL mock_IterateOverReachableObjects( + jvmtiEnv *, jvmtiHeapRootCallback heap_root_cb, + jvmtiStackReferenceCallback, jvmtiObjectReferenceCallback, + const void *user_data) { + for (auto &e : active_fixture->script) { + if (e.referrer_idx != -1) { + continue; + } + jlong class_tag = 0; + if (e.class_idx >= 0) { + class_tag = active_fixture->tags[active_fixture->classes[e.class_idx].klass]; + } + jlong *tag_ptr = &active_fixture->node_tags[e.referee_idx]; + jvmtiIterationControl ctl = heap_root_cb( + JVMTI_HEAP_ROOT_JNI_GLOBAL, class_tag, /*size=*/0, tag_ptr, + const_cast(user_data)); + if (*tag_ptr != 0) { + active_fixture->tags_ever_assigned[e.referee_idx] = *tag_ptr; + } + if (ctl == JVMTI_ITERATION_ABORT) { + break; + } + } + return JVMTI_ERROR_NONE; + } + + // Resolves each requested tag to its node's synthetic identity (&node_tags[idx]) by scanning + // node_tags for a matching current value - mirroring real GetObjectsWithTags()'s "only + // currently-live tags come back" contract. + static jvmtiError JNICALL mock_GetObjectsWithTags( + jvmtiEnv *, jint tag_count, const jlong *req_tags, jint *count_ptr, + jobject **object_result_ptr, jlong **tag_result_ptr) { + if (active_fixture->fail_get_objects_with_tags) { + // Deliberately leave *count_ptr/*object_result_ptr/*tag_result_ptr untouched - a real + // failed JVMTI call makes no promise about them, and releaseSearchTags() must not read + // them on this path. + return JVMTI_ERROR_OUT_OF_MEMORY; + } + if (active_fixture->gotw_delay_ns != 0) { + u64 until = OS::nanotime() + active_fixture->gotw_delay_ns; + while (OS::nanotime() < until) { + // busy-wait: a sleep could overshoot by scheduler latency, and the overshoot + // direction matters for the one-batch deadline arithmetic the callers of this knob + // rely on. + } + } + std::vector objs; + std::vector found; + for (jint i = 0; i < tag_count; i++) { + jlong want = req_tags[i]; + if (want == 0 || active_fixture->dead_tags.count(want) > 0) { + continue; + } + // The decoder resolves a referrer CLASS from its raw (negative) tag - no node carries + // one, so the tag -> fake jclass map (field_decode_classes, see its own comment) serves + // it. + auto fd = active_fixture->field_decode_classes.find(want); + if (fd != active_fixture->field_decode_classes.end()) { + objs.push_back((jobject)fd->second); + found.push_back(want); + break; + } + for (size_t idx = 0; idx < active_fixture->node_tags.size(); idx++) { + if (active_fixture->node_tags[idx] == want) { + objs.push_back((jobject)&active_fixture->node_tags[idx]); + found.push_back(want); + break; + } + } + } + *count_ptr = (jint)objs.size(); + *object_result_ptr = objs.empty() + ? nullptr : (jobject *)malloc(sizeof(jobject) * objs.size()); + *tag_result_ptr = found.empty() + ? nullptr : (jlong *)malloc(sizeof(jlong) * found.size()); + for (size_t i = 0; i < objs.size(); i++) { + (*object_result_ptr)[i] = objs[i]; + (*tag_result_ptr)[i] = found[i]; + } + return JVMTI_ERROR_NONE; + } + + // Plays back `script` against the real production heap_reference_callback, modelling enough of + // FollowReferences' actual semantics for these heap-walk tests to be meaningful: - "a reference + // from A to B is not traversed until A is visited" - an edge whose referrer was not returned + // JVMTI_VISIT_OBJECTS for (or was never itself visited) is skipped, exactly as a real traversal + // would never reach it. + static jvmtiError JNICALL mock_FollowReferences( + jvmtiEnv *, jint, jclass, jobject initial_object, + const jvmtiHeapCallbacks *callbacks, const void *user_data) { + std::unordered_map expandable; // seed_idx == -2 marks the root walk (initial_object == NULL); any + // other value marks an expansion walk seeded from one or more boundary objects, in which + // case root edges are never replayed. + int seed_idx = -2; + // The transient holder array itself is never tagged (mirrors real production: + // admitStaticFieldRoots()/expandFrontier() never call SetTag on the frontier-holder array + // they build), so every holder->element ARRAY_ELEMENT edge below is replayed with a + // referrer tag of 0. + static jlong holder_tag = 0; + if (initial_object != nullptr) { + auto holder_it = active_fixture->holders.find(initial_object); + if (holder_it != active_fixture->holders.end()) { + // Array-holder walk (expandFrontier()'s already-tagged boundary batch, or + // admitStaticFieldRoots()'s negative- tagged class-object seed): actually invoke + // the production callback for each holder->element edge, exactly like a real + // FollowReferences(initial_object=holder_array) call would - this is what lets + // heap_reference_callback()'s own tag-sign/reference_kind logic (e.g. the *tag_ptr + // < 0 early-return and its admitStaticFieldRoots() carve-out) actually run, rather + // than assuming every element is expandable. + seed_idx = -1; + for (jobject elem : holder_it->second) { + int idx = active_fixture->indexOfNode(elem); + if (idx < 0) { + continue; + } + jlong *tag_ptr = &active_fixture->node_tags[idx]; + jint ctl = callbacks->heap_reference_callback( + JVMTI_HEAP_REFERENCE_ARRAY_ELEMENT, nullptr, + /*class_tag=*/0, /*referrer_class_tag=*/0, + /*size=*/0, tag_ptr, &holder_tag, + /*length=*/-1, const_cast(user_data)); + if (*tag_ptr != 0) { + active_fixture->tags_ever_assigned[idx] = *tag_ptr; + } + if (ctl & JVMTI_VISIT_ABORT) { + return JVMTI_ERROR_NONE; + } + expandable[idx] = (ctl & JVMTI_VISIT_OBJECTS) != 0; } + } else { + seed_idx = active_fixture->indexOfNode(initial_object); + expandable[seed_idx] = true; + } + } + for (auto &e : active_fixture->script) { + if (e.referrer_idx == -1) { + if (seed_idx != -2) { + continue; // resumed pass: never replay root edges + } + } else { + auto it = expandable.find(e.referrer_idx); + if (it == expandable.end() || !it->second) { continue; + } + } jlong class_tag = 0; + if (e.class_idx >= 0) { + class_tag = active_fixture->tags[active_fixture->classes[e.class_idx].klass]; + } + jlong *referrer_tag_ptr = e.referrer_idx >= 0 + ? &active_fixture->node_tags[e.referrer_idx] : nullptr; + jlong *tag_ptr = &active_fixture->node_tags[e.referee_idx]; + jint ctl = callbacks->heap_reference_callback( + e.kind, nullptr, class_tag, /*referrer_class_tag=*/0, + /*size=*/0, tag_ptr, referrer_tag_ptr, /*length=*/-1, + const_cast(user_data)); + if (*tag_ptr != 0) { + active_fixture->tags_ever_assigned[e.referee_idx] = *tag_ptr; + } + if (ctl & JVMTI_VISIT_ABORT) { + return JVMTI_ERROR_NONE; + } + expandable[e.referee_idx] = (ctl & JVMTI_VISIT_OBJECTS) != 0; + } + return JVMTI_ERROR_NONE; + } +}; + +ReferenceChainsBfsTest *ReferenceChainsBfsTest::active_fixture = nullptr; + +TEST_F(ReferenceChainsBfsTest, ReconstructsChainForSyntheticGraph) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + void *classA = (void *)0x2001, *classB = (void *)0x2002, + *classTarget = (void *)0x2003; + int ca = addClass(classA, "Lcom/rc/phase3/graph/A;"); + int cb = addClass(classB, "Lcom/rc/phase3/graph/B;"); + int ct = addClass(classTarget, "Lcom/rc/phase3/graph/Target;"); + + int nodeA = addNode(); + int nodeB = addNode(); + int nodeTarget = addNode(); + + script = { + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodeA, ca}, + {JVMTI_HEAP_REFERENCE_FIELD, nodeA, nodeB, cb}, + {JVMTI_HEAP_REFERENCE_FIELD, nodeB, nodeTarget, ct}, + }; + + bool truncated = true; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_FALSE(truncated); + + // A single pass that reaches full exhaustion of the reachable graph completes the search and + // releases every tag it assigned - so the tag must be fetched via tags_ever_assigned (captured + // at assignment time), not node_tags (already reset to 0 by releaseSearchTags() by the time + // runPass() returns; see ReleasesTagsOnCompletion below for the release itself). + EXPECT_EQ(SearchState::COMPLETED, tracker->searchState()); + jlong targetTag = tags_ever_assigned[nodeTarget]; + ASSERT_NE(0, targetTag); + EXPECT_EQ(0, node_tags[nodeTarget]); // released - see the comment above + + std::vector chain; + ASSERT_TRUE(tracker->frontierTable()->reconstructChain(targetTag, &chain)); + ASSERT_EQ(3u, chain.size()); + + int expectedTarget = Profiler::instance()->lookupClass( + "com/rc/phase3/graph/Target", strlen("com/rc/phase3/graph/Target")); + int expectedB = Profiler::instance()->lookupClass( + "com/rc/phase3/graph/B", strlen("com/rc/phase3/graph/B")); + int expectedA = Profiler::instance()->lookupClass( + "com/rc/phase3/graph/A", strlen("com/rc/phase3/graph/A")); + ASSERT_NE(-1, expectedTarget); + ASSERT_NE(-1, expectedB); + ASSERT_NE(-1, expectedA); + + EXPECT_EQ((u32)expectedTarget, chain[0]); + EXPECT_EQ((u32)expectedB, chain[1]); + EXPECT_EQ((u32)expectedA, chain[2]); + + // buildChainEvent() wraps the same reconstructChain() call into the ReferenceChainEvent shape + // Recording::recordReferenceChain() (flightRecorder.cpp) expects - same chain/order, plus the + // target's own depth from FrontierEntry. + ReferenceChainEvent event; + ASSERT_TRUE(tracker->buildChainEvent(&mock_jvmti, &mock_jni, targetTag, + &event)); + EXPECT_EQ((u64)targetTag, event._target_tag); + EXPECT_EQ(2u, event._depth); // root(A, depth0) -> B(depth1) -> Target(depth2) + ASSERT_EQ(chain.size(), event._hops.size()); + for (size_t i = 0; i < chain.size(); i++) { + EXPECT_EQ(chain[i], event._hops[i].klass_id); + } + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, BuildChainEventFailsForUnknownTag) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + ReferenceChainEvent event; + EXPECT_FALSE(tracker->buildChainEvent(&mock_jvmti, &mock_jni, 12345, + &event)); + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, BuildAbandonedEventFailsUnlessSearchAbandoned) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + // Freshly started: never abandoned (never even run a pass yet). + ReferenceChainAbandonedEvent event; + EXPECT_FALSE(tracker->buildAbandonedEvent(&event)); + + int nodeA = addNode(); + script = {{JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodeA, -1}}; + bool truncated = true; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + // Small graph, no caps hit -> COMPLETED, not ABANDONED. + ASSERT_EQ(SearchState::COMPLETED, tracker->searchState()); + EXPECT_FALSE(tracker->buildAbandonedEvent(&event)); + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, HopCapStopsAdmittingBeyondCap) { + Arguments args; + // hops=1: only depth 0 (direct root references) may be admitted. + ASSERT_FALSE(args.parse("referencechains=true:hops=1:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int nodeA = addNode(); + int nodeB = addNode(); + + script = { + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodeA, -1}, // depth 0 - admitted + {JVMTI_HEAP_REFERENCE_FIELD, nodeA, nodeB, -1}, // depth 1 - capped + }; + + bool truncated = true; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_FALSE(truncated); // hop cap is not truncation - a normal boundary + // Not truncated -> graph fully explored within the hop cap -> the search completes and releases + // its tags in the same call (see the previous test's comment) - fetch nodeA's tag via + // tags_ever_assigned, not node_tags. + EXPECT_EQ(SearchState::COMPLETED, tracker->searchState()); + + EXPECT_NE(0, tags_ever_assigned[nodeA]); + EXPECT_EQ(0, node_tags[nodeB]); // never admitted into the frontier + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, BudgetExhaustionTruncatesAndIsReported) { + Arguments args; + // budget=1: root enumeration and the expand phase draw from separate budget pools (see + // runPassManualWalk()'s own comment), each sized 1 here - root enum admits nodeA, then the + // expand phase's own 1-unit budget admits exactly one of nodeA's two children before + // exhausting. + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1:firstpassbudget=1")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int nodeA = addNode(); + int nodeB = addNode(); + int nodeC = addNode(); + + script = { + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodeA, -1}, // admitted via root enum + {JVMTI_HEAP_REFERENCE_FIELD, nodeA, nodeB, -1}, // admitted via expand + {JVMTI_HEAP_REFERENCE_FIELD, nodeA, nodeC, -1}, // expand budget exhausted + }; + + bool truncated = false; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_TRUE(truncated); + // Budget exhaustion for *this pass* leaves pending work (nodeC's own edge was never even + // attempted) - the search stays RUNNING, not COMPLETED, so no tag release happens yet and + // node_tags[nodeA]/[nodeB] are still the real assigned tags. + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); + + EXPECT_NE(0, node_tags[nodeA]); + EXPECT_NE(0, node_tags[nodeB]); + EXPECT_EQ(0, node_tags[nodeC]); + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, PreTaggedClassObjectsAreNeverExpandedOrAdmitted) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int classNode = addNode(); + node_tags[classNode] = -7; // simulate a class object already tagged by + // resolveLoadedClasses() before this pass - see ClassTagTable's + // tag-sign convention. + int fieldTargetNode = addNode(); + + script = { + // A root reference straight to the class object (e.g. a JVMTI_HEAP_REFERENCE_SYSTEM_CLASS + // root edge in a real walk). + {JVMTI_HEAP_REFERENCE_SYSTEM_CLASS, -1, classNode, -1}, + // A static field of that class - must never be delivered by a real FollowReferences call, + // since the class-object edge above must not return JVMTI_VISIT_OBJECTS; + // mock_FollowReferences enforces this the same way a real traversal would (see its own + // comment). + {JVMTI_HEAP_REFERENCE_STATIC_FIELD, classNode, fieldTargetNode, -1}, + }; + + bool truncated = true; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_FALSE(truncated); + + EXPECT_EQ(-7, node_tags[classNode]); // untouched - never treated as a + // frontier object + EXPECT_EQ(0, node_tags[fieldTargetNode]); // never reached - the class + // edge above must not expand + + tracker->stop(); +} + +// Regression test for admitStaticFieldRoots(): an object retained solely by a static field (no +// other GC root reaches it) must still be discovered. +TEST_F(ReferenceChainsBfsTest, DiscoversObjectRetainedOnlyByStaticField) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int classNode = addNode(); + int fieldTargetNode = addNode(); + addClass((void *)&node_tags[classNode], "Lcom/rc/statics/Holder;"); + + script = { + // No GC-root path to fieldTargetNode at all - it is reachable only via classNode's static + // field. + {JVMTI_HEAP_REFERENCE_STATIC_FIELD, classNode, fieldTargetNode, -1}, + }; + + bool truncated = true; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_FALSE(truncated); + + // classNode got tagged negative by the real resolveLoadedClasses() scan (not manually, unlike + // PreTaggedClassObjectsAreNeverExpandedOrAdmitted above), and was never itself admitted as a + // frontier object. + EXPECT_LT(node_tags[classNode], 0); + + jlong target_tag = tags_ever_assigned[fieldTargetNode]; + ASSERT_NE(0, target_tag); + + std::vector chain; + ASSERT_TRUE(tracker->frontierTable()->reconstructChain(target_tag, &chain)); + ASSERT_EQ(1u, chain.size()); + + FrontierEntry entry{}; + ASSERT_TRUE(tracker->frontierTable()->lookup(target_tag, &entry)); + EXPECT_EQ(0, entry.parent_tag); // root-attached, not a child hop + EXPECT_EQ((u8)JVMTI_HEAP_REFERENCE_STATIC_FIELD, entry.root_kind); + + // buildChainEvent() appends the root TYPE (the declaring class) as the chain's root-side end: + // the frontier path's terminal element is the static field's holder instance, one hop below the + // root type. + ReferenceChainEvent event; + ASSERT_TRUE(tracker->buildChainEvent(&mock_jvmti, &mock_jni, target_tag, + &event)); + int expectedHolder = Profiler::instance()->lookupClass( + "com/rc/statics/Holder", strlen("com/rc/statics/Holder")); + ASSERT_NE(-1, expectedHolder); + ASSERT_EQ(2u, event._hops.size()); + EXPECT_EQ(chain[0], event._hops[0].klass_id); // the target's own class, unchanged + EXPECT_EQ((u32)expectedHolder, event._hops[1].klass_id); + // One label per hop (recordReferenceChain() drops ALL labels when any is empty); the root-type + // hop's own edge is the unlabeled root edge (field_index -1). + ASSERT_EQ(2u, event._hops.size()); + EXPECT_FALSE(event._hops[0].edge_label.empty()); + EXPECT_FALSE(event._hops[1].edge_label.empty()); + + tracker->stop(); +} + +// Regression test for the resolveLoadedClasses() scan-skip guard: it must compare `class_count != +// _last_resolved_class_count`, not `class_count > _last_resolved_class_count`. +TEST_F(ReferenceChainsBfsTest, ResolveLoadedClassesRescansAfterClassCountShrinksAndPartiallyRegrows) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + void *classA = (void *)0x3001, *classB = (void *)0x3002, *classC = (void *)0x3003; + addClass(classA, "Lcom/rc/regress/A;"); + int idxB = addClass(classB, "Lcom/rc/regress/B;"); + + // Pass 1: both A and B loaded (count == 2) - both get resolved/tagged. + ReferenceChainsTestAccessor::resolveLoadedClasses(&mock_jvmti, &mock_jni); + EXPECT_EQ(2, ReferenceChainsTestAccessor::lastResolvedClassCount()); + ASSERT_NE(0u, tags.count(classA)); + ASSERT_NE(0u, tags.count(classB)); + EXPECT_NE(0, tags[classA]); + EXPECT_NE(0, tags[classB]); + + // Simulate B's classloader being GC'd: GetLoadedClasses() now reports only A (count shrinks 2 + // -> 1), exactly like a real class unload. + classes.erase(classes.begin() + idxB); + ReferenceChainsTestAccessor::resolveLoadedClasses(&mock_jvmti, &mock_jni); + EXPECT_EQ(1, ReferenceChainsTestAccessor::lastResolvedClassCount()); + + // Simulate a *different* class C loading back in, bringing the count back to 2 - the same count + // as pass 1's peak, but not the same class set. + addClass(classC, "Lcom/rc/regress/C;"); + ReferenceChainsTestAccessor::resolveLoadedClasses(&mock_jvmti, &mock_jni); + EXPECT_EQ(2, ReferenceChainsTestAccessor::lastResolvedClassCount()); + ASSERT_NE(0u, tags.count(classC)); + EXPECT_NE(0, tags[classC]); // the regression this test guards against + + tracker->stop(); +} + +// Incremental resumption across passes (ReferenceChainTracker:: + +TEST_F(ReferenceChainsBfsTest, MultiPassResumptionReconstructsChainAcrossPasses) { + Arguments args; + // budget=1 forces each pass to admit at most one new frontier entry, so this 3-hop chain cannot + // be discovered within a single pass - exercising expandFrontier() (resumed passes), not just + // the first pass's root walk. + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1:firstpassbudget=1")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int nodeA = addNode(); + int nodeB = addNode(); + int nodeTarget = addNode(); + + script = { + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodeA, -1}, + {JVMTI_HEAP_REFERENCE_FIELD, nodeA, nodeB, -1}, + {JVMTI_HEAP_REFERENCE_FIELD, nodeB, nodeTarget, -1}, + }; + + // Drive the search to completion one pass at a time, exactly as threadLoop() would once wired + // up (each call bounded by `budget`). + bool truncated = true; + int passes_issued = 0; + while (tracker->searchState() == SearchState::RUNNING && passes_issued < 20) { + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + passes_issued++; + } + + ASSERT_EQ(SearchState::COMPLETED, tracker->searchState()); + EXPECT_GT(tracker->passesRun(), 1); // did not fit in a single pass + EXPECT_EQ(tracker->passesRun(), passes_issued); + + jlong targetTag = tags_ever_assigned[nodeTarget]; + ASSERT_NE(0, targetTag); + std::vector chain; + ASSERT_TRUE(tracker->frontierTable()->reconstructChain(targetTag, &chain)); + // Depth/parent_tag linkage survived resumption intact - all 3 hops walk back to a root-attached + // (depth 0) entry, which reconstructChain() requires to succeed at all (see its own "reaching + // parent_tag == 0" contract). + EXPECT_EQ(3u, chain.size()); + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, FrontierCapHitAbandonsImmediately) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000:framecap=1")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + ASSERT_EQ(1, tracker->frontierTable()->maxCapacity()); + + int nodeA = addNode(); + int nodeB = addNode(); + + script = { + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodeA, -1}, // fits (the one slot) + {JVMTI_HEAP_REFERENCE_FIELD, nodeA, nodeB, -1}, // frontier cap hit + }; + + bool truncated = false; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_TRUE(truncated); + // Frontier-size cap hit -- the search abandons immediately (design doc's Termination-section + // priority 1). + ASSERT_EQ(SearchState::ABANDONED, tracker->searchState()); + ASSERT_EQ(SearchAbandonReason::FRONTIER_CAP, tracker->abandonReason()); + + // nodeA was admitted (frontier cap=1 allowed one entry), then its tag was released as part of + // this same pass's abandon handling (the mock GetObjectsWithTags() succeeds by default). + EXPECT_NE(0, tags_ever_assigned[nodeA]); + EXPECT_EQ(0, node_tags[nodeA]); + // nodeB was never admitted (frontier cap hit). + EXPECT_EQ(0, node_tags[nodeB]); + + tracker->stop(); +} + +// releaseSearchTags()'s GetObjectsWithTags() call failing must NOT be treated as "released" - see +// that method's own comment for why: marking a tag ABANDONED (or resetting _next_tag on restart) +// while its object might still be live would let a restarted search's fresh tags collide with it, +// corrupting FrontierTable's tag-uniqueness invariant. +TEST_F(ReferenceChainsBfsTest, ReleaseSearchTagsFailureBlocksTagReuseUntilItSucceeds) { + Arguments args; + // framecap=1 with a self-cycle: pass 1 admits nodeA (the frontier's only slot); the + // nodeA->nodeA edge then finds nodeA ALREADY_ADMITTED rather than hitting the frontier cap (no + // new slot is needed for an edge back to an already-tagged object), so the search stays RUNNING + // and only the no-progress detector - after NO_PROGRESS_PASS_LIMIT stale passes - can abandon + // it. + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000:framecap=1:ttl=0")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int nodeA = addNode(); + // nodeA -> nodeA self-cycle. With framecap=1, pass 1 admits nodeA; the self-cycle edge is + // ALREADY_ADMITTED, not a fresh frontier slot. + script = { + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodeA, -1}, + {JVMTI_HEAP_REFERENCE_FIELD, nodeA, nodeA, -1}, // self-cycle + }; + + long long failedBefore = + Counters::getCounter(REFERENCE_CHAIN_TAG_RELEASE_FAILED); + + fail_get_objects_with_tags = true; + bool truncated = false; + // Pass 1: admits nodeA; the self-cycle keeps the pass truncated without growing the frontier + // further. + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_TRUE(truncated); + ASSERT_EQ(SearchState::RUNNING, tracker->searchState()); + EXPECT_NE(0, tags_ever_assigned[nodeA]); + EXPECT_NE(0, node_tags[nodeA]); + + // Run enough stale passes to trigger no-progress abandonment. + for (int i = 1; i < ReferenceChainTracker::NO_PROGRESS_PASS_LIMIT + 1; i++) { + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + ASSERT_EQ(SearchState::RUNNING, tracker->searchState()); + } + // The next pass should abandon via no-progress. + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + ASSERT_EQ(SearchState::ABANDONED, tracker->searchState()); + + // GetObjectsWithTags() failed - nodeA's still-live tag must NOT have been cleared, and the + // failure must be counted. + EXPECT_NE(0, tags_ever_assigned[nodeA]); + EXPECT_NE(0, node_tags[nodeA]) << "tag must not be cleared when the " + "release batch itself failed"; + EXPECT_FALSE(ReferenceChainsTestAccessor::tagsReleased()); + EXPECT_EQ(failedBefore + 1, + Counters::getCounter(REFERENCE_CHAIN_TAG_RELEASE_FAILED)); + + // While the release is still outstanding, shouldRunPass() must force a retry unconditionally. + EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(1)); + EXPECT_EQ(SearchState::ABANDONED, tracker->searchState()) + << "must retry the release in place, not restart, while tags are " + "still unreleased"; + + // A further runPass() call retries the release; still failing. + int passesBefore = tracker->passesRun(); + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_EQ(passesBefore, tracker->passesRun()); + EXPECT_NE(0, node_tags[nodeA]); + EXPECT_FALSE(ReferenceChainsTestAccessor::tagsReleased()); + EXPECT_EQ(failedBefore + 2, + Counters::getCounter(REFERENCE_CHAIN_TAG_RELEASE_FAILED)); + + // Once GetObjectsWithTags() starts succeeding again. + fail_get_objects_with_tags = false; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_EQ(0, node_tags[nodeA]); + EXPECT_TRUE(ReferenceChainsTestAccessor::tagsReleased()); + EXPECT_EQ(failedBefore + 2, + Counters::getCounter(REFERENCE_CHAIN_TAG_RELEASE_FAILED)) + << "a successful release must not itself count as a failure"; + + tracker->stop(); +} + +// Regression test for the CANARY_STUCK/frontier-wipe convergence bug: prior to this fix, +// runPass()'s canary-stuck branch fired purely off _passes_since_last_candidate_progress, so a +// search whose candidate simply had not been found yet was abandoned - and its frontier +// destructively wiped by the next restartSearch() - after only CANARY_NO_PROGRESS_PASS_LIMIT +// passes, even while the whole-graph frontier was still growing every single pass. +TEST_F(ReferenceChainsBfsTest, CanaryStuckRequiresWholeGraphFrontierAlsoStalled) { + Arguments args; + // budget=1: exactly one new frontier admission per pass, so the frontier grows every single + // pass for as long as the chain has unexplored nodes left - _passes_since_last_progress never + // leaves 0. + ASSERT_FALSE(args.parse( + "referencechains=true:hops=200:budget=1:ttl=0:firstpassbudget=1")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + // A chain longer than the number of passes driven below, so the frontier still has pending work + // - and is still growing one node per pass - at every pass this test checks. + constexpr int kChainLength = 50; + std::vector nodes; + for (int i = 0; i < kChainLength; i++) { + nodes.push_back(addNode()); + } + script.push_back({JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodes[0], -1}); + for (int i = 1; i < kChainLength; i++) { + script.push_back( + {JVMTI_HEAP_REFERENCE_FIELD, nodes[i - 1], nodes[i], -1}); + } + + // A canary candidate that this graph never actually contains (no node is ever tagged with the + // candidate's marker tag) - the candidate-specific stuck counter + // (_passes_since_last_candidate_progress) climbs every pass with zero discovery progress, + // exactly like the live-pod scenario chasing a candidate deeper than the old fixed + // CANARY_NO_PROGRESS_PASS_LIMIT (30) passes could reach. + ReferenceChainsTestAccessor::setCandidateCountForTest(1); + + bool truncated = true; + // One more pass than the old fixed CANARY_NO_PROGRESS_PASS_LIMIT: long enough that the pre-fix + // single-condition check would already have abandoned the search, but short enough that the + // 40-node chain still has unexplored work left, so the frontier is still genuinely growing + // every pass. + for (int i = 0; i < ReferenceChainTracker::CANARY_NO_PROGRESS_PASS_LIMIT + 2; + i++) { + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + ASSERT_EQ(0, tracker->passesSinceLastProgressForTest()) + << "pass " << i << ": frontier must still be growing every pass"; + ASSERT_EQ(SearchState::RUNNING, tracker->searchState()) + << "pass " << i + << ": a canary search must not be abandoned while the " + "whole-graph frontier is still growing, even if its specific " + "candidate has not yet been found"; + } + // The narrower candidate-stuck counter climbed the whole time - this is what the old, + // single-condition check would have abandoned on alone. + EXPECT_GE(ReferenceChainsTestAccessor::passesSinceLastCandidateProgress(), + ReferenceChainTracker::CANARY_NO_PROGRESS_PASS_LIMIT); + + tracker->stop(); +} + +// Canary-lane backoff pacing (option A): a chase with unresolved candidates runs back-to-back only +// while it is fresh or making candidate progress; each pass with NO candidate progress doubles the +// spacing multiplier (_canary_backoff_mult) up to CANARY_BACKOFF_MULT_MAX, progress resets it to 1, +// and the OOM urgency ramp overrides the gate entirely. +TEST_F(ReferenceChainsBfsTest, CanaryLaneBacksOffWithoutProgressAndResetsOnProgress) { + Arguments args; + // Same shape as CanaryStuckRequiresWholeGraphFrontierAlsoStalled above: budget=1 with a long + // chain keeps the frontier growing one node per pass, so the CANARY_STUCK detector (which also + // requires a stalled frontier) never fires and the chase stays RUNNING through the whole loop + // below. + ASSERT_FALSE(args.parse( + "referencechains=true:hops=200:budget=1:ttl=0:firstpassbudget=1")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + constexpr int kChainLength = 50; + std::vector nodes; + for (int i = 0; i < kChainLength; i++) { + nodes.push_back(addNode()); + } + script.push_back({JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodes[0], -1}); + for (int i = 1; i < kChainLength; i++) { + script.push_back( + {JVMTI_HEAP_REFERENCE_FIELD, nodes[i - 1], nodes[i], -1}); + } + + ReferenceChainsTestAccessor::setCandidateCountForTest(1); + bool truncated = true; + + // Pass 1: the candidate admission itself raises the progress mark (0 -> 1), so this counts as + // progress and the multiplier stays at 1 - back-to-back. + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_EQ(1, ReferenceChainsTestAccessor::canaryBackoffMult()); + EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(OS::nanotime())) + << "a fresh chase must be allowed to run back-to-back"; + + // Pass 2: no candidate progress -> first doubling (1 -> 2). + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_EQ(2, ReferenceChainsTestAccessor::canaryBackoffMult()); + ReferenceChainsTestAccessor::setCanaryBackoffForTest( + /*mult=*/2, /*ema_ms=*/100, OS::nanotime()); + EXPECT_FALSE(ReferenceChainsTestAccessor::shouldRunPass(OS::nanotime())) + << "a no-progress canary pass must hold off the next one"; + // Beyond the spacing, the chase is allowed again - the backoff paces, it never abandons. + EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass( + ReferenceChainsTestAccessor::lastCanaryPassNs() + + 2ULL * 100ULL * 1000000ULL + 1)) + << "elapsed spacing must re-admit the canary pass"; + + // The OOM urgency ramp overrides the backoff gate entirely. + ReferenceChainsTestAccessor::setOomRampActive(true); + EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(OS::nanotime())) + << "urgency must bypass the canary backoff"; + ReferenceChainsTestAccessor::setOomRampActive(false); + + // Consecutive no-progress passes double the multiplier up to the cap (seeded at 8 so one more + // pass reaches it, the next holds it). + ReferenceChainsTestAccessor::setCanaryBackoffForTest( + /*mult=*/8, /*ema_ms=*/100, OS::nanotime()); + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_EQ(ReferenceChainTracker::CANARY_BACKOFF_MULT_MAX, + ReferenceChainsTestAccessor::canaryBackoffMult()); + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_EQ(ReferenceChainTracker::CANARY_BACKOFF_MULT_MAX, + ReferenceChainsTestAccessor::canaryBackoffMult()) + << "the multiplier must hold at its cap, not grow past it"; + + // Candidate progress (a new candidate admitted into a slot) resets the lane to back-to-back. + ReferenceChainsTestAccessor::setCandidateCountForTest(2); + ReferenceChainsTestAccessor::setCandidateKlassIdForTest(1, /*klass_id=*/987); + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_EQ(1, ReferenceChainsTestAccessor::canaryBackoffMult()) + << "candidate progress must reset the spacing multiplier to 1"; + EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(OS::nanotime())); + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, ChainCompletesWithoutAbandonment) { + Arguments args; + // budget=1 on a graph where each pass admits exactly one new edge until the chain is exhausted, + // then the frontier stops growing. + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1:ttl=0:firstpassbudget=1")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int nodeA = addNode(); + int nodeB = addNode(); + int nodeC = addNode(); + int nodeD = addNode(); + + // A chain one node longer than either pass's 1-edge expand budget can fully drain in a single + // call, so each pass still ends truncated (see mock_FollowReferences()'s own comment: an + // array-holder walk chains through as many script edges as it can admit before budget aborts + // it) and there is still pending work left for the no-progress check to catch once the chain is + // fully discovered. + script = { + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodeA, -1}, + {JVMTI_HEAP_REFERENCE_FIELD, nodeA, nodeB, -1}, + {JVMTI_HEAP_REFERENCE_FIELD, nodeB, nodeC, -1}, + {JVMTI_HEAP_REFERENCE_FIELD, nodeC, nodeD, -1}, + }; + + bool truncated = false; + // Pass 1: root enum admits nodeA, expand admits nodeB, aborts on nodeB->nodeC for lack of + // budget - truncated, frontier grew (progress). + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_TRUE(truncated); + ASSERT_EQ(SearchState::RUNNING, tracker->searchState()); + + // Pass 2: admits nodeC, aborts on nodeC->nodeD - still truncated, still growing (progress). + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_TRUE(truncated); + ASSERT_EQ(SearchState::RUNNING, tracker->searchState()); + + // Pass 3: admits nodeD, chain exhausted - no longer truncated, no pending frontier, natural + // completion. + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_FALSE(truncated); + EXPECT_EQ(SearchState::COMPLETED, tracker->searchState()); + + // Tags released. + EXPECT_NE(0, tags_ever_assigned[nodeA]); + EXPECT_EQ(0, node_tags[nodeA]); + EXPECT_NE(0, tags_ever_assigned[nodeB]); + EXPECT_EQ(0, node_tags[nodeB]); + + // No-progress (not TTL) is reported as the reason when the frontier stalls. + EXPECT_EQ(SearchAbandonReason::NONE, tracker->abandonReason()); + EXPECT_EQ(SearchState::COMPLETED, tracker->searchState()); + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, NoProgressAbandonsSearchAndReleasesTags) { + // Verify the progress-based abandonment wiring: the no-progress limit is accessible, positive, + // and reset to 0 by resetSearchStateForTest(). + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1:ttl=0:firstpassbudget=1")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + // Verify the no-progress limit is accessible and positive. + EXPECT_GT(ReferenceChainTracker::NO_PROGRESS_PASS_LIMIT, 0); + + // Verify that a fresh search starts with zero passes since last progress. + EXPECT_EQ(0, tracker->passesSinceLastProgressForTest()); + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, ResolveOrDropPrunesDeadFrontierEntries) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1:firstpassbudget=1")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int nodeA = addNode(); + int nodeB = addNode(); + // A second root that root enumeration's own 1-unit budget (firstpassbudget=1) can't reach this + // pass - the resulting root-enum truncation makes runPassManualWalk() return before + // expandFrontier() ever runs (see its own comment on frontier-cap-hit/budget-exhausted + // root-enum truncation), so nodeA is admitted but never gets a chance to expand nodeA->nodeB. + int decoyRoot = addNode(); + + script = { + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodeA, -1}, + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, decoyRoot, -1}, + {JVMTI_HEAP_REFERENCE_FIELD, nodeA, nodeB, -1}, + }; + + bool truncated = false; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); // pass 1 + ASSERT_TRUE(truncated); + ASSERT_EQ(SearchState::RUNNING, tracker->searchState()); + + jlong aTag = tags_ever_assigned[nodeA]; + ASSERT_NE(0, aTag); + // Simulate nodeA dying (collected) between pass 1 and pass 2 - GetObjectsWithTags will no + // longer report it as live. + dead_tags.insert(aTag); + + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); // pass 2: resolve-or-drop + EXPECT_FALSE(truncated); + // The dead branch was pruned for free - with nothing else pending, the search completes rather + // than staying RUNNING or being ABANDONED. + EXPECT_EQ(SearchState::COMPLETED, tracker->searchState()); + EXPECT_EQ(2, tracker->passesRun()); + + FrontierEntry entry{}; + ASSERT_TRUE(tracker->frontierTable()->lookup(aTag, &entry)); + EXPECT_EQ(FrontierEntryState::ABANDONED, entry.state); + + // nodeB was never discovered - nodeA's subtree was pruned, not expanded. + EXPECT_EQ(0, tags_ever_assigned[nodeB]); + + tracker->stop(); +} + +// pollWatchedTargets() (design doc's Open Question 3 bridging + +// =========================================================================== Pod-in-a-jar system +// harness (design node: design-pod-in-a-jar-harness; meta-whackamole-analysis): the REAL tracker +// loop (shouldRunPass -> runPass -> pollWatchedTargets, the exact threadLoop body) driven over the +// scripted mock heap, asserting SYSTEM INVARIANTS instead of unit symptoms. diff --git a/ddprof-lib/src/test/cpp/referenceChainsCoreTests.inc b/ddprof-lib/src/test/cpp/referenceChainsCoreTests.inc new file mode 100644 index 000000000..ba561244d --- /dev/null +++ b/ddprof-lib/src/test/cpp/referenceChainsCoreTests.inc @@ -0,0 +1,1288 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +TEST(RcDebugLevelTest, ParseAcceptsTrimmedSingleDigits) { + EXPECT_EQ(parseRcDebugLevel(nullptr), -1); + EXPECT_EQ(parseRcDebugLevel(""), -1); + EXPECT_EQ(parseRcDebugLevel("0"), 0); + EXPECT_EQ(parseRcDebugLevel("1"), 1); + EXPECT_EQ(parseRcDebugLevel("2"), 2); + EXPECT_EQ(parseRcDebugLevel(" 2\n"), 2); // the `echo 2 >` form + EXPECT_EQ(parseRcDebugLevel("\t1\r\n"), 1); + EXPECT_EQ(parseRcDebugLevel("3"), -1); + EXPECT_EQ(parseRcDebugLevel("12"), -1); + EXPECT_EQ(parseRcDebugLevel("-1"), -1); + EXPECT_EQ(parseRcDebugLevel("abc"), -1); + EXPECT_EQ(parseRcDebugLevel("2 garbage"), -1); +} + +TEST(RcDebugLevelTest, ReadFileParsesTrimmedAndRejectsInvalid) { + char path[] = "/tmp/rc_dbg_test_XXXXXX"; + int fd = mkstemp(path); + ASSERT_GE(fd, 0); + close(fd); + struct Case { + const char *content; + int expected; + }; + const Case cases[] = { + {"2\n", 2}, {"1", 1}, {" 2 ", 2}, {"\n1\n", 1}, + {"", -1}, {"3", -1}, {"22", -1}, {"abc", -1}, {"x", -1}, + }; + for (const Case &c : cases) { + FILE *f = fopen(path, "w"); + ASSERT_NE(f, nullptr); + EXPECT_GE(fputs(c.content, f), 0); // non-negative on success + fclose(f); + EXPECT_EQ(readRcDebugLevelFile(path), c.expected) << "content='" << c.content << "'"; + } + unlink(path); + EXPECT_EQ(readRcDebugLevelFile(path), -1); // now missing + EXPECT_EQ(readRcDebugLevelFile(nullptr), -1); +} + +TEST(RcDebugLevelTest, ReadFileRejectsSymlinkedOverride) { + // The knob path lives under world-writable /tmp: a symlink planted by a + // local user must not be followed (see readRcDebugLevelFile's lstat + // guard) - only regular files owned by root or the current user are read. + char target[] = "/tmp/rc_dbg_target_XXXXXX"; + int fd = mkstemp(target); + ASSERT_GE(fd, 0); + ASSERT_GE(write(fd, "2\n", 2), 0); + close(fd); + char link[] = "/tmp/rc_dbg_link_XXXXXX"; + int lfd = mkstemp(link); + ASSERT_GE(lfd, 0); + close(lfd); + ASSERT_EQ(unlink(link), 0); + ASSERT_EQ(symlink(target, link), 0); + EXPECT_EQ(readRcDebugLevelFile(link), -1) // symlink itself: rejected + << "symlinked override must not be followed"; + EXPECT_EQ(readRcDebugLevelFile(target), 2); // plain regular file: accepted + unlink(link); + unlink(target); +} + +TEST(RcDebugLevelTest, RefreshFollowsEnvWhenNoOverrideFile) { + // The refresh's fixed override path is machine-global; skip rather than flake on a developer + // machine that happens to have the file. + if (access("/tmp/ddprof_root/refchains_debug_level", F_OK) == 0) { + GTEST_SKIP() << "override file present on this machine"; + } + setenv("DD_PROFILING_REFERENCE_CHAINS_DEBUG", "1", 1); + rcDebugLevelRefresh(true); + EXPECT_EQ(rcDebugLevel(), 1); + setenv("DD_PROFILING_REFERENCE_CHAINS_DEBUG", "2", 1); + rcDebugLevelRefresh(true); + EXPECT_EQ(rcDebugLevel(), 2); + // invalid env value means silent, not "keep the previous level" + setenv("DD_PROFILING_REFERENCE_CHAINS_DEBUG", "bogus", 1); + rcDebugLevelRefresh(true); + EXPECT_EQ(rcDebugLevel(), 0); + // restore the pinned level for any later test's diagnostics + setenv("DD_PROFILING_REFERENCE_CHAINS_DEBUG", "2", 1); + rcDebugLevelRefresh(true); + EXPECT_EQ(rcDebugLevel(), 2); +} + +// VMTestAccessor - friend of VM (vmEntry.h), lets tests swap VM::_jvmti for a +class VMTestAccessor { +public: + static jvmtiEnv* getJvmti() { return VM::_jvmti; } + static void setJvmti(jvmtiEnv* env) { VM::_jvmti = env; } +}; + +// ReferenceChainsTestAccessor - same pattern as VMTestAccessor above, for the +class ReferenceChainsTestAccessor { +public: + static void reset() { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + delete t->_frontier; + t->_frontier = nullptr; + t->_class_tags = ClassTagTable(); + t->_last_resolved_class_count = 0; + // Without these two, a prior test's fully-swept (or partially-swept) + // admitStaticFieldRoots() state survives in this process-wide singleton and can wrongly + // skip the sweep entirely on this test's first pass if its resolved class count happens to + // match whatever an earlier test last left behind - see resetForRestart()'s identical reset + // of these same fields for the production-restart equivalent of this same contract. + t->_last_static_field_class_count = -1; + t->_static_field_sweep_cursor = 0; + t->_static_field_sweep_cycle_truncated = false; + t->_next_tag = 1; + // Shared with LivenessTracker (classTagAllocator.h) - process-wide, not + // per-ReferenceChainTracker-instance, so it needs its own reset seam rather than being a + // plain member write. + ClassTagAllocator::resetForTest(); + t->_search_started = false; + t->_tags_released = true; + t->_search_state = SearchState::RUNNING; + t->_abandon_reason = SearchAbandonReason::NONE; + t->_search_start_ns = 0; + t->_pending_expand.clear(); + t->_priority_expand.clear(); + t->_priority_expand_set.clear(); + // B' at-risk FIFO + its indexes/counters, and the round-15 fresh lane: production clears + // all of these on every restartSearch()/ resetSearchStateForTest(), but this seam predates + // the FIFO and was never given the clears - until round 16 a test left at-risk entries + // behind and only survived because LATER tests' own runPass()s drained the residue with the + // full Bfs mock. + t->_static_anchor_fifo.clear(); + t->_static_anchor_fifo_set.clear(); + t->_static_anchor_fifo_klass_counts.clear(); + t->_static_anchor_fresh_queue.clear(); + t->_last_pass_gc_finish_epoch = 0; + t->_last_pass_ns = 0; + t->_passes_run = 0; + t->_passes_since_last_progress = 0; + t->_candidate_count = 0; + t->_candidate_found_bits = 0; + memset(t->_candidate_discovered_count, 0, sizeof(t->_candidate_discovered_count)); + t->_passes_since_last_candidate_progress = 0; + t->_last_candidate_progress_mark = 0; + t->_canary_stuck_restart_count = 0; + t->_resolved_chains.clear(); + t->_safepoint_pain_budget = PainBudget(); + t->_cpu_pain_budget = PainBudget(); + t->_search_pain_ms = 0; + t->_root_kind_rotation_cursor = 1; + t->_stale_expanded_rotation_cursor = 1; + t->_static_anchor_index.clear(); + t->_static_anchor_own_class_tags.clear(); + t->_static_anchor_index_tags.clear(); + t->_anchor_container_cursor = 0; + t->_anchor_other_cursor = 0; + t->_class_shape_cache.clear(); + t->_thread_walk_anchor_cursor = 0; + memset(t->_candidate_qualifying_tid_count, 0, + sizeof(t->_candidate_qualifying_tid_count)); + t->_hop_label_cache.clear(); + t->_watched_leak_klass_count = 0; + t->_leak_signature_totals.clear(); + t->_leak_signature_prev_totals.clear(); + t->_leak_parent_fanout.clear(); + t->_borrowed_budget = 0; + t->_consecutive_under_target_passes = 0; + // Adaptive batch + lane state: NOT covered by anything above, and a prior test that drove + // expansion leaves a non-zero EMA, a live batch size, a stale pass deadline, and/or a + // mid-alternation lane toggle behind - all of which silently change the next test's + // expandFrontier() arithmetic (exact-value asserts on batch sizing only pass standalone + // otherwise). + t->_gotw_ema_call_ns = 0; + t->_gotw_batch_size = 0; + t->_pass_deadline_ns = 0; + t->_expand_lane_prefer_priority = true; + } + + // Search restart + pain budget (SearchRestartTest below) - same rationale as the pacing + // accessors above: private state a test needs to drive/observe directly. + static bool canAffordNewSearch(u64 now_ns) { + return ReferenceChainTracker::instance()->canAffordNewSearch(now_ns); + } + + static bool shouldRunPass(u64 now_ns) { + return ReferenceChainTracker::instance()->shouldRunPass(now_ns); + } + + static void setSearchPainMs(u64 ms) { + ReferenceChainTracker::instance()->_search_pain_ms = ms; + } + + static void setCandidateFrontierTagForTest(int idx, jlong tag) { + ReferenceChainTracker::instance()->setCandidateFrontierTagForTest(idx, tag); + } + static void setCandidateParentTagForTest(int idx, jlong tag) { + ReferenceChainTracker::instance()->setCandidateParentTagForTest(idx, tag); + } + static void setCandidateReferrerKlassForTest(int idx, u32 klass_id) { + ReferenceChainTracker::instance()->setCandidateReferrerKlassForTest(idx, klass_id); + } + static void setCandidateDepthForTest(int idx, u32 depth) { + ReferenceChainTracker::instance()->setCandidateDepthForTest(idx, depth); + } + + static void setCandidateCountForTest(int n) { + ReferenceChainTracker::instance()->setCandidateCountForTest(n); + } + + // Canary-lane backoff state wrappers - see _canary_backoff_mult's own comment + // (referenceChains.h). + static int canaryBackoffMult() { + return ReferenceChainTracker::instance()->canaryBackoffMultForTest(); + } + static u64 lastCanaryPassNs() { + return ReferenceChainTracker::instance()->lastCanaryPassNsForTest(); + } + static void setCanaryBackoffForTest(int mult, u64 ema_ms, u64 last_pass_ns) { + ReferenceChainTracker::instance()->setCanaryBackoffForTest(mult, ema_ms, + last_pass_ns); + } + static void setOomRampActive(bool active) { + ReferenceChainTracker::instance()->setOomRampActiveForTest(active); + } + + static int passesSinceLastCandidateProgress() { + return ReferenceChainTracker::instance()->passesSinceLastCandidateProgressForTest(); + } + + static int canaryStuckRestartCount() { + return ReferenceChainTracker::instance()->canaryStuckRestartCountForTest(); + } + + static u64 searchPainMs() { + return ReferenceChainTracker::instance()->_search_pain_ms; + } + + // Resolved-chain cache: read-only size peek and a pass-through to the private snapshot + // (drainPendingChainEvents()) and insert (cacheResolvedChain()), for ResolvedChainCacheTest + // below - same rationale as hasResolvedChainForTag()/resolvedChainCount() below. + static size_t resolvedChainCount() { + return ReferenceChainTracker::instance()->_resolved_chains.size(); + } + + static void drain(std::vector *out) { + ReferenceChainTracker::instance()->drainPendingChainEvents(out); + } + + static void cacheChain(jlong source_tag, ReferenceChainEvent event, + jlong source_tag_val, u64 source_search_ns) { + ReferenceChainTracker::instance()->cacheResolvedChain( + source_tag, std::move(event), source_tag_val, source_search_ns); + } + + static int maxResolvedChains() { + return ReferenceChainTracker::MAX_RESOLVED_CHAINS; + } + + // Target-selection bridging step: read-only peeks into the resolved-chain cache, for asserting + // exactly which klass a chain was resolved for and the tag it was reconstructed from - see + // PollWatchedTargetsTest below. + static bool hasResolvedChainForTag(jlong tag) { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + return t->_resolved_chains.find(tag) != t->_resolved_chains.end(); + } + + static jlong resolvedChainSourceTag(jlong tag) { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + auto it = t->_resolved_chains.find(tag); + return it == t->_resolved_chains.end() ? 0 : it->second.source_tag; + } + + // Leak-tag correlation (design C): read a frontier entry's stored leak tag, and a pass-through + // to the private buildChainEvent(), for LeakTagInterceptionTest below - same friend-accessor + // rationale as hasResolvedChainForTag() above. + static jlong frontierLeakTag(jlong tag) { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + FrontierEntry entry{}; + if (t->_frontier == nullptr || !t->_frontier->lookup(tag, &entry)) { + return -1; + } + return entry.leak_tag; + } + + static void setCandidateKlassIdForTest(int idx, u32 klass_id) { + ReferenceChainTracker::instance()->setCandidateKlassIdForTest(idx, klass_id); + } + + // Round-19: the leak-tag canary found criterion (pod 289f8 — the chase was structurally + // unresolvable after the marker->leak-tag migration; see buildDiscoveredInstanceChains' own + // comment). + static u64 candidateFoundBitsForTest() { + return ReferenceChainTracker::instance()->_candidate_found_bits; + } + + static jlong candidateFrontierTagForTest(int slot) { + return ReferenceChainTracker::instance()->_candidate_frontier_tags[slot]; + } + + static void buildDiscoveredInstanceChainsForTest(u32 klass_id, + u64 current_search_ns) { + // jvmti/jni null is safe: resolveHopEdgeLabel() null-guards and degrades hop labels to kind + // labels. + ReferenceChainTracker::instance()->buildDiscoveredInstanceChains( + nullptr, nullptr, klass_id, current_search_ns); + } + + // ---- pod-in-a-jar system harness (design-pod-in-a-jar-harness) ---- + static u8 searchStateForTest() { + return ReferenceChainTracker::instance()->_search_state; + } + + static int sweepGateResolvedCountForTest() { + return ReferenceChainTracker::instance()->_last_resolved_class_count; + } + + static int sweepGateStaticCountForTest() { + return ReferenceChainTracker::instance()->_last_static_field_class_count; + } + + static int sweepCursorForTest() { + return ReferenceChainTracker::instance()->_static_field_sweep_cursor; + } + + static int passesRunForTest() { + return ReferenceChainTracker::instance()->_passes_run; + } + + static int candidateCountForTest() { + return ReferenceChainTracker::instance()->_candidate_count; + } + + static u32 candidateKlassIdForTest(int slot) { + return ReferenceChainTracker::instance()->_candidate_klass_ids[slot]; + } + + static size_t resolvedChainCountForTest() { + return ReferenceChainTracker::instance()->_resolved_chains.size(); + } + + static std::vector resolvedChainTargetsForTest() { + std::vector out; + for (auto &kv : + ReferenceChainTracker::instance()->_resolved_chains) { + out.push_back(kv.second.event._target_tag); + } + return out; + } + + static size_t staticAnchorFreshQueueSizeForTest() { + return ReferenceChainTracker::instance() + ->_static_anchor_fresh_queue.size(); + } + + static jlong candidateDiscoveredTagForTest(int slot, int idx) { + return ReferenceChainTracker::instance()->candidateDiscoveredTagForTest(slot, idx); + } + + static int candidateDiscoveredCountForTest(int slot) { + return ReferenceChainTracker::instance()->candidateDiscoveredCountForTest(slot); + } + + // recordDiscoveredInstance()/correlateAdmittedLeakTag() are the production paths for the + // leak-correlation tests below. + static void recordDiscoveredInstanceForTest(u32 klass_id, jlong tag, + bool leak_correlated) { + ReferenceChainTracker::instance()->recordDiscoveredInstance(klass_id, tag, + leak_correlated); + } + + // Drive restartSearch() directly (the accessor base already set _tags_released, so its assert + // is satisfied). + static void restartSearchForTest() { + ReferenceChainTracker::instance()->restartSearch(); + } + + static bool anchorIndexIsEmptyForTest() { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + return t->_static_anchor_index.empty() && + t->_static_anchor_own_class_tags.empty() && + t->_static_anchor_index_tags.empty(); + } + + // Read back discovered-instance slots (frontier tags recorded by recordDiscoveredInstance). + static jlong discoveredTagForTest(int slot, int idx) { + return ReferenceChainTracker::instance() + ->_candidate_discovered_tags[slot][idx]; + } + + static int discoveredCountForTest(int slot) { + return ReferenceChainTracker::instance() + ->_candidate_discovered_count[slot]; + } + + static size_t priorityExpandCap() { + return ReferenceChainTracker::PRIORITY_EXPAND_CAP; + } + + static int maxDiscoveredPerClass() { + return ReferenceChainTracker::MAX_DISCOVERED_INSTANCES_PER_CLASS; + } + + static bool buildChainEventForTest(jvmtiEnv *jvmti, JNIEnv *jni, + jlong tag, ReferenceChainEvent *out) { + return ReferenceChainTracker::instance()->buildChainEvent(jvmti, jni, + tag, out); + } + + // Direct expandFrontier() drive for the AIMD batch test: a full runPass() drains a small graph + // to completion and its rotation phase adds extra GetObjectsWithTags calls, so per-call AIMD + // assertions cannot be made deterministic through runPass(). + static void pushPendingExpandForTest(jlong tag) { + ReferenceChainTracker::instance()->_pending_expand.push_back(tag); + } + + static void expandFrontierForTest(jvmtiEnv *jvmti, JNIEnv *jni, + int *edges_admitted) { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + bool truncated = false; + bool cap_hit = false; + u64 safepoint_ticks = 0; + t->expandFrontier(jvmti, jni, t->_hop_cap, 1000, edges_admitted, + &truncated, &cap_hit, &safepoint_ticks); + } + + // Pause-time pacing controller: read-only peeks at the controller's derived values, and a + // pass-through to the private updatePacing() itself, for ReferenceChainsPacingTest below - same + // rationale as hasResolvedChainForTag()/resolvedChainCount() above (the target-selection + // bridging step): private state a test needs to drive/ observe directly, exposed via this + // existing friend accessor rather than adding public getters/setters to ReferenceChainTracker + // itself. + static int effectiveBudget() { + return ReferenceChainTracker::instance()->_effective_budget; + } + + static u64 effectiveCadenceNs() { + return ReferenceChainTracker::instance()->_effective_cadence_ns; + } + + static void updatePacing(u64 pass_wall_ns) { + ReferenceChainTracker::instance()->updatePacing(pass_wall_ns); + } + + static u64 baselineCadenceNs() { return ReferenceChainTracker::PASS_CADENCE_NS; } + + // Test-only seams for PacingGrowsBudgetBackAndRelaxesCadenceWhenUnderCeiling below, which needs + // to start from a controlled below-ceiling/above- baseline point with a freshly reset + // controller (see that test's own comment for why chaining directly off a prior constant-input + // sequence would leave _pause_pid's integral state mid-recovery from that sequence's windup, + // muddying this method's per-step direction assertions with a transient the test is not about). + static void setEffectiveBudget(int v) { + ReferenceChainTracker::instance()->_effective_budget = v; + } + + static void setEffectiveCadenceNs(u64 v) { + ReferenceChainTracker::instance()->_effective_cadence_ns = v; + } + + static void resetPacingController() { + ReferenceChainTracker::instance()->_pause_pid.reset(); + } + + // Budget-borrowing (referenceChains.h's _borrowed_budget comment): the configured multiplier + // PacingGrowsBudgetBackAndRelaxesCadenceWhenUnderCeiling below asserts convergence against, + // instead of hardcoding it a second time in the test itself. + static int borrowCeilingMultiplier() { + return ReferenceChainTracker::BORROW_CEILING_MULTIPLIER; + } + + static int64_t borrowedBudget() { + return ReferenceChainTracker::instance()->_borrowed_budget; + } + + // MaybeRevokeBorrowForRootEnumPass* tests below: drive the borrow state directly into "already + // granted" before exercising the revocation-only seam, and a pass-through to that seam itself - + // same rationale as updatePacing()'s own accessor above. + static void setBorrowedBudget(int64_t v) { + ReferenceChainTracker::instance()->_borrowed_budget = v; + } + + static int consecutiveUnderTargetPasses() { + return ReferenceChainTracker::instance()->_consecutive_under_target_passes; + } + + static void setConsecutiveUnderTargetPasses(int v) { + ReferenceChainTracker::instance()->_consecutive_under_target_passes = v; + } + + static void maybeRevokeBorrowForRootEnumPass(u64 pass_wall_ticks) { + ReferenceChainTracker::instance()->maybeRevokeBorrowForRootEnumPass( + pass_wall_ticks); + } + + // ReleaseSearchTagsFailureTest below: read-only peek at whether the tracker still owes a tag + // release before it can allow a restart - see _tags_released's own comment. + static bool tagsReleased() { + return ReferenceChainTracker::instance()->_tags_released; + } + + // ResolveLoadedClassesRescansAfterClassCountShrinksAndPartiallyRegrows below: direct + // pass-through to the private resolveLoadedClasses(), plus a read-only peek at the count it + // stashes - the same rationale as tagsReleased() above (private state/behavior a test needs to + // drive/observe directly, without going through a full runPass()/search lifecycle that + // resolveLoadedClasses() alone does not need). + static void resolveLoadedClasses(jvmtiEnv *jvmti, JNIEnv *jni) { + ReferenceChainTracker::instance()->resolveLoadedClasses(jvmti, jni); + } + + static int lastResolvedClassCount() { + return ReferenceChainTracker::instance()->_last_resolved_class_count; + } + + // Durability re-verification test seams: direct pass-throughs to the private tie-break/rotation + // methods, plus FrontierTable::insert() itself (also private-by-convention here in the sense + // that production code only ever calls it via admitObject()) so tests can set up a frontier + // entry's exact starting root_kind/state/parent_tag without needing a live JVMTI mock for + // IterateOverReachableObjects/FollowReferences (neither is mocked in this file - see the file + // header's FollowReferences- only mock rationale). + // Direct seam for buildCanaryChainEvent() - private in production (only + // pollWatchedTargets() calls it), but the bounded parent-chain walk and its + // cycle-corruption behavior are unit-testable only through it. + static bool buildCanaryChainEventForTest(int candidate_idx, + ReferenceChainEvent *out) { + return ReferenceChainTracker::instance()->buildCanaryChainEvent( + candidate_idx, out); + } + + static bool insertFrontierEntry(FrontierTable *frontier, jlong tag, + jlong parent_tag, u32 depth, u8 state, + u8 root_kind, u32 referrer_klass = 0, + jlong class_tag = 0, + jint referrer_field_index = -1, + u8 edge_kind = 0, + jlong referrer_class_tag = 0) { + return frontier->insert(tag, parent_tag, referrer_klass, depth, + state, root_kind, class_tag, + referrer_field_index, edge_kind, + referrer_class_tag); + } + + static bool maybeUpgradeRootAttachedRootKind(FrontierTable *frontier, + jlong tag, + u8 new_root_kind) { + return ReferenceChainTracker::instance() + ->maybeUpgradeRootAttachedRootKind(frontier, tag, new_root_kind); + } + + static std::vector collectStaleRootKindEntriesForRotation( + int max_count) { + return ReferenceChainTracker::instance() + ->collectStaleRootKindEntriesForRotation(max_count); + } + + static std::vector collectStaleExpandedEntriesForRotation( + int max_count) { + return ReferenceChainTracker::instance() + ->collectStaleExpandedEntriesForRotation(max_count); + } + + // Candidate-scoped reach (descendFromAnchor()/walkCandidateThreadLocals()/ + // walkStaticFieldAnchors()): direct drives for the same reason as expandFrontierForTest() above + // - a full runPass() drains a small graph to completion and its other phases add interference, + // so the walk phases are exercised on their own. + static void walkCandidateThreadLocalsForTest(jvmtiEnv *jvmti, JNIEnv *jni, + int budget, + int *edges_admitted) { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + bool truncated = false; + bool cap_hit = false; + u64 safepoint_ticks = 0; + t->walkCandidateThreadLocals(jvmti, jni, budget, edges_admitted, + &truncated, &cap_hit, &safepoint_ticks); + } + + static void walkStaticFieldAnchorsForTest(jvmtiEnv *jvmti, JNIEnv *jni, + const std::vector &tags, + int budget, + int *edges_admitted) { + walkStaticAnchorFifoForTest(jvmti, jni, tags, budget, edges_admitted, + nullptr); + } + + static std::vector + collectStaticFieldAnchorsForRotationForTest(int max_count) { + return ReferenceChainTracker::instance() + ->collectStaticFieldAnchorsForRotation(max_count); + } + + static void addToStaticAnchorIndexForTest(jlong tag, jlong own_class_tag, + u8 root_kind) { + ReferenceChainTracker::instance() + ->addToStaticAnchorIndex(tag, own_class_tag, root_kind); + } + + // Prime the class-shape cache as if reconcileAnchorClassShapes() had classified `class_tag` + // (tests script shapes instead of driving the JNI interface walk, which needs real classes). + static void primeClassShapeForTest(jlong class_tag, bool container) { + ReferenceChainTracker::instance()->_class_shape_cache[class_tag] = + container + ? (u8)ReferenceChainTracker::AnchorClassShape::CONTAINER + : (u8)ReferenceChainTracker::AnchorClassShape::NON_CONTAINER; + } + + // B' at-risk static-anchor FIFO (see _static_anchor_fifo's declaration comment in + // referenceChains.h). + using AtRiskAnchor = ReferenceChainTracker::AtRiskAnchor; + static constexpr u32 kAtRiskPerKlassCap = + ReferenceChainTracker::STATIC_ANCHOR_ATRISK_PER_KLASS_CAP; + + static void pushStaticAnchorFifoForTest(jlong tag, u32 klass_id) { + ReferenceChainTracker::instance()->pushAtRiskStaticAnchor(tag, + klass_id); + } + + static int drainStaticAnchorFifoForTest( + int max_count, + std::vector &out) { + return ReferenceChainTracker::instance()->drainStaticAnchorFifo( + max_count, out); + } + + static void requeueStaticAnchorFifoFrontForTest( + const std::vector &entries) { + ReferenceChainTracker::instance()->requeueStaticAnchorFifoFront( + entries); + } + + static size_t staticAnchorFifoSizeForTest() { + return ReferenceChainTracker::instance()->_static_anchor_fifo.size(); + } + + static bool staticAnchorFifoContainsForTest(jlong tag) { + return ReferenceChainTracker::instance() + ->_static_anchor_fifo_set.contains(tag); + } + + static void walkStaticAnchorFifoForTest(jvmtiEnv *jvmti, JNIEnv *jni, + const std::vector &tags, + int budget, int *edges_admitted, + std::vector *unwalked) { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + bool truncated = false; + bool cap_hit = false; + u64 safepoint_ticks = 0; + t->walkStaticFieldAnchors(jvmti, jni, tags, budget, edges_admitted, + &truncated, &cap_hit, &safepoint_ticks, + unwalked); + } + + // Direct candidate-slot seeding (the production path fills these via pollWatchedTargets()'s + // snapshot loop - see _candidate_qualifying_tids' own comment): the walk phase tests need + // exactly one (slot, klass, tid) combination without driving LivenessTracker's hysteresis + // machinery. + static void seedCandidateSlotForTest(int slot, u32 klass_id, + const jint *tids, int tid_count) { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + t->_candidate_klass_ids[slot] = klass_id; + for (int i = 0; i < tid_count; i++) { + t->_candidate_qualifying_tids[slot][i] = tids[i]; + } + t->_candidate_qualifying_tid_count[slot] = tid_count; + if (slot + 1 > t->_candidate_count) { + t->_candidate_count = slot + 1; + } + } + + static int candidateQualifyingTidCountForTest(int slot) { + return ReferenceChainTracker::instance() + ->_candidate_qualifying_tid_count[slot]; + } + + static jlong getTagForTest(jvmtiEnv *jvmti, jobject obj) { + return ReferenceChainTracker::instance()->getTag(jvmti, obj); + } + + // Snapshot of _priority_expand's current contents, in queue order - used by tests to check for + // duplicate tags after both rotation collectors have run against it within the same simulated + // pass. + static std::vector priorityExpandContents() { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + return std::vector(t->_priority_expand.begin(), + t->_priority_expand.end()); + } + + // StaleExpandedRotationSkipsPreexistingQueueEntries below: simulates a tag left in + // _priority_expand by a prior pass's truncated expandFrontier() batch (expandFrontier()'s own + // "leave the batch at the front of the source queue for a later pass to retry" comment) without + // driving a full expandFrontier()/JVMTI round-trip to produce one. + static void pushPriorityExpand(jlong tag) { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + t->_priority_expand.push_back(tag); + t->_priority_expand_set.insert(tag); + } + + // Simulates expandFrontier() having fully drained _priority_expand at the end of a pass (the + // common case: rotation's whole selection fit within that pass's rotation_budget slice) - see + // StaleExpandedRotationStarvesHighTagEntryBehindLowTagPopulation below, which needs this to + // model collectStaleExpandedEntriesForRotation() being called fresh on each of several + // simulated passes, the way runPassManualWalk() actually does it once per real pass. + static void clearPriorityExpand() { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + t->_priority_expand.clear(); + t->_priority_expand_set.clear(); + } + + static void setRootKindRotationCursor(jlong tag) { + ReferenceChainTracker::instance()->_root_kind_rotation_cursor = tag; + } + + static jlong rootKindRotationCursor() { + return ReferenceChainTracker::instance()->_root_kind_rotation_cursor; + } + + static int rootKindRotationBudget() { + return ReferenceChainTracker::ROOT_KIND_ROTATION_BUDGET; + } + + static int staleExpandedRotationBudget() { + return ReferenceChainTracker::STALE_EXPANDED_ROTATION_BUDGET; + } + + static size_t priorityExpandSize() { + return ReferenceChainTracker::instance()->_priority_expand.size(); + } + + // Snapshot of _pending_expand's current contents, in queue order - used by the rolling-resume + // smoke test to verify that a truncated expandFrontier() batch pops fully-processed entries + // (mark EXPANDED) and leaves only the partially-processed and unvisited entries at the front of + // the queue for the next pass to retry. + static std::vector pendingExpandContents() { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + return std::vector(t->_pending_expand.begin(), + t->_pending_expand.end()); + } + + static size_t pendingExpandSize() { + return ReferenceChainTracker::instance()->_pending_expand.size(); + } + + // Self-calibrating adaptive batch size (AIMD): read/write the per-call EMA and live batch size + // so tests can verify the AIMD dynamics. + static u64 gotwEmaCallNs() { + return ReferenceChainTracker::instance()->_gotw_ema_call_ns; + } + + static void setGotwEmaCallNs(u64 v) { + ReferenceChainTracker::instance()->_gotw_ema_call_ns = v; + } + + static size_t gotwBatchSize() { + return ReferenceChainTracker::instance()->_gotw_batch_size; + } + + static void setGotwBatchSize(size_t v) { + ReferenceChainTracker::instance()->_gotw_batch_size = v; + } + + // Read-only peeks at the batch-control constants (private statics - friendship applies inside + // this class's methods, not in test bodies). + static u64 gotwCpuBudgetNs() { + return ReferenceChainTracker::GOTW_CPU_BUDGET_NS; + } + + static size_t gotwInitialBatchSize() { + return (size_t)ReferenceChainTracker::GOTW_INITIAL_BATCH_SIZE; + } + + static size_t gotwMinBatch() { + return ReferenceChainTracker::GOTW_MIN_BATCH; + } + + static size_t gotwMaxBatch() { + return ReferenceChainTracker::GOTW_MAX_BATCH; + } + + static size_t gotwBacklogMinDepth() { + return ReferenceChainTracker::GOTW_BACKLOG_MIN_DEPTH; + } + + static u64 gotwBacklogWindowMult() { + return ReferenceChainTracker::GOTW_BACKLOG_WINDOW_MULT; + } + + // gotwWindowNs() is a pure function of (remaining window, lane depth) and the seeded EMA - + // directly unit-testable without a mock JVMTI call. + static u64 gotwWindowNs(u64 remaining_ns, size_t lane_depth) { + return ReferenceChainTracker::instance()->gotwWindowNs(remaining_ns, + lane_depth); + } + + static void setPassDeadlineNs(u64 v) { + ReferenceChainTracker::instance()->_pass_deadline_ns = v; + } + + static bool expandLanePreferPriority() { + return ReferenceChainTracker::instance()->_expand_lane_prefer_priority; + } + + // Leak-tag pool range base (private static) - same friend-access rationale as the AIMD + // constants above. + static jlong leakTagBase() { + return ReferenceChainTracker::LEAK_TAG_BASE; + } + + // Leak-accumulation rotation test seams (collectLeakAccumulationCandidatesForRotation()). + static void setWatchedLeakKlassIdsForTest(const std::vector &ids) { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + int n = (int)std::min(ids.size(), + (size_t)ReferenceChainTracker::MAX_WATCHED_LEAK_KLASSES); + for (int i = 0; i < n; i++) { + t->_watched_leak_klass_ids[i] = ids[i]; + } + t->_watched_leak_klass_count = n; + } + + static void trackLeakAccumulation(FrontierTable *frontier, u32 referrer_klass, + jlong parent_tag, jlong tag) { + ReferenceChainTracker::instance()->trackLeakAccumulation( + frontier, referrer_klass, parent_tag, tag); + } + + static std::vector collectLeakAccumulationCandidatesForRotation( + int max_count) { + return ReferenceChainTracker::instance() + ->collectLeakAccumulationCandidatesForRotation(max_count); + } + + static int leakAccumulationRotationBudget() { + return ReferenceChainTracker::LEAK_ACCUMULATION_ROTATION_BUDGET; + } + + static u32 leakSignatureTotal(u32 leaf_klass_id, u32 parent_class_id) { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + u64 key = t->leakSignatureKey(leaf_klass_id, parent_class_id); + auto it = t->_leak_signature_totals.find(key); + return it != t->_leak_signature_totals.end() ? it->second : 0; + } + + static u32 leakParentFanout(jlong parent_tag) { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + auto it = t->_leak_parent_fanout.find(parent_tag); + return it != t->_leak_parent_fanout.end() ? it->second.fanout : 0; + } + + static size_t leakSignatureCount() { + return ReferenceChainTracker::instance()->_leak_signature_totals.size(); + } + + static void seedLeakAccumulationForNewlyWatchedKlass(u32 klass_id) { + ReferenceChainTracker::instance() + ->seedLeakAccumulationForNewlyWatchedKlass(klass_id); + } +}; + +static jvmtiError JNICALL mock_SetEventNotificationMode(jvmtiEnv *, jvmtiEventMode, + jvmtiEvent, jthread, ...) { + return JVMTI_ERROR_NONE; +} + +class ReferenceChainsTest : public ::testing::Test { +protected: + jvmtiInterface_1_ tbl{}; + _jvmtiEnv mock_env{}; + jvmtiEnv *orig_jvmti = nullptr; + + void SetUp() override { + orig_jvmti = VMTestAccessor::getJvmti(); + tbl = jvmtiInterface_1_{}; + tbl.SetEventNotificationMode = &mock_SetEventNotificationMode; + mock_env.functions = &tbl; + VMTestAccessor::setJvmti(&mock_env); + } + + void TearDown() override { + VMTestAccessor::setJvmti(orig_jvmti); + } +}; + +TEST_F(ReferenceChainsTest, DefaultDisabled) { + Arguments args; + EXPECT_FALSE(args._reference_chains); +} + +TEST_F(ReferenceChainsTest, FlagParsesEnabled) { + Arguments args; + Error error = args.parse("referencechains=true"); + EXPECT_FALSE(error); + EXPECT_TRUE(args._reference_chains); +} + +TEST_F(ReferenceChainsTest, FlagParsesDisabled) { + Arguments args; + Error error = args.parse("referencechains=false"); + EXPECT_FALSE(error); + EXPECT_FALSE(args._reference_chains); +} + +TEST_F(ReferenceChainsTest, FlagParsesSubOptions) { + Arguments args; + Error error = args.parse("referencechains=true:hops=64:budget=2000:ttl=5000:framecap=128"); + EXPECT_FALSE(error); + EXPECT_TRUE(args._reference_chains); + EXPECT_EQ(64, args._reference_chains_hop_cap); + EXPECT_EQ(2000, args._reference_chains_budget); + EXPECT_EQ(5000, args._reference_chains_ttl_ms); + EXPECT_EQ(128, args._reference_chains_frontier_cap); +} + +// Negative/out-of-range sub-options must be floored/clamped at the parse boundary +// (Arguments::parse(), arguments.cpp) rather than stored verbatim - see that call site's own +// comment for why an unclamped negative hops in particular is dangerous: `depth >= +// (u32)ctx->hop_cap` (referenceChains.cpp) casts a negative int to u32, wrapping to ~4e9 and +// silently disabling the hop cap entirely. +TEST_F(ReferenceChainsTest, FlagClampsNegativeSubOptions) { + Arguments args; + Error error = args.parse( + "referencechains=true:hops=-1:budget=-5:ttl=-1:framecap=-3:" + "pausetarget=-1:painbudget=-10"); + EXPECT_FALSE(error); + EXPECT_TRUE(args._reference_chains); + // Floored to a sane minimum (1), not left negative - a negative value cast to u32 downstream + // would otherwise wrap to a huge positive number. + EXPECT_GT(args._reference_chains_hop_cap, 0); + EXPECT_GT(args._reference_chains_budget, 0); + EXPECT_GT(args._reference_chains_frontier_cap, 0); + // ttl/pausetarget are floored at 0 (their own downstream gates already treat 0 as "disabled", + // so 0 - not 1 - is the correct floor). + EXPECT_GE(args._reference_chains_ttl_ms, 0); + EXPECT_GE(args._reference_chains_pause_target_ms, 0); + // painbudget is a percentage - clamped into [0, 100]. + EXPECT_GE(args._reference_chains_pain_budget_percent, 0); + EXPECT_LE(args._reference_chains_pain_budget_percent, 100); +} + +// A too-large painbudget must be clamped down to 100, not stored verbatim - the sibling of +// FlagClampsNegativeSubOptions above, for the upper bound rather than the lower one. +TEST_F(ReferenceChainsTest, FlagClampsOversizedPainBudgetPercent) { + Arguments args; + Error error = args.parse("referencechains=true:painbudget=250"); + EXPECT_FALSE(error); + EXPECT_EQ(100, args._reference_chains_pain_budget_percent); +} + +TEST_F(ReferenceChainsTest, FlagWithOtherArgsDoesNotClobberOuterParse) { + Arguments args; + Error error = args.parse("event=cpu,referencechains=true:hops=32,interval=1000000"); + EXPECT_FALSE(error); + EXPECT_TRUE(args._reference_chains); + EXPECT_EQ(32, args._reference_chains_hop_cap); + EXPECT_STREQ("cpu", args._event); + EXPECT_EQ(1000000, args._interval); +} + +TEST_F(ReferenceChainsTest, StartStopDisabledDoesNotCrash) { + Arguments args; + Error error = args.parse("referencechains=false"); + ASSERT_FALSE(error); + + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + Error startError = tracker->start(args); + EXPECT_FALSE(startError); + EXPECT_FALSE(tracker->enabled()); + tracker->stop(); +} + +TEST_F(ReferenceChainsTest, StartStopEnabledDoesNotCrash) { + Arguments args; + Error error = args.parse("referencechains=true:hops=10:budget=100"); + ASSERT_FALSE(error); + + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + Error startError = tracker->start(args); + EXPECT_FALSE(startError); + EXPECT_TRUE(tracker->enabled()); + tracker->stop(); +} + +// GC signal (GarbageCollectionStart/Finish -> epoch counters). + +TEST_F(ReferenceChainsTest, GCCallbacksIncrementEpochWhenEnabled) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + u64 startBefore = tracker->gcStartEpoch(); + u64 finishBefore = tracker->gcFinishEpoch(); + + ReferenceChainTracker::GarbageCollectionStart(nullptr); + ReferenceChainTracker::GarbageCollectionFinish(nullptr); + + EXPECT_EQ(startBefore + 1, tracker->gcStartEpoch()); + EXPECT_EQ(finishBefore + 1, tracker->gcFinishEpoch()); + + tracker->stop(); +} + +TEST_F(ReferenceChainsTest, GCCallbacksAreNoOpWhenDisabled) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=false")); + + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + ASSERT_FALSE(tracker->enabled()); + + u64 startBefore = tracker->gcStartEpoch(); + u64 finishBefore = tracker->gcFinishEpoch(); + + ReferenceChainTracker::GarbageCollectionStart(nullptr); + ReferenceChainTracker::GarbageCollectionFinish(nullptr); + + EXPECT_EQ(startBefore, tracker->gcStartEpoch()); + EXPECT_EQ(finishBefore, tracker->gcFinishEpoch()); +} + +// Tag round-trip (SetTag/GetTag/clear). + +class ReferenceChainsTagTest : public ::testing::Test { +protected: + jvmtiInterface_1_ tbl{}; + _jvmtiEnv mock_env{}; + std::unordered_map tags; + + static ReferenceChainsTagTest *active_fixture; + + void SetUp() override { + active_fixture = this; + tbl = jvmtiInterface_1_{}; + tbl.SetTag = &mock_SetTag; + tbl.GetTag = &mock_GetTag; + mock_env.functions = &tbl; + } + + void TearDown() override { + active_fixture = nullptr; + } + + static jvmtiError JNICALL mock_SetTag(jvmtiEnv *, jobject object, jlong tag) { + if (tag == 0) { + active_fixture->tags.erase(object); + } else { + active_fixture->tags[object] = tag; + } + return JVMTI_ERROR_NONE; + } + + static jvmtiError JNICALL mock_GetTag(jvmtiEnv *, jobject object, jlong *tag_ptr) { + auto it = active_fixture->tags.find(object); + *tag_ptr = it != active_fixture->tags.end() ? it->second : 0; + return JVMTI_ERROR_NONE; + } +}; + +ReferenceChainsTagTest *ReferenceChainsTagTest::active_fixture = nullptr; + +TEST_F(ReferenceChainsTagTest, TagRoundTripsThenClears) { + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + int fake_object_storage = 0; + jobject obj = reinterpret_cast(&fake_object_storage); + + jlong tag = tracker->tagObject(&mock_env, obj); + EXPECT_NE(0, tag); + EXPECT_EQ(tag, tracker->getTag(&mock_env, obj)); + + tracker->clearTag(&mock_env, obj); + EXPECT_EQ(0, tracker->getTag(&mock_env, obj)); +} + +TEST_F(ReferenceChainsTagTest, TagsAreUniqueAndNeverZero) { + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + int a = 0, b = 0; + jlong tagA = tracker->tagObject(&mock_env, reinterpret_cast(&a)); + jlong tagB = tracker->tagObject(&mock_env, reinterpret_cast(&b)); + + EXPECT_NE(0, tagA); + EXPECT_NE(0, tagB); + EXPECT_NE(tagA, tagB); +} + +TEST_F(ReferenceChainsTagTest, UntaggedObjectReadsBackZero) { + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + int untagged = 0; + EXPECT_EQ(0, tracker->getTag(&mock_env, reinterpret_cast(&untagged))); +} + +// FrontierTable (tag-indexed frontier metadata table). + +TEST(FrontierTableTest, InsertThenLookupRoundTrips) { + FrontierTable table(64); + + ASSERT_TRUE(table.insert(1, /*parent_tag=*/0, /*referrer_klass=*/7, + /*depth=*/0, FrontierEntryState::FRONTIER)); + + FrontierEntry entry{}; + ASSERT_TRUE(table.lookup(1, &entry)); + EXPECT_EQ(0, entry.parent_tag); + EXPECT_EQ(7u, entry.referrer_klass); + EXPECT_EQ(0u, entry.depth); + EXPECT_EQ(FrontierEntryState::FRONTIER, entry.state); +} + +TEST(FrontierTableTest, LookupOfNeverInsertedTagFails) { + FrontierTable table(64); + FrontierEntry entry{}; + EXPECT_FALSE(table.lookup(1, &entry)); + EXPECT_FALSE(table.lookup(5, &entry)); +} + +TEST(FrontierTableTest, NonPositiveTagIsRejected) { + FrontierTable table(64); + FrontierEntry entry{}; + EXPECT_FALSE(table.insert(0, 0, 0, 0)); + EXPECT_FALSE(table.insert(-1, 0, 0, 0)); + EXPECT_FALSE(table.lookup(0, &entry)); + EXPECT_FALSE(table.lookup(-1, &entry)); +} + +TEST(FrontierTableTest, LookupLockedRejectsNonPositiveTag) { + FrontierTable table(64); + ASSERT_TRUE(table.insert(1, 0, 7, 0, FrontierEntryState::FRONTIER)); + FrontierEntry entry{}; + // tag=0 must be rejected the same way lookup() rejects it - callers index the table with tag-1, + // so a `tag <= 0` check (not just `tag < 0`) is required to keep that subtraction from wrapping + // into a valid slot. + EXPECT_FALSE(table.lookupLocked(0, &entry)); + EXPECT_FALSE(table.lookupLocked(-1, &entry)); +} + +TEST(FrontierTableTest, LookupLockedRejectsTagPastCurrentSize) { + FrontierTable table(64); + ASSERT_TRUE(table.insert(1, 0, 7, 0, FrontierEntryState::FRONTIER)); + FrontierEntry entry{}; + // Only tag=1 has ever been inserted (table_size == 1); tag=2 maps to idx=1, exactly at the + // current size boundary, and must be rejected rather than read out of bounds. + EXPECT_FALSE(table.lookupLocked(2, &entry)); +} + +TEST(FrontierTableTest, ParentTagChainReconstructsAcrossHops) { + // Mirrors how the heap-walk engine walks parent_tag links back to a root: insert a small chain + // root(tag=1) <- mid(tag=2) <- leaf(tag=3) and confirm the links resolve in order. + FrontierTable table(64); + ASSERT_TRUE(table.insert(1, 0, 100, 0, FrontierEntryState::EDGE)); + ASSERT_TRUE(table.insert(2, 1, 200, 1, FrontierEntryState::EDGE)); + ASSERT_TRUE(table.insert(3, 2, 300, 2, FrontierEntryState::EDGE)); + + FrontierEntry entry{}; + jlong tag = 3; + std::vector chain; + while (tag != 0) { + ASSERT_TRUE(table.lookup(tag, &entry)); + chain.push_back(entry.referrer_klass); + tag = entry.parent_tag; + } + + ASSERT_EQ(3u, chain.size()); + EXPECT_EQ(300u, chain[0]); + EXPECT_EQ(200u, chain[1]); + EXPECT_EQ(100u, chain[2]); +} + +TEST(FrontierTableTest, ClearMarksAbandonedWithoutRemovingEntry) { + FrontierTable table(64); + ASSERT_TRUE(table.insert(1, 0, 7, 0, FrontierEntryState::FRONTIER)); + + table.clear(1); + + FrontierEntry entry{}; + ASSERT_TRUE(table.lookup(1, &entry)); + EXPECT_EQ(FrontierEntryState::ABANDONED, entry.state); +} + +TEST(FrontierTableTest, ClearOfNeverInsertedTagIsNoOp) { + FrontierTable table(64); + table.clear(1); // must not crash + FrontierEntry entry{}; + EXPECT_FALSE(table.lookup(1, &entry)); +} + +TEST(FrontierTableTest, GrowsPastInitialCapacityUpToMaxCap) { + // Force at least one resize by inserting beyond the small max_cap. + const int max_cap = 10; + FrontierTable table(max_cap); + ASSERT_LE(table.capacity(), max_cap); + + for (jlong tag = 1; tag <= max_cap; tag++) { + ASSERT_TRUE(table.insert(tag, tag - 1, (u32)tag, (u32)(tag - 1))) + << "insert failed for tag " << tag; + } + EXPECT_EQ(max_cap, table.capacity()); + + for (jlong tag = 1; tag <= max_cap; tag++) { + FrontierEntry entry{}; + ASSERT_TRUE(table.lookup(tag, &entry)); + EXPECT_EQ((u32)tag, entry.referrer_klass); + } +} + +TEST(FrontierTableTest, CapacityExhaustedReportsFailureInsteadOfCrashing) { + const int max_cap = 4; + FrontierTable table(max_cap); + + for (jlong tag = 1; tag <= max_cap; tag++) { + ASSERT_TRUE(table.insert(tag, 0, 0, 0)); + } + // One past max_cap must be rejected, not silently dropped-but-crashing. + EXPECT_FALSE(table.insert(max_cap + 1, 0, 0, 0)); + EXPECT_EQ(max_cap, table.capacity()); + + // Existing entries remain intact after the failed insert. + FrontierEntry entry{}; + EXPECT_TRUE(table.lookup(1, &entry)); +} + +TEST(FrontierTableTest, ZeroMaxCapRejectsEveryInsert) { + FrontierTable table(0); + EXPECT_EQ(0, table.capacity()); + EXPECT_FALSE(table.insert(1, 0, 0, 0)); +} + +TEST(FrontierTableTest, ConcurrentInsertWhileGrowingDoesNotCrash) { + // Small max_cap relative to thread/tag count forces repeated resizes while other threads are + // concurrently inserting distinct tags. + const int max_cap = 4096; + const int thread_count = 8; + const int tags_per_thread = 256; + FrontierTable table(max_cap); + + std::vector threads; + for (int t = 0; t < thread_count; t++) { + threads.emplace_back([&table, t, tags_per_thread]() { + for (int i = 0; i < tags_per_thread; i++) { + jlong tag = (jlong)t * tags_per_thread + i + 1; + table.insert(tag, 0, (u32)tag, 0); + } + }); + } + for (auto &th : threads) { + th.join(); + } + + int found = 0; + for (jlong tag = 1; tag <= (jlong)thread_count * tags_per_thread; tag++) { + FrontierEntry entry{}; + if (table.lookup(tag, &entry)) { + EXPECT_EQ((u32)tag, entry.referrer_klass); + found++; + } + } + // Every tag fits well within max_cap, so all inserts must have succeeded and be independently + // readable. + EXPECT_EQ(thread_count * tags_per_thread, found); +} + +TEST(FrontierTableTest, ReconstructChainWalksParentTagsAndPreservesState) { + FrontierTable table(64); + ASSERT_TRUE(table.insert(1, 0, 100, 0, FrontierEntryState::FRONTIER)); + ASSERT_TRUE(table.insert(2, 1, 200, 1, FrontierEntryState::FRONTIER)); + ASSERT_TRUE(table.insert(3, 2, 300, 2, FrontierEntryState::EXPANDED)); + + std::vector chain; + ASSERT_TRUE(table.reconstructChain(3, &chain)); + ASSERT_EQ(3u, chain.size()); + EXPECT_EQ(300u, chain[0]); + EXPECT_EQ(200u, chain[1]); + EXPECT_EQ(100u, chain[2]); + + // Every visited entry must KEEP its pre-walk state: the earlier EDGE demotion made + // resolved-path holders invisible to the rotation collectors (which select EXPANDED entries), + // so later leak instances behind a changed holder were never re-discovered. + for (jlong tag = 1; tag <= 2; tag++) { + FrontierEntry entry{}; + ASSERT_TRUE(table.lookup(tag, &entry)); + EXPECT_EQ(FrontierEntryState::FRONTIER, entry.state); + } + FrontierEntry entry{}; + ASSERT_TRUE(table.lookup(3, &entry)); + EXPECT_EQ(FrontierEntryState::EXPANDED, entry.state); +} + +TEST(FrontierTableTest, ReconstructChainOfNeverInsertedTagFails) { + FrontierTable table(64); + std::vector chain; + EXPECT_FALSE(table.reconstructChain(1, &chain)); +} + +// Heap-walk engine (ReferenceChainTracker::runPass()/ + diff --git a/ddprof-lib/src/test/cpp/referenceChainsEventTests.inc b/ddprof-lib/src/test/cpp/referenceChainsEventTests.inc new file mode 100644 index 000000000..314c3b4ec --- /dev/null +++ b/ddprof-lib/src/test/cpp/referenceChainsEventTests.inc @@ -0,0 +1,455 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +TEST_F(ReferenceChainsBfsTest, HopEdgeLabelsDecodeSpecFieldOrdinals) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + // Hierarchy mirroring the spec's own numbering example shape: interface IBase { int p; } 1 own + // field interface ISub extends IBase { int x; } 1 own field class Base { int base_f; } 1 own + // field, no super class Holder extends Base implements ISub { int holder_a; Object leakList; } + // interface ISink { Object CONST_A; Object CONST_B; } Spec ordinal spaces + // (jvmtiHeapReferenceInfoField): Holder (class branch): base = ISub(1) + IBase(1) = 2 + // (transitive interfaces, each once); then the superclass chain root-first: base_f@2; then own + // fields in GetClassFields order: holder_a@3, leakList@4. + void *ibase = (void *)0x5001, *isub = (void *)0x5002, *base = (void *)0x5003, + *holder = (void *)0x5004, *isink = (void *)0x5005; + addClass(ibase, "Lcom/rc/labels/IBase;"); + addClass(isub, "Lcom/rc/labels/ISub;"); + addClass(base, "Lcom/rc/labels/Base;"); + addClass(holder, "Lcom/rc/labels/Holder;"); + addClass(isink, "Lcom/rc/labels/ISink;"); + ReferenceChainsTestAccessor::resolveLoadedClasses(&mock_jvmti, &mock_jni); + // resolveLoadedClasses minted each class's raw (negative) tag via the mock's tag map - read + // them back for the decoder's tag->class lookup and the entries' referrer_class_tag values. + auto tagOf = [&](void *k) -> jlong { return tags[k]; }; + field_decode_classes[tagOf(holder)] = holder; + field_decode_classes[tagOf(isink)] = isink; + field_decode_classes[tagOf(base)] = base; + // ibase deliberately NOT registered into field_decode_classes: an unresolvable referrer class + // below must degrade to a kind label. + field_decode_hierarchy[ibase] = {true, nullptr, {}, {{(void *)0x6001, "p"}}}; + field_decode_hierarchy[isub] = + {true, nullptr, {ibase}, {{(void *)0x6002, "x"}}}; + field_decode_hierarchy[base] = {false, nullptr, {}, {{(void *)0x6003, "base_f"}}}; + field_decode_hierarchy[holder] = + {false, base, {isub}, + {{(void *)0x6004, "holder_a"}, {(void *)0x6005, "leakList"}}}; + field_decode_hierarchy[isink] = + {true, nullptr, {}, + {{(void *)0x6006, "CONST_A"}, {(void *)0x6007, "CONST_B"}}}; + + FrontierTable *frontier = tracker->frontierTable(); + // Chain: [chunk(3)] <- Base.base_f(ordinal 0 over Base's space) <- [value2(2), class Base] <- + // Holder.leakList(ordinal 4, the static root edge with the declaring class as referrer) <- + // [static value(1), class Holder] <- [class Holder (the ROOT TYPE - buildChainEvent() appends + // the root-attached entry's referrer_class_tag for static-field roots)]. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 1, 0, 0, FrontierEntryState::EDGE, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, + /*referrer_klass=*/0, /*class_tag=*/tagOf(holder), + /*referrer_field_index=*/4, /*edge_kind=*/0, + /*referrer_class_tag=*/tagOf(holder))); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 2, 1, 1, FrontierEntryState::EDGE, /*root_kind=*/0, + /*referrer_klass=*/0, /*class_tag=*/tagOf(base), + /*referrer_field_index=*/3, JVMTI_HEAP_REFERENCE_FIELD)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 3, 2, 2, FrontierEntryState::EDGE, /*root_kind=*/0, + /*referrer_klass=*/0, /*class_tag=*/tagOf(holder), + /*referrer_field_index=*/0, JVMTI_HEAP_REFERENCE_FIELD)); + + ReferenceChainEvent event; + ASSERT_TRUE(ReferenceChainsTestAccessor::buildChainEventForTest( + &mock_jvmti, &mock_jni, /*target_tag=*/3, &event)); + ASSERT_EQ(4u, event._hops.size()); + // Leaf first: chunk is retained via Base.base_f (parent entry's class is Base, ordinal 0 in + // Base's own space), then value2 via Holder's holder_a (ordinal 3 = interface offset 2 + Base's + // 1 + own position 0), then the static root edge's field name leakList (ordinal 4), then the + // root-type hop (class Holder) - the root edge itself, kind label only. + EXPECT_EQ("base_f", event._hops[0].edge_label); + EXPECT_EQ("holder_a", event._hops[1].edge_label); + EXPECT_EQ("leakList", event._hops[2].edge_label); + EXPECT_EQ("static_field", event._hops[3].edge_label); + int expectedRootType = Profiler::instance()->lookupClass( + "com/rc/labels/Holder", strlen("com/rc/labels/Holder")); + ASSERT_NE(-1, expectedRootType); + EXPECT_EQ((u32)expectedRootType, event._hops[3].klass_id); + + // Interface-referrer branch: ISink's own-field ordinals have NO superclass-chain component + // (base = superinterfaces' fields only). + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 4, 0, 0, FrontierEntryState::EDGE, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, + /*referrer_klass=*/0, /*class_tag=*/tagOf(isink), + /*referrer_field_index=*/1, /*edge_kind=*/0, + /*referrer_class_tag=*/tagOf(isink))); + ReferenceChainEvent iface_event; + ASSERT_TRUE(ReferenceChainsTestAccessor::buildChainEventForTest( + &mock_jvmti, &mock_jni, /*target_tag=*/4, &iface_event)); + // Same root-type append as above: the root-side end gains ISink (the declaring class of the + // static field) plus its kind-only root edge. + ASSERT_EQ(2u, iface_event._hops.size()); + EXPECT_EQ("CONST_B", iface_event._hops[0].edge_label); + EXPECT_EQ("static_field", iface_event._hops[1].edge_label); + int expectedSinkRoot = Profiler::instance()->lookupClass( + "com/rc/labels/ISink", strlen("com/rc/labels/ISink")); + ASSERT_NE(-1, expectedSinkRoot); + EXPECT_EQ((u32)expectedSinkRoot, iface_event._hops[1].klass_id); + + // Fail-safe: a referrer class that cannot be resolved degrades to the edge KIND label, never a + // fabricated name. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 5, 0, 0, FrontierEntryState::EDGE, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, + /*referrer_klass=*/0, /*class_tag=*/tagOf(holder), + /*referrer_field_index=*/0, /*edge_kind=*/0, + /*referrer_class_tag=*/tagOf(ibase))); + ReferenceChainEvent degraded_event; + ASSERT_TRUE(ReferenceChainsTestAccessor::buildChainEventForTest( + &mock_jvmti, &mock_jni, /*target_tag=*/5, °raded_event)); + // Both hops degrade to kind labels: the holder hop's referrer class (IBase) is deliberately + // unregistered from the decoder, and the appended root-type hop (IBase itself) carries no field + // identity. + ASSERT_EQ(2u, degraded_event._hops.size()); + EXPECT_EQ("static_field", degraded_event._hops[0].edge_label); + EXPECT_EQ("static_field", degraded_event._hops[1].edge_label); + int expectedIbaseRoot = Profiler::instance()->lookupClass( + "com/rc/labels/IBase", strlen("com/rc/labels/IBase")); + ASSERT_NE(-1, expectedIbaseRoot); + EXPECT_EQ((u32)expectedIbaseRoot, degraded_event._hops[1].klass_id); + + tracker->stop(); +} + +// PRIORITY_EXPAND_CAP backpressure: with the fast lane at the cap, the rotation collectors must +// stop pushing. +TEST_F(ReferenceChainsBfsTest, PriorityExpandCapStopsRotationCollectorPushes) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + // One eligible stale-EXPANDED entry. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 1, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + + for (size_t i = 0; i < ReferenceChainsTestAccessor::priorityExpandCap(); i++) { + ReferenceChainsTestAccessor::pushPriorityExpand((jlong)(100 + i)); + } + std::vector selected = + ReferenceChainsTestAccessor::collectStaleExpandedEntriesForRotation(10); + EXPECT_TRUE(selected.empty()) + << "collector must stop pushing once _priority_expand hits the cap"; + + // With the lane drained (a pass's expand phase consumed it), the collector selects again. + ReferenceChainsTestAccessor::clearPriorityExpand(); + selected = + ReferenceChainsTestAccessor::collectStaleExpandedEntriesForRotation(10); + ASSERT_EQ(1u, selected.size()); + EXPECT_EQ((jlong)1, selected[0]); + + tracker->stop(); +} + +// The stale-expansion rotation must select leak parents from _leak_parent_fanout ahead of the blind +// table lap: the fanout entries are the EXPANDED parents that actually lead to watched leak-klass +// children, and neither the blind lap (~table_size/budget passes, hundreds live) nor the +// growth-gated leak-accumulation tier reaches them in steady state. +TEST_F(ReferenceChainsBfsTest, StaleRotationPrefersLeakParentsOverBlindLap) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + constexpr u32 kLeafKlass = 987; + ReferenceChainsTestAccessor::setWatchedLeakKlassIdsForTest({kLeafKlass}); + + // Fanout parent 1 and an unrelated stale-EXPANDED entry 3. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 1, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, + /*class_tag=*/42)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 3, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, 1, 10); + + // Budget 1: the fanout parent wins over the blind-lap entry. + std::vector selected = + ReferenceChainsTestAccessor::collectStaleExpandedEntriesForRotation(1); + ASSERT_EQ(1u, selected.size()); + EXPECT_EQ((jlong)1, selected[0]); + + // Budget covering both: fanout parent first, blind lap fills the rest. + ReferenceChainsTestAccessor::clearPriorityExpand(); + selected = + ReferenceChainsTestAccessor::collectStaleExpandedEntriesForRotation(2); + ASSERT_EQ(2u, selected.size()); + EXPECT_EQ((jlong)1, selected[0]); + EXPECT_EQ((jlong)3, selected[1]); + + tracker->stop(); +} + +// reparentToDurableRoot: a depth-1 entry first admitted through a transient root (stack/JNI local) +// is re-parented to a durable root-attached parent at equal depth - the case improveChain() cannot +// express (it requires a strictly deeper path), and exactly the hotdog shape where the singleton +// collection is a depth-0 static root and its elements depth 1. +TEST_F(ReferenceChainsBfsTest, ReparentToDurableRootSwapsTransientForDurable) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + // tag 1: transient root (old parent). tag 2: target at depth 1 under it. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 1, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_JNI_LOCAL)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 2, 1, 1, FrontierEntryState::EXPANDED, /*root_kind=*/0)); + // tag 5: durable static root (new parent). + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 5, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + // tag 6: another transient root - must never be swapped TO. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 6, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STACK_LOCAL)); + + EXPECT_TRUE(frontier->reparentToDurableRoot(2, 5, 42)); + FrontierEntry entry{}; + ASSERT_TRUE(frontier->lookup(2, &entry)); + EXPECT_EQ((jlong)5, entry.parent_tag); + EXPECT_EQ((u32)42, entry.referrer_klass); + + // Transient new parent: no swap (would trade one noise root for another). + EXPECT_FALSE(frontier->reparentToDurableRoot(2, 6, 43)); + ASSERT_TRUE(frontier->lookup(2, &entry)); + EXPECT_EQ((jlong)5, entry.parent_tag) << "parent must be unchanged"; + + // Depth-2 targets are out of scope (judging root durability there would require walking both + // chains). + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 7, 2, 2, FrontierEntryState::EXPANDED, /*root_kind=*/0)); + EXPECT_FALSE(frontier->reparentToDurableRoot(7, 5, 44)); + + tracker->stop(); +} + +// recordDiscoveredInstance eviction: noise instances fill discovery slots first-come-first-served, +// but a leak-correlated discovery must evict a noise slot when all are full - without eviction, the +// 8 noise instances observed on-pod permanently blocked every later leak-tagged instance of the +// watched class. +TEST_F(ReferenceChainsBfsTest, RecordDiscoveredInstanceEvictsNoiseSlots) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + constexpr u32 kKlass = 3; + ReferenceChainsTestAccessor::setCandidateCountForTest(1); + ReferenceChainsTestAccessor::setCandidateKlassIdForTest(0, kKlass); + + const int cap = ReferenceChainsTestAccessor::maxDiscoveredPerClass(); + for (int d = 0; d < cap; d++) { + ReferenceChainsTestAccessor::recordDiscoveredInstanceForTest( + kKlass, /*tag=*/100 + d, /*leak_correlated=*/false); + } + EXPECT_EQ(cap, ReferenceChainsTestAccessor::candidateDiscoveredCountForTest(0)); + + // Noise beyond the cap is dropped, slots unchanged. + ReferenceChainsTestAccessor::recordDiscoveredInstanceForTest( + kKlass, 108, false); + EXPECT_EQ(cap, ReferenceChainsTestAccessor::candidateDiscoveredCountForTest(0)); + EXPECT_EQ((jlong)100, + ReferenceChainsTestAccessor::candidateDiscoveredTagForTest(0, 0)); + + // Leak-correlated discovery evicts the first noise slot (tag 100 has no frontier entry -> + // treated as uncorrelated). + ReferenceChainsTestAccessor::recordDiscoveredInstanceForTest( + kKlass, 200, true); + EXPECT_EQ(cap, ReferenceChainsTestAccessor::candidateDiscoveredCountForTest(0)); + EXPECT_EQ((jlong)200, + ReferenceChainsTestAccessor::candidateDiscoveredTagForTest(0, 0)); + EXPECT_EQ((jlong)101, + ReferenceChainsTestAccessor::candidateDiscoveredTagForTest(0, 1)); + + // Once every slot is leak-correlated, a further leak discovery is dropped (no eviction of real + // signal). + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 200, 0, 1, FrontierEntryState::EXPANDED, /*root_kind=*/0)); + frontier->setLeakTag(200, ReferenceChainsTestAccessor::leakTagBase() + 1); + // Entries for the remaining noise slots so the eviction scan finds all slots leak-tagged. + for (int d = 1; d < cap; d++) { + jlong tag = 101 + (d - 1); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, tag, 0, 1, FrontierEntryState::EXPANDED, /*root_kind=*/0)); + frontier->setLeakTag(tag, ReferenceChainsTestAccessor::leakTagBase() + 2); + } + ReferenceChainsTestAccessor::recordDiscoveredInstanceForTest( + kKlass, 201, true); + EXPECT_EQ(cap, ReferenceChainsTestAccessor::candidateDiscoveredCountForTest(0)); + for (int d = 0; d < cap; d++) { + EXPECT_NE((jlong)201, + ReferenceChainsTestAccessor::candidateDiscoveredTagForTest(0, d)); + } + + tracker->stop(); +} + +// correlateAdmittedLeakTag: a tracked instance the BFS admitted BEFORE it was leak-tagged carries a +// frontier tag on the object; correlating stores the leak tag ON the entry (chain events then emit +// targetTag = leak tag) and records the instance as discovered. +TEST_F(ReferenceChainsBfsTest, CorrelateAdmittedLeakTagSetsEntryAndDiscovers) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + constexpr u32 kKlass = 3; + constexpr jlong kLeakTag = 0x40000000LL + 5; + ReferenceChainsTestAccessor::setCandidateCountForTest(1); + ReferenceChainsTestAccessor::setCandidateKlassIdForTest(0, kKlass); + + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 300, 0, 1, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + + EXPECT_TRUE(tracker->correlateAdmittedLeakTag(300, kLeakTag, kKlass)); + EXPECT_EQ(kLeakTag, (jlong)ReferenceChainsTestAccessor::frontierLeakTag(300)); + EXPECT_EQ(1, ReferenceChainsTestAccessor::candidateDiscoveredCountForTest(0)); + EXPECT_EQ((jlong)300, + ReferenceChainsTestAccessor::candidateDiscoveredTagForTest(0, 0)); + + // Idempotent: an already-correlated entry just returns true. + EXPECT_TRUE(tracker->correlateAdmittedLeakTag(300, kLeakTag, kKlass)); + EXPECT_EQ(kLeakTag, (jlong)ReferenceChainsTestAccessor::frontierLeakTag(300)); + + // Unknown tag: no crash, no discovery side effects. + EXPECT_FALSE(tracker->correlateAdmittedLeakTag(999, kLeakTag, kKlass)); + EXPECT_EQ(1, ReferenceChainsTestAccessor::candidateDiscoveredCountForTest(0)); + + tracker->stop(); +} + +// Retention-explanation gate on the discovered-instance chains (depth==0 always suppressed; +// depth==1 suppressed only for TRANSIENT roots - a depth-1 chain from a durable root is the real +// direct-retention shape): transient depth-1 must NOT be cached, durable depth-1 and depth-2 must. +TEST_F(PollWatchedTargetsTest, DiscoveredChainGateSuppressesTransientDepthOne) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int fake_object_storage = 0; + jobject obj = reinterpret_cast(&fake_object_storage); + seedGrowingCandidate(/*klass_id=*/3, /*rep=*/(jweak)obj); + + // First poll populates the candidate slots from LivenessTracker's population. + tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); + + FrontierTable *frontier = tracker->frontierTable(); + // Noise shape: transient root (JNI local frame) -> depth-1 instance. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 6, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_JNI_LOCAL)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 7, 6, 1, FrontierEntryState::EXPANDED, /*root_kind=*/0)); + // Real direct-retention shape: static-field root -> depth-1 instance. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 8, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 9, 8, 1, FrontierEntryState::EXPANDED, /*root_kind=*/0)); + // Deeper chain through the transient root: passes on depth alone. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 10, 7, 2, FrontierEntryState::EXPANDED, /*root_kind=*/0)); + // Depth-0 transient root: the candidate instance itself held by a live frame - suppressed like + // the depth-1 transient shape. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 11, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STACK_LOCAL)); + // Depth-0 durable root: the candidate instance IS the static field's value (the + // singleton-collection-itself shape) - a real direct-retention chain, NOT suppressible as + // noise. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 12, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + + ReferenceChainsTestAccessor::recordDiscoveredInstanceForTest(3, 7, false); + ReferenceChainsTestAccessor::recordDiscoveredInstanceForTest(3, 9, false); + ReferenceChainsTestAccessor::recordDiscoveredInstanceForTest(3, 10, false); + ReferenceChainsTestAccessor::recordDiscoveredInstanceForTest(3, 11, false); + ReferenceChainsTestAccessor::recordDiscoveredInstanceForTest(3, 12, false); + ASSERT_EQ(5, ReferenceChainsTestAccessor::candidateDiscoveredCountForTest(0)); + + tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); + + EXPECT_FALSE(ReferenceChainsTestAccessor::hasResolvedChainForTag(7)) + << "depth-1 chain rooted at a transient (JNI local) root is noise"; + EXPECT_TRUE(ReferenceChainsTestAccessor::hasResolvedChainForTag(9)) + << "depth-1 chain rooted at a durable (static field) root is a real " + "direct-retention chain"; + EXPECT_TRUE(ReferenceChainsTestAccessor::hasResolvedChainForTag(10)) + << "depth-2 chain passes the gate regardless of root kind"; + EXPECT_FALSE(ReferenceChainsTestAccessor::hasResolvedChainForTag(11)) + << "depth-0 chain rooted at a transient (stack local) root is noise"; + EXPECT_TRUE(ReferenceChainsTestAccessor::hasResolvedChainForTag(12)) + << "depth-0 chain rooted at a durable (static field) root is the " + "direct-retention shape the search exists to report"; + + tracker->stop(); +} + +// Orphan slot sweep: a candidate that qualified long enough for the walk to record discovered +// instances, then stopped qualifying (its trend aged out of the poll's candidate list), must still +// get chains built for those instances. +TEST_F(PollWatchedTargetsTest, OrphanedSlotBuildsDiscoveredChainsAfterCandidateDropsOut) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int fake_object_storage = 0; + jobject obj = reinterpret_cast(&fake_object_storage); + seedGrowingCandidate(/*klass_id=*/3, /*rep=*/(jweak)obj); + + // First poll admits klass 3 into candidate slot 0 (nothing discovered yet, so nothing is built + // here). + tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); + ASSERT_EQ(0, ReferenceChainsTestAccessor::candidateDiscoveredCountForTest(0)); + + // The walk discovered an instance while the candidate still qualified: the real + // direct-retention shape (static-field root -> depth-1 instance), which the discovered-chain + // gate lets through. + FrontierTable *frontier = tracker->frontierTable(); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 9, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 8, 9, 1, FrontierEntryState::EXPANDED, /*root_kind=*/0)); + ReferenceChainsTestAccessor::recordDiscoveredInstanceForTest(3, 8, false); + ASSERT_EQ(1, ReferenceChainsTestAccessor::candidateDiscoveredCountForTest(0)); + + // The candidate stops qualifying: LivenessTracker's population table is wiped, so + // selectLeakCandidates() returns 0 on every poll from here on. + LivenessTracker::instance()->klassPopulationResetForTest(); + tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); + + EXPECT_TRUE(ReferenceChainsTestAccessor::hasResolvedChainForTag(8)) + << "discovered instances recorded while the candidate qualified must " + "still get chains built after it stops qualifying"; + + tracker->stop(); +} diff --git a/ddprof-lib/src/test/cpp/referenceChainsPodTests.inc b/ddprof-lib/src/test/cpp/referenceChainsPodTests.inc new file mode 100644 index 000000000..bc4bbafa8 --- /dev/null +++ b/ddprof-lib/src/test/cpp/referenceChainsPodTests.inc @@ -0,0 +1,479 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +class PodInAJarTest : public ReferenceChainsBfsTest { +protected: + struct PodTopology { + int holder_class_node = -1; + int wrapper_node = -1; + int list_node = -1; + int leak_cls_idx = -1; + int wrapper_cls_idx = -1; + int list_cls_idx = -1; + std::vector chunk_nodes; + std::vector chunk_leak_tags; + std::vector flood_class_nodes; + std::vector flood_nodes; + std::vector filler_nodes; + }; + + static constexpr int kChunks = 6; // < MAX_DISCOVERED_INSTANCES_PER_CLASS + static constexpr u64 kCycleNs = 2000000000ULL; // 2s fake-clock step + + void SetUp() override { + ReferenceChainsBfsTest::SetUp(); + // resolveCandidateRepresentative() NewLocalRef()s the stored representative; the Bfs + // fixture never wires that JNI slot (PollWatchedTargetsTest has its own). + jni_tbl.NewLocalRef = &mock_NewLocalRefPassthrough; + // NOTE: no liveness calls here - the Bfs tests run without them, and liveness state changes + // (setGcGenerationsForTest) alter the pass machinery's behavior; each harness test resets + // liveness explicitly where it wants it (resetLivenessForPod below). + } + + static jobject JNICALL mock_NewLocalRefPassthrough(JNIEnv *, jobject ref) { + return ref; + } + + void resetLivenessForPod() { + LivenessTracker::instance()->klassPopulationResetForTest(); + LivenessTracker::instance()->setGcGenerationsForTest(true); + LivenessTracker::instance()->leakTagPoolResetForTest(); + } + + void TearDown() override { + ReferenceChainsBfsTest::TearDown(); + // Hygiene for whatever suite runs next: clear any population this harness seeded. + LivenessTracker::instance()->klassPopulationResetForTest(); + } + + // Builds the hotdog leak topology: holder-class static -> synchronized wrapper (with its + // mutex==this self-edge) -> c -> K leak-tagged chunks, plus 2 flood classes x 16 static-held + // nodes for anchor-tier volume. + PodTopology buildLeakPod() { + PodTopology topo; + // CAPACITY FIXTURE CONTRACT: classes registered via + // classes.push_back({(void*)&node_tags[node], ...}) capture the node's ADDRESS as the + // jclass identity - std::vector growth would reallocate node_tags and dangle every captured + // pointer, silently making indexOfNode() fail for those classes in the static sweep (the + // holder array seeds no expandable class, the STATIC_FIELD edges never replay, nothing + // admits). + node_tags.reserve(600); + tags_ever_assigned.reserve(600); + topo.leak_cls_idx = addClass((void *)0x5001, "[B"); + topo.wrapper_cls_idx = addClass( + (void *)0x5002, + "Ljava/util/Collections$SynchronizedRandomAccessList;"); + topo.list_cls_idx = addClass((void *)0x5003, "Ljava/util/ArrayList;"); + + topo.holder_class_node = addNode(); + topo.wrapper_node = addNode(); + topo.list_node = addNode(); + // The holder class node doubles as the jclass identity (see + // DiscoversObjectRetainedOnlyByStaticField): register it as a loaded class so the + // static-field sweep admits the wrapper root-attached. + classes.push_back({(void *)&node_tags[topo.holder_class_node], + "Lcom/rc/pod/Holder;"}); + // Flood classes get their own jclass-identity nodes too, so their statics are separate + // anchors (not more statics on the holder). + const char *flood_sigs[2] = {"Lcom/rc/pod/FloodA;", + "Lcom/rc/pod/FloodB;"}; + for (int c = 0; c < 2; c++) { + topo.flood_class_nodes.push_back(addNode()); + classes.push_back( + {(void *)&node_tags[topo.flood_class_nodes[c]], flood_sigs[c]}); + } + + for (int i = 0; i < kChunks; i++) { + topo.chunk_nodes.push_back(addNode()); + } + for (int i = 0; i < 32; i++) { + topo.flood_nodes.push_back(addNode()); + } + // Backlog volume: a 300-edge deep chain under one flood root. + for (int i = 0; i < 300; i++) { + topo.filler_nodes.push_back(addNode()); + } + + script = { + // LEAK_BUFFER: the holder class's static field -> wrapper. + {JVMTI_HEAP_REFERENCE_STATIC_FIELD, topo.holder_class_node, + topo.wrapper_node, topo.wrapper_cls_idx}, + // The synchronized wrapper's mutex == this self-edge (round-16 fix-A shape: must not + // demote the root-attached wrapper). + {JVMTI_HEAP_REFERENCE_FIELD, topo.wrapper_node, topo.wrapper_node, + topo.wrapper_cls_idx}, + // wrapper -> c -> chunks. + {JVMTI_HEAP_REFERENCE_FIELD, topo.wrapper_node, topo.list_node, + topo.list_cls_idx}, + }; + for (int i = 0; i < kChunks; i++) { + script.push_back({JVMTI_HEAP_REFERENCE_FIELD, topo.list_node, + topo.chunk_nodes[i], topo.leak_cls_idx}); + } + // Flood volume: each flood class holds 16 statics. + for (int c = 0; c < 2; c++) { + for (int i = 0; i < 16; i++) { + script.push_back({JVMTI_HEAP_REFERENCE_STATIC_FIELD, + topo.flood_class_nodes[c], + topo.flood_nodes[16 * c + i], + topo.leak_cls_idx /* any class; volume only */}); + } + } + // The filler chain hangs off flood node 0 (already a static anchor). + script.push_back({JVMTI_HEAP_REFERENCE_FIELD, topo.flood_nodes[0], + topo.filler_nodes[0], topo.leak_cls_idx}); + for (int i = 0; i + 1 < (int)topo.filler_nodes.size(); i++) { + script.push_back({JVMTI_HEAP_REFERENCE_FIELD, + topo.filler_nodes[i], topo.filler_nodes[i + 1], + topo.leak_cls_idx}); + } + reseedChunkLeakTags(topo); + return topo; + } + + // Leak tags model the poll's tagLeakInstances() assignment (the pod re-tags within minutes of + // every restart - the verify-not-retag state machine); the walk's leak-tag interception path + // consumes them for real from the live node tags. + void reseedChunkLeakTags(const PodTopology &topo) { + for (int i = 0; i < (int)topo.chunk_nodes.size(); i++) { + jlong leak_tag = 1073741824LL + 100 + i; + node_tags[topo.chunk_nodes[i]] = leak_tag; + tags_ever_assigned[topo.chunk_nodes[i]] = leak_tag; + } + } + + // Resolves the leak class's tracker-side klass id from the classTags table via the SAME tag + // value the mock passes as the chunk edges' class_tag (tags[klass_ptr] - the sweep's negative + // class tag, set during phase 1's static sweep). + u32 resolveLeakKlassId(const PodTopology &topo) { + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + EXPECT_GT(ReferenceChainsTestAccessor::passesRunForTest(), 0) + << "phase-1 pass never ran"; + jlong class_tag = tags[classes[topo.leak_cls_idx].klass]; + EXPECT_LT(class_tag, 0) << "leak class never sweep-tagged (class_tag=" + << class_tag << ")"; + return tracker->classTags()->resolve(class_tag); + } + + // Seeds the leak-side liveness (population growth + qualifying tid + representative = the first + // chunk), mirroring PollWatchedTargetsTest::seedGrowingCandidate. + void seedLeakLiveness(u32 klass_id, const PodTopology &topo) { + int slot; + bool created; + for (u16 i = 1; i <= 20; i++) { + LivenessTracker::instance()->klassPopulationRecordForTest( + klass_id, i, i, &slot, &created); + LivenessTracker::instance()->tidTrendRecordForTest( + klass_id, /*tid=*/4242, (u32)i, (u64)i); + } + LivenessTracker::instance()->klassPopulationSetRepresentativeForTest( + nullptr, klass_id, (jweak)&node_tags[topo.chunk_nodes[0]]); + // The representative's GetTag() identity: the mock tags map is keyed by object pointer; + // getTag(rep) must report the rep's leak tag (the reseed in + // reseedChunkLeakTags() above made node_tags hold it - chunk_leak_tags + // itself is not kept). + tags[&node_tags[topo.chunk_nodes[0]]] = + node_tags[topo.chunk_nodes[0]]; + ASSERT_NE(0, tags[&node_tags[topo.chunk_nodes[0]]]) + << "representative's leak tag missing - L1/L2 assertions below " + "would silently test an untagged representative"; + } + + // One threadLoop iteration, fake clock: the exact body order from + // ReferenceChainTracker::threadLoop() (shouldRunPass -> runPass -> pollWatchedTargets, poll + // unconditional). + bool drivePodCycle(u64 &fake_now) { + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + bool should_run = tracker->shouldRunPassForTest(fake_now); + if (should_run) { + bool truncated = false; + tracker->runPass(&mock_jvmti, &mock_jni, &truncated); + } + tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); + fake_now += kCycleNs; + return should_run; + } + + // Seeds a throwaway candidate klass (NO instances, never matches any real class) whose only job + // is arming the leak signal so the phase-1 pass runs: the dormancy invariant (L8) proved + // shouldRunPass stays false without a candidate, and the klass-id resolution needs a pass. + void seedThrowawayLiveness(u32 klass_id) { + int slot; + bool created; + for (u16 i = 1; i <= 20; i++) { + LivenessTracker::instance()->klassPopulationRecordForTest( + klass_id, i, i, &slot, &created); + LivenessTracker::instance()->tidTrendRecordForTest( + klass_id, /*tid=*/4242, (u32)i, (u64)i); + } + LivenessTracker::instance()->klassPopulationSetRepresentativeForTest( + nullptr, klass_id, (jweak)0xBADC0DE); + } + + // Two-phase pod bring-up. Phase 1: a throwaway candidate arms the leak signal so cycle 1's pass + // runs - the sweep tags the classes and the walk admits the wrapper subtree, which makes the + // leak class's tracker-side klass id resolvable (the only reliable source of the id is the + // system's own resolve() over a real frontier entry). + void bringUpPod(const PodTopology &topo, u32 &leak_klass_id_out, + u64 &fake_now) { + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + // Seed the fake clock from the REAL monotonic clock: the pain budgets' _last_update_ns is + // OS::nanotime()-based, so a fake epoch starting at ~0 sits before every budget's + // initialization and canStartNow() blocks every pass (the dormancy test passes either way - + // no passes at all - so only the pass-running tests caught this). + fake_now = OS::nanotime(); + resetLivenessForPod(); + seedThrowawayLiveness(/*klass_id=*/999); + // The candidate arms in the POLL, which runs after shouldRunPass in each cycle - so the + // first cycle only arms, and a pass actually runs one cycle later. + bool ran = false; + for (int i = 0; i < 6 && !ran; i++) { + ran = drivePodCycle(fake_now); + } + leak_klass_id_out = resolveLeakKlassId(topo); + LivenessTracker::instance()->klassPopulationResetForTest(); + ReferenceChainsTestAccessor::restartSearchForTest(); + reseedChunkLeakTags(topo); + seedLeakLiveness(leak_klass_id_out, topo); + // Poll once BEFORE the first pass of the new search: the candidate slot registers in the + // poll, and the pass's admission auto-mark requires the slot to exist + // (heapReferenceCallback's auto-mark guards on _candidate_count > 0). + tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); + } + + // Captures stdout per distinct ReferenceChainTracker/LivenessTracker line prefix (L4: the + // log-budget invariant - the round-15 800k/min flood and the round-18 tick stragglers become + // assertion failures). + struct StdoutCapture { + int saved_fd = -1; + int tmp_fd = -1; + char path[64] = {0}; + bool done = false; + StdoutCapture() { + snprintf(path, sizeof(path), "/tmp/podjar_stdout_XXXXXX"); + tmp_fd = mkstemp(path); + fflush(stdout); + saved_fd = dup(1); + dup2(tmp_fd, 1); + } + std::map perPrefixCounts() { + if (done) { + return {}; + } + done = true; + fflush(stdout); + dup2(saved_fd, 1); + close(saved_fd); + saved_fd = -1; + lseek(tmp_fd, 0, SEEK_SET); + std::map counts; + FILE *f = fdopen(tmp_fd, "r"); + char line[512]; + while (fgets(line, sizeof(line), f)) { + char cls[96], fn[96]; + if (sscanf(line, "[TEST::INFO] %95[^:]::%95[a-zA-Z]", cls, + fn) == 2) { + counts[std::string(cls) + "::" + fn]++; + } + } + fclose(f); + unlink(path); + tmp_fd = -1; + return counts; + } + }; +}; + +TEST_F(PodInAJarTest, SystemLivenessLeakChainsBuildAndCanaryResolves) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=10")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + PodTopology topo = buildLeakPod(); + u32 leak_klass_id; + u64 fake_now; + bringUpPod(topo, leak_klass_id, fake_now); + + int cycles_run = 0; + while (cycles_run < 120 && + ReferenceChainsTestAccessor::resolvedChainCountForTest() < + (size_t)kChunks && + ReferenceChainsTestAccessor::searchStateForTest() == + SearchState::RUNNING) { + drivePodCycle(fake_now); + cycles_run++; + } + + // L1: every leak-tagged chunk's tag appears as a cached chain target (leak-correlated events - + // the pod's round-16 end-goal state). + auto targets = ReferenceChainsTestAccessor::resolvedChainTargetsForTest(); + for (int i = 0; i < kChunks; i++) { + jlong leak_tag = 1073741824LL + 100 + i; + EXPECT_NE(std::find(targets.begin(), targets.end(), (u64)leak_tag), + targets.end()) + << "leak tag " << leak_tag + << " never became a cached chain target after " << cycles_run + << " cycles"; + } + + // L2: the canary resolved for the LEAK klass (found bit set on its slot - the round-19 + // criterion; fails on any pre-788d7b2a7 build). + int leak_slot = -1; + for (int s2 = 0; s2 < ReferenceChainsTestAccessor::candidateCountForTest(); + s2++) { + if (ReferenceChainsTestAccessor::candidateKlassIdForTest(s2) == + leak_klass_id) { + leak_slot = s2; + break; + } + } + ASSERT_GE(leak_slot, 0) << "leak klass never registered a candidate slot"; + EXPECT_TRUE(ReferenceChainsTestAccessor::candidateFoundBitsForTest() & + (1ULL << leak_slot)) + << "leak candidate slot never marked found despite leak-tag chains"; +} + +TEST_F(PodInAJarTest, SystemSearchCompletesNaturally) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=10")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + PodTopology topo = buildLeakPod(); + u32 leak_klass_id; + u64 fake_now; + bringUpPod(topo, leak_klass_id, fake_now); + + int cycles = 0; + while (cycles < 200 && + ReferenceChainsTestAccessor::searchStateForTest() == + SearchState::RUNNING) { + drivePodCycle(fake_now); + cycles++; + } + + // L3: a healthy-topology search must end COMPLETED (all candidates found), never ABANDONED + // (TTL/frontier-cap). + EXPECT_EQ((u8)SearchState::COMPLETED, + ReferenceChainsTestAccessor::searchStateForTest()) + << "search did not complete naturally within " << cycles + << " cycles (state=" + << (int)ReferenceChainsTestAccessor::searchStateForTest() << ")"; + EXPECT_GT(ReferenceChainsTestAccessor::passesRunForTest(), 0); +} + +TEST_F(PodInAJarTest, SystemRestartLeavesNothingBehind) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=10")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + PodTopology topo = buildLeakPod(); + u32 leak_klass_id; + u64 fake_now; + bringUpPod(topo, leak_klass_id, fake_now); + for (int i = 0; i < 120 && + ReferenceChainsTestAccessor::resolvedChainCountForTest() == 0; + i++) { + drivePodCycle(fake_now); + } + ASSERT_GT(ReferenceChainsTestAccessor::resolvedChainCountForTest(), + (size_t)0); + + ReferenceChainsTestAccessor::restartSearchForTest(); + + // L6: the restart contract as a test instead of discipline. + EXPECT_GT(ReferenceChainsTestAccessor::resolvedChainCountForTest(), + (size_t)0) + << "resolved chains should persist across restarts by design"; + EXPECT_EQ((size_t)0, + ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); + EXPECT_EQ((size_t)0, + ReferenceChainsTestAccessor::staticAnchorFreshQueueSizeForTest()); + EXPECT_TRUE(ReferenceChainsTestAccessor::anchorIndexIsEmptyForTest()); + for (int s = 0; s < 5; s++) { + EXPECT_EQ(0, + ReferenceChainsTestAccessor::candidateDiscoveredCountForTest(s)) + << "discovered slot " << s << " survived restartSearch()"; + } +} + +TEST_F(PodInAJarTest, TopologyCapacityContractStaticAdmits) { + // The topology-builder capacity contract as a regression: the FULL buildLeakPod topology + one + // direct pass must admit the wrapper static (and with it the whole leak subtree). + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + PodTopology topo = buildLeakPod(); + bool truncated = true; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_GT(tags_ever_assigned[topo.wrapper_node], 0) + << "wrapper static never admitted (holder_cls_tag=" + << node_tags[topo.holder_class_node] + << " sweep_gate=" << ReferenceChainsTestAccessor::sweepGateStaticCountForTest() + << "/" << ReferenceChainsTestAccessor::sweepGateResolvedCountForTest() + << ")"; + + tracker->stop(); +} + +TEST_F(PodInAJarTest, SystemHealthyAppIsDormant) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=10")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + // L8: the accidental 3h pod control, encoded - a heap with no leak candidates must produce ZERO + // searches (the candidate/generations gate holds the machinery dormant on a healthy app). + PodTopology topo = buildLeakPod(); + // No seedLeakLiveness(): no candidate ever qualifies. bringUpPod is also skipped - it seeds + // liveness; drive raw cycles instead. + (void)topo; + resetLivenessForPod(); + u64 fake_now = OS::nanotime(); + for (int i = 0; i < 20; i++) { + drivePodCycle(fake_now); + } + EXPECT_EQ(0, ReferenceChainsTestAccessor::passesRunForTest()) + << "searches ran on a healthy (candidate-less) app"; + EXPECT_EQ((size_t)0, + ReferenceChainsTestAccessor::resolvedChainCountForTest()); +} + +TEST_F(PodInAJarTest, SystemLogBudgetPerPass) { +#ifndef DEBUG + // The gtest binary compiles the main sources WITHOUT DEBUG (round-16 lesson: TEST_LOG is a + // no-op here) - the log-budget invariant can only run in a DEBUG-built test binary. + GTEST_SKIP() << "log budget needs a DEBUG-built gtest binary"; +#else + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=10")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + PodTopology topo = buildLeakPod(); + u32 leak_klass_id; + u64 fake_now; + bringUpPod(topo, leak_klass_id, fake_now); + for (int i = 0; i < 3; i++) { + drivePodCycle(fake_now); + } + + // L4: capture one full cycle (pass + poll) and bound every distinct line shape. + StdoutCapture capture; + drivePodCycle(fake_now); + auto counts = capture.perPrefixCounts(); + ASSERT_FALSE(counts.empty()) << "no diagnostics captured at level 2"; + for (const auto &kv : counts) { + EXPECT_LE(kv.second, 300) << "log line shape '" << kv.first + << "' fired " << kv.second + << " times in one pass+poll cycle"; + } +#endif +} + diff --git a/ddprof-lib/src/test/cpp/referenceChainsRotationTests.inc b/ddprof-lib/src/test/cpp/referenceChainsRotationTests.inc new file mode 100644 index 000000000..5790d23cc --- /dev/null +++ b/ddprof-lib/src/test/cpp/referenceChainsRotationTests.inc @@ -0,0 +1,824 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +TEST_F(ReferenceChainsBfsTest, StaleRootAttributionUpgradesOnRediscovery) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + // Synthetic stack-local root: admitted, root-attached (parent_tag == 0), its owning frame has + // since "gone away" from the design doc's scenario (nothing further to model here - the entry + // simply stays as-is until a more durable root is discovered). + jlong tag = 1; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, tag, /*parent_tag=*/0, /*depth=*/0, + FrontierEntryState::EXPANDED, JVMTI_HEAP_REFERENCE_STACK_LOCAL)); + + // A second, equally-or-less durable root discovery does not overwrite the recorded root_kind. + EXPECT_FALSE(ReferenceChainsTestAccessor::maybeUpgradeRootAttachedRootKind( + frontier, tag, JVMTI_HEAP_REFERENCE_JNI_LOCAL)); + FrontierEntry entry{}; + ASSERT_TRUE(frontier->lookup(tag, &entry)); + EXPECT_EQ(JVMTI_HEAP_REFERENCE_STACK_LOCAL, entry.root_kind); + + // A durable root (JNI global) attaching to the same object upgrades it. + EXPECT_TRUE(ReferenceChainsTestAccessor::maybeUpgradeRootAttachedRootKind( + frontier, tag, JVMTI_HEAP_REFERENCE_JNI_GLOBAL)); + ASSERT_TRUE(frontier->lookup(tag, &entry)); + EXPECT_EQ(JVMTI_HEAP_REFERENCE_JNI_GLOBAL, entry.root_kind); + EXPECT_EQ(0, entry.parent_tag); // still root-attached, unchanged + + // An even less durable root discovered afterwards cannot downgrade it. + EXPECT_FALSE(ReferenceChainsTestAccessor::maybeUpgradeRootAttachedRootKind( + frontier, tag, JVMTI_HEAP_REFERENCE_MONITOR)); + ASSERT_TRUE(frontier->lookup(tag, &entry)); + EXPECT_EQ(JVMTI_HEAP_REFERENCE_JNI_GLOBAL, entry.root_kind); + + tracker->stop(); +} + +// Exercises the invariant conflict that durability re-verification exists to catch: a non-root +// entry (parent_tag != 0) rediscovered as if via a root context must never have its root_kind +// overwritten - doing so would leave a non-zero root_kind on an entry nothing else treats as +// root-attached (referenceChains.h's FrontierEntry::root_kind comment), since this mutator never +// touches parent_tag. +TEST_F(ReferenceChainsBfsTest, NonRootAttachedEntryNeverUpgraded) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + // Parent Y (root-attached) and child X, admitted the way frontier re-expansion admits a + // non-root child: non-root (parent_tag == Y's tag), root_kind == 0. + jlong yTag = 1; + jlong xTag = 2; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, yTag, /*parent_tag=*/0, /*depth=*/0, + FrontierEntryState::EXPANDED, JVMTI_HEAP_REFERENCE_JNI_GLOBAL)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, xTag, /*parent_tag=*/yTag, /*depth=*/1, + FrontierEntryState::EXPANDED, /*root_kind=*/0)); + + // Re-expanding Y rediscovers an edge to X (already tracked) - even if this rediscovery is + // (incorrectly) attempted with a durable root_kind, it must be rejected because X is not + // root-attached. + EXPECT_FALSE(ReferenceChainsTestAccessor::maybeUpgradeRootAttachedRootKind( + frontier, xTag, JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + FrontierEntry entry{}; + ASSERT_TRUE(frontier->lookup(xTag, &entry)); + EXPECT_EQ(0, entry.root_kind); + EXPECT_EQ(yTag, entry.parent_tag); + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, RotationSelectsOnlyTransientExpandedRootAttachedEntries) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + // Eligible: root-attached, EXPANDED, transient root_kind. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 1, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STACK_LOCAL)); + // Not eligible: durable root_kind. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 2, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_JNI_GLOBAL)); + // Not eligible: transient but still FRONTIER, not yet EXPANDED. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 3, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_JNI_LOCAL)); + // Not eligible: transient root_kind but not root-attached. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 4, /*parent_tag=*/1, 1, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_JNI_LOCAL)); + // Eligible: root-attached, EXPANDED, transient (JNI local this time). + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 5, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_JNI_LOCAL)); + + std::vector selected = + ReferenceChainsTestAccessor::collectStaleRootKindEntriesForRotation(10); + std::sort(selected.begin(), selected.end()); + EXPECT_EQ((std::vector{1, 5}), selected); + + // Selected tags are queued for re-expansion, exactly like an ordinary admission would queue a + // newly-discovered tag. + EXPECT_EQ(2u, ReferenceChainsTestAccessor::priorityExpandSize()); + + tracker->stop(); +} + +// N transient-root_kind entries, rotation size R: every entry must be selected at least once within +// ceil(N/R) calls, regardless of where the cursor happened to start. +TEST_F(ReferenceChainsBfsTest, RotationCoversAllEntriesWithinCeilNOverR) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + const int N = 10; + const int R = 3; + for (jlong tag = 1; tag <= N; tag++) { + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, tag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STACK_LOCAL)); + } + + std::unordered_set covered; + int calls = (N + R - 1) / R; + for (int i = 0; i < calls; i++) { + std::vector selected = + ReferenceChainsTestAccessor::collectStaleRootKindEntriesForRotation(R); + for (jlong tag : selected) { + covered.insert(tag); + } + } + EXPECT_EQ((size_t)N, covered.size()); + + tracker->stop(); +} + +// collectStaleExpandedEntriesForRotation()'s own EXPANDED-only criterion is a strict superset of +// collectStaleRootKindEntriesForRotation()'s (which also requires parent_tag == 0 and a transient +// root_kind), and runPassManualWalk() calls the root-kind collector first, into the very same +// _priority_expand deque. +TEST_F(ReferenceChainsBfsTest, StaleExpandedRotationDoesNotDuplicateRootKindSelection) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + // Eligible for both collectors: EXPANDED, root-attached, transient root_kind - exactly the + // overlap collectStaleRootKindEntriesForRotation() will pick up first. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 1, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STACK_LOCAL)); + // Eligible only for the EXPANDED-only sweep: EXPANDED but not root-attached. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 2, /*parent_tag=*/1, 1, FrontierEntryState::EXPANDED, + /*root_kind=*/0)); + + std::vector root_kind_selected = + ReferenceChainsTestAccessor::collectStaleRootKindEntriesForRotation( + ReferenceChainsTestAccessor::rootKindRotationBudget()); + EXPECT_EQ((std::vector{1}), root_kind_selected); + + std::vector stale_expanded_selected = + ReferenceChainsTestAccessor::collectStaleExpandedEntriesForRotation( + ReferenceChainsTestAccessor::staleExpandedRotationBudget()); + // Tag 1 is already queued from the root-kind collector above and must not be selected again; + // tag 2 is newly discovered by this sweep. + EXPECT_EQ((std::vector{2}), stale_expanded_selected); + + std::vector queued = ReferenceChainsTestAccessor::priorityExpandContents(); + EXPECT_EQ((std::vector{1, 2}), queued); + std::unordered_set unique_queued(queued.begin(), queued.end()); + EXPECT_EQ(queued.size(), unique_queued.size()); + + tracker->stop(); +} + +// A tag left over in _priority_expand from a prior pass's truncated expandFrontier() batch (see +// expandFrontier()'s own "leave the batch at the front of the source queue for a later pass to +// retry" comment) must also be skipped by collectStaleExpandedEntriesForRotation() - not just tags +// queued by collectStaleRootKindEntriesForRotation() earlier in the same call. +TEST_F(ReferenceChainsBfsTest, StaleExpandedRotationSkipsPreexistingQueueEntries) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 1, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STACK_LOCAL)); + + // Simulate a truncated batch from a prior pass still sitting at the front of _priority_expand, + // without going through the root-kind collector at all - the leftover entry alone must still be + // enough to suppress a duplicate. + ReferenceChainsTestAccessor::pushPriorityExpand(1); + + std::vector stale_expanded_selected = + ReferenceChainsTestAccessor::collectStaleExpandedEntriesForRotation( + ReferenceChainsTestAccessor::staleExpandedRotationBudget()); + EXPECT_TRUE(stale_expanded_selected.empty()); + + std::vector queued = ReferenceChainsTestAccessor::priorityExpandContents(); + EXPECT_EQ((std::vector{1}), queued); + + tracker->stop(); +} + +// End-to-end proof of the prof-analyzer-hotdog-jb pod's actual leak shape: a static-field-rooted +// collection (like ProfileAnalyzer.LEAK_BUFFER) whose owning node is admitted and fully EXPANDED +// once, then has a *new* element appended to it afterward - mirroring a Java List field being +// mutated in place, never reassigned, well after admitStaticFieldRoots()'s one-time sweep. +TEST_F(ReferenceChainsBfsTest, RotationDiscoversLateElementOfExpandedStaticFieldCollectionWithoutSearchCompleting) { + Arguments args; + // budget=8 -> rotation_reserved_budget = min(8/2, 272) = 4, ordinary = 4: both slices non-zero, + // unlike a budget=1 pattern which would zero out rotation's reserved slice entirely (min(0, + // 272) == 0). + ASSERT_FALSE(args.parse("referencechains=true:hops=5000:budget=8:firstpassbudget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int classNode = addNode(); + int listNode = addNode(); + int seedChildNode = addNode(); + int lateChildNode = addNode(); + + // Distractor chain: a long, independently root-seeded chain that never fully drains within this + // test's bounded pass loops below, so the overall search always has forward progress available + // and never reaches SearchState::COMPLETED (nor NO_PROGRESS_PASS_LIMIT-triggered ABANDONED) + // purely as a side effect of this test's own loop bounds. + const int kDistractorNodes = 500; + std::vector distractor(kDistractorNodes); + for (int i = 0; i < kDistractorNodes; i++) { + distractor[i] = addNode(); + } + + // addClass() captures classNode's address in node_tags' backing storage - must come after every + // addNode() call above (including the distractor loop), or a later push_back reallocating + // node_tags would silently leave this pointer dangling (indexOfNode() would then never match + // it). + addClass((void *)&node_tags[classNode], "Lcom/rc/statics/GrowingListHolder;"); + + script = { + // listNode is retained only via classNode's static field - the same shape as + // DiscoversObjectRetainedOnlyByStaticField above. + {JVMTI_HEAP_REFERENCE_STATIC_FIELD, classNode, listNode, -1}, + // listNode's one pre-existing element, discovered the first time listNode itself is + // expanded. + {JVMTI_HEAP_REFERENCE_FIELD, listNode, seedChildNode, -1}, + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, distractor[0], -1}, + }; + for (int i = 0; i + 1 < kDistractorNodes; i++) { + script.push_back({JVMTI_HEAP_REFERENCE_FIELD, distractor[i], distractor[i + 1], -1}); + } + + // Phase 1: run passes until listNode has been fully expanded (its one pre-existing child + // discovered), without ever letting the search complete. + bool truncated = true; + FrontierEntry listEntry{}; + bool listExpanded = false; + for (int i = 0; i < 200 && !listExpanded; i++) { + ASSERT_EQ(SearchState::RUNNING, tracker->searchState()); + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + jlong listTag = tags_ever_assigned[listNode]; + if (listTag != 0 && tracker->frontierTable()->lookup(listTag, &listEntry) + && listEntry.state == FrontierEntryState::EXPANDED) { + listExpanded = true; + } + } + ASSERT_TRUE(listExpanded); + ASSERT_NE(0, tags_ever_assigned[seedChildNode]); + ASSERT_EQ(SearchState::RUNNING, tracker->searchState()); + + // Phase 2: simulate a new element appended to the leaking static field's list *after* + // listNode's one-time expansion - the exact "growing collection" shape found in the real pod's + // leak generator. + script.push_back({JVMTI_HEAP_REFERENCE_FIELD, listNode, lateChildNode, -1}); + + for (int i = 0; i < 200 && tags_ever_assigned[lateChildNode] == 0; i++) { + ASSERT_EQ(SearchState::RUNNING, tracker->searchState()); + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + } + + // The late element was discovered purely via rotation re-expanding listNode - and, critically, + // without the search ever completing (no dependency on a full heap walk finishing). + ASSERT_NE(0, tags_ever_assigned[lateChildNode]); + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); + + std::vector chain; + ASSERT_TRUE(tracker->frontierTable()->reconstructChain( + tags_ever_assigned[lateChildNode], &chain)); + FrontierEntry lateEntry{}; + ASSERT_TRUE(tracker->frontierTable()->lookup( + tags_ever_assigned[lateChildNode], &lateEntry)); + EXPECT_EQ(tags_ever_assigned[listNode], lateEntry.parent_tag); + + tracker->stop(); +} + +// Proof of the fix for the actual prof-analyzer-hotdog-jb stall: +// collectStaleExpandedEntriesForRotation() (referenceChains.cpp) used to always rescan +// FrontierTable slots starting from tag 1, unlike its sibling +// collectStaleRootKindEntriesForRotation() which already carried its own persistent cursor. +TEST_F(ReferenceChainsBfsTest, StaleExpandedRotationCoversHighTagEntryBehindLowTagPopulationWithinBoundedPasses) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + const int lowTagBudget = ReferenceChainsTestAccessor::staleExpandedRotationBudget(); + // Comfortably above the 256-entry cap, so the low-tag population alone would fill every sweep + // before an always-from-1 scan could ever reach the high-tag entry below - mirrors a real + // multi-GiB heap's frontier table, which accumulates far more than 256 long-lived, perpetually- + // EXPANDED entries (bootstrap classes, caches, etc.) well before any one leak-candidate class + // even loads. + const int lowTagPopulation = lowTagBudget + 50; + for (jlong tag = 1; tag <= lowTagPopulation; tag++) { + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, tag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STACK_LOCAL)); + } + + // The leak candidate's own owning node - e.g. LEAK_BUFFER's list, admitted via a static field + // only once its class loads, well after the JVM's own bootstrap population already occupies + // every low tag number. + const jlong highTag = lowTagPopulation + 1; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, highTag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + + // Several simulated passes: each iteration mirrors one real pass - + // collectStaleExpandedEntriesForRotation() runs once, then clearPriorityExpand() mirrors + // expandFrontier() having drained whatever it selected before the next pass's sweep resumes + // from the cursor. + const int table_size = lowTagPopulation + 1; + const int calls = (table_size + lowTagBudget - 1) / lowTagBudget; + bool highTagSelected = false; + std::unordered_set covered; + for (int pass = 0; pass < calls && !highTagSelected; pass++) { + std::vector selected = + ReferenceChainsTestAccessor::collectStaleExpandedEntriesForRotation( + lowTagBudget); + for (jlong tag : selected) { + covered.insert(tag); + if (tag == highTag) { + highTagSelected = true; + } + } + ReferenceChainsTestAccessor::clearPriorityExpand(); + } + + EXPECT_TRUE(highTagSelected) + << "highTag was never selected within ceil(table_size / max_count) " + "passes - the fix's coverage guarantee does not hold"; + EXPECT_EQ((size_t)table_size, covered.size()); + + tracker->stop(); +} + +// trackLeakAccumulation() - the admission-time hook (called from + +TEST_F(ReferenceChainsBfsTest, TrackLeakAccumulationAggregatesBySignatureAndFanout) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + constexpr u32 kLeafKlass = 987; + constexpr u32 kParent1Klass = 100, kParent2Klass = 200; + ReferenceChainsTestAccessor::setWatchedLeakKlassIdsForTest({kLeafKlass}); + + jlong parent1Tag = 1, parent2Tag = 2; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, parent1Tag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, kParent1Klass)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, parent2Tag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, kParent2Klass)); + + // 3 children of the watched leaf klass under parent1, 1 under parent2 - each call simulates one + // admission (the childTag argument is only used by production code for logging/future use, not + // read by this method). + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, parent1Tag, 10); + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, parent1Tag, 11); + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, parent1Tag, 12); + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, parent2Tag, 20); + + EXPECT_EQ(3u, ReferenceChainsTestAccessor::leakSignatureTotal(kLeafKlass, kParent1Klass)); + EXPECT_EQ(1u, ReferenceChainsTestAccessor::leakSignatureTotal(kLeafKlass, kParent2Klass)); + EXPECT_EQ(3u, ReferenceChainsTestAccessor::leakParentFanout(parent1Tag)); + EXPECT_EQ(1u, ReferenceChainsTestAccessor::leakParentFanout(parent2Tag)); + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, TrackLeakAccumulationSkipsUnwatchedKlass) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + ReferenceChainsTestAccessor::setWatchedLeakKlassIdsForTest({987}); + jlong parentTag = 1; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, parentTag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, /*class_tag=*/100)); + + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, /*class_tag=*/555, + parentTag, 10); + + EXPECT_EQ(0u, ReferenceChainsTestAccessor::leakSignatureCount()); + EXPECT_EQ(0u, ReferenceChainsTestAccessor::leakParentFanout(parentTag)); + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, TrackLeakAccumulationSkipsRootAttachedChild) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + ReferenceChainsTestAccessor::setWatchedLeakKlassIdsForTest({987}); + + // parent_tag == 0 - a root-attached leaf itself, nothing to attribute a container to. + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, 987, /*parent_tag=*/0, 10); + + EXPECT_EQ(0u, ReferenceChainsTestAccessor::leakSignatureCount()); + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, TrackLeakAccumulationSkipsWhenParentNotFound) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + ReferenceChainsTestAccessor::setWatchedLeakKlassIdsForTest({987}); + + // parent_tag=99 was never inserted - graceful no-op, not a crash. + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, 987, /*parent_tag=*/99, 10); + + EXPECT_EQ(0u, ReferenceChainsTestAccessor::leakSignatureCount()); + + tracker->stop(); +} + +// Proof of the actual bug this design was found fixing: the classMap dictionary id (referrer_klass) +// for the exact same class can differ depending on which subsystem/generation resolved it (see +// class_tag's own comment, referenceChains.h, for the real-world case - "[B" resolving to two +// different classMap ids for LivenessTracker vs. +TEST_F(ReferenceChainsBfsTest, TrackLeakAccumulationMatchesByClassTagEvenWhenReferrerKlassDiffers) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + constexpr u32 kLeafClassTag = 987; + constexpr u32 kParentClassTag = 100; + // Deliberately different, "wrong" classMap ids - simulating exactly the compaction/regeneration + // scenario that broke referrer_klass-based matching. + constexpr u32 kParentStaleReferrerKlass = 555555; + ReferenceChainsTestAccessor::setWatchedLeakKlassIdsForTest({kLeafClassTag}); + + jlong parentTag = 1; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, parentTag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, kParentStaleReferrerKlass, + kParentClassTag)); + + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafClassTag, + parentTag, 10); + + EXPECT_EQ(1u, ReferenceChainsTestAccessor::leakSignatureTotal(kLeafClassTag, + kParentClassTag)); + EXPECT_EQ(1u, ReferenceChainsTestAccessor::leakParentFanout(parentTag)); + + tracker->stop(); +} + +// collectLeakAccumulationCandidatesForRotation() - the two-tier design + +// The central discriminating test for the whole design (per the "ubiquitous common leaf class held +// by many small unrelated parents" concern this design exists to solve): a signature with a LARGE +// but FLAT total (many unrelated parents, e.g. a common leaf class scattered across a real +// classpath) must NOT outrank a signature with a SMALLER but GROWING total (the actual leak) once a +// growth history exists - retained-size-style ranking alone would pick the wrong one every time. +TEST_F(ReferenceChainsBfsTest, LeakAccumulationRotationPrioritizesGrowingSignatureOverLargeFlatOne) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + constexpr u32 kLeafKlass = 987; + constexpr u32 kGrowingParentKlass = 100; // signature A: the real leak + constexpr u32 kUbiquitousParentKlass = 999; // signature B: common, but flat + ReferenceChainsTestAccessor::setWatchedLeakKlassIdsForTest({kLeafKlass}); + + // Signature A: one parent, growing. + jlong growingParentTag = 1; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, growingParentTag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, kGrowingParentKlass)); + + // Signature B: 20 distinct, unrelated parents, each holding just 1-2 instances of the same + // common leaf klass - a much LARGER total than A, but it will not grow between passes. + constexpr int kUbiquitousParentCount = 20; + std::vector ubiquitousParentTags; + for (int i = 0; i < kUbiquitousParentCount; i++) { + jlong tag = 100 + i; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, tag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, kUbiquitousParentKlass)); + ubiquitousParentTags.push_back(tag); + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, tag, 1000 + i); + } + // Pass 1: A has fanout 5, B has total 20 (20 parents x 1 each) - B is larger. + for (int i = 0; i < 5; i++) { + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, + growingParentTag, 2000 + i); + } + ReferenceChainsTestAccessor::collectLeakAccumulationCandidatesForRotation( + ReferenceChainsTestAccessor::leakAccumulationRotationBudget()); + + // Pass 2: B stays exactly flat (no new admissions); A grows from 5 to 8. + for (int i = 0; i < 3; i++) { + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, + growingParentTag, 3000 + i); + } + std::vector selected = + ReferenceChainsTestAccessor::collectLeakAccumulationCandidatesForRotation( + ReferenceChainsTestAccessor::leakAccumulationRotationBudget()); + + ASSERT_EQ(1u, selected.size()); + EXPECT_EQ(growingParentTag, selected[0]) + << "the growing signature's parent must be selected, even though " + "the flat-but-larger signature has a much bigger absolute total"; + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, LeakAccumulationRotationRanksByFanoutWithinWinningSignature) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + constexpr u32 kLeafKlass = 987, kParentKlass = 100; + ReferenceChainsTestAccessor::setWatchedLeakKlassIdsForTest({kLeafKlass}); + + jlong lowFanoutTag = 1, highFanoutTag = 2; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, lowFanoutTag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, kParentKlass)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, highFanoutTag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, kParentKlass)); + + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, lowFanoutTag, 10); + for (int i = 0; i < 5; i++) { + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, + highFanoutTag, 20 + i); + } + + std::vector selected = + ReferenceChainsTestAccessor::collectLeakAccumulationCandidatesForRotation(10); + ASSERT_EQ(2u, selected.size()); + EXPECT_EQ(highFanoutTag, selected[0]) << "higher fanout ranks first"; + EXPECT_EQ(lowFanoutTag, selected[1]); + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, LeakAccumulationRotationRespectsMaxCountAndDedup) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + constexpr u32 kLeafKlass = 987, kParentKlass = 100; + ReferenceChainsTestAccessor::setWatchedLeakKlassIdsForTest({kLeafKlass}); + + jlong tag1 = 1, tag2 = 2, tag3 = 3; + for (jlong tag : {tag1, tag2, tag3}) { + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, tag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, kParentKlass)); + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, tag, 10); + } + // tag2 already queued from an earlier collector this same pass - must be skipped even though it + // qualifies structurally. + ReferenceChainsTestAccessor::pushPriorityExpand(tag2); + + std::vector selected = + ReferenceChainsTestAccessor::collectLeakAccumulationCandidatesForRotation( + /*max_count=*/1); + EXPECT_EQ(1u, selected.size()) << "capped at max_count"; + EXPECT_NE(tag2, selected[0]) << "already-queued tag must not be re-selected"; + + tracker->stop(); +} + +// Reversed on round-4 pod evidence (ev-leaktag-onpod-round4): the previous EXPANDED-only selection +// made the targeted tier select ZERO every pass on a live leak - the growing holders are +// un-expanded FRONTIER-state backlog entries that the starved pending lane never reaches (a 127k +// backlog at ~120-200 objects/min). +TEST_F(ReferenceChainsBfsTest, LeakAccumulationRotationSelectsUnexpandedFrontierParentAheadOfBacklog) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + constexpr u32 kLeafKlass = 987, kParentKlass = 100; + ReferenceChainsTestAccessor::setWatchedLeakKlassIdsForTest({kLeafKlass}); + + // Stale re-walks already sitting in the priority lane (push_back, as the other two collectors + // do). + ReferenceChainsTestAccessor::pushPriorityExpand(900); + ReferenceChainsTestAccessor::pushPriorityExpand(901); + + jlong notYetExpandedTag = 1, expandedLowFanoutTag = 2; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, notYetExpandedTag, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, kParentKlass)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, expandedLowFanoutTag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, kParentKlass)); + for (int i = 0; i < 10; i++) { + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, + notYetExpandedTag, 10 + i); + } + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, + expandedLowFanoutTag, 100); + + std::vector selected = + ReferenceChainsTestAccessor::collectLeakAccumulationCandidatesForRotation(10); + ASSERT_EQ(2u, selected.size()); + EXPECT_EQ(notYetExpandedTag, selected[0]) + << "the FRONTIER-state parent qualifies and outranks the " + "lower-fanout EXPANDED one"; + EXPECT_EQ(expandedLowFanoutTag, selected[1]); + + std::vector queue = ReferenceChainsTestAccessor::priorityExpandContents(); + ASSERT_GE(queue.size(), 4u); + EXPECT_EQ(notYetExpandedTag, queue[0]) + << "the targeted un-expanded holder must JUMP the backlog, not " + "queue behind the stale re-walks"; + EXPECT_EQ(expandedLowFanoutTag, queue[1]) + << "selection order must be preserved at the head (fanout rank)"; + EXPECT_EQ(900, queue[2]); + EXPECT_EQ(901, queue[3]); + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, LeakAccumulationRotationReturnsEmptyWhenNothingHasGrownSincePreviousPass) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + constexpr u32 kLeafKlass = 987, kParentKlass = 100; + ReferenceChainsTestAccessor::setWatchedLeakKlassIdsForTest({kLeafKlass}); + + jlong parentTag = 1; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, parentTag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, kParentKlass)); + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, parentTag, 10); + + // First call establishes the baseline (delta == total, since there is no prior snapshot) and + // selects it. + std::vector firstPass = + ReferenceChainsTestAccessor::collectLeakAccumulationCandidatesForRotation(10); + ASSERT_EQ(1u, firstPass.size()); + ReferenceChainsTestAccessor::clearPriorityExpand(); + + // Second call, nothing new admitted - delta is now 0 for every signature, so nothing should be + // selected. + std::vector secondPass = + ReferenceChainsTestAccessor::collectLeakAccumulationCandidatesForRotation(10); + EXPECT_TRUE(secondPass.empty()) + << "no signature grew since the previous pass's snapshot"; + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, LeakAccumulationRotationReturnsEmptyWhenNoSignaturesTracked) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + std::vector selected = + ReferenceChainsTestAccessor::collectLeakAccumulationCandidatesForRotation(10); + EXPECT_TRUE(selected.empty()); + + tracker->stop(); +} + +// seedLeakAccumulationForNewlyWatchedKlass() - the cold-start fix: retroactively + +TEST_F(ReferenceChainsBfsTest, SeedLeakAccumulationPopulatesFromAlreadyAdmittedEntries) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + constexpr u32 kLeafKlass = 987, kParentKlass = 100; + // Entries inserted directly (as if admitted by an earlier pass), with no watched klass_id set + // at all yet at insertion time - trackLeakAccumulation() was never called for any of these. + jlong parentTag = 1; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, parentTag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, kParentKlass)); + for (int i = 0; i < 4; i++) { + jlong childTag = 10 + i; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, childTag, parentTag, 1, FrontierEntryState::EXPANDED, + /*root_kind=*/0, /*referrer_klass=*/0, kLeafKlass)); + } + ASSERT_EQ(0u, ReferenceChainsTestAccessor::leakSignatureCount()) + << "nothing tracked yet - trackLeakAccumulation() was never called"; + + ReferenceChainsTestAccessor::seedLeakAccumulationForNewlyWatchedKlass(kLeafKlass); + + EXPECT_EQ(4u, ReferenceChainsTestAccessor::leakSignatureTotal(kLeafKlass, kParentKlass)); + EXPECT_EQ(4u, ReferenceChainsTestAccessor::leakParentFanout(parentTag)); + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, SeedLeakAccumulationSkipsNonMatchingAndNonExpandedEntries) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + constexpr u32 kLeafKlass = 987, kOtherKlass = 555, kParentKlass = 100; + jlong parentTag = 1; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, parentTag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, kParentKlass)); + // Wrong class - must not be counted. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 10, parentTag, 1, FrontierEntryState::EXPANDED, + /*root_kind=*/0, /*referrer_klass=*/0, kOtherKlass)); + // Right class, but still FRONTIER (not yet EXPANDED) - must not be counted. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 11, parentTag, 1, FrontierEntryState::FRONTIER, + /*root_kind=*/0, /*referrer_klass=*/0, kLeafKlass)); + // Right class, root-attached (no real parent) - must not be counted. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 12, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, kLeafKlass)); + + ReferenceChainsTestAccessor::seedLeakAccumulationForNewlyWatchedKlass(kLeafKlass); + + EXPECT_EQ(0u, ReferenceChainsTestAccessor::leakSignatureCount()); + + tracker->stop(); +} + +TEST_F(ReferenceChainsBfsTest, SeedLeakAccumulationComposesWithOngoingIncrementalUpdates) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + constexpr u32 kLeafKlass = 987, kParentKlass = 100; + jlong parentTag = 1; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, parentTag, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, kParentKlass)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 10, parentTag, 1, FrontierEntryState::EXPANDED, + /*root_kind=*/0, /*referrer_klass=*/0, kLeafKlass)); + + // Retroactive seed sees the one pre-existing child. + ReferenceChainsTestAccessor::seedLeakAccumulationForNewlyWatchedKlass(kLeafKlass); + ASSERT_EQ(1u, ReferenceChainsTestAccessor::leakParentFanout(parentTag)); + + // A genuinely new admission after watching starts must add on top of the retroactive baseline, + // not reset or double it. + ReferenceChainsTestAccessor::setWatchedLeakKlassIdsForTest({kLeafKlass}); + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, parentTag, 11); + + EXPECT_EQ(2u, ReferenceChainsTestAccessor::leakParentFanout(parentTag)); + EXPECT_EQ(2u, ReferenceChainsTestAccessor::leakSignatureTotal(kLeafKlass, kParentKlass)); + + tracker->stop(); +} + +// Smoke test simulating the hotdog pod conditions that starved BFS: + diff --git a/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc b/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc new file mode 100644 index 000000000..2bb5b2cd0 --- /dev/null +++ b/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc @@ -0,0 +1,905 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +class PollWatchedTargetsTest : public ::testing::Test { +protected: + jvmtiInterface_1_ jvmti_tbl{}; + _jvmtiEnv mock_jvmti{}; + JNINativeInterface_ jni_tbl{}; + JNIEnv_ mock_jni{}; + + std::unordered_map tags; + std::unordered_set dead_refs; // NewLocalRef returns NULL for these + + jvmtiEnv *orig_jvmti = nullptr; + static PollWatchedTargetsTest *active_fixture; + + void SetUp() override { + active_fixture = this; + ReferenceChainsTestAccessor::reset(); + LivenessTracker::instance()->klassPopulationResetForTest(); + LivenessTracker::instance()->setGcGenerationsForTest(true); + + jvmti_tbl = jvmtiInterface_1_{}; + jvmti_tbl.SetEventNotificationMode = &mock_SetEventNotificationMode; + jvmti_tbl.GetTag = &mock_GetTag; + jvmti_tbl.SetTag = &mock_SetTag; + jvmti_tbl.GetClassSignature = &mock_GetClassSignature; + jvmti_tbl.Deallocate = &mock_Deallocate; + mock_jvmti.functions = &jvmti_tbl; + orig_jvmti = VMTestAccessor::getJvmti(); + VMTestAccessor::setJvmti(&mock_jvmti); + + jni_tbl = JNINativeInterface_{}; + jni_tbl.NewLocalRef = &mock_NewLocalRef; + jni_tbl.DeleteLocalRef = &mock_DeleteLocalRef; + jni_tbl.GetObjectClass = &mock_GetObjectClass; + mock_jni.functions = &jni_tbl; + } + + void TearDown() override { + VMTestAccessor::setJvmti(orig_jvmti); + LivenessTracker::instance()->klassPopulationResetForTest(); + LivenessTracker::instance()->setGcGenerationsForTest(false); + active_fixture = nullptr; + } + + static jvmtiError JNICALL mock_GetTag(jvmtiEnv *, jobject object, jlong *tag_ptr) { + auto it = active_fixture->tags.find(object); + *tag_ptr = it != active_fixture->tags.end() ? it->second : 0; + return JVMTI_ERROR_NONE; + } + + static jvmtiError JNICALL mock_SetTag(jvmtiEnv *, jobject object, jlong tag) { + active_fixture->tags[object] = tag; + return JVMTI_ERROR_NONE; + } + + static jobject JNICALL mock_NewLocalRef(JNIEnv *, jobject ref) { + if (active_fixture->dead_refs.count(ref) > 0) { + return nullptr; + } + return ref; // identity passthrough - see this fixture's own comment + } + + static void JNICALL mock_DeleteLocalRef(JNIEnv *, jobject) { + // no-op: this fixture's fake jobject values are not real JNI refs. + } + + // pollWatchedTargets()'s diagnostic class-name lookup on the candidate's representative: this + // fixture's fake jobjects carry no real class identity, so a fixed non-null jclass plus a fixed + // signature is all GetObjectClass()/GetClassSignature() need to return for that lookup to + // complete without touching a real JVM. + static jclass JNICALL mock_GetObjectClass(JNIEnv *, jobject) { + return (jclass)0xC1A55; + } + + static jvmtiError JNICALL mock_GetClassSignature(jvmtiEnv *, jclass, + char **signature_ptr, + char **generic_ptr) { + *signature_ptr = strdup("Ltest/FakeKlass;"); + if (generic_ptr != nullptr) { + *generic_ptr = nullptr; + } + return JVMTI_ERROR_NONE; + } + + static jvmtiError JNICALL mock_Deallocate(jvmtiEnv *, unsigned char *mem) { + free(mem); + return JVMTI_ERROR_NONE; + } + + // Seeds LivenessTracker's real population table with a growing series for `klass_id` (20 + // strictly-increasing samples - satisfies selectLeakCandidates()'s min-fill, growth/floor + // magnitude, and sustained-trend hysteresis requirements, livenessTracker.h; 20 rather than the + // 10-sample minimum fill leaves comfortable margin past the hysteresis threshold rather than + // sitting exactly on its boundary) and points its representative at `rep`. + void seedGrowingCandidate(u32 klass_id, jweak rep) { + int slot; + bool created; + for (u16 i = 1; i <= 20; i++) { + LivenessTracker::instance()->klassPopulationRecordForTest( + klass_id, i, i, &slot, &created); + // Per-(klass, tid) qualification: selectLeakCandidates() also requires a qualifying + // allocating thread. + LivenessTracker::instance()->tidTrendRecordForTest( + klass_id, /*tid=*/4242, (u32)i, (u64)i); + } + LivenessTracker::instance()->klassPopulationSetRepresentativeForTest(nullptr, klass_id, rep); + } +}; + +PollWatchedTargetsTest *PollWatchedTargetsTest::active_fixture = nullptr; + +TEST_F(PollWatchedTargetsTest, EmitsEventForAlreadyDiscoveredCandidate) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int fake_object_storage = 0; + jobject obj = reinterpret_cast(&fake_object_storage); + seedGrowingCandidate(/*source_tag=*/1, /*rep=*/(jweak)obj); + + // Model "already discovered by an ordinary runPass()": a root-level FrontierTable entry plus a + // matching GetTag() result, mirroring referenceChainJfrRoundtrip_ut.cpp's seeding style. + ASSERT_TRUE(tracker->frontierTable()->insert( + /*tag=*/7, /*parent_tag=*/0, /*referrer_klass=*/1, /*depth=*/0, + FrontierEntryState::EDGE)); + tags[obj] = 7; + // With class-tag matching, _candidate_frontier_tags must be set so buildCanaryChainEvent() can + // reconstruct the chain. + ReferenceChainsTestAccessor::setCandidateFrontierTagForTest(0, 7); + + tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); + + EXPECT_EQ(1u, ReferenceChainsTestAccessor::resolvedChainCount()); + EXPECT_TRUE(ReferenceChainsTestAccessor::hasResolvedChainForTag(7)); + EXPECT_EQ(7, ReferenceChainsTestAccessor::resolvedChainSourceTag(7)); + + tracker->stop(); +} + +TEST_F(PollWatchedTargetsTest, NoEventForNotYetDiscoveredCandidate) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int fake_object_storage = 0; + jobject obj = reinterpret_cast(&fake_object_storage); + seedGrowingCandidate(/*source_tag=*/1, /*rep=*/(jweak)obj); + // GetTag() reports 0 (default) - no pass has reached this object yet. + + tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); + + EXPECT_EQ(0u, ReferenceChainsTestAccessor::resolvedChainCount()); + + tracker->stop(); +} + +TEST_F(PollWatchedTargetsTest, NoDuplicateOnRepeatPoll) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int fake_object_storage = 0; + jobject obj = reinterpret_cast(&fake_object_storage); + seedGrowingCandidate(/*source_tag=*/1, /*rep=*/(jweak)obj); + + ASSERT_TRUE(tracker->frontierTable()->insert( + 7, 0, 1, 0, FrontierEntryState::EDGE)); + tags[obj] = 7; + ReferenceChainsTestAccessor::setCandidateFrontierTagForTest(0, 7); + + tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); + ASSERT_EQ(1u, ReferenceChainsTestAccessor::resolvedChainCount()); + + // Klass 1 is still flagged (LivenessTracker's ranking doesn't know an event was already emitted + // for it) - a second, third, ... + tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); + tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); + EXPECT_EQ(1u, ReferenceChainsTestAccessor::resolvedChainCount()); + EXPECT_TRUE(ReferenceChainsTestAccessor::hasResolvedChainForTag(7)); + EXPECT_EQ(7, ReferenceChainsTestAccessor::resolvedChainSourceTag(7)); + + tracker->stop(); +} + +TEST_F(PollWatchedTargetsTest, SkipsCandidateWhoseWeakReferenceDied) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int fake_object_storage = 0; + jobject obj = reinterpret_cast(&fake_object_storage); + seedGrowingCandidate(/*source_tag=*/1, /*rep=*/(jweak)obj); + dead_refs.insert(obj); // NewLocalRef(rep) -> NULL, as if GC'd + + ASSERT_TRUE(tracker->frontierTable()->insert( + 7, 0, 1, 0, FrontierEntryState::EDGE)); + tags[obj] = 7; // would resolve to a discovered tag, if it could resolve + + tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); + + EXPECT_EQ(0u, ReferenceChainsTestAccessor::resolvedChainCount()); + + tracker->stop(); +} + +// A cached chain must not re-emit forever: once the klass's representative stops resolving +// (collected, or LRU-evicted from LivenessTracker's population table - +// klassPopulationSetRepresentativeForTest()'s ref is the stand-in for either), the very next poll +// must prune it from _resolved_chains rather than leaving a dump keep re-emitting a chain for a +// sample that is gone (see _resolved_chains' own comment, referenceChains.h, and +// pollWatchedTargets()'s "candidate died, or was evicted" branch). +TEST_F(PollWatchedTargetsTest, ChainPersistsAfterRepresentativeDies) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int fake_object_storage = 0; + jobject obj = reinterpret_cast(&fake_object_storage); + seedGrowingCandidate(/*source_tag=*/1, /*rep=*/(jweak)obj); + + ASSERT_TRUE(tracker->frontierTable()->insert( + 7, 0, 1, 0, FrontierEntryState::EDGE)); + tags[obj] = 7; + ReferenceChainsTestAccessor::setCandidateFrontierTagForTest(0, 7); + + tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); + ASSERT_EQ(1u, ReferenceChainsTestAccessor::resolvedChainCount()); + ASSERT_TRUE(ReferenceChainsTestAccessor::hasResolvedChainForTag(7)); + + // The representative died. Per-instance caching: the chain persists (it describes a reference + // path that was valid at resolution time). + dead_refs.insert(obj); + + tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); + + EXPECT_EQ(1u, ReferenceChainsTestAccessor::resolvedChainCount()) + << "per-instance chains persist after representative dies; " + "they expire on search restart, not on representative death"; + EXPECT_TRUE(ReferenceChainsTestAccessor::hasResolvedChainForTag(7)); + + tracker->stop(); +} + +// Bounded canary parent-chain walk: a cyclic/corrupt parent chain must fail +// safe instead of spinning the poll thread (the bound mirrors +// FrontierTable::reconstructChain()'s maxCapacity() bound), and legitimate +// deep chains must still reconstruct fully - the bound must not cost +// convergence. +TEST_F(PollWatchedTargetsTest, CanaryChainWalkFailsSafeOnCyclicParentChain) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + FrontierTable *frontier = tracker->frontierTable(); + // Corrupt chain: 1 -> 2 -> 1 (a cycle no legitimate BFS admission could + // produce, but insert() does not validate parent chains - exactly the + // failure mode reconstructChain()'s own bound guards against). + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 1, 2, 2, FrontierEntryState::EDGE, /*root_kind=*/0, + /*referrer_klass=*/11)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 2, 1, 3, FrontierEntryState::EDGE, /*root_kind=*/0, + /*referrer_klass=*/22)); + + ReferenceChainsTestAccessor::setCandidateCountForTest(1); + ReferenceChainsTestAccessor::setCandidateKlassIdForTest(0, 99); + ReferenceChainsTestAccessor::setCandidateParentTagForTest(0, 1); + ReferenceChainsTestAccessor::setCandidateFrontierTagForTest(0, 2); + ReferenceChainsTestAccessor::setCandidateReferrerKlassForTest(0, 99); + ReferenceChainsTestAccessor::setCandidateDepthForTest(0, 4); + + ReferenceChainEvent event; + // The unbounded predecessor of this walk spun forever here; the bound + // must make it return promptly (the test itself is the termination + // proof - a regression to an unbounded walk hangs this test). + EXPECT_FALSE(ReferenceChainsTestAccessor::buildCanaryChainEventForTest( + 0, &event)); + + tracker->stop(); +} + +TEST_F(PollWatchedTargetsTest, CanaryChainWalkStillReconstructsDeepChains) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + FrontierTable *frontier = tracker->frontierTable(); + // A 200-hop linear chain, root-attached at tag 1 (parent_tag == 0): far + // past the 64-hop default _hop_cap, but the walk bounds at the FRONTIER's + // maxCapacity() (65536 by default), not the BFS hop cap - legitimate deep + // chains reconstruct exactly as before the bound. + const int kDepth = 200; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 1, 0, 0, FrontierEntryState::EDGE, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/1)); + for (int i = 2; i <= kDepth; i++) { + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, i, i - 1, i - 1, FrontierEntryState::EDGE, /*root_kind=*/0, + /*referrer_klass=*/(u32)i)); + } + + ReferenceChainsTestAccessor::setCandidateCountForTest(1); + ReferenceChainsTestAccessor::setCandidateKlassIdForTest(0, 999); + ReferenceChainsTestAccessor::setCandidateParentTagForTest(0, kDepth); + ReferenceChainsTestAccessor::setCandidateFrontierTagForTest(0, 1); + ReferenceChainsTestAccessor::setCandidateReferrerKlassForTest(0, 999); + ReferenceChainsTestAccessor::setCandidateDepthForTest(0, kDepth); + + ReferenceChainEvent event; + ASSERT_TRUE(ReferenceChainsTestAccessor::buildCanaryChainEventForTest( + 0, &event)); + // Candidate's own klass first, then the reversed walk: root-side hop + // (tag 1, klass 1) first, parent-side hop (tag 200, klass 200) last. + ASSERT_EQ((size_t)(kDepth + 1), event._hops.size()); + EXPECT_EQ(999u, event._hops[0].klass_id); + for (int i = 1; i <= kDepth; i++) { + EXPECT_EQ((u32)i, event._hops[i].klass_id) + << "hop " << i; + } + EXPECT_EQ(1u, event._target_tag); // the candidate's frontier tag + EXPECT_EQ((u32)kDepth, event._depth); + // The root kind describes the chain's ROOT (the root-attached tag-1 entry + // this walk terminates at), not the candidate-side parent entry - same + // terminal-root semantics reconstructChain() uses. + EXPECT_EQ((u8)JVMTI_HEAP_REFERENCE_STATIC_FIELD, event._root_kind); + + tracker->stop(); +} + +TEST_F(PollWatchedTargetsTest, NoOpWhenGcGenerationsDisabled) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + // Overrides this fixture's own SetUp() default - exercises the pollWatchedTargets() guard + // covering LivenessTracker's own _gc_generations gate (population tracking's own gate), not + // just this tracker's own _enabled. + LivenessTracker::instance()->setGcGenerationsForTest(false); + + int fake_object_storage = 0; + jobject obj = reinterpret_cast(&fake_object_storage); + seedGrowingCandidate(/*source_tag=*/1, /*rep=*/(jweak)obj); + ASSERT_TRUE(tracker->frontierTable()->insert( + 7, 0, 1, 0, FrontierEntryState::EDGE)); + tags[obj] = 7; + + tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); + + EXPECT_EQ(0u, ReferenceChainsTestAccessor::resolvedChainCount()); + + tracker->stop(); +} + +// Resolved-chain cache (ReferenceChainTracker::cacheResolvedChain()/ + +class ResolvedChainCacheTest : public ::testing::Test { +protected: + void SetUp() override { + ReferenceChainsTestAccessor::reset(); + } + + void TearDown() override { + ReferenceChainsTestAccessor::reset(); + } + + static ReferenceChainEvent makeEvent(u64 target_tag) { + ReferenceChainEvent event; + event._target_tag = target_tag; + event._depth = 0; + return event; + } +}; + +// The defining property of the "stick around" model: a cached chain is re-emitted on every dump, +// not drained once. +TEST_F(ResolvedChainCacheTest, SnapshotReEmitsOnEveryDumpWithoutClearing) { + ReferenceChainsTestAccessor::cacheChain(/*source_tag=*/1, makeEvent(7), + /*source_tag=*/7, /*search_ns=*/0); + + std::vector firstDump; + ReferenceChainsTestAccessor::drain(&firstDump); + ASSERT_EQ(1u, firstDump.size()); + EXPECT_EQ(7u, firstDump[0]._target_tag); + EXPECT_EQ(1u, ReferenceChainsTestAccessor::resolvedChainCount()) + << "drain must not clear the cache"; + + // A second dump with nothing changed re-emits the same chain. + std::vector secondDump; + ReferenceChainsTestAccessor::drain(&secondDump); + ASSERT_EQ(1u, secondDump.size()); + EXPECT_EQ(7u, secondDump[0]._target_tag); + EXPECT_EQ(1u, ReferenceChainsTestAccessor::resolvedChainCount()); +} + +// Re-resolving the same klass (a restart re-tags its sample, or a fresh walk finds a deeper path) +// refreshes its single cache slot in place rather than accumulating duplicates - so a dump re-emits +// one current chain per klass, not one per resolution. +TEST_F(ResolvedChainCacheTest, RefreshReplacesSameKlassInPlace) { + ReferenceChainsTestAccessor::cacheChain(1, makeEvent(7), 7, 0); + ASSERT_EQ(1u, ReferenceChainsTestAccessor::resolvedChainCount()); + EXPECT_EQ(7, ReferenceChainsTestAccessor::resolvedChainSourceTag(1)); + + // Same klass, rebuilt from a new tag (e.g. after a search restart). + ReferenceChainsTestAccessor::cacheChain(1, makeEvent(9), 9, 0); + EXPECT_EQ(1u, ReferenceChainsTestAccessor::resolvedChainCount()) + << "refresh must overwrite, not append"; + EXPECT_EQ(9, ReferenceChainsTestAccessor::resolvedChainSourceTag(1)); + + std::vector dump; + ReferenceChainsTestAccessor::drain(&dump); + ASSERT_EQ(1u, dump.size()); + EXPECT_EQ(9u, dump[0]._target_tag); +} + +// Distinct klasses each get their own slot and all re-emit together in one dump (order is +// unspecified - the cache is a map keyed by klass_id). +TEST_F(ResolvedChainCacheTest, MultipleKlassesAllSnapshotTogether) { + ReferenceChainsTestAccessor::cacheChain(1, makeEvent(1), 1, 0); + ReferenceChainsTestAccessor::cacheChain(2, makeEvent(2), 2, 0); + ReferenceChainsTestAccessor::cacheChain(3, makeEvent(3), 3, 0); + ASSERT_EQ(3u, ReferenceChainsTestAccessor::resolvedChainCount()); + + std::vector dump; + ReferenceChainsTestAccessor::drain(&dump); + ASSERT_EQ(3u, dump.size()); + std::set tags; + for (const auto &e : dump) { + tags.insert(e._target_tag); + } + EXPECT_EQ((std::set{1, 2, 3}), tags); +} + +// A brand-new klass arriving with the cache already at MAX_RESOLVED_CHAINS is dropped (and counted +// via REFERENCE_CHAIN_EVENTS_DROPPED, this codebase's own "dropped-event-without-counter" review +// lens) rather than evicting some other still-live sample's chain - but refreshing a klass that is +// already cached still succeeds even at capacity. +TEST_F(ResolvedChainCacheTest, OverflowDropsNewKlassButAllowsRefresh) { + const int cap = ReferenceChainsTestAccessor::maxResolvedChains(); + long long droppedBefore = Counters::getCounter(REFERENCE_CHAIN_EVENTS_DROPPED); + + for (int i = 0; i < cap; i++) { + ReferenceChainsTestAccessor::cacheChain((jlong)i, makeEvent((jlong)i), + (jlong)i, 0); + } + ASSERT_EQ((size_t)cap, ReferenceChainsTestAccessor::resolvedChainCount()); + EXPECT_EQ(droppedBefore, Counters::getCounter(REFERENCE_CHAIN_EVENTS_DROPPED)) + << "filling exactly to capacity must not drop anything yet"; + + // A brand-new klass at capacity is dropped and counted. + ReferenceChainsTestAccessor::cacheChain((jlong)cap, makeEvent((jlong)cap), + (jlong)cap, 0); + EXPECT_EQ((size_t)cap, ReferenceChainsTestAccessor::resolvedChainCount()) + << "cache must stay capped, not grow past MAX_RESOLVED_CHAINS"; + EXPECT_EQ(droppedBefore + 1, Counters::getCounter(REFERENCE_CHAIN_EVENTS_DROPPED)); + EXPECT_FALSE(ReferenceChainsTestAccessor::hasResolvedChainForTag((u32)cap)); + + // Refreshing an already-cached klass at capacity must still succeed - it reuses that klass's + // existing slot rather than needing a free one. + ReferenceChainsTestAccessor::cacheChain(/*source_tag=*/0, makeEvent(999), + /*source_tag=*/999, 0); + EXPECT_EQ((size_t)cap, ReferenceChainsTestAccessor::resolvedChainCount()); + EXPECT_EQ(999, ReferenceChainsTestAccessor::resolvedChainSourceTag(0)); + EXPECT_EQ(droppedBefore + 1, Counters::getCounter(REFERENCE_CHAIN_EVENTS_DROPPED)) + << "an in-place refresh must not count as a drop"; +} + +// Pause-time pacing controller: pause-time-SLO feedback loop + +TEST_F(ReferenceChainsTest, PacingHoldsSteadyWhenPassesLandExactlyOnCeiling) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:budget=4000:pausetarget=5")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int startBudget = ReferenceChainsTestAccessor::effectiveBudget(); + u64 startCadence = ReferenceChainsTestAccessor::effectiveCadenceNs(); + ASSERT_EQ(4000, startBudget); // starts pinned at the configured ceiling + + // A pass landing exactly on the pause-time target is a zero error every call - the controller + // should never move away from its starting point, regardless of how many such passes are + // observed in a row. + for (int i = 0; i < 10; i++) { + ReferenceChainsTestAccessor::updatePacing(5 * 1000000ULL); // 5ms + EXPECT_EQ(startBudget, ReferenceChainsTestAccessor::effectiveBudget()); + EXPECT_EQ(startCadence, ReferenceChainsTestAccessor::effectiveCadenceNs()); + } + + tracker->stop(); +} + +TEST_F(ReferenceChainsTest, PacingShrinksBudgetAndWidensCadenceWhenOverCeiling) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:budget=4000:pausetarget=5")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int initialBudget = ReferenceChainsTestAccessor::effectiveBudget(); + u64 initialCadence = ReferenceChainsTestAccessor::effectiveCadenceNs(); + + // A pass taking 10x the pause-time ceiling, fed repeatedly (a constant input - the plan's own + // "does not oscillate indefinitely" scenario). + int lastBudget = initialBudget; + u64 lastCadence = initialCadence; + for (int i = 0; i < 20; i++) { + ReferenceChainsTestAccessor::updatePacing(50 * 1000000ULL); // 50ms + int budget = ReferenceChainsTestAccessor::effectiveBudget(); + u64 cadence = ReferenceChainsTestAccessor::effectiveCadenceNs(); + EXPECT_LE(budget, lastBudget); // never grows while still over ceiling + EXPECT_GE(cadence, lastCadence); // never shrinks while still over ceiling + lastBudget = budget; + lastCadence = cadence; + } + + // Moved in the correct direction... + EXPECT_LT(lastBudget, initialBudget); + EXPECT_GT(lastCadence, initialCadence); + // ...and converged to a fixed point rather than oscillating: one more identical input produces + // no further change. + ReferenceChainsTestAccessor::updatePacing(50 * 1000000ULL); + EXPECT_EQ(lastBudget, ReferenceChainsTestAccessor::effectiveBudget()); + EXPECT_EQ(lastCadence, ReferenceChainsTestAccessor::effectiveCadenceNs()); + + tracker->stop(); +} + +TEST_F(ReferenceChainsTest, PacingGrowsBudgetBackAndRelaxesCadenceWhenUnderCeiling) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:budget=4000:pausetarget=5")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + // Start from a controlled below-ceiling/above-baseline point (as if an earlier over-ceiling run + // had already shrunk/widened them - see the previous test) with a freshly reset controller, + // rather than chaining directly off a constant-input sequence like the previous test's own: + // _pause_pid's integral state would otherwise still be recovering from that sequence's windup + // for many iterations after switching to a smaller-magnitude error, muddying this test's + // per-step "moves in the correct direction every step" assertions with a transient this test is + // not about. + ReferenceChainsTestAccessor::setEffectiveBudget(2400); + ReferenceChainsTestAccessor::setEffectiveCadenceNs( + 2 * ReferenceChainsTestAccessor::baselineCadenceNs()); + ReferenceChainsTestAccessor::resetPacingController(); + int shrunkBudget = ReferenceChainsTestAccessor::effectiveBudget(); + u64 widenedCadence = ReferenceChainsTestAccessor::effectiveCadenceNs(); + + // Now feed passes comfortably under the ceiling, repeatedly (a constant input, to check + // convergence rather than oscillation). + int lastBudget = shrunkBudget; + u64 lastCadence = widenedCadence; + for (int i = 0; i < 200; i++) { + ReferenceChainsTestAccessor::updatePacing(0); // effectively instant + int budget = ReferenceChainsTestAccessor::effectiveBudget(); + u64 cadence = ReferenceChainsTestAccessor::effectiveCadenceNs(); + EXPECT_GE(budget, lastBudget); // never shrinks while comfortably under + EXPECT_LE(cadence, lastCadence); // never widens while comfortably under + lastBudget = budget; + lastCadence = cadence; + } + + // Moved in the correct direction... and, since 50 identical comfortably-under-target passes is + // well past BORROW_WARMUP_PASSES, past the configured ceiling too - budget-borrowing lets it + // converge at the borrowed ceiling (configured budget * multiplier) instead of stalling at the + // plain configured budget. + EXPECT_GT(lastBudget, shrunkBudget); + EXPECT_EQ(4000 * ReferenceChainsTestAccessor::borrowCeilingMultiplier(), lastBudget); + EXPECT_LT(lastCadence, widenedCadence); + // ...and converged: one more identical input produces no further change. + ReferenceChainsTestAccessor::updatePacing(0); + EXPECT_EQ(lastBudget, ReferenceChainsTestAccessor::effectiveBudget()); + EXPECT_EQ(lastCadence, ReferenceChainsTestAccessor::effectiveCadenceNs()); + + tracker->stop(); +} + +TEST_F(ReferenceChainsTest, MaybeRevokeBorrowForRootEnumPassPreservesBorrowAtBoundary) { + Arguments args; + // BORROW_UNDER_TARGET_FRACTION (referenceChains.h) is 0.5, so with pausetarget=10 the + // comfortably-under-target boundary is exactly 5ms. + ASSERT_FALSE(args.parse("referencechains=true:budget=1000:pausetarget=10")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + ReferenceChainsTestAccessor::setBorrowedBudget(500); + ReferenceChainsTestAccessor::setConsecutiveUnderTargetPasses(5); + + // Exactly at the boundary: comfortably_under_target's `<=` check must still treat this as + // comfortably under, so the borrow is preserved. + ReferenceChainsTestAccessor::maybeRevokeBorrowForRootEnumPass(5 * 1000000ULL); + EXPECT_EQ(500, ReferenceChainsTestAccessor::borrowedBudget()); + EXPECT_EQ(5, ReferenceChainsTestAccessor::consecutiveUnderTargetPasses()); + + tracker->stop(); +} + +TEST_F(ReferenceChainsTest, MaybeRevokeBorrowForRootEnumPassRevokesJustPastBoundary) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:budget=1000:pausetarget=10")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + ReferenceChainsTestAccessor::setBorrowedBudget(500); + ReferenceChainsTestAccessor::setConsecutiveUnderTargetPasses(5); + ReferenceChainsTestAccessor::setEffectiveBudget(1500); // as if borrow had raised the ceiling + + // Just past the boundary: no longer comfortably under target, so the grant is revoked + // immediately, including re-clamping _effective_budget down to the plain (non-borrowed) budget + // rather than leaving it borrow-inflated until the next ordinary pass's updatePacing() call. + ReferenceChainsTestAccessor::maybeRevokeBorrowForRootEnumPass(6 * 1000000ULL); + EXPECT_EQ(0, ReferenceChainsTestAccessor::borrowedBudget()); + EXPECT_EQ(0, ReferenceChainsTestAccessor::consecutiveUnderTargetPasses()); + EXPECT_EQ(1000, ReferenceChainsTestAccessor::effectiveBudget()); + + tracker->stop(); +} + +// PainBudget (painBudget.h) - standalone, no ReferenceChainTracker singleton + +TEST(PainBudgetTest, ClearBeforeAnythingIsEverSpent) { + PainBudget budget(0.01); + EXPECT_TRUE(budget.canStartNow(1000)); +} + +TEST(PainBudgetTest, SpendCreatesDebtThatBlocksAnImmediateSecondCall) { + PainBudget budget(0.01); // 1% + ASSERT_TRUE(budget.canStartNow(1000)); // establishes the drain baseline + budget.spend(100); // 100ms of debt + // No time has elapsed since the baseline call above - the debt cannot have drained at all yet. + EXPECT_FALSE(budget.canStartNow(1000)); +} + +TEST(PainBudgetTest, DebtDrainsProportionallyToElapsedTimeAndRefillRate) { + PainBudget budget(0.01); // 1% -> 1ms of debt needs 100ms elapsed to clear + ASSERT_TRUE(budget.canStartNow(0)); + budget.spend(10); // 10ms of debt -> needs 1000ms elapsed to fully clear + EXPECT_FALSE(budget.canStartNow(500ULL * 1000000ULL)); // 500ms elapsed - not enough + EXPECT_TRUE(budget.canStartNow(1500ULL * 1000000ULL)); // 1500ms total - enough +} + +TEST(PainBudgetTest, ZeroRefillRateNeverClearsDebt) { + PainBudget budget(0.0); + ASSERT_TRUE(budget.canStartNow(0)); + budget.spend(1); + // An enormous elapsed time still drains nothing at a 0 refill rate. + EXPECT_FALSE(budget.canStartNow(1000000000000ULL)); +} + +// Search restart (referenceChains.h's own header comment: gating a + +class SearchRestartTest : public ::testing::Test { +protected: + jvmtiInterface_1_ jvmti_tbl{}; + _jvmtiEnv mock_jvmti{}; + jvmtiEnv *orig_jvmti = nullptr; + + void SetUp() override { + ReferenceChainsTestAccessor::reset(); + LivenessTracker::instance()->klassPopulationResetForTest(); + LivenessTracker::instance()->setGcGenerationsForTest(false); + + jvmti_tbl = jvmtiInterface_1_{}; + jvmti_tbl.SetEventNotificationMode = &mock_SetEventNotificationMode; + jvmti_tbl.GetLoadedClasses = &mock_GetLoadedClasses; + jvmti_tbl.FollowReferences = &mock_FollowReferences; + jvmti_tbl.IterateOverReachableObjects = &mock_IterateOverReachableObjects; + jvmti_tbl.GetAvailableProcessors = &mock_GetAvailableProcessors; + mock_jvmti.functions = &jvmti_tbl; + orig_jvmti = VMTestAccessor::getJvmti(); + VMTestAccessor::setJvmti(&mock_jvmti); + } + + void TearDown() override { + VMTestAccessor::setJvmti(orig_jvmti); + LivenessTracker::instance()->klassPopulationResetForTest(); + LivenessTracker::instance()->setGcGenerationsForTest(false); + // UrgentOOMProjectionBypassesCandidateGate below sets this - reset it here (TearDown always + // runs, even after a fatal ASSERT_* return) rather than as a trailing statement in that + // test body, so a failed assertion can't leak a stale max-heap value into the next test + // sharing this singleton. + LivenessTracker::instance()->setMaxHeapBytesForTest(-1); + } + + // No loaded classes to resolve - resolveLoadedClasses() reports 0 and does nothing further. + static jvmtiError JNICALL mock_GetLoadedClasses(jvmtiEnv *, jint *count, + jclass **out) { + *count = 0; + *out = nullptr; + return JVMTI_ERROR_NONE; + } + + // ReferenceChainTracker::start() -> autoTuneDefaults() queries this whenever LivenessTracker + // reports a max heap > 0 - which UrgentOOMProjectionBypassesCandidateGate below sets. + static jvmtiError JNICALL mock_GetAvailableProcessors(jvmtiEnv *, + jint *nprocs) { + *nprocs = 1; + return JVMTI_ERROR_NONE; + } + + // Never invokes the callback - models a heap with nothing reachable from any root, so the very + // first pass completes immediately (0 admitted edges, not truncated). + static jvmtiError JNICALL mock_FollowReferences( + jvmtiEnv *, jint, jclass, jobject, const jvmtiHeapCallbacks *, + const void *) { + return JVMTI_ERROR_NONE; + } + + // runPassManualWalk()'s root enumeration - never invokes the root callback, same "nothing + // reachable from any root" heap model as mock_FollowReferences() above, so the first pass still + // completes immediately with 0 admitted edges. + static jvmtiError JNICALL mock_IterateOverReachableObjects( + jvmtiEnv *, jvmtiHeapRootCallback, jvmtiStackReferenceCallback, + jvmtiObjectReferenceCallback, const void *) { + return JVMTI_ERROR_NONE; + } + + // Same seeding helper as PollWatchedTargetsTest above (20 strictly- increasing samples - + // satisfies selectLeakCandidates()'s min-fill, growth/floor magnitude, and sustained-trend + // hysteresis requirements). + void seedGrowingCandidate(u32 klass_id, jweak rep) { + int slot; + bool created; + for (u16 i = 1; i <= 20; i++) { + LivenessTracker::instance()->klassPopulationRecordForTest( + klass_id, i, i, &slot, &created); + // Per-(klass, tid) qualification: selectLeakCandidates() also requires a qualifying + // allocating thread. + LivenessTracker::instance()->tidTrendRecordForTest( + klass_id, /*tid=*/4242, (u32)i, (u64)i); + } + LivenessTracker::instance()->klassPopulationSetRepresentativeForTest(nullptr, klass_id, rep); + } +}; + +TEST_F(SearchRestartTest, WithoutGenerationsSignalRestartStaysUnconditional) { + // gc_generations off (this fixture's SetUp default): canAffordNewSearch() has no candidate + // signal to gate on at all, so a terminal search is immediately eligible to restart - preserves + // this tracker's pre-restart behavior for a referencechains-without-generations setup (this + // class's own header comment, last paragraph). + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + ASSERT_TRUE(tracker->runPass(&mock_jvmti, nullptr)); + ASSERT_EQ(SearchState::COMPLETED, tracker->searchState()); + + EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(1)); + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); + + tracker->stop(); +} + +TEST_F(SearchRestartTest, GenerationsEnabledButNoCandidateBlocksFirstSearch) { + // A brand-new tracker must not pay for the initial whole-heap walk/tagging pass either when + // there is no leak candidate yet - shouldRunPass()'s !_search_started branch now shares + // canAffordNewSearch() with the restart gate below (this class's own header comment). + LivenessTracker::instance()->setGcGenerationsForTest(true); + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + EXPECT_FALSE(ReferenceChainsTestAccessor::shouldRunPass(1)); + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); + EXPECT_EQ(0, tracker->passesRun()); + + int fake_object_storage = 0; + seedGrowingCandidate(/*source_tag=*/1, /*rep=*/(jweak)&fake_object_storage); + + EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(2)); + + tracker->stop(); +} + +TEST_F(SearchRestartTest, GenerationsEnabledButNoCandidateBlocksRestart) { + LivenessTracker::instance()->setGcGenerationsForTest(true); + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + ASSERT_TRUE(tracker->runPass(&mock_jvmti, nullptr)); + ASSERT_EQ(SearchState::COMPLETED, tracker->searchState()); + + // No leak candidate flagged - nothing to justify the cost of a restart. + EXPECT_FALSE(ReferenceChainsTestAccessor::shouldRunPass(1)); + EXPECT_EQ(SearchState::COMPLETED, tracker->searchState()); + + tracker->stop(); +} + +TEST_F(SearchRestartTest, RestartsOnceACandidateAppearsAndResetsPerSearchState) { + LivenessTracker::instance()->setGcGenerationsForTest(true); + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + ASSERT_TRUE(tracker->runPass(&mock_jvmti, nullptr)); + ASSERT_EQ(SearchState::COMPLETED, tracker->searchState()); + ASSERT_EQ(1, tracker->passesRun()); + + int fake_object_storage = 0; + seedGrowingCandidate(/*source_tag=*/1, /*rep=*/(jweak)&fake_object_storage); + + EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(1)); // restartSearch() runs inline + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); + EXPECT_EQ(0, tracker->passesRun()); // restartSearch() zeroed per-search state + + // The next runPass() call takes the "first pass of a search" branch again, exactly like a + // brand-new tracker. + ASSERT_TRUE(tracker->runPass(&mock_jvmti, nullptr)); + EXPECT_EQ(SearchState::COMPLETED, tracker->searchState()); + EXPECT_EQ(1, tracker->passesRun()); + + tracker->stop(); +} + +TEST_F(SearchRestartTest, PainBudgetBlocksARestartUntilItDrains) { + LivenessTracker::instance()->setGcGenerationsForTest(true); + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:painbudget=1")); // 1% + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int fake_object_storage = 0; + seedGrowingCandidate(/*source_tag=*/1, /*rep=*/(jweak)&fake_object_storage); + + // First-ever search: called via runPass() directly here, bypassing shouldRunPass()'s + // canAffordNewSearch() gate entirely - the candidate seeded above would satisfy that gate + // anyway (this class's own header comment). + ASSERT_TRUE(tracker->runPass(&mock_jvmti, nullptr)); + ASSERT_EQ(SearchState::COMPLETED, tracker->searchState()); + + // Restart #1: _safepoint_pain_budget has never had anything spent into it yet, so this is + // always immediately affordable regardless of this first search's own cost - the cost a search + // incurs only debits the *next* restart's affordability (restartSearch()'s own spend-then-reset + // order), not its own. + ASSERT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(1)); + ASSERT_EQ(SearchState::RUNNING, tracker->searchState()); + ASSERT_TRUE(tracker->runPass(&mock_jvmti, nullptr)); + ASSERT_EQ(SearchState::COMPLETED, tracker->searchState()); + + // Pretend this second search cost 1000ms of safepoint time - a mocked FollowReferences call in + // this fixture takes ~0 real wall-clock time, so this accessor stands in for what a real, + // expensive pass would have accumulated into _search_pain_ms on its own. + ReferenceChainsTestAccessor::setSearchPainMs(1000); + + // Restart #2: the terminal gate charges the finished search's OWN 1000ms cost BEFORE checking + // affordability (canAffordNewSearch() must see the cost of the search that just ended, or an + // expensive search would earn one free immediate successor). + EXPECT_FALSE(ReferenceChainsTestAccessor::shouldRunPass(2)); + EXPECT_EQ(SearchState::COMPLETED, tracker->searchState()); + + // Well past the drain point - the debt has cleared, restart #2 proceeds. + EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(1ULL + 200000000000ULL)); + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); + ASSERT_TRUE(tracker->runPass(&mock_jvmti, nullptr)); + ASSERT_EQ(SearchState::COMPLETED, tracker->searchState()); + + tracker->stop(); +} + +// hasLeakSignal()'s OOM_URGENT_THRESHOLD_S fast path (referenceChains.h/.cpp): a heap-wide leak +// growing fast enough to project exhaustion sooner than the threshold must start a search +// immediately, without waiting for any klass to clear selectLeakCandidates()'s own per-klass +// ring-fill/hysteresis gate - this is the aggressive-leak gap +// GenerationsEnabledButNoCandidateBlocksFirstSearch above documents for the non-urgent case. +TEST_F(SearchRestartTest, UrgentOOMProjectionBypassesCandidateGate) { + LivenessTracker::instance()->setGcGenerationsForTest(true); + constexpr u64 SEC_NS = 1000000000ULL; + constexpr u64 MiB = 1ULL << 20; + // Same worked example as livenessTracker_ut.cpp's + // SecondsToOOMTest.RisingFloorProjectsExpectedSeconds: 700MiB rise over 7s against a 2800MiB + // max heap projects to 10s - comfortably under OOM_URGENT_THRESHOLD_S (5 minutes). + LivenessTracker::instance()->setMaxHeapBytesForTest((jlong)(2800 * MiB)); + for (int i = 0; i < 10; i++) { + LivenessTracker::instance()->heapFloorRecordForTest( + 1000 * MiB + (u64)i * 100 * MiB, (u64)i * SEC_NS); + } + + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(1)); + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); + + tracker->stop(); +} + +// Durability re-verification (correctness hardening). + diff --git a/ddprof-lib/src/test/cpp/referenceChainsTraversalTests.inc b/ddprof-lib/src/test/cpp/referenceChainsTraversalTests.inc new file mode 100644 index 000000000..361944bba --- /dev/null +++ b/ddprof-lib/src/test/cpp/referenceChainsTraversalTests.inc @@ -0,0 +1,606 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +TEST_F(ReferenceChainsBfsTest, RollingResumePopsProcessedEntriesOnTruncatedBatch) { + Arguments args; + // budget=4: small enough that expand truncates mid-batch after admitting a few children. + ASSERT_FALSE(args.parse( + "referencechains=true:hops=5000:budget=4:firstpassbudget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + // A static-field root: classNode -> listNode (the leaking collection). + int classNode = addNode(); + int listNode = addNode(); + + // A chain of 20 children hanging off listNode. With budget=4, the callback admits 4 children + // then returns JVMTI_VISIT_ABORT (BUDGET_EXHAUSTED), truncating mid-batch. + constexpr int kChainLen = 20; + std::vector chainNodes(kChainLen); + for (int i = 0; i < kChainLen; i++) { + chainNodes[i] = addNode(); + } + + // Distractor roots: 20 independent JNI-global roots, each with one child. + constexpr int kDistractors = 20; + std::vector distractorRoots(kDistractors); + std::vector distractorChildren(kDistractors); + for (int i = 0; i < kDistractors; i++) { + distractorRoots[i] = addNode(); + distractorChildren[i] = addNode(); + } + + // addClass() must come after all addNode() calls. + addClass((void *)&node_tags[classNode], "Lcom/rc/SmokeTestHolder;"); + + script = { + {JVMTI_HEAP_REFERENCE_STATIC_FIELD, classNode, listNode, -1}, + {JVMTI_HEAP_REFERENCE_FIELD, listNode, chainNodes[0], -1}, + }; + for (int i = 0; i + 1 < kChainLen; i++) { + script.push_back({JVMTI_HEAP_REFERENCE_FIELD, chainNodes[i], chainNodes[i + 1], -1}); + } + for (int i = 0; i < kDistractors; i++) { + script.push_back({JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, distractorRoots[i], -1}); + script.push_back({JVMTI_HEAP_REFERENCE_FIELD, distractorRoots[i], distractorChildren[i], -1}); + } + + // Phase 1: run passes until listNode is admitted via the static-field sweep. + bool truncated = true; + jlong listTag = 0; + for (int i = 0; i < 200 && listTag == 0; i++) { + ASSERT_EQ(SearchState::RUNNING, tracker->searchState()); + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + listTag = tags_ever_assigned[listNode]; + } + ASSERT_NE(0, listTag) << "listNode was never admitted to the frontier"; + + // Phase 2: run passes until listNode is expanded (rolling resume pops it). + for (int i = 0; i < 200; i++) { + ASSERT_EQ(SearchState::RUNNING, tracker->searchState()); + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + FrontierEntry entry{}; + if (frontier->lookup(listTag, &entry) && + entry.state == FrontierEntryState::EXPANDED) { + break; + } + } + FrontierEntry listEntry{}; + ASSERT_TRUE(frontier->lookup(listTag, &listEntry)); + EXPECT_EQ(FrontierEntryState::EXPANDED, listEntry.state) + << "listNode should be EXPANDED after rolling resume popped it"; + + // Verify some chain children were admitted. + int admittedChildren = 0; + for (int i = 0; i < kChainLen; i++) { + if (tags_ever_assigned[chainNodes[i]] != 0) admittedChildren++; + } + EXPECT_GT(admittedChildren, 0) + << "No chain children were admitted — expand never ran"; + + // Phase 3: run more passes until all chain children are admitted. + for (int i = 0; i < 500 && admittedChildren < kChainLen; i++) { + ASSERT_EQ(SearchState::RUNNING, tracker->searchState()); + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + admittedChildren = 0; + for (int j = 0; j < kChainLen; j++) { + if (tags_ever_assigned[chainNodes[j]] != 0) admittedChildren++; + } + } + EXPECT_EQ(kChainLen, admittedChildren) + << "Not all chain children were admitted within bounded passes"; + + tracker->stop(); +} + +// Verify the AIMD adaptive batch_size: with the per-call EMA over the CPU budget, expandFrontier +// should multiplicatively decrease the batch; under the budget it should additively increase toward +// the cap. +TEST_F(ReferenceChainsBfsTest, AdaptiveBatchSizeProportionalToWindow) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + // Adaptive-batch state is zeroed by reset() (SetUp) but zeroed here too for the same reason as + // before: exact per-phase arithmetic below. + ReferenceChainsTestAccessor::setGotwEmaCallNs(0); + ReferenceChainsTestAccessor::setGotwBatchSize(0); + ReferenceChainsTestAccessor::setPassDeadlineNs(0); + + // Seed a frontier root manually (mirrors PollWatchedTargetsTest's seeding style): node carries + // frontier tag 1, pending expansion has exactly that tag. + int rootNode = addNode(); + int childNode = addNode(); + node_tags[rootNode] = 1; + ASSERT_TRUE(tracker->frontierTable()->insert( + 1, 0, 1, 0, FrontierEntryState::EDGE)); + ReferenceChainsTestAccessor::pushPendingExpandForTest(1); + int edges = 0; + const u64 budget = ReferenceChainsTestAccessor::gotwCpuBudgetNs(); + + // --- Populate phase: first GetObjectsWithTags call. The EMA should be non-zero afterwards, and + // the near-zero mock call time means the window (nominal budget, no deadline) fits ~unbounded + // many calls - the proportion scales the batch all the way to the cap. + ReferenceChainsTestAccessor::expandFrontierForTest(&mock_jvmti, + &mock_jni, &edges); + EXPECT_NE(0u, ReferenceChainsTestAccessor::gotwEmaCallNs()) + << "per-call EMA should be populated after first GetObjectsWithTags"; + EXPECT_EQ(ReferenceChainsTestAccessor::gotwMaxBatch(), + ReferenceChainsTestAccessor::gotwBatchSize()) + << "near-free call should scale the batch to the cap"; + + // --- Shrink phase: EMA at 2x the window with no deadline -> batch halves (512 x 1 / 1.6 after + // the EMA update). + ReferenceChainsTestAccessor::setGotwEmaCallNs(budget * 2); + ReferenceChainsTestAccessor::setGotwBatchSize(512); + ReferenceChainsTestAccessor::pushPendingExpandForTest(1); + ReferenceChainsTestAccessor::expandFrontierForTest(&mock_jvmti, + &mock_jni, &edges); + // EMA after the call: 2x window x 0.8 + mock elapsed/5 - slightly above 1.6x window, so the + // exact expectation is computed from the actual EMA the same way the control law does (window = + // nominal budget, no deadline): next = 512 x window / ema. + EXPECT_EQ((size_t)(512ULL * budget / + std::max(ReferenceChainsTestAccessor::gotwEmaCallNs(), + 1ULL)), + ReferenceChainsTestAccessor::gotwBatchSize()) + << "EMA at ~1.6x the window should scale the batch to 512/1.6"; + + // --- Grow phase: EMA at half the window -> batch scales up 2.5x, i.e. the floor-dominated + // regime GROWS the batch (the whole point of the proportional law - the old AIMD could not grow + // past a fixed budget even when bigger batches were nearly free). + ReferenceChainsTestAccessor::setGotwBatchSize(64); + ReferenceChainsTestAccessor::setGotwEmaCallNs(budget / 2); + ReferenceChainsTestAccessor::pushPendingExpandForTest(1); + ReferenceChainsTestAccessor::expandFrontierForTest(&mock_jvmti, + &mock_jni, &edges); + // Same computation from the actual post-call EMA (~0.4x window): next = 64 x window / ema. + EXPECT_EQ((size_t)(64ULL * budget / + std::max(ReferenceChainsTestAccessor::gotwEmaCallNs(), + 1ULL)), + ReferenceChainsTestAccessor::gotwBatchSize()) + << "EMA under the window should scale the batch up proportionally"; + + // --- Deadline-window phase: with a live pass deadline the window is the REMAINING time, not + // the nominal budget. + ReferenceChainsTestAccessor::setPassDeadlineNs( + OS::nanotime() + budget * 10); + ReferenceChainsTestAccessor::setGotwBatchSize(64); + ReferenceChainsTestAccessor::setGotwEmaCallNs(budget); + ReferenceChainsTestAccessor::pushPendingExpandForTest(1); + ReferenceChainsTestAccessor::expandFrontierForTest(&mock_jvmti, + &mock_jni, &edges); + EXPECT_EQ(ReferenceChainsTestAccessor::gotwMaxBatch(), + ReferenceChainsTestAccessor::gotwBatchSize()) + << "a wide remaining deadline should grow the batch to the cap"; + ReferenceChainsTestAccessor::setPassDeadlineNs(0); + + // --- Admission sanity: expansion still walks the graph. Root -> child edge, one more drive, + // child must be admitted. + script.push_back({JVMTI_HEAP_REFERENCE_FIELD, rootNode, childNode, -1}); + ReferenceChainsTestAccessor::pushPendingExpandForTest(1); + ReferenceChainsTestAccessor::expandFrontierForTest(&mock_jvmti, + &mock_jni, &edges); + EXPECT_NE(0, tags_ever_assigned[childNode]) + << "expandFrontier failed to admit childNode with adaptive batch_size"; + + tracker->stop(); +} + +// gotwWindowNs() backlog-pressure widening, unit level: the pod regime is a remaining pass window +// (~10ms) smaller than the measured per-call floor (~22-40ms at a 242k-entry tag map), against a +// lane 127k deep. +TEST_F(ReferenceChainsBfsTest, GotwWindowWidensOnlyUnderBacklogPressure) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + const u64 budget = ReferenceChainsTestAccessor::gotwCpuBudgetNs(); + const size_t depth = ReferenceChainsTestAccessor::gotwBacklogMinDepth(); + const u64 mult = ReferenceChainsTestAccessor::gotwBacklogWindowMult(); + const u64 floor = budget * 2; // any floor above the nominal window + + // No deadline and no EMA yet: the nominal budget window. + ReferenceChainsTestAccessor::setGotwEmaCallNs(0); + EXPECT_EQ(budget, ReferenceChainsTestAccessor::gotwWindowNs(0, depth)); + + ReferenceChainsTestAccessor::setGotwEmaCallNs(floor); + + // Floor above the remaining window but a SHALLOW lane: no widening - the remaining window + // stands (rotation fast-lane stays cheap). + EXPECT_EQ(1u, ReferenceChainsTestAccessor::gotwWindowNs(1, 1)); + + // Floor above the remaining window and a DEEP lane: widened to EMA x mult, never below the + // remaining window itself. + EXPECT_EQ(floor * mult, + ReferenceChainsTestAccessor::gotwWindowNs(1, depth)); + + // Floor BELOW the remaining window: no widening even at depth - the ordinary proportional law + // already fits the call in the window. + ReferenceChainsTestAccessor::setGotwEmaCallNs(budget / 2); + EXPECT_EQ(budget, + ReferenceChainsTestAccessor::gotwWindowNs(budget, depth)); + ReferenceChainsTestAccessor::setGotwEmaCallNs(floor); + + // Floor above the NOMINAL window (deadline already passed, the exact pod's post-call state) at + // depth: still widened - the floor is paid by the next call regardless, so the batch must + // amortize it. + EXPECT_EQ(floor * mult, + ReferenceChainsTestAccessor::gotwWindowNs(0, depth)); + + tracker->stop(); +} + +// The widened window in action through the real control loop: one GetObjectsWithTags call whose +// floor (simulated by the mock's busy-wait) exceeds both the remaining pass deadline and the +// nominal window, with a backlog deeper than GOTW_BACKLOG_MIN_DEPTH, must GROW the calibrated batch +// (calib x mult, exactly - the window scales with the measured EMA) instead of collapsing it to +// GOTW_MIN_BATCH. +TEST_F(ReferenceChainsBfsTest, AdaptiveBatchGrowsWhenFloorExceedsWindowUnderDeepBacklog) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + // A pass deadline a fraction of the simulated per-call floor: the call overruns it (exactly the + // pod's 10ms window vs 22-40ms floor), so after the call the loop's deadline check stops the + // invocation with ONE control update - deterministic arithmetic for the assertion below. + gotw_delay_ns = ReferenceChainsTestAccessor::gotwCpuBudgetNs(); // 25ms floor + ReferenceChainsTestAccessor::setPassDeadlineNs(OS::nanotime() + 5000000ULL); + ReferenceChainsTestAccessor::setGotwBatchSize(ReferenceChainsTestAccessor::gotwMinBatch()); + ReferenceChainsTestAccessor::setGotwEmaCallNs(0); // seeded by the call below + + // A pending lane deep enough to cross GOTW_BACKLOG_MIN_DEPTH. + const size_t depth = ReferenceChainsTestAccessor::gotwBacklogMinDepth() + 1; + for (size_t i = 0; i < depth; i++) { + ReferenceChainsTestAccessor::pushPendingExpandForTest( + (jlong)(1000000 + i)); + } + + int edges = 0; + ReferenceChainsTestAccessor::expandFrontierForTest(&mock_jvmti, + &mock_jni, &edges); + + // EMA after the call = the busy-wait floor (~25ms). The pass deadline is long past, so the + // window is widened to EMA x GOTW_BACKLOG_WINDOW_MULT, and the control law computes calib x + // window / ema = calib x mult - exactly, because the window is a whole multiple of the same EMA + // it divides by. + EXPECT_EQ(ReferenceChainsTestAccessor::gotwMinBatch() * + ReferenceChainsTestAccessor::gotwBacklogWindowMult(), + ReferenceChainsTestAccessor::gotwBatchSize()) + << "the floor-dominated deep-backlog regime must GROW the batch, " + "not clamp it to GOTW_MIN_BATCH"; + + ReferenceChainsTestAccessor::setPassDeadlineNs(0); + gotw_delay_ns = 0; + tracker->stop(); +} + +// FAIR-SHARE DRAIN persistence: the lane toggle must survive across expandFrontier() invocations. +TEST_F(ReferenceChainsBfsTest, FairShareLaneAlternationPersistsAcrossInvocations) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int rootNode = addNode(); + int otherRoot = addNode(); + // Two live boundary objects: tag 1 in pending, tag 2 in priority. + node_tags[rootNode] = 1; + node_tags[otherRoot] = 2; + ASSERT_TRUE(tracker->frontierTable()->insert( + 1, 0, 1, 0, FrontierEntryState::EDGE)); + ASSERT_TRUE(tracker->frontierTable()->insert( + 2, 0, 1, 0, FrontierEntryState::EDGE)); + ReferenceChainsTestAccessor::pushPendingExpandForTest(1); + ReferenceChainsTestAccessor::pushPriorityExpand(2); + int edges = 0; + + // Mock GetObjectsWithTags calls are ~free, so without a deadline a single expandFrontier() + // invocation would drain BOTH lanes in one loop. + gotw_delay_ns = 1 * 1000 * 1000; // 1ms + ReferenceChainsTestAccessor::setPassDeadlineNs(OS::nanotime() + 200 * 1000); + + // Invocation 1: priority first (the standing preference). + ReferenceChainsTestAccessor::expandFrontierForTest(&mock_jvmti, + &mock_jni, &edges); + EXPECT_EQ(0u, ReferenceChainsTestAccessor::priorityExpandSize()) + << "first invocation should drain the priority lane"; + EXPECT_EQ(1u, ReferenceChainsTestAccessor::pendingExpandSize()) + << "first invocation must leave the pending lane for the next one"; + EXPECT_FALSE(ReferenceChainsTestAccessor::expandLanePreferPriority()); + + // Rotation refills the priority lane; invocation 2 must STILL prefer the pending lane - the + // toggle persists, it is not reset per call. + ReferenceChainsTestAccessor::pushPriorityExpand(2); + ReferenceChainsTestAccessor::setPassDeadlineNs(OS::nanotime() + 200 * 1000); + ReferenceChainsTestAccessor::expandFrontierForTest(&mock_jvmti, + &mock_jni, &edges); + EXPECT_EQ(0u, ReferenceChainsTestAccessor::pendingExpandSize()) + << "second invocation should drain the pending lane"; + EXPECT_EQ(1u, ReferenceChainsTestAccessor::priorityExpandSize()) + << "second invocation must leave the refilled priority lane alone"; + EXPECT_TRUE(ReferenceChainsTestAccessor::expandLanePreferPriority()); + + tracker->stop(); +} +// FANOUT HYGIENE: a _leak_parent_fanout entry whose parent no longer resolves in the frontier +// (pruned: dead object, or a search-restart wipe) can never be re-walked, so +// collectStaleExpandedEntriesForRotation() must erase it during selection rather than skip it +// forever - without the erase, the fanout grows monotonically with corpses (observed live at ~11k +// entries of overwhelmingly-dead old backing arrays), which both bloats the selection scan and +// turns the fanout cursor's lap arithmetic into mostly wasted skips. +TEST_F(ReferenceChainsBfsTest, StaleRotationEvictsDeadFanoutParents) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + constexpr u32 kLeafKlass = 987; + ReferenceChainsTestAccessor::setWatchedLeakKlassIdsForTest({kLeafKlass}); + FrontierTable *frontier = tracker->frontierTable(); + + // Live fanout parent 1 and dead fanout parent 5 (frontier entry pruned). + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 1, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, + /*class_tag=*/42)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 5, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD, /*referrer_klass=*/0, + /*class_tag=*/43)); + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, 1, 10); + ReferenceChainsTestAccessor::trackLeakAccumulation(frontier, kLeafKlass, 5, 20); + ASSERT_EQ(1u, ReferenceChainsTestAccessor::leakParentFanout(1)); + ASSERT_EQ(1u, ReferenceChainsTestAccessor::leakParentFanout(5)); + + frontier->clear(5); // parent 5's object died / search restart pruned it + + std::vector selected = + ReferenceChainsTestAccessor::collectStaleExpandedEntriesForRotation(4); + ASSERT_EQ(1u, selected.size()); + EXPECT_EQ((jlong)1, selected[0]); + EXPECT_EQ(0u, ReferenceChainsTestAccessor::leakParentFanout(5)) + << "dead fanout parent must be erased during selection"; + EXPECT_EQ(1u, ReferenceChainsTestAccessor::leakParentFanout(1)) + << "live fanout parent must survive"; + + tracker->stop(); +} + + +// Leak-tag interception (design A + C): an object pre-tagged with a leak tag (as +// LivenessTracker::tagLeakInstances() would have set on a tracked leaking instance) must be +// admitted by converting the leak tag to a frontier tag, with the leak tag preserved in the +// frontier entry so buildChainEvent() emits it as target_tag - the ReferenceChain <-> +// HeapLiveObject correlation key. +TEST_F(ReferenceChainsBfsTest, LeakTagInterceptionConvertsToFrontierTagAndCorrelates) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + const jlong leak_tag = ReferenceChainsTestAccessor::leakTagBase(); + + int rootNode = addNode(); + int leakChild = addNode(); + int plainChild = addNode(); + // Simulate tagLeakInstances(): the tracked leaking instance already carries a leak tag; the + // sibling does not. + node_tags[leakChild] = leak_tag; + script = { + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, rootNode, -1}, + {JVMTI_HEAP_REFERENCE_FIELD, rootNode, leakChild, -1}, + {JVMTI_HEAP_REFERENCE_FIELD, rootNode, plainChild, -1}, + }; + + // A single pass drains this tiny graph to completion, and a completed search releases all JVMTI + // tags (releaseSearchTags(), "tagsReleased" in runPass's own log) - so read the tags from + // tags_ever_assigned, which records each tag at assignment time and is never reset (see its own + // comment), not from node_tags (which reads 0 after release). + bool truncated = true; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + + // The leak-tagged child's tag was REPLACED by a frontier tag (small positive, outside the leak + // range). + jlong leak_ftag = tags_ever_assigned[leakChild]; + ASSERT_NE(leak_tag, leak_ftag) + << "leak tag was never intercepted - BFS did not reach the object"; + ASSERT_GT(leak_ftag, 0); + EXPECT_LT(leak_ftag, leak_tag) << "frontier tag must be outside leak range"; + + // The frontier entry preserves the leak tag for correlation. + EXPECT_EQ(leak_tag, ReferenceChainsTestAccessor::frontierLeakTag(leak_ftag)); + + // The untagged sibling got an ordinary admit: frontier tag assigned, but no leak tag stored. + jlong plain_ftag = tags_ever_assigned[plainChild]; + ASSERT_GT(plain_ftag, 0); + EXPECT_EQ(0, ReferenceChainsTestAccessor::frontierLeakTag(plain_ftag)); + + // Design C: buildChainEvent() reports the leak tag as target_tag for the leak-tagged instance, + // and the plain frontier tag for the sibling. + ReferenceChainEvent event; + ASSERT_TRUE(ReferenceChainsTestAccessor::buildChainEventForTest( + &mock_jvmti, &mock_jni, leak_ftag, &event)); + EXPECT_EQ((u64)leak_tag, event._target_tag) + << "chain target tag must be the leak tag (correlation key)"; + EXPECT_GE(event._depth, 1u) << "leak child sits behind the root, not at it"; + + ReferenceChainEvent plain_event; + ASSERT_TRUE(ReferenceChainsTestAccessor::buildChainEventForTest( + &mock_jvmti, &mock_jni, plain_ftag, &plain_event)); + EXPECT_EQ((u64)plain_ftag, plain_event._target_tag) + << "untagged instance must keep the frontier tag as target tag"; + + tracker->stop(); +} + +// Candidate-scoped reach, prong 1 (walkCandidateThreadLocals()): a leak held through the leaking +// thread's ThreadLocalMap must be intercepted with its full chain by ONE bounded walk from the +// Thread object, no matter what the ordinary BFS backlog state is - and the walk's gates must keep +// it off the Thread's non-thread-local fields entirely. +TEST_F(ReferenceChainsBfsTest, ThreadWalkDescendsOnlyThreadLocalMapAndInterceptsLeak) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + // The classes descendFromAnchor()'s resolutions look up by FindClass name (see + // registerClassForFindClass' own comment) + the scripted graph's own classes. + void *threadCls = (void *)0x3001, *tlmapCls = (void *)0x3002, + *loaderCls = (void *)0x3003, *holderCls = (void *)0x3004, + *chunkCls = (void *)0x3005; + int tlmapIdx = + registerClassForFindClass(tlmapCls, + "java/lang/ThreadLocal$ThreadLocalMap", + "Ljava/lang/ThreadLocal$ThreadLocalMap;"); + int loaderIdx = + registerClassForFindClass(loaderCls, "java/lang/ClassLoader", + "Ljava/lang/ClassLoader;"); + registerClassForFindClass(threadCls, "java/lang/Thread", + "Ljava/lang/Thread;"); + int holder = addClass(holderCls, "Lcom/rc/descendwalk/Holder;"); + int chunk = addClass(chunkCls, "Lcom/rc/descendwalk/LeakChunk;"); + thread_class = threadCls; + ReferenceChainsTestAccessor::resolveLoadedClasses(&mock_jvmti, &mock_jni); + + int threadNode = addNode(); + int threadNode2 = addNode(); // second candidate thread: fresh-admission path + int tlmapNode = addNode(); + int loaderNode = addNode(); // Thread's contextClassLoader: anchor gate + int loaderNode2 = addNode(); // a ClassLoader below the gate: no-descend + int holderNode = addNode(); + int leakChunk = addNode(); + + const jlong leak_tag = ReferenceChainsTestAccessor::leakTagBase(); + node_tags[leakChunk] = leak_tag; + + // Topological order (mock_FollowReferences replays edges in script order, expanding only refs + // the production callback said to descend into). + script = { + {JVMTI_HEAP_REFERENCE_FIELD, threadNode, tlmapNode, + /*class_idx=*/-1}, + {JVMTI_HEAP_REFERENCE_FIELD, threadNode, loaderNode, + /*class_idx=*/-1}, + {JVMTI_HEAP_REFERENCE_FIELD, tlmapNode, holderNode, holder}, + {JVMTI_HEAP_REFERENCE_FIELD, tlmapNode, loaderNode2, + /*class_idx=*/-1}, + {JVMTI_HEAP_REFERENCE_FIELD, holderNode, leakChunk, chunk}, + }; + // The anchor gate compares the REFEREE's class tag, so the thread edges' class_idx values + // matter: the tlmap edge carries ThreadLocalMap's tag, and the loader edges ClassLoader's. + script[0].class_idx = tlmapIdx; + script[1].class_idx = loaderIdx; + script[3].class_idx = loaderIdx; + + // The first thread walks the REUSE path: its Thread object is already admitted (root-attached + // THREAD entry + JVMTI tag) exactly as it is in production after the first walk pass or root + // enumeration. + FrontierTable *frontier = tracker->frontierTable(); + jlong anchor_tag = + tracker->tagObject(&mock_jvmti, + reinterpret_cast(&node_tags[threadNode])); + ASSERT_GT(anchor_tag, 0); + node_tags[threadNode] = anchor_tag; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, anchor_tag, 0, 0, FrontierEntryState::FRONTIER, + (u8)JVMTI_HEAP_REFERENCE_THREAD)); + + // Two candidate slots, two qualifying tids: tid 777's thread is pre-anchored (reuse path), tid + // 778's is untagged (fresh-admission path - GetObjectClass + tagObject + insert, the path a + // never-walked thread takes in production). + jint tids0[] = {777}; + jint tids1[] = {778}; + ReferenceChainsTestAccessor::seedCandidateSlotForTest( + /*slot=*/0, /*klass_id=*/6, tids0, 1); + ReferenceChainsTestAccessor::seedCandidateSlotForTest( + /*slot=*/1, /*klass_id=*/6, tids1, 1); + tracker->registerThreadObject( + &mock_jni, 777, reinterpret_cast(&node_tags[threadNode])); + tracker->registerThreadObject( + &mock_jni, 778, reinterpret_cast(&node_tags[threadNode2])); + + int edges = 0; + ReferenceChainsTestAccessor::walkCandidateThreadLocalsForTest( + &mock_jvmti, &mock_jni, 1000, &edges); + + // The ThreadLocalMap-held chain was admitted end-to-end and the leak-tagged chunk was + // intercepted (tag replaced by a frontier tag, leak tag preserved for correlation). + jlong thread_ftag = anchor_tag; + jlong tlmap_ftag = tags_ever_assigned[tlmapNode]; + ASSERT_GT(tlmap_ftag, 0) << "anchor gate did not descend into ThreadLocalMap"; + jlong holder_ftag = tags_ever_assigned[holderNode]; + ASSERT_GT(holder_ftag, 0) << "walk did not descend below ThreadLocalMap"; + jlong chunk_ftag = tags_ever_assigned[leakChunk]; + ASSERT_NE(chunk_ftag, leak_tag) + << "leak-tagged chunk under the ThreadLocalMap was never intercepted"; + ASSERT_GT(chunk_ftag, 0); + EXPECT_EQ(leak_tag, ReferenceChainsTestAccessor::frontierLeakTag(chunk_ftag)); + + // Chain shape: Thread (root-attached, THREAD root kind) -> ThreadLocalMap -> holder -> chunk. + FrontierEntry thread_entry{}; + ASSERT_TRUE(frontier->lookup(thread_ftag, &thread_entry)); + EXPECT_EQ(0, thread_entry.parent_tag); + EXPECT_EQ((u8)JVMTI_HEAP_REFERENCE_THREAD, thread_entry.root_kind); + FrontierEntry chunk_entry{}; + ASSERT_TRUE(frontier->lookup(chunk_ftag, &chunk_entry)); + EXPECT_EQ(holder_ftag, chunk_entry.parent_tag); + EXPECT_EQ(3u, chunk_entry.depth); + + // The gates kept the walk off the metadata branches: neither the Thread's own + // contextClassLoader edge (anchor gate) nor a ClassLoader below ThreadLocalMap (no-descend + // gate) was admitted. + EXPECT_EQ(0, tags_ever_assigned[loaderNode]) + << "anchor gate must not admit the Thread's non-ThreadLocalMap fields"; + EXPECT_EQ(0, tags_ever_assigned[loaderNode2]) + << "no-descend gate must not admit fat-metadata classes below the anchor"; + + // The second thread took the fresh-admission path (no prior tag/entry): its Thread object was + // admitted root-attached with the THREAD root kind. + jlong thread2_ftag = ReferenceChainsTestAccessor::getTagForTest( + &mock_jvmti, reinterpret_cast(&node_tags[threadNode2])); + ASSERT_GT(thread2_ftag, 0) << "fresh thread anchor was never admitted"; + FrontierEntry thread2_entry{}; + ASSERT_TRUE(frontier->lookup(thread2_ftag, &thread2_entry)); + EXPECT_EQ(0, thread2_entry.parent_tag); + EXPECT_EQ((u8)JVMTI_HEAP_REFERENCE_THREAD, thread2_entry.root_kind); + + tracker->stop(); +} + +// unregisterThreadObject() must defer the global-ref deletion to releaseEndedThreadRefs(): +// walkCandidateThreadLocals() copies the jobject out of _thread_objects under _thread_objects_lock, +// releases the lock, and can still be using it as a FollowReferences anchor when a concurrent +// ThreadEnd erases the entry - deleting there would be JNI use-after-free (see +// _thread_refs_pending_delete's comment). +TEST_F(ReferenceChainsBfsTest, ThreadRefUnregisterDefersGlobalRefDeletion) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int threadNode = addNode(); + tracker->registerThreadObject( + &mock_jni, 555, reinterpret_cast(&node_tags[threadNode])); + tracker->unregisterThreadObject(&mock_jni, 555); + // The erasing side only enqueues - no DeleteGlobalRef yet. + EXPECT_EQ(0, global_refs_deleted_); + + // The drain deletes exactly the queued ref, and draining an empty list is a no-op. + tracker->releaseEndedThreadRefs(&mock_jni); + EXPECT_EQ(1, global_refs_deleted_); + tracker->releaseEndedThreadRefs(&mock_jni); + EXPECT_EQ(1, global_refs_deleted_); + + tracker->stop(); +} + +// Candidate-scoped reach, prong 2 (collectStaticFieldAnchorsForRotation()/ +// walkStaticFieldAnchors()): the collector selects exactly the root-attached static-holder entries +// with a wrapping cursor, and the walk reaches a leak held 3-4 hops inside a static collection in +// one bounded call - the shape the one-hop Tier-2 rotation demonstrably cannot reach from an +// un-expanded FRONTIER holder on a rising heap (pod rounds 5-6). diff --git a/ddprof-lib/src/test/cpp/referenceChains_ut.cpp b/ddprof-lib/src/test/cpp/referenceChains_ut.cpp new file mode 100644 index 000000000..a64e28382 --- /dev/null +++ b/ddprof-lib/src/test/cpp/referenceChains_ut.cpp @@ -0,0 +1,54 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "arguments.h" +#include "counters.h" +#include "livenessTracker.h" +#include "os.h" +#include "profiler.h" +#include "rcDebugLevel.h" +#include "referenceChains.h" +#include "vmEntry.h" +#include "../../main/cpp/gtest_crash_handler.h" +#include +#include +#include + +static constexpr char REFERENCE_CHAINS_TEST_NAME[] = "ReferenceChainsTest"; + +class ReferenceChainsGlobalSetup { +public: + ReferenceChainsGlobalSetup() { + installGtestCrashHandler(); + } + ~ReferenceChainsGlobalSetup() { + restoreDefaultSignalHandlers(); + } +}; + +static ReferenceChainsGlobalSetup global_setup; + +// Enable diagnostics for tests. +[[maybe_unused]] static const bool kRcDebugLevelPinnedForTests = + setenv("DD_PROFILING_REFERENCE_CHAINS_DEBUG", "2", 1) == 0; + +#include "referenceChainsCoreTests.inc" +#include "referenceChainsBfsTests.inc" +#include "referenceChainsPodTests.inc" +#include "referenceChainsTrackerTests.inc" +#include "referenceChainsRotationTests.inc" +#include "referenceChainsTraversalTests.inc" +#include "referenceChainsAnchorTests.inc" +#include "referenceChainsEventTests.inc"