diff --git a/main.go b/main.go index 2e5b43f6..f2f66e59 100644 --- a/main.go +++ b/main.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "path/filepath" + "strconv" "strings" "sync" "time" @@ -187,6 +188,9 @@ func run() error { if collector := metricsRegistry.DispatchCollector(); collector != nil { collector.Start(runCtx, dispatchMonitorSources(runtimes), raftMetricsObserveInterval) } + if collector := metricsRegistry.PebbleCollector(); collector != nil { + collector.Start(runCtx, pebbleMonitorSources(runtimes), raftMetricsObserveInterval) + } compactor := kv.NewFSMCompactor( fsmCompactionRuntimes(runtimes), kv.WithFSMCompactorActiveTimestampTracker(readTracker), @@ -447,6 +451,31 @@ func raftMonitorRuntimes(runtimes []*raftGroupRuntime) []monitoring.RaftRuntime return out } +// pebbleMonitorSources extracts the MVCC stores that expose +// *pebble.DB.Metrics() so monitoring can poll LSM internals (L0 +// sublevels, compaction debt, memtable, block cache) for the +// elastickv_pebble_* metrics family. Stores that do not satisfy the +// interface (non-Pebble backends, if any are added later) are skipped +// silently. +func pebbleMonitorSources(runtimes []*raftGroupRuntime) []monitoring.PebbleSource { + out := make([]monitoring.PebbleSource, 0, len(runtimes)) + for _, runtime := range runtimes { + if runtime == nil || runtime.store == nil { + continue + } + src, ok := runtime.store.(monitoring.PebbleMetricsSource) + if !ok { + continue + } + out = append(out, monitoring.PebbleSource{ + GroupID: runtime.spec.id, + GroupIDStr: strconv.FormatUint(runtime.spec.id, 10), + Source: src, + }) + } + return out +} + // dispatchMonitorSources extracts the raft engines that expose etcd // dispatch counters so monitoring can poll them for the hot-path // dashboard. Engines that do not satisfy the interface (hashicorp diff --git a/monitoring/pebble.go b/monitoring/pebble.go new file mode 100644 index 00000000..7f4e3be7 --- /dev/null +++ b/monitoring/pebble.go @@ -0,0 +1,279 @@ +package monitoring + +import ( + "context" + "sync" + "time" + + "github.com/cockroachdb/pebble/v2" + "github.com/prometheus/client_golang/prometheus" +) + +// Pebble LSM metrics. These mirror the most operationally useful +// fields from *pebble.DB.Metrics() so operators can graph/alert on +// write-stall signals (L0 sublevels, compaction debt) and capacity +// trends (memtable, block cache) without importing Pebble from every +// dashboard. +// +// The point-in-time fields (Sublevels, NumFiles, EstimatedDebt, +// MemTable.*, NumInProgress, BlockCache.Size) are exposed as +// Prometheus GAUGES — each poll overwrites the previous value. +// Monotonic fields (Compact.Count, BlockCache.Hits/Misses) are exposed +// as COUNTERS; the collector emits only the positive delta against the +// last snapshot so a store reset (Restore/swap) does not produce +// negative values. +// +// Name convention: elastickv_pebble_* to keep a consistent node_id / +// node_address label prefix with the rest of the registry. + +const defaultPebblePollInterval = 5 * time.Second + +// PebbleMetrics owns the Prometheus vectors for Pebble LSM internals. +// One instance per registry; shared by all groups (labelled by group +// ID + level where relevant). +type PebbleMetrics struct { + // L0 pressure: incident signals. + l0Sublevels *prometheus.GaugeVec + l0NumFiles *prometheus.GaugeVec + + // Compaction queue depth / debt. + compactEstimatedDebt *prometheus.GaugeVec + compactInProgress *prometheus.GaugeVec + compactCountTotal *prometheus.CounterVec + + // Memtable footprint. + memtableCount *prometheus.GaugeVec + memtableSizeBytes *prometheus.GaugeVec + memtableZombieCount *prometheus.GaugeVec + + // Block cache. + blockCacheSizeBytes *prometheus.GaugeVec + blockCacheHitsTotal *prometheus.CounterVec + blockCacheMissesTotal *prometheus.CounterVec +} + +func newPebbleMetrics(registerer prometheus.Registerer) *PebbleMetrics { + m := &PebbleMetrics{ + l0Sublevels: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "elastickv_pebble_l0_sublevels", + Help: "Current L0 sublevel count reported by Pebble. Climbing sublevels are the canonical precursor to a write stall; alert when this exceeds the L0CompactionThreshold for a sustained period.", + }, + []string{"group"}, + ), + l0NumFiles: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "elastickv_pebble_l0_num_files", + Help: "Current number of sstables in L0 reported by Pebble. Paired with elastickv_pebble_l0_sublevels to diagnose L0 pressure.", + }, + []string{"group"}, + ), + compactEstimatedDebt: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "elastickv_pebble_compact_estimated_debt_bytes", + Help: "Estimated number of bytes Pebble still needs to compact for the LSM to reach a stable state. Growth indicates compactions are falling behind ingest.", + }, + []string{"group"}, + ), + compactInProgress: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "elastickv_pebble_compact_in_progress", + Help: "Number of compactions currently in progress.", + }, + []string{"group"}, + ), + compactCountTotal: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "elastickv_pebble_compact_count_total", + Help: "Cumulative number of compactions completed by Pebble since the process started.", + }, + []string{"group"}, + ), + memtableCount: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "elastickv_pebble_memtable_count", + Help: "Current count of memtables (active + queued for flush).", + }, + []string{"group"}, + ), + memtableSizeBytes: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "elastickv_pebble_memtable_size_bytes", + Help: "Current bytes allocated by memtables and large flushable batches.", + }, + []string{"group"}, + ), + memtableZombieCount: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "elastickv_pebble_memtable_zombie_count", + Help: "Current count of zombie memtables (no longer referenced by the DB but pinned by open iterators).", + }, + []string{"group"}, + ), + blockCacheSizeBytes: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "elastickv_pebble_block_cache_size_bytes", + Help: "Current bytes in use by Pebble's block cache.", + }, + []string{"group"}, + ), + blockCacheHitsTotal: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "elastickv_pebble_block_cache_hits_total", + Help: "Cumulative block cache hits reported by Pebble.", + }, + []string{"group"}, + ), + blockCacheMissesTotal: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "elastickv_pebble_block_cache_misses_total", + Help: "Cumulative block cache misses reported by Pebble.", + }, + []string{"group"}, + ), + } + + registerer.MustRegister( + m.l0Sublevels, + m.l0NumFiles, + m.compactEstimatedDebt, + m.compactInProgress, + m.compactCountTotal, + m.memtableCount, + m.memtableSizeBytes, + m.memtableZombieCount, + m.blockCacheSizeBytes, + m.blockCacheHitsTotal, + m.blockCacheMissesTotal, + ) + return m +} + +// PebbleMetricsSource abstracts the per-group access to a Pebble DB's +// Metrics(). The concrete *store pebbleStore satisfies this via its +// Metrics() accessor. Returning nil (e.g. store closed mid-restore) is +// allowed; the collector will skip that group for the tick. +type PebbleMetricsSource interface { + Metrics() *pebble.Metrics +} + +// PebbleSource binds a raft group ID to its Pebble store. Multiple +// groups can be polled by a single collector on a sharded node. +// GroupIDStr is the pre-formatted decimal form of GroupID used as the +// "group" Prometheus label; pre-computing it avoids a per-tick +// strconv.FormatUint allocation in observeOnce. +type PebbleSource struct { + GroupID uint64 + GroupIDStr string + Source PebbleMetricsSource +} + +// PebbleCollector polls each registered Pebble store on a fixed +// interval and mirrors the snapshot into the Prometheus vectors. +// Gauges are overwritten; counters advance by the positive delta +// against the previous snapshot. +type PebbleCollector struct { + metrics *PebbleMetrics + + mu sync.Mutex + previous map[uint64]pebbleSnapshot +} + +type pebbleSnapshot struct { + compactCount int64 + blockCacheHits int64 + blockCacheMisses int64 +} + +func newPebbleCollector(metrics *PebbleMetrics) *PebbleCollector { + return &PebbleCollector{ + metrics: metrics, + previous: map[uint64]pebbleSnapshot{}, + } +} + +// Start begins polling sources on interval until ctx is canceled. +// Passing interval <= 0 uses defaultPebblePollInterval (5 s), matching +// the DispatchCollector cadence so operators see consistent refresh +// rates across dashboards. Pebble.Metrics() acquires internal mutexes +// but is not expensive; 5 s gives ample headroom. +func (c *PebbleCollector) Start(ctx context.Context, sources []PebbleSource, interval time.Duration) { + if c == nil || c.metrics == nil || len(sources) == 0 { + return + } + if interval <= 0 { + interval = defaultPebblePollInterval + } + c.observeOnce(sources) + ticker := time.NewTicker(interval) + go func() { + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + c.observeOnce(sources) + } + } + }() +} + +// ObserveOnce is exposed for tests and single-shot callers. +func (c *PebbleCollector) ObserveOnce(sources []PebbleSource) { + c.observeOnce(sources) +} + +func (c *PebbleCollector) observeOnce(sources []PebbleSource) { + if c == nil || c.metrics == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + for _, src := range sources { + if src.Source == nil { + continue + } + snap := src.Source.Metrics() + if snap == nil { + continue + } + group := src.GroupIDStr + + // L0 pressure: gauges, overwritten each tick. + c.metrics.l0Sublevels.WithLabelValues(group).Set(float64(snap.Levels[0].Sublevels)) + c.metrics.l0NumFiles.WithLabelValues(group).Set(float64(snap.Levels[0].TablesCount)) + + // Compaction. + c.metrics.compactEstimatedDebt.WithLabelValues(group).Set(float64(snap.Compact.EstimatedDebt)) + c.metrics.compactInProgress.WithLabelValues(group).Set(float64(snap.Compact.NumInProgress)) + + // Memtable. + c.metrics.memtableCount.WithLabelValues(group).Set(float64(snap.MemTable.Count)) + c.metrics.memtableSizeBytes.WithLabelValues(group).Set(float64(snap.MemTable.Size)) + c.metrics.memtableZombieCount.WithLabelValues(group).Set(float64(snap.MemTable.ZombieCount)) + + // Block cache gauge. + c.metrics.blockCacheSizeBytes.WithLabelValues(group).Set(float64(snap.BlockCache.Size)) + + // Monotonic counters: emit only the positive delta. A smaller + // value means the source was reset (store reopened); rebase + // silently without emitting negative. + prev := c.previous[src.GroupID] + curr := pebbleSnapshot{ + compactCount: snap.Compact.Count, + blockCacheHits: snap.BlockCache.Hits, + blockCacheMisses: snap.BlockCache.Misses, + } + if curr.compactCount > prev.compactCount { + c.metrics.compactCountTotal.WithLabelValues(group).Add(float64(curr.compactCount - prev.compactCount)) + } + if curr.blockCacheHits > prev.blockCacheHits { + c.metrics.blockCacheHitsTotal.WithLabelValues(group).Add(float64(curr.blockCacheHits - prev.blockCacheHits)) + } + if curr.blockCacheMisses > prev.blockCacheMisses { + c.metrics.blockCacheMissesTotal.WithLabelValues(group).Add(float64(curr.blockCacheMisses - prev.blockCacheMisses)) + } + c.previous[src.GroupID] = curr + } +} diff --git a/monitoring/pebble_test.go b/monitoring/pebble_test.go new file mode 100644 index 00000000..93b05a72 --- /dev/null +++ b/monitoring/pebble_test.go @@ -0,0 +1,202 @@ +package monitoring + +import ( + "strings" + "sync" + "testing" + + "github.com/cockroachdb/pebble/v2" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +// fakePebbleSource implements PebbleMetricsSource with canned values +// so tests can exercise the collector without opening a real Pebble +// DB. A nil stored value makes Metrics() return nil (exercising the +// "store closed / mid-swap" skip path). +type fakePebbleSource struct { + mu sync.Mutex + metrics *pebble.Metrics +} + +func (f *fakePebbleSource) set(m *pebble.Metrics) { + f.mu.Lock() + defer f.mu.Unlock() + f.metrics = m +} + +func (f *fakePebbleSource) Metrics() *pebble.Metrics { + f.mu.Lock() + defer f.mu.Unlock() + return f.metrics +} + +// newFakeMetrics builds a *pebble.Metrics populated only with the +// fields the collector reads. Other fields stay at their zero value. +func newFakeMetrics(l0Sub int32, l0Files int64, debt uint64, inProg int64, compactCount int64, + memCount int64, memSize uint64, memZombie int64, + cacheSize int64, hits int64, misses int64, +) *pebble.Metrics { + m := &pebble.Metrics{} + m.Levels[0].Sublevels = l0Sub + m.Levels[0].TablesCount = l0Files + m.Compact.EstimatedDebt = debt + m.Compact.NumInProgress = inProg + m.Compact.Count = compactCount + m.MemTable.Count = memCount + m.MemTable.Size = memSize + m.MemTable.ZombieCount = memZombie + m.BlockCache.Size = cacheSize + m.BlockCache.Hits = hits + m.BlockCache.Misses = misses + return m +} + +func TestPebbleCollectorMirrorsGaugesAndCounters(t *testing.T) { + registry := NewRegistry("n1", "10.0.0.1:50051") + collector := registry.PebbleCollector() + require.NotNil(t, collector) + + src := &fakePebbleSource{} + sources := []PebbleSource{{GroupID: 1, GroupIDStr: "1", Source: src}} + + // Baseline tick: initial counter values establish the delta + // baseline, gauges reflect the snapshot immediately. + src.set(newFakeMetrics( + 2, 7, 1024, 1, 10, + 3, 2048, 1, + 8192, 100, 20, + )) + collector.ObserveOnce(sources) + + // Advance: gauges change, monotonic counters grow. + src.set(newFakeMetrics( + 5, 12, 4096, 2, 15, + 4, 8192, 2, + 16384, 150, 25, + )) + collector.ObserveOnce(sources) + + // Idempotent second pass with the same snapshot must not + // double-count the counters. + collector.ObserveOnce(sources) + + err := testutil.GatherAndCompare( + registry.Gatherer(), + strings.NewReader(` +# HELP elastickv_pebble_block_cache_hits_total Cumulative block cache hits reported by Pebble. +# TYPE elastickv_pebble_block_cache_hits_total counter +elastickv_pebble_block_cache_hits_total{group="1",node_address="10.0.0.1:50051",node_id="n1"} 150 +# HELP elastickv_pebble_block_cache_misses_total Cumulative block cache misses reported by Pebble. +# TYPE elastickv_pebble_block_cache_misses_total counter +elastickv_pebble_block_cache_misses_total{group="1",node_address="10.0.0.1:50051",node_id="n1"} 25 + +# HELP elastickv_pebble_block_cache_size_bytes Current bytes in use by Pebble's block cache. +# TYPE elastickv_pebble_block_cache_size_bytes gauge +elastickv_pebble_block_cache_size_bytes{group="1",node_address="10.0.0.1:50051",node_id="n1"} 16384 +# HELP elastickv_pebble_compact_count_total Cumulative number of compactions completed by Pebble since the process started. +# TYPE elastickv_pebble_compact_count_total counter +elastickv_pebble_compact_count_total{group="1",node_address="10.0.0.1:50051",node_id="n1"} 15 +# HELP elastickv_pebble_compact_estimated_debt_bytes Estimated number of bytes Pebble still needs to compact for the LSM to reach a stable state. Growth indicates compactions are falling behind ingest. +# TYPE elastickv_pebble_compact_estimated_debt_bytes gauge +elastickv_pebble_compact_estimated_debt_bytes{group="1",node_address="10.0.0.1:50051",node_id="n1"} 4096 +# HELP elastickv_pebble_compact_in_progress Number of compactions currently in progress. +# TYPE elastickv_pebble_compact_in_progress gauge +elastickv_pebble_compact_in_progress{group="1",node_address="10.0.0.1:50051",node_id="n1"} 2 +# HELP elastickv_pebble_l0_num_files Current number of sstables in L0 reported by Pebble. Paired with elastickv_pebble_l0_sublevels to diagnose L0 pressure. +# TYPE elastickv_pebble_l0_num_files gauge +elastickv_pebble_l0_num_files{group="1",node_address="10.0.0.1:50051",node_id="n1"} 12 +# HELP elastickv_pebble_l0_sublevels Current L0 sublevel count reported by Pebble. Climbing sublevels are the canonical precursor to a write stall; alert when this exceeds the L0CompactionThreshold for a sustained period. +# TYPE elastickv_pebble_l0_sublevels gauge +elastickv_pebble_l0_sublevels{group="1",node_address="10.0.0.1:50051",node_id="n1"} 5 +# HELP elastickv_pebble_memtable_count Current count of memtables (active + queued for flush). +# TYPE elastickv_pebble_memtable_count gauge +elastickv_pebble_memtable_count{group="1",node_address="10.0.0.1:50051",node_id="n1"} 4 +# HELP elastickv_pebble_memtable_size_bytes Current bytes allocated by memtables and large flushable batches. +# TYPE elastickv_pebble_memtable_size_bytes gauge +elastickv_pebble_memtable_size_bytes{group="1",node_address="10.0.0.1:50051",node_id="n1"} 8192 +# HELP elastickv_pebble_memtable_zombie_count Current count of zombie memtables (no longer referenced by the DB but pinned by open iterators). +# TYPE elastickv_pebble_memtable_zombie_count gauge +elastickv_pebble_memtable_zombie_count{group="1",node_address="10.0.0.1:50051",node_id="n1"} 2 +`), + "elastickv_pebble_l0_sublevels", + "elastickv_pebble_l0_num_files", + "elastickv_pebble_compact_estimated_debt_bytes", + "elastickv_pebble_compact_in_progress", + "elastickv_pebble_compact_count_total", + "elastickv_pebble_memtable_count", + "elastickv_pebble_memtable_size_bytes", + "elastickv_pebble_memtable_zombie_count", + "elastickv_pebble_block_cache_size_bytes", + "elastickv_pebble_block_cache_hits_total", + "elastickv_pebble_block_cache_misses_total", + ) + require.NoError(t, err) +} + +func TestPebbleCollectorHandlesSourceReset(t *testing.T) { + // If the underlying DB is replaced (Restore reopens it) the + // monotonic counters may go DOWN. The collector must not emit + // negative deltas; instead, it rebases silently. + registry := NewRegistry("n1", "10.0.0.1:50051") + collector := registry.PebbleCollector() + require.NotNil(t, collector) + + src := &fakePebbleSource{} + sources := []PebbleSource{{GroupID: 7, GroupIDStr: "7", Source: src}} + + src.set(newFakeMetrics(0, 0, 0, 0, 10, 0, 0, 0, 0, 100, 5)) + collector.ObserveOnce(sources) // baseline: 10 compactions, 100 hits + + src.set(newFakeMetrics(0, 0, 0, 0, 3, 0, 0, 0, 0, 20, 5)) // simulated reset + collector.ObserveOnce(sources) + + src.set(newFakeMetrics(0, 0, 0, 0, 5, 0, 0, 0, 0, 30, 5)) // +2 compactions, +10 hits from post-reset baseline + collector.ObserveOnce(sources) + + // Expected: baseline (10 / 100) + 0 + post-reset delta (2 / 10). + err := testutil.GatherAndCompare( + registry.Gatherer(), + strings.NewReader(` +# HELP elastickv_pebble_block_cache_hits_total Cumulative block cache hits reported by Pebble. +# TYPE elastickv_pebble_block_cache_hits_total counter +elastickv_pebble_block_cache_hits_total{group="7",node_address="10.0.0.1:50051",node_id="n1"} 110 +# HELP elastickv_pebble_compact_count_total Cumulative number of compactions completed by Pebble since the process started. +# TYPE elastickv_pebble_compact_count_total counter +elastickv_pebble_compact_count_total{group="7",node_address="10.0.0.1:50051",node_id="n1"} 12 +`), + "elastickv_pebble_compact_count_total", + "elastickv_pebble_block_cache_hits_total", + ) + require.NoError(t, err) +} + +func TestPebbleCollectorSkipsNilSnapshot(t *testing.T) { + // A source that returns nil (store closed mid-restore) must not + // panic and must not populate any series for that group. + registry := NewRegistry("n1", "10.0.0.1:50051") + collector := registry.PebbleCollector() + require.NotNil(t, collector) + + src := &fakePebbleSource{} + sources := []PebbleSource{{GroupID: 9, GroupIDStr: "9", Source: src}} + + // Both nil-source and nil-snapshot should be safe. + require.NotPanics(t, func() { collector.ObserveOnce(sources) }) + require.NotPanics(t, func() { collector.ObserveOnce([]PebbleSource{{GroupID: 1, GroupIDStr: "1", Source: nil}}) }) + + // No gauges or counters should exist yet. + require.Equal(t, 0, testutil.CollectAndCount(registry.pebble.l0Sublevels)) + require.Equal(t, 0, testutil.CollectAndCount(registry.pebble.compactCountTotal)) +} + +func TestPebbleCollectorZeroRegistryIsSafe(t *testing.T) { + // Code paths that bypass the registry (tests, bootstrap helpers) + // must tolerate a nil collector / empty sources without panicking. + var c *PebbleCollector + require.NotPanics(t, func() { c.ObserveOnce(nil) }) + + registry := NewRegistry("n1", "10.0.0.1:50051") + collector := registry.PebbleCollector() + require.NotPanics(t, func() { collector.ObserveOnce(nil) }) +} diff --git a/monitoring/registry.go b/monitoring/registry.go index 2145f979..3c8ed09e 100644 --- a/monitoring/registry.go +++ b/monitoring/registry.go @@ -19,6 +19,7 @@ type Registry struct { raft *RaftMetrics lua *LuaMetrics hotPath *HotPathMetrics + pebble *PebbleMetrics } // NewRegistry builds a registry with constant labels that identify the local node. @@ -39,6 +40,7 @@ func NewRegistry(nodeID string, nodeAddress string) *Registry { r.raft = newRaftMetrics(registerer) r.lua = newLuaMetrics(registerer) r.hotPath = newHotPathMetrics(registerer) + r.pebble = newPebbleMetrics(registerer) return r } @@ -131,3 +133,15 @@ func (r *Registry) DispatchCollector() *DispatchCollector { } return newDispatchCollector(r.hotPath) } + +// PebbleCollector returns a collector that polls each Pebble store's +// Metrics() snapshot and mirrors the operationally useful fields +// (L0 sublevels, compaction debt, memtable, block cache) into +// Prometheus. Start it with the node's Pebble sources after the +// stores have been opened. +func (r *Registry) PebbleCollector() *PebbleCollector { + if r == nil || r.pebble == nil { + return nil + } + return newPebbleCollector(r.pebble) +} diff --git a/store/lsm_store.go b/store/lsm_store.go index c593c19d..4572d980 100644 --- a/store/lsm_store.go +++ b/store/lsm_store.go @@ -1790,3 +1790,25 @@ func (s *pebbleStore) Close() error { defer s.dbMu.Unlock() return errors.WithStack(s.db.Close()) } + +// Metrics returns a snapshot of the underlying Pebble DB's operational +// metrics (LSM shape, compaction debt, memtable, block cache). The +// return value is a freshly allocated *pebble.Metrics owned by the +// caller. +// +// Returns nil only before the first Open has installed a DB or after a +// failed Open left s.db unset; callers during an in-flight Restore block +// on dbMu (which Restore holds exclusively) rather than observing nil, +// and Close() does not clear s.db. Callers must still handle nil for the +// pre-Open case. +// +// Safe for concurrent use: takes the dbMu read lock to protect against +// Restore swapping the DB pointer. +func (s *pebbleStore) Metrics() *pebble.Metrics { + s.dbMu.RLock() + defer s.dbMu.RUnlock() + if s.db == nil { + return nil + } + return s.db.Metrics() +}