diff --git a/sei-db/state_db/sc/flatkv/api.go b/sei-db/state_db/sc/flatkv/api.go index 3dd3b1fe04..634243a34f 100644 --- a/sei-db/state_db/sc/flatkv/api.go +++ b/sei-db/state_db/sc/flatkv/api.go @@ -108,11 +108,14 @@ type Store interface { // CommittedRootHash returns the 32-byte checksum of the last committed LtHash. CommittedRootHash() []byte - // HashCategories returns the hash logger category names this store reports (the global root plus one - // per data DB). The set is fixed. The caller registers these on the logger. + // HashCategories returns the hash logger category names this store reports: the global root, one per + // data DB, and one per (data DB, module) pair currently tracked for that DB. The per-DB set is fixed, + // but the per-module set is dynamic — it grows as new modules first write into a DB — so callers must + // recompute this every block to detect changes and keep the logger's registered set in sync. HashCategories() []string - // RecordHashes reports this store's hashes (root + per-DB) for blockNumber. Call right after Commit. + // RecordHashes reports this store's hashes (root + per-DB + per-module) for blockNumber. Call right + // after Commit. RecordHashes(hl hashlog.HashLogger, blockNumber uint64) error // Version returns the latest committed version. diff --git a/sei-db/state_db/sc/flatkv/hashlog.go b/sei-db/state_db/sc/flatkv/hashlog.go index 63a7ed084a..30ab4b4dcc 100644 --- a/sei-db/state_db/sc/flatkv/hashlog.go +++ b/sei-db/state_db/sc/flatkv/hashlog.go @@ -3,46 +3,103 @@ package flatkv import ( "fmt" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashlog" ) // Hash logger category names owned by the flatKV backend. flatKVDBHashPrefix is joined with a data DB -// directory name (e.g. "flatKV/db/account"). The metadata DB is intentionally excluded — it holds only -// watermarks, not state. +// directory name for that DB's per-DB LtHash (e.g. "flatKV/db/account"), or additionally with +// flatKVModHashPrefix and a module name for a per-module LtHash within that DB (e.g. "flatKV/db/account/mod/evm"). +// The metadata DB is intentionally excluded — it holds no state. const ( - FlatKVRootHashType = "flatKV/root" - flatKVDBHashPrefix = "flatKV/db/" + FlatKVRootHashType = "flatKV/root" + flatKVDBHashPrefix = "flatKV/db/" + flatKVModHashPrefix = "mod" ) -// HashCategories returns the hash logger categories this store reports: the global flatKV root plus one -// per data DB. The set is fixed (the data DBs never change), so callers can use it to detect when the -// overall logged category set has changed. +// dbHashCategory returns the hash logger category for a single data DB's per-DB LtHash, e.g. +// dbHashCategory("account") == "flatKV/db/account". +func dbHashCategory(dir string) string { + return flatKVDBHashPrefix + dir +} + +// moduleHashCategory returns the hash logger category for a single module's per-module LtHash within a +// data DB, e.g. moduleHashCategory("account", "evm") == "flatKV/db/account/mod/evm". A module that spans +// several DBs (e.g. "evm" touches account/code/storage) reports one category per DB it actually has an +// entry in, since ModuleLtHashes is keyed per-DB, not aggregated across DBs. +func moduleHashCategory(dir, module string) string { + return flatKVDBHashPrefix + dir + "/" + flatKVModHashPrefix + "/" + module +} + +// HashCategories returns the hash logger categories this store reports: the global flatKV root, one per +// data DB, and one per (data DB, module) pair currently tracked in that DB's LocalMeta.ModuleLtHashes. +// The per-DB set is fixed (the data DBs never change), but the per-module set is dynamic — it grows as +// new modules first write into a DB — so callers must recompute this every block to detect changes +// (handled upstream by reopening/rotating the logger when the set changes). func (s *CommitStore) HashCategories() []string { categories := make([]string, 0, len(dataDBDirs)+1) categories = append(categories, FlatKVRootHashType) for _, dir := range dataDBDirs { - categories = append(categories, flatKVDBHashPrefix+dir) + categories = append(categories, dbHashCategory(dir)) + if meta := s.localMeta[dir]; meta != nil { + for module := range meta.ModuleLtHashes { + categories = append(categories, moduleHashCategory(dir, module)) + } + } } return categories } -// RecordHashes reports this store's hashes for blockNumber: the committed global root and each data DB's -// committed per-DB LtHash checksum. Intended to be called right after Commit, when localMeta holds the -// just-committed per-DB hashes and CommittedRootHash reflects the same version. +// RecordHashes reports this store's hashes for blockNumber: the committed global root, each data DB's +// committed per-DB LtHash checksum, and each (data DB, module) pair's committed per-module LtHash +// checksum. Intended to be called right after Commit, when localMeta holds the just-committed per-DB and +// per-module hashes and CommittedRootHash reflects the same version. func (s *CommitStore) RecordHashes(hl hashlog.HashLogger, blockNumber uint64) error { if err := hl.ReportHash(blockNumber, FlatKVRootHashType, s.CommittedRootHash()); err != nil { return fmt.Errorf("failed to report flatkv root hash: %w", err) } for _, dir := range dataDBDirs { - var hash []byte - if meta := s.localMeta[dir]; meta != nil && meta.LtHash != nil { - checksum := meta.LtHash.Checksum() - hash = checksum[:] + meta := s.localMeta[dir] + + var dbHash *lthash.LtHash + if meta != nil { + dbHash = meta.LtHash } - category := flatKVDBHashPrefix + dir - if err := hl.ReportHash(blockNumber, category, hash); err != nil { + category := dbHashCategory(dir) + if err := hl.ReportHash(blockNumber, category, checksumBytesOrNil(dbHash)); err != nil { return fmt.Errorf("failed to report flatkv db hash %q: %w", category, err) } + + if err := reportModuleHashes(hl, blockNumber, dir, meta); err != nil { + return err + } } return nil } + +// reportModuleHashes reports the checksum of every module's per-module LtHash tracked within one data DB +// (meta.ModuleLtHashes). A nil meta (DB never committed) reports nothing. +func reportModuleHashes(hl hashlog.HashLogger, blockNumber uint64, dir string, meta *ktype.LocalMeta) error { + if meta == nil { + return nil + } + for module, moduleHash := range meta.ModuleLtHashes { + category := moduleHashCategory(dir, module) + if err := hl.ReportHash(blockNumber, category, checksumBytesOrNil(moduleHash)); err != nil { + return fmt.Errorf("failed to report flatkv module hash %q: %w", category, err) + } + } + return nil +} + +// checksumBytesOrNil returns h's checksum as a byte slice, or nil if h is nil (nothing committed yet). +// Mirrors checksumOrNil (verify.go), which renders the same "nil-tolerant checksum" for error messages +// instead of a []byte. +func checksumBytesOrNil(h *lthash.LtHash) []byte { + if h == nil { + return nil + } + checksum := h.Checksum() + return checksum[:] +} diff --git a/sei-db/state_db/sc/flatkv/hashlog_test.go b/sei-db/state_db/sc/flatkv/hashlog_test.go index 4d12f78f24..293c8c44df 100644 --- a/sei-db/state_db/sc/flatkv/hashlog_test.go +++ b/sei-db/state_db/sc/flatkv/hashlog_test.go @@ -50,20 +50,23 @@ func TestFlatKVHashReporting(t *testing.T) { _, err := s.Commit() require.NoError(t, err) - // Categories: the global root plus one per data DB (metadata DB excluded). - require.Equal(t, []string{ + // Categories: the global root, one per data DB (metadata DB excluded), plus one per (DB, module) + // pair that actually has an entry. Only the storage write above produced a module hash, and only in + // storageDB, so "evm" only shows up there. + require.ElementsMatch(t, []string{ "flatKV/root", "flatKV/db/account", "flatKV/db/code", "flatKV/db/storage", "flatKV/db/misc", + "flatKV/db/storage/mod/evm", }, s.HashCategories()) logger := newCaptureLogger() for _, category := range s.HashCategories() { require.NoError(t, logger.RegisterHashType(category)) } - require.Len(t, logger.registered, 5) + require.Len(t, logger.registered, 6) require.NoError(t, s.RecordHashes(logger, 1)) @@ -80,10 +83,25 @@ func TestFlatKVHashReporting(t *testing.T) { require.Equal(t, checksum[:], logger.hashes["flatKV/db/"+dir]) } + // The per-module hash reported for storage/evm is the checksum of that module's committed LtHash, + // and it is non-zero (the module actually has data). + evmModuleHash := s.localMeta[storageDBDir].ModuleLtHashes["evm"] + require.NotNil(t, evmModuleHash) + require.False(t, evmModuleHash.IsZero()) + evmChecksum := evmModuleHash.Checksum() + require.Equal(t, evmChecksum[:], logger.hashes["flatKV/db/storage/mod/evm"]) + // Homomorphic invariant: the per-DB LtHashes sum to the committed global LtHash. sum := lthash.New() for _, dir := range dataDBDirs { sum.MixIn(s.localMeta[dir].LtHash) } require.True(t, sum.Equal(s.committedLtHash)) + + // Homomorphic invariant: storageDB's per-module hashes sum to its per-DB root. + moduleSum := lthash.New() + for _, h := range s.localMeta[storageDBDir].ModuleLtHashes { + moduleSum.MixIn(h) + } + require.True(t, moduleSum.Equal(s.localMeta[storageDBDir].LtHash)) } diff --git a/sei-db/state_db/sc/flatkv/metrics.go b/sei-db/state_db/sc/flatkv/metrics.go index f1964763f3..ed2a345920 100644 --- a/sei-db/state_db/sc/flatkv/metrics.go +++ b/sei-db/state_db/sc/flatkv/metrics.go @@ -8,6 +8,8 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash" ) var ( @@ -33,6 +35,8 @@ var ( ImportKVPairs metric.Int64Counter ImportWorkerFlushLatency metric.Float64Histogram FlushLatency metric.Float64Histogram + ModuleKeyCount metric.Int64Gauge + ModuleTotalBytes metric.Int64Gauge }{ OpenLatency: must(flatkvMeter.Float64Histogram( "flatkv_open_latency", @@ -129,6 +133,16 @@ var ( metric.WithDescription("Time taken to flush a FlatKV data DB"), metric.WithUnit("s"), )), + ModuleKeyCount: must(flatkvMeter.Int64Gauge( + "flatkv_module_key_count", + metric.WithDescription("Current number of live keys for a (data DB, module) pair"), + metric.WithUnit("{count}"), + )), + ModuleTotalBytes: must(flatkvMeter.Int64Gauge( + "flatkv_module_total_bytes", + metric.WithDescription("Current total serialized key+value bytes for a (data DB, module) pair"), + metric.WithUnit("By"), + )), } ) @@ -151,6 +165,22 @@ func dbAttr(db string) attribute.KeyValue { return attribute.String("db", db) } +func moduleAttr(module string) attribute.KeyValue { + return attribute.String("module", module) +} + +// recordModuleStats records the current per-module key-count / byte totals for one data DB. Only called +// from CommitStore.recordAllModuleStats, which is itself only reached from Commit's live success path +// (see its doc comment). stats is keyed by module name, mirroring LocalMeta.ModuleStats; a nil/empty map +// is a no-op (fresh DB with no modules yet). +func recordModuleStats(ctx context.Context, db string, stats map[string]lthash.ModuleStats) { + for module, s := range stats { + attrs := metric.WithAttributes(dbAttr(db), moduleAttr(module)) + otelMetrics.ModuleKeyCount.Record(ctx, s.KeyCount, attrs) + otelMetrics.ModuleTotalBytes.Record(ctx, s.Bytes, attrs) + } +} + func recordPendingWrites(ctx context.Context, db string, count int) { otelMetrics.PendingWrites.Record(ctx, int64(count), metric.WithAttributes(dbAttr(db))) } diff --git a/sei-db/state_db/sc/flatkv/store_write.go b/sei-db/state_db/sc/flatkv/store_write.go index eaee86f44c..832e61719b 100644 --- a/sei-db/state_db/sc/flatkv/store_write.go +++ b/sei-db/state_db/sc/flatkv/store_write.go @@ -111,6 +111,7 @@ func (s *CommitStore) Commit() (version int64, err error) { s.phaseTimer.SetPhase("commit_done") otelMetrics.CurrentVersion.Record(s.ctx, version) + s.recordAllModuleStats() logger.Info("FlatKV Commit complete", "version", version, "totalWriteCount", pendingAccount+pendingCode+pendingStorage+pendingMisc, @@ -225,7 +226,12 @@ func (s *CommitStore) commitBatches(version int64) error { } } - // Update in-memory local meta after all commits succeed. + // Update in-memory local meta after all commits succeed. Per-module stats gauges are deliberately + // NOT recorded here: commitBatches is also called by catchup (WAL replay on open, import, rollback, + // and openReadOnly's historical read-only clone), and recording from every replayed entry would + // publish stale/historical values into the process-global gauges, clobbering the live store's real + // values. Only Commit's success path (an actual new block being committed) records via + // recordAllModuleStats. for _, p := range pending { s.localMeta[p.dbDir] = &ktype.LocalMeta{ CommittedVersion: version, @@ -237,6 +243,16 @@ func (s *CommitStore) commitBatches(version int64) error { return nil } +// recordAllModuleStats records the current per-module key-count / byte totals for every data DB. Called +// only from Commit's success path — i.e. only for an actual new block being committed, never for open +// (LoadVersion/catchup), import (FinalizeImport), rollback, or a read-only historical replay — so the +// process-global gauges always reflect the live store, not a replay. +func (s *CommitStore) recordAllModuleStats() { + for _, dir := range dataDBDirs { + recordModuleStats(s.ctx, dir, s.perDBModuleWorkingStats[dir]) + } +} + func prepareBatch[T vtype.VType]( db types.KeyValueDB, writes map[string]T, diff --git a/sei-ibc-go/modules/core/03-connection/types/msgs_test.go b/sei-ibc-go/modules/core/03-connection/types/msgs_test.go index 82f82d91e0..f56da61d2e 100644 --- a/sei-ibc-go/modules/core/03-connection/types/msgs_test.go +++ b/sei-ibc-go/modules/core/03-connection/types/msgs_test.go @@ -55,6 +55,9 @@ func (suite *MsgTestSuite) SetupTest() { scConfig.MemIAVLConfig.SnapshotMinTimeInterval = 0 ssConfig := seidbconfig.StateStoreConfig{} store := storev2rootmulti.NewStore(suite.T().TempDir(), scConfig, ssConfig, nil) + defer func() { + suite.Require().NoError(store.Close()) + }() storeKey := storetypes.NewKVStoreKey("iavlStoreKey") store.MountStoreWithDB(storeKey, storetypes.StoreTypeIAVL, nil) diff --git a/sei-ibc-go/modules/core/23-commitment/types/commitment_test.go b/sei-ibc-go/modules/core/23-commitment/types/commitment_test.go index 1fa20e16c8..69cc3971d7 100644 --- a/sei-ibc-go/modules/core/23-commitment/types/commitment_test.go +++ b/sei-ibc-go/modules/core/23-commitment/types/commitment_test.go @@ -36,6 +36,10 @@ func (suite *MerkleTestSuite) SetupTest() { suite.iavlStore = suite.store.GetCommitKVStore(suite.storeKey) } +func (suite *MerkleTestSuite) TearDownTest() { + suite.Require().NoError(suite.store.Close()) +} + func TestMerkleTestSuite(t *testing.T) { suite.Run(t, new(MerkleTestSuite)) }