Add hashLogger for per module hash and metrics for per module stats

sei-db/state_db/sc/flatkv/api.go

@@ -108,11 +108,14 @@ type Store interface {
 	// CommittedRootHash returns the 32-byte checksum of the last committed LtHash.
 	CommittedRootHash() []byte
 
-	// HashCategories returns the hash logger category names this store reports (the global root plus one
-	// per data DB). The set is fixed. The caller registers these on the logger.
+	// HashCategories returns the hash logger category names this store reports: the global root, one per
+	// data DB, and one per (data DB, module) pair currently tracked for that DB. The per-DB set is fixed,
+	// but the per-module set is dynamic — it grows as new modules first write into a DB — so callers must
+	// recompute this every block to detect changes and keep the logger's registered set in sync.
 	HashCategories() []string
 
-	// RecordHashes reports this store's hashes (root + per-DB) for blockNumber. Call right after Commit.
+	// RecordHashes reports this store's hashes (root + per-DB + per-module) for blockNumber. Call right
+	// after Commit.
 	RecordHashes(hl hashlog.HashLogger, blockNumber uint64) error
 
 	// Version returns the latest committed version.

sei-db/state_db/sc/flatkv/hashlog.go

@@ -3,46 +3,103 @@ package flatkv
 import (
 	"fmt"
 
+	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype"
+	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash"
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashlog"
 )
 
 // Hash logger category names owned by the flatKV backend. flatKVDBHashPrefix is joined with a data DB
-// directory name (e.g. "flatKV/db/account"). The metadata DB is intentionally excluded — it holds only
-// watermarks, not state.
+// directory name for that DB's per-DB LtHash (e.g. "flatKV/db/account"), or additionally with
+// flatKVModHashPrefix and a module name for a per-module LtHash within that DB (e.g. "flatKV/db/account/mod/evm").
+// The metadata DB is intentionally excluded — it holds no state.
 const (
-	FlatKVRootHashType = "flatKV/root"
-	flatKVDBHashPrefix = "flatKV/db/"
+	FlatKVRootHashType  = "flatKV/root"
+	flatKVDBHashPrefix  = "flatKV/db/"
+	flatKVModHashPrefix = "mod"
 )
 
-// HashCategories returns the hash logger categories this store reports: the global flatKV root plus one
-// per data DB. The set is fixed (the data DBs never change), so callers can use it to detect when the
-// overall logged category set has changed.
+// dbHashCategory returns the hash logger category for a single data DB's per-DB LtHash, e.g.
+// dbHashCategory("account") == "flatKV/db/account".
+func dbHashCategory(dir string) string {
+	return flatKVDBHashPrefix + dir
+}
+
+// moduleHashCategory returns the hash logger category for a single module's per-module LtHash within a
+// data DB, e.g. moduleHashCategory("account", "evm") == "flatKV/db/account/mod/evm". A module that spans
+// several DBs (e.g. "evm" touches account/code/storage) reports one category per DB it actually has an
+// entry in, since ModuleLtHashes is keyed per-DB, not aggregated across DBs.
+func moduleHashCategory(dir, module string) string {
+	return flatKVDBHashPrefix + dir + "/" + flatKVModHashPrefix + "/" + module
+}
+
+// HashCategories returns the hash logger categories this store reports: the global flatKV root, one per
+// data DB, and one per (data DB, module) pair currently tracked in that DB's LocalMeta.ModuleLtHashes.
+// The per-DB set is fixed (the data DBs never change), but the per-module set is dynamic — it grows as
+// new modules first write into a DB — so callers must recompute this every block to detect changes
+// (handled upstream by reopening/rotating the logger when the set changes).
 func (s *CommitStore) HashCategories() []string {
 	categories := make([]string, 0, len(dataDBDirs)+1)
 	categories = append(categories, FlatKVRootHashType)
 	for _, dir := range dataDBDirs {
-		categories = append(categories, flatKVDBHashPrefix+dir)
+		categories = append(categories, dbHashCategory(dir))
+		if meta := s.localMeta[dir]; meta != nil {
+			for module := range meta.ModuleLtHashes {
+				categories = append(categories, moduleHashCategory(dir, module))
+			}
+		}
 	}
 	return categories
 }
 
-// RecordHashes reports this store's hashes for blockNumber: the committed global root and each data DB's
-// committed per-DB LtHash checksum. Intended to be called right after Commit, when localMeta holds the
-// just-committed per-DB hashes and CommittedRootHash reflects the same version.
+// RecordHashes reports this store's hashes for blockNumber: the committed global root, each data DB's
+// committed per-DB LtHash checksum, and each (data DB, module) pair's committed per-module LtHash
+// checksum. Intended to be called right after Commit, when localMeta holds the just-committed per-DB and
+// per-module hashes and CommittedRootHash reflects the same version.
 func (s *CommitStore) RecordHashes(hl hashlog.HashLogger, blockNumber uint64) error {
 	if err := hl.ReportHash(blockNumber, FlatKVRootHashType, s.CommittedRootHash()); err != nil {
 		return fmt.Errorf("failed to report flatkv root hash: %w", err)
 	}
 	for _, dir := range dataDBDirs {
-		var hash []byte
-		if meta := s.localMeta[dir]; meta != nil && meta.LtHash != nil {
-			checksum := meta.LtHash.Checksum()
-			hash = checksum[:]
+		meta := s.localMeta[dir]
+
+		var dbHash *lthash.LtHash
+		if meta != nil {
+			dbHash = meta.LtHash
 		}
-		category := flatKVDBHashPrefix + dir
-		if err := hl.ReportHash(blockNumber, category, hash); err != nil {
+		category := dbHashCategory(dir)
+		if err := hl.ReportHash(blockNumber, category, checksumBytesOrNil(dbHash)); err != nil {
 			return fmt.Errorf("failed to report flatkv db hash %q: %w", category, err)
 		}
+
+		if err := reportModuleHashes(hl, blockNumber, dir, meta); err != nil {
+			return err
+		}
 	}
 	return nil
 }
+
+// reportModuleHashes reports the checksum of every module's per-module LtHash tracked within one data DB
+// (meta.ModuleLtHashes). A nil meta (DB never committed) reports nothing.
+func reportModuleHashes(hl hashlog.HashLogger, blockNumber uint64, dir string, meta *ktype.LocalMeta) error {
+	if meta == nil {
+		return nil
+	}
+	for module, moduleHash := range meta.ModuleLtHashes {
+		category := moduleHashCategory(dir, module)
+		if err := hl.ReportHash(blockNumber, category, checksumBytesOrNil(moduleHash)); err != nil {
+			return fmt.Errorf("failed to report flatkv module hash %q: %w", category, err)
+		}
+	}
+	return nil
+}
+
+// checksumBytesOrNil returns h's checksum as a byte slice, or nil if h is nil (nothing committed yet).
+// Mirrors checksumOrNil (verify.go), which renders the same "nil-tolerant checksum" for error messages
+// instead of a []byte.
+func checksumBytesOrNil(h *lthash.LtHash) []byte {
+	if h == nil {
+		return nil
+	}
+	checksum := h.Checksum()
+	return checksum[:]
+}

sei-db/state_db/sc/flatkv/hashlog_test.go

@@ -50,20 +50,23 @@ func TestFlatKVHashReporting(t *testing.T) {
 	_, err := s.Commit()
 	require.NoError(t, err)
 
-	// Categories: the global root plus one per data DB (metadata DB excluded).
-	require.Equal(t, []string{
+	// Categories: the global root, one per data DB (metadata DB excluded), plus one per (DB, module)
+	// pair that actually has an entry. Only the storage write above produced a module hash, and only in
+	// storageDB, so "evm" only shows up there.
+	require.ElementsMatch(t, []string{
 		"flatKV/root",
 		"flatKV/db/account",
 		"flatKV/db/code",
 		"flatKV/db/storage",
 		"flatKV/db/misc",
+		"flatKV/db/storage/mod/evm",
 	}, s.HashCategories())
 
 	logger := newCaptureLogger()
 	for _, category := range s.HashCategories() {
 		require.NoError(t, logger.RegisterHashType(category))
 	}
-	require.Len(t, logger.registered, 5)
+	require.Len(t, logger.registered, 6)
 
 	require.NoError(t, s.RecordHashes(logger, 1))
 
@@ -80,10 +83,25 @@ func TestFlatKVHashReporting(t *testing.T) {
 		require.Equal(t, checksum[:], logger.hashes["flatKV/db/"+dir])
 	}
 
+	// The per-module hash reported for storage/evm is the checksum of that module's committed LtHash,
+	// and it is non-zero (the module actually has data).
+	evmModuleHash := s.localMeta[storageDBDir].ModuleLtHashes["evm"]
+	require.NotNil(t, evmModuleHash)
+	require.False(t, evmModuleHash.IsZero())
+	evmChecksum := evmModuleHash.Checksum()
+	require.Equal(t, evmChecksum[:], logger.hashes["flatKV/db/storage/mod/evm"])
+
 	// Homomorphic invariant: the per-DB LtHashes sum to the committed global LtHash.
 	sum := lthash.New()
 	for _, dir := range dataDBDirs {
 		sum.MixIn(s.localMeta[dir].LtHash)
 	}
 	require.True(t, sum.Equal(s.committedLtHash))
+
+	// Homomorphic invariant: storageDB's per-module hashes sum to its per-DB root.
+	moduleSum := lthash.New()
+	for _, h := range s.localMeta[storageDBDir].ModuleLtHashes {
+		moduleSum.MixIn(h)
+	}
+	require.True(t, moduleSum.Equal(s.localMeta[storageDBDir].LtHash))
 }

sei-db/state_db/sc/flatkv/metrics.go

@@ -8,6 +8,8 @@ import (
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
+
+	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash"
 )
 
 var (
@@ -33,6 +35,8 @@ var (
 		ImportKVPairs             metric.Int64Counter
 		ImportWorkerFlushLatency  metric.Float64Histogram
 		FlushLatency              metric.Float64Histogram
+		ModuleKeyCount            metric.Int64Gauge
+		ModuleTotalBytes          metric.Int64Gauge
 	}{
 		OpenLatency: must(flatkvMeter.Float64Histogram(
 			"flatkv_open_latency",
@@ -129,6 +133,16 @@ var (
 			metric.WithDescription("Time taken to flush a FlatKV data DB"),
 			metric.WithUnit("s"),
 		)),
+		ModuleKeyCount: must(flatkvMeter.Int64Gauge(
+			"flatkv_module_key_count",
+			metric.WithDescription("Current number of live keys for a (data DB, module) pair"),
+			metric.WithUnit("{count}"),
+		)),
+		ModuleTotalBytes: must(flatkvMeter.Int64Gauge(
+			"flatkv_module_total_bytes",
+			metric.WithDescription("Current total serialized key+value bytes for a (data DB, module) pair"),
+			metric.WithUnit("By"),
+		)),
 	}
 )
 
@@ -151,6 +165,22 @@ func dbAttr(db string) attribute.KeyValue {
 	return attribute.String("db", db)
 }
 
+func moduleAttr(module string) attribute.KeyValue {
+	return attribute.String("module", module)
+}
+
+// recordModuleStats records the current per-module key-count / byte totals for one data DB. Only called
+// from CommitStore.recordAllModuleStats, which is itself only reached from Commit's live success path
+// (see its doc comment). stats is keyed by module name, mirroring LocalMeta.ModuleStats; a nil/empty map
+// is a no-op (fresh DB with no modules yet).
+func recordModuleStats(ctx context.Context, db string, stats map[string]lthash.ModuleStats) {
+	for module, s := range stats {
+		attrs := metric.WithAttributes(dbAttr(db), moduleAttr(module))
+		otelMetrics.ModuleKeyCount.Record(ctx, s.KeyCount, attrs)
+		otelMetrics.ModuleTotalBytes.Record(ctx, s.Bytes, attrs)
+	}
+}
+
 func recordPendingWrites(ctx context.Context, db string, count int) {
 	otelMetrics.PendingWrites.Record(ctx, int64(count), metric.WithAttributes(dbAttr(db)))
 }

sei-db/state_db/sc/flatkv/store_write.go

@@ -111,6 +111,7 @@ func (s *CommitStore) Commit() (version int64, err error) {
 
 	s.phaseTimer.SetPhase("commit_done")
 	otelMetrics.CurrentVersion.Record(s.ctx, version)
+	s.recordAllModuleStats()
 	logger.Info("FlatKV Commit complete",
 		"version", version,
 		"totalWriteCount", pendingAccount+pendingCode+pendingStorage+pendingMisc,
@@ -225,7 +226,12 @@ func (s *CommitStore) commitBatches(version int64) error {
 		}
 	}
 
-	// Update in-memory local meta after all commits succeed.
+	// Update in-memory local meta after all commits succeed. Per-module stats gauges are deliberately
+	// NOT recorded here: commitBatches is also called by catchup (WAL replay on open, import, rollback,
+	// and openReadOnly's historical read-only clone), and recording from every replayed entry would
+	// publish stale/historical values into the process-global gauges, clobbering the live store's real
+	// values. Only Commit's success path (an actual new block being committed) records via
+	// recordAllModuleStats.
 	for _, p := range pending {
 		s.localMeta[p.dbDir] = &ktype.LocalMeta{
 			CommittedVersion: version,
@@ -237,6 +243,16 @@ func (s *CommitStore) commitBatches(version int64) error {
 	return nil
 }
 
+// recordAllModuleStats records the current per-module key-count / byte totals for every data DB. Called
+// only from Commit's success path — i.e. only for an actual new block being committed, never for open
+// (LoadVersion/catchup), import (FinalizeImport), rollback, or a read-only historical replay — so the
+// process-global gauges always reflect the live store, not a replay.
+func (s *CommitStore) recordAllModuleStats() {
+	for _, dir := range dataDBDirs {
+		recordModuleStats(s.ctx, dir, s.perDBModuleWorkingStats[dir])
+	}
+}
+
 func prepareBatch[T vtype.VType](
 	db types.KeyValueDB,
 	writes map[string]T,

sei-ibc-go/modules/core/03-connection/types/msgs_test.go

@@ -55,6 +55,9 @@ func (suite *MsgTestSuite) SetupTest() {
 	scConfig.MemIAVLConfig.SnapshotMinTimeInterval = 0
 	ssConfig := seidbconfig.StateStoreConfig{}
 	store := storev2rootmulti.NewStore(suite.T().TempDir(), scConfig, ssConfig, nil)
+	defer func() {
+		suite.Require().NoError(store.Close())
+	}()
 	storeKey := storetypes.NewKVStoreKey("iavlStoreKey")
 
 	store.MountStoreWithDB(storeKey, storetypes.StoreTypeIAVL, nil)

sei-ibc-go/modules/core/23-commitment/types/commitment_test.go

@@ -36,6 +36,10 @@ func (suite *MerkleTestSuite) SetupTest() {
 	suite.iavlStore = suite.store.GetCommitKVStore(suite.storeKey)
 }
 
+func (suite *MerkleTestSuite) TearDownTest() {
+	suite.Require().NoError(suite.store.Close())
+}
+
 func TestMerkleTestSuite(t *testing.T) {
 	suite.Run(t, new(MerkleTestSuite))
 }