StateWAL + FlatKV

sei-cosmos/storev2/rootmulti/flatkv_helpers_test.go

@@ -325,7 +325,9 @@ func rollbackFlatKV(t *testing.T, dir string, cfg seidbconfig.StateCommitConfig,
 	t.Helper()
 	flatkvCfg := cfg.FlatKVConfig
 	flatkvCfg.DataDir = utils.GetFlatKVPath(dir)
-	evmStore, err := flatkv.NewCommitStore(context.Background(), &flatkvCfg)
+	stateWAL, err := flatkv.OpenStateWAL(&flatkvCfg)
+	require.NoError(t, err)
+	evmStore, err := flatkv.NewCommitStore(context.Background(), &flatkvCfg, stateWAL)
 	require.NoError(t, err)
 	_, err = evmStore.LoadVersion(0, false)
 	require.NoError(t, err)
@@ -351,7 +353,9 @@ func openFlatKVReadOnly(t *testing.T, dir string, cfg seidbconfig.StateCommitCon
 	t.Helper()
 	flatkvCfg := cfg.FlatKVConfig
 	flatkvCfg.DataDir = utils.GetFlatKVPath(dir)
-	store, err := flatkv.NewCommitStore(context.Background(), &flatkvCfg)
+	stateWAL, err := flatkv.OpenStateWAL(&flatkvCfg)
+	require.NoError(t, err)
+	store, err := flatkv.NewCommitStore(context.Background(), &flatkvCfg, stateWAL)
 	require.NoError(t, err)
 	ro, err := store.LoadVersion(version, true)
 	require.NoError(t, err)
@@ -414,7 +418,9 @@ func collectFlatKVEVM(t *testing.T, dir string, cfg seidbconfig.StateCommitConfi
 	flatkvCfg := cfg.FlatKVConfig
 	flatkvCfg.DataDir = utils.GetFlatKVPath(dir)
 
-	s, err := flatkv.NewCommitStore(context.Background(), &flatkvCfg)
+	stateWAL, err := flatkv.OpenStateWAL(&flatkvCfg)
+	require.NoError(t, err)
+	s, err := flatkv.NewCommitStore(context.Background(), &flatkvCfg, stateWAL)
 	require.NoError(t, err)
 	defer func() { require.NoError(t, s.Close()) }()
 

sei-cosmos/storev2/rootmulti/flatkv_migration_test.go

@@ -36,7 +36,9 @@ func migrationVersionInFlatKV(t *testing.T, dir string, cfg seidbconfig.StateCom
 	t.Helper()
 	flatkvCfg := cfg.FlatKVConfig
 	flatkvCfg.DataDir = utils.GetFlatKVPath(dir)
-	s, err := flatkv.NewCommitStore(context.Background(), &flatkvCfg)
+	stateWAL, err := flatkv.OpenStateWAL(&flatkvCfg)
+	require.NoError(t, err)
+	s, err := flatkv.NewCommitStore(context.Background(), &flatkvCfg, stateWAL)
 	require.NoError(t, err)
 	_, err = s.LoadVersion(0, false)
 	require.NoError(t, err)

sei-db/seiwal/seiwal_lock_test.go

@@ -34,6 +34,10 @@ func TestFileLockPreventsOfflineWhileLive(t *testing.T) {
 
 	err = VerifyIntegrity(dir)
 	require.ErrorIs(t, err, commonerrors.ErrFileLockUnavailable)
+
+	err = DeleteAll(dir)
+	require.ErrorIs(t, err, commonerrors.ErrFileLockUnavailable)
+	require.DirExists(t, dir, "a wipe blocked by the lock must not have removed anything")
 }
 
 // TestFileLockReleasedOnClose verifies that Close releases the lock so a later WAL and the offline utilities
@@ -60,6 +64,47 @@ func TestFileLockReleasedOnClose(t *testing.T) {
 
 	w2 := openWAL(t, testConfig(dir))
 	require.NoError(t, w2.Close())
+
+	require.NoError(t, DeleteAll(dir))
+}
+
+// TestDeleteAllRemovesDirectoryAndLockStillExcludes verifies that DeleteAll removes the whole directory,
+// lock file included, and that the lock recreated by the next open still excludes a second owner. Asserting
+// only that the directory is gone would pass even if the replacement lock file no longer excluded anything.
+func TestDeleteAllRemovesDirectoryAndLockStillExcludes(t *testing.T) {
+	dir := t.TempDir()
+
+	w := openWAL(t, testConfig(dir))
+	for index := uint64(1); index <= 5; index++ {
+		appendRecord(t, w, index)
+	}
+	require.NoError(t, w.Flush())
+	require.NoError(t, w.Close())
+	require.FileExists(t, filepath.Join(dir, lockFileName))
+
+	require.NoError(t, DeleteAll(dir))
+	require.NoDirExists(t, dir)
+
+	// Reopening recreates the directory and its lock file, and that fresh lock must exclude just as the
+	// original did.
+	w2 := openWAL(t, testConfig(dir))
+	defer func() { require.NoError(t, w2.Close()) }()
+	require.FileExists(t, filepath.Join(dir, lockFileName))
+
+	_, err := NewWAL(testConfig(dir))
+	require.ErrorIs(t, err, commonerrors.ErrFileLockUnavailable)
+
+	ok, _, _, err := w2.Bounds()
+	require.NoError(t, err)
+	require.False(t, ok, "WAL should be empty after DeleteAll")
+}
+
+// TestDeleteAllMissingDirIsNoop verifies DeleteAll is a clean no-op on a directory that does not exist, and
+// that it does not create one on the way out.
+func TestDeleteAllMissingDirIsNoop(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "never-created")
+	require.NoError(t, DeleteAll(dir))
+	require.NoDirExists(t, dir)
 }
 
 // TestFileLockSequentialOpenClose verifies that repeated open/close cycles succeed: the lock leaves no stale

sei-db/seiwal/seiwal_offline.go

@@ -72,6 +72,38 @@ func PruneAfter(path string, highestIndexToKeep uint64) error {
 	return nil
 }
 
+// DeleteAll removes the WAL directory at path and everything in it, lock file included. It is a no-op if the
+// directory does not exist. It requires the exclusive directory lock and returns
+// commonerrors.ErrFileLockUnavailable if a WAL is open on the same directory.
+func DeleteAll(path string) error {
+	info, err := os.Stat(path)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("failed to stat WAL directory %s: %w", path, err)
+	}
+	if !info.IsDir() {
+		return fmt.Errorf("WAL path %s exists but is not a directory", path)
+	}
+
+	lock, err := acquireDirLock(path)
+	if err != nil {
+		return fmt.Errorf("failed to lock WAL directory %s: %w", path, err)
+	}
+	// Releasing after the removal below is safe: the lock acts on the open file descriptor, which outlives
+	// the unlink of the file it was opened from on both backends.
+	defer releaseDirLock(lock, path)
+
+	// This must remain the last mutation under the lock. It unlinks the lock file, so a concurrent NewWAL can
+	// recreate the directory and take a fresh, unrelated lock; anything added after this point would run
+	// against that second owner with no exclusion at all.
+	if err := os.RemoveAll(path); err != nil {
+		return fmt.Errorf("failed to delete WAL directory %s: %w", path, err)
+	}
+	return nil
+}
+
 // VerifyIntegrity checks every sealed file in the WAL directory at path: each record's CRC is intact, each
 // file's content covers the index range its name promises, and the sealed sequence has no gaps or duplicates.
 // It does not modify the directory. It requires the exclusive directory lock and returns

sei-db/state_db/bench/cryptosim/config/basic-config.json

@@ -13,7 +13,7 @@
     "UseDefaultComparer": false,
     "EVMDBDirectory": ""
   },
-  "BlocksPerCommit": 32,
+  "BlocksPerCommit": 1,
   "CannedRandomSize": 1073741824,
   "ConstantThreadCount": 0,
   "ConsoleUpdateIntervalSeconds": 1,

sei-db/state_db/bench/cryptosim/cryptosim_config.go

@@ -84,6 +84,7 @@ type CryptoSimConfig struct {
 	TransactionsPerBlock int
 
 	// Commit is called on the database after this many blocks have been processed.
+	// Must be 1 for the FlatKV backend, which persists exactly one block per commit.
 	BlocksPerCommit int
 
 	// The directory to store the benchmark data.
@@ -353,6 +354,10 @@ func (c *CryptoSimConfig) Validate() error {
 	if c.BlocksPerCommit < 1 {
 		return fmt.Errorf("BlocksPerCommit must be at least 1 (got %d)", c.BlocksPerCommit)
 	}
+	if c.Backend == wrappers.FlatKV && c.BlocksPerCommit != 1 {
+		return fmt.Errorf("BlocksPerCommit must be 1 for the %s backend (got %d): FlatKV persists exactly "+
+			"one block per commit", wrappers.FlatKV, c.BlocksPerCommit)
+	}
 	if c.CannedRandomSize < 8 {
 		return fmt.Errorf("CannedRandomSize must be at least 8 (got %d)", c.CannedRandomSize)
 	}

sei-db/state_db/bench/cryptosim/cryptosim_config_test.go

@@ -50,6 +50,41 @@ func TestLoadConfigFromFile_InvalidStateStoreBackend(t *testing.T) {
 	require.ErrorContains(t, err, `StateStoreConfig.Backend must be one of "pebbledb" or "rocksdb"`)
 }
 
+func TestLoadConfigFromFile_FlatKVRejectsBatchedBlocks(t *testing.T) {
+	t.Parallel()
+
+	configPath := filepath.Join(t.TempDir(), "cryptosim.json")
+	err := os.WriteFile(configPath, []byte(`{
+  "Backend": "FlatKV",
+  "BlocksPerCommit": 32,
+  "DataDir": "data",
+  "LogDir": "logs"
+}`), 0o600)
+	require.NoError(t, err)
+
+	_, err = LoadConfigFromFile(configPath)
+	require.ErrorContains(t, err, "BlocksPerCommit must be 1 for the FlatKV backend")
+}
+
+// TestShippedConfigsAreValid keeps every config file under config/ runnable: a
+// config that fails Validate is a broken benchmark nobody discovers until they run
+// it.
+func TestShippedConfigsAreValid(t *testing.T) {
+	t.Parallel()
+
+	paths, err := filepath.Glob(filepath.Join("config", "*.json"))
+	require.NoError(t, err)
+	require.NotEmpty(t, paths, "no shipped configs found")
+
+	for _, path := range paths {
+		t.Run(filepath.Base(path), func(t *testing.T) {
+			t.Parallel()
+			_, err := LoadConfigFromFile(path)
+			require.NoError(t, err)
+		})
+	}
+}
+
 func TestLoadConfigFromFile_DisableTransactionReadsOverride(t *testing.T) {
 	t.Parallel()
 

sei-db/state_db/bench/wrappers/db_implementations.go

@@ -77,8 +77,13 @@ func newFlatKVCommitStore(ctx context.Context, dbDir string, config *flatkvConfi
 	config.DataDir = dbDir
 
 	fmt.Printf("Opening flatKV from directory %s\n", dbDir)
-	cs, err := flatkv.NewCommitStore(ctx, config)
+	stateWAL, err := flatkv.OpenStateWAL(config)
 	if err != nil {
+		return nil, fmt.Errorf("failed to open FlatKV state WAL: %w", err)
+	}
+	cs, err := flatkv.NewCommitStore(ctx, config, stateWAL)
+	if err != nil {
+		_ = stateWAL.Close()
 		return nil, fmt.Errorf("failed to create FlatKV commit store: %w", err)
 	}
 	_, err = cs.LoadVersion(0, false)

sei-db/state_db/bench/wrappers/flatkv_wrapper.go

@@ -11,9 +11,10 @@ import (
 var _ DBWrapper = (*flatKVWrapper)(nil)
 
 // flatKVWrapper wraps a flatkv commit store to implement the DBWrapper interface.
-// Version() and Commit() consult the base store's PendingVersion() so that
-// benchmarks may call ApplyChangeSets multiple times (one per block) before a
-// single Commit, e.g. when BlocksPerCommit > 1.
+// FlatKV persists exactly one block per Commit, so benchmarks must commit every
+// block (BlocksPerCommit == 1, enforced by cryptosim's config validation). Several
+// ApplyChangeSets calls may still precede one Commit as long as they all target the
+// same height; Commit() consults PendingVersion() to find that height.
 type flatKVWrapper struct {
 	base flatkv.Store
 }
@@ -46,23 +47,14 @@ func (f *flatKVWrapper) LoadVersion(version int64) error {
 	return err
 }
 
-// Version returns the working version: the height stamped by the most
-// recent pending ApplyChangeSets call, or committedVersion+1's predecessor
-// (committedVersion) when nothing is pending.
 func (f *flatKVWrapper) Version() int64 {
-	if p := f.base.PendingVersion(); p != 0 {
-		return p
-	}
 	return f.base.Version()
 }
 
-// nextVersion computes the height for the next ApplyChangeSets call: one
-// past the last pending call's height, or one past the committed version
-// when no writes are pending.
+// nextVersion computes the height for the next ApplyChangeSets call: one past the
+// committed version. It deliberately ignores PendingVersion() — a pending block's
+// writes may be extended at its own height, never continued at the next one.
 func (f *flatKVWrapper) nextVersion() int64 {
-	if p := f.base.PendingVersion(); p != 0 {
-		return p + 1
-	}
 	return f.base.Version() + 1
 }
 

sei-db/state_db/bench/wrappers/flatkv_wrapper_test.go

@@ -8,48 +8,50 @@ import (
 	"github.com/sei-protocol/sei-chain/sei-db/proto"
 )
 
-// TestFlatKVWrapperBatchesMultipleBlocksBeforeCommit reproduces the
-// cryptosim.Database.FinalizeBlock pattern used when BlocksPerCommit > 1:
-// ApplyChangeSets is called once per block (each time computing
-// entry.Version as wrapper.Version()+1) and Commit is only called every
-// N blocks. Regression test for "flatkv: cannot apply at height N; pending
-// writes already stamped at N-1".
-func TestFlatKVWrapperBatchesMultipleBlocksBeforeCommit(t *testing.T) {
+func flatKVEntry(version int64, value byte) *proto.ChangelogEntry {
+	return &proto.ChangelogEntry{
+		Version: version,
+		Changesets: []*proto.NamedChangeSet{{
+			Name:      EVMStoreName,
+			Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: []byte("key"), Value: []byte{value}}}},
+		}},
+	}
+}
+
+// TestFlatKVWrapperCommitsOneBlockPerCommit drives the cryptosim
+// Database.FinalizeBlock pattern at BlocksPerCommit == 1 against a real state
+// WAL: each block is applied at Version()+1 and committed immediately. It runs
+// several cycles because the WAL only rejects a non-contiguous block number on
+// the commit after the first one.
+func TestFlatKVWrapperCommitsOneBlockPerCommit(t *testing.T) {
 	wrapper, err := NewDBImpl(t.Context(), FlatKV, t.TempDir(), nil)
 	require.NoError(t, err)
 	defer func() { require.NoError(t, wrapper.Close()) }()
 
-	const blocksPerCommit = 5
-	for block := 1; block <= blocksPerCommit; block++ {
-		key := []byte("key")
-		value := []byte{byte(block)}
-		entry := &proto.ChangelogEntry{
-			Version: wrapper.Version() + 1,
-			Changesets: []*proto.NamedChangeSet{{
-				Name:      EVMStoreName,
-				Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: key, Value: value}}},
-			}},
-		}
-		require.NoError(t, wrapper.ApplyChangeSets(entry), "block %d", block)
-		require.Equal(t, int64(block), wrapper.Version(), "working version after applying block %d", block)
+	for block := 1; block <= 5; block++ {
+		require.NoError(t, wrapper.ApplyChangeSets(flatKVEntry(wrapper.Version()+1, byte(block))), "block %d", block)
+		committed, err := wrapper.Commit()
+		require.NoError(t, err, "block %d", block)
+		require.Equal(t, int64(block), committed)
+		require.Equal(t, int64(block), wrapper.Version())
 	}
+}
 
-	committed, err := wrapper.Commit()
+// TestFlatKVWrapperRejectsSecondBlockBeforeCommit pins the barrier against
+// batched blocks: FlatKV persists exactly one block per Commit, and the refusal
+// lands on the offending ApplyChangeSets rather than on a later Commit.
+func TestFlatKVWrapperRejectsSecondBlockBeforeCommit(t *testing.T) {
+	wrapper, err := NewDBImpl(t.Context(), FlatKV, t.TempDir(), nil)
 	require.NoError(t, err)
-	require.Equal(t, int64(blocksPerCommit), committed)
-	require.Equal(t, int64(blocksPerCommit), wrapper.Version())
+	defer func() { require.NoError(t, wrapper.Close()) }()
 
-	// A subsequent single-block cycle (BlocksPerCommit == 1 shape) must
-	// still work after a batched commit.
-	entry := &proto.ChangelogEntry{
-		Version: wrapper.Version() + 1,
-		Changesets: []*proto.NamedChangeSet{{
-			Name:      EVMStoreName,
-			Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: []byte("key2"), Value: []byte{1}}}},
-		}},
-	}
-	require.NoError(t, wrapper.ApplyChangeSets(entry))
-	committed, err = wrapper.Commit()
+	require.NoError(t, wrapper.ApplyChangeSets(flatKVEntry(1, 0x01)))
+
+	err = wrapper.ApplyChangeSets(flatKVEntry(2, 0x02))
+	require.ErrorContains(t, err, "only one block may be buffered per commit")
+
+	// The rejected call left the pending block intact and committable.
+	committed, err := wrapper.Commit()
 	require.NoError(t, err)
-	require.Equal(t, int64(blocksPerCommit+1), committed)
+	require.Equal(t, int64(1), committed)
 }

sei-db/state_db/sc/composite/random_test_framework_test.go

@@ -1608,7 +1608,9 @@ func rollbackFlatKVIndependently(t *testing.T, dir string, cfg config.StateCommi
 	t.Helper()
 	flatkvCfg := cfg.FlatKVConfig
 	flatkvCfg.DataDir = utils.GetFlatKVPath(dir)
-	evmStore, err := flatkv.NewCommitStore(t.Context(), &flatkvCfg)
+	flatkvWAL, err := flatkv.OpenStateWAL(&flatkvCfg)
+	require.NoError(t, err)
+	evmStore, err := flatkv.NewCommitStore(t.Context(), &flatkvCfg, flatkvWAL)
 	require.NoError(t, err)
 	_, err = evmStore.LoadVersion(0, false)
 	require.NoError(t, err)

sei-db/state_db/sc/composite/store.go

@@ -161,8 +161,13 @@ func NewCompositeCommitStore(
 
 	var flatKV flatkv.Store
 	if openFlatKV {
-		fkv, err := flatkv.NewCommitStore(ctx, &cfg.FlatKVConfig)
+		stateWAL, err := flatkv.OpenStateWAL(&cfg.FlatKVConfig)
 		if err != nil {
+			return nil, fmt.Errorf("failed to open FlatKV state WAL: %w", err)
+		}
+		fkv, err := flatkv.NewCommitStore(ctx, &cfg.FlatKVConfig, stateWAL)
+		if err != nil {
+			_ = stateWAL.Close()
 			return nil, fmt.Errorf("failed to create FlatKV commit store: %w", err)
 		}
 		flatKV = fkv
@@ -676,8 +681,13 @@ func (cs *CompositeCommitStore) SetWriteMode(targetWriteMode types.WriteMode) er
 func (cs *CompositeCommitStore) newFlatKVInstance() (flatkv.Store, error) {
 	flatKVConfig := cs.config.FlatKVConfig
 	flatKVConfig.DataDir = utils.GetFlatKVPath(cs.homeDir)
-	created, err := flatkv.NewCommitStore(cs.ctx, &flatKVConfig)
+	stateWAL, err := flatkv.OpenStateWAL(&flatKVConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open FlatKV state WAL: %w", err)
+	}
+	created, err := flatkv.NewCommitStore(cs.ctx, &flatKVConfig, stateWAL)
 	if err != nil {
+		_ = stateWAL.Close()
 		return nil, fmt.Errorf("failed to create FlatKV commit store: %w", err)
 	}
 	return created, nil