diff --git a/sei-cosmos/storev2/rootmulti/flatkv_helpers_test.go b/sei-cosmos/storev2/rootmulti/flatkv_helpers_test.go index f963d73d4f..4586b842c4 100644 --- a/sei-cosmos/storev2/rootmulti/flatkv_helpers_test.go +++ b/sei-cosmos/storev2/rootmulti/flatkv_helpers_test.go @@ -325,7 +325,9 @@ func rollbackFlatKV(t *testing.T, dir string, cfg seidbconfig.StateCommitConfig, t.Helper() flatkvCfg := cfg.FlatKVConfig flatkvCfg.DataDir = utils.GetFlatKVPath(dir) - evmStore, err := flatkv.NewCommitStore(context.Background(), &flatkvCfg) + stateWAL, err := flatkv.OpenStateWAL(&flatkvCfg) + require.NoError(t, err) + evmStore, err := flatkv.NewCommitStore(context.Background(), &flatkvCfg, stateWAL) require.NoError(t, err) _, err = evmStore.LoadVersion(0, false) require.NoError(t, err) @@ -351,7 +353,9 @@ func openFlatKVReadOnly(t *testing.T, dir string, cfg seidbconfig.StateCommitCon t.Helper() flatkvCfg := cfg.FlatKVConfig flatkvCfg.DataDir = utils.GetFlatKVPath(dir) - store, err := flatkv.NewCommitStore(context.Background(), &flatkvCfg) + stateWAL, err := flatkv.OpenStateWAL(&flatkvCfg) + require.NoError(t, err) + store, err := flatkv.NewCommitStore(context.Background(), &flatkvCfg, stateWAL) require.NoError(t, err) ro, err := store.LoadVersion(version, true) require.NoError(t, err) @@ -414,7 +418,9 @@ func collectFlatKVEVM(t *testing.T, dir string, cfg seidbconfig.StateCommitConfi flatkvCfg := cfg.FlatKVConfig flatkvCfg.DataDir = utils.GetFlatKVPath(dir) - s, err := flatkv.NewCommitStore(context.Background(), &flatkvCfg) + stateWAL, err := flatkv.OpenStateWAL(&flatkvCfg) + require.NoError(t, err) + s, err := flatkv.NewCommitStore(context.Background(), &flatkvCfg, stateWAL) require.NoError(t, err) defer func() { require.NoError(t, s.Close()) }() diff --git a/sei-cosmos/storev2/rootmulti/flatkv_migration_test.go b/sei-cosmos/storev2/rootmulti/flatkv_migration_test.go index e9706a7a7d..acb7291fc3 100644 --- a/sei-cosmos/storev2/rootmulti/flatkv_migration_test.go +++ b/sei-cosmos/storev2/rootmulti/flatkv_migration_test.go @@ -36,7 +36,9 @@ func migrationVersionInFlatKV(t *testing.T, dir string, cfg seidbconfig.StateCom t.Helper() flatkvCfg := cfg.FlatKVConfig flatkvCfg.DataDir = utils.GetFlatKVPath(dir) - s, err := flatkv.NewCommitStore(context.Background(), &flatkvCfg) + stateWAL, err := flatkv.OpenStateWAL(&flatkvCfg) + require.NoError(t, err) + s, err := flatkv.NewCommitStore(context.Background(), &flatkvCfg, stateWAL) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) diff --git a/sei-db/seiwal/seiwal_lock_test.go b/sei-db/seiwal/seiwal_lock_test.go index 005d0a3b7f..945a5ad822 100644 --- a/sei-db/seiwal/seiwal_lock_test.go +++ b/sei-db/seiwal/seiwal_lock_test.go @@ -34,6 +34,10 @@ func TestFileLockPreventsOfflineWhileLive(t *testing.T) { err = VerifyIntegrity(dir) require.ErrorIs(t, err, commonerrors.ErrFileLockUnavailable) + + err = DeleteAll(dir) + require.ErrorIs(t, err, commonerrors.ErrFileLockUnavailable) + require.DirExists(t, dir, "a wipe blocked by the lock must not have removed anything") } // TestFileLockReleasedOnClose verifies that Close releases the lock so a later WAL and the offline utilities @@ -60,6 +64,47 @@ func TestFileLockReleasedOnClose(t *testing.T) { w2 := openWAL(t, testConfig(dir)) require.NoError(t, w2.Close()) + + require.NoError(t, DeleteAll(dir)) +} + +// TestDeleteAllRemovesDirectoryAndLockStillExcludes verifies that DeleteAll removes the whole directory, +// lock file included, and that the lock recreated by the next open still excludes a second owner. Asserting +// only that the directory is gone would pass even if the replacement lock file no longer excluded anything. +func TestDeleteAllRemovesDirectoryAndLockStillExcludes(t *testing.T) { + dir := t.TempDir() + + w := openWAL(t, testConfig(dir)) + for index := uint64(1); index <= 5; index++ { + appendRecord(t, w, index) + } + require.NoError(t, w.Flush()) + require.NoError(t, w.Close()) + require.FileExists(t, filepath.Join(dir, lockFileName)) + + require.NoError(t, DeleteAll(dir)) + require.NoDirExists(t, dir) + + // Reopening recreates the directory and its lock file, and that fresh lock must exclude just as the + // original did. + w2 := openWAL(t, testConfig(dir)) + defer func() { require.NoError(t, w2.Close()) }() + require.FileExists(t, filepath.Join(dir, lockFileName)) + + _, err := NewWAL(testConfig(dir)) + require.ErrorIs(t, err, commonerrors.ErrFileLockUnavailable) + + ok, _, _, err := w2.Bounds() + require.NoError(t, err) + require.False(t, ok, "WAL should be empty after DeleteAll") +} + +// TestDeleteAllMissingDirIsNoop verifies DeleteAll is a clean no-op on a directory that does not exist, and +// that it does not create one on the way out. +func TestDeleteAllMissingDirIsNoop(t *testing.T) { + dir := filepath.Join(t.TempDir(), "never-created") + require.NoError(t, DeleteAll(dir)) + require.NoDirExists(t, dir) } // TestFileLockSequentialOpenClose verifies that repeated open/close cycles succeed: the lock leaves no stale diff --git a/sei-db/seiwal/seiwal_offline.go b/sei-db/seiwal/seiwal_offline.go index c001d2fa1c..bf6c5703d9 100644 --- a/sei-db/seiwal/seiwal_offline.go +++ b/sei-db/seiwal/seiwal_offline.go @@ -72,6 +72,38 @@ func PruneAfter(path string, highestIndexToKeep uint64) error { return nil } +// DeleteAll removes the WAL directory at path and everything in it, lock file included. It is a no-op if the +// directory does not exist. It requires the exclusive directory lock and returns +// commonerrors.ErrFileLockUnavailable if a WAL is open on the same directory. +func DeleteAll(path string) error { + info, err := os.Stat(path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return fmt.Errorf("failed to stat WAL directory %s: %w", path, err) + } + if !info.IsDir() { + return fmt.Errorf("WAL path %s exists but is not a directory", path) + } + + lock, err := acquireDirLock(path) + if err != nil { + return fmt.Errorf("failed to lock WAL directory %s: %w", path, err) + } + // Releasing after the removal below is safe: the lock acts on the open file descriptor, which outlives + // the unlink of the file it was opened from on both backends. + defer releaseDirLock(lock, path) + + // This must remain the last mutation under the lock. It unlinks the lock file, so a concurrent NewWAL can + // recreate the directory and take a fresh, unrelated lock; anything added after this point would run + // against that second owner with no exclusion at all. + if err := os.RemoveAll(path); err != nil { + return fmt.Errorf("failed to delete WAL directory %s: %w", path, err) + } + return nil +} + // VerifyIntegrity checks every sealed file in the WAL directory at path: each record's CRC is intact, each // file's content covers the index range its name promises, and the sealed sequence has no gaps or duplicates. // It does not modify the directory. It requires the exclusive directory lock and returns diff --git a/sei-db/state_db/bench/cryptosim/config/basic-config.json b/sei-db/state_db/bench/cryptosim/config/basic-config.json index 67907014e9..0414a9af58 100644 --- a/sei-db/state_db/bench/cryptosim/config/basic-config.json +++ b/sei-db/state_db/bench/cryptosim/config/basic-config.json @@ -13,7 +13,7 @@ "UseDefaultComparer": false, "EVMDBDirectory": "" }, - "BlocksPerCommit": 32, + "BlocksPerCommit": 1, "CannedRandomSize": 1073741824, "ConstantThreadCount": 0, "ConsoleUpdateIntervalSeconds": 1, diff --git a/sei-db/state_db/bench/cryptosim/cryptosim_config.go b/sei-db/state_db/bench/cryptosim/cryptosim_config.go index d0a09cbfba..fd52820241 100644 --- a/sei-db/state_db/bench/cryptosim/cryptosim_config.go +++ b/sei-db/state_db/bench/cryptosim/cryptosim_config.go @@ -84,6 +84,7 @@ type CryptoSimConfig struct { TransactionsPerBlock int // Commit is called on the database after this many blocks have been processed. + // Must be 1 for the FlatKV backend, which persists exactly one block per commit. BlocksPerCommit int // The directory to store the benchmark data. @@ -353,6 +354,10 @@ func (c *CryptoSimConfig) Validate() error { if c.BlocksPerCommit < 1 { return fmt.Errorf("BlocksPerCommit must be at least 1 (got %d)", c.BlocksPerCommit) } + if c.Backend == wrappers.FlatKV && c.BlocksPerCommit != 1 { + return fmt.Errorf("BlocksPerCommit must be 1 for the %s backend (got %d): FlatKV persists exactly "+ + "one block per commit", wrappers.FlatKV, c.BlocksPerCommit) + } if c.CannedRandomSize < 8 { return fmt.Errorf("CannedRandomSize must be at least 8 (got %d)", c.CannedRandomSize) } diff --git a/sei-db/state_db/bench/cryptosim/cryptosim_config_test.go b/sei-db/state_db/bench/cryptosim/cryptosim_config_test.go index f88a89df73..86b5b164de 100644 --- a/sei-db/state_db/bench/cryptosim/cryptosim_config_test.go +++ b/sei-db/state_db/bench/cryptosim/cryptosim_config_test.go @@ -50,6 +50,41 @@ func TestLoadConfigFromFile_InvalidStateStoreBackend(t *testing.T) { require.ErrorContains(t, err, `StateStoreConfig.Backend must be one of "pebbledb" or "rocksdb"`) } +func TestLoadConfigFromFile_FlatKVRejectsBatchedBlocks(t *testing.T) { + t.Parallel() + + configPath := filepath.Join(t.TempDir(), "cryptosim.json") + err := os.WriteFile(configPath, []byte(`{ + "Backend": "FlatKV", + "BlocksPerCommit": 32, + "DataDir": "data", + "LogDir": "logs" +}`), 0o600) + require.NoError(t, err) + + _, err = LoadConfigFromFile(configPath) + require.ErrorContains(t, err, "BlocksPerCommit must be 1 for the FlatKV backend") +} + +// TestShippedConfigsAreValid keeps every config file under config/ runnable: a +// config that fails Validate is a broken benchmark nobody discovers until they run +// it. +func TestShippedConfigsAreValid(t *testing.T) { + t.Parallel() + + paths, err := filepath.Glob(filepath.Join("config", "*.json")) + require.NoError(t, err) + require.NotEmpty(t, paths, "no shipped configs found") + + for _, path := range paths { + t.Run(filepath.Base(path), func(t *testing.T) { + t.Parallel() + _, err := LoadConfigFromFile(path) + require.NoError(t, err) + }) + } +} + func TestLoadConfigFromFile_DisableTransactionReadsOverride(t *testing.T) { t.Parallel() diff --git a/sei-db/state_db/bench/wrappers/db_implementations.go b/sei-db/state_db/bench/wrappers/db_implementations.go index 64fcf818cd..48d39ace14 100644 --- a/sei-db/state_db/bench/wrappers/db_implementations.go +++ b/sei-db/state_db/bench/wrappers/db_implementations.go @@ -77,8 +77,13 @@ func newFlatKVCommitStore(ctx context.Context, dbDir string, config *flatkvConfi config.DataDir = dbDir fmt.Printf("Opening flatKV from directory %s\n", dbDir) - cs, err := flatkv.NewCommitStore(ctx, config) + stateWAL, err := flatkv.OpenStateWAL(config) if err != nil { + return nil, fmt.Errorf("failed to open FlatKV state WAL: %w", err) + } + cs, err := flatkv.NewCommitStore(ctx, config, stateWAL) + if err != nil { + _ = stateWAL.Close() return nil, fmt.Errorf("failed to create FlatKV commit store: %w", err) } _, err = cs.LoadVersion(0, false) diff --git a/sei-db/state_db/bench/wrappers/flatkv_wrapper.go b/sei-db/state_db/bench/wrappers/flatkv_wrapper.go index cadf92ddcf..2f29f138f6 100644 --- a/sei-db/state_db/bench/wrappers/flatkv_wrapper.go +++ b/sei-db/state_db/bench/wrappers/flatkv_wrapper.go @@ -11,9 +11,10 @@ import ( var _ DBWrapper = (*flatKVWrapper)(nil) // flatKVWrapper wraps a flatkv commit store to implement the DBWrapper interface. -// Version() and Commit() consult the base store's PendingVersion() so that -// benchmarks may call ApplyChangeSets multiple times (one per block) before a -// single Commit, e.g. when BlocksPerCommit > 1. +// FlatKV persists exactly one block per Commit, so benchmarks must commit every +// block (BlocksPerCommit == 1, enforced by cryptosim's config validation). Several +// ApplyChangeSets calls may still precede one Commit as long as they all target the +// same height; Commit() consults PendingVersion() to find that height. type flatKVWrapper struct { base flatkv.Store } @@ -46,23 +47,14 @@ func (f *flatKVWrapper) LoadVersion(version int64) error { return err } -// Version returns the working version: the height stamped by the most -// recent pending ApplyChangeSets call, or committedVersion+1's predecessor -// (committedVersion) when nothing is pending. func (f *flatKVWrapper) Version() int64 { - if p := f.base.PendingVersion(); p != 0 { - return p - } return f.base.Version() } -// nextVersion computes the height for the next ApplyChangeSets call: one -// past the last pending call's height, or one past the committed version -// when no writes are pending. +// nextVersion computes the height for the next ApplyChangeSets call: one past the +// committed version. It deliberately ignores PendingVersion() — a pending block's +// writes may be extended at its own height, never continued at the next one. func (f *flatKVWrapper) nextVersion() int64 { - if p := f.base.PendingVersion(); p != 0 { - return p + 1 - } return f.base.Version() + 1 } diff --git a/sei-db/state_db/bench/wrappers/flatkv_wrapper_test.go b/sei-db/state_db/bench/wrappers/flatkv_wrapper_test.go index e42519d5c3..dc21084657 100644 --- a/sei-db/state_db/bench/wrappers/flatkv_wrapper_test.go +++ b/sei-db/state_db/bench/wrappers/flatkv_wrapper_test.go @@ -8,48 +8,50 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/proto" ) -// TestFlatKVWrapperBatchesMultipleBlocksBeforeCommit reproduces the -// cryptosim.Database.FinalizeBlock pattern used when BlocksPerCommit > 1: -// ApplyChangeSets is called once per block (each time computing -// entry.Version as wrapper.Version()+1) and Commit is only called every -// N blocks. Regression test for "flatkv: cannot apply at height N; pending -// writes already stamped at N-1". -func TestFlatKVWrapperBatchesMultipleBlocksBeforeCommit(t *testing.T) { +func flatKVEntry(version int64, value byte) *proto.ChangelogEntry { + return &proto.ChangelogEntry{ + Version: version, + Changesets: []*proto.NamedChangeSet{{ + Name: EVMStoreName, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: []byte("key"), Value: []byte{value}}}}, + }}, + } +} + +// TestFlatKVWrapperCommitsOneBlockPerCommit drives the cryptosim +// Database.FinalizeBlock pattern at BlocksPerCommit == 1 against a real state +// WAL: each block is applied at Version()+1 and committed immediately. It runs +// several cycles because the WAL only rejects a non-contiguous block number on +// the commit after the first one. +func TestFlatKVWrapperCommitsOneBlockPerCommit(t *testing.T) { wrapper, err := NewDBImpl(t.Context(), FlatKV, t.TempDir(), nil) require.NoError(t, err) defer func() { require.NoError(t, wrapper.Close()) }() - const blocksPerCommit = 5 - for block := 1; block <= blocksPerCommit; block++ { - key := []byte("key") - value := []byte{byte(block)} - entry := &proto.ChangelogEntry{ - Version: wrapper.Version() + 1, - Changesets: []*proto.NamedChangeSet{{ - Name: EVMStoreName, - Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: key, Value: value}}}, - }}, - } - require.NoError(t, wrapper.ApplyChangeSets(entry), "block %d", block) - require.Equal(t, int64(block), wrapper.Version(), "working version after applying block %d", block) + for block := 1; block <= 5; block++ { + require.NoError(t, wrapper.ApplyChangeSets(flatKVEntry(wrapper.Version()+1, byte(block))), "block %d", block) + committed, err := wrapper.Commit() + require.NoError(t, err, "block %d", block) + require.Equal(t, int64(block), committed) + require.Equal(t, int64(block), wrapper.Version()) } +} - committed, err := wrapper.Commit() +// TestFlatKVWrapperRejectsSecondBlockBeforeCommit pins the barrier against +// batched blocks: FlatKV persists exactly one block per Commit, and the refusal +// lands on the offending ApplyChangeSets rather than on a later Commit. +func TestFlatKVWrapperRejectsSecondBlockBeforeCommit(t *testing.T) { + wrapper, err := NewDBImpl(t.Context(), FlatKV, t.TempDir(), nil) require.NoError(t, err) - require.Equal(t, int64(blocksPerCommit), committed) - require.Equal(t, int64(blocksPerCommit), wrapper.Version()) + defer func() { require.NoError(t, wrapper.Close()) }() - // A subsequent single-block cycle (BlocksPerCommit == 1 shape) must - // still work after a batched commit. - entry := &proto.ChangelogEntry{ - Version: wrapper.Version() + 1, - Changesets: []*proto.NamedChangeSet{{ - Name: EVMStoreName, - Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: []byte("key2"), Value: []byte{1}}}}, - }}, - } - require.NoError(t, wrapper.ApplyChangeSets(entry)) - committed, err = wrapper.Commit() + require.NoError(t, wrapper.ApplyChangeSets(flatKVEntry(1, 0x01))) + + err = wrapper.ApplyChangeSets(flatKVEntry(2, 0x02)) + require.ErrorContains(t, err, "only one block may be buffered per commit") + + // The rejected call left the pending block intact and committable. + committed, err := wrapper.Commit() require.NoError(t, err) - require.Equal(t, int64(blocksPerCommit+1), committed) + require.Equal(t, int64(1), committed) } diff --git a/sei-db/state_db/sc/composite/random_test_framework_test.go b/sei-db/state_db/sc/composite/random_test_framework_test.go index 9af946a17e..847631f324 100644 --- a/sei-db/state_db/sc/composite/random_test_framework_test.go +++ b/sei-db/state_db/sc/composite/random_test_framework_test.go @@ -1608,7 +1608,9 @@ func rollbackFlatKVIndependently(t *testing.T, dir string, cfg config.StateCommi t.Helper() flatkvCfg := cfg.FlatKVConfig flatkvCfg.DataDir = utils.GetFlatKVPath(dir) - evmStore, err := flatkv.NewCommitStore(t.Context(), &flatkvCfg) + flatkvWAL, err := flatkv.OpenStateWAL(&flatkvCfg) + require.NoError(t, err) + evmStore, err := flatkv.NewCommitStore(t.Context(), &flatkvCfg, flatkvWAL) require.NoError(t, err) _, err = evmStore.LoadVersion(0, false) require.NoError(t, err) diff --git a/sei-db/state_db/sc/composite/store.go b/sei-db/state_db/sc/composite/store.go index 6bdac399f7..43097dd52c 100644 --- a/sei-db/state_db/sc/composite/store.go +++ b/sei-db/state_db/sc/composite/store.go @@ -161,8 +161,13 @@ func NewCompositeCommitStore( var flatKV flatkv.Store if openFlatKV { - fkv, err := flatkv.NewCommitStore(ctx, &cfg.FlatKVConfig) + stateWAL, err := flatkv.OpenStateWAL(&cfg.FlatKVConfig) if err != nil { + return nil, fmt.Errorf("failed to open FlatKV state WAL: %w", err) + } + fkv, err := flatkv.NewCommitStore(ctx, &cfg.FlatKVConfig, stateWAL) + if err != nil { + _ = stateWAL.Close() return nil, fmt.Errorf("failed to create FlatKV commit store: %w", err) } flatKV = fkv @@ -676,8 +681,13 @@ func (cs *CompositeCommitStore) SetWriteMode(targetWriteMode types.WriteMode) er func (cs *CompositeCommitStore) newFlatKVInstance() (flatkv.Store, error) { flatKVConfig := cs.config.FlatKVConfig flatKVConfig.DataDir = utils.GetFlatKVPath(cs.homeDir) - created, err := flatkv.NewCommitStore(cs.ctx, &flatKVConfig) + stateWAL, err := flatkv.OpenStateWAL(&flatKVConfig) + if err != nil { + return nil, fmt.Errorf("failed to open FlatKV state WAL: %w", err) + } + created, err := flatkv.NewCommitStore(cs.ctx, &flatKVConfig, stateWAL) if err != nil { + _ = stateWAL.Close() return nil, fmt.Errorf("failed to create FlatKV commit store: %w", err) } return created, nil diff --git a/sei-db/state_db/sc/composite/store_test.go b/sei-db/state_db/sc/composite/store_test.go index 13abe3b32e..d86c922ab1 100644 --- a/sei-db/state_db/sc/composite/store_test.go +++ b/sei-db/state_db/sc/composite/store_test.go @@ -1368,7 +1368,9 @@ func TestReconcileVersionsAfterCrash(t *testing.T) { flatkvCfg := cfg.FlatKVConfig flatkvCfg.DataDir = utils.GetFlatKVPath(dir) - evmStore, err := flatkv.NewCommitStore(t.Context(), &flatkvCfg) + flatkvWAL, err := flatkv.OpenStateWAL(&flatkvCfg) + require.NoError(t, err) + evmStore, err := flatkv.NewCommitStore(t.Context(), &flatkvCfg, flatkvWAL) require.NoError(t, err) _, err = evmStore.LoadVersion(0, false) require.NoError(t, err) @@ -1430,7 +1432,9 @@ func TestReconcileVersionsThenContinueCommitting(t *testing.T) { // Simulate crash: roll FlatKV back to version 2. flatkvCfg := cfg.FlatKVConfig flatkvCfg.DataDir = utils.GetFlatKVPath(dir) - evmStore, err := flatkv.NewCommitStore(t.Context(), &flatkvCfg) + flatkvWAL, err := flatkv.OpenStateWAL(&flatkvCfg) + require.NoError(t, err) + evmStore, err := flatkv.NewCommitStore(t.Context(), &flatkvCfg, flatkvWAL) require.NoError(t, err) _, err = evmStore.LoadVersion(0, false) require.NoError(t, err) @@ -1862,7 +1866,9 @@ func TestReconcileVersionsCosmosAheadByMultiple(t *testing.T) { // Rollback FlatKV to version 3 (simulating 2 lost commits) flatkvCfg := cfg.FlatKVConfig flatkvCfg.DataDir = utils.GetFlatKVPath(dir) - evmStore, err := flatkv.NewCommitStore(t.Context(), &flatkvCfg) + flatkvWAL, err := flatkv.OpenStateWAL(&flatkvCfg) + require.NoError(t, err) + evmStore, err := flatkv.NewCommitStore(t.Context(), &flatkvCfg, flatkvWAL) require.NoError(t, err) _, err = evmStore.LoadVersion(0, false) require.NoError(t, err) diff --git a/sei-db/state_db/sc/flatkv/api.go b/sei-db/state_db/sc/flatkv/api.go index 4b8e98ae90..db838c9c43 100644 --- a/sei-db/state_db/sc/flatkv/api.go +++ b/sei-db/state_db/sc/flatkv/api.go @@ -25,23 +25,30 @@ type Options struct { // Key format: x/evm memiavl keys (mapped internally to account/code/storage DBs). type Store interface { // LoadVersion opens the database at the given version (0 = latest). - // When readOnly is true an isolated, read-only store is returned; - // the caller must Close it when done. + // + // Sharp edges. The two modes differ in what they load: readOnly=true returns + // a new isolated store and leaves the receiver untouched (the caller must + // Close it when done), while readOnly=false loads and returns the receiver + // itself. And a writable load at a non-zero version leaves the state WAL + // holding its blocks above targetVersion, so Commit fails as non-contiguous + // and such a store can only be read from; Rollback rewinds and prunes + // together, and is the only way to reach an older version and keep + // committing. LoadVersion(targetVersion int64, readOnly bool) (Store, error) // ApplyChangeSets buffers changesets at the given version, to be // persisted by the next Commit. - // May be called multiple times before a single Commit, either - // repeatedly at the same version (e.g. one block's writes split across - // several calls) or at increasing versions (e.g. batching several - // blocks together). version must never decrease across calls (see - // PendingVersion), and Commit must be called with the highest version - // applied since the last Commit. + // + // Exactly one block may be buffered per Commit: repeated calls at the + // same version are allowed, any other version is rejected (see + // PendingVersion), and Commit must be called with that version. + // Batching several blocks into one Commit is not supported. ApplyChangeSets(version int64, cs []*proto.NamedChangeSet) error // Commit persists buffered writes at the given version (block height). - // If ApplyChangeSets has buffered writes, version must equal the highest - // height those rows were stamped with (see PendingVersion). + // One Commit persists exactly one block. If ApplyChangeSets has buffered + // writes, version must equal the height those rows were stamped with + // (see PendingVersion). Commit(version int64) (int64, error) // CommitBlock is a Giga-only helper that applies changesets and commits @@ -134,14 +141,10 @@ type Store interface { // Version returns the latest committed version. Version() int64 - // PendingVersion returns the height stamped by the most recent - // ApplyChangeSets call since the last Commit, or 0 when there are no - // buffered writes. Multiple ApplyChangeSets calls may accumulate at - // strictly increasing heights before a single Commit persists them all - // (e.g. batching several blocks); callers that need to know "the next - // version to apply" should use PendingVersion()+1 when non-zero, falling - // back to Version()+1 otherwise, rather than assuming Version()+1 always - // reflects the next height. + // PendingVersion returns the height of the block currently buffered by + // ApplyChangeSets, or 0 when there are no buffered writes. It is the + // version the next Commit must be called with, and the only version + // further ApplyChangeSets calls may use. PendingVersion() int64 // GetLatestVersion returns the latest committed version persisted to diff --git a/sei-db/state_db/sc/flatkv/empty_value_replay_test.go b/sei-db/state_db/sc/flatkv/empty_value_replay_test.go index 96028c1626..126ea2655b 100644 --- a/sei-db/state_db/sc/flatkv/empty_value_replay_test.go +++ b/sei-db/state_db/sc/flatkv/empty_value_replay_test.go @@ -29,7 +29,7 @@ func reopenCommittedRoot(t *testing.T, dir string, readOnly bool) []byte { t.Helper() cfg := config.DefaultConfig() cfg.DataDir = dir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) ro, err := s.LoadVersion(0, readOnly) require.NoError(t, err) diff --git a/sei-db/state_db/sc/flatkv/import_export_test.go b/sei-db/state_db/sc/flatkv/import_export_test.go index a7be3154b0..a5194e3c34 100644 --- a/sei-db/state_db/sc/flatkv/import_export_test.go +++ b/sei-db/state_db/sc/flatkv/import_export_test.go @@ -316,7 +316,7 @@ func TestImportSurvivesReopen(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbPath - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -334,7 +334,7 @@ func TestImportSurvivesReopen(t *testing.T) { cfg2 := config.DefaultTestConfig(t) cfg2.DataDir = dbPath - s2, err := NewCommitStore(t.Context(), cfg2) + s2, err := newCommitStoreWithWAL(t.Context(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(1, false) require.NoError(t, err) @@ -365,7 +365,7 @@ func TestImportPurgesStaleData(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbPath - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -444,7 +444,7 @@ func TestImportPurgesStaleData(t *testing.T) { // --- Phase 3: import snapshot into the existing store --- require.NoError(t, s.Close()) - s, err = NewCommitStore(t.Context(), cfg) + s, err = newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -484,7 +484,7 @@ func TestImportPurgesStaleData(t *testing.T) { // Verify the store survives a reopen. require.NoError(t, s.Close()) - s, err = NewCommitStore(t.Context(), cfg) + s, err = newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(1, false) require.NoError(t, err) @@ -505,7 +505,7 @@ func TestImporterFailsWhenResetCannotRemoveCurrentLink(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbPath - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -555,7 +555,7 @@ func TestImporterHeightNonZeroSkipped(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -584,7 +584,7 @@ func TestImporterNilKeySkipped(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -609,7 +609,7 @@ func TestImporterEmptyStore(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -629,7 +629,7 @@ func TestImporterCorruptKeyDataPropagatesError(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -655,7 +655,7 @@ func TestImporterDoubleImport(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -807,7 +807,7 @@ func TestExportImportLargerDataset(t *testing.T) { dir2 := t.TempDir() cfg2 := config.DefaultTestConfig(t) cfg2.DataDir = filepath.Join(dir2, flatkvRootDir) - s2, err := NewCommitStore(t.Context(), cfg2) + s2, err := newCommitStoreWithWAL(t.Context(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) diff --git a/sei-db/state_db/sc/flatkv/lthash_correctness_test.go b/sei-db/state_db/sc/flatkv/lthash_correctness_test.go index a9b9e6160a..0ee6abaf8e 100644 --- a/sei-db/state_db/sc/flatkv/lthash_correctness_test.go +++ b/sei-db/state_db/sc/flatkv/lthash_correctness_test.go @@ -559,7 +559,7 @@ func TestLtHashPersistenceAfterReopen(t *testing.T) { // Phase 1: create state and close cfg := config.DefaultTestConfig(t) cfg.DataDir = dir - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -580,7 +580,7 @@ func TestLtHashPersistenceAfterReopen(t *testing.T) { // Phase 2: reopen and verify cfg = config.DefaultTestConfig(t) cfg.DataDir = dir - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1148,7 +1148,7 @@ func TestLtHashReadOnlyMatchesParent(t *testing.T) { cfg.SnapshotInterval = 1 cfg.SnapshotKeepRecent = 5 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1273,7 +1273,7 @@ func TestLtHashSnapshotCatchupFullScan(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -1295,7 +1295,7 @@ func TestLtHashSnapshotCatchupFullScan(t *testing.T) { // Reopen — snapshot is at v3, WAL catchup replays v4-v7 cfg2 := config.DefaultTestConfig(t) cfg2.DataDir = dbDir - s2, err := NewCommitStore(t.Context(), cfg2) + s2, err := newCommitStoreWithWAL(t.Context(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1321,7 +1321,7 @@ func TestLtHashRollbackFullScan(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1400,7 +1400,7 @@ func TestLtHashMultipleRollbacks(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) diff --git a/sei-db/state_db/sc/flatkv/perdb_lthash_test.go b/sei-db/state_db/sc/flatkv/perdb_lthash_test.go index 1e1835261e..d2d3172f75 100644 --- a/sei-db/state_db/sc/flatkv/perdb_lthash_test.go +++ b/sei-db/state_db/sc/flatkv/perdb_lthash_test.go @@ -97,7 +97,7 @@ func TestPerDBLtHashSkewRecovery(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -125,7 +125,7 @@ func TestPerDBLtHashSkewRecovery(t *testing.T) { cfg2 := config.DefaultTestConfig(t) cfg2.DataDir = dbDir - s2, err := NewCommitStore(t.Context(), cfg2) + s2, err := newCommitStoreWithWAL(t.Context(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -144,7 +144,7 @@ func TestPerDBLtHashPersistenceAfterReopen(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -159,7 +159,7 @@ func TestPerDBLtHashPersistenceAfterReopen(t *testing.T) { cfg2 := config.DefaultTestConfig(t) cfg2.DataDir = dbDir - s2, err := NewCommitStore(t.Context(), cfg2) + s2, err := newCommitStoreWithWAL(t.Context(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -257,7 +257,7 @@ func TestPerDBLtHashCatchupReplay(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -280,7 +280,7 @@ func TestPerDBLtHashCatchupReplay(t *testing.T) { cfg2 := config.DefaultTestConfig(t) cfg2.DataDir = dbDir - s2, err := NewCommitStore(t.Context(), cfg2) + s2, err := newCommitStoreWithWAL(t.Context(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -326,7 +326,7 @@ func TestPerDBLtHashAfterImport(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -366,7 +366,7 @@ func TestPerDBLtHashRollback(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -395,7 +395,7 @@ func TestPerDBLtHashPersistedInLocalMeta(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -429,7 +429,7 @@ func TestPerDBLtHashAfterDirectImport(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) diff --git a/sei-db/state_db/sc/flatkv/permodule_lthash_test.go b/sei-db/state_db/sc/flatkv/permodule_lthash_test.go index bea288204a..9f08d07bb8 100644 --- a/sei-db/state_db/sc/flatkv/permodule_lthash_test.go +++ b/sei-db/state_db/sc/flatkv/permodule_lthash_test.go @@ -154,7 +154,7 @@ func TestPerModuleLtHashPersistenceAfterReopen(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -175,7 +175,7 @@ func TestPerModuleLtHashPersistenceAfterReopen(t *testing.T) { cfg2 := config.DefaultTestConfig(t) cfg2.DataDir = dbDir - s2, err := NewCommitStore(t.Context(), cfg2) + s2, err := newCommitStoreWithWAL(t.Context(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -246,7 +246,7 @@ func TestPerModuleLtHashAfterImport(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -288,7 +288,7 @@ func TestPerModuleLtHashStateSyncImportSurvivesRestart(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -325,7 +325,7 @@ func TestPerModuleLtHashStateSyncImportSurvivesRestart(t *testing.T) { // snapshot written by the import. cfg2 := config.DefaultTestConfig(t) cfg2.DataDir = dbDir - s2, err := NewCommitStore(t.Context(), cfg2) + s2, err := newCommitStoreWithWAL(t.Context(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) diff --git a/sei-db/state_db/sc/flatkv/permodule_stats_test.go b/sei-db/state_db/sc/flatkv/permodule_stats_test.go index 8b4ea09278..189749ccf6 100644 --- a/sei-db/state_db/sc/flatkv/permodule_stats_test.go +++ b/sei-db/state_db/sc/flatkv/permodule_stats_test.go @@ -139,7 +139,7 @@ func TestPerModuleStatsPersistenceAfterReopen(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -160,7 +160,7 @@ func TestPerModuleStatsPersistenceAfterReopen(t *testing.T) { cfg2 := config.DefaultTestConfig(t) cfg2.DataDir = dbDir - s2, err := NewCommitStore(t.Context(), cfg2) + s2, err := newCommitStoreWithWAL(t.Context(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -202,7 +202,7 @@ func TestPerModuleStatsAfterImportSurvivesRestart(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -238,7 +238,7 @@ func TestPerModuleStatsAfterImportSurvivesRestart(t *testing.T) { cfg2 := config.DefaultTestConfig(t) cfg2.DataDir = dbDir - s2, err := NewCommitStore(t.Context(), cfg2) + s2, err := newCommitStoreWithWAL(t.Context(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) diff --git a/sei-db/state_db/sc/flatkv/snapshot.go b/sei-db/state_db/sc/flatkv/snapshot.go index 1905746d7d..a84ba6e60c 100644 --- a/sei-db/state_db/sc/flatkv/snapshot.go +++ b/sei-db/state_db/sc/flatkv/snapshot.go @@ -14,8 +14,8 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/db_engine/pebbledb" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" - "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" + "github.com/sei-protocol/sei-chain/sei-db/state_db/statewal" "go.opentelemetry.io/otel/metric" ) @@ -562,10 +562,68 @@ func (s *CommitStore) pruneSnapshots(dir string, currentVersion int64) int { return pruned } +// rollbackBaseVersion returns the snapshot version Rollback should rewind to for targetVersion, and reports +// an error if the target cannot be reached from it. A target is reachable when a snapshot at or below it +// exists and either sits exactly on it, or the WAL still holds every block between that snapshot and the +// target. With a nil WAL there is no replay, so only a snapshot sitting exactly on the target qualifies. +// When the only snapshot behind the target is the initial one, the store's history starts at the WAL's first +// block rather than at block 1, so a target is reachable from there iff the WAL holds it. +// +// It reads only: no snapshot, symlink, or WAL state is modified, so Rollback can consult it before touching +// anything and refuse an impossible target outright. +func (s *CommitStore) rollbackBaseVersion(dir string, targetVersion int64) (int64, error) { + if targetVersion < 1 { + return 0, fmt.Errorf("rollback target %d is invalid: version 0 means no state, so there is nothing "+ + "to roll back to", targetVersion) + } + + baseVersion, err := seekSnapshot(dir, targetVersion) + if err != nil { + return 0, fmt.Errorf("seek snapshot for rollback: %w", err) + } + if baseVersion == targetVersion { + return baseVersion, nil + } + + // The snapshot lands below the target, so the WAL has to supply the blocks in between. + if s.wal == nil { + return 0, fmt.Errorf("cannot roll back to version %d: nearest snapshot is %d and this store has no "+ + "WAL to replay the difference", targetVersion, baseVersion) + } + ok, first, last, err := s.wal.GetStoredRange() + if err != nil { + return 0, fmt.Errorf("read WAL range for rollback: %w", err) + } + if !ok { + return 0, fmt.Errorf("cannot roll back to version %d: nearest snapshot is %d and the WAL is empty, "+ + "so blocks %d-%d are unavailable", targetVersion, baseVersion, baseVersion+1, targetVersion) + } + needFrom := uint64(baseVersion) + 1 //nolint:gosec // baseVersion >= 0 + needTo := uint64(targetVersion) //nolint:gosec // targetVersion >= 1 checked above + if baseVersion == 0 { + // Only the initial snapshot sits behind the target, so replay starts at the WAL's first block rather + // than at block 1 — the same clamp catchup and replayInto apply when committedVersion is 0. Below + // that block nothing can rebuild the target: no snapshot names it and the WAL does not reach it. + needFrom = first + if needTo < needFrom { + return 0, fmt.Errorf("cannot roll back to version %d: no snapshot covers it and the WAL "+ + "starts at block %d, so it cannot be reconstructed", targetVersion, first) + } + } + if first > needFrom || last < needTo { + return 0, fmt.Errorf("cannot roll back to version %d: nearest snapshot is %d, so blocks %d-%d are "+ + "needed, but the WAL only holds %d-%d", + targetVersion, baseVersion, needFrom, needTo, first, last) + } + return baseVersion, nil +} + // Rollback restores state to targetVersion by rewinding to the highest // snapshot <= targetVersion, replaying WAL to reach the target, and // truncating all WAL entries and snapshots beyond that point. // +// An unreachable target is rejected before anything is modified. +// // Crash safety: the WAL is truncated BEFORE catchup writes any data to // PebbleDB. If the process crashes after truncation but before catchup // completes, the next restart will simply re-run catchup against the @@ -584,13 +642,15 @@ func (s *CommitStore) Rollback(targetVersion int64) (err error) { dir := s.flatkvDir() - if err := s.closeDBsOnly(); err != nil { - return fmt.Errorf("close before rollback: %w", err) + // Establish reachability first: everything below this point mutates the store irreversibly, and closing + // the DBs would leave it unusable on an early return. + baseVersion, err := s.rollbackBaseVersion(dir, targetVersion) + if err != nil { + return err } - baseVersion, err := seekSnapshot(dir, targetVersion) - if err != nil { - return fmt.Errorf("seek snapshot for rollback: %w", err) + if err := s.closeDBsOnly(); err != nil { + return fmt.Errorf("close before rollback: %w", err) } if err := updateCurrentSymlink(dir, snapshotName(baseVersion)); err != nil { @@ -607,30 +667,25 @@ func (s *CommitStore) Rollback(targetVersion int64) (err error) { return fmt.Errorf("open for rollback: %w", err) } - // Truncate WAL beyond targetVersion BEFORE catchup (crash safety). - if s.changelog != nil { - off, err := s.walOffsetForVersion(targetVersion) - if err != nil { - return fmt.Errorf("compute WAL offset for version %d: %w", targetVersion, err) + // Reset the WAL to targetVersion BEFORE catchup: drop every block after it so a later open-to-latest + // can't replay past target and the write head resumes at targetVersion+1. Rollback is a startup/offline + // operation (no concurrent commits), so rather than mutating a live instance we close the injected WAL, + // prune it offline, and reopen it with its original config. The prune only ever runs for a target + // rollbackBaseVersion already established is reachable, including the case where the target predates every + // retained block and the prune empties the WAL. Skipped when the WAL is nil — the outer context owns it. + if s.wal != nil { + cfg := stateWALConfig(&s.config) + if err := s.wal.Close(); err != nil { + return fmt.Errorf("close WAL for rollback: %w", err) } - if off > 0 { - if err := s.changelog.TruncateAfter(off); err != nil { - return fmt.Errorf("truncate WAL after version %d (offset %d): %w", targetVersion, off, err) - } - if err := s.verifyWALTail(targetVersion); err != nil { - return err - } - } else { - // Target predates all WAL entries; clear the entire WAL to - // prevent re-application. tidwall/wal cannot truncate to empty, - // so we close, delete, and reopen. - lastOff, lErr := s.changelog.LastOffset() - if lErr == nil && lastOff > 0 { - if err := s.clearChangelog(); err != nil { - return fmt.Errorf("clear WAL (target %d predates first entry): %w", targetVersion, err) - } - } + if err := statewal.PruneAfter(cfg, uint64(targetVersion)); err != nil { //nolint:gosec // targetVersion >= 0 + return fmt.Errorf("prune WAL after version %d: %w", targetVersion, err) + } + w, err := statewal.New(cfg) + if err != nil { + return fmt.Errorf("reopen WAL after rollback prune: %w", err) } + s.wal = w } if err := s.catchup(targetVersion); err != nil { @@ -657,37 +712,19 @@ func (s *CommitStore) Rollback(targetVersion int64) (err error) { return nil } -// verifyWALTail checks that the last WAL entry has the expected version. -func (s *CommitStore) verifyWALTail(expectedVersion int64) error { - lastOff, err := s.changelog.LastOffset() - if err != nil { - return fmt.Errorf("verify WAL last offset: %w", err) - } - var lastVer int64 - if err := s.changelog.Replay(lastOff, lastOff, func(_ uint64, entry proto.ChangelogEntry) error { - lastVer = entry.Version - return nil - }); err != nil { - return fmt.Errorf("verify WAL last entry: %w", err) - } - if lastVer != expectedVersion { - return fmt.Errorf("WAL integrity check failed: last entry is version %d, expected %d", lastVer, expectedVersion) - } - return nil -} - -// tryTruncateWAL is a best-effort truncation of WAL entries that are older -// than the earliest snapshot. This prevents unbounded WAL growth while -// keeping enough entries for rollback to any retained snapshot. +// tryTruncateWAL truncates WAL entries older than the earliest snapshot, keeping enough entries for rollback +// to any retained snapshot. Scheduling the truncation is best-effort in that it is skipped when there is +// nothing to prune against, but a prune that fails is not a benign outcome: it only fails when the WAL is +// already dead, which means commits will fail from that point on. func (s *CommitStore) tryTruncateWAL() { - if s.changelog == nil { + if s.wal == nil { return } dir := s.flatkvDir() - // Find the earliest (lowest-version) snapshot — we must keep WAL entries - // from that point onward so rollback to it is possible. + // Find the earliest (lowest-version) snapshot — we must keep WAL blocks from that point onward so + // rollback to it is possible. var earliestSnapVersion int64 _ = traverseSnapshots(dir, true, func(v int64) (bool, error) { earliestSnapVersion = v @@ -697,17 +734,12 @@ func (s *CommitStore) tryTruncateWAL() { return } - off, err := s.walOffsetForVersion(earliestSnapVersion) - if err != nil || off == 0 { - return - } - - firstOff, err := s.changelog.FirstOffset() - if err != nil || off <= firstOff { - return - } - - if err := s.changelog.TruncateBefore(off); err != nil { - logger.Error("failed to truncate WAL", "err", err, "truncateOffset", off) + // Index == version, so prune below the earliest snapshot directly — no offset mapping. + if err := s.wal.Prune(uint64(earliestSnapVersion)); err != nil { //nolint:gosec // earliestSnapVersion > 0 + // A prune only fails when the WAL is already dead or shutting down, so this is not a retryable + // housekeeping miss. Name the consequence: the next commit fails at its WAL write with this same + // cause, and that failure would otherwise look unrelated to this line. + logger.Error("WAL is unusable; FlatKV commits will fail from here", + "err", err, "lowestBlockToKeep", earliestSnapVersion) } } diff --git a/sei-db/state_db/sc/flatkv/snapshot_test.go b/sei-db/state_db/sc/flatkv/snapshot_test.go index 4390e7d48a..228e5532c6 100644 --- a/sei-db/state_db/sc/flatkv/snapshot_test.go +++ b/sei-db/state_db/sc/flatkv/snapshot_test.go @@ -39,7 +39,7 @@ func TestSnapshotCreatesDir(t *testing.T) { dir := t.TempDir() cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -69,7 +69,7 @@ func TestSnapshotIdempotent(t *testing.T) { dir := t.TempDir() cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -92,7 +92,7 @@ func TestOpenFromSnapshot(t *testing.T) { // Phase 1: create store, commit v1 and v2, snapshot at v2, commit v3 cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -112,7 +112,7 @@ func TestOpenFromSnapshot(t *testing.T) { // Phase 2: reopen - should catchup from v2 snapshot + WAL entry for v3 cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -137,7 +137,7 @@ func TestCatchupUpdatesLtHash(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -158,7 +158,7 @@ func TestCatchupUpdatesLtHash(t *testing.T) { // Reopen: catchup from v2 snapshot through v3,v4,v5 via WAL cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -172,7 +172,7 @@ func TestCatchupUpdatesLtHash(t *testing.T) { func TestRollbackRewindsState(t *testing.T) { cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -209,7 +209,7 @@ func TestRollbackRewindsState(t *testing.T) { func TestRollbackToSnapshotExact(t *testing.T) { cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -233,7 +233,7 @@ func TestPartialSnapshotCleanup(t *testing.T) { dir := t.TempDir() cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -295,7 +295,7 @@ func TestMigrationFromFlatLayout(t *testing.T) { // Open the store - should trigger migration cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -329,7 +329,7 @@ func TestOpenVersionValidation(t *testing.T) { // Phase 1: create store, commit some data cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -357,7 +357,7 @@ func TestOpenVersionValidation(t *testing.T) { // Phase 3: reopen - should detect skew and catchup cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -429,7 +429,7 @@ func TestLoadVersionWithTarget(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -445,7 +445,7 @@ func TestLoadVersionWithTarget(t *testing.T) { // Reopen at specific version 3 cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(3, false) require.NoError(t, err) @@ -467,7 +467,7 @@ func TestSnapshotThenCatchupThenVerifyCorrectness(t *testing.T) { // Phase 1: build baseline at v2 and snapshot it. cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -491,7 +491,7 @@ func TestSnapshotThenCatchupThenVerifyCorrectness(t *testing.T) { // baseline in place, we'd incorrectly read 0x04 here. cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(2, false) require.NoError(t, err) @@ -503,7 +503,7 @@ func TestSnapshotThenCatchupThenVerifyCorrectness(t *testing.T) { // Phase 4: reopen latest again to ensure catchup/replay still reaches v4. cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s3, err := NewCommitStore(t.Context(), cfg) + s3, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s3.LoadVersion(0, false) require.NoError(t, err) @@ -526,7 +526,7 @@ func TestLoadVersionMixedSequence(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -544,7 +544,7 @@ func TestLoadVersionMixedSequence(t *testing.T) { // Round 1: load exactly v2 cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(2, false) require.NoError(t, err) @@ -558,7 +558,7 @@ func TestLoadVersionMixedSequence(t *testing.T) { // Round 2: load latest (catches up through v3, v4) cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -572,7 +572,7 @@ func TestLoadVersionMixedSequence(t *testing.T) { // Round 3: load v2 AGAIN — snapshot must still be clean. cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s3, err := NewCommitStore(t.Context(), cfg) + s3, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s3.LoadVersion(2, false) require.NoError(t, err, "LoadVersion(2) must succeed after LoadVersion(0) dirtied working dir") @@ -584,19 +584,20 @@ func TestLoadVersionMixedSequence(t *testing.T) { require.NoError(t, s3.Close()) } -// TestRollbackTargetBeforeWALStart: rollback to a version predating all WAL -// entries. The WAL must be cleared entirely to prevent re-application. -func TestRollbackTargetBeforeWALStart(t *testing.T) { +// TestRollbackToSnapshotVersion: rollback to a version that is exactly a snapshot boundary. The WAL tail +// beyond the target is dropped (via close → offline PruneAfter → reopen) so a restart does not re-apply the +// later blocks and the store stays at the rolled-back version. +func TestRollbackToSnapshotVersion(t *testing.T) { dir := t.TempDir() cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) - // Build: v1..v5, snapshot at v2 + // Build: v1..v5, snapshot at v2. commitStorageEntry(t, s, ktype.Address{0x90}, ktype.Slot{0x01}, []byte{0x01}) commitStorageEntry(t, s, ktype.Address{0x90}, ktype.Slot{0x02}, []byte{0x02}) hashAtV2 := s.RootHash() @@ -606,36 +607,221 @@ func TestRollbackTargetBeforeWALStart(t *testing.T) { commitStorageEntry(t, s, ktype.Address{0x90}, ktype.Slot{0x04}, []byte{0x04}) commitStorageEntry(t, s, ktype.Address{0x90}, ktype.Slot{0x05}, []byte{0x05}) - // Front-truncate WAL so first entry is now v4 (simulates prior pruning). - off, err := s.walOffsetForVersion(4) - require.NoError(t, err) - require.NoError(t, s.changelog.TruncateBefore(off)) - - // Rollback to v2: target predates first WAL entry; should clear WAL - // and land at the v2 snapshot exactly. + // Rollback to v2: lands at the v2 snapshot exactly, with the WAL tail beyond v2 pruned. require.NoError(t, s.Rollback(2)) require.Equal(t, int64(2), s.Version()) require.Equal(t, hashAtV2, s.RootHash()) - // Verify WAL is empty so a restart won't re-apply v4/v5. - firstOff, err := s.changelog.FirstOffset() - require.NoError(t, err) - lastOff, err := s.changelog.LastOffset() + // The WAL must not hold anything above the rolled-back version, or a restart would re-apply v3..v5. + ok, _, last, err := s.wal.GetStoredRange() require.NoError(t, err) - require.True(t, lastOff == 0 || firstOff > lastOff, "WAL should be empty after rollback past WAL start") + require.True(t, !ok || last <= 2, "WAL must not retain blocks above the rolled-back version") + + // A fresh commit resumes contiguously at v3. + commitStorageEntry(t, s, ktype.Address{0x90}, ktype.Slot{0x06}, []byte{0x06}) + require.Equal(t, int64(3), s.Version()) - // Simulate restart: should stay at v2. + // Simulate restart from the rolled-back-then-advanced state: should land at v3. + hashAtV3 := s.RootHash() require.NoError(t, s.Close()) cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) defer s2.Close() - require.Equal(t, int64(2), s2.Version()) - require.Equal(t, hashAtV2, s2.RootHash()) + require.Equal(t, int64(3), s2.Version()) + require.Equal(t, hashAtV3, s2.RootHash()) +} + +// rollbackFixture returns a store with v1..v5 committed and a snapshot at v2. +func rollbackFixture(t *testing.T) *CommitStore { + t.Helper() + cfg := config.DefaultTestConfig(t) + cfg.DataDir = filepath.Join(t.TempDir(), flatkvRootDir) + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + _, err = s.LoadVersion(0, false) + require.NoError(t, err) + t.Cleanup(func() { _ = s.Close() }) + + for i := byte(1); i <= 5; i++ { + commitStorageEntry(t, s, ktype.Address{0x91}, ktype.Slot{i}, []byte{i}) + if i == 2 { + require.NoError(t, s.WriteSnapshot("")) + } + } + return s +} + +// requireRollbackRejected asserts Rollback(target) fails up front with wantErr and leaves the store completely +// untouched: the same snapshot is current, the WAL holds the same blocks, the version has not moved, and the +// store is still usable. wantErr must name a message only the pre-flight check produces — some unreachable +// targets also fail late, after the store has been rewound, so matching the message is what distinguishes +// "refused before touching anything" from "attempted and then reported". The trailing commit catches the +// reachability check drifting below the point where Rollback closes the DBs. +func requireRollbackRejected(t *testing.T, s *CommitStore, target int64, wantErr string) { + t.Helper() + dir := s.flatkvDir() + + _, curBefore, err := currentSnapshotDir(dir) + require.NoError(t, err) + okBefore, firstBefore, lastBefore, err := s.wal.GetStoredRange() + require.NoError(t, err) + versionBefore := s.Version() + + rollbackErr := s.Rollback(target) + require.Error(t, rollbackErr, "an unreachable rollback target must be rejected") + require.Contains(t, rollbackErr.Error(), wantErr, "must be refused by the pre-flight check, not mid-flight") + + _, curAfter, err := currentSnapshotDir(dir) + require.NoError(t, err) + require.Equal(t, curBefore, curAfter, "a rejected rollback must not move the current snapshot") + + okAfter, firstAfter, lastAfter, err := s.wal.GetStoredRange() + require.NoError(t, err) + require.Equal(t, okBefore, okAfter, "a rejected rollback must not touch the WAL") + require.Equal(t, firstBefore, firstAfter, "a rejected rollback must not prune the WAL") + require.Equal(t, lastBefore, lastAfter, "a rejected rollback must not truncate the WAL") + require.Equal(t, versionBefore, s.Version(), "a rejected rollback must not move the version") + + commitStorageEntry(t, s, ktype.Address{0x9F}, ktype.Slot{0xFF}, []byte{0xFF}) + require.Equal(t, versionBefore+1, s.Version(), "the store must remain usable after a rejected rollback") +} + +// TestRollbackRejectsTargetBeyondWALEnd verifies a target above everything the WAL holds is refused up front, +// rather than discovered after the store has already been rewound and pruned. +func TestRollbackRejectsTargetBeyondWALEnd(t *testing.T) { + requireRollbackRejected(t, rollbackFixture(t), 9, "the WAL only holds") +} + +// rollbackFixtureEmptyWALAtV2 returns a store with v1..v2 committed, a snapshot at v2, and an emptied WAL, so +// the caller can choose exactly which blocks the WAL retains. Prune is file-granular and asynchronous, so it +// cannot be used to shape a small WAL; wiping and re-committing is deterministic. +func rollbackFixtureEmptyWALAtV2(t *testing.T) *CommitStore { + t.Helper() + cfg := config.DefaultTestConfig(t) + cfg.DataDir = filepath.Join(t.TempDir(), flatkvRootDir) + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + _, err = s.LoadVersion(0, false) + require.NoError(t, err) + t.Cleanup(func() { _ = s.Close() }) + + commitStorageEntry(t, s, ktype.Address{0x92}, ktype.Slot{0x01}, []byte{0x01}) + commitStorageEntry(t, s, ktype.Address{0x92}, ktype.Slot{0x02}, []byte{0x02}) + require.NoError(t, s.WriteSnapshot("")) + require.NoError(t, s.resetWAL()) + return s +} + +// TestRollbackRejectsTargetTheWALNoLongerCovers verifies that when the snapshot lands below the target and the +// WAL does not reach back to the blocks in between, the rollback is refused instead of landing on the snapshot +// with the intervening blocks destroyed. +func TestRollbackRejectsTargetTheWALNoLongerCovers(t *testing.T) { + s := rollbackFixtureEmptyWALAtV2(t) + + // Resume at v4, so reaching v4 from the v2 snapshot would need block 3, which the WAL never held. + cs := makeChangeSet(evmStorageKey(ktype.Address{0x92}, ktype.Slot{0x04}), padLeft32(0x04), false) + require.NoError(t, s.ApplyChangeSets(4, []*proto.NamedChangeSet{cs})) + _, err := s.Commit(4) + require.NoError(t, err) + + ok, first, _, err := s.wal.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(4), first, "fixture precondition: the WAL must start above the v2 snapshot") + + requireRollbackRejected(t, s, 4, "the WAL only holds") +} + +// TestRollbackRejectsVersionZero verifies version 0 is refused: it means no state, so there is nothing to roll +// back to, and it is the one target that would reach PruneAfter's retains-block-zero boundary. +func TestRollbackRejectsVersionZero(t *testing.T) { + requireRollbackRejected(t, rollbackFixture(t), 0, "nothing to roll back to") +} + +// rollbackFixtureMidChainWALStart returns a store whose only snapshot is the initial one and whose WAL begins +// at block 10 — the topology TestReadOnlyAcceptsMidChainWALStart pins for reads, i.e. a store that started +// mid-chain and so has no history behind its first WAL block. +func rollbackFixtureMidChainWALStart(t *testing.T) *CommitStore { + t.Helper() + cfg := config.DefaultTestConfig(t) + cfg.DataDir = filepath.Join(t.TempDir(), flatkvRootDir) + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + _, err = s.LoadVersion(0, false) + require.NoError(t, err) + t.Cleanup(func() { _ = s.Close() }) + + for _, v := range []int64{10, 11, 12} { + cs := makeChangeSet(evmStorageKey(ktype.Address{0x93}, ktype.Slot{byte(v)}), padLeft32(byte(v)), false) + require.NoError(t, s.ApplyChangeSets(v, []*proto.NamedChangeSet{cs})) + _, err := s.Commit(v) + require.NoError(t, err) + } + + base, err := seekSnapshot(s.flatkvDir(), 11) + require.NoError(t, err) + require.Equal(t, int64(0), base, "fixture precondition: the only snapshot must be the initial one") + ok, first, last, err := s.wal.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(10), first, "fixture precondition: the WAL must begin above block 1") + require.Equal(t, uint64(12), last) + return s +} + +// TestRollbackAcceptsMidChainWALStart verifies a target inside the WAL is reachable when the only snapshot is +// the initial one. catchup treats the WAL's first block as the start of history whenever committedVersion is 0, +// so the reachability pre-flight must apply that same clamp rather than demanding the WAL reach back to block 1. +func TestRollbackAcceptsMidChainWALStart(t *testing.T) { + s := rollbackFixtureMidChainWALStart(t) + + require.NoError(t, s.Rollback(11)) + require.Equal(t, int64(11), s.Version()) + + ok, first, last, err := s.wal.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(10), first, "rollback must not drop blocks at or below the target") + require.Equal(t, uint64(11), last, "rollback must truncate the WAL to the target") +} + +// TestRollbackRejectsTargetBelowMidChainWALStart verifies a target below the WAL's first block stays an up-front +// rejection. With only the initial snapshot there is no state at such a target, and PruneAfter(target) would +// drop every block the WAL holds, so discovering it after the rewind would destroy the store's whole history. +func TestRollbackRejectsTargetBelowMidChainWALStart(t *testing.T) { + requireRollbackRejected(t, rollbackFixtureMidChainWALStart(t), 5, "cannot be reconstructed") +} + +// TestRollbackToTargetPredatingWALSucceeds covers the truncate-to-empty case: the target sits exactly on a +// snapshot and below every block the WAL still holds, so pruning to it empties the WAL. No replay is needed, +// so this is reachable and must succeed. +func TestRollbackToTargetPredatingWALSucceeds(t *testing.T) { + s := rollbackFixtureEmptyWALAtV2(t) + + // The WAL holds only blocks above the v2 snapshot, so rolling back to v2 must empty it. + for i := byte(3); i <= 5; i++ { + commitStorageEntry(t, s, ktype.Address{0x92}, ktype.Slot{i}, []byte{i}) + } + ok, first, _, err := s.wal.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(3), first) + + require.NoError(t, s.Rollback(2)) + require.Equal(t, int64(2), s.Version()) + + ok, _, _, err = s.wal.GetStoredRange() + require.NoError(t, err) + require.False(t, ok, "pruning to a target below every retained block must empty the WAL") + + // The emptied WAL accepts the next contiguous block, so the store keeps working. + commitStorageEntry(t, s, ktype.Address{0x92}, ktype.Slot{0x06}, []byte{0x06}) + require.Equal(t, int64(3), s.Version()) } // ============================================================================= @@ -829,7 +1015,7 @@ func TestPruneSnapshotsKeepsRecent(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(t.TempDir(), flatkvRootDir) cfg.SnapshotKeepRecent = 1 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -854,7 +1040,7 @@ func TestPruneSnapshotsKeepsRecent(t *testing.T) { func TestPruneSnapshotsKeepAll(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.SnapshotKeepRecent = 100 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -892,7 +1078,7 @@ func TestOrphanSnapshotRecovery(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -949,95 +1135,6 @@ func TestTraverseSnapshotsEarlyStop(t *testing.T) { require.Equal(t, int64(20), visited[0], "descending should visit highest first") } -// ============================================================================= -// verifyWALTail -// ============================================================================= - -func TestVerifyWALTailSuccess(t *testing.T) { - cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) - require.NoError(t, err) - _, err = s.LoadVersion(0, false) - require.NoError(t, err) - defer s.Close() - - commitStorageEntry(t, s, ktype.Address{0x01}, ktype.Slot{0x01}, []byte{0x01}) - commitStorageEntry(t, s, ktype.Address{0x01}, ktype.Slot{0x02}, []byte{0x02}) - commitStorageEntry(t, s, ktype.Address{0x01}, ktype.Slot{0x03}, []byte{0x03}) - - require.NoError(t, s.verifyWALTail(3)) -} - -func TestVerifyWALTailMismatch(t *testing.T) { - cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) - require.NoError(t, err) - _, err = s.LoadVersion(0, false) - require.NoError(t, err) - defer s.Close() - - commitStorageEntry(t, s, ktype.Address{0x01}, ktype.Slot{0x01}, []byte{0x01}) - commitStorageEntry(t, s, ktype.Address{0x01}, ktype.Slot{0x02}, []byte{0x02}) - - err = s.verifyWALTail(5) - require.Error(t, err) - require.Contains(t, err.Error(), "WAL integrity check failed") -} - -// ============================================================================= -// tryTruncateWAL -// ============================================================================= - -func TestTryTruncateWAL(t *testing.T) { - dir := t.TempDir() - // SnapshotKeepRecent=0 so pruneSnapshots removes snapshot-0 once - // the manual snapshot at v5 is created; this makes v5 the earliest - // snapshot and gives tryTruncateWAL a positive truncation offset. - cfg := config.DefaultTestConfig(t) - cfg.DataDir = filepath.Join(dir, flatkvRootDir) - cfg.SnapshotKeepRecent = 0 - s, err := NewCommitStore(t.Context(), cfg) - require.NoError(t, err) - _, err = s.LoadVersion(0, false) - require.NoError(t, err) - defer s.Close() - - for i := 0; i < 5; i++ { - commitStorageEntry(t, s, ktype.Address{byte(i + 1)}, ktype.Slot{byte(i + 1)}, []byte{byte(i + 1)}) - } - - require.NoError(t, s.WriteSnapshot("")) - - for i := 5; i < 10; i++ { - commitStorageEntry(t, s, ktype.Address{byte(i + 1)}, ktype.Slot{byte(i + 1)}, []byte{byte(i + 1)}) - } - - firstBefore, _ := s.changelog.FirstOffset() - - s.tryTruncateWAL() - - firstAfter, _ := s.changelog.FirstOffset() - require.Greater(t, firstAfter, firstBefore, "WAL should be truncated after snapshot") -} - -func TestTryTruncateWALNoSnapshot(t *testing.T) { - cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) - require.NoError(t, err) - _, err = s.LoadVersion(0, false) - require.NoError(t, err) - defer s.Close() - - commitStorageEntry(t, s, ktype.Address{0x01}, ktype.Slot{0x01}, []byte{0x01}) - - firstBefore, _ := s.changelog.FirstOffset() - - s.tryTruncateWAL() - - firstAfter, _ := s.changelog.FirstOffset() - require.Equal(t, firstBefore, firstAfter, "no snapshot means no truncation") -} - // ============================================================================= // Rollback removes post-target snapshots // ============================================================================= @@ -1046,7 +1143,7 @@ func TestRollbackRemovesPostTargetSnapshots(t *testing.T) { dir := t.TempDir() cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1148,7 +1245,7 @@ func TestMultipleSnapshotsAndReopen(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) cfg.SnapshotKeepRecent = 10 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1166,7 +1263,7 @@ func TestMultipleSnapshotsAndReopen(t *testing.T) { cfg2 := config.DefaultTestConfig(t) cfg2.DataDir = filepath.Join(dir, flatkvRootDir) cfg2.SnapshotKeepRecent = 10 - s2, err := NewCommitStore(t.Context(), cfg2) + s2, err := newCommitStoreWithWAL(t.Context(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(ver, false) require.NoError(t, err) @@ -1184,7 +1281,7 @@ func TestWriteSnapshotUpdatesSnapshotBase(t *testing.T) { dir := t.TempDir() cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(context.Background(), cfg) + s, err := newCommitStoreWithWAL(context.Background(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1213,7 +1310,7 @@ func TestWriteSnapshotUpdatesSnapshotBase(t *testing.T) { // (from the snapshot). Catchup should replay 0 entries. cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s2, err := NewCommitStore(context.Background(), cfg) + s2, err := newCommitStoreWithWAL(context.Background(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1227,7 +1324,7 @@ func TestSnapshotPreservesAllKeyTypes(t *testing.T) { dir := t.TempDir() cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1251,7 +1348,7 @@ func TestSnapshotPreservesAllKeyTypes(t *testing.T) { cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1286,7 +1383,7 @@ func TestReopenAfterEmptyCommits(t *testing.T) { cfg := config.DefaultConfig() cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1303,7 +1400,7 @@ func TestReopenAfterEmptyCommits(t *testing.T) { cfg2 := config.DefaultConfig() cfg2.DataDir = dbDir - s2, err := NewCommitStore(context.Background(), cfg2) + s2, err := newCommitStoreWithWAL(context.Background(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1323,7 +1420,7 @@ func TestReopenAfterDeletes(t *testing.T) { cfg := config.DefaultConfig() cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1363,7 +1460,7 @@ func TestReopenAfterDeletes(t *testing.T) { cfg2 := config.DefaultConfig() cfg2.DataDir = dbDir - s2, err := NewCommitStore(context.Background(), cfg2) + s2, err := newCommitStoreWithWAL(context.Background(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1399,7 +1496,7 @@ func TestWALTruncationThenRollback(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.SnapshotInterval = 5 cfg.SnapshotKeepRecent = 1 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1441,7 +1538,7 @@ func TestReopenAfterSnapshotAndTruncation(t *testing.T) { cfg.SnapshotInterval = 5 cfg.SnapshotKeepRecent = 1 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1454,7 +1551,7 @@ func TestReopenAfterSnapshotAndTruncation(t *testing.T) { hashBefore := s.RootHash() require.NoError(t, s.Close()) - s2, err := NewCommitStore(context.Background(), cfg) + s2, err := newCommitStoreWithWAL(context.Background(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1483,7 +1580,7 @@ func TestSingleDBOpenFailure(t *testing.T) { cfg := config.DefaultConfig() cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1505,7 +1602,7 @@ func TestSingleDBOpenFailure(t *testing.T) { cfg2 := config.DefaultConfig() cfg2.DataDir = dbDir - s2, err := NewCommitStore(context.Background(), cfg2) + s2, err := newCommitStoreWithWAL(context.Background(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.Error(t, err, "open should fail when storageDB is corrupted in both working and snapshot") @@ -1521,7 +1618,7 @@ func TestGlobalMetadataCorruption(t *testing.T) { cfg := config.DefaultConfig() cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1550,7 +1647,7 @@ func TestGlobalMetadataCorruption(t *testing.T) { cfg2 := config.DefaultConfig() cfg2.DataDir = dbDir - s2, err := NewCommitStore(context.Background(), cfg2) + s2, err := newCommitStoreWithWAL(context.Background(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.Error(t, err, "open should fail when global metadata is corrupted") @@ -1566,7 +1663,7 @@ func TestWALDirectoryDeleted(t *testing.T) { cfg := config.DefaultConfig() cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1581,7 +1678,7 @@ func TestWALDirectoryDeleted(t *testing.T) { cfg2 := config.DefaultConfig() cfg2.DataDir = dbDir - s2, err := NewCommitStore(context.Background(), cfg2) + s2, err := newCommitStoreWithWAL(context.Background(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1604,7 +1701,7 @@ func TestLocalMetaCorruption(t *testing.T) { cfg := config.DefaultConfig() cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1637,7 +1734,7 @@ func TestLocalMetaCorruption(t *testing.T) { cfg2 := config.DefaultConfig() cfg2.DataDir = dbDir - s2, err := NewCommitStore(context.Background(), cfg2) + s2, err := newCommitStoreWithWAL(context.Background(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.Error(t, err, "open should fail when meta version is corrupted") @@ -1654,7 +1751,7 @@ func TestWALSegmentCorruption(t *testing.T) { cfg := config.DefaultConfig() cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1696,7 +1793,7 @@ func TestWALSegmentCorruption(t *testing.T) { // Request version 2: global says v1, WAL auto-truncated (empty), can't catchup to v2. cfg2 := config.DefaultConfig() cfg2.DataDir = dbDir - s2, err := NewCommitStore(context.Background(), cfg2) + s2, err := newCommitStoreWithWAL(context.Background(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(2, false) require.Error(t, err, "LoadVersion should fail: WAL corrupted, can't reach requested version") @@ -1713,7 +1810,7 @@ func TestAccountRowDeletePersistsAfterReopen(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s, err := NewCommitStore(context.Background(), cfg) + s, err := newCommitStoreWithWAL(context.Background(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1748,7 +1845,7 @@ func TestAccountRowDeletePersistsAfterReopen(t *testing.T) { hashBefore := s.RootHash() require.NoError(t, s.Close()) - s2, err := NewCommitStore(context.Background(), cfg) + s2, err := newCommitStoreWithWAL(context.Background(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1768,7 +1865,7 @@ func TestAccountRowDeleteSurvivesWALReplay(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir - s, err := NewCommitStore(context.Background(), cfg) + s, err := newCommitStoreWithWAL(context.Background(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1808,7 +1905,7 @@ func TestAccountRowDeleteSurvivesWALReplay(t *testing.T) { require.NoError(t, mdb.Set(ktype.MetaVersionKey, versionBuf, types.WriteOptions{Sync: true})) require.NoError(t, mdb.Close()) - s2, err := NewCommitStore(context.Background(), cfg) + s2, err := newCommitStoreWithWAL(context.Background(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1829,7 +1926,7 @@ func TestAccountRowDeleteAfterSnapshotRollback(t *testing.T) { cfg.SnapshotInterval = 1 cfg.SnapshotKeepRecent = 2 - s, err := NewCommitStore(context.Background(), cfg) + s, err := newCommitStoreWithWAL(context.Background(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -2002,7 +2099,7 @@ func TestRollbackPreservesWALContinuity(t *testing.T) { cfg.DataDir = filepath.Join(dir, flatkvRootDir) cfg.SnapshotInterval = 2 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -2030,7 +2127,7 @@ func TestRollbackPreservesWALContinuity(t *testing.T) { require.NoError(t, s.Close()) // Reopen and verify WAL continuity is intact. - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) diff --git a/sei-db/state_db/sc/flatkv/store.go b/sei-db/state_db/sc/flatkv/store.go index 7c86351862..16886d8342 100644 --- a/sei-db/state_db/sc/flatkv/store.go +++ b/sei-db/state_db/sc/flatkv/store.go @@ -27,7 +27,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/vtype" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" - "github.com/sei-protocol/sei-chain/sei-db/wal" + "github.com/sei-protocol/sei-chain/sei-db/state_db/statewal" "github.com/sei-protocol/seilog" ) @@ -172,14 +172,24 @@ type CommitStore struct { storageWrites map[string]*vtype.StorageData miscWrites map[string]*vtype.MiscData - changelog wal.ChangelogWAL + // The state WAL. Injected at construction: non-nil ⇒ FlatKV writes/replays/prunes it; nil ⇒ the outer + // context owns the whole WAL pipeline and FlatKV no-ops every WAL operation. FlatKV owns Close of whatever + // instance it currently holds (rollback/restore may replace it via close→prune/delete→reopen); its + // lifecycle is decoupled from the DB open/close cycle, so closeDBsOnly does not touch it. + wal statewal.StateWAL + + // Whether FlatKV manages a WAL (a non-nil instance was injected at construction). It records the intent + // even across a Close that nil-es the live instance, so import reset can reconstruct the WAL rather than + // mistaking a closed-but-owned WAL for the nil "outer context owns it" case. + manageWAL bool // Changes to feed into the WAL at the next commit. pendingChangeSets []*proto.NamedChangeSet // pendingBlockHeight is the version stamped by the current buffered - // ApplyChangeSets. 0 means no pending apply. Commit requires version to - // match when this is non-zero. + // ApplyChangeSets. 0 means no pending apply. Further ApplyChangeSets + // calls and Commit both require version to match when this is non-zero: + // only one block may be buffered per commit. pendingBlockHeight int64 lastSnapshotTime time.Time @@ -276,6 +286,7 @@ func (s *CommitStore) routePhysicalKey(physicalKey []byte) (seidbtypes.KeyValueD func NewCommitStore( ctx context.Context, cfg *config.Config, + stateWAL statewal.StateWAL, ) (*CommitStore, error) { InitializeDataDirectories(cfg) @@ -318,6 +329,8 @@ func NewCommitStore( miscPool: miscPool, ltHashPool: ltHashPool, ltCalc: ltCalc, + wal: stateWAL, + manageWAL: stateWAL != nil, }, nil } @@ -357,6 +370,7 @@ var errReadOnly = errors.New("flatkv: store is read-only") // LoadVersion opens the database at the given version (0 = latest). // When readOnly is true an isolated, read-only CommitStore is returned; // the caller must Close it when done. +// See the Store interface for this method's sharp edges. func (s *CommitStore) LoadVersion(targetVersion int64, readOnly bool) (opened Store, retErr error) { logger.Info("FlatKV LoadVersion", "targetVersion", targetVersion, "readOnly", readOnly) obs := s.observeOp("LoadVersion", otelMetrics.OpenLatency, @@ -398,6 +412,9 @@ func (s *CommitStore) LoadVersion(targetVersion int64, readOnly bool) (opened St } }() + // TODO: this should also truncate the WAL to targetVersion, the way Rollback + // does, so the loaded store can go on committing instead of being read-only + // in practice. if targetVersion > 0 { if err := os.MkdirAll(dir, 0750); err != nil { return nil, fmt.Errorf("create flatkv dir: %w", err) @@ -448,15 +465,8 @@ func (s *CommitStore) loadVersionReadOnly(targetVersion int64) (_ Store, retErr return nil, fmt.Errorf("loadVersionReadOnly: pre-init cleanup: %w", err) } } - // Give the clone an independent context rather than deriving it from the - // parent (s.ctx). The read-only store owns its own resources (thread pools, - // pebble handles) and is torn down by its own Close, which cancels the - // context NewCommitStore derives here. Rooting it at s.ctx instead would - // cancel the clone's context — and abort in-flight reads with "context - // canceled" — the moment the parent is closed, even though the caller may - // still be using the returned view. The clone's lifecycle is therefore - // decoupled from the parent's. - ro, err := NewCommitStore(context.Background(), &s.config) + + ro, err := NewCommitStore(context.Background(), &s.config, nil) if err != nil { return nil, fmt.Errorf("failed to create readonly store: %w", err) } @@ -492,12 +502,115 @@ func (s *CommitStore) loadVersionReadOnly(targetVersion int64) (_ Store, retErr return nil, fmt.Errorf("readonly open: %w", err) } + // The clone is open at a snapshot boundary with a nil WAL. Replay this (primary) store's WAL into it up + // to targetVersion so it reflects the exact requested height. The clone is not yet marked read-only, so + // the replay's ApplyChangeSets calls are permitted; mark it read-only only once replay succeeds. + if err := s.replayInto(ro, targetVersion); err != nil { + return nil, err + } + + if targetVersion > 0 && ro.committedVersion != targetVersion { + return nil, fmt.Errorf("readonly version mismatch: requested %d, reached %d", + targetVersion, ro.committedVersion) + } + + ro.readOnly = true + + logger.Info("FlatKV readonly store opened", "version", ro.committedVersion, "dir", ro.readOnlyWorkDir) return ro, nil } -// openReadOnly opens PebbleDBs in readOnlyWorkDir, replays the WAL to -// targetVersion, then closes the WAL and marks the store as read-only. -// It never modifies the global "current" symlink. +// replayInto replays this store's WAL into a read-only clone, advancing the clone from the snapshot +// boundary it opened at up to targetVersion (or this store's latest WAL block when targetVersion <= 0). It +// exists because the clone has a nil WAL: the primary owns the WAL, so the primary reads it and feeds each +// block into the clone via applyAndCommit (which never touches a WAL). +// +// The WAL must still reach back to the block after the clone's snapshot boundary. If it begins later than +// that, the blocks the clone needs are gone and this fails rather than serving a clone with a hole in it. +// +// Concurrency: export runs in a background goroutine while this (primary) store may still be committing. +// The iterator is constructed under s.mu, serializing against a concurrent Commit's WAL-wrapper access; +// iteration then proceeds lock-free, because a seiwal iterator reads a consistent point-in-time (hard-link) +// snapshot that concurrent appends/prunes cannot disturb. A concurrent Commit is the only overlap this +// tolerates: closing the primary while an export is in flight is not permitted (see Close). +func (s *CommitStore) replayInto(clone *CommitStore, targetVersion int64) (retErr error) { + if s.wal == nil { + // nil WAL: the outer context owns the pipeline, so no between-snapshot replay is available here. The + // clone can only serve the snapshot boundary it opened at. + if targetVersion > 0 && clone.committedVersion != targetVersion { + return fmt.Errorf("readonly: nil WAL cannot replay to version %d (opened at %d)", + targetVersion, clone.committedVersion) + } + return nil + } + + s.mu.Lock() + ok, first, last, err := s.wal.GetStoredRange() + if err != nil { + s.mu.Unlock() + return fmt.Errorf("readonly: WAL range: %w", err) + } + if !ok { + s.mu.Unlock() + return nil // empty WAL: nothing to replay + } + + // Replay from the block after the snapshot boundary the clone opened at. A clone at version 0 has no + // history behind it, so the WAL's first block is where this store's history begins rather than a gap — + // the same reasoning as catchup, which this must match or a store whose first block is above 1 (chain + // initial_height > 1) would be rejected as gapped. + start := first + if clone.committedVersion > 0 { + start = uint64(clone.committedVersion) + 1 //nolint:gosec // committedVersion > 0 checked above + } + end := last + if targetVersion > 0 && uint64(targetVersion) < end { + end = uint64(targetVersion) + } + if end < start { + s.mu.Unlock() + return nil // clone already at or beyond target + } + if first > start { + // We are about to replay, but the primary's WAL no longer reaches back to this clone's snapshot, so + // the blocks the clone needs are gone. Only the export fails — the primary's own state is untouched. + s.mu.Unlock() + return fmt.Errorf("readonly: WAL starts at block %d but the clone needs block %d: "+ + "blocks %d-%d are missing (data loss or corruption)", first, start, start, first-1) + } + it, err := s.wal.Iterator(start, end) + s.mu.Unlock() + if err != nil { + return fmt.Errorf("readonly: WAL iterator [%d,%d]: %w", start, end, err) + } + defer func() { + if cerr := it.Close(); cerr != nil && retErr == nil { + retErr = fmt.Errorf("readonly: close WAL iterator: %w", cerr) + } + }() + + for { + hasNext, nErr := it.Next() + if nErr != nil { + return fmt.Errorf("readonly: WAL iterate: %w", nErr) + } + if !hasNext { + break + } + block, changesets := it.Entry() + if err := clone.applyAndCommit(int64(block), changesets); err != nil { //nolint:gosec // block <= end + return fmt.Errorf("readonly: replay block %d: %w", block, err) + } + } + return nil +} + +// openReadOnly opens PebbleDBs in readOnlyWorkDir at the snapshot boundary at or below targetVersion, +// leaving committedVersion at that snapshot version. It never modifies the global "current" symlink. +// +// This clone has a nil WAL of its own, so it does NOT replay: advancing from the snapshot boundary up to +// targetVersion — and marking the store read-only — is driven by the primary via loadVersionReadOnly / +// replayInto, which feeds the primary's WAL into this clone. func (s *CommitStore) openReadOnly(targetVersion int64) error { s.clearPendingWrites() @@ -522,7 +635,7 @@ func (s *CommitStore) openReadOnly(targetVersion int64) error { return fmt.Errorf("create readonly working dir: %w", err) } - if err := s.openDBs(s.readOnlyWorkDir, dir); err != nil { + if err := s.openDBs(s.readOnlyWorkDir); err != nil { return err } @@ -530,26 +643,7 @@ func (s *CommitStore) openReadOnly(targetVersion int64) error { return err } - if err := s.catchup(targetVersion); err != nil { - return fmt.Errorf("readonly catchup: %w", err) - } - - if targetVersion > 0 && s.committedVersion != targetVersion { - return fmt.Errorf("readonly version mismatch: requested %d, reached %d", - targetVersion, s.committedVersion) - } - - if s.changelog != nil { - closeErr := s.changelog.Close() - s.changelog = nil - if closeErr != nil { - return fmt.Errorf("close readonly changelog: %w", closeErr) - } - } - - s.readOnly = true - - logger.Info("FlatKV readonly store opened", "version", s.committedVersion, + logger.Info("FlatKV readonly base opened", "version", s.committedVersion, "dir", s.readOnlyWorkDir) return nil } @@ -619,7 +713,7 @@ func (s *CommitStore) open() (retErr error) { return fmt.Errorf("create working dir: %w", err) } - if err := s.openDBs(workDir, dir); err != nil { + if err := s.openDBs(workDir); err != nil { return err } @@ -666,9 +760,12 @@ func (s *CommitStore) openPebbleDB(cfg *pebbledb.PebbleDBConfig, cacheCfg *dbcac return db, nil } -// openDBs opens all PebbleDBs from dbDir and optionally the changelog WAL -// from changelogRoot. On failure all already-opened handles are closed. -func (s *CommitStore) openDBs(dbDir, changelogRoot string) (retErr error) { +// openDBs opens all PebbleDBs from dbDir. On failure all already-opened handles are closed. +// +// It does not touch the WAL: the WAL is injected at construction and its lifecycle is decoupled from the +// DB open/close cycle (it must survive LoadVersion/Rollback DB reopens), so it is neither opened nor +// cleared here. +func (s *CommitStore) openDBs(dbDir string) (retErr error) { var toClose []io.Closer defer func() { @@ -681,7 +778,6 @@ func (s *CommitStore) openDBs(dbDir, changelogRoot string) (retErr error) { s.codeDB = nil s.storageDB = nil s.miscDB = nil - s.changelog = nil s.localMeta = make(map[string]*ktype.LocalMeta) } }() @@ -717,19 +813,6 @@ func (s *CommitStore) openDBs(dbDir, changelogRoot string) (retErr error) { } toClose = append(toClose, s.metadataDB) - if changelogRoot != "" { - changelogPath := filepath.Join(changelogRoot, changelogDir) - s.changelog, err = wal.NewChangelogWAL(changelogPath, wal.Config{ - WriteBufferSize: 0, - KeepRecent: 0, - PruneInterval: 0, - }) - if err != nil { - return fmt.Errorf("failed to open changelog: %w", err) - } - toClose = append(toClose, s.changelog) - } - for _, ndb := range s.namedDataDBs() { meta, err := loadLocalMeta(ndb.db) if err != nil { @@ -798,36 +881,12 @@ func (s *CommitStore) loadGlobalMetadata() error { return nil } -// clearChangelog closes the WAL, removes its directory, and reopens an empty -// WAL. Used by Rollback when the target version predates all WAL entries and -// the entire log must be discarded to prevent re-application on restart. -func (s *CommitStore) clearChangelog() error { - if s.changelog == nil { - return nil - } - dir := filepath.Join(s.flatkvDir(), changelogDir) - if err := s.changelog.Close(); err != nil { - return fmt.Errorf("close changelog: %w", err) - } - s.changelog = nil - if err := os.RemoveAll(dir); err != nil { - return fmt.Errorf("remove changelog dir: %w", err) - } - var err error - s.changelog, err = wal.NewChangelogWAL(dir, wal.Config{}) - if err != nil { - return fmt.Errorf("reopen changelog: %w", err) - } - return nil -} - func (s *CommitStore) Version() int64 { return s.committedVersion } -// PendingVersion returns s.pendingBlockHeight: the height stamped by the most -// recent ApplyChangeSets call since the last Commit, or 0 when there are no -// buffered writes. +// PendingVersion returns s.pendingBlockHeight: the height of the block currently +// buffered by ApplyChangeSets, or 0 when there are no buffered writes. func (s *CommitStore) PendingVersion() int64 { return s.pendingBlockHeight } @@ -911,8 +970,11 @@ func (s *CommitStore) resetForImport() error { return fmt.Errorf("resetForImport: remove %s: %w", currentLink, err) } - if err := atomicRemoveDir(filepath.Join(dir, changelogDir)); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("resetForImport: remove %s: %w", changelogDir, err) + // Reset the WAL through the injected instance rather than removing its directory directly: the instance + // is still open here, so a raw removal would strand it (and break its Close). resetWAL wipes it to a + // clean empty log aligned with the import; it is a no-op when the outer context owns the WAL (nil). + if err := s.resetWAL(); err != nil { + return fmt.Errorf("resetForImport: reset WAL: %w", err) } // Reopen from a pristine empty state. open() will load metadata @@ -931,6 +993,35 @@ func (s *CommitStore) resetForImport() error { return nil } +// resetWAL discards this store's WAL and reopens an empty one, so a restore starts from a clean WAL aligned +// with the imported snapshot (called by resetForImport). Import bypasses the WAL, so any pre-existing WAL is +// stale relative to the imported version; under statewal's contiguity a stale non-empty WAL (a re-syncing +// node's old-chain entries) would reject the next commit. Wiping is a no-op on a fresh node and the fix on +// a re-sync. +// +// When the store does not manage a WAL (constructed with nil — the outer context owns the pipeline) this is +// a no-op. The instance may already be closed here, since rootmulti.Restore closes the store before +// importing; closing twice is a no-op. On failure the store keeps the closed instance rather than dropping to +// nil, so a later write fails loudly instead of being skipped. +func (s *CommitStore) resetWAL() error { + if !s.manageWAL { + return nil + } + cfg := stateWALConfig(&s.config) + if err := s.wal.Close(); err != nil { + return fmt.Errorf("close WAL for reset: %w", err) + } + if err := statewal.Delete(cfg); err != nil { + return fmt.Errorf("delete WAL for reset: %w", err) + } + w, err := statewal.New(cfg) + if err != nil { + return fmt.Errorf("reopen WAL after reset: %w", err) + } + s.wal = w + return nil +} + func (s *CommitStore) GetPhaseTimer() *metrics.PhaseTimer { return s.phaseTimer } diff --git a/sei-db/state_db/sc/flatkv/store_apply.go b/sei-db/state_db/sc/flatkv/store_apply.go index c9a681ad90..2dbc0ed488 100644 --- a/sei-db/state_db/sc/flatkv/store_apply.go +++ b/sei-db/state_db/sc/flatkv/store_apply.go @@ -35,21 +35,15 @@ func (s *CommitStore) ApplyChangeSets(version int64, changeSets []*proto.NamedCh return fmt.Errorf("flatkv: apply version %d must be ahead of committed version %d", version, s.committedVersion) } - // Two legitimate multi-call patterns must both be accepted here: - // - Same-height repeats: a single block's writes may arrive across - // several ApplyChangeSets calls at the same height (e.g. a - // ModuleRouter fanning one block's changesets out to multiple - // routes that all target flatKV). - // - Strictly increasing heights: several blocks' writes may be - // batched before a single Commit (e.g. a benchmark harness with - // BlocksPerCommit > 1), each call stamping its own rows with its - // own height. - // Only a height that goes backwards indicates a bug. - // Commit(version) below requires version == pendingBlockHeight, i.e. - // the highest height applied since the last Commit. - if s.pendingBlockHeight != 0 && version < s.pendingBlockHeight { - return fmt.Errorf("flatkv: cannot apply at height %d; pending writes already stamped up to %d", - version, s.pendingBlockHeight) + // A single block's writes may arrive across several ApplyChangeSets calls at the same height (e.g. a + // ModuleRouter fanning one block's changesets out to multiple routes that all target flatKV), so + // same-height repeats are accepted. Any other height is a bug: batching several blocks before one + // Commit is not supported, because changesets carry no block number and the batch would collapse into + // a single WAL entry at its highest height — see the ApplyChangeSets contract in api.go. Rejecting the + // jump here rather than letting the WAL reject the following Commit keeps the error at the caller. + if s.pendingBlockHeight != 0 && version != s.pendingBlockHeight { + return fmt.Errorf("flatkv: cannot apply at height %d; pending writes are stamped at %d and only "+ + "one block may be buffered per commit", version, s.pendingBlockHeight) } s.phaseTimer.SetPhase("apply_change_sets_prepare") diff --git a/sei-db/state_db/sc/flatkv/store_catchup.go b/sei-db/state_db/sc/flatkv/store_catchup.go index 55cf2cc52a..83e8074585 100644 --- a/sei-db/state_db/sc/flatkv/store_catchup.go +++ b/sei-db/state_db/sc/flatkv/store_catchup.go @@ -6,227 +6,130 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/proto" ) -// walOffsetForVersion returns the WAL offset whose entry has the given version. -// Returns 0 if the WAL is empty or the version predates all WAL entries. -// -// Strategy: try the arithmetic shortcut (O(1) reads) first -- it works when -// each version maps 1:1 to a sequential offset. On mismatch, fall back to -// binary search (O(log N) reads) which handles gaps and batched versions. -func (s *CommitStore) walOffsetForVersion(version int64) (uint64, error) { - if s.changelog == nil { - return 0, fmt.Errorf("changelog not open") - } - firstOff, err := s.changelog.FirstOffset() - if err != nil { - return 0, fmt.Errorf("WAL first offset: %w", err) - } - if firstOff == 0 { - return 0, nil - } - lastOff, err := s.changelog.LastOffset() - if err != nil { - return 0, fmt.Errorf("WAL last offset: %w", err) - } - if lastOff == 0 || firstOff > lastOff { - return 0, nil - } - - firstVer, err := s.walVersionAtOffset(firstOff) - if err != nil { - return 0, fmt.Errorf("read first WAL entry: %w", err) - } - if firstVer <= 0 || version < firstVer { - return 0, nil - } - - // Fast path: O(1) arithmetic guess. - guess := firstOff + uint64(version-firstVer) //nolint:gosec // version >= firstVer checked above - if guess >= firstOff && guess <= lastOff { - if v, err := s.walVersionAtOffset(guess); err == nil && v == version { - return guess, nil - } - } - - // Slow path: binary search over [firstOff, lastOff]. - lo, hi := firstOff, lastOff - for lo <= hi { - mid := lo + (hi-lo)/2 - v, err := s.walVersionAtOffset(mid) - if err != nil { - return 0, fmt.Errorf("WAL binary search at offset %d: %w", mid, err) - } - switch { - case v == version: - return mid, nil - case v < version: - lo = mid + 1 - default: - if mid == 0 { - break - } - hi = mid - 1 - } - } - return 0, fmt.Errorf("WAL version %d not found (range %d-%d)", version, firstOff, lastOff) -} - -// walVersionAtOffset reads a single WAL entry and returns its version. -func (s *CommitStore) walVersionAtOffset(off uint64) (int64, error) { - var ver int64 - err := s.changelog.Replay(off, off, func(_ uint64, entry proto.ChangelogEntry) error { - ver = entry.Version - return nil - }) - return ver, err +// applyAndCommit replays a single block into the store: it applies the changesets, commits the per-DB +// batches, advances the committed version, clones the working LtHash to committed, and clears the pending +// buffers. It is the shared per-block step used both by catchup (replaying this store's own WAL) and by +// read-only export replay (replaying the primary's WAL into a clone, see replayInto). It never touches the +// WAL — the data being applied was itself read from a WAL, so re-writing it would double-append. +func (s *CommitStore) applyAndCommit(version int64, changesets []*proto.NamedChangeSet) error { + if err := s.ApplyChangeSets(version, changesets); err != nil { + return fmt.Errorf("apply v%d: %w", version, err) + } + if err := s.commitBatches(version); err != nil { + return fmt.Errorf("commit v%d: %w", version, err) + } + s.committedVersion = version + s.committedLtHash = s.workingLtHash.Clone() + s.clearPendingWrites() + recordPendingWrites(s.ctx, accountDBDir, 0) + recordPendingWrites(s.ctx, codeDBDir, 0) + recordPendingWrites(s.ctx, storageDBDir, 0) + recordPendingWrites(s.ctx, miscDBDir, 0) + return nil } -// catchup replays WAL entries from the current committedVersion up to (and -// including) targetVersion. If targetVersion <= 0, replay continues to the -// end of the WAL. +// catchup replays this store's WAL from the current committedVersion up to (and including) targetVersion. +// If targetVersion <= 0, replay continues to the end of the WAL. Each block runs through applyAndCommit; a +// single global-metadata commit follows. +// +// catchup runs at startup (open/openTo) and during Rollback — never concurrently with live commits — so it +// reads s.wal without additional locking. (Concurrent read for state-sync export goes through replayInto, +// which locks around iterator construction.) // -// Each replayed entry runs through ApplyChangeSets(entry.Version, ...) (so -// row last-modified heights match the WAL commit version) and commitBatches. -// After all entries are replayed, global metadata is flushed once. +// The WAL must still reach back to the block after committedVersion. If it begins later than that, the +// blocks in between are gone and catchup fails rather than silently skipping them. // -// Version gaps are allowed: Commit/CommitBlock may persist non-sequential -// heights (batch last-block height, empty commits that jump ahead), so the -// next WAL entry may be strictly greater than committedVersion+1. Replay -// still rejects versions that do not advance past the last applied entry. +// With a nil WAL there is no replay: the store can only be at a version that is already committed or that +// exactly matches the snapshot it opened at. A load that would need to advance past the committed version +// is rejected — the nil-WAL contract, where the outer context owns the WAL pipeline. func (s *CommitStore) catchup(targetVersion int64) (err error) { var replayed int - var startOff, endOff uint64 - obs := s.observeOp("catchup", otelMetrics.CatchupLatency, - "targetVersion", targetVersion) - // Replayed blocks are reported regardless of catchup outcome: even on a - // later error, the blocks that did replay are real progress that moved - // committedVersion forward. - // - // CurrentVersion is intentionally NOT recorded here: the write-mode - // callers (LoadVersion, Rollback) record it on success themselves, and - // recording it from catchup would pollute the gauge when openReadOnly - // invokes catchup during a read-only historical load. + var startBlock, endBlock uint64 + obs := s.observeOp("catchup", otelMetrics.CatchupLatency, "targetVersion", targetVersion) + // Replayed blocks are reported regardless of outcome: even on a later error, blocks that did replay are + // real progress. CurrentVersion is intentionally NOT recorded here — write-mode callers record it on + // success themselves. defer func() { if replayed > 0 { otelMetrics.CatchupReplayNumBlocks.Add(s.ctx, int64(replayed)) } - obs.done(&err, nil, - "startOffset", startOff, - "endOffset", endOff, - "replayed", replayed) + obs.done(&err, nil, "startBlock", startBlock, "endBlock", endBlock, "replayed", replayed) }() - if s.changelog == nil { - return fmt.Errorf("catchup: changelog not open") + if s.wal == nil { + if targetVersion > 0 && s.committedVersion != targetVersion { + return fmt.Errorf("catchup: nil WAL cannot replay to version %d (committed %d)", + targetVersion, s.committedVersion) + } + return nil } - firstOff, err := s.changelog.FirstOffset() + ok, first, last, err := s.wal.GetStoredRange() if err != nil { - return fmt.Errorf("catchup: first offset: %w", err) + return fmt.Errorf("catchup: WAL range: %w", err) } - lastOff, err := s.changelog.LastOffset() - if err != nil { - return fmt.Errorf("catchup: last offset: %w", err) - } - - if lastOff == 0 || firstOff > lastOff { + if !ok || last <= uint64(s.committedVersion) { + // Empty WAL, or nothing past committedVersion: clean no-op. return nil } - // Resolve the WAL boundary versions once so we can pick the right start - // offset below. - walFirstVer, err := s.walVersionAtOffset(firstOff) - if err != nil { - return fmt.Errorf("catchup: read first WAL entry: %w", err) + // Replay from the block after committedVersion. On a fresh load committedVersion is the snapshot version + // (>= first); when there is no committed state yet (0) start at the WAL's first block. + startBlock = first + if s.committedVersion > 0 { + startBlock = uint64(s.committedVersion) + 1 //nolint:gosec // committedVersion > 0 checked above } - walLastVer, err := s.walVersionAtOffset(lastOff) - if err != nil { - return fmt.Errorf("catchup: read last WAL entry: %w", err) + endBlock = last + if targetVersion > 0 && uint64(targetVersion) < endBlock { + endBlock = uint64(targetVersion) } - - startOff = firstOff - if s.committedVersion > 0 { - // Nothing past committedVersion in the WAL: clean no-op. - if walLastVer <= s.committedVersion { - return nil - } - if walFirstVer <= s.committedVersion { - // The WAL is guaranteed to still hold the entry for - // committedVersion: Commit always writes an entry's WAL record - // before it advances committedVersion, and truncation never - // removes entries at or after committedVersion. Locate it - // directly; the replay loop below skips it - // (entry.Version <= s.committedVersion) and applies from there. - off, err := s.walOffsetForVersion(s.committedVersion) - if err != nil { - return fmt.Errorf("catchup: resolve WAL offset for committed version %d: %w", s.committedVersion, err) - } - startOff = off - } + if endBlock < startBlock { + return nil } - - // Bound end offset to avoid deserializing entries past the target: - // O(target - snapshot) instead of O(WAL_size). - endOff = lastOff - if targetVersion > 0 { - off, err := s.walOffsetForVersion(targetVersion) - if err != nil { - return fmt.Errorf("catchup: resolve WAL offset for target version %d: %w", targetVersion, err) - } - if off > 0 && off < endOff { - endOff = off - } + if first > startBlock { + // We are about to replay, but the blocks between where this store sits and where the WAL begins are + // gone. Starting at first would silently skip them and commit a state whose LtHash matches no real + // chain history. Retention never drops blocks a store still needs, so this is loss or a bug. + return fmt.Errorf("catchup: WAL starts at block %d but replay must start at block %d: "+ + "blocks %d-%d are missing (data loss or corruption)", first, startBlock, startBlock, first-1) } logger.Info("FlatKV catchup start", - "targetVersion", targetVersion, - "committedVersion", s.committedVersion, - "startOffset", startOff, - "endOffset", endOff) - lastApplied := s.committedVersion - err = s.changelog.Replay(startOff, endOff, func(_ uint64, entry proto.ChangelogEntry) error { - if entry.Version <= s.committedVersion { - return nil - } - if targetVersion > 0 && entry.Version > targetVersion { - return nil - } - // WAL versions must advance. Forward jumps are OK (gapped/batch - // commits); going backwards indicates corruption. - if entry.Version <= lastApplied { - return fmt.Errorf("catchup: WAL version not advancing: lastApplied=%d, got %d", lastApplied, entry.Version) + "targetVersion", targetVersion, "committedVersion", s.committedVersion, + "startBlock", startBlock, "endBlock", endBlock) + + it, err := s.wal.Iterator(startBlock, endBlock) + if err != nil { + return fmt.Errorf("catchup: WAL iterator [%d,%d]: %w", startBlock, endBlock, err) + } + defer func() { + if cerr := it.Close(); cerr != nil && err == nil { + err = fmt.Errorf("catchup: close WAL iterator: %w", cerr) } + }() - if err := s.ApplyChangeSets(entry.Version, entry.Changesets); err != nil { - return fmt.Errorf("catchup apply v%d: %w", entry.Version, err) + for { + hasNext, nErr := it.Next() + if nErr != nil { + return fmt.Errorf("catchup: WAL iterate: %w", nErr) } - if err := s.commitBatches(entry.Version); err != nil { - return fmt.Errorf("catchup commit v%d: %w", entry.Version, err) + if !hasNext { + break + } + block, changesets := it.Entry() + if err := s.applyAndCommit(int64(block), changesets); err != nil { //nolint:gosec // block <= endBlock + return fmt.Errorf("catchup: replay block %d: %w", block, err) } - - s.committedVersion = entry.Version - s.committedLtHash = s.workingLtHash.Clone() - s.clearPendingWrites() - recordPendingWrites(s.ctx, accountDBDir, 0) - recordPendingWrites(s.ctx, codeDBDir, 0) - recordPendingWrites(s.ctx, storageDBDir, 0) - recordPendingWrites(s.ctx, miscDBDir, 0) - lastApplied = entry.Version - replayed++ if replayed%1000 == 0 { - logger.Info("FlatKV catchup progress", "replayed", replayed, "version", entry.Version) + logger.Info("FlatKV catchup progress", "replayed", replayed, "version", block) } - return nil - }) - if err != nil { - return fmt.Errorf("catchup replay: %w", err) } if replayed > 0 { if !s.config.Fsync { - // During catchup with Sync=false, per-entry batch commits can leave - // data only in OS/page cache. Flush once before advancing global - // metadata so global watermark won't get ahead of data durability. + // With Fsync=false, per-block batch commits may leave data only in OS/page cache. Flush once + // before advancing global metadata so the global watermark never gets ahead of data durability. if err := s.flushAllDBs(); err != nil { return fmt.Errorf("catchup flush: %w", err) } @@ -235,10 +138,7 @@ func (s *CommitStore) catchup(targetVersion int64) (err error) { return fmt.Errorf("catchup global meta: %w", err) } logger.Info("FlatKV catchup complete", - "replayed", replayed, - "version", s.committedVersion, - "elapsed", obs.elapsed()) + "replayed", replayed, "version", s.committedVersion, "elapsed", obs.elapsed()) } - return nil } diff --git a/sei-db/state_db/sc/flatkv/store_catchup_test.go b/sei-db/state_db/sc/flatkv/store_catchup_test.go index 530f8cf483..b15f6e926e 100644 --- a/sei-db/state_db/sc/flatkv/store_catchup_test.go +++ b/sei-db/state_db/sc/flatkv/store_catchup_test.go @@ -8,43 +8,16 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash" "github.com/stretchr/testify/require" ) -// TestCatchupAllowsVersionGaps verifies that catchup can jump from -// committedVersion to a later WAL entry when intermediate heights were never -// committed (gapped / batched CommitBlock, or empty commits that jump ahead). -func TestCatchupAllowsVersionGaps(t *testing.T) { - cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) - require.NoError(t, err) - _, err = s.LoadVersion(0, false) - require.NoError(t, err) - defer s.Close() - - for i := byte(1); i <= 5; i++ { - commitStorageEntry(t, s, ktype.Address{i}, ktype.Slot{i}, []byte{i}) - } - require.Equal(t, int64(5), s.committedVersion) - - off, err := s.walOffsetForVersion(4) - require.NoError(t, err) - require.NoError(t, s.changelog.TruncateBefore(off)) - - // Simulate a store whose committed watermark is behind the truncated WAL - // tip. Catchup should advance by replaying v4/v5 even though v3 is absent. - s.committedVersion = 2 - - require.NoError(t, s.catchup(0)) - require.Equal(t, int64(5), s.committedVersion) -} - // TestCatchupNoOpWhenWALBehindCommittedVersion verifies catchup is a clean // no-op when the WAL only contains entries that are already covered by // committedVersion (the normal post-truncation steady state). func TestCatchupNoOpWhenWALBehindCommittedVersion(t *testing.T) { cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -69,7 +42,7 @@ func TestCatchupRecoversGappedCommitBlockAfterMetadataLag(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -97,3 +70,130 @@ func TestCatchupRecoversGappedCommitBlockAfterMetadataLag(t *testing.T) { require.NoError(t, s.Close()) } + +// gappedWALStore returns a store whose WAL holds exactly one block, at firstBlock, with nothing before it. +// Committing block 10 as the very first commit is legitimate on its own (a chain whose initial_height is +// above 1, or a store materialized mid-chain); callers create the gap by moving committedVersion. +func gappedWALStore(t *testing.T, firstBlock int64) *CommitStore { + t.Helper() + cfg := config.DefaultTestConfig(t) + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + _, err = s.LoadVersion(0, false) + require.NoError(t, err) + + key := keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(ktype.Address{0xAB}, ktype.Slot{0xCD})) + cs := makeChangeSet(key, padLeft32(0x11), false) + require.NoError(t, s.CommitBlock(firstBlock, []*proto.NamedChangeSet{cs})) + return s +} + +// TestCatchupRejectsWALStartingAfterReplayStart verifies catchup refuses to replay when the WAL no longer +// reaches back to the block after committedVersion. Replaying from the WAL's first block would silently skip +// the missing blocks and commit a state whose LtHash matches no real chain history. +func TestCatchupRejectsWALStartingAfterReplayStart(t *testing.T) { + s := gappedWALStore(t, 10) + defer func() { require.NoError(t, s.Close()) }() + + // Replay must start at 6, but the WAL begins at 10: blocks 6-9 are gone. + s.committedVersion = 5 + err := s.catchup(0) + require.Error(t, err) + require.Contains(t, err.Error(), "blocks 6-9 are missing") + require.Equal(t, int64(5), s.committedVersion, "committedVersion must not advance over a gap") +} + +// TestCatchupAcceptsWALStartingExactlyAtReplayStart pins the boundary: a WAL whose first block is exactly +// committedVersion+1 has no gap and must replay. An off-by-one here would reject healthy stores. +func TestCatchupAcceptsWALStartingExactlyAtReplayStart(t *testing.T) { + s := gappedWALStore(t, 10) + defer func() { require.NoError(t, s.Close()) }() + + s.committedVersion = 9 + require.NoError(t, s.catchup(0)) + require.Equal(t, int64(10), s.committedVersion) +} + +// TestLoadVersionSurfacesCatchupGap verifies the gap error reaches the edge of the flatKV API rather than +// being swallowed somewhere between catchup and LoadVersion. +func TestLoadVersionSurfacesCatchupGap(t *testing.T) { + cfg := config.DefaultTestConfig(t) + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + _, err = s.LoadVersion(0, false) + require.NoError(t, err) + + key := keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(ktype.Address{0xAB}, ktype.Slot{0xCD})) + cs := makeChangeSet(key, padLeft32(0x11), false) + require.NoError(t, s.CommitBlock(10, []*proto.NamedChangeSet{cs})) + + // Rewind the persisted watermark so the reopened store needs blocks 6-9, which this WAL never held. + require.NoError(t, s.commitGlobalMetadata(5, lthash.New())) + require.NoError(t, s.Close()) + + reopened, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + defer func() { _ = reopened.Close() }() + + _, err = reopened.LoadVersion(0, false) + require.Error(t, err) + require.Contains(t, err.Error(), "blocks 6-9 are missing") +} + +// TestReadOnlyAcceptsMidChainWALStart pins the clamp replayInto needs: a clone that opens with no history +// behind it must treat the WAL's first block as the start of history, not as a gap. Without the clamp every +// historical read on a store whose first block is above 1 would be rejected. +func TestReadOnlyAcceptsMidChainWALStart(t *testing.T) { + cfg := config.DefaultTestConfig(t) + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + _, err = s.LoadVersion(0, false) + require.NoError(t, err) + defer func() { require.NoError(t, s.Close()) }() + + key := keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(ktype.Address{0xAB}, ktype.Slot{0xCD})) + for _, v := range []int64{10, 11, 12} { + cs := makeChangeSet(key, padLeft32(byte(v)), false) + require.NoError(t, s.CommitBlock(v, []*proto.NamedChangeSet{cs})) + } + + // The only snapshot is the initial one, so the clone opens at version 0 while the WAL begins at 10. + ro, err := s.LoadVersion(12, true) + require.NoError(t, err) + defer func() { require.NoError(t, ro.Close()) }() + require.Equal(t, int64(12), ro.Version()) +} + +// TestReadOnlySurfacesReplayGap verifies a clone that holds a real position is refused when the primary's WAL +// no longer reaches back to it, instead of being served with a hole in it. Only the export fails: the primary +// keeps committing, since its own state is untouched. +func TestReadOnlySurfacesReplayGap(t *testing.T) { + cfg := config.DefaultTestConfig(t) + // Snapshot every other block and retain them, so the clone below can open at version 2. + cfg.SnapshotInterval = 2 + cfg.SnapshotKeepRecent = 8 + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + _, err = s.LoadVersion(0, false) + require.NoError(t, err) + defer func() { require.NoError(t, s.Close()) }() + + key := keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(ktype.Address{0xAB}, ktype.Slot{0xCD})) + commit := func(v int64, val byte) { + require.NoError(t, s.CommitBlock(v, []*proto.NamedChangeSet{makeChangeSet(key, padLeft32(val), false)})) + } + for v := int64(1); v <= 4; v++ { + commit(v, byte(v)) + } + + // Wipe the WAL and resume far ahead so it no longer reaches back to the snapshot at version 2. + require.NoError(t, s.resetWAL()) + commit(10, 0x99) + + _, err = s.LoadVersion(3, true) + require.Error(t, err) + require.Contains(t, err.Error(), "are missing") + + commit(11, 0xAA) + require.Equal(t, int64(11), s.Version()) +} diff --git a/sei-db/state_db/sc/flatkv/store_lifecycle.go b/sei-db/state_db/sc/flatkv/store_lifecycle.go index 9631e0fb02..0de307e7fe 100644 --- a/sei-db/state_db/sc/flatkv/store_lifecycle.go +++ b/sei-db/state_db/sc/flatkv/store_lifecycle.go @@ -17,18 +17,13 @@ func (s *CommitStore) isClosed() bool { s.codeDB == nil && s.storageDB == nil && s.miscDB == nil } -// closeDBsOnly closes all database handles and the WAL but retains the -// file lock, preventing a race window during Rollback or LoadVersion. +// closeDBsOnly closes all database handles but retains the file lock, preventing a race window during +// Rollback or LoadVersion. It deliberately does NOT close the WAL: the injected WAL's lifecycle is decoupled +// from the DB open/close cycle and must survive the reopen that Rollback/LoadVersion perform. The WAL is +// closed only by top-level Close (or replaced in place by Rollback/restore). func (s *CommitStore) closeDBsOnly() error { var errs []error - if s.changelog != nil { - if err := s.changelog.Close(); err != nil { - errs = append(errs, fmt.Errorf("changelog close: %w", err)) - } - } - s.changelog = nil - if s.metadataDB != nil { if err := s.metadataDB.Close(); err != nil { errs = append(errs, fmt.Errorf("metadataDB close: %w", err)) @@ -73,6 +68,11 @@ func (s *CommitStore) closeDBsOnly() error { // Close drains thread pools, closes all database instances, cancels the // store's context to stop background goroutines (caches, metrics), and // releases the file lock. +// +// NOT SAFE FOR CONCURRENT USE with any other operation on this store: Close releases resources that +// live operations still hold references to, and no lock guards them against it. The caller must +// quiesce every operation first — including background work it does not drive itself, such as an +// export goroutine replaying this store's WAL into a read-only clone (see replayInto). func (s *CommitStore) Close() error { if s.readPool != nil { s.readPool.Close() @@ -91,6 +91,18 @@ func (s *CommitStore) Close() error { s.ltCalc = nil err := s.closeDBsOnly() + + // FlatKV owns Close of whatever WAL instance it currently holds (the injected one, or a replacement made + // by rollback/restore). A nil WAL means the outer context owns the pipeline — nothing to close. The + // closed instance is deliberately retained rather than nilled: a store constructed with a WAL holds one + // for its whole life, so a later write fails loudly against the closed WAL instead of being silently + // skipped as it would be against a nil one. + if s.wal != nil { + if walErr := s.wal.Close(); walErr != nil { + err = errors.Join(err, fmt.Errorf("WAL close: %w", walErr)) + } + } + s.cancel() if s.fileLock != nil { diff --git a/sei-db/state_db/sc/flatkv/store_meta_test.go b/sei-db/state_db/sc/flatkv/store_meta_test.go index a5d9fb6335..5a1c2308f9 100644 --- a/sei-db/state_db/sc/flatkv/store_meta_test.go +++ b/sei-db/state_db/sc/flatkv/store_meta_test.go @@ -136,7 +136,7 @@ func TestLoadRejectsStoreMissingPerModuleMetadata(t *testing.T) { cfg := config.DefaultConfig() cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -167,7 +167,7 @@ func TestLoadRejectsStoreMissingPerModuleMetadata(t *testing.T) { // Reopening must reject the tampered store loudly rather than corrupt it. cfg2 := config.DefaultConfig() cfg2.DataDir = dbDir - s2, err := NewCommitStore(context.Background(), cfg2) + s2, err := newCommitStoreWithWAL(context.Background(), cfg2) require.NoError(t, err) defer s2.Close() _, err = s2.LoadVersion(0, false) @@ -317,7 +317,7 @@ func TestSetInitialVersion_GenesisSkipsSeededSnapshot(t *testing.T) { func TestSetInitialVersion_PersistsEarliestVersion(t *testing.T) { cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -328,7 +328,7 @@ func TestSetInitialVersion_PersistsEarliestVersion(t *testing.T) { require.Equal(t, int64(99), s.EarliestVersion()) require.NoError(t, s.Close()) - reopened, err := NewCommitStore(t.Context(), cfg) + reopened, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = reopened.LoadVersion(0, false) require.NoError(t, err) @@ -388,7 +388,7 @@ func TestSetInitialVersion_SurvivesReopen(t *testing.T) { cfg := config.DefaultConfig() cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -398,7 +398,7 @@ func TestSetInitialVersion_SurvivesReopen(t *testing.T) { cfg2 := config.DefaultConfig() cfg2.DataDir = dbDir - s2, err := NewCommitStore(context.Background(), cfg2) + s2, err := newCommitStoreWithWAL(context.Background(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -446,7 +446,7 @@ func TestGlobalMetadataPersistence(t *testing.T) { cfg := config.DefaultConfig() cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -467,7 +467,7 @@ func TestGlobalMetadataPersistence(t *testing.T) { cfg2 := config.DefaultConfig() cfg2.DataDir = dbDir - s2, err := NewCommitStore(context.Background(), cfg2) + s2, err := newCommitStoreWithWAL(context.Background(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -496,7 +496,7 @@ func TestGetLatestVersionAfterCommitsReadsWorkingMeta(t *testing.T) { cfg := config.DefaultConfig() cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -519,7 +519,7 @@ func TestGetLatestVersionMissingKeyReturnsZero(t *testing.T) { cfg := config.DefaultConfig() cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -554,7 +554,7 @@ func TestCommitStoreGetLatestVersionFallsBackToDiskWhenUnloaded(t *testing.T) { cfg := config.DefaultConfig() cfg.DataDir = dbDir - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -564,7 +564,7 @@ func TestCommitStoreGetLatestVersionFallsBackToDiskWhenUnloaded(t *testing.T) { cfg2 := config.DefaultConfig() cfg2.DataDir = dbDir - s2, err := NewCommitStore(context.Background(), cfg2) + s2, err := newCommitStoreWithWAL(context.Background(), cfg2) require.NoError(t, err) defer s2.Close() diff --git a/sei-db/state_db/sc/flatkv/store_read_test.go b/sei-db/state_db/sc/flatkv/store_read_test.go index 99099803e5..ca9f84580d 100644 --- a/sei-db/state_db/sc/flatkv/store_read_test.go +++ b/sei-db/state_db/sc/flatkv/store_read_test.go @@ -591,7 +591,7 @@ func TestGetAfterReopenAllKeyTypes(t *testing.T) { // Phase 1: write everything and close cfg := config.DefaultTestConfig(t) cfg.DataDir = dir - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) defer s1.Close() _, err = s1.LoadVersion(0, false) @@ -613,7 +613,7 @@ func TestGetAfterReopenAllKeyTypes(t *testing.T) { // Phase 2: reopen and verify all reads cfg2 := config.DefaultTestConfig(t) cfg2.DataDir = dir - s2, err := NewCommitStore(t.Context(), cfg2) + s2, err := newCommitStoreWithWAL(t.Context(), cfg2) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) diff --git a/sei-db/state_db/sc/flatkv/store_test.go b/sei-db/state_db/sc/flatkv/store_test.go index 57c3b4303c..2c265d974b 100644 --- a/sei-db/state_db/sc/flatkv/store_test.go +++ b/sei-db/state_db/sc/flatkv/store_test.go @@ -39,7 +39,7 @@ func TestCommitStoreImplementsStore(t *testing.T) { func TestStoreOpenClose(t *testing.T) { cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -68,7 +68,7 @@ func TestInitializeDataDirectoriesPropagatesPebbleMetrics(t *testing.T) { func TestStoreClose(t *testing.T) { cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -284,7 +284,7 @@ func TestStorePersistence(t *testing.T) { // Write and close cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -297,7 +297,7 @@ func TestStorePersistence(t *testing.T) { // Reopen and verify cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -417,14 +417,17 @@ func TestFileLockPreventsDoubleOpen(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) + // The second store uses a nil WAL so its construction does not take the WAL's changelog-directory lock; + // that isolates the flatkv writer LOCK, which LoadVersion must find held by s1. (With a real WAL the + // conflict would instead surface at construction, from the WAL's own directory lock.) cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := NewCommitStore(t.Context(), cfg, nil) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.Error(t, err, "second open on same dir should fail due to file lock") @@ -437,39 +440,13 @@ func TestFileLockPreventsDoubleOpen(t *testing.T) { require.NoError(t, s2.Close()) } -// ============================================================================= -// clearChangelog -// ============================================================================= - -func TestClearChangelog(t *testing.T) { - cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) - require.NoError(t, err) - _, err = s.LoadVersion(0, false) - require.NoError(t, err) - defer s.Close() - - commitStorageEntry(t, s, ktype.Address{0x01}, ktype.Slot{0x01}, []byte{0x01}) - commitStorageEntry(t, s, ktype.Address{0x02}, ktype.Slot{0x02}, []byte{0x02}) - - last, _ := s.changelog.LastOffset() - require.Greater(t, last, uint64(0), "WAL should have entries") - - require.NoError(t, s.clearChangelog()) - - require.NotNil(t, s.changelog, "changelog should be reopened") - - last, _ = s.changelog.LastOffset() - require.Equal(t, uint64(0), last, "WAL should be empty after clear") -} - // ============================================================================= // closeDBsOnly and Close idempotent // ============================================================================= func TestCloseDBsOnlyIdempotent(t *testing.T) { cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -490,7 +467,7 @@ func TestLoadVersionTargetBeyondWALFails(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -502,7 +479,7 @@ func TestLoadVersionTargetBeyondWALFails(t *testing.T) { cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(100, false) require.Error(t, err, "loading version beyond WAL should fail") @@ -517,7 +494,7 @@ func TestReopenReusesWorkingDir(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -533,7 +510,7 @@ func TestReopenReusesWorkingDir(t *testing.T) { cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -542,65 +519,6 @@ func TestReopenReusesWorkingDir(t *testing.T) { require.Equal(t, int64(1), s2.Version()) } -// ============================================================================= -// walOffsetForVersion -// ============================================================================= - -func TestWalOffsetForVersionFastPath(t *testing.T) { - cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) - require.NoError(t, err) - _, err = s.LoadVersion(0, false) - require.NoError(t, err) - defer s.Close() - - for i := 0; i < 5; i++ { - commitStorageEntry(t, s, ktype.Address{byte(i + 1)}, ktype.Slot{byte(i + 1)}, []byte{byte(i + 1)}) - } - - for v := int64(1); v <= 5; v++ { - off, err := s.walOffsetForVersion(v) - require.NoError(t, err) - require.Greater(t, off, uint64(0), "offset for version %d should be nonzero", v) - - ver, err := s.walVersionAtOffset(off) - require.NoError(t, err) - require.Equal(t, v, ver) - } -} - -func TestWalOffsetForVersionBeforeWAL(t *testing.T) { - cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) - require.NoError(t, err) - _, err = s.LoadVersion(0, false) - require.NoError(t, err) - defer s.Close() - - for i := 0; i < 3; i++ { - commitStorageEntry(t, s, ktype.Address{byte(i + 1)}, ktype.Slot{byte(i + 1)}, []byte{byte(i + 1)}) - } - - off, err := s.walOffsetForVersion(0) - require.NoError(t, err) - require.Equal(t, uint64(0), off, "version 0 predates WAL") -} - -func TestWalOffsetForVersionNotFound(t *testing.T) { - cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) - require.NoError(t, err) - _, err = s.LoadVersion(0, false) - require.NoError(t, err) - defer s.Close() - - commitStorageEntry(t, s, ktype.Address{0x01}, ktype.Slot{0x01}, []byte{0x01}) - commitStorageEntry(t, s, ktype.Address{0x02}, ktype.Slot{0x02}, []byte{0x02}) - - _, err = s.walOffsetForVersion(10) - require.Error(t, err, "version 10 should not be found in WAL with only 2 entries") -} - // ============================================================================= // Catchup from specific version // ============================================================================= @@ -609,7 +527,7 @@ func TestCatchupFromSpecificVersion(t *testing.T) { dir := t.TempDir() cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -624,7 +542,7 @@ func TestCatchupFromSpecificVersion(t *testing.T) { cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -695,7 +613,7 @@ func TestPersistenceAllKeyTypes(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s1.LoadVersion(0, false) require.NoError(t, err) @@ -717,7 +635,7 @@ func TestPersistenceAllKeyTypes(t *testing.T) { cfg = config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -744,7 +662,7 @@ func TestPersistenceAllKeyTypes(t *testing.T) { // ============================================================================= func TestReadOnlyBasicLoadAndRead(t *testing.T) { - s, err := NewCommitStore(t.Context(), config.DefaultTestConfig(t)) + s, err := newCommitStoreWithWAL(t.Context(), config.DefaultTestConfig(t)) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -771,7 +689,7 @@ func TestReadOnlyBasicLoadAndRead(t *testing.T) { func TestReadOnlyLoadFromUnopenedStore(t *testing.T) { cfg := config.DefaultTestConfig(t) - writer, err := NewCommitStore(t.Context(), cfg) + writer, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = writer.LoadVersion(0, false) require.NoError(t, err) @@ -785,7 +703,7 @@ func TestReadOnlyLoadFromUnopenedStore(t *testing.T) { commitAndCheck(t, writer) require.NoError(t, writer.Close()) - fresh, err := NewCommitStore(t.Context(), cfg) + fresh, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) ro, err := fresh.LoadVersion(0, true) require.NoError(t, err) @@ -799,7 +717,7 @@ func TestReadOnlyLoadFromUnopenedStore(t *testing.T) { func TestReadOnlyAtSpecificVersion(t *testing.T) { cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -827,7 +745,7 @@ func TestReadOnlyAtSpecificVersion(t *testing.T) { func TestReadOnlyWriteGuards(t *testing.T) { cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -856,7 +774,7 @@ func TestReadOnlyWriteGuards(t *testing.T) { func TestReadOnlyParentWritesDuringReadOnly(t *testing.T) { cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -888,7 +806,7 @@ func TestReadOnlyConcurrentInstances(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.SnapshotInterval = 2 cfg.SnapshotKeepRecent = 10 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -925,7 +843,7 @@ func TestReadOnlyConcurrentInstances(t *testing.T) { func TestReadOnlyFailureDoesNotAffectParent(t *testing.T) { cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -951,7 +869,7 @@ func TestReadOnlyFailureDoesNotAffectParent(t *testing.T) { func TestReadOnlyCloseRemovesTempDir(t *testing.T) { cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -975,7 +893,7 @@ func TestReadOnlyCloseRemovesTempDir(t *testing.T) { func TestCleanupOrphanedReadOnlyDirs(t *testing.T) { cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) defer func() { require.NoError(t, s.Close()) }() @@ -999,12 +917,14 @@ func TestCleanupOrphanedReadOnlyDirs(t *testing.T) { func TestCleanupOrphanedReadOnlyDirsHoldsWriterLock(t *testing.T) { cfg := config.DefaultTestConfig(t) - s1, err := NewCommitStore(t.Context(), cfg) + s1, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) defer func() { require.NoError(t, s1.Close()) }() require.NoError(t, s1.CleanupOrphanedReadOnlyDirs()) - s2, err := NewCommitStore(t.Context(), cfg) + // nil WAL on the second store so its construction does not take the WAL's changelog-directory lock; + // this isolates the flatkv writer LOCK that CleanupOrphanedReadOnlyDirs must find held by s1. + s2, err := NewCommitStore(t.Context(), cfg, nil) require.NoError(t, err) defer func() { require.NoError(t, s2.Close()) }() @@ -1019,7 +939,7 @@ func TestLoadVersionReload(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1100,7 +1020,7 @@ func TestLoadVersionEmptyWAL(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1117,7 +1037,7 @@ func TestCloseWithPendingUncommittedWrites(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1137,7 +1057,7 @@ func TestCloseWithPendingUncommittedWrites(t *testing.T) { require.NoError(t, s.Close()) // Reopen: uncommitted data should be lost. - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1218,7 +1138,7 @@ func TestCatchupSkipsAlreadyCommittedEntries(t *testing.T) { cfg.DataDir = filepath.Join(dir, flatkvRootDir) cfg.SnapshotInterval = 2 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1236,7 +1156,7 @@ func TestCatchupSkipsAlreadyCommittedEntries(t *testing.T) { // Reopen: catchup should replay only entries after the committed version // and skip already-committed entries. - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1252,7 +1172,7 @@ func TestCatchupTargetVersionMiddleOfWAL(t *testing.T) { cfg.DataDir = filepath.Join(dir, flatkvRootDir) cfg.SnapshotInterval = 2 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1270,7 +1190,7 @@ func TestCatchupTargetVersionMiddleOfWAL(t *testing.T) { require.NoError(t, s.Close()) // Open at v3 (middle of WAL). - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(3, false) require.NoError(t, err) @@ -1280,26 +1200,13 @@ func TestCatchupTargetVersionMiddleOfWAL(t *testing.T) { require.Equal(t, hashes[3], s2.RootHash()) } -func TestWalOffsetForVersionNilChangelog(t *testing.T) { - s := setupTestStore(t) - savedChangelog := s.changelog - s.changelog = nil - - _, err := s.walOffsetForVersion(1) - require.Error(t, err) - require.Contains(t, err.Error(), "changelog not open") - - s.changelog = savedChangelog - require.NoError(t, s.Close()) -} - func TestCrashRecoverySkewedPerDBVersions(t *testing.T) { dir := t.TempDir() cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) cfg.SnapshotInterval = 3 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1332,7 +1239,7 @@ func TestCrashRecoverySkewedPerDBVersions(t *testing.T) { require.NoError(t, s.Close()) // Reopen: loadGlobalMetadata detects version skew and catchup replays. - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1360,7 +1267,7 @@ func TestCrashRecoveryGlobalMetadataAheadOfDataDBs(t *testing.T) { cfg.DataDir = filepath.Join(dir, flatkvRootDir) cfg.SnapshotInterval = 3 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1385,7 +1292,7 @@ func TestCrashRecoveryGlobalMetadataAheadOfDataDBs(t *testing.T) { require.NoError(t, s.Close()) - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1408,7 +1315,7 @@ func TestCrashRecoveryWALReplayLargeGap(t *testing.T) { cfg.DataDir = filepath.Join(dir, flatkvRootDir) cfg.SnapshotInterval = 5 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1425,7 +1332,7 @@ func TestCrashRecoveryWALReplayLargeGap(t *testing.T) { require.NoError(t, s.Close()) // Reopen normally -- large WAL gap between snapshot and HEAD. - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1449,7 +1356,7 @@ func TestCrashRecoveryEmptyWALAfterSnapshot(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1466,11 +1373,11 @@ func TestCrashRecoveryEmptyWALAfterSnapshot(t *testing.T) { expectedVersion := s.Version() // Clear the WAL entirely (simulate WAL lost after snapshot). - require.NoError(t, s.clearChangelog()) + require.NoError(t, s.resetWAL()) require.NoError(t, s.Close()) // Reopen: should work from snapshot alone. - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1491,6 +1398,92 @@ func TestCrashRecoveryEmptyWALAfterSnapshot(t *testing.T) { require.Equal(t, expectedVersion+1, v) } +// walStoreAtV1 returns an open store managing a real WAL, with one committed block. +func walStoreAtV1(t *testing.T, cfg *config.Config) *CommitStore { + t.Helper() + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + _, err = s.LoadVersion(0, false) + require.NoError(t, err) + + key := keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(addrN(0x07), slotN(0x01))) + cs := makeChangeSet(key, padLeft32(0xAA), false) + require.NoError(t, s.ApplyChangeSets(1, []*proto.NamedChangeSet{cs})) + _, err = s.Commit(1) + require.NoError(t, err) + return s +} + +// TestCloseRetainsWALInstance verifies Close leaves the closed WAL in place instead of nilling it. A store +// built with a WAL must hold one for its whole life: a nil field is indistinguishable from "the outer +// context owns the pipeline", which is what made a WAL-less writer commit with no log. +func TestCloseRetainsWALInstance(t *testing.T) { + cfg := config.DefaultTestConfig(t) + s := walStoreAtV1(t, cfg) + + require.True(t, s.manageWAL) + require.NoError(t, s.Close()) + require.NotNil(t, s.wal, "Close must retain the closed WAL instance, not nil it") + require.True(t, s.manageWAL, "manageWAL is fixed at construction") +} + +// TestCommitWithClosedWALFailsLoudly verifies the state a failed resetWAL now leaves behind — DBs open, WAL +// closed — refuses to commit rather than persisting to the four Pebble DBs with nothing to replay from. That +// is the whole point of retaining the closed instance: against a nil WAL this commit would silently succeed. +func TestCommitWithClosedWALFailsLoudly(t *testing.T) { + cfg := config.DefaultTestConfig(t) + s := walStoreAtV1(t, cfg) + + // Exactly the state left by a resetWAL whose Delete or New failed after the old WAL was closed. + require.NoError(t, s.wal.Close()) + + key := keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(addrN(0x07), slotN(0x02))) + cs := makeChangeSet(key, padLeft32(0xBB), false) + require.NoError(t, s.ApplyChangeSets(2, []*proto.NamedChangeSet{cs})) + _, err := s.Commit(2) + require.Error(t, err, "a store whose WAL is closed must not commit silently") + require.Contains(t, err.Error(), "closed") + require.Equal(t, int64(1), s.committedVersion, "the failed commit must not advance the version") + + // Closing the store closes the WAL a second time, which must be a no-op. + require.NoError(t, s.Close()) +} + +// TestResetWALAfterCloseReopens covers the live import path: rootmulti.Restore closes the store before +// calling Importer, so resetWAL runs against an already-closed instance. It must still read the config off +// it and close it a second time without error. +func TestResetWALAfterCloseReopens(t *testing.T) { + cfg := config.DefaultTestConfig(t) + s := walStoreAtV1(t, cfg) + require.NoError(t, s.Close()) + + require.NoError(t, s.resetWAL()) + require.NotNil(t, s.wal) + + ok, _, _, err := s.wal.GetStoredRange() + require.NoError(t, err) + require.False(t, ok, "resetWAL must leave an empty WAL") + require.NoError(t, s.Close()) +} + +// TestRollbackRetainsWALInstance pins the behavior Rollback already had — it never nils s.wal — so a future +// edit cannot reintroduce the fail-silent shape there. +func TestRollbackRetainsWALInstance(t *testing.T) { + cfg := config.DefaultTestConfig(t) + s := walStoreAtV1(t, cfg) + defer func() { require.NoError(t, s.Close()) }() + + key := keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(addrN(0x07), slotN(0x03))) + cs := makeChangeSet(key, padLeft32(0xCC), false) + require.NoError(t, s.ApplyChangeSets(2, []*proto.NamedChangeSet{cs})) + _, err := s.Commit(2) + require.NoError(t, err) + + require.NoError(t, s.Rollback(1)) + require.NotNil(t, s.wal) + require.Equal(t, int64(1), s.committedVersion) +} + func TestCrashRecoveryCorruptedAccountValueInDB(t *testing.T) { s := setupTestStore(t) defer s.Close() @@ -1529,7 +1522,7 @@ func TestCrashRecoveryCrashAfterWALBeforeDBCommit(t *testing.T) { cfg.DataDir = filepath.Join(dir, flatkvRootDir) cfg.SnapshotInterval = 1 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1547,12 +1540,10 @@ func TestCrashRecoveryCrashAfterWALBeforeDBCommit(t *testing.T) { cs2 := makeChangeSet(key, padLeft32(0x22), false) require.NoError(t, s.ApplyChangeSets(s.Version()+1, []*proto.NamedChangeSet{cs2})) - // Write v2 to WAL manually (like Commit step 1). - changelogEntry := proto.ChangelogEntry{ - Version: 2, - Changesets: s.pendingChangeSets, - } - require.NoError(t, s.changelog.Write(changelogEntry)) + // Write v2 to the WAL manually (like Commit step 1) without committing to the DBs. + require.NoError(t, s.wal.Write(2, s.pendingChangeSets)) + require.NoError(t, s.wal.SignalEndOfBlock()) + require.NoError(t, s.wal.Flush()) // Do NOT call commitBatches or update global metadata. // Reset in-memory state to v1 to simulate crash. @@ -1561,7 +1552,7 @@ func TestCrashRecoveryCrashAfterWALBeforeDBCommit(t *testing.T) { require.NoError(t, s.Close()) // Reopen: catchup should replay v2 from WAL. - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1582,7 +1573,7 @@ func TestCrashRecoveryLtHashConsistencyAfterAllPaths(t *testing.T) { cfg.DataDir = filepath.Join(dir, flatkvRootDir) cfg.SnapshotInterval = 3 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1608,7 +1599,7 @@ func TestCrashRecoveryLtHashConsistencyAfterAllPaths(t *testing.T) { require.NoError(t, s.Close()) // Path 1: Normal reopen - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s2.LoadVersion(0, false) require.NoError(t, err) @@ -1633,7 +1624,7 @@ func TestCrashRecoveryLtHashConsistencyAfterAllPaths(t *testing.T) { require.NoError(t, s2.Close()) // Path 4: Reopen after rollback + new commit - s3, err := NewCommitStore(t.Context(), cfg) + s3, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s3.LoadVersion(0, false) require.NoError(t, err) @@ -1646,7 +1637,7 @@ func TestCrashRecoveryCorruptLtHashBlobInMetadata(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1668,7 +1659,7 @@ func TestCrashRecoveryCorruptLtHashBlobInMetadata(t *testing.T) { require.NoError(t, s.Close()) // Reopen should fail with an LtHash unmarshal error. - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) defer s2.Close() _, err = s2.LoadVersion(0, false) @@ -1681,7 +1672,7 @@ func TestCrashRecoveryCorruptLtHashBlobInPerDBMeta(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1703,7 +1694,7 @@ func TestCrashRecoveryCorruptLtHashBlobInPerDBMeta(t *testing.T) { require.NoError(t, s.Close()) // Reopen should fail with an LtHash unmarshal error from per-DB meta. - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) defer s2.Close() _, err = s2.LoadVersion(0, false) @@ -1716,7 +1707,7 @@ func TestCrashRecoveryGlobalVersionOverflow(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -1740,7 +1731,7 @@ func TestCrashRecoveryGlobalVersionOverflow(t *testing.T) { require.NoError(t, s.Close()) // Reopen should fail with an overflow error. - s2, err := NewCommitStore(t.Context(), cfg) + s2, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) defer s2.Close() _, err = s2.LoadVersion(0, false) diff --git a/sei-db/state_db/sc/flatkv/store_write.go b/sei-db/state_db/sc/flatkv/store_write.go index 6014926d45..651bdf25aa 100644 --- a/sei-db/state_db/sc/flatkv/store_write.go +++ b/sei-db/state_db/sc/flatkv/store_write.go @@ -28,7 +28,10 @@ func (s *CommitStore) CommitBlock(version int64, changesets []*proto.NamedChange return nil } -// Commit persists buffered writes at the given version (block height). +// Commit persists buffered writes at the given version (block height). One Commit persists exactly one +// block; version must equal the height the pending writes were stamped with. Consecutive commits must also +// be contiguous: the state WAL rejects a version that skips a height, though the first block written to an +// empty WAL may be any height. // Protocol: WAL → per-DB batch (with LocalMeta) → flush → update metaDB. // On crash, catchup replays WAL to recover incomplete commits. func (s *CommitStore) Commit(version int64) (committed int64, err error) { @@ -73,14 +76,21 @@ func (s *CommitStore) Commit(version int64) (committed int64, err error) { version, s.pendingBlockHeight) } - // Step 1: Write Changelog (WAL) - source of truth (always sync) - s.phaseTimer.SetPhase("commit_write_changelog") - changelogEntry := proto.ChangelogEntry{ - Version: version, - Changesets: s.pendingChangeSets, - } - if err := s.changelog.Write(changelogEntry); err != nil { - return version, fmt.Errorf("changelog write: %w", err) + // Step 1: Write the WAL (source of truth) before the DBs, so crash recovery via catchup stays valid. + // Write buffers this block's changesets, SignalEndOfBlock seals them as one record, and Flush makes the + // record durable. An empty block (no ApplyChangeSets) writes an empty but contiguous record. Skipped + // entirely when the WAL is nil — the outer context then owns the WAL pipeline. + if s.wal != nil { + s.phaseTimer.SetPhase("commit_write_wal") + if err := s.wal.Write(uint64(version), s.pendingChangeSets); err != nil { //nolint:gosec // version > committed >= 0 + return version, fmt.Errorf("WAL write: %w", err) + } + if err := s.wal.SignalEndOfBlock(); err != nil { + return version, fmt.Errorf("WAL end of block: %w", err) + } + if err := s.wal.Flush(); err != nil { + return version, fmt.Errorf("WAL flush: %w", err) + } } // Step 2: Commit to each DB (data + LocalMeta.CommittedVersion atomically) diff --git a/sei-db/state_db/sc/flatkv/store_write_test.go b/sei-db/state_db/sc/flatkv/store_write_test.go index a2bfc21a0b..b9ad3a6a34 100644 --- a/sei-db/state_db/sc/flatkv/store_write_test.go +++ b/sei-db/state_db/sc/flatkv/store_write_test.go @@ -151,6 +151,31 @@ func TestStoreWriteEmptyCommit(t *testing.T) { requireAllLocalMetaAt(t, s, 2) } +// TestCommitRejectsNonContiguousVersion verifies a forward jump in commit height is rejected by the WAL's +// contiguity rule rather than silently accepted. The version guard only rejects versions at or below +// committedVersion, so a gapped height reaches the WAL and must fail there. +// +// This is a behavior change from the old changelog, which permitted forward jumps for gapped/batch commits. +// Note the asymmetry: the *first* block written to an empty WAL may be any number, which is why a fresh +// store can start mid-chain (see TestCatchupRecoversGappedCommitBlockAfterMetadataLag) while an established +// one cannot skip. +func TestCommitRejectsNonContiguousVersion(t *testing.T) { + s := setupTestStore(t) + defer s.Close() + + key := evmStorageKey(ktype.Address{0x11}, ktype.Slot{0x22}) + require.NoError(t, s.ApplyChangeSets(1, []*proto.NamedChangeSet{makeChangeSet(key, padLeft32(0xAA), false)})) + _, err := s.Commit(1) + require.NoError(t, err) + + // Skip version 2. This clears the version guard (3 > committedVersion) and fails at the WAL. + require.NoError(t, s.ApplyChangeSets(3, []*proto.NamedChangeSet{makeChangeSet(key, padLeft32(0xBB), false)})) + _, err = s.Commit(3) + require.Error(t, err) + require.Contains(t, err.Error(), "contiguous") + require.Equal(t, int64(1), s.committedVersion, "a rejected commit must not advance the version") +} + func TestStoreWriteAccountAndCode(t *testing.T) { s := setupTestStore(t) defer s.Close() @@ -502,7 +527,7 @@ func TestStoreMiscEmptyCommitLocalMeta(t *testing.T) { func TestStoreFsyncConfig(t *testing.T) { t.Run("DefaultConfig", func(t *testing.T) { cfg := config.DefaultTestConfig(t) - store, err := NewCommitStore(t.Context(), cfg) + store, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = store.LoadVersion(0, false) require.NoError(t, err) @@ -516,7 +541,7 @@ func TestStoreFsyncConfig(t *testing.T) { t.Run("FsyncDisabled", func(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.Fsync = false - store, err := NewCommitStore(t.Context(), cfg) + store, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = store.LoadVersion(0, false) require.NoError(t, err) @@ -549,7 +574,7 @@ func TestAutoSnapshotTriggeredByInterval(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.SnapshotInterval = 5 cfg.SnapshotKeepRecent = 2 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -572,7 +597,7 @@ func TestAutoSnapshotNotTriggeredBeforeInterval(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.SnapshotInterval = 10 cfg.SnapshotKeepRecent = 2 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -600,7 +625,7 @@ func TestAutoSnapshotNotTriggeredBeforeInterval(t *testing.T) { func TestAutoSnapshotDisabledWhenIntervalZero(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.SnapshotInterval = 0 - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -693,7 +718,7 @@ func TestMultipleApplyAccountFieldsPreservesOther(t *testing.T) { func TestLtHashDeterministicAcrossReopen(t *testing.T) { writeAndGetHash := func() []byte { cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -818,7 +843,7 @@ func TestEmptyCommitAdvancesVersion(t *testing.T) { func TestStoreFsyncEnabled(t *testing.T) { cfg := config.DefaultTestConfig(t) cfg.Fsync = true - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -840,7 +865,7 @@ func TestStoreFsyncEnabled(t *testing.T) { func TestLastSnapshotTimeUpdated(t *testing.T) { cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -861,7 +886,7 @@ func TestLastSnapshotTimeUpdated(t *testing.T) { func TestWALRecordsChangesets(t *testing.T) { cfg := config.DefaultTestConfig(t) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -870,17 +895,7 @@ func TestWALRecordsChangesets(t *testing.T) { commitStorageEntry(t, s, ktype.Address{0x02}, ktype.Slot{0x02}, []byte{0xBB}) commitStorageEntry(t, s, ktype.Address{0x03}, ktype.Slot{0x03}, []byte{0xCC}) - first, _ := s.changelog.FirstOffset() - last, _ := s.changelog.LastOffset() - require.Greater(t, last, uint64(0)) - - var versions []int64 - err = s.changelog.Replay(first, last, func(_ uint64, entry proto.ChangelogEntry) error { - versions = append(versions, entry.Version) - return nil - }) - require.NoError(t, err) - require.Equal(t, []int64{1, 2, 3}, versions) + require.Equal(t, []uint64{1, 2, 3}, walBlockNumbers(t, s)) require.NoError(t, s.Close()) } @@ -1007,26 +1022,11 @@ func TestEmptyCommitWALPayloadsDiffer(t *testing.T) { require.NoError(t, sEmpty.ApplyChangeSets(sEmpty.Version()+1, []*proto.NamedChangeSet{emptyCS})) commitAndCheck(t, sEmpty) - nilFirst, _ := sNil.changelog.FirstOffset() - nilLast, _ := sNil.changelog.LastOffset() - var nilEntry proto.ChangelogEntry - err := sNil.changelog.Replay(nilFirst, nilLast, func(_ uint64, e proto.ChangelogEntry) error { - nilEntry = e - return nil - }) - require.NoError(t, err) + nilChangesets := singleWALBlockChangesets(t, sNil) + emptyChangesets := singleWALBlockChangesets(t, sEmpty) - emptyFirst, _ := sEmpty.changelog.FirstOffset() - emptyLast, _ := sEmpty.changelog.LastOffset() - var emptyEntry proto.ChangelogEntry - err = sEmpty.changelog.Replay(emptyFirst, emptyLast, func(_ uint64, e proto.ChangelogEntry) error { - emptyEntry = e - return nil - }) - require.NoError(t, err) - - require.Len(t, nilEntry.Changesets, 0, "nil ApplyChangeSets produces 0 WAL changesets") - require.Len(t, emptyEntry.Changesets, 1, "[empty NamedChangeSet] produces 1 WAL changeset") + require.Len(t, nilChangesets, 0, "nil ApplyChangeSets produces 0 WAL changesets") + require.Len(t, emptyChangesets, 1, "[empty NamedChangeSet] produces 1 WAL changeset") } // ============================================================================= @@ -1798,48 +1798,45 @@ func TestCommitRejectsApplyHeightMismatch(t *testing.T) { require.Equal(t, int64(1), v) } -// TestApplyChangeSetsAllowsBatchingMultipleHeightsBeforeCommit is a -// regression test for a bug where ApplyChangeSets rejected any call whose -// version did not exactly equal the previous pending call's version. That -// broke two legitimate patterns: (1) a caller batching several blocks' -// writes before a single Commit (each call at a strictly higher height), -// and (2) a caller splitting one block's writes across multiple calls at -// the same height. Only a version that goes backwards is a bug. -func TestApplyChangeSetsAllowsBatchingMultipleHeightsBeforeCommit(t *testing.T) { +// TestApplyChangeSetsAllowsSameHeightRepeatsOnly pins the one-block-per-commit +// contract: a single block's writes may arrive across several ApplyChangeSets +// calls at the same height, but any other height is rejected, in either +// direction. Batching several blocks before one Commit is not supported — +// changesets carry no block number, so the batch would collapse into a single WAL +// entry at its highest height, leaving the skipped heights unreplayable. +func TestApplyChangeSetsAllowsSameHeightRepeatsOnly(t *testing.T) { s := setupTestStore(t) defer s.Close() addr := addrN(0x01) key1 := keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(addr, slotN(0x01))) key2 := keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(addr, slotN(0x02))) - key3 := keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(addr, slotN(0x03))) // Same-height repeat: splitting one block's writes across two calls. - require.NoError(t, s.ApplyChangeSets(1, []*proto.NamedChangeSet{makeChangeSet(key1, padLeft32(0x11), false)})) - require.NoError(t, s.ApplyChangeSets(1, []*proto.NamedChangeSet{makeChangeSet(key2, padLeft32(0x22), false)})) + require.NoError(t, s.ApplyChangeSets(5, []*proto.NamedChangeSet{makeChangeSet(key1, padLeft32(0x11), false)})) + require.NoError(t, s.ApplyChangeSets(5, []*proto.NamedChangeSet{makeChangeSet(key2, padLeft32(0x22), false)})) + require.Equal(t, int64(5), s.PendingVersion()) - // Strictly increasing: a second block batched before the commit. - require.NoError(t, s.ApplyChangeSets(2, []*proto.NamedChangeSet{makeChangeSet(key3, padLeft32(0x33), false)})) - require.Equal(t, int64(2), s.PendingVersion()) + // Advancing to the next block before committing this one is rejected. + err := s.ApplyChangeSets(6, []*proto.NamedChangeSet{makeChangeSet(key1, padLeft32(0x33), false)}) + require.Error(t, err) + require.Contains(t, err.Error(), "only one block may be buffered per commit") - // Going backwards is rejected. - err := s.ApplyChangeSets(1, []*proto.NamedChangeSet{makeChangeSet(key1, padLeft32(0x44), false)}) + // So is going backwards. + err = s.ApplyChangeSets(4, []*proto.NamedChangeSet{makeChangeSet(key1, padLeft32(0x44), false)}) require.Error(t, err) - require.Contains(t, err.Error(), "pending writes already stamped up to") + require.Contains(t, err.Error(), "only one block may be buffered per commit") - v, err := s.Commit(2) + v, err := s.Commit(5) require.NoError(t, err) - require.Equal(t, int64(2), v) + require.Equal(t, int64(5), v) require.Equal(t, int64(0), s.PendingVersion()) - for _, k := range []struct { - key []byte - height int64 - }{{key1, 1}, {key2, 1}, {key3, 2}} { - height, found, err := s.GetBlockHeightModified(keys.EVMStoreKey, k.key) + for _, key := range [][]byte{key1, key2} { + height, found, err := s.GetBlockHeightModified(keys.EVMStoreKey, key) require.NoError(t, err) require.True(t, found) - require.Equal(t, k.height, height) + require.Equal(t, int64(5), height) } } diff --git a/sei-db/state_db/sc/flatkv/testutil_test.go b/sei-db/state_db/sc/flatkv/testutil_test.go index cd24c4f3b0..75d98e85d5 100644 --- a/sei-db/state_db/sc/flatkv/testutil_test.go +++ b/sei-db/state_db/sc/flatkv/testutil_test.go @@ -69,7 +69,7 @@ func setupTestDB(t *testing.T) types.KeyValueDB { // setupTestStore creates a minimal test store func setupTestStore(t *testing.T) *CommitStore { t.Helper() - s, err := NewCommitStore(t.Context(), config.DefaultTestConfig(t)) + s, err := newCommitStoreWithWAL(t.Context(), config.DefaultTestConfig(t)) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) @@ -81,7 +81,7 @@ func setupTestStoreWithConfig(t *testing.T, cfg *config.Config) *CommitStore { t.Helper() dir := t.TempDir() cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := NewCommitStore(t.Context(), cfg) + s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) diff --git a/sei-db/state_db/sc/flatkv/wal_glue.go b/sei-db/state_db/sc/flatkv/wal_glue.go new file mode 100644 index 0000000000..dba603d608 --- /dev/null +++ b/sei-db/state_db/sc/flatkv/wal_glue.go @@ -0,0 +1,30 @@ +package flatkv + +import ( + "path/filepath" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" + "github.com/sei-protocol/sei-chain/sei-db/state_db/statewal" +) + +// This file holds how FlatKV locates and opens its injected state WAL. The WAL's use during normal +// operation lives with the operations that use it — writes in Commit (store_write.go), replay in catchup +// (store_catchup.go), tail-truncate/prune in Rollback and tryTruncateWAL (snapshot.go), and the +// import/restore reset in resetWAL (store.go, next to resetForImport). + +// OpenStateWAL opens (or creates, recovering any prior session) the state WAL for a FlatKV store configured +// by cfg, at the conventional changelog directory under the store's data dir. The returned instance is +// injected into NewCommitStore; the outer context constructs it here as the intermediate step toward +// managing the WAL entirely outside FlatKV. Pass nil to NewCommitStore instead to have FlatKV skip all WAL +// operations (e.g. read-only export clones). This is the single seam outside package flatkv that needs to +// know FlatKV's on-disk WAL layout. +func OpenStateWAL(cfg *config.Config) (statewal.StateWAL, error) { + return statewal.New(stateWALConfig(cfg)) +} + +// stateWALConfig builds the state WAL configuration for a store configured by cfg: a "changelog" +// subdirectory of the data dir, with a fixed instance name used only to label metrics. It is the single +// definition of that layout convention. +func stateWALConfig(cfg *config.Config) *statewal.Config { + return statewal.DefaultConfig(filepath.Join(cfg.DataDir, changelogDir), "flatkv") +} diff --git a/sei-db/state_db/sc/flatkv/wal_testutil_test.go b/sei-db/state_db/sc/flatkv/wal_testutil_test.go new file mode 100644 index 0000000000..d9b3aeafcf --- /dev/null +++ b/sei-db/state_db/sc/flatkv/wal_testutil_test.go @@ -0,0 +1,67 @@ +package flatkv + +import ( + "context" + "testing" + + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" + "github.com/stretchr/testify/require" +) + +// newCommitStoreWithWAL constructs a CommitStore with a real state WAL opened from cfg's changelog +// directory — the test-suite equivalent of how the composite wires production stores. It stands in for the +// former 2-arg NewCommitStore calls now that the state WAL is an injected constructor argument; the suite's +// call sites were mechanically rewritten to use it. +func newCommitStoreWithWAL(ctx context.Context, cfg *config.Config) (*CommitStore, error) { + stateWAL, err := OpenStateWAL(cfg) + if err != nil { + return nil, err + } + return NewCommitStore(ctx, cfg, stateWAL) +} + +// walBlockNumbers returns the block numbers stored in the store's WAL, in ascending order. Test-only. +func walBlockNumbers(t *testing.T, s *CommitStore) []uint64 { + t.Helper() + ok, first, last, err := s.wal.GetStoredRange() + require.NoError(t, err) + if !ok { + return nil + } + it, err := s.wal.Iterator(first, last) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + var blocks []uint64 + for { + hasNext, err := it.Next() + require.NoError(t, err) + if !hasNext { + break + } + block, _ := it.Entry() + blocks = append(blocks, block) + } + return blocks +} + +// singleWALBlockChangesets returns the changesets of the sole block stored in the store's WAL, asserting +// there is exactly one. Test-only. +func singleWALBlockChangesets(t *testing.T, s *CommitStore) []*proto.NamedChangeSet { + t.Helper() + ok, first, last, err := s.wal.GetStoredRange() + require.NoError(t, err) + require.True(t, ok, "expected a WAL block") + require.Equal(t, first, last, "expected exactly one WAL block") + + it, err := s.wal.Iterator(first, last) + require.NoError(t, err) + defer func() { require.NoError(t, it.Close()) }() + + hasNext, err := it.Next() + require.NoError(t, err) + require.True(t, hasNext) + _, changesets := it.Entry() + return changesets +} diff --git a/sei-db/state_db/sc/flatkv/wal_torn_write_test.go b/sei-db/state_db/sc/flatkv/wal_torn_write_test.go deleted file mode 100644 index 17981542ce..0000000000 --- a/sei-db/state_db/sc/flatkv/wal_torn_write_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package flatkv - -import ( - "os" - "path/filepath" - "testing" - - "github.com/sei-protocol/sei-chain/sei-db/common/keys" - "github.com/sei-protocol/sei-chain/sei-db/proto" - "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" - "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" - "github.com/stretchr/testify/require" -) - -func TestStoreOpenAfterWALTornWrite(t *testing.T) { - t.Run("mid_record_truncation_discards_torn_commit", func(t *testing.T) { - cfg, key, rootAtV4, _ := prepareStoreWithManualWALTail(t) - truncateLastWALSegmentBy(t, filepath.Join(cfg.DataDir, changelogDir), 4) - - s := reopenTestStore(t, cfg) - defer func() { require.NoError(t, s.Close()) }() - - require.Equal(t, int64(4), s.Version()) - require.Equal(t, rootAtV4, s.CommittedRootHash()) - val, found := s.Get(keys.EVMStoreKey, key) - require.True(t, found) - require.Equal(t, padLeft32(0x04), val) - verifyLtHashConsistency(t, s) - }) - - t.Run("partial_length_prefix_discards_torn_commit", func(t *testing.T) { - cfg, key, rootAtV4, tailStart := prepareStoreWithManualWALTail(t) - replaceManualWALTailWithPartialLengthPrefix(t, filepath.Join(cfg.DataDir, changelogDir), tailStart) - - s := reopenTestStore(t, cfg) - defer func() { require.NoError(t, s.Close()) }() - - require.Equal(t, int64(4), s.Version()) - require.Equal(t, rootAtV4, s.CommittedRootHash()) - val, found := s.Get(keys.EVMStoreKey, key) - require.True(t, found) - require.Equal(t, padLeft32(0x04), val) - verifyLtHashConsistency(t, s) - }) - - t.Run("clean_tail_replays_last_wal_commit", func(t *testing.T) { - cfg, key, rootAtV4, _ := prepareStoreWithManualWALTail(t) - - s := reopenTestStore(t, cfg) - defer func() { require.NoError(t, s.Close()) }() - - require.Equal(t, int64(5), s.Version()) - require.NotEqual(t, rootAtV4, s.CommittedRootHash()) - val, found := s.Get(keys.EVMStoreKey, key) - require.True(t, found) - require.Equal(t, padLeft32(0x05), val) - verifyLtHashConsistency(t, s) - }) -} - -func prepareStoreWithManualWALTail(t *testing.T) (*config.Config, []byte, []byte, int64) { - t.Helper() - - cfg := config.DefaultTestConfig(t) - cfg.DataDir = filepath.Join(t.TempDir(), flatkvRootDir) - cfg.SnapshotInterval = 0 - - s := reopenTestStore(t, cfg) - addr := ktype.Address{0x44} - slot := ktype.Slot{0x55} - key := keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(addr, slot)) - for v := byte(1); v <= 4; v++ { - commitStorageEntry(t, s, addr, slot, []byte{v}) - } - require.Equal(t, int64(4), s.Version()) - rootAtV4 := append([]byte(nil), s.CommittedRootHash()...) - tailStart := walSegmentSize(t, filepath.Join(cfg.DataDir, changelogDir)) - - cs := makeChangeSet(key, padLeft32(0x05), false) - require.NoError(t, s.ApplyChangeSets(s.Version()+1, []*proto.NamedChangeSet{cs})) - require.NoError(t, s.changelog.Write(proto.ChangelogEntry{ - Version: 5, - Changesets: s.pendingChangeSets, - })) - - // Simulate a process dying after WAL append and before DB batch commit. - // Close releases file handles for deterministic test-time file mutation. - s.clearPendingWrites() - require.NoError(t, s.Close()) - return cfg, key, rootAtV4, tailStart -} - -func reopenTestStore(t *testing.T, cfg *config.Config) *CommitStore { - t.Helper() - s, err := NewCommitStore(t.Context(), cfg) - require.NoError(t, err) - _, err = s.LoadVersion(0, false) - require.NoError(t, err) - return s -} - -func truncateLastWALSegmentBy(t *testing.T, walDir string, n int64) { - t.Helper() - path := lastWALSegment(t, walDir) - info, err := os.Stat(path) - require.NoError(t, err) - require.Greater(t, info.Size(), n) - require.NoError(t, os.Truncate(path, info.Size()-n)) -} - -func replaceManualWALTailWithPartialLengthPrefix(t *testing.T, walDir string, tailStart int64) { - t.Helper() - path := lastWALSegment(t, walDir) - require.NoError(t, os.Truncate(path, tailStart)) - f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0) - require.NoError(t, err) - _, err = f.Write([]byte{0x80}) - require.NoError(t, err) - require.NoError(t, f.Close()) -} - -func walSegmentSize(t *testing.T, walDir string) int64 { - t.Helper() - info, err := os.Stat(lastWALSegment(t, walDir)) - require.NoError(t, err) - return info.Size() -} - -func lastWALSegment(t *testing.T, walDir string) string { - t.Helper() - entries, err := os.ReadDir(walDir) - require.NoError(t, err) - var last string - for _, entry := range entries { - if entry.IsDir() || len(entry.Name()) < 20 { - continue - } - last = entry.Name() - } - require.NotEmpty(t, last, "expected at least one WAL segment in %s", walDir) - return filepath.Join(walDir, last) -} diff --git a/sei-db/state_db/sc/migration/migration_test_framework_test.go b/sei-db/state_db/sc/migration/migration_test_framework_test.go index 7c2afa793c..e9999d7558 100644 --- a/sei-db/state_db/sc/migration/migration_test_framework_test.go +++ b/sei-db/state_db/sc/migration/migration_test_framework_test.go @@ -622,7 +622,11 @@ func NewTestFlatKVCommitStore(t *testing.T, dir string) *flatkv.CommitStore { t.Helper() cfg := flatkvconfig.DefaultTestConfig(t) cfg.DataDir = dir - s, err := flatkv.NewCommitStore(t.Context(), cfg) + stateWAL, err := flatkv.OpenStateWAL(cfg) + if err != nil { + t.Fatalf("NewTestFlatKVCommitStore: OpenStateWAL: %v", err) + } + s, err := flatkv.NewCommitStore(t.Context(), cfg, stateWAL) if err != nil { t.Fatalf("NewTestFlatKVCommitStore: NewCommitStore: %v", err) } diff --git a/sei-db/state_db/statewal/state_wal.go b/sei-db/state_db/statewal/state_wal.go index cb7fa7dc74..543593cc34 100644 --- a/sei-db/state_db/statewal/state_wal.go +++ b/sei-db/state_db/statewal/state_wal.go @@ -27,12 +27,14 @@ type StateWAL interface { // A nil entry in cs is rejected synchronously with an error and leaves the WAL usable; cs itself may be // nil or empty. // - // The StateWAL rejects writes for blocks if provided out of order. To avoid errors, observe - // the following rules: + // Blocks must be written in contiguous ascending order, one at a time: // - // - The block numbers passed to Write() may never decrease. - // - If data has been written for block N, you cannot write data for block N+1 until you have called - // SignalEndOfBlock(). + // - The first block written to an empty WAL may be any number. Every block after it must be exactly one + // greater than the last, so forward jumps are rejected. This survives close and reopen. + // - Any number of Writes may target the current block until SignalEndOfBlock; afterwards that block is + // closed and writing to it again is an error. + // + // Violations are reported synchronously and leave the WAL usable. Write( // The block number associated with the changeset. blockNumber uint64, @@ -40,8 +42,8 @@ type StateWAL interface { cs []*proto.NamedChangeSet, ) error - // Signal that there will be no more writes for the current block number. Attempting to write additional - // changes for the same block number after calling this method may result in an error. + // Signal that there will be no more writes for the current block number. Writing additional changes for + // the same block number after calling this method is an error. // // Similar to Write(), this method is asynchronous. Calling this method does not, by itself, make // data immediately crash durable. diff --git a/sei-db/state_db/statewal/state_wal_impl.go b/sei-db/state_db/statewal/state_wal_impl.go index 5dbc9e7de1..053b0d75b7 100644 --- a/sei-db/state_db/statewal/state_wal_impl.go +++ b/sei-db/state_db/statewal/state_wal_impl.go @@ -100,6 +100,24 @@ func VerifyIntegrity(config *Config) error { return nil } +// Delete removes the configured state WAL directory and everything in it, so a subsequent New yields a +// fresh, empty WAL that recreates the directory. It is a no-op if the directory does not exist, and it does +// not construct a live StateWAL. +// +// It takes the exclusive WAL directory lock, so it fails with commonerrors.ErrFileLockUnavailable if a +// StateWAL is open on the same directory, and serializes against GetRange/PruneAfter/VerifyIntegrity there. +// FlatKV closes the WAL before deleting and reopening it on a state-sync restore, so the receiving node +// rebuilds a clean WAL aligned with the imported snapshot rather than splicing onto stale entries. +// +// Removing the directory unlinks the lock file, so exclusion ends there rather than when this returns. A +// caller that must hold the directory across the delete-then-reopen sequence needs its own outer lock. +func Delete(config *Config) error { + if err := seiwal.DeleteAll(config.Path); err != nil { + return fmt.Errorf("state WAL delete failed: %w", err) + } + return nil +} + func newStateWAL(wal seiwal.WAL[[]*proto.NamedChangeSet]) (StateWAL, error) { w := &stateWALImpl{wal: wal} diff --git a/sei-db/state_db/statewal/state_wal_reopen_test.go b/sei-db/state_db/statewal/state_wal_reopen_test.go new file mode 100644 index 0000000000..f4e028af22 --- /dev/null +++ b/sei-db/state_db/statewal/state_wal_reopen_test.go @@ -0,0 +1,141 @@ +package statewal + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + commonerrors "github.com/sei-protocol/sei-chain/sei-db/common/errors" +) + +// TestDeleteRemovesWALDirectory verifies Delete removes the WAL directory outright and that New recreates it, +// yielding a fresh, empty WAL. +func TestDeleteRemovesWALDirectory(t *testing.T) { + cfg := testConfig(t.TempDir()) + w := openWAL(t, cfg) + for block := uint64(1); block <= 3; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + require.NoError(t, w.Close()) + + require.NoError(t, Delete(cfg)) + require.NoDirExists(t, cfg.Path) + + w2 := openWAL(t, cfg) + defer func() { require.NoError(t, w2.Close()) }() + require.DirExists(t, cfg.Path, "New must recreate the directory Delete removed") + ok, _, _, err := w2.GetStoredRange() + require.NoError(t, err) + require.False(t, ok, "WAL should be empty after Delete") +} + +// TestDeleteMissingDirIsNoop verifies Delete on a directory that was never created is a clean no-op that does +// not create the directory. +func TestDeleteMissingDirIsNoop(t *testing.T) { + cfg := testConfig(filepath.Join(t.TempDir(), "never-created")) + require.NoError(t, Delete(cfg)) + require.NoDirExists(t, cfg.Path) +} + +// TestDeleteRejectedWhileWALOpen verifies Delete fails fast rather than wiping the directory out from under a +// live StateWAL, which would leave the writer appending to unlinked files. +func TestDeleteRejectedWhileWALOpen(t *testing.T) { + cfg := testConfig(t.TempDir()) + w := openWAL(t, cfg) + defer func() { require.NoError(t, w.Close()) }() + + writeBlock(t, w, 1) + require.NoError(t, w.Flush()) + + err := Delete(cfg) + require.ErrorIs(t, err, commonerrors.ErrFileLockUnavailable) + require.DirExists(t, cfg.Path) + + // The live WAL is untouched and still usable. + writeBlock(t, w, 2) + require.NoError(t, w.Flush()) + ok, first, last, err := w.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first) + require.Equal(t, uint64(2), last) +} + +// TestCloseDeleteReopenAcceptsFarAheadBlock models the state-sync restore case (D7): after wiping a WAL that +// held an old chain's blocks, the reopened WAL accepts a first write at a far-ahead height with no +// contiguity error — this is why restore must wipe rather than splice. +func TestCloseDeleteReopenAcceptsFarAheadBlock(t *testing.T) { + cfg := testConfig(t.TempDir()) + w := openWAL(t, cfg) + for block := uint64(1); block <= 5; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + require.NoError(t, w.Close()) + + require.NoError(t, Delete(cfg)) + + w2 := openWAL(t, cfg) + defer func() { require.NoError(t, w2.Close()) }() + + const restoredHeight = uint64(1000) + writeBlock(t, w2, restoredHeight) // fresh WAL accepts any first block + require.NoError(t, w2.Flush()) + + ok, start, end, err := w2.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, restoredHeight, start) + require.Equal(t, restoredHeight, end) +} + +// TestClosePruneAfterReopenResetsHead models the rollback case (D3): close, offline PruneAfter to the +// target, reopen — the stored range is truncated to the target and the write head resets, so the next write +// is contiguous at target+1. +func TestClosePruneAfterReopenResetsHead(t *testing.T) { + cfg := testConfig(t.TempDir()) + w := openWAL(t, cfg) + for block := uint64(1); block <= 5; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + require.NoError(t, w.Close()) + + require.NoError(t, PruneAfter(cfg, 3)) + + w2 := openWAL(t, cfg) + defer func() { require.NoError(t, w2.Close()) }() + + ok, start, end, err := w2.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), start) + require.Equal(t, uint64(3), end) + + writeBlock(t, w2, 4) // contiguous with the rolled-back head + require.NoError(t, w2.Flush()) + require.Equal(t, []uint64{1, 2, 3, 4}, collectBlocks(t, w2, 1, 4)) +} + +// TestReopenEnforcesContiguity verifies the contiguity rule survives close and reopen: the reopened WAL +// recovers its write head from the highest stored block, so a forward jump is still rejected. Only the +// positive direction (resuming at head+1) was covered before. +func TestReopenEnforcesContiguity(t *testing.T) { + cfg := testConfig(t.TempDir()) + w := openWAL(t, cfg) + for block := uint64(1); block <= 3; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + require.NoError(t, w.Close()) + + w2 := openWAL(t, cfg) + defer func() { require.NoError(t, w2.Close()) }() + + require.Error(t, w2.Write(5, nil), "a forward jump must be rejected after reopen, not just in-session") + writeBlock(t, w2, 4) + require.NoError(t, w2.Flush()) + require.Equal(t, []uint64{1, 2, 3, 4}, collectBlocks(t, w2, 1, 4)) +} diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_open.go b/sei-db/tools/cmd/seidb/operations/flatkv_open.go index 04c9ebe138..6c90e8e4fa 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_open.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_open.go @@ -12,16 +12,19 @@ import ( "strings" "syscall" - "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" - "github.com/sei-protocol/sei-chain/sei-db/wal" + "github.com/sei-protocol/sei-chain/sei-db/state_db/statewal" ) const ( flatkvSnapshotPrefix = "snapshot-" flatkvSnapshotDirLen = len(flatkvSnapshotPrefix) + 20 + // flatkvStateWALName matches the WAL instance name FlatKV opens its state WAL with; it only labels + // metrics, and the offline GetRange used here does not emit any, but keep it consistent. + flatkvStateWALName = "flatkv" + // maxCloneRetries bounds the number of retries when the source snapshot // is pruned mid-clone by a live writer (atomicRemoveDir race) or when the // live writer truncates the WAL past our snapshot between snapshot and @@ -89,8 +92,14 @@ func openFlatKVReadOnly(dbDir string, height int64) (*openedFlatKV, error) { cfg := config.DefaultConfig() cfg.DataDir = tempDir - store, err := flatkv.NewCommitStore(context.Background(), cfg) + stateWAL, err := flatkv.OpenStateWAL(cfg) if err != nil { + _ = os.RemoveAll(tempDir) + return nil, fmt.Errorf("failed to open FlatKV state WAL: %w", err) + } + store, err := flatkv.NewCommitStore(context.Background(), cfg, stateWAL) + if err != nil { + _ = stateWAL.Close() _ = os.RemoveAll(tempDir) return nil, fmt.Errorf("failed to create FlatKV store: %w", err) } @@ -198,56 +207,29 @@ func tryPrepareFlatKVToolingClone(dbDir string, height int64) (string, error) { return tempDir, nil } -// verifyClonedWALCovers opens the cloned WAL just long enough to ensure it +// verifyClonedWALCovers inspects the cloned WAL just long enough to ensure it // either is empty, ends at or before snapshotVersion (no replay needed), or -// starts at or before snapshotVersion+1 (catchup can resume cleanly). +// starts at or before snapshotVersion+1 (catchup can resume cleanly). The state +// WAL is keyed by block number, so its stored range is directly the version +// range; GetRange reads it offline without a live WAL instance. func verifyClonedWALCovers(dstChangelogDir string, snapshotVersion int64) error { - walLog, err := wal.NewChangelogWAL(dstChangelogDir, wal.Config{}) - if err != nil { - return fmt.Errorf("open cloned changelog for validation: %w", err) - } - defer func() { _ = walLog.Close() }() - - firstOff, err := walLog.FirstOffset() - if err != nil { - return fmt.Errorf("cloned changelog first offset: %w", err) - } - lastOff, err := walLog.LastOffset() + ok, firstVer, lastVer, err := statewal.GetRange(statewal.DefaultConfig(dstChangelogDir, flatkvStateWALName)) if err != nil { - return fmt.Errorf("cloned changelog last offset: %w", err) + return fmt.Errorf("read cloned changelog range: %w", err) } - if firstOff == 0 || lastOff == 0 || firstOff > lastOff { + if !ok { return nil } - - firstVer, err := readWALEntryVersion(walLog, firstOff) - if err != nil { - return fmt.Errorf("read first cloned changelog entry: %w", err) - } - lastVer, err := readWALEntryVersion(walLog, lastOff) - if err != nil { - return fmt.Errorf("read last cloned changelog entry: %w", err) - } - - if lastVer <= snapshotVersion { + if int64(lastVer) <= snapshotVersion { //nolint:gosec // version fits int64 return nil } - if firstVer <= snapshotVersion+1 { + if int64(firstVer) <= snapshotVersion+1 { //nolint:gosec // version fits int64 return nil } return fmt.Errorf("%w: cloned WAL starts at version %d but snapshot is %d (truncated past snapshot mid-clone)", errSourceChurning, firstVer, snapshotVersion) } -func readWALEntryVersion(walLog wal.ChangelogWAL, off uint64) (int64, error) { - var ver int64 - err := walLog.Replay(off, off, func(_ uint64, entry proto.ChangelogEntry) error { - ver = entry.Version - return nil - }) - return ver, err -} - func selectFlatKVSnapshot(dbDir string, height int64) (string, error) { if height == 0 { target, err := os.Readlink(filepath.Join(dbDir, "current")) diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go b/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go index 469a085e1b..7921e22912 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go @@ -13,7 +13,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" flatkvconfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" - "github.com/sei-protocol/sei-chain/sei-db/wal" + "github.com/sei-protocol/sei-chain/sei-db/state_db/statewal" "github.com/stretchr/testify/require" ) @@ -201,24 +201,24 @@ func TestPrepareFlatKVToolingCloneDetectsWALTruncationRace(t *testing.T) { } require.NoError(t, store.Close()) + // Simulate tryTruncateWAL having dropped v1..v3 from the WAL after a newer snapshot rolled — i.e. the + // WAL now starts at v4 while the selected snapshot is still v1. Rebuild the source changelog as a fresh + // state WAL that starts at v4 (the state WAL permits any first block number), which is the exact shape a + // front-truncation past the snapshot leaves behind. walDir := filepath.Join(dbDir, "changelog") - walLog, err := wal.NewChangelogWAL(walDir, wal.Config{}) + walCfg := statewal.DefaultConfig(walDir, "flatkv") + require.NoError(t, statewal.Delete(walCfg)) + w, err := statewal.New(walCfg) require.NoError(t, err) - first, err := walLog.FirstOffset() - require.NoError(t, err) - last, err := walLog.LastOffset() - require.NoError(t, err) - - var v4Off uint64 - require.NoError(t, walLog.Replay(first, last, func(off uint64, entry proto.ChangelogEntry) error { - if entry.Version == 4 && v4Off == 0 { - v4Off = off - } - return nil - })) - require.Greater(t, v4Off, uint64(0), "WAL should contain a v4 entry") - require.NoError(t, walLog.TruncateBefore(v4Off)) - require.NoError(t, walLog.Close()) + for v := uint64(4); v <= 5; v++ { + require.NoError(t, w.Write(v, []*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(byte(v)), v)}}, + }})) + require.NoError(t, w.SignalEndOfBlock()) + } + require.NoError(t, w.Flush()) + require.NoError(t, w.Close()) _, err = prepareFlatKVToolingClone(dbDir, 0) require.Error(t, err) @@ -297,7 +297,9 @@ func newDiskBackedFlatKVStore(t *testing.T) (*flatkv.CommitStore, string) { t.Helper() cfg := flatkvconfig.DefaultTestConfig(t) - store, err := flatkv.NewCommitStore(context.Background(), cfg) + stateWAL, err := flatkv.OpenStateWAL(cfg) + require.NoError(t, err) + store, err := flatkv.NewCommitStore(context.Background(), cfg, stateWAL) require.NoError(t, err) _, err = store.LoadVersion(0, false) require.NoError(t, err) diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_state_size_test.go b/sei-db/tools/cmd/seidb/operations/flatkv_state_size_test.go index d1bd4d0a3c..7e304ff973 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_state_size_test.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_state_size_test.go @@ -220,7 +220,10 @@ func TestFlatKVStateSizeAnalysisShape(t *testing.T) { func newTestFlatKVStore(t *testing.T) *flatkv.CommitStore { t.Helper() - s, err := flatkv.NewCommitStore(context.Background(), flatkvconfig.DefaultTestConfig(t)) + cfg := flatkvconfig.DefaultTestConfig(t) + stateWAL, err := flatkv.OpenStateWAL(cfg) + require.NoError(t, err) + s, err := flatkv.NewCommitStore(context.Background(), cfg, stateWAL) require.NoError(t, err) _, err = s.LoadVersion(0, false) require.NoError(t, err) diff --git a/sei-db/tools/cmd/seidb/operations/import_flatkv_from_memiavl.go b/sei-db/tools/cmd/seidb/operations/import_flatkv_from_memiavl.go index 6ce0446aa2..d914515226 100644 --- a/sei-db/tools/cmd/seidb/operations/import_flatkv_from_memiavl.go +++ b/sei-db/tools/cmd/seidb/operations/import_flatkv_from_memiavl.go @@ -198,8 +198,13 @@ func importMemiavlModulesToFlatKV(ctx context.Context, homeDir string, modules [ cfg := flatkvconfig.DefaultConfig() cfg.DataDir = utils.GetFlatKVPath(homeDir) - store, err := flatkv.NewCommitStore(ctx, cfg) + stateWAL, err := flatkv.OpenStateWAL(cfg) if err != nil { + return fmt.Errorf("failed to open FlatKV state WAL: %w", err) + } + store, err := flatkv.NewCommitStore(ctx, cfg, stateWAL) + if err != nil { + _ = stateWAL.Close() return fmt.Errorf("failed to create FlatKV store: %w", err) } defer func() { _ = store.Close() }() diff --git a/sei-db/tools/cmd/seidb/operations/import_flatkv_from_memiavl_test.go b/sei-db/tools/cmd/seidb/operations/import_flatkv_from_memiavl_test.go index 33b9c4be5e..c448d01516 100644 --- a/sei-db/tools/cmd/seidb/operations/import_flatkv_from_memiavl_test.go +++ b/sei-db/tools/cmd/seidb/operations/import_flatkv_from_memiavl_test.go @@ -355,7 +355,9 @@ func newTestFlatKVStoreAtHome(t *testing.T, homeDir string) *flatkv.CommitStore t.Helper() cfg := flatkvconfig.DefaultTestConfig(t) cfg.DataDir = utils.GetFlatKVPath(homeDir) - store, err := flatkv.NewCommitStore(context.Background(), cfg) + stateWAL, err := flatkv.OpenStateWAL(cfg) + require.NoError(t, err) + store, err := flatkv.NewCommitStore(context.Background(), cfg, stateWAL) require.NoError(t, err) _, err = store.LoadVersion(0, false) require.NoError(t, err)