From 326e505a97620c3aaf5dce8c245eb523ae35109f Mon Sep 17 00:00:00 2001 From: YimingZang Date: Thu, 30 Jul 2026 12:48:52 -0700 Subject: [PATCH 01/15] Unify and make garbage collector more generic --- sei-db/config/giga_config.go | 35 +++ sei-db/ledger_db/event/placeholder.go | 1 - sei-db/ledger_db/transaction/placeholder.go | 1 - .../management/storage_garbage_collector.go | 256 +++++++++++++----- .../storage_garbage_collector_test.go | 254 +++++++++++++++-- sei-db/management/store_types.go | 72 +++-- 6 files changed, 504 insertions(+), 115 deletions(-) create mode 100644 sei-db/config/giga_config.go delete mode 100644 sei-db/ledger_db/event/placeholder.go delete mode 100644 sei-db/ledger_db/transaction/placeholder.go diff --git a/sei-db/config/giga_config.go b/sei-db/config/giga_config.go new file mode 100644 index 0000000000..581b67fd7a --- /dev/null +++ b/sei-db/config/giga_config.go @@ -0,0 +1,35 @@ +package config + +import ( + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/littblock" + flatkvConfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" +) + +const DefaultRollbackWindow = 1000 +const DefaultRetention = 1000000 + +type GigaStorageConfig struct { + RollbackWindow uint64 + BlockRetention uint64 + DataDirectory string + FlatKVConfig *flatkvConfig.Config + SSConfig StateStoreConfig + ReceiptDBConfig ReceiptStoreConfig + BlockDBConfig *littblock.LittBlockConfig +} + +func DefaultGigaStorageConfig(dbDir string) GigaStorageConfig { + blockDBConfig, err := littblock.DefaultConfig(dbDir) + if err != nil { + panic(err) + } + return GigaStorageConfig{ + RollbackWindow: DefaultRollbackWindow, + BlockRetention: DefaultRetention, + DataDirectory: dbDir, + FlatKVConfig: flatkvConfig.DefaultConfig(), + SSConfig: DefaultStateStoreConfig(), + ReceiptDBConfig: DefaultReceiptStoreConfig(), + BlockDBConfig: blockDBConfig, + } +} diff --git a/sei-db/ledger_db/event/placeholder.go b/sei-db/ledger_db/event/placeholder.go deleted file mode 100644 index 0e4b82e85c..0000000000 --- a/sei-db/ledger_db/event/placeholder.go +++ /dev/null @@ -1 +0,0 @@ -package event diff --git a/sei-db/ledger_db/transaction/placeholder.go b/sei-db/ledger_db/transaction/placeholder.go deleted file mode 100644 index 0619207533..0000000000 --- a/sei-db/ledger_db/transaction/placeholder.go +++ /dev/null @@ -1 +0,0 @@ -package transaction diff --git a/sei-db/management/storage_garbage_collector.go b/sei-db/management/storage_garbage_collector.go index 20ad25f2e2..040d629d06 100644 --- a/sei-db/management/storage_garbage_collector.go +++ b/sei-db/management/storage_garbage_collector.go @@ -11,7 +11,10 @@ import ( var logger = seilog.NewLogger("db", "management") -// StorageGarbageCollector manages deletion of stored data across a set of state stores and the StateWAL. +// StorageGarbageCollector manages deletion of stored data across a set of stores. Each store declares its +// StoreType, which determines both how the collector observes what the store retains and how far the store is +// allowed to prune: a SnapshotStore retains whole snapshots, while a StreamStore (e.g. the state WAL) retains a +// contiguous range of blocks that the snapshot stores are replayed from. // // The StorageGarbageCollector performs state deletion while maintaining the following invariant: // "If it's possible to roll back at least config.RollbackWindow blocks, then any state deletion operation @@ -26,11 +29,8 @@ type StorageGarbageCollector struct { // Configuration for this storage garbage collector. config *StorageGarbageCollectorConfig - // The stores whose data can be rebuilt by replaying the state WAL. - stateStores []SnapshotStore - - // The WAL containing changesets for each block (i.e. the key-value pairs that change each block). - stateWAL StreamStore + // The stores this collector prunes. + stores []PrunableStore // Cancelled to signal the run loop to stop. ctx context.Context @@ -45,20 +45,26 @@ type StorageGarbageCollector struct { func NewStorageGarbageCollector( ctx context.Context, config *StorageGarbageCollectorConfig, - stateStores []SnapshotStore, - stateWAL StreamStore, + stores []PrunableStore, ) (*StorageGarbageCollector, error) { if err := config.Validate(); err != nil { return nil, fmt.Errorf("invalid storage garbage collector config: %w", err) } + for _, store := range stores { + switch storeType := store.GetStoreType(); storeType { + case SnapshotStore, StreamStore: + default: + return nil, fmt.Errorf("store %s has unsupported store type %s", store.Name(), storeType) + } + } + s := &StorageGarbageCollector{ - config: config, - stateStores: stateStores, - stateWAL: stateWAL, - ctx: ctx, - stopCh: make(chan struct{}), + config: config, + stores: stores, + ctx: ctx, + stopCh: make(chan struct{}), } s.wg.Add(1) @@ -90,7 +96,7 @@ func (s *StorageGarbageCollector) run() { case <-s.ctx.Done(): return case <-ticker.C: - if err := prune(s.config.RollbackWindow, s.stateStores, s.stateWAL); err != nil { + if err := prune(s.config.RollbackWindow, s.stores); err != nil { logger.Error("prune cycle failed", "err", err) } } @@ -99,43 +105,50 @@ func (s *StorageGarbageCollector) run() { // prune performs a single prune cycle: it observes the blocks retained by each managed store, computes how far each // store may prune while preserving the rollback window, and issues the prune commands. -func prune( - rollbackWindow uint64, - stateStores []SnapshotStore, - stateWAL StreamStore, -) error { - stateWALStart, stateWALEnd, stateWALHasData, err := stateWAL.GetStoredBlocks() +func prune(rollbackWindow uint64, stores []PrunableStore) error { + observations, err := observeStores(stores) if err != nil { - return fmt.Errorf("failed to read stored blocks from %s: %w", stateWAL.Name(), err) + return err } - stateStoreBlocks := make([][]uint64, len(stateStores)) - anyStoreEmpty := false - for i, store := range stateStores { - blocks, err := store.GetStoredBlocks() - if err != nil { - return fmt.Errorf("failed to read stored blocks from %s: %w", store.Name(), err) - } - stateStoreBlocks[i] = blocks - if len(blocks) == 0 { - anyStoreEmpty = true + if len(observations) == 0 { + return nil + } + + allStoresHaveData := true + for _, obs := range observations { + if !obs.hasData { + allStoresHaveData = false } } - if !stateWALHasData || anyStoreEmpty { - // We only prune if the WAL and every provided store has at least some data. An empty store is treated as - // unknown rather than empty (a snapshot may be mid-write and take hours), and pruning against it would risk - // breaking the primary invariant (i.e. breaking the ability to roll back by the act of deletion). - logArgs := []any{"stateWAL", blockRange(stateWALStart, stateWALEnd), "stateWALHasData", stateWALHasData} - for i, store := range stateStores { - logArgs = append(logArgs, store.Name(), stateStoreBlocks[i]) + if !allStoresHaveData { + // We only prune if every provided store has at least some data. An empty store is treated as unknown rather + // than empty (a snapshot may be mid-write and take hours), and pruning against it would risk breaking the + // primary invariant (i.e. breaking the ability to roll back by the act of deletion). + logArgs := make([]any, 0, 2*len(observations)) + for _, obs := range observations { + logArgs = append(logArgs, obs.store.Name(), obs.describeRetained()) } logger.Info("skipping pruning, not all stores have data", logArgs...) return nil } - // The latest committed block is defined by the head of the state WAL. - latestBlock := stateWALEnd + latestBlock, highestBlock := committedBlockRange(observations) + + // The lowest head sets the rollback target for every store, so a store trailing the rest by more than the whole + // rollback window is throttling how much anything else can prune. That errs toward retention and is safe, but it + // usually means the store is stalled or is reporting the wrong height, and it is otherwise invisible: pruning + // still looks like it is working, it just stops reclaiming much. + if highestBlock-latestBlock > rollbackWindow { + logArgs := make([]any, 0, 2+2*len(observations)) + logArgs = append(logArgs, "rollbackWindow", rollbackWindow) + for _, obs := range observations { + logArgs = append(logArgs, obs.store.Name()+"Head", obs.latestBlock) + } + logger.Warn("store heads disagree by more than the rollback window, pruning is limited by the "+ + "furthest behind store", logArgs...) + } // The oldest block we must remain able to roll back to. var oldestBlockNeeded uint64 @@ -143,51 +156,160 @@ func prune( oldestBlockNeeded = latestBlock - rollbackWindow } - floors := make([]uint64, len(stateStores)) - for i, blocks := range stateStoreBlocks { - floors[i] = snapshotPruningFloor(blocks, oldestBlockNeeded) - } - - // The WAL retains from the oldest block any state store still needs. With no state stores, nothing depends on the - // WAL for rollback, so it retains only the rollback window. - stateWALFloor := oldestBlockNeeded - for i, floor := range floors { - if i == 0 || floor < stateWALFloor { - stateWALFloor = floor - } - } + floors := pruningFloors(observations, oldestBlockNeeded) logArgs := []any{"latestBlock", latestBlock, "oldestBlockNeeded", oldestBlockNeeded} - for i, store := range stateStores { + for i, obs := range observations { logArgs = append(logArgs, - store.Name()+"Initial", stateStoreBlocks[i], - store.Name()+"Final", blocksAtOrAbove(stateStoreBlocks[i], floors[i]), + obs.store.Name()+"Initial", obs.describeRetained(), + obs.store.Name()+"Final", obs.describeRetainedAfterPruning(floors[i]), ) } - logArgs = append(logArgs, - "stateWALInitial", blockRange(stateWALStart, stateWALEnd), - "stateWALFinal", blockRange(stateWALFloor, stateWALEnd), - ) logger.Info("pruning storage", logArgs...) - for i, store := range stateStores { - if err := store.PruneBelow(floors[i]); err != nil { - return fmt.Errorf("failed to prune %s below %d: %w", store.Name(), floors[i], err) + for i, obs := range observations { + if err := obs.store.PruneBelow(floors[i]); err != nil { + return fmt.Errorf("failed to prune %s below %d: %w", obs.store.Name(), floors[i], err) } } - if err := stateWAL.PruneBelow(stateWALFloor); err != nil { - return fmt.Errorf("failed to prune %s below %d: %w", stateWAL.Name(), stateWALFloor, err) - } return nil } +// observation is what a single prune cycle read back from one store. +type observation struct { + store PrunableStore + storeType StoreType + + // The snapshots held by a SnapshotStore, ascending. Always empty for a StreamStore. + snapshots []uint64 + + // The inclusive range of blocks held by a StreamStore. Meaningful only when hasData is true. + start uint64 + end uint64 + + // The highest block the store has ingested. Meaningful only when hasData is true. + latestBlock uint64 + + // Whether the store holds any blocks at all. + hasData bool +} + +// observeStores reads what every store currently retains, using the accessor that matches each store's type. +// Observation happens up front so that a read failure aborts the cycle before anything has been deleted. +func observeStores(stores []PrunableStore) ([]observation, error) { + observations := make([]observation, len(stores)) + for i, store := range stores { + obs := observation{store: store, storeType: store.GetStoreType()} + + switch obs.storeType { + case SnapshotStore: + snapshots, err := store.GetStoredSnapshots() + if err != nil { + return nil, fmt.Errorf("failed to read stored snapshots from %s: %w", store.Name(), err) + } + obs.snapshots = snapshots + obs.hasData = len(snapshots) > 0 + case StreamStore: + start, end, hasData, err := store.GetBlockRange() + if err != nil { + return nil, fmt.Errorf("failed to read block range from %s: %w", store.Name(), err) + } + obs.start, obs.end, obs.hasData = start, end, hasData + default: + return nil, fmt.Errorf("store %s has unsupported store type %s", store.Name(), obs.storeType) + } + + latestBlock, err := store.GetLastCommittedBlock() + if err != nil { + return nil, fmt.Errorf("failed to read last committed block from %s: %w", store.Name(), err) + } + obs.latestBlock = latestBlock + + observations[i] = obs + } + return observations, nil +} + +// committedBlockRange returns the lowest and highest heads reported across the stores. +// +// The lowest head is the head of the chain as the collector understands it, because it bounds what the system can +// actually serve; treating a lagging store as caught up would place the rollback target above the blocks that store +// still holds. The highest is reported alongside it so callers can see how far the stores disagree. Callers must have +// verified that there is at least one store and that every store has data. +func committedBlockRange(observations []observation) (lowest uint64, highest uint64) { + for i, obs := range observations { + if i == 0 || obs.latestBlock < lowest { + lowest = obs.latestBlock + } + if i == 0 || obs.latestBlock > highest { + highest = obs.latestBlock + } + } + return lowest, highest +} + +// pruningFloors computes the block each store may prune below, indexed in parallel with observations. +// +// A snapshot store retains the newest snapshot at or below oldestBlockNeeded, since that is the snapshot a rollback to +// oldestBlockNeeded would start from. Every stream store retains from the oldest block any snapshot store still needs, +// because a rollback replays the stream forward from that snapshot. With no snapshot stores nothing depends on the +// streams for rollback, so they retain only the rollback window. +// +// A stream store must not simply retain from oldestBlockNeeded: snapshots land at arbitrary heights, so the newest +// snapshot at or below oldestBlockNeeded can sit far below it. With snapshots at 80,000 and 92,000 and +// oldestBlockNeeded of 90,000, the 80,000 snapshot is the only one a rollback to 90,000 can start from, and replaying +// forward to 90,000 needs stream entries 80,001 onward. Pruning the stream to 90,000 would strand that snapshot. +func pruningFloors(observations []observation, oldestBlockNeeded uint64) []uint64 { + floors := make([]uint64, len(observations)) + + streamFloor := oldestBlockNeeded + seenSnapshotStore := false + for i, obs := range observations { + if obs.storeType != SnapshotStore { + continue + } + floors[i] = snapshotPruningFloor(obs.snapshots, oldestBlockNeeded) + if !seenSnapshotStore || floors[i] < streamFloor { + streamFloor = floors[i] + } + seenSnapshotStore = true + } + + for i, obs := range observations { + if obs.storeType == StreamStore { + floors[i] = streamFloor + } + } + + return floors +} + +// describeRetained renders what the store currently holds, for logging. +func (o observation) describeRetained() string { + if !o.hasData { + return "empty" + } + if o.storeType == SnapshotStore { + return fmt.Sprintf("%v", o.snapshots) + } + return blockRange(o.start, o.end) +} + +// describeRetainedAfterPruning renders what the store is expected to hold once PruneBelow(floor) completes. +func (o observation) describeRetainedAfterPruning(floor uint64) string { + if o.storeType == SnapshotStore { + return fmt.Sprintf("%v", blocksAtOrAbove(o.snapshots, floor)) + } + return blockRange(floor, o.end) +} + // Given a list of snapshot block numbers, determine the lowest snapshot we need to keep in order to be able // to roll back to the target block number. // // Returns the highest numbered block from snapshotBlocks that is less than or equal to the rollbackTarget. If every // snapshot is greater than rollbackTarget, the lowest snapshot is returned, since none can be safely pruned. -// snapshotBlocks must be non-empty and sorted in ascending order, per the SnapshotStore contract. +// snapshotBlocks must be non-empty and sorted in ascending order, per the PrunableStore contract. func snapshotPruningFloor( // Blocks we have snapshots for, in ascending order. Must be non-empty. snapshotBlocks []uint64, diff --git a/sei-db/management/storage_garbage_collector_test.go b/sei-db/management/storage_garbage_collector_test.go index 8e8d0997e9..65a7fb5567 100644 --- a/sei-db/management/storage_garbage_collector_test.go +++ b/sei-db/management/storage_garbage_collector_test.go @@ -12,22 +12,35 @@ import ( "github.com/stretchr/testify/require" ) -// mockSnapshotStore is a hand-written SnapshotStore for tests. It records the last PruneBelow argument and returns -// canned GetStoredBlocks data or errors. +// mockSnapshotStore is a hand-written PrunableStore of type SnapshotStore for tests. It records the last PruneBelow +// argument and returns canned GetStoredSnapshots data or errors. type mockSnapshotStore struct { name string blocks []uint64 - getErr error + // The committed height, which for a snapshot store normally sits above its newest snapshot. + latestHeight uint64 + getErr error pruneBelowCalled bool prunedBelow uint64 pruneErr error } -func (m *mockSnapshotStore) GetStoredBlocks() ([]uint64, error) { +func (m *mockSnapshotStore) GetStoredSnapshots() ([]uint64, error) { return m.blocks, m.getErr } +func (m *mockSnapshotStore) GetLastCommittedBlock() (uint64, error) { + return m.latestHeight, m.getErr +} + +func (m *mockSnapshotStore) GetBlockRange() (uint64, uint64, bool, error) { + if len(m.blocks) == 0 { + return 0, 0, false, m.getErr + } + return m.blocks[0], m.blocks[len(m.blocks)-1], true, m.getErr +} + func (m *mockSnapshotStore) PruneBelow(blockNumber uint64) error { m.pruneBelowCalled = true m.prunedBelow = blockNumber @@ -38,7 +51,11 @@ func (m *mockSnapshotStore) Name() string { return m.name } -// mockStreamStore is a hand-written StreamStore for tests. +func (m *mockSnapshotStore) GetStoreType() StoreType { + return SnapshotStore +} + +// mockStreamStore is a hand-written PrunableStore of type StreamStore for tests. type mockStreamStore struct { name string start uint64 @@ -51,10 +68,18 @@ type mockStreamStore struct { pruneErr error } -func (m *mockStreamStore) GetStoredBlocks() (uint64, uint64, bool, error) { +func (m *mockStreamStore) GetStoredSnapshots() ([]uint64, error) { + return nil, m.getErr +} + +func (m *mockStreamStore) GetBlockRange() (uint64, uint64, bool, error) { return m.start, m.end, m.hasData, m.getErr } +func (m *mockStreamStore) GetLastCommittedBlock() (uint64, error) { + return m.end, m.getErr +} + func (m *mockStreamStore) PruneBelow(blockNumber uint64) error { m.pruneBelowCalled = true m.prunedBelow = blockNumber @@ -65,10 +90,27 @@ func (m *mockStreamStore) Name() string { return m.name } -func toStores(s []*mockSnapshotStore) []SnapshotStore { - result := make([]SnapshotStore, len(s)) - for i, store := range s { - result[i] = store +func (m *mockStreamStore) GetStoreType() StoreType { + return StreamStore +} + +// mockUnknownStore reports a store type the collector does not understand. +type mockUnknownStore struct { + mockSnapshotStore +} + +func (m *mockUnknownStore) GetStoreType() StoreType { + return StoreType(0) +} + +// storeList assembles the single slice prune operates on: the snapshot stores followed by the stream stores. +func storeList(snapshotStores []*mockSnapshotStore, streamStores ...*mockStreamStore) []PrunableStore { + result := make([]PrunableStore, 0, len(snapshotStores)+len(streamStores)) + for _, store := range snapshotStores { + result = append(result, store) + } + for _, store := range streamStores { + result = append(result, store) } return result } @@ -194,13 +236,18 @@ func TestPruneDecisions(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { + // Every snapshot store is caught up with the WAL, so the WAL head decides the latest block. mocks := make([]*mockSnapshotStore, len(tc.storeBlocks)) for i, blocks := range tc.storeBlocks { - mocks[i] = &mockSnapshotStore{name: fmt.Sprintf("store%d", i), blocks: blocks} + mocks[i] = &mockSnapshotStore{ + name: fmt.Sprintf("store%d", i), + blocks: blocks, + latestHeight: tc.wal.end, + } } tc.wal.name = "stateWAL" - require.NoError(t, prune(tc.rollbackWindow, toStores(mocks), tc.wal)) + require.NoError(t, prune(tc.rollbackWindow, storeList(mocks, tc.wal))) for i, want := range tc.wantStores { if want == nil { @@ -220,12 +267,82 @@ func TestPruneDecisions(t *testing.T) { } } +// TestPruneMultipleStreamStores checks that the lowest stream head defines the latest committed block, and that every +// stream store is pruned to the floor of the snapshot stores. +func TestPruneMultipleStreamStores(t *testing.T) { + a := &mockSnapshotStore{name: "a", blocks: []uint64{80_000, 90_000}, latestHeight: 100_000} + ahead := &mockStreamStore{name: "ahead", start: 1, end: 100_000, hasData: true} + behind := &mockStreamStore{name: "behind", start: 1, end: 95_000, hasData: true} + + // latestBlock is the lower head (95,000), so oldestBlockNeeded is 85,000 and the newest snapshot at or below it + // is 80,000. Had the higher head won, the snapshot floor would have been 90,000 instead. + require.NoError(t, prune(10_000, storeList([]*mockSnapshotStore{a}, ahead, behind))) + + require.Equal(t, uint64(80_000), a.prunedBelow) + require.Equal(t, uint64(80_000), ahead.prunedBelow) + require.Equal(t, uint64(80_000), behind.prunedBelow) +} + +func TestPruneMultipleStreamStoresOneEmpty(t *testing.T) { + a := &mockSnapshotStore{name: "a", blocks: []uint64{80_000}, latestHeight: 100_000} + populated := &mockStreamStore{name: "populated", start: 1, end: 100_000, hasData: true} + empty := &mockStreamStore{name: "empty", hasData: false} + + require.NoError(t, prune(10_000, storeList([]*mockSnapshotStore{a}, populated, empty))) + + require.False(t, a.pruneBelowCalled) + require.False(t, populated.pruneBelowCalled) + require.False(t, empty.pruneBelowCalled) +} + +// TestPruneLaggingSnapshotStoreLowersLatestBlock checks that the head comes from every store, not just the streams: a +// snapshot store that has fallen behind the WAL pulls the rollback target down with it. +func TestPruneLaggingSnapshotStoreLowersLatestBlock(t *testing.T) { + // Caught up with the WAL: oldestBlockNeeded is 90,000, so the 90,000 snapshot is the one to keep. + caughtUp := &mockSnapshotStore{name: "caughtUp", blocks: []uint64{80_000, 90_000}, latestHeight: 100_000} + require.NoError(t, prune(10_000, storeList( + []*mockSnapshotStore{caughtUp}, + &mockStreamStore{name: "stateWAL", start: 1, end: 100_000, hasData: true}, + ))) + require.Equal(t, uint64(90_000), caughtUp.prunedBelow) + + // Lagging at 95,000 against the same WAL: oldestBlockNeeded drops to 85,000, so 80,000 must be retained. + lagging := &mockSnapshotStore{name: "lagging", blocks: []uint64{80_000, 90_000}, latestHeight: 95_000} + wal := &mockStreamStore{name: "stateWAL", start: 1, end: 100_000, hasData: true} + require.NoError(t, prune(10_000, storeList([]*mockSnapshotStore{lagging}, wal))) + require.Equal(t, uint64(80_000), lagging.prunedBelow) + require.Equal(t, uint64(80_000), wal.prunedBelow) +} + +// TestPruneWithoutStreamStore checks that a snapshot-only store set still prunes; nothing depends on a stream store +// being present. +func TestPruneWithoutStreamStore(t *testing.T) { + a := &mockSnapshotStore{name: "a", blocks: []uint64{80_000, 90_000}, latestHeight: 100_000} + b := &mockSnapshotStore{name: "b", blocks: []uint64{85_000, 92_000}, latestHeight: 100_000} + + require.NoError(t, prune(10_000, storeList([]*mockSnapshotStore{a, b}))) + + require.Equal(t, uint64(90_000), a.prunedBelow) + require.Equal(t, uint64(85_000), b.prunedBelow) +} + +// TestPruneStoreOrderDoesNotMatter checks that stores are classified by type rather than by position. +func TestPruneStoreOrderDoesNotMatter(t *testing.T) { + a := &mockSnapshotStore{name: "a", blocks: []uint64{85_000, 92_000}, latestHeight: 100_000} + wal := &mockStreamStore{name: "stateWAL", start: 1, end: 100_000, hasData: true} + + require.NoError(t, prune(10_000, []PrunableStore{wal, a})) + + require.Equal(t, uint64(85_000), a.prunedBelow) + require.Equal(t, uint64(85_000), wal.prunedBelow) +} + func TestPruneWALGetError(t *testing.T) { sentinel := errors.New("boom") - a := &mockSnapshotStore{name: "a", blocks: []uint64{80_000}} + a := &mockSnapshotStore{name: "a", blocks: []uint64{80_000}, latestHeight: 100_000} wal := &mockStreamStore{name: "stateWAL", getErr: sentinel} - err := prune(10_000, toStores([]*mockSnapshotStore{a}), wal) + err := prune(10_000, storeList([]*mockSnapshotStore{a}, wal)) require.ErrorIs(t, err, sentinel) require.ErrorContains(t, err, "stateWAL") require.False(t, a.pruneBelowCalled) @@ -237,7 +354,7 @@ func TestPruneStoreGetError(t *testing.T) { a := &mockSnapshotStore{name: "commitStore", getErr: sentinel} wal := &mockStreamStore{name: "stateWAL", start: 1, end: 100_000, hasData: true} - err := prune(10_000, toStores([]*mockSnapshotStore{a}), wal) + err := prune(10_000, storeList([]*mockSnapshotStore{a}, wal)) require.ErrorIs(t, err, sentinel) require.ErrorContains(t, err, "commitStore") require.False(t, wal.pruneBelowCalled) @@ -246,11 +363,11 @@ func TestPruneStoreGetError(t *testing.T) { func TestPruneStorePruneErrorStopsBeforeLaterStoresAndWAL(t *testing.T) { sentinel := errors.New("boom") // The first store fails to prune; later stores and the WAL must be left untouched. - a := &mockSnapshotStore{name: "a", blocks: []uint64{80_000}, pruneErr: sentinel} - b := &mockSnapshotStore{name: "b", blocks: []uint64{80_000}} + a := &mockSnapshotStore{name: "a", blocks: []uint64{80_000}, latestHeight: 100_000, pruneErr: sentinel} + b := &mockSnapshotStore{name: "b", blocks: []uint64{80_000}, latestHeight: 100_000} wal := &mockStreamStore{name: "stateWAL", start: 1, end: 100_000, hasData: true} - err := prune(10_000, toStores([]*mockSnapshotStore{a, b}), wal) + err := prune(10_000, storeList([]*mockSnapshotStore{a, b}, wal)) require.ErrorIs(t, err, sentinel) require.ErrorContains(t, err, "a") require.True(t, a.pruneBelowCalled) @@ -261,11 +378,11 @@ func TestPruneStorePruneErrorStopsBeforeLaterStoresAndWAL(t *testing.T) { func TestPruneWALPruneError(t *testing.T) { sentinel := errors.New("boom") // All stores prune successfully, then the WAL prune fails. - a := &mockSnapshotStore{name: "a", blocks: []uint64{80_000}} - b := &mockSnapshotStore{name: "b", blocks: []uint64{85_000}} + a := &mockSnapshotStore{name: "a", blocks: []uint64{80_000}, latestHeight: 100_000} + b := &mockSnapshotStore{name: "b", blocks: []uint64{85_000}, latestHeight: 100_000} wal := &mockStreamStore{name: "stateWAL", start: 1, end: 100_000, hasData: true, pruneErr: sentinel} - err := prune(10_000, toStores([]*mockSnapshotStore{a, b}), wal) + err := prune(10_000, storeList([]*mockSnapshotStore{a, b}, wal)) require.ErrorIs(t, err, sentinel) require.ErrorContains(t, err, "stateWAL") require.ErrorContains(t, err, "80000") // WAL floor = min(store floors) @@ -274,6 +391,54 @@ func TestPruneWALPruneError(t *testing.T) { require.True(t, wal.pruneBelowCalled) } +func TestCommittedBlockRange(t *testing.T) { + observationsWithHeads := func(heads ...uint64) []observation { + observations := make([]observation, len(heads)) + for i, head := range heads { + observations[i] = observation{latestBlock: head} + } + return observations + } + + cases := []struct { + name string + heads []uint64 + wantLowest uint64 + wantHighest uint64 + }{ + {name: "single store", heads: []uint64{100}, wantLowest: 100, wantHighest: 100}, + {name: "all agree", heads: []uint64{100, 100, 100}, wantLowest: 100, wantHighest: 100}, + {name: "lowest first", heads: []uint64{80, 100}, wantLowest: 80, wantHighest: 100}, + {name: "lowest last", heads: []uint64{100, 80}, wantLowest: 80, wantHighest: 100}, + {name: "lowest in the middle", heads: []uint64{100, 50, 90}, wantLowest: 50, wantHighest: 100}, + // A store reporting zero must not be mistaken for a store that was skipped. + {name: "a store reports zero", heads: []uint64{0, 100}, wantLowest: 0, wantHighest: 100}, + {name: "every store reports zero", heads: []uint64{0, 0}, wantLowest: 0, wantHighest: 0}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + lowest, highest := committedBlockRange(observationsWithHeads(tc.heads...)) + require.Equal(t, tc.wantLowest, lowest, "lowest head") + require.Equal(t, tc.wantHighest, highest, "highest head") + }) + } +} + +// TestPruneWithLaggingStoreStillPrunes checks that a head spread wide enough to trigger the warning does not change the +// outcome: the cycle still prunes, using the lowest head. +func TestPruneWithLaggingStoreStillPrunes(t *testing.T) { + lagging := &mockSnapshotStore{name: "lagging", blocks: []uint64{50_000, 80_000}, latestHeight: 60_000} + wal := &mockStreamStore{name: "stateWAL", start: 1, end: 100_000, hasData: true} + + // Heads are 60,000 and 100,000, a spread of 40,000 against a 10,000 window. The lowest head wins, so + // oldestBlockNeeded is 50,000 and the 50,000 snapshot is the one retained. + require.NoError(t, prune(10_000, storeList([]*mockSnapshotStore{lagging}, wal))) + + require.Equal(t, uint64(50_000), lagging.prunedBelow) + require.Equal(t, uint64(50_000), wal.prunedBelow) +} + func TestSnapshotPruningFloor(t *testing.T) { cases := []struct { name string @@ -387,21 +552,58 @@ func TestNewStorageGarbageCollectorInvalidConfig(t *testing.T) { sm, err := NewStorageGarbageCollector( context.Background(), &StorageGarbageCollectorConfig{PruneIntervalSeconds: 0}, - toStores([]*mockSnapshotStore{{name: "a"}}), - &mockStreamStore{name: "stateWAL"}, + storeList([]*mockSnapshotStore{{name: "a"}}, &mockStreamStore{name: "stateWAL"}), ) require.Error(t, err) require.Nil(t, sm) } +func TestNewStorageGarbageCollectorRejectsUnsupportedStoreType(t *testing.T) { + sm, err := NewStorageGarbageCollector( + context.Background(), + &StorageGarbageCollectorConfig{RollbackWindow: 10, PruneIntervalSeconds: 60}, + []PrunableStore{ + &mockUnknownStore{mockSnapshotStore{name: "mystery"}}, + &mockStreamStore{name: "stateWAL"}, + }, + ) + require.ErrorContains(t, err, "mystery") + require.ErrorContains(t, err, "unsupported store type") + require.Nil(t, sm) +} + +func TestNewStorageGarbageCollectorAcceptsAnyMixOfStoreTypes(t *testing.T) { + config := &StorageGarbageCollectorConfig{RollbackWindow: 10, PruneIntervalSeconds: 60} + + for _, tc := range []struct { + name string + stores []PrunableStore + }{ + {name: "snapshot stores only", stores: storeList([]*mockSnapshotStore{{name: "a", blocks: []uint64{100}}})}, + {name: "stream stores only", stores: storeList(nil, &mockStreamStore{name: "stateWAL"})}, + {name: "no stores", stores: nil}, + } { + t.Run(tc.name, func(t *testing.T) { + sm, err := NewStorageGarbageCollector(context.Background(), config, tc.stores) + require.NoError(t, err) + require.NoError(t, sm.Close()) + }) + } +} + +func TestStoreTypeString(t *testing.T) { + require.Equal(t, "SnapshotStore", SnapshotStore.String()) + require.Equal(t, "StreamStore", StreamStore.String()) + require.Equal(t, "UnknownStoreType(0)", StoreType(0).String()) +} + func TestNewStorageGarbageCollectorConstructAndClose(t *testing.T) { - a := &mockSnapshotStore{name: "a", blocks: []uint64{100}} + a := &mockSnapshotStore{name: "a", blocks: []uint64{100}, latestHeight: 100} wal := &mockStreamStore{name: "stateWAL", start: 1, end: 100, hasData: true} sm, err := NewStorageGarbageCollector( context.Background(), &StorageGarbageCollectorConfig{RollbackWindow: 10, PruneIntervalSeconds: 60}, - toStores([]*mockSnapshotStore{a}), - wal, + storeList([]*mockSnapshotStore{a}, wal), ) require.NoError(t, err) require.NotNil(t, sm) diff --git a/sei-db/management/store_types.go b/sei-db/management/store_types.go index d53c157870..1710b8ac8f 100644 --- a/sei-db/management/store_types.go +++ b/sei-db/management/store_types.go @@ -1,34 +1,66 @@ package management -// A store that is persisted via snapshots (e.g. SC, SS). -type SnapshotStore interface { +import "fmt" - // Fetch a list of all block numbers that this store has snapshots for, in ascending sorted order. - GetStoredBlocks() ([]uint64, error) +// StoreType describes how a store retains its data, which determines how the StorageGarbageCollector +// interprets what the store currently holds. +type StoreType int - // Instruct the store that it may drop snapshots for all blocks below a specified number. - // Store may drop data asynchronously. - PruneBelow(blockNumber uint64) error +const ( + // SnapshotStore is a store that is persisted via snapshots (e.g. SC, SS). It retains a discrete set of + // snapshotted heights, reported by GetStoredSnapshots. + SnapshotStore StoreType = iota + 1 - // Return the name of the snapshot store. - Name() string + // StreamStore is a store that is made up of a stream of data (e.g. a WAL). It retains a contiguous range + // of blocks, reported by GetBlockRange. + StreamStore +) + +func (t StoreType) String() string { + switch t { + case SnapshotStore: + return "SnapshotStore" + case StreamStore: + return "StreamStore" + default: + return fmt.Sprintf("UnknownStoreType(%d)", int(t)) + } } -// A store that is made up of a stream of data (e.g. a WAL). -type StreamStore interface { +// PrunableStore is a store whose old data may be dropped by the StorageGarbageCollector. +// +// Both GetStoredSnapshots and GetBlockRange must be implemented, but only the accessor matching the store's +// StoreType is required to carry information: a StreamStore has no discrete snapshots to report, and a +// SnapshotStore reports the range spanned by the snapshots it holds. +type PrunableStore interface { + // Name Return the name of the store. + Name() string + + // GetStoreType returns the type of the store. + GetStoreType() StoreType - // Fetch the range of blocks stored within this stream. - GetStoredBlocks() ( + // PruneBelow tells the store that it may drop snapshots/data for all blocks below a specified number. + // Store can drop data asynchronously in the background. + PruneBelow(blockNumber uint64) error + + // GetStoredSnapshots return a list of all block heights that this store has snapshots for, in ascending + // sorted order. A StreamStore has no snapshots and returns nil. + GetStoredSnapshots() ([]uint64, error) + + // GetBlockRange fetch the range of blocks stored in this store. + GetBlockRange() ( start uint64, // inclusive; meaningful only when hasData is true end uint64, // inclusive; meaningful only when hasData is true - hasData bool, // true if the stream contains at least one block + hasData bool, // true if the store contains at least one block err error, ) - // Instruct the store that it may drop data for all blocks below a specified number. - // Store may drop data asynchronously. - PruneBelow(blockNumber uint64) error - - // Return the name of the stream store. - Name() string + // GetLastCommittedBlock returns the highest block this store has ingested. Meaningful only when the store has + // data. + // + // This is the store's committed height, not the newest data it has durably retained. For a SnapshotStore the + // two differ: snapshots are written periodically while the store keeps ingesting blocks, so the committed + // height normally sits above the newest snapshot. Reporting the newest snapshot height instead is safe but + // makes the garbage collector systematically under-prune. + GetLastCommittedBlock() (uint64, error) } From 01683ce82ba0b09723e891f6ff1ed92eb8f53c1c Mon Sep 17 00:00:00 2001 From: YimingZang Date: Thu, 30 Jul 2026 16:14:52 -0700 Subject: [PATCH 02/15] Further simplification --- sei-db/management/gc/api.go | 29 + .../gc/storage_garbage_collector.go | 191 ++++++ .../gc/storage_garbage_collector_config.go | 67 ++ .../gc/storage_garbage_collector_test.go | 459 +++++++++++++ .../management/storage_garbage_collector.go | 345 ---------- .../storage_garbage_collector_config.go | 45 -- .../storage_garbage_collector_test.go | 617 ------------------ sei-db/management/store_types.go | 66 -- 8 files changed, 746 insertions(+), 1073 deletions(-) create mode 100644 sei-db/management/gc/api.go create mode 100644 sei-db/management/gc/storage_garbage_collector.go create mode 100644 sei-db/management/gc/storage_garbage_collector_config.go create mode 100644 sei-db/management/gc/storage_garbage_collector_test.go delete mode 100644 sei-db/management/storage_garbage_collector.go delete mode 100644 sei-db/management/storage_garbage_collector_config.go delete mode 100644 sei-db/management/storage_garbage_collector_test.go delete mode 100644 sei-db/management/store_types.go diff --git a/sei-db/management/gc/api.go b/sei-db/management/gc/api.go new file mode 100644 index 0000000000..b112da56b0 --- /dev/null +++ b/sei-db/management/gc/api.go @@ -0,0 +1,29 @@ +package gc + +// PrunableStore is a store whose old data may be dropped by the StorageGarbageCollector. +type PrunableStore interface { + // Name Return the name of the store. + Name() string + + // PruneBelow tells the store that it may drop snapshots/data for all blocks below a specified number. + // Store can drop data asynchronously in the background. + PruneBelow(blockNumber uint64) error + + // GetOldestBlockToRetain returns the oldest block this store must keep in order to remain able to serve cutLine, + // or 0 when the store holds nothing. A chain's first block is block 1, so 0 is free to serve as that sentinel: no + // store can legitimately need to retain from block 0. The collector ignores a store that answers 0, since a store + // holding nothing cannot serve a rollback to any height. + // + // A store that retains a contiguous range of blocks (blockDB, receiptDB, the state WAL) can be restored to any + // block it holds, so it needs nothing below the cut line and returns cutLine unchanged. + // A store with checkpoints can only be restored to a height it has a snapshot for, + // so it returns the newest snapshot at or below cutLine, or its oldest snapshot when every snapshot is above the cut line, + // since in that case none of them can be dropped. + // + // The collector prunes every store below the lowest answer it receives, so an answer that is too low only costs temporary + // disk, while one that is too high deletes blocks the system still needs. + GetOldestBlockToRetain(cutLine uint64) uint64 + + // GetLastCommittedBlock returns the highest block this store has ingested. + GetLastCommittedBlock() (uint64, error) +} diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go new file mode 100644 index 0000000000..595bc4a493 --- /dev/null +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -0,0 +1,191 @@ +package gc + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/sei-protocol/seilog" +) + +var logger = seilog.NewLogger("db", "management") + +// StorageGarbageCollector manages deletion of stored data across a set of stores. +// +// Each cycle locates the head of the chain, derives the cut line the system must remain able to serve, asks every store +// how far back it must retain data to serve that cut line, and prunes every store below the lowest answer. Deleting to +// the lowest answer rather than to each store's own is what keeps the stores mutually usable: a snapshot is only +// restorable if the contiguous stores still hold the blocks that follow it. +// +// The StorageGarbageCollector performs state deletion while maintaining the following invariant: +// "If it's possible to roll back at least config.RollbackWindow blocks, then any state deletion operation +// will retain the system's ability to roll back at least config.RollbackWindow blocks." That is to say, +// while the StorageGarbageCollector cannot unilaterally guarantee the ability to roll back config.RollbackWindow +// blocks by itself, it guarantees that the act of deleting old state does not prevent rollback within this +// rollback window. +// +// StorageGarbageCollector is not thread safe. +type StorageGarbageCollector struct { + + // Configuration for this storage garbage collector. + config *StorageGarbageCollectorConfig + + // The stores this collector prunes. + stores []PrunableStore + + // Cancelled to signal the run loop to stop. + ctx context.Context + + // Closed by Close to signal the run loop to stop. + stopCh chan struct{} + + // Tracks the run loop goroutine so Close can wait for it to exit. + wg sync.WaitGroup +} + +func NewStorageGarbageCollector( + ctx context.Context, + config *StorageGarbageCollectorConfig, + stores []PrunableStore, +) (*StorageGarbageCollector, error) { + if err := config.Validate(); err != nil { + return nil, fmt.Errorf("invalid storage garbage collector config: %w", err) + } + + s := &StorageGarbageCollector{ + config: config, + stores: stores, + ctx: ctx, + stopCh: make(chan struct{}), + } + + s.wg.Add(1) + go s.run() + + return s, nil +} + +// Close stops the storage garbage collector and waits for its run loop to exit. It must be called exactly once. +func (s *StorageGarbageCollector) Close() error { + close(s.stopCh) + s.wg.Wait() + return nil +} + +// run periodically drives prune cycles until the manager is stopped. All decision logic lives in prune so it can +// be unit tested without threading. +func (s *StorageGarbageCollector) run() { + defer s.wg.Done() + + //nolint:gosec // G115 - Config.Validate rejects PruneIntervalSeconds large enough to overflow this conversion. + ticker := time.NewTicker(time.Duration(s.config.PruneIntervalSeconds) * time.Second) + defer ticker.Stop() + + for { + select { + case <-s.stopCh: + return + case <-s.ctx.Done(): + return + case <-ticker.C: + if err := s.prune(); err != nil { + logger.Error("prune cycle failed", "err", err) + } + } + } +} + +// prune performs a single prune cycle: it locates the head of the chain, derives the cut line from it, asks every store +// how far back it must retain data to serve that cut line, and prunes every store below the lowest answer. +func (s *StorageGarbageCollector) prune() error { + if len(s.stores) == 0 { + return nil + } + globalLatestBlock, err := getGlobalLastCommittedBlock(s.stores) + if err != nil { + return err + } + if globalLatestBlock == 0 { + logger.Debug("skipping pruning, no store has committed a block") + return nil + } + + cutLine := getCutLine(globalLatestBlock, s.config.RollbackWindow, s.config.StoreRetention) + if cutLine == 0 { + logger.Debug("skipping pruning, the chain is younger than the retain window", + "globalLatestBlock", globalLatestBlock) + return nil + } + + // Every store must keep back to its own answer, so the system may only prune below the lowest of them. Pruning + // below a higher answer would delete blocks that a lower-answering store just reported it still needs, which is + // exactly how deletion would break the rollback invariant. + var pruneHeight uint64 + for _, store := range s.stores { + oldestBlockToRetain := store.GetOldestBlockToRetain(cutLine) + if oldestBlockToRetain == 0 { + // The store holds nothing, so it cannot serve a rollback to any height and has no stake in how far the + // other stores prune. + continue + } + // A pruneHeight of 0 means no store has answered yet, since the only zero answer is skipped above. + if pruneHeight == 0 || oldestBlockToRetain < pruneHeight { + pruneHeight = oldestBlockToRetain + } + } + if pruneHeight == 0 { + logger.Debug("skipping pruning, no store has data to retain", "cutLine", cutLine) + return nil + } + + logger.Info("pruning storage", "pruneHeight", pruneHeight) + for _, store := range s.stores { + if err := store.PruneBelow(pruneHeight); err != nil { + return fmt.Errorf("failed to prune %s below %d: %w", store.Name(), pruneHeight, err) + } + } + return nil +} + +// getCutLine returns the oldest block the system must remain able to serve, which is the head of the chain less the +// rollback window and the extra retention. +// +// Returns 0 to mean "prune nothing", which is the case for a chain younger than that combined window. The comparison has +// to happen before the subtraction: these are unsigned, so subtracting past zero wraps to a cut line far above the head +// of the chain, and pruning to it would delete everything. +// +// Config.Validate rejects a rollback window and retention that overflow when summed, so retainWindow below is exact. +func getCutLine(globalLatestBlock uint64, rollbackWindow uint64, retention uint64) uint64 { + retainWindow := rollbackWindow + retention + if globalLatestBlock <= retainWindow { + return 0 + } + return globalLatestBlock - retainWindow +} + +// getGlobalLastCommittedBlock reads the head height of every store and returns the smallest. +// +// The smallest head is the head of the chain as far as the collector can tell, because it bounds what the system can +// actually serve. Measuring the cut line from a store that has run ahead of the others would let the collector prune a +// lagging store past blocks that store still needs in order to serve a rollback. +// +// Heads of 0 are skipped: a store that has ingested nothing would otherwise hold the head of the whole system at 0 and +// stall pruning everywhere. Skipping it does not put its data at risk, because whatever it holds, its +// GetOldestBlockToRetain answer still bounds the prune height. Returns 0 when no store has committed a block. +func getGlobalLastCommittedBlock(stores []PrunableStore) (uint64, error) { + var blockNum uint64 + for _, store := range stores { + storeHeight, err := store.GetLastCommittedBlock() + if err != nil { + return 0, fmt.Errorf("failed to read last committed block from %s: %w", store.Name(), err) + } + if storeHeight == 0 { + continue + } + if blockNum == 0 || storeHeight < blockNum { + blockNum = storeHeight + } + } + return blockNum, nil +} diff --git a/sei-db/management/gc/storage_garbage_collector_config.go b/sei-db/management/gc/storage_garbage_collector_config.go new file mode 100644 index 0000000000..802155db98 --- /dev/null +++ b/sei-db/management/gc/storage_garbage_collector_config.go @@ -0,0 +1,67 @@ +package gc + +import ( + "fmt" + "math" + "time" +) + +// Configures a StorageGarbageCollector. +type StorageGarbageCollectorConfig struct { + + // The maximum number of blocks the system should be able to roll back at any point in time. + // Storage garbage collector ensures that enough data is kept on disk such that the system can always + // roll back this many blocks. + // + // Note that the "always able to rollback" invariant may be broken after a rollback. For example, if we normally + // require a rollback window of 10,000 blocks and then we rollback 5,000 blocks, we can then only rollback an + // additional 5,000 blocks after that first rollback. + RollbackWindow uint64 + + // Additional blocks to retain beyond the rollback window, applied to every store. + // + // This is retention the operator wants rather than retention correctness requires: RollbackWindow is what the + // rollback invariant depends on, while StoreRetention buys history that is served long after it is needed for + // rollback, such as queries against old blocks and receipts. The two are added together to form the cut line. + // + // TODO: make this per-store if the shared value proves too blunt. Because every store is pruned to the lowest + // block any single store still needs, retention bought for one store is paid for by all of them: raising this to + // keep more block and receipt history also holds that much extra state in the snapshotted stores, which is far + // more expensive per block. Separating them means giving the collector a retention value alongside each store + // (e.g. a []ManagedStore{Store, RetentionBlocks} in place of []PrunableStore) and computing one cut line per + // store. Note that this only reduces retention where a store's own answer is what bounds the minimum; the + // contiguous stores must still cover the oldest snapshot anyone kept, or the snapshot becomes unreplayable. + StoreRetention uint64 + + // The frequency of prune operations, in seconds. + PruneIntervalSeconds uint64 +} + +// Construct a default storage garbage collector config. +func DefaultStorageGarbageCollectorConfig() *StorageGarbageCollectorConfig { + return &StorageGarbageCollectorConfig{ + RollbackWindow: 1000, + StoreRetention: 100_000, + PruneIntervalSeconds: 600, + } +} + +// Validate the storage garbage collector's config. +func (c *StorageGarbageCollectorConfig) Validate() error { + // A zero rollback window is legal: it means the system prunes as aggressively as possible. + if c.PruneIntervalSeconds == 0 { + return fmt.Errorf("prune interval must be greater than 0 seconds") + } + // The prune interval is converted to a time.Duration (int64 nanoseconds) when the ticker is created. Reject values + // that would overflow that conversion so the ticker can never be handed a zero/negative duration. + if c.PruneIntervalSeconds > math.MaxInt64/uint64(time.Second) { + return fmt.Errorf("prune interval must be at most %d seconds", math.MaxInt64/uint64(time.Second)) + } + // The cut line is derived by subtracting RollbackWindow + StoreRetention from the head of the chain. Reject a pair + // that overflows when summed, so that subtraction cannot wrap around to a cut line above the head. + if c.RollbackWindow+c.StoreRetention < c.RollbackWindow { + return fmt.Errorf("rollback window (%d) plus store retention (%d) must be at most %d", + c.RollbackWindow, c.StoreRetention, uint64(math.MaxUint64)) + } + return nil +} diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go new file mode 100644 index 0000000000..36bf48c4d3 --- /dev/null +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -0,0 +1,459 @@ +package gc + +import ( + "context" + "errors" + "math" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// mockStore is a PrunableStore with canned answers. Build one with snapshotStore or contiguousStore. +type mockStore struct { + name string + latestHeight uint64 + // oldestToRetain answers GetOldestBlockToRetain for a given cut line. + oldestToRetain func(cutLine uint64) uint64 + getErr error + pruneErr error + + pruneBelowCalled bool + prunedBelow uint64 +} + +// snapshotStore models a store that can only be restored to a height it holds a snapshot for, such as SC or SS: it +// answers with the newest snapshot at or below the cut line, or with its oldest snapshot when every snapshot is above the +// cut line. Snapshots must be in ascending order; a store given none holds no data. +func snapshotStore(name string, latestHeight uint64, snapshots ...uint64) *mockStore { + return &mockStore{ + name: name, + latestHeight: latestHeight, + oldestToRetain: func(cutLine uint64) uint64 { + if len(snapshots) == 0 { + return 0 + } + oldest := snapshots[0] + for _, snapshot := range snapshots { + if snapshot > cutLine { + break + } + oldest = snapshot + } + return oldest + }, + } +} + +// contiguousStore models a store that retains every block in a range, such as blockDB, receiptDB or the state WAL. It can +// be restored to any block it holds, so it needs nothing below the cut line. +func contiguousStore(name string, latestHeight uint64, hasData bool) *mockStore { + return &mockStore{ + name: name, + latestHeight: latestHeight, + oldestToRetain: func(cutLine uint64) uint64 { + if !hasData { + return 0 + } + return cutLine + }, + } +} + +func (m *mockStore) Name() string { + return m.name +} + +func (m *mockStore) GetLastCommittedBlock() (uint64, error) { + return m.latestHeight, m.getErr +} + +func (m *mockStore) GetOldestBlockToRetain(cutLine uint64) uint64 { + return m.oldestToRetain(cutLine) +} + +func (m *mockStore) PruneBelow(blockNumber uint64) error { + m.pruneBelowCalled = true + m.prunedBelow = blockNumber + return m.pruneErr +} + +// prunableStores adapts the mocks to the interface slice the collector holds. +func prunableStores(list ...*mockStore) []PrunableStore { + result := make([]PrunableStore, len(list)) + for i, store := range list { + result[i] = store + } + return result +} + +// newTestCollector builds a collector without starting its run loop, so prune can be driven one cycle at a time. +func newTestCollector( + t *testing.T, + rollbackWindow uint64, + storeRetention uint64, + stores ...*mockStore, +) *StorageGarbageCollector { + t.Helper() + + config := &StorageGarbageCollectorConfig{ + RollbackWindow: rollbackWindow, + StoreRetention: storeRetention, + PruneIntervalSeconds: 60, + } + // getCutLine subtracts without an overflow guard because Validate rejects a retain window that would overflow, so + // test configs have to clear the same bar as real ones. + require.NoError(t, config.Validate()) + + return &StorageGarbageCollector{config: config, stores: prunableStores(stores...)} +} + +// TestPruneDecisions is the decision matrix for a single prune cycle. wantPruneBelow == nil means no store may be pruned. +func TestPruneDecisions(t *testing.T) { + cases := []struct { + name string + rollbackWindow uint64 + storeRetention uint64 + stores []*mockStore + wantPruneBelow *uint64 + }{ + { + name: "one snapshotted store and a WAL", + rollbackWindow: 10_000, + stores: []*mockStore{ + snapshotStore("sc", 100_000, 80_000, 85_000, 92_000), + contiguousStore("stateWAL", 100_000, true), + }, + // Cut line 90,000. The WAL needs only 90,000, but sc must keep the 85,000 snapshot a rollback to the cut + // line would start from, so everything is pruned to 85,000. Pruning to the higher answer would delete + // that snapshot. + wantPruneBelow: ptr(85_000), + }, + { + name: "the lowest need across many stores wins", + rollbackWindow: 10_000, + stores: []*mockStore{ + snapshotStore("sc", 100_000, 80_000, 85_000, 92_000), + snapshotStore("ss", 100_000, 70_000, 88_000, 95_000), + snapshotStore("flatKV", 100_000, 50_000, 90_000, 99_000), + contiguousStore("stateWAL", 100_000, true), + }, + // Needs are 85,000, 88,000, 90,000 and 90,000. + wantPruneBelow: ptr(85_000), + }, + { + name: "zero rollback window prunes up to the head", + rollbackWindow: 0, + stores: []*mockStore{ + snapshotStore("sc", 100_000, 80_000, 100_000), + contiguousStore("stateWAL", 100_000, true), + }, + wantPruneBelow: ptr(100_000), + }, + { + name: "store retention deepens the cut line", + rollbackWindow: 10_000, + storeRetention: 5_000, + stores: []*mockStore{ + snapshotStore("sc", 100_000, 80_000, 90_000), + contiguousStore("stateWAL", 100_000, true), + }, + // Cut line 85,000 rather than 90,000, so the 80,000 snapshot is now the newest at or below it. Without the + // extra retention sc would have needed only 90,000. + wantPruneBelow: ptr(80_000), + }, + { + name: "a snapshot exactly at the cut line is kept", + rollbackWindow: 10_000, + stores: []*mockStore{ + snapshotStore("sc", 100_000, 50_000, 90_000), + contiguousStore("stateWAL", 100_000, true), + }, + wantPruneBelow: ptr(90_000), + }, + { + name: "every snapshot above the cut line pins the store to its oldest", + rollbackWindow: 10_000, + stores: []*mockStore{ + snapshotStore("sc", 100_000, 95_000, 97_000), + contiguousStore("stateWAL", 100_000, true), + }, + // sc cannot drop either snapshot and answers 95,000, so the WAL's 90,000 is the lowest. + wantPruneBelow: ptr(90_000), + }, + { + name: "a lagging store pulls the cut line down with it", + rollbackWindow: 10_000, + stores: []*mockStore{ + snapshotStore("lagging", 50_000, 30_000, 50_000), + contiguousStore("stateWAL", 100_000, true), + }, + // The head is the lowest of the two, 50,000, putting the cut line at 40,000 and making 30,000 the newest + // snapshot at or below it. Measuring from the WAL's head instead would put the cut line at 90,000, and the + // 30,000 snapshot needed to roll back to 40,000 would be deleted. + wantPruneBelow: ptr(30_000), + }, + { + name: "a store holding nothing is ignored", + rollbackWindow: 10_000, + stores: []*mockStore{ + snapshotStore("empty", 100_000), + snapshotStore("sc", 100_000, 80_000), + contiguousStore("stateWAL", 100_000, true), + }, + wantPruneBelow: ptr(80_000), + }, + { + name: "a head of zero is ignored, the data behind it is not", + rollbackWindow: 10_000, + stores: []*mockStore{ + snapshotStore("stalled", 0, 50_000), + contiguousStore("stateWAL", 100_000, true), + }, + // The stalled store is left out of the head, so the cut line comes from the WAL at 90,000. It still + // reports that it needs its 50,000 snapshot, and that is the lowest answer. + wantPruneBelow: ptr(50_000), + }, + { + name: "chain younger than the retain window", + rollbackWindow: 10_000, + stores: []*mockStore{ + snapshotStore("sc", 5_000, 1_000, 2_000), + contiguousStore("stateWAL", 5_000, true), + }, + wantPruneBelow: nil, + }, + { + name: "head exactly at the retain window", + rollbackWindow: 60_000, + storeRetention: 40_000, + stores: []*mockStore{ + snapshotStore("sc", 100_000, 500, 1_000), + contiguousStore("stateWAL", 100_000, true), + }, + wantPruneBelow: nil, + }, + { + name: "no store has committed a block", + rollbackWindow: 10_000, + stores: []*mockStore{ + snapshotStore("sc", 0), + contiguousStore("stateWAL", 0, false), + }, + wantPruneBelow: nil, + }, + { + name: "stores are committing blocks but none retains any", + rollbackWindow: 10_000, + stores: []*mockStore{ + snapshotStore("sc", 100_000), + contiguousStore("stateWAL", 100_000, false), + }, + // A fresh node whose stores have ingested blocks but have not yet written a first snapshot. + wantPruneBelow: nil, + }, + { + name: "no stores at all", + rollbackWindow: 10_000, + stores: nil, + wantPruneBelow: nil, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + collector := newTestCollector(t, tc.rollbackWindow, tc.storeRetention, tc.stores...) + require.NoError(t, collector.prune()) + + for _, store := range tc.stores { + if tc.wantPruneBelow == nil { + require.Falsef(t, store.pruneBelowCalled, "%s should not be pruned", store.name) + continue + } + require.Truef(t, store.pruneBelowCalled, "%s should be pruned", store.name) + require.Equalf(t, *tc.wantPruneBelow, store.prunedBelow, "%s prune height", store.name) + } + }) + } +} + +// TestPruneOnYoungChainWithDefaultConfig guards the unsigned subtraction in getCutLine using the real defaults, where the +// retain window is 101,000 blocks. Every node spends its early life here, and a wrapped cut line would land far above the +// head of the chain and delete everything. +func TestPruneOnYoungChainWithDefaultConfig(t *testing.T) { + sc := snapshotStore("sc", 100, 50, 100) + wal := contiguousStore("stateWAL", 100, true) + + collector := &StorageGarbageCollector{ + config: DefaultStorageGarbageCollectorConfig(), + stores: prunableStores(sc, wal), + } + require.NoError(t, collector.prune()) + + require.False(t, sc.pruneBelowCalled) + require.False(t, wal.pruneBelowCalled) +} + +// TestPruneGetLastCommittedBlockError checks that a failure to read a head aborts the cycle before anything is deleted. +func TestPruneGetLastCommittedBlockError(t *testing.T) { + sentinel := errors.New("boom") + sc := snapshotStore("sc", 100_000, 80_000) + broken := contiguousStore("brokenStore", 100_000, true) + broken.getErr = sentinel + + err := newTestCollector(t, 10_000, 0, sc, broken).prune() + require.ErrorIs(t, err, sentinel) + require.ErrorContains(t, err, "brokenStore") + require.False(t, sc.pruneBelowCalled) + require.False(t, broken.pruneBelowCalled) +} + +// TestPruneBelowErrorStopsRemainingStores checks that the first failed deletion aborts the cycle, leaving later stores +// untouched. +func TestPruneBelowErrorStopsRemainingStores(t *testing.T) { + sentinel := errors.New("boom") + first := snapshotStore("first", 100_000, 80_000) + first.pruneErr = sentinel + second := snapshotStore("second", 100_000, 80_000) + wal := contiguousStore("stateWAL", 100_000, true) + + err := newTestCollector(t, 10_000, 0, first, second, wal).prune() + require.ErrorIs(t, err, sentinel) + require.ErrorContains(t, err, "first") + require.ErrorContains(t, err, "80000") // the prune height, i.e. the lowest need across the stores + require.True(t, first.pruneBelowCalled) + require.False(t, second.pruneBelowCalled) + require.False(t, wal.pruneBelowCalled) +} + +func TestGetCutLine(t *testing.T) { + cases := []struct { + name string + head uint64 + rollbackWindow uint64 + retention uint64 + want uint64 + }{ + {name: "rollback window only", head: 100_000, rollbackWindow: 10_000, want: 90_000}, + {name: "retention only", head: 100_000, retention: 10_000, want: 90_000}, + {name: "the two windows add", head: 100_000, rollbackWindow: 10_000, retention: 5_000, want: 85_000}, + {name: "no retain window at all", head: 100_000, want: 100_000}, + {name: "head one above the window", head: 10_001, rollbackWindow: 10_000, want: 1}, + {name: "head exactly at the window", head: 10_000, rollbackWindow: 10_000, want: 0}, + {name: "head one below the window", head: 9_999, rollbackWindow: 10_000, want: 0}, + // The default retain window against a young chain: the subtraction must not wrap. + {name: "head far below the window", head: 100, rollbackWindow: 1_000, retention: 100_000, want: 0}, + {name: "head at genesis", head: 0, rollbackWindow: 10_000, want: 0}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, getCutLine(tc.head, tc.rollbackWindow, tc.retention)) + }) + } +} + +func TestGetGlobalLastCommittedBlock(t *testing.T) { + storesWithHeads := func(heads ...uint64) []PrunableStore { + list := make([]*mockStore, len(heads)) + for i, head := range heads { + list[i] = contiguousStore("store", head, true) + } + return prunableStores(list...) + } + + cases := []struct { + name string + heads []uint64 + want uint64 + }{ + {name: "single store", heads: []uint64{100}, want: 100}, + {name: "all agree", heads: []uint64{100, 100, 100}, want: 100}, + {name: "smallest first", heads: []uint64{80, 100}, want: 80}, + {name: "smallest last", heads: []uint64{100, 80}, want: 80}, + {name: "smallest in the middle", heads: []uint64{100, 50, 90}, want: 50}, + // An uninitialized store must not drag the head of the chain down with it. + {name: "leading zero ignored", heads: []uint64{0, 100}, want: 100}, + {name: "trailing zero ignored", heads: []uint64{100, 0}, want: 100}, + {name: "zero among many ignored", heads: []uint64{100, 0, 80}, want: 80}, + {name: "every store reports zero", heads: []uint64{0, 0}, want: 0}, + {name: "no stores", heads: nil, want: 0}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + head, err := getGlobalLastCommittedBlock(storesWithHeads(tc.heads...)) + require.NoError(t, err) + require.Equal(t, tc.want, head) + }) + } +} + +func TestDefaultStorageGarbageCollectorConfig(t *testing.T) { + cfg := DefaultStorageGarbageCollectorConfig() + require.Equal(t, uint64(1_000), cfg.RollbackWindow) + require.Equal(t, uint64(100_000), cfg.StoreRetention) + require.Equal(t, uint64(600), cfg.PruneIntervalSeconds) + require.NoError(t, cfg.Validate()) +} + +func TestValidate(t *testing.T) { + // A zero rollback window is legal, as is a zero store retention. + require.NoError(t, (&StorageGarbageCollectorConfig{PruneIntervalSeconds: 60}).Validate()) + + require.ErrorContains(t, (&StorageGarbageCollectorConfig{PruneIntervalSeconds: 0}).Validate(), "prune interval") + + // The largest interval that does not overflow time.Duration is accepted; one larger is rejected. + maxInterval := uint64(math.MaxInt64) / uint64(time.Second) + require.NoError(t, (&StorageGarbageCollectorConfig{PruneIntervalSeconds: maxInterval}).Validate()) + require.ErrorContains(t, (&StorageGarbageCollectorConfig{PruneIntervalSeconds: maxInterval + 1}).Validate(), "at most") + + // The retain window is the sum of the two, so the pair must not overflow. getCutLine subtracts it without a guard. + require.NoError(t, (&StorageGarbageCollectorConfig{ + RollbackWindow: math.MaxUint64, + PruneIntervalSeconds: 60, + }).Validate()) + require.ErrorContains(t, (&StorageGarbageCollectorConfig{ + RollbackWindow: math.MaxUint64, + StoreRetention: 1, + PruneIntervalSeconds: 60, + }).Validate(), "store retention") +} + +func TestNewStorageGarbageCollectorInvalidConfig(t *testing.T) { + sm, err := NewStorageGarbageCollector( + context.Background(), + &StorageGarbageCollectorConfig{PruneIntervalSeconds: 0}, + prunableStores(snapshotStore("sc", 100)), + ) + require.Error(t, err) + require.Nil(t, sm) +} + +func TestNewStorageGarbageCollectorConstructAndClose(t *testing.T) { + sm, err := NewStorageGarbageCollector( + context.Background(), + DefaultStorageGarbageCollectorConfig(), + prunableStores(snapshotStore("sc", 100, 100), contiguousStore("stateWAL", 100, true)), + ) + require.NoError(t, err) + require.NotNil(t, sm) + + // Close must return without hanging. + require.NoError(t, sm.Close()) +} + +func TestCloseAfterContextCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + sm, err := NewStorageGarbageCollector(ctx, DefaultStorageGarbageCollectorConfig(), nil) + require.NoError(t, err) + + // Cancelling the context lets the run loop exit on its own, so Close must still return. + cancel() + require.NoError(t, sm.Close()) +} + +func ptr(v uint64) *uint64 { + return &v +} diff --git a/sei-db/management/storage_garbage_collector.go b/sei-db/management/storage_garbage_collector.go deleted file mode 100644 index 040d629d06..0000000000 --- a/sei-db/management/storage_garbage_collector.go +++ /dev/null @@ -1,345 +0,0 @@ -package management - -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/sei-protocol/seilog" -) - -var logger = seilog.NewLogger("db", "management") - -// StorageGarbageCollector manages deletion of stored data across a set of stores. Each store declares its -// StoreType, which determines both how the collector observes what the store retains and how far the store is -// allowed to prune: a SnapshotStore retains whole snapshots, while a StreamStore (e.g. the state WAL) retains a -// contiguous range of blocks that the snapshot stores are replayed from. -// -// The StorageGarbageCollector performs state deletion while maintaining the following invariant: -// "If it's possible to roll back at least config.RollbackWindow blocks, then any state deletion operation -// will retain the system's ability to roll back at least config.RollbackWindow blocks." That is to say, -// while the StorageGarbageCollector cannot unilaterally guarantee the ability to roll back config.RollbackWindow -// blocks by itself, it guarantees that the act of deleting old state does not prevent rollback within this -// rollback window. -// -// StorageGarbageCollector is not thread safe. -type StorageGarbageCollector struct { - - // Configuration for this storage garbage collector. - config *StorageGarbageCollectorConfig - - // The stores this collector prunes. - stores []PrunableStore - - // Cancelled to signal the run loop to stop. - ctx context.Context - - // Closed by Close to signal the run loop to stop. - stopCh chan struct{} - - // Tracks the run loop goroutine so Close can wait for it to exit. - wg sync.WaitGroup -} - -func NewStorageGarbageCollector( - ctx context.Context, - config *StorageGarbageCollectorConfig, - stores []PrunableStore, -) (*StorageGarbageCollector, error) { - - if err := config.Validate(); err != nil { - return nil, fmt.Errorf("invalid storage garbage collector config: %w", err) - } - - for _, store := range stores { - switch storeType := store.GetStoreType(); storeType { - case SnapshotStore, StreamStore: - default: - return nil, fmt.Errorf("store %s has unsupported store type %s", store.Name(), storeType) - } - } - - s := &StorageGarbageCollector{ - config: config, - stores: stores, - ctx: ctx, - stopCh: make(chan struct{}), - } - - s.wg.Add(1) - go s.run() - - return s, nil -} - -// Close stops the storage garbage collector and waits for its run loop to exit. It must be called exactly once. -func (s *StorageGarbageCollector) Close() error { - close(s.stopCh) - s.wg.Wait() - return nil -} - -// run periodically drives prune cycles until the manager is stopped. All decision logic lives in prune so it can -// be unit tested without threading. -func (s *StorageGarbageCollector) run() { - defer s.wg.Done() - - //nolint:gosec // G115 - Config.Validate rejects PruneIntervalSeconds large enough to overflow this conversion. - ticker := time.NewTicker(time.Duration(s.config.PruneIntervalSeconds) * time.Second) - defer ticker.Stop() - - for { - select { - case <-s.stopCh: - return - case <-s.ctx.Done(): - return - case <-ticker.C: - if err := prune(s.config.RollbackWindow, s.stores); err != nil { - logger.Error("prune cycle failed", "err", err) - } - } - } -} - -// prune performs a single prune cycle: it observes the blocks retained by each managed store, computes how far each -// store may prune while preserving the rollback window, and issues the prune commands. -func prune(rollbackWindow uint64, stores []PrunableStore) error { - observations, err := observeStores(stores) - if err != nil { - return err - } - - if len(observations) == 0 { - return nil - } - - allStoresHaveData := true - for _, obs := range observations { - if !obs.hasData { - allStoresHaveData = false - } - } - - if !allStoresHaveData { - // We only prune if every provided store has at least some data. An empty store is treated as unknown rather - // than empty (a snapshot may be mid-write and take hours), and pruning against it would risk breaking the - // primary invariant (i.e. breaking the ability to roll back by the act of deletion). - logArgs := make([]any, 0, 2*len(observations)) - for _, obs := range observations { - logArgs = append(logArgs, obs.store.Name(), obs.describeRetained()) - } - logger.Info("skipping pruning, not all stores have data", logArgs...) - return nil - } - - latestBlock, highestBlock := committedBlockRange(observations) - - // The lowest head sets the rollback target for every store, so a store trailing the rest by more than the whole - // rollback window is throttling how much anything else can prune. That errs toward retention and is safe, but it - // usually means the store is stalled or is reporting the wrong height, and it is otherwise invisible: pruning - // still looks like it is working, it just stops reclaiming much. - if highestBlock-latestBlock > rollbackWindow { - logArgs := make([]any, 0, 2+2*len(observations)) - logArgs = append(logArgs, "rollbackWindow", rollbackWindow) - for _, obs := range observations { - logArgs = append(logArgs, obs.store.Name()+"Head", obs.latestBlock) - } - logger.Warn("store heads disagree by more than the rollback window, pruning is limited by the "+ - "furthest behind store", logArgs...) - } - - // The oldest block we must remain able to roll back to. - var oldestBlockNeeded uint64 - if latestBlock > rollbackWindow { - oldestBlockNeeded = latestBlock - rollbackWindow - } - - floors := pruningFloors(observations, oldestBlockNeeded) - - logArgs := []any{"latestBlock", latestBlock, "oldestBlockNeeded", oldestBlockNeeded} - for i, obs := range observations { - logArgs = append(logArgs, - obs.store.Name()+"Initial", obs.describeRetained(), - obs.store.Name()+"Final", obs.describeRetainedAfterPruning(floors[i]), - ) - } - logger.Info("pruning storage", logArgs...) - - for i, obs := range observations { - if err := obs.store.PruneBelow(floors[i]); err != nil { - return fmt.Errorf("failed to prune %s below %d: %w", obs.store.Name(), floors[i], err) - } - } - - return nil -} - -// observation is what a single prune cycle read back from one store. -type observation struct { - store PrunableStore - storeType StoreType - - // The snapshots held by a SnapshotStore, ascending. Always empty for a StreamStore. - snapshots []uint64 - - // The inclusive range of blocks held by a StreamStore. Meaningful only when hasData is true. - start uint64 - end uint64 - - // The highest block the store has ingested. Meaningful only when hasData is true. - latestBlock uint64 - - // Whether the store holds any blocks at all. - hasData bool -} - -// observeStores reads what every store currently retains, using the accessor that matches each store's type. -// Observation happens up front so that a read failure aborts the cycle before anything has been deleted. -func observeStores(stores []PrunableStore) ([]observation, error) { - observations := make([]observation, len(stores)) - for i, store := range stores { - obs := observation{store: store, storeType: store.GetStoreType()} - - switch obs.storeType { - case SnapshotStore: - snapshots, err := store.GetStoredSnapshots() - if err != nil { - return nil, fmt.Errorf("failed to read stored snapshots from %s: %w", store.Name(), err) - } - obs.snapshots = snapshots - obs.hasData = len(snapshots) > 0 - case StreamStore: - start, end, hasData, err := store.GetBlockRange() - if err != nil { - return nil, fmt.Errorf("failed to read block range from %s: %w", store.Name(), err) - } - obs.start, obs.end, obs.hasData = start, end, hasData - default: - return nil, fmt.Errorf("store %s has unsupported store type %s", store.Name(), obs.storeType) - } - - latestBlock, err := store.GetLastCommittedBlock() - if err != nil { - return nil, fmt.Errorf("failed to read last committed block from %s: %w", store.Name(), err) - } - obs.latestBlock = latestBlock - - observations[i] = obs - } - return observations, nil -} - -// committedBlockRange returns the lowest and highest heads reported across the stores. -// -// The lowest head is the head of the chain as the collector understands it, because it bounds what the system can -// actually serve; treating a lagging store as caught up would place the rollback target above the blocks that store -// still holds. The highest is reported alongside it so callers can see how far the stores disagree. Callers must have -// verified that there is at least one store and that every store has data. -func committedBlockRange(observations []observation) (lowest uint64, highest uint64) { - for i, obs := range observations { - if i == 0 || obs.latestBlock < lowest { - lowest = obs.latestBlock - } - if i == 0 || obs.latestBlock > highest { - highest = obs.latestBlock - } - } - return lowest, highest -} - -// pruningFloors computes the block each store may prune below, indexed in parallel with observations. -// -// A snapshot store retains the newest snapshot at or below oldestBlockNeeded, since that is the snapshot a rollback to -// oldestBlockNeeded would start from. Every stream store retains from the oldest block any snapshot store still needs, -// because a rollback replays the stream forward from that snapshot. With no snapshot stores nothing depends on the -// streams for rollback, so they retain only the rollback window. -// -// A stream store must not simply retain from oldestBlockNeeded: snapshots land at arbitrary heights, so the newest -// snapshot at or below oldestBlockNeeded can sit far below it. With snapshots at 80,000 and 92,000 and -// oldestBlockNeeded of 90,000, the 80,000 snapshot is the only one a rollback to 90,000 can start from, and replaying -// forward to 90,000 needs stream entries 80,001 onward. Pruning the stream to 90,000 would strand that snapshot. -func pruningFloors(observations []observation, oldestBlockNeeded uint64) []uint64 { - floors := make([]uint64, len(observations)) - - streamFloor := oldestBlockNeeded - seenSnapshotStore := false - for i, obs := range observations { - if obs.storeType != SnapshotStore { - continue - } - floors[i] = snapshotPruningFloor(obs.snapshots, oldestBlockNeeded) - if !seenSnapshotStore || floors[i] < streamFloor { - streamFloor = floors[i] - } - seenSnapshotStore = true - } - - for i, obs := range observations { - if obs.storeType == StreamStore { - floors[i] = streamFloor - } - } - - return floors -} - -// describeRetained renders what the store currently holds, for logging. -func (o observation) describeRetained() string { - if !o.hasData { - return "empty" - } - if o.storeType == SnapshotStore { - return fmt.Sprintf("%v", o.snapshots) - } - return blockRange(o.start, o.end) -} - -// describeRetainedAfterPruning renders what the store is expected to hold once PruneBelow(floor) completes. -func (o observation) describeRetainedAfterPruning(floor uint64) string { - if o.storeType == SnapshotStore { - return fmt.Sprintf("%v", blocksAtOrAbove(o.snapshots, floor)) - } - return blockRange(floor, o.end) -} - -// Given a list of snapshot block numbers, determine the lowest snapshot we need to keep in order to be able -// to roll back to the target block number. -// -// Returns the highest numbered block from snapshotBlocks that is less than or equal to the rollbackTarget. If every -// snapshot is greater than rollbackTarget, the lowest snapshot is returned, since none can be safely pruned. -// snapshotBlocks must be non-empty and sorted in ascending order, per the PrunableStore contract. -func snapshotPruningFloor( - // Blocks we have snapshots for, in ascending order. Must be non-empty. - snapshotBlocks []uint64, - // The target block that the system needs to be able to roll back to. - rollbackTarget uint64, -) (floor uint64) { - - floor = snapshotBlocks[0] // guaranteed non-empty - for _, b := range snapshotBlocks { - if b > rollbackTarget { - break - } - floor = b - } - return floor -} - -// blocksAtOrAbove returns the subset of blocks that are >= floor, preserving order. This is the set a snapshot store is -// expected to hold after PruneBelow(floor) completes. -func blocksAtOrAbove(blocks []uint64, floor uint64) []uint64 { - result := make([]uint64, 0, len(blocks)) - for _, b := range blocks { - if b >= floor { - result = append(result, b) - } - } - return result -} - -// blockRange renders an inclusive block range for logging. -func blockRange(start uint64, end uint64) string { - return fmt.Sprintf("[%d, %d]", start, end) -} diff --git a/sei-db/management/storage_garbage_collector_config.go b/sei-db/management/storage_garbage_collector_config.go deleted file mode 100644 index 41ba06af13..0000000000 --- a/sei-db/management/storage_garbage_collector_config.go +++ /dev/null @@ -1,45 +0,0 @@ -package management - -import ( - "fmt" - "math" - "time" -) - -// Configures a StorageGarbageCollector. -type StorageGarbageCollectorConfig struct { - - // The maximum number of blocks the system should be able to roll back at any point in time. - // Storage garbage collector ensures that enough data is kept on disk such that the system can always - // roll back this many blocks. - // - // Note that the "always able to rollback" invariant may be broken after a rollback. For example, if we normally - // require a rollback window of 10,000 blocks and then we rollback 5,000 blocks, we can then only rollback an - // additional 5,000 blocks after that first rollback. - RollbackWindow uint64 - - // The frequency of prune operations, in seconds. - PruneIntervalSeconds uint64 -} - -// Construct a default storage garbage collector config. -func DefaultStorageGarbageCollectorConfig() *StorageGarbageCollectorConfig { - return &StorageGarbageCollectorConfig{ - RollbackWindow: 10_000, - PruneIntervalSeconds: 60, - } -} - -// Validate the storage garbage collector's config. -func (c *StorageGarbageCollectorConfig) Validate() error { - // A zero rollback window is legal: it means the system prunes as aggressively as possible. - if c.PruneIntervalSeconds == 0 { - return fmt.Errorf("prune interval must be greater than 0 seconds") - } - // The prune interval is converted to a time.Duration (int64 nanoseconds) when the ticker is created. Reject values - // that would overflow that conversion so the ticker can never be handed a zero/negative duration. - if c.PruneIntervalSeconds > math.MaxInt64/uint64(time.Second) { - return fmt.Errorf("prune interval must be at most %d seconds", math.MaxInt64/uint64(time.Second)) - } - return nil -} diff --git a/sei-db/management/storage_garbage_collector_test.go b/sei-db/management/storage_garbage_collector_test.go deleted file mode 100644 index 65a7fb5567..0000000000 --- a/sei-db/management/storage_garbage_collector_test.go +++ /dev/null @@ -1,617 +0,0 @@ -package management - -import ( - "context" - "errors" - "fmt" - "math" - "math/rand" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -// mockSnapshotStore is a hand-written PrunableStore of type SnapshotStore for tests. It records the last PruneBelow -// argument and returns canned GetStoredSnapshots data or errors. -type mockSnapshotStore struct { - name string - blocks []uint64 - // The committed height, which for a snapshot store normally sits above its newest snapshot. - latestHeight uint64 - getErr error - - pruneBelowCalled bool - prunedBelow uint64 - pruneErr error -} - -func (m *mockSnapshotStore) GetStoredSnapshots() ([]uint64, error) { - return m.blocks, m.getErr -} - -func (m *mockSnapshotStore) GetLastCommittedBlock() (uint64, error) { - return m.latestHeight, m.getErr -} - -func (m *mockSnapshotStore) GetBlockRange() (uint64, uint64, bool, error) { - if len(m.blocks) == 0 { - return 0, 0, false, m.getErr - } - return m.blocks[0], m.blocks[len(m.blocks)-1], true, m.getErr -} - -func (m *mockSnapshotStore) PruneBelow(blockNumber uint64) error { - m.pruneBelowCalled = true - m.prunedBelow = blockNumber - return m.pruneErr -} - -func (m *mockSnapshotStore) Name() string { - return m.name -} - -func (m *mockSnapshotStore) GetStoreType() StoreType { - return SnapshotStore -} - -// mockStreamStore is a hand-written PrunableStore of type StreamStore for tests. -type mockStreamStore struct { - name string - start uint64 - end uint64 - hasData bool - getErr error - - pruneBelowCalled bool - prunedBelow uint64 - pruneErr error -} - -func (m *mockStreamStore) GetStoredSnapshots() ([]uint64, error) { - return nil, m.getErr -} - -func (m *mockStreamStore) GetBlockRange() (uint64, uint64, bool, error) { - return m.start, m.end, m.hasData, m.getErr -} - -func (m *mockStreamStore) GetLastCommittedBlock() (uint64, error) { - return m.end, m.getErr -} - -func (m *mockStreamStore) PruneBelow(blockNumber uint64) error { - m.pruneBelowCalled = true - m.prunedBelow = blockNumber - return m.pruneErr -} - -func (m *mockStreamStore) Name() string { - return m.name -} - -func (m *mockStreamStore) GetStoreType() StoreType { - return StreamStore -} - -// mockUnknownStore reports a store type the collector does not understand. -type mockUnknownStore struct { - mockSnapshotStore -} - -func (m *mockUnknownStore) GetStoreType() StoreType { - return StoreType(0) -} - -// storeList assembles the single slice prune operates on: the snapshot stores followed by the stream stores. -func storeList(snapshotStores []*mockSnapshotStore, streamStores ...*mockStreamStore) []PrunableStore { - result := make([]PrunableStore, 0, len(snapshotStores)+len(streamStores)) - for _, store := range snapshotStores { - result = append(result, store) - } - for _, store := range streamStores { - result = append(result, store) - } - return result -} - -// TestPruneDecisions covers the full decision matrix: which floor each store is pruned to, what the WAL is pruned to, -// and when a cycle is skipped entirely. want == nil means "PruneBelow must not be called". -func TestPruneDecisions(t *testing.T) { - cases := []struct { - name string - rollbackWindow uint64 - wal *mockStreamStore - storeBlocks [][]uint64 - wantStores []*uint64 // expected prunedBelow per store; nil => not pruned - wantWAL *uint64 // expected WAL prunedBelow; nil => not pruned - }{ - { - name: "single store (flatKV) normal", - rollbackWindow: 10_000, - wal: &mockStreamStore{start: 1, end: 100_000, hasData: true}, - storeBlocks: [][]uint64{{80_000, 85_000, 92_000}}, - wantStores: []*uint64{ptr(85_000)}, - wantWAL: ptr(85_000), - }, - { - name: "three stores, WAL floor is the min", - rollbackWindow: 10_000, - wal: &mockStreamStore{start: 1, end: 100_000, hasData: true}, - storeBlocks: [][]uint64{{80_000, 85_000, 92_000}, {70_000, 88_000, 95_000}, {50_000, 90_000, 99_000}}, - wantStores: []*uint64{ptr(85_000), ptr(88_000), ptr(90_000)}, - wantWAL: ptr(85_000), - }, - { - name: "rollback window zero => oldestNeeded == latest", - rollbackWindow: 0, - wal: &mockStreamStore{start: 1, end: 100_000, hasData: true}, - storeBlocks: [][]uint64{{80_000, 100_000}, {90_000, 100_000}}, - wantStores: []*uint64{ptr(100_000), ptr(100_000)}, - wantWAL: ptr(100_000), - }, - { - name: "latest == rollback window => oldestNeeded == 0", - rollbackWindow: 100_000, - wal: &mockStreamStore{start: 1, end: 100_000, hasData: true}, - storeBlocks: [][]uint64{{500, 1_000}}, - // oldestNeeded 0, no snapshot <= 0, so floor is the lowest snapshot. - wantStores: []*uint64{ptr(500)}, - wantWAL: ptr(500), - }, - { - name: "latest below rollback window => underflow clamped to 0", - rollbackWindow: 10_000, - wal: &mockStreamStore{start: 1, end: 5_000, hasData: true}, - storeBlocks: [][]uint64{{1_000, 2_000}, {1_500}}, - wantStores: []*uint64{ptr(1_000), ptr(1_500)}, - wantWAL: ptr(1_000), - }, - { - name: "snapshot exactly at oldestNeeded is retained", - rollbackWindow: 10_000, - wal: &mockStreamStore{start: 1, end: 100_000, hasData: true}, - storeBlocks: [][]uint64{{50_000, 90_000}, {90_000}}, - wantStores: []*uint64{ptr(90_000), ptr(90_000)}, - wantWAL: ptr(90_000), - }, - { - name: "all snapshots newer than oldestNeeded => floor is oldest snapshot", - rollbackWindow: 10_000, - wal: &mockStreamStore{start: 1, end: 100_000, hasData: true}, - // oldestNeeded 90,000; every snapshot is above it. - storeBlocks: [][]uint64{{95_000, 97_000}, {92_000, 99_000}}, - wantStores: []*uint64{ptr(95_000), ptr(92_000)}, - wantWAL: ptr(92_000), - }, - { - name: "mixed: one store at/below oldest, one all-above => WAL is the min", - rollbackWindow: 10_000, - wal: &mockStreamStore{start: 1, end: 100_000, hasData: true}, - storeBlocks: [][]uint64{{85_000, 92_000}, {95_000, 99_000}}, - wantStores: []*uint64{ptr(85_000), ptr(95_000)}, - wantWAL: ptr(85_000), - }, - { - name: "no stores + WAL has data => prune WAL to the rollback window", - rollbackWindow: 10_000, - wal: &mockStreamStore{start: 1, end: 100_000, hasData: true}, - storeBlocks: nil, - wantStores: nil, - wantWAL: ptr(90_000), - }, - { - name: "no stores + empty WAL => skip", - rollbackWindow: 10_000, - wal: &mockStreamStore{hasData: false}, - storeBlocks: nil, - wantStores: nil, - wantWAL: nil, - }, - { - name: "empty WAL => skip everything", - rollbackWindow: 10_000, - wal: &mockStreamStore{hasData: false}, - storeBlocks: [][]uint64{{80_000}, {80_000}}, - wantStores: []*uint64{nil, nil}, - wantWAL: nil, - }, - { - name: "one store empty among many => skip everything", - rollbackWindow: 10_000, - wal: &mockStreamStore{start: 1, end: 100_000, hasData: true}, - storeBlocks: [][]uint64{{80_000}, nil, {80_000}}, - wantStores: []*uint64{nil, nil, nil}, - wantWAL: nil, - }, - { - name: "all stores empty => skip everything", - rollbackWindow: 10_000, - wal: &mockStreamStore{start: 1, end: 100_000, hasData: true}, - storeBlocks: [][]uint64{nil, nil}, - wantStores: []*uint64{nil, nil}, - wantWAL: nil, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - // Every snapshot store is caught up with the WAL, so the WAL head decides the latest block. - mocks := make([]*mockSnapshotStore, len(tc.storeBlocks)) - for i, blocks := range tc.storeBlocks { - mocks[i] = &mockSnapshotStore{ - name: fmt.Sprintf("store%d", i), - blocks: blocks, - latestHeight: tc.wal.end, - } - } - tc.wal.name = "stateWAL" - - require.NoError(t, prune(tc.rollbackWindow, storeList(mocks, tc.wal))) - - for i, want := range tc.wantStores { - if want == nil { - require.Falsef(t, mocks[i].pruneBelowCalled, "store%d should not be pruned", i) - } else { - require.Truef(t, mocks[i].pruneBelowCalled, "store%d should be pruned", i) - require.Equalf(t, *want, mocks[i].prunedBelow, "store%d prune floor", i) - } - } - if tc.wantWAL == nil { - require.False(t, tc.wal.pruneBelowCalled, "WAL should not be pruned") - } else { - require.True(t, tc.wal.pruneBelowCalled, "WAL should be pruned") - require.Equal(t, *tc.wantWAL, tc.wal.prunedBelow, "WAL prune floor") - } - }) - } -} - -// TestPruneMultipleStreamStores checks that the lowest stream head defines the latest committed block, and that every -// stream store is pruned to the floor of the snapshot stores. -func TestPruneMultipleStreamStores(t *testing.T) { - a := &mockSnapshotStore{name: "a", blocks: []uint64{80_000, 90_000}, latestHeight: 100_000} - ahead := &mockStreamStore{name: "ahead", start: 1, end: 100_000, hasData: true} - behind := &mockStreamStore{name: "behind", start: 1, end: 95_000, hasData: true} - - // latestBlock is the lower head (95,000), so oldestBlockNeeded is 85,000 and the newest snapshot at or below it - // is 80,000. Had the higher head won, the snapshot floor would have been 90,000 instead. - require.NoError(t, prune(10_000, storeList([]*mockSnapshotStore{a}, ahead, behind))) - - require.Equal(t, uint64(80_000), a.prunedBelow) - require.Equal(t, uint64(80_000), ahead.prunedBelow) - require.Equal(t, uint64(80_000), behind.prunedBelow) -} - -func TestPruneMultipleStreamStoresOneEmpty(t *testing.T) { - a := &mockSnapshotStore{name: "a", blocks: []uint64{80_000}, latestHeight: 100_000} - populated := &mockStreamStore{name: "populated", start: 1, end: 100_000, hasData: true} - empty := &mockStreamStore{name: "empty", hasData: false} - - require.NoError(t, prune(10_000, storeList([]*mockSnapshotStore{a}, populated, empty))) - - require.False(t, a.pruneBelowCalled) - require.False(t, populated.pruneBelowCalled) - require.False(t, empty.pruneBelowCalled) -} - -// TestPruneLaggingSnapshotStoreLowersLatestBlock checks that the head comes from every store, not just the streams: a -// snapshot store that has fallen behind the WAL pulls the rollback target down with it. -func TestPruneLaggingSnapshotStoreLowersLatestBlock(t *testing.T) { - // Caught up with the WAL: oldestBlockNeeded is 90,000, so the 90,000 snapshot is the one to keep. - caughtUp := &mockSnapshotStore{name: "caughtUp", blocks: []uint64{80_000, 90_000}, latestHeight: 100_000} - require.NoError(t, prune(10_000, storeList( - []*mockSnapshotStore{caughtUp}, - &mockStreamStore{name: "stateWAL", start: 1, end: 100_000, hasData: true}, - ))) - require.Equal(t, uint64(90_000), caughtUp.prunedBelow) - - // Lagging at 95,000 against the same WAL: oldestBlockNeeded drops to 85,000, so 80,000 must be retained. - lagging := &mockSnapshotStore{name: "lagging", blocks: []uint64{80_000, 90_000}, latestHeight: 95_000} - wal := &mockStreamStore{name: "stateWAL", start: 1, end: 100_000, hasData: true} - require.NoError(t, prune(10_000, storeList([]*mockSnapshotStore{lagging}, wal))) - require.Equal(t, uint64(80_000), lagging.prunedBelow) - require.Equal(t, uint64(80_000), wal.prunedBelow) -} - -// TestPruneWithoutStreamStore checks that a snapshot-only store set still prunes; nothing depends on a stream store -// being present. -func TestPruneWithoutStreamStore(t *testing.T) { - a := &mockSnapshotStore{name: "a", blocks: []uint64{80_000, 90_000}, latestHeight: 100_000} - b := &mockSnapshotStore{name: "b", blocks: []uint64{85_000, 92_000}, latestHeight: 100_000} - - require.NoError(t, prune(10_000, storeList([]*mockSnapshotStore{a, b}))) - - require.Equal(t, uint64(90_000), a.prunedBelow) - require.Equal(t, uint64(85_000), b.prunedBelow) -} - -// TestPruneStoreOrderDoesNotMatter checks that stores are classified by type rather than by position. -func TestPruneStoreOrderDoesNotMatter(t *testing.T) { - a := &mockSnapshotStore{name: "a", blocks: []uint64{85_000, 92_000}, latestHeight: 100_000} - wal := &mockStreamStore{name: "stateWAL", start: 1, end: 100_000, hasData: true} - - require.NoError(t, prune(10_000, []PrunableStore{wal, a})) - - require.Equal(t, uint64(85_000), a.prunedBelow) - require.Equal(t, uint64(85_000), wal.prunedBelow) -} - -func TestPruneWALGetError(t *testing.T) { - sentinel := errors.New("boom") - a := &mockSnapshotStore{name: "a", blocks: []uint64{80_000}, latestHeight: 100_000} - wal := &mockStreamStore{name: "stateWAL", getErr: sentinel} - - err := prune(10_000, storeList([]*mockSnapshotStore{a}, wal)) - require.ErrorIs(t, err, sentinel) - require.ErrorContains(t, err, "stateWAL") - require.False(t, a.pruneBelowCalled) - require.False(t, wal.pruneBelowCalled) -} - -func TestPruneStoreGetError(t *testing.T) { - sentinel := errors.New("boom") - a := &mockSnapshotStore{name: "commitStore", getErr: sentinel} - wal := &mockStreamStore{name: "stateWAL", start: 1, end: 100_000, hasData: true} - - err := prune(10_000, storeList([]*mockSnapshotStore{a}, wal)) - require.ErrorIs(t, err, sentinel) - require.ErrorContains(t, err, "commitStore") - require.False(t, wal.pruneBelowCalled) -} - -func TestPruneStorePruneErrorStopsBeforeLaterStoresAndWAL(t *testing.T) { - sentinel := errors.New("boom") - // The first store fails to prune; later stores and the WAL must be left untouched. - a := &mockSnapshotStore{name: "a", blocks: []uint64{80_000}, latestHeight: 100_000, pruneErr: sentinel} - b := &mockSnapshotStore{name: "b", blocks: []uint64{80_000}, latestHeight: 100_000} - wal := &mockStreamStore{name: "stateWAL", start: 1, end: 100_000, hasData: true} - - err := prune(10_000, storeList([]*mockSnapshotStore{a, b}, wal)) - require.ErrorIs(t, err, sentinel) - require.ErrorContains(t, err, "a") - require.True(t, a.pruneBelowCalled) - require.False(t, b.pruneBelowCalled) - require.False(t, wal.pruneBelowCalled) -} - -func TestPruneWALPruneError(t *testing.T) { - sentinel := errors.New("boom") - // All stores prune successfully, then the WAL prune fails. - a := &mockSnapshotStore{name: "a", blocks: []uint64{80_000}, latestHeight: 100_000} - b := &mockSnapshotStore{name: "b", blocks: []uint64{85_000}, latestHeight: 100_000} - wal := &mockStreamStore{name: "stateWAL", start: 1, end: 100_000, hasData: true, pruneErr: sentinel} - - err := prune(10_000, storeList([]*mockSnapshotStore{a, b}, wal)) - require.ErrorIs(t, err, sentinel) - require.ErrorContains(t, err, "stateWAL") - require.ErrorContains(t, err, "80000") // WAL floor = min(store floors) - require.True(t, a.pruneBelowCalled) - require.True(t, b.pruneBelowCalled) - require.True(t, wal.pruneBelowCalled) -} - -func TestCommittedBlockRange(t *testing.T) { - observationsWithHeads := func(heads ...uint64) []observation { - observations := make([]observation, len(heads)) - for i, head := range heads { - observations[i] = observation{latestBlock: head} - } - return observations - } - - cases := []struct { - name string - heads []uint64 - wantLowest uint64 - wantHighest uint64 - }{ - {name: "single store", heads: []uint64{100}, wantLowest: 100, wantHighest: 100}, - {name: "all agree", heads: []uint64{100, 100, 100}, wantLowest: 100, wantHighest: 100}, - {name: "lowest first", heads: []uint64{80, 100}, wantLowest: 80, wantHighest: 100}, - {name: "lowest last", heads: []uint64{100, 80}, wantLowest: 80, wantHighest: 100}, - {name: "lowest in the middle", heads: []uint64{100, 50, 90}, wantLowest: 50, wantHighest: 100}, - // A store reporting zero must not be mistaken for a store that was skipped. - {name: "a store reports zero", heads: []uint64{0, 100}, wantLowest: 0, wantHighest: 100}, - {name: "every store reports zero", heads: []uint64{0, 0}, wantLowest: 0, wantHighest: 0}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - lowest, highest := committedBlockRange(observationsWithHeads(tc.heads...)) - require.Equal(t, tc.wantLowest, lowest, "lowest head") - require.Equal(t, tc.wantHighest, highest, "highest head") - }) - } -} - -// TestPruneWithLaggingStoreStillPrunes checks that a head spread wide enough to trigger the warning does not change the -// outcome: the cycle still prunes, using the lowest head. -func TestPruneWithLaggingStoreStillPrunes(t *testing.T) { - lagging := &mockSnapshotStore{name: "lagging", blocks: []uint64{50_000, 80_000}, latestHeight: 60_000} - wal := &mockStreamStore{name: "stateWAL", start: 1, end: 100_000, hasData: true} - - // Heads are 60,000 and 100,000, a spread of 40,000 against a 10,000 window. The lowest head wins, so - // oldestBlockNeeded is 50,000 and the 50,000 snapshot is the one retained. - require.NoError(t, prune(10_000, storeList([]*mockSnapshotStore{lagging}, wal))) - - require.Equal(t, uint64(50_000), lagging.prunedBelow) - require.Equal(t, uint64(50_000), wal.prunedBelow) -} - -func TestSnapshotPruningFloor(t *testing.T) { - cases := []struct { - name string - blocks []uint64 - target uint64 - wantFloor uint64 - }{ - {name: "single below", blocks: []uint64{50}, target: 100, wantFloor: 50}, - {name: "single above", blocks: []uint64{150}, target: 100, wantFloor: 150}, - {name: "single exact", blocks: []uint64{100}, target: 100, wantFloor: 100}, - {name: "all below picks max", blocks: []uint64{10, 20, 30}, target: 100, wantFloor: 30}, - {name: "all above picks min", blocks: []uint64{150, 200, 300}, target: 100, wantFloor: 150}, - {name: "exact match preferred", blocks: []uint64{50, 100, 150}, target: 100, wantFloor: 100}, - {name: "mixed", blocks: []uint64{10, 40, 90, 150, 200}, target: 100, wantFloor: 90}, - {name: "target zero all above", blocks: []uint64{5, 10}, target: 0, wantFloor: 5}, - {name: "duplicate blocks", blocks: []uint64{50, 50, 100, 100}, target: 100, wantFloor: 100}, - {name: "brackets target", blocks: []uint64{99, 101}, target: 100, wantFloor: 99}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.wantFloor, snapshotPruningFloor(tc.blocks, tc.target)) - }) - } -} - -// TestSnapshotPruningFloorProperty checks snapshotPruningFloor against a brute-force oracle over random sorted input: -// the result must be the highest block <= target, or the lowest block when none qualify. -func TestSnapshotPruningFloorProperty(t *testing.T) { - rng := rand.New(rand.NewSource(1)) - for iter := 0; iter < 2000; iter++ { - n := 1 + rng.Intn(8) - blocks := make([]uint64, n) - var next uint64 - for i := range blocks { - next += uint64(rng.Intn(5)) // ascending, allows duplicates - blocks[i] = next - } - target := uint64(rng.Intn(int(next) + 5)) - - // Oracle: highest block <= target, or the lowest block (blocks[0]) when none qualify. - want := blocks[0] - for _, b := range blocks { - if b <= target { - want = b - } - } - - require.Equalf(t, want, snapshotPruningFloor(blocks, target), "blocks=%v target=%d", blocks, target) - } -} - -func TestBlocksAtOrAbove(t *testing.T) { - require.Empty(t, blocksAtOrAbove(nil, 100)) - require.Equal(t, []uint64{100, 150}, blocksAtOrAbove([]uint64{50, 100, 150}, 100)) - require.Equal(t, []uint64{10, 20, 30}, blocksAtOrAbove([]uint64{10, 20, 30}, 0)) - require.Empty(t, blocksAtOrAbove([]uint64{10, 20}, 100)) - // Floor strictly between elements. - require.Equal(t, []uint64{60, 70}, blocksAtOrAbove([]uint64{50, 60, 70}, 55)) -} - -// TestBlocksAtOrAboveProperty checks blocksAtOrAbove equals the filter b >= floor, preserving order. -func TestBlocksAtOrAboveProperty(t *testing.T) { - rng := rand.New(rand.NewSource(2)) - for iter := 0; iter < 2000; iter++ { - n := rng.Intn(8) - blocks := make([]uint64, n) - for i := range blocks { - blocks[i] = uint64(rng.Intn(50)) - } - floor := uint64(rng.Intn(50)) - - want := []uint64{} - for _, b := range blocks { - if b >= floor { - want = append(want, b) - } - } - got := blocksAtOrAbove(blocks, floor) - require.Equalf(t, want, got, "blocks=%v floor=%d", blocks, floor) - } -} - -func TestBlockRange(t *testing.T) { - require.Equal(t, "[1, 100]", blockRange(1, 100)) - require.Equal(t, "[0, 0]", blockRange(0, 0)) -} - -func TestDefaultStorageGarbageCollectorConfig(t *testing.T) { - cfg := DefaultStorageGarbageCollectorConfig() - require.Equal(t, uint64(10_000), cfg.RollbackWindow) - require.Equal(t, uint64(60), cfg.PruneIntervalSeconds) -} - -func TestValidate(t *testing.T) { - require.NoError(t, DefaultStorageGarbageCollectorConfig().Validate()) - - // A zero rollback window is legal. - require.NoError(t, (&StorageGarbageCollectorConfig{RollbackWindow: 0, PruneIntervalSeconds: 60}).Validate()) - - err := (&StorageGarbageCollectorConfig{PruneIntervalSeconds: 0}).Validate() - require.ErrorContains(t, err, "prune interval") - - // The largest interval that does not overflow time.Duration is accepted; one larger is rejected. - maxInterval := uint64(math.MaxInt64) / uint64(time.Second) - require.NoError(t, (&StorageGarbageCollectorConfig{PruneIntervalSeconds: maxInterval}).Validate()) - require.ErrorContains(t, (&StorageGarbageCollectorConfig{PruneIntervalSeconds: maxInterval + 1}).Validate(), "at most") -} - -func TestNewStorageGarbageCollectorInvalidConfig(t *testing.T) { - sm, err := NewStorageGarbageCollector( - context.Background(), - &StorageGarbageCollectorConfig{PruneIntervalSeconds: 0}, - storeList([]*mockSnapshotStore{{name: "a"}}, &mockStreamStore{name: "stateWAL"}), - ) - require.Error(t, err) - require.Nil(t, sm) -} - -func TestNewStorageGarbageCollectorRejectsUnsupportedStoreType(t *testing.T) { - sm, err := NewStorageGarbageCollector( - context.Background(), - &StorageGarbageCollectorConfig{RollbackWindow: 10, PruneIntervalSeconds: 60}, - []PrunableStore{ - &mockUnknownStore{mockSnapshotStore{name: "mystery"}}, - &mockStreamStore{name: "stateWAL"}, - }, - ) - require.ErrorContains(t, err, "mystery") - require.ErrorContains(t, err, "unsupported store type") - require.Nil(t, sm) -} - -func TestNewStorageGarbageCollectorAcceptsAnyMixOfStoreTypes(t *testing.T) { - config := &StorageGarbageCollectorConfig{RollbackWindow: 10, PruneIntervalSeconds: 60} - - for _, tc := range []struct { - name string - stores []PrunableStore - }{ - {name: "snapshot stores only", stores: storeList([]*mockSnapshotStore{{name: "a", blocks: []uint64{100}}})}, - {name: "stream stores only", stores: storeList(nil, &mockStreamStore{name: "stateWAL"})}, - {name: "no stores", stores: nil}, - } { - t.Run(tc.name, func(t *testing.T) { - sm, err := NewStorageGarbageCollector(context.Background(), config, tc.stores) - require.NoError(t, err) - require.NoError(t, sm.Close()) - }) - } -} - -func TestStoreTypeString(t *testing.T) { - require.Equal(t, "SnapshotStore", SnapshotStore.String()) - require.Equal(t, "StreamStore", StreamStore.String()) - require.Equal(t, "UnknownStoreType(0)", StoreType(0).String()) -} - -func TestNewStorageGarbageCollectorConstructAndClose(t *testing.T) { - a := &mockSnapshotStore{name: "a", blocks: []uint64{100}, latestHeight: 100} - wal := &mockStreamStore{name: "stateWAL", start: 1, end: 100, hasData: true} - sm, err := NewStorageGarbageCollector( - context.Background(), - &StorageGarbageCollectorConfig{RollbackWindow: 10, PruneIntervalSeconds: 60}, - storeList([]*mockSnapshotStore{a}, wal), - ) - require.NoError(t, err) - require.NotNil(t, sm) - - // Close must return without hanging. - require.NoError(t, sm.Close()) -} - -func ptr(v uint64) *uint64 { - return &v -} diff --git a/sei-db/management/store_types.go b/sei-db/management/store_types.go deleted file mode 100644 index 1710b8ac8f..0000000000 --- a/sei-db/management/store_types.go +++ /dev/null @@ -1,66 +0,0 @@ -package management - -import "fmt" - -// StoreType describes how a store retains its data, which determines how the StorageGarbageCollector -// interprets what the store currently holds. -type StoreType int - -const ( - // SnapshotStore is a store that is persisted via snapshots (e.g. SC, SS). It retains a discrete set of - // snapshotted heights, reported by GetStoredSnapshots. - SnapshotStore StoreType = iota + 1 - - // StreamStore is a store that is made up of a stream of data (e.g. a WAL). It retains a contiguous range - // of blocks, reported by GetBlockRange. - StreamStore -) - -func (t StoreType) String() string { - switch t { - case SnapshotStore: - return "SnapshotStore" - case StreamStore: - return "StreamStore" - default: - return fmt.Sprintf("UnknownStoreType(%d)", int(t)) - } -} - -// PrunableStore is a store whose old data may be dropped by the StorageGarbageCollector. -// -// Both GetStoredSnapshots and GetBlockRange must be implemented, but only the accessor matching the store's -// StoreType is required to carry information: a StreamStore has no discrete snapshots to report, and a -// SnapshotStore reports the range spanned by the snapshots it holds. -type PrunableStore interface { - // Name Return the name of the store. - Name() string - - // GetStoreType returns the type of the store. - GetStoreType() StoreType - - // PruneBelow tells the store that it may drop snapshots/data for all blocks below a specified number. - // Store can drop data asynchronously in the background. - PruneBelow(blockNumber uint64) error - - // GetStoredSnapshots return a list of all block heights that this store has snapshots for, in ascending - // sorted order. A StreamStore has no snapshots and returns nil. - GetStoredSnapshots() ([]uint64, error) - - // GetBlockRange fetch the range of blocks stored in this store. - GetBlockRange() ( - start uint64, // inclusive; meaningful only when hasData is true - end uint64, // inclusive; meaningful only when hasData is true - hasData bool, // true if the store contains at least one block - err error, - ) - - // GetLastCommittedBlock returns the highest block this store has ingested. Meaningful only when the store has - // data. - // - // This is the store's committed height, not the newest data it has durably retained. For a SnapshotStore the - // two differ: snapshots are written periodically while the store keeps ingesting blocks, so the committed - // height normally sits above the newest snapshot. Reporting the newest snapshot height instead is safe but - // makes the garbage collector systematically under-prune. - GetLastCommittedBlock() (uint64, error) -} From e1b473a3d1972875b736c9740e73266f92a6255f Mon Sep 17 00:00:00 2001 From: YimingZang Date: Thu, 30 Jul 2026 16:36:42 -0700 Subject: [PATCH 03/15] Fix giga config path --- sei-db/common/utils/path.go | 7 +++++++ sei-db/common/utils/path_test.go | 8 ++++++++ sei-db/config/giga_config.go | 32 +++++++++++++++++++++++++------- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/sei-db/common/utils/path.go b/sei-db/common/utils/path.go index d98751d3b0..5148ec59c3 100644 --- a/sei-db/common/utils/path.go +++ b/sei-db/common/utils/path.go @@ -74,6 +74,13 @@ func GetReceiptStorePath(homePath string, backend string) string { return filepath.Join(homePath, "data", "ledger", "receipt", backend) } +// GetBlockStorePath returns the path for the ledger block store, which sits alongside the +// receipt store under data/ledger. There is no legacy fallback: the block store has only +// ever lived at this path. +func GetBlockStorePath(homePath string) string { + return filepath.Join(homePath, "data", "ledger", "block") +} + func GetChangelogPath(dbPath string) string { return filepath.Join(dbPath, "changelog") } diff --git a/sei-db/common/utils/path_test.go b/sei-db/common/utils/path_test.go index a5b6b009db..3a5a90657a 100644 --- a/sei-db/common/utils/path_test.go +++ b/sei-db/common/utils/path_test.go @@ -135,6 +135,14 @@ func TestGetReceiptStorePath_LegacyExists(t *testing.T) { assert.Equal(t, legacy, got) } +// --- GetBlockStorePath --- + +func TestGetBlockStorePath_NewNode(t *testing.T) { + home := t.TempDir() + got := GetBlockStorePath(home) + assert.Equal(t, filepath.Join(home, "data", "ledger", "block"), got) +} + // --- GetChangelogPath (unchanged, but verify) --- func TestGetChangelogPath(t *testing.T) { diff --git a/sei-db/config/giga_config.go b/sei-db/config/giga_config.go index 581b67fd7a..eed9b1db24 100644 --- a/sei-db/config/giga_config.go +++ b/sei-db/config/giga_config.go @@ -1,6 +1,7 @@ package config import ( + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/littblock" flatkvConfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" ) @@ -11,25 +12,42 @@ const DefaultRetention = 1000000 type GigaStorageConfig struct { RollbackWindow uint64 BlockRetention uint64 - DataDirectory string + HomePath string FlatKVConfig *flatkvConfig.Config SSConfig StateStoreConfig ReceiptDBConfig ReceiptStoreConfig BlockDBConfig *littblock.LittBlockConfig } -func DefaultGigaStorageConfig(dbDir string) GigaStorageConfig { - blockDBConfig, err := littblock.DefaultConfig(dbDir) +// DefaultGigaStorageConfig returns a GigaStorageConfig whose store directories match the +// below layout: +// +// data/state_commit/flatkv +// data/state_store/evm/{backend} +// data/ledger/receipt/{backend} +// data/ledger/block +func DefaultGigaStorageConfig(homePath string) GigaStorageConfig { + blockDBConfig, err := littblock.DefaultConfig(utils.GetBlockStorePath(homePath)) if err != nil { panic(err) } + + flatKV := flatkvConfig.DefaultConfig() + flatKV.DataDir = utils.GetFlatKVPath(homePath) + + ssConfig := DefaultStateStoreConfig() + ssConfig.DBDirectory = utils.GetEVMStateStorePath(homePath, ssConfig.Backend) + + receiptConfig := DefaultReceiptStoreConfig() + receiptConfig.DBDirectory = utils.GetReceiptStorePath(homePath, receiptConfig.Backend) + return GigaStorageConfig{ RollbackWindow: DefaultRollbackWindow, BlockRetention: DefaultRetention, - DataDirectory: dbDir, - FlatKVConfig: flatkvConfig.DefaultConfig(), - SSConfig: DefaultStateStoreConfig(), - ReceiptDBConfig: DefaultReceiptStoreConfig(), + HomePath: homePath, + FlatKVConfig: flatKV, + SSConfig: ssConfig, + ReceiptDBConfig: receiptConfig, BlockDBConfig: blockDBConfig, } } From 1d3c1806b34be2bc50b08dc1fa5d3c2e06d593bf Mon Sep 17 00:00:00 2001 From: YimingZang Date: Thu, 30 Jul 2026 16:53:11 -0700 Subject: [PATCH 04/15] Fix edge case for 0 --- sei-db/config/giga_config.go | 24 +++++++++++------ sei-db/management/gc/api.go | 9 ++++--- .../gc/storage_garbage_collector.go | 26 ++++++++++++------- .../gc/storage_garbage_collector_test.go | 6 +++-- 4 files changed, 42 insertions(+), 23 deletions(-) diff --git a/sei-db/config/giga_config.go b/sei-db/config/giga_config.go index eed9b1db24..7916952f02 100644 --- a/sei-db/config/giga_config.go +++ b/sei-db/config/giga_config.go @@ -3,24 +3,31 @@ package config import ( "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/littblock" + "github.com/sei-protocol/sei-chain/sei-db/management/gc" flatkvConfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" ) -const DefaultRollbackWindow = 1000 -const DefaultRetention = 1000000 - +// GigaStorageConfig is an in-process composition of store configs for Giga storage. +// It is intentionally not read from app.toml (no mapstructure tags, no TOML section): +// callers build it via DefaultGigaStorageConfig. Nested knobs live on the store and +// collector configs they already own (StateStoreConfig, ReceiptStoreConfig, +// gc.StorageGarbageCollectorConfig) rather than being redeclared here — in particular +// RollbackWindow and StoreRetention have a single source of truth in +// gc.DefaultStorageGarbageCollectorConfig. type GigaStorageConfig struct { - RollbackWindow uint64 - BlockRetention uint64 HomePath string FlatKVConfig *flatkvConfig.Config SSConfig StateStoreConfig ReceiptDBConfig ReceiptStoreConfig BlockDBConfig *littblock.LittBlockConfig + // PruningConfig is the garbage-collector config shared by every store under this + // Giga layout. Defaults come from gc.DefaultStorageGarbageCollectorConfig; do not + // reintroduce parallel RollbackWindow / StoreRetention fields on this struct. + PruningConfig *gc.StorageGarbageCollectorConfig } // DefaultGigaStorageConfig returns a GigaStorageConfig whose store directories match the -// below layout: +// layout below, and whose pruning knobs are gc.DefaultStorageGarbageCollectorConfig(): // // data/state_commit/flatkv // data/state_store/evm/{backend} @@ -41,13 +48,14 @@ func DefaultGigaStorageConfig(homePath string) GigaStorageConfig { receiptConfig := DefaultReceiptStoreConfig() receiptConfig.DBDirectory = utils.GetReceiptStorePath(homePath, receiptConfig.Backend) + pruningConfig := gc.DefaultStorageGarbageCollectorConfig() + return GigaStorageConfig{ - RollbackWindow: DefaultRollbackWindow, - BlockRetention: DefaultRetention, HomePath: homePath, FlatKVConfig: flatKV, SSConfig: ssConfig, ReceiptDBConfig: receiptConfig, BlockDBConfig: blockDBConfig, + PruningConfig: pruningConfig, } } diff --git a/sei-db/management/gc/api.go b/sei-db/management/gc/api.go index b112da56b0..4fd2c31a3e 100644 --- a/sei-db/management/gc/api.go +++ b/sei-db/management/gc/api.go @@ -2,7 +2,7 @@ package gc // PrunableStore is a store whose old data may be dropped by the StorageGarbageCollector. type PrunableStore interface { - // Name Return the name of the store. + // Name Returns the name of the store. Name() string // PruneBelow tells the store that it may drop snapshots/data for all blocks below a specified number. @@ -11,8 +11,11 @@ type PrunableStore interface { // GetOldestBlockToRetain returns the oldest block this store must keep in order to remain able to serve cutLine, // or 0 when the store holds nothing. A chain's first block is block 1, so 0 is free to serve as that sentinel: no - // store can legitimately need to retain from block 0. The collector ignores a store that answers 0, since a store - // holding nothing cannot serve a rollback to any height. + // store can legitimately need to retain from block 0. + // + // The collector skips the whole prune cycle when any store answers 0. An empty answer is treated as unknown rather + // than empty — a snapshot may be mid-write and take hours to land — and pruning the other stores against it would + // risk deleting contiguous blocks that snapshot will need once it appears. // // A store that retains a contiguous range of blocks (blockDB, receiptDB, the state WAL) can be restored to any // block it holds, so it needs nothing below the cut line and returns cutLine unchanged. diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go index 595bc4a493..fa391dc7c3 100644 --- a/sei-db/management/gc/storage_garbage_collector.go +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -9,7 +9,7 @@ import ( "github.com/sei-protocol/seilog" ) -var logger = seilog.NewLogger("db", "management") +var logger = seilog.NewLogger("db", "gc") // StorageGarbageCollector manages deletion of stored data across a set of stores. // @@ -121,25 +121,31 @@ func (s *StorageGarbageCollector) prune() error { // Every store must keep back to its own answer, so the system may only prune below the lowest of them. Pruning // below a higher answer would delete blocks that a lower-answering store just reported it still needs, which is // exactly how deletion would break the rollback invariant. + // + // A store that answers 0 is treated as unknown rather than empty: a snapshot may be mid-write and take hours to + // land. Skipping that store and pruning everyone else would delete the contiguous blocks the in-flight snapshot + // will need once it appears, so the whole cycle is skipped until every store reports something to retain. var pruneHeight uint64 + oldestBlockToRetainByStore := make(map[string]uint64, len(s.stores)) for _, store := range s.stores { oldestBlockToRetain := store.GetOldestBlockToRetain(cutLine) + oldestBlockToRetainByStore[store.Name()] = oldestBlockToRetain if oldestBlockToRetain == 0 { - // The store holds nothing, so it cannot serve a rollback to any height and has no stake in how far the - // other stores prune. - continue + logger.Debug("skipping pruning, a store holds no data to retain", + "cutLine", cutLine, "oldestBlockToRetainByStore", oldestBlockToRetainByStore) + return nil } - // A pruneHeight of 0 means no store has answered yet, since the only zero answer is skipped above. if pruneHeight == 0 || oldestBlockToRetain < pruneHeight { pruneHeight = oldestBlockToRetain } } - if pruneHeight == 0 { - logger.Debug("skipping pruning, no store has data to retain", "cutLine", cutLine) - return nil - } - logger.Info("pruning storage", "pruneHeight", pruneHeight) + logger.Info("pruning storage", + "globalLatestBlock", globalLatestBlock, + "cutLine", cutLine, + "pruneHeight", pruneHeight, + "oldestBlockToRetainByStore", oldestBlockToRetainByStore, + ) for _, store := range s.stores { if err := store.PruneBelow(pruneHeight); err != nil { return fmt.Errorf("failed to prune %s below %d: %w", store.Name(), pruneHeight, err) diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go index 36bf48c4d3..2919482c89 100644 --- a/sei-db/management/gc/storage_garbage_collector_test.go +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -195,14 +195,16 @@ func TestPruneDecisions(t *testing.T) { wantPruneBelow: ptr(30_000), }, { - name: "a store holding nothing is ignored", + name: "a store holding nothing stalls the whole cycle", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("empty", 100_000), snapshotStore("sc", 100_000, 80_000), contiguousStore("stateWAL", 100_000, true), }, - wantPruneBelow: ptr(80_000), + // empty may be mid-write of a first snapshot below the cut line; pruning the WAL to 80,000 would leave + // that snapshot unreplayable once it lands. + wantPruneBelow: nil, }, { name: "a head of zero is ignored, the data behind it is not", From 74629a0112c3cbc17298cd45557c6f115dc20e2b Mon Sep 17 00:00:00 2001 From: YimingZang Date: Thu, 30 Jul 2026 16:56:07 -0700 Subject: [PATCH 05/15] Avoid panic in default config --- sei-db/config/giga_config.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sei-db/config/giga_config.go b/sei-db/config/giga_config.go index 7916952f02..f8c0aa01b1 100644 --- a/sei-db/config/giga_config.go +++ b/sei-db/config/giga_config.go @@ -1,6 +1,8 @@ package config import ( + "fmt" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/littblock" "github.com/sei-protocol/sei-chain/sei-db/management/gc" @@ -33,10 +35,13 @@ type GigaStorageConfig struct { // data/state_store/evm/{backend} // data/ledger/receipt/{backend} // data/ledger/block -func DefaultGigaStorageConfig(homePath string) GigaStorageConfig { +// +// Unlike the other Default*Config helpers in this package, this can fail: the block-store +// default wraps littdb.DefaultConfig, which validates the directory path. +func DefaultGigaStorageConfig(homePath string) (GigaStorageConfig, error) { blockDBConfig, err := littblock.DefaultConfig(utils.GetBlockStorePath(homePath)) if err != nil { - panic(err) + return GigaStorageConfig{}, fmt.Errorf("failed to build block db config: %w", err) } flatKV := flatkvConfig.DefaultConfig() @@ -48,14 +53,12 @@ func DefaultGigaStorageConfig(homePath string) GigaStorageConfig { receiptConfig := DefaultReceiptStoreConfig() receiptConfig.DBDirectory = utils.GetReceiptStorePath(homePath, receiptConfig.Backend) - pruningConfig := gc.DefaultStorageGarbageCollectorConfig() - return GigaStorageConfig{ HomePath: homePath, FlatKVConfig: flatKV, SSConfig: ssConfig, ReceiptDBConfig: receiptConfig, BlockDBConfig: blockDBConfig, - PruningConfig: pruningConfig, - } + PruningConfig: gc.DefaultStorageGarbageCollectorConfig(), + }, nil } From f6501953ba99fddfdb5e125ca47d3d49370d05cc Mon Sep 17 00:00:00 2001 From: YimingZang Date: Thu, 30 Jul 2026 17:03:14 -0700 Subject: [PATCH 06/15] Address comments --- .../gc/storage_garbage_collector.go | 6 +- .../gc/storage_garbage_collector_config.go | 22 ++- .../gc/storage_garbage_collector_test.go | 136 ++++++++++++++---- 3 files changed, 118 insertions(+), 46 deletions(-) diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go index fa391dc7c3..16e9d538a4 100644 --- a/sei-db/management/gc/storage_garbage_collector.go +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -74,12 +74,12 @@ func (s *StorageGarbageCollector) Close() error { } // run periodically drives prune cycles until the manager is stopped. All decision logic lives in prune so it can -// be unit tested without threading. +// be unit tested without threading; the ticker path is covered by constructing a collector with a short +// config.PruneInterval. func (s *StorageGarbageCollector) run() { defer s.wg.Done() - //nolint:gosec // G115 - Config.Validate rejects PruneIntervalSeconds large enough to overflow this conversion. - ticker := time.NewTicker(time.Duration(s.config.PruneIntervalSeconds) * time.Second) + ticker := time.NewTicker(s.config.PruneInterval) defer ticker.Stop() for { diff --git a/sei-db/management/gc/storage_garbage_collector_config.go b/sei-db/management/gc/storage_garbage_collector_config.go index 802155db98..328433b48b 100644 --- a/sei-db/management/gc/storage_garbage_collector_config.go +++ b/sei-db/management/gc/storage_garbage_collector_config.go @@ -33,29 +33,27 @@ type StorageGarbageCollectorConfig struct { // contiguous stores must still cover the oldest snapshot anyone kept, or the snapshot becomes unreplayable. StoreRetention uint64 - // The frequency of prune operations, in seconds. - PruneIntervalSeconds uint64 + // How often the collector drives a prune cycle. + PruneInterval time.Duration } // Construct a default storage garbage collector config. func DefaultStorageGarbageCollectorConfig() *StorageGarbageCollectorConfig { return &StorageGarbageCollectorConfig{ - RollbackWindow: 1000, - StoreRetention: 100_000, - PruneIntervalSeconds: 600, + RollbackWindow: 1000, + StoreRetention: 100_000, + PruneInterval: 10 * time.Minute, } } // Validate the storage garbage collector's config. func (c *StorageGarbageCollectorConfig) Validate() error { - // A zero rollback window is legal: it means the system prunes as aggressively as possible. - if c.PruneIntervalSeconds == 0 { - return fmt.Errorf("prune interval must be greater than 0 seconds") + if c == nil { + return fmt.Errorf("config is required") } - // The prune interval is converted to a time.Duration (int64 nanoseconds) when the ticker is created. Reject values - // that would overflow that conversion so the ticker can never be handed a zero/negative duration. - if c.PruneIntervalSeconds > math.MaxInt64/uint64(time.Second) { - return fmt.Errorf("prune interval must be at most %d seconds", math.MaxInt64/uint64(time.Second)) + // A zero rollback window is legal: it means the system prunes as aggressively as possible. + if c.PruneInterval <= 0 { + return fmt.Errorf("prune interval must be greater than 0") } // The cut line is derived by subtracting RollbackWindow + StoreRetention from the head of the chain. Reject a pair // that overflows when summed, so that subtraction cannot wrap around to a cut line above the head. diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go index 2919482c89..f56b4cd189 100644 --- a/sei-db/management/gc/storage_garbage_collector_test.go +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "math" + "sync/atomic" "testing" "time" @@ -19,8 +20,9 @@ type mockStore struct { getErr error pruneErr error - pruneBelowCalled bool - prunedBelow uint64 + // Atomically observed so the run-loop ticker tests can assert without racing the collector goroutine. + pruneBelowCalled atomic.Bool + prunedBelow atomic.Uint64 } // snapshotStore models a store that can only be restored to a height it holds a snapshot for, such as SC or SS: it @@ -74,8 +76,8 @@ func (m *mockStore) GetOldestBlockToRetain(cutLine uint64) uint64 { } func (m *mockStore) PruneBelow(blockNumber uint64) error { - m.pruneBelowCalled = true - m.prunedBelow = blockNumber + m.pruneBelowCalled.Store(true) + m.prunedBelow.Store(blockNumber) return m.pruneErr } @@ -98,9 +100,9 @@ func newTestCollector( t.Helper() config := &StorageGarbageCollectorConfig{ - RollbackWindow: rollbackWindow, - StoreRetention: storeRetention, - PruneIntervalSeconds: 60, + RollbackWindow: rollbackWindow, + StoreRetention: storeRetention, + PruneInterval: time.Minute, } // getCutLine subtracts without an overflow guard because Validate rejects a retain window that would overflow, so // test configs have to clear the same bar as real ones. @@ -270,11 +272,11 @@ func TestPruneDecisions(t *testing.T) { for _, store := range tc.stores { if tc.wantPruneBelow == nil { - require.Falsef(t, store.pruneBelowCalled, "%s should not be pruned", store.name) + require.Falsef(t, store.pruneBelowCalled.Load(), "%s should not be pruned", store.name) continue } - require.Truef(t, store.pruneBelowCalled, "%s should be pruned", store.name) - require.Equalf(t, *tc.wantPruneBelow, store.prunedBelow, "%s prune height", store.name) + require.Truef(t, store.pruneBelowCalled.Load(), "%s should be pruned", store.name) + require.Equalf(t, *tc.wantPruneBelow, store.prunedBelow.Load(), "%s prune height", store.name) } }) } @@ -293,8 +295,8 @@ func TestPruneOnYoungChainWithDefaultConfig(t *testing.T) { } require.NoError(t, collector.prune()) - require.False(t, sc.pruneBelowCalled) - require.False(t, wal.pruneBelowCalled) + require.False(t, sc.pruneBelowCalled.Load()) + require.False(t, wal.pruneBelowCalled.Load()) } // TestPruneGetLastCommittedBlockError checks that a failure to read a head aborts the cycle before anything is deleted. @@ -307,8 +309,8 @@ func TestPruneGetLastCommittedBlockError(t *testing.T) { err := newTestCollector(t, 10_000, 0, sc, broken).prune() require.ErrorIs(t, err, sentinel) require.ErrorContains(t, err, "brokenStore") - require.False(t, sc.pruneBelowCalled) - require.False(t, broken.pruneBelowCalled) + require.False(t, sc.pruneBelowCalled.Load()) + require.False(t, broken.pruneBelowCalled.Load()) } // TestPruneBelowErrorStopsRemainingStores checks that the first failed deletion aborts the cycle, leaving later stores @@ -324,9 +326,9 @@ func TestPruneBelowErrorStopsRemainingStores(t *testing.T) { require.ErrorIs(t, err, sentinel) require.ErrorContains(t, err, "first") require.ErrorContains(t, err, "80000") // the prune height, i.e. the lowest need across the stores - require.True(t, first.pruneBelowCalled) - require.False(t, second.pruneBelowCalled) - require.False(t, wal.pruneBelowCalled) + require.True(t, first.pruneBelowCalled.Load()) + require.False(t, second.pruneBelowCalled.Load()) + require.False(t, wal.pruneBelowCalled.Load()) } func TestGetCutLine(t *testing.T) { @@ -396,43 +398,47 @@ func TestDefaultStorageGarbageCollectorConfig(t *testing.T) { cfg := DefaultStorageGarbageCollectorConfig() require.Equal(t, uint64(1_000), cfg.RollbackWindow) require.Equal(t, uint64(100_000), cfg.StoreRetention) - require.Equal(t, uint64(600), cfg.PruneIntervalSeconds) + require.Equal(t, 10*time.Minute, cfg.PruneInterval) require.NoError(t, cfg.Validate()) } func TestValidate(t *testing.T) { - // A zero rollback window is legal, as is a zero store retention. - require.NoError(t, (&StorageGarbageCollectorConfig{PruneIntervalSeconds: 60}).Validate()) + require.ErrorContains(t, (*StorageGarbageCollectorConfig)(nil).Validate(), "config is required") - require.ErrorContains(t, (&StorageGarbageCollectorConfig{PruneIntervalSeconds: 0}).Validate(), "prune interval") + // A zero rollback window is legal, as is a zero store retention. + require.NoError(t, (&StorageGarbageCollectorConfig{PruneInterval: time.Minute}).Validate()) - // The largest interval that does not overflow time.Duration is accepted; one larger is rejected. - maxInterval := uint64(math.MaxInt64) / uint64(time.Second) - require.NoError(t, (&StorageGarbageCollectorConfig{PruneIntervalSeconds: maxInterval}).Validate()) - require.ErrorContains(t, (&StorageGarbageCollectorConfig{PruneIntervalSeconds: maxInterval + 1}).Validate(), "at most") + require.ErrorContains(t, (&StorageGarbageCollectorConfig{PruneInterval: 0}).Validate(), "prune interval") + require.ErrorContains(t, (&StorageGarbageCollectorConfig{PruneInterval: -time.Second}).Validate(), "prune interval") // The retain window is the sum of the two, so the pair must not overflow. getCutLine subtracts it without a guard. require.NoError(t, (&StorageGarbageCollectorConfig{ - RollbackWindow: math.MaxUint64, - PruneIntervalSeconds: 60, + RollbackWindow: math.MaxUint64, + PruneInterval: time.Minute, }).Validate()) require.ErrorContains(t, (&StorageGarbageCollectorConfig{ - RollbackWindow: math.MaxUint64, - StoreRetention: 1, - PruneIntervalSeconds: 60, + RollbackWindow: math.MaxUint64, + StoreRetention: 1, + PruneInterval: time.Minute, }).Validate(), "store retention") } func TestNewStorageGarbageCollectorInvalidConfig(t *testing.T) { sm, err := NewStorageGarbageCollector( context.Background(), - &StorageGarbageCollectorConfig{PruneIntervalSeconds: 0}, + &StorageGarbageCollectorConfig{PruneInterval: 0}, prunableStores(snapshotStore("sc", 100)), ) require.Error(t, err) require.Nil(t, sm) } +func TestNewStorageGarbageCollectorNilConfig(t *testing.T) { + sm, err := NewStorageGarbageCollector(context.Background(), nil, nil) + require.ErrorContains(t, err, "config is required") + require.Nil(t, sm) +} + func TestNewStorageGarbageCollectorConstructAndClose(t *testing.T) { sm, err := NewStorageGarbageCollector( context.Background(), @@ -456,6 +462,74 @@ func TestCloseAfterContextCancelled(t *testing.T) { require.NoError(t, sm.Close()) } +// startTestCollectorWithInterval builds a running collector with a short PruneInterval so the ticker path can be +// exercised without waiting on the production default. +func startTestCollectorWithInterval( + t *testing.T, + interval time.Duration, + stores []PrunableStore, +) *StorageGarbageCollector { + t.Helper() + + config := &StorageGarbageCollectorConfig{ + RollbackWindow: 10_000, + PruneInterval: interval, + } + require.NoError(t, config.Validate()) + + s := &StorageGarbageCollector{ + config: config, + stores: stores, + ctx: context.Background(), + stopCh: make(chan struct{}), + } + s.wg.Add(1) + go s.run() + t.Cleanup(func() { require.NoError(t, s.Close()) }) + return s +} + +// TestRunFiresPruneCycle checks that the ticker path actually invokes prune. +func TestRunFiresPruneCycle(t *testing.T) { + sc := snapshotStore("sc", 100_000, 80_000) + wal := contiguousStore("stateWAL", 100_000, true) + + _ = startTestCollectorWithInterval(t, 5*time.Millisecond, prunableStores(sc, wal)) + + require.Eventually(t, func() bool { + return sc.pruneBelowCalled.Load() && wal.pruneBelowCalled.Load() + }, time.Second, 5*time.Millisecond) + require.Equal(t, uint64(80_000), sc.prunedBelow.Load()) + require.Equal(t, uint64(80_000), wal.prunedBelow.Load()) +} + +// countingStore counts how many times GetLastCommittedBlock is called so a failing prune cycle can be observed +// without depending on log output. +type countingStore struct { + *mockStore + calls atomic.Uint64 +} + +func (c *countingStore) GetLastCommittedBlock() (uint64, error) { + c.calls.Add(1) + return c.mockStore.GetLastCommittedBlock() +} + +// TestRunContinuesAfterPruneError checks that a failed prune cycle does not kill the run loop: the ticker keeps +// firing and prune is invoked again. +func TestRunContinuesAfterPruneError(t *testing.T) { + broken := &countingStore{ + mockStore: contiguousStore("broken", 100_000, true), + } + broken.getErr = errors.New("boom") + + _ = startTestCollectorWithInterval(t, 5*time.Millisecond, []PrunableStore{broken}) + + require.Eventually(t, func() bool { + return broken.calls.Load() >= 2 + }, time.Second, 5*time.Millisecond) +} + func ptr(v uint64) *uint64 { return &v } From c27486669f191bf395def345c06e1641efe33615 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Thu, 30 Jul 2026 17:19:37 -0700 Subject: [PATCH 07/15] Address more comments --- sei-db/management/gc/api.go | 21 ++++---- .../gc/storage_garbage_collector.go | 28 ++++++---- .../gc/storage_garbage_collector_config.go | 5 +- .../gc/storage_garbage_collector_test.go | 53 ++++++++++++++----- 4 files changed, 70 insertions(+), 37 deletions(-) diff --git a/sei-db/management/gc/api.go b/sei-db/management/gc/api.go index 4fd2c31a3e..ae9bf47227 100644 --- a/sei-db/management/gc/api.go +++ b/sei-db/management/gc/api.go @@ -10,21 +10,22 @@ type PrunableStore interface { PruneBelow(blockNumber uint64) error // GetOldestBlockToRetain returns the oldest block this store must keep in order to remain able to serve cutLine, - // or 0 when the store holds nothing. A chain's first block is block 1, so 0 is free to serve as that sentinel: no - // store can legitimately need to retain from block 0. + // or 0 when the store opts out of pruning entirely (e.g. infinite retention or truly uninitialized). A chain's + // first block is block 1, so 0 is free to serve as that sentinel. // - // The collector skips the whole prune cycle when any store answers 0. An empty answer is treated as unknown rather - // than empty — a snapshot may be mid-write and take hours to land — and pruning the other stores against it would - // risk deleting contiguous blocks that snapshot will need once it appears. + // The collector ignores a store that answers 0 when choosing the prune height and does not call PruneBelow on it, + // so other stores can still be pruned. // // A store that retains a contiguous range of blocks (blockDB, receiptDB, the state WAL) can be restored to any // block it holds, so it needs nothing below the cut line and returns cutLine unchanged. - // A store with checkpoints can only be restored to a height it has a snapshot for, - // so it returns the newest snapshot at or below cutLine, or its oldest snapshot when every snapshot is above the cut line, - // since in that case none of them can be dropped. + // A store with checkpoints can only be restored to a height it has a snapshot for, so it returns the newest + // snapshot at or below cutLine, or its oldest snapshot when every snapshot is above the cut line, since in that + // case none of them can be dropped. If it has not completed any snapshot yet, it returns its last committed + // height — including an in-progress snapshot height — so contiguous stores are not pruned past blocks that + // snapshot will need once it lands. // - // The collector prunes every store below the lowest answer it receives, so an answer that is too low only costs temporary - // disk, while one that is too high deletes blocks the system still needs. + // The collector prunes every participating store below the lowest non-zero answer it receives, so an answer that is + // too low only costs temporary disk, while one that is too high deletes blocks the system still needs. GetOldestBlockToRetain(cutLine uint64) uint64 // GetLastCommittedBlock returns the highest block this store has ingested. diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go index 16e9d538a4..e78e27d186 100644 --- a/sei-db/management/gc/storage_garbage_collector.go +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -107,38 +107,41 @@ func (s *StorageGarbageCollector) prune() error { return err } if globalLatestBlock == 0 { - logger.Debug("skipping pruning, no store has committed a block") + logger.Info("skipping pruning, no store has committed a block") return nil } cutLine := getCutLine(globalLatestBlock, s.config.RollbackWindow, s.config.StoreRetention) if cutLine == 0 { - logger.Debug("skipping pruning, the chain is younger than the retain window", + logger.Info("skipping pruning, the chain is younger than the retain window", "globalLatestBlock", globalLatestBlock) return nil } - // Every store must keep back to its own answer, so the system may only prune below the lowest of them. Pruning - // below a higher answer would delete blocks that a lower-answering store just reported it still needs, which is - // exactly how deletion would break the rollback invariant. + // Every participating store must keep back to its own answer, so the system may only prune below the lowest of + // them. Pruning below a higher answer would delete blocks that a lower-answering store just reported it still + // needs, which is exactly how deletion would break the rollback invariant. // - // A store that answers 0 is treated as unknown rather than empty: a snapshot may be mid-write and take hours to - // land. Skipping that store and pruning everyone else would delete the contiguous blocks the in-flight snapshot - // will need once it appears, so the whole cycle is skipped until every store reports something to retain. + // A store that answers 0 is left out of both the prune-height vote and PruneBelow: it may be uninitialized or + // intentionally retaining forever. Snapshotted stores that have not finished a first snapshot yet still vote with + // their last committed height (see GetOldestBlockToRetain). var pruneHeight uint64 oldestBlockToRetainByStore := make(map[string]uint64, len(s.stores)) for _, store := range s.stores { oldestBlockToRetain := store.GetOldestBlockToRetain(cutLine) oldestBlockToRetainByStore[store.Name()] = oldestBlockToRetain if oldestBlockToRetain == 0 { - logger.Debug("skipping pruning, a store holds no data to retain", - "cutLine", cutLine, "oldestBlockToRetainByStore", oldestBlockToRetainByStore) - return nil + continue } if pruneHeight == 0 || oldestBlockToRetain < pruneHeight { pruneHeight = oldestBlockToRetain } } + if pruneHeight == 0 { + logger.Info("skipping pruning, no store has data to retain", + "cutLine", cutLine, "oldestBlockToRetainByStore", oldestBlockToRetainByStore) + return nil + } logger.Info("pruning storage", "globalLatestBlock", globalLatestBlock, @@ -147,6 +150,9 @@ func (s *StorageGarbageCollector) prune() error { "oldestBlockToRetainByStore", oldestBlockToRetainByStore, ) for _, store := range s.stores { + if oldestBlockToRetainByStore[store.Name()] == 0 { + continue + } if err := store.PruneBelow(pruneHeight); err != nil { return fmt.Errorf("failed to prune %s below %d: %w", store.Name(), pruneHeight, err) } diff --git a/sei-db/management/gc/storage_garbage_collector_config.go b/sei-db/management/gc/storage_garbage_collector_config.go index 328433b48b..0a912ab5f7 100644 --- a/sei-db/management/gc/storage_garbage_collector_config.go +++ b/sei-db/management/gc/storage_garbage_collector_config.go @@ -2,7 +2,6 @@ package gc import ( "fmt" - "math" "time" ) @@ -58,8 +57,8 @@ func (c *StorageGarbageCollectorConfig) Validate() error { // The cut line is derived by subtracting RollbackWindow + StoreRetention from the head of the chain. Reject a pair // that overflows when summed, so that subtraction cannot wrap around to a cut line above the head. if c.RollbackWindow+c.StoreRetention < c.RollbackWindow { - return fmt.Errorf("rollback window (%d) plus store retention (%d) must be at most %d", - c.RollbackWindow, c.StoreRetention, uint64(math.MaxUint64)) + return fmt.Errorf("rollback window (%d) plus store retention (%d) overflows uint64", + c.RollbackWindow, c.StoreRetention) } return nil } diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go index f56b4cd189..068841dbcb 100644 --- a/sei-db/management/gc/storage_garbage_collector_test.go +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -27,14 +27,14 @@ type mockStore struct { // snapshotStore models a store that can only be restored to a height it holds a snapshot for, such as SC or SS: it // answers with the newest snapshot at or below the cut line, or with its oldest snapshot when every snapshot is above the -// cut line. Snapshots must be in ascending order; a store given none holds no data. +// cut line. Snapshots must be in ascending order. With no completed snapshot yet it answers its last committed height. func snapshotStore(name string, latestHeight uint64, snapshots ...uint64) *mockStore { return &mockStore{ name: name, latestHeight: latestHeight, oldestToRetain: func(cutLine uint64) uint64 { if len(snapshots) == 0 { - return 0 + return latestHeight } oldest := snapshots[0] for _, snapshot := range snapshots { @@ -197,16 +197,40 @@ func TestPruneDecisions(t *testing.T) { wantPruneBelow: ptr(30_000), }, { - name: "a store holding nothing stalls the whole cycle", + name: "a store answering 0 is ignored", rollbackWindow: 10_000, stores: []*mockStore{ - snapshotStore("empty", 100_000), + contiguousStore("optOut", 100_000, false), snapshotStore("sc", 100_000, 80_000), contiguousStore("stateWAL", 100_000, true), }, - // empty may be mid-write of a first snapshot below the cut line; pruning the WAL to 80,000 would leave - // that snapshot unreplayable once it lands. - wantPruneBelow: nil, + // optOut answers 0 (e.g. infinite retention) and is left out of the vote and of PruneBelow. + wantPruneBelow: ptr(80_000), + }, + { + name: "no snapshots yet votes the last committed height", + rollbackWindow: 10_000, + stores: []*mockStore{ + snapshotStore("sc", 100_000), + contiguousStore("stateWAL", 100_000, true), + }, + // sc has no completed snapshot, so it answers its committed height (100,000) and still participates — + // unlike answering 0, which would leave it out of PruneBelow. The WAL's cut line of 90,000 is lower. + wantPruneBelow: ptr(90_000), + }, + { + name: "an in-progress first snapshot binds below the cut line", + rollbackWindow: 10_000, + stores: []*mockStore{ + { + name: "sc", + latestHeight: 100_000, + // First snapshot is being written at 50,000 while the tip has moved on. + oldestToRetain: func(uint64) uint64 { return 50_000 }, + }, + contiguousStore("stateWAL", 100_000, true), + }, + wantPruneBelow: ptr(50_000), }, { name: "a head of zero is ignored, the data behind it is not", @@ -248,13 +272,12 @@ func TestPruneDecisions(t *testing.T) { wantPruneBelow: nil, }, { - name: "stores are committing blocks but none retains any", + name: "only an opt-out store has data", rollbackWindow: 10_000, stores: []*mockStore{ - snapshotStore("sc", 100_000), - contiguousStore("stateWAL", 100_000, false), + contiguousStore("optOut", 100_000, false), }, - // A fresh node whose stores have ingested blocks but have not yet written a first snapshot. + // Every store answered 0, so there is no prune height. wantPruneBelow: nil, }, { @@ -270,8 +293,12 @@ func TestPruneDecisions(t *testing.T) { collector := newTestCollector(t, tc.rollbackWindow, tc.storeRetention, tc.stores...) require.NoError(t, collector.prune()) + head, err := getGlobalLastCommittedBlock(prunableStores(tc.stores...)) + require.NoError(t, err) + cutLine := getCutLine(head, tc.rollbackWindow, tc.storeRetention) + for _, store := range tc.stores { - if tc.wantPruneBelow == nil { + if tc.wantPruneBelow == nil || store.GetOldestBlockToRetain(cutLine) == 0 { require.Falsef(t, store.pruneBelowCalled.Load(), "%s should not be pruned", store.name) continue } @@ -420,7 +447,7 @@ func TestValidate(t *testing.T) { RollbackWindow: math.MaxUint64, StoreRetention: 1, PruneInterval: time.Minute, - }).Validate(), "store retention") + }).Validate(), "overflows uint64") } func TestNewStorageGarbageCollectorInvalidConfig(t *testing.T) { From 9d8f9be29b01edc6fb51dbbc51c3be9659573882 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Thu, 30 Jul 2026 22:15:55 -0700 Subject: [PATCH 08/15] Address comments --- sei-db/management/gc/api.go | 21 +++++---- .../gc/storage_garbage_collector.go | 43 +++++++++++++------ .../gc/storage_garbage_collector_test.go | 39 +++++++++-------- 3 files changed, 59 insertions(+), 44 deletions(-) diff --git a/sei-db/management/gc/api.go b/sei-db/management/gc/api.go index ae9bf47227..3845be5153 100644 --- a/sei-db/management/gc/api.go +++ b/sei-db/management/gc/api.go @@ -10,22 +10,21 @@ type PrunableStore interface { PruneBelow(blockNumber uint64) error // GetOldestBlockToRetain returns the oldest block this store must keep in order to remain able to serve cutLine, - // or 0 when the store opts out of pruning entirely (e.g. infinite retention or truly uninitialized). A chain's - // first block is block 1, so 0 is free to serve as that sentinel. + // or 0 when the store opts out of pruning entirely (e.g. infinite retention or disabled). // // The collector ignores a store that answers 0 when choosing the prune height and does not call PruneBelow on it, - // so other stores can still be pruned. + // so that other stores can still be pruned. // // A store that retains a contiguous range of blocks (blockDB, receiptDB, the state WAL) can be restored to any - // block it holds, so it needs nothing below the cut line and returns cutLine unchanged. - // A store with checkpoints can only be restored to a height it has a snapshot for, so it returns the newest - // snapshot at or below cutLine, or its oldest snapshot when every snapshot is above the cut line, since in that - // case none of them can be dropped. If it has not completed any snapshot yet, it returns its last committed - // height — including an in-progress snapshot height — so contiguous stores are not pruned past blocks that - // snapshot will need once it lands. + // block it holds, so it should simply return the cutLine unchanged. // - // The collector prunes every participating store below the lowest non-zero answer it receives, so an answer that is - // too low only costs temporary disk, while one that is too high deletes blocks the system still needs. + // A store with checkpoints can only be restored to a height it has a snapshot for, so it answers the newest + // completed snapshot at or below cutLine, or its oldest completed snapshot when every snapshot is above the cut + // line, since in that case none of them can be dropped. With no completed snapshot at all it answers 0. + // + // Snapshot creation is assumed to complete within seconds, so a snapshot write in flight is not modeled: a store + // need not reserve blocks for a snapshot that has not landed yet. A store whose snapshots take long enough for the + // cut line to advance past them would need to report the in-progress height instead, and that is out of scope here. GetOldestBlockToRetain(cutLine uint64) uint64 // GetLastCommittedBlock returns the highest block this store has ingested. diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go index e78e27d186..a4177cb15b 100644 --- a/sei-db/management/gc/storage_garbage_collector.go +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -3,6 +3,7 @@ package gc import ( "context" "fmt" + "strings" "sync" "time" @@ -122,24 +123,25 @@ func (s *StorageGarbageCollector) prune() error { // them. Pruning below a higher answer would delete blocks that a lower-answering store just reported it still // needs, which is exactly how deletion would break the rollback invariant. // - // A store that answers 0 is left out of both the prune-height vote and PruneBelow: it may be uninitialized or - // intentionally retaining forever. Snapshotted stores that have not finished a first snapshot yet still vote with - // their last committed height (see GetOldestBlockToRetain). + // A store that answers 0 is left out of both the prune-height vote and PruneBelow: it holds no snapshot yet, is + // disabled, or is intentionally retaining forever (see GetOldestBlockToRetain). + // + // The answers are kept positionally rather than keyed by Name so that two stores sharing a name cannot make the + // collector prune one that opted out. Names are for logs only. var pruneHeight uint64 - oldestBlockToRetainByStore := make(map[string]uint64, len(s.stores)) - for _, store := range s.stores { - oldestBlockToRetain := store.GetOldestBlockToRetain(cutLine) - oldestBlockToRetainByStore[store.Name()] = oldestBlockToRetain - if oldestBlockToRetain == 0 { + oldestBlockToRetain := make([]uint64, len(s.stores)) + for i, store := range s.stores { + oldestBlockToRetain[i] = store.GetOldestBlockToRetain(cutLine) + if oldestBlockToRetain[i] == 0 { continue } - if pruneHeight == 0 || oldestBlockToRetain < pruneHeight { - pruneHeight = oldestBlockToRetain + if pruneHeight == 0 || oldestBlockToRetain[i] < pruneHeight { + pruneHeight = oldestBlockToRetain[i] } } if pruneHeight == 0 { logger.Info("skipping pruning, no store has data to retain", - "cutLine", cutLine, "oldestBlockToRetainByStore", oldestBlockToRetainByStore) + "cutLine", cutLine, "oldestBlockToRetainByStore", s.describeAnswers(oldestBlockToRetain)) return nil } @@ -147,10 +149,10 @@ func (s *StorageGarbageCollector) prune() error { "globalLatestBlock", globalLatestBlock, "cutLine", cutLine, "pruneHeight", pruneHeight, - "oldestBlockToRetainByStore", oldestBlockToRetainByStore, + "oldestBlockToRetainByStore", s.describeAnswers(oldestBlockToRetain), ) - for _, store := range s.stores { - if oldestBlockToRetainByStore[store.Name()] == 0 { + for i, store := range s.stores { + if oldestBlockToRetain[i] == 0 { continue } if err := store.PruneBelow(pruneHeight); err != nil { @@ -160,6 +162,19 @@ func (s *StorageGarbageCollector) prune() error { return nil } +// describeAnswers renders the per-store retain answers for a log line, in store order so duplicate names stay +// distinguishable. +func (s *StorageGarbageCollector) describeAnswers(oldestBlockToRetain []uint64) string { + var sb strings.Builder + for i, store := range s.stores { + if i > 0 { + sb.WriteString(" ") + } + fmt.Fprintf(&sb, "%s=%d", store.Name(), oldestBlockToRetain[i]) + } + return sb.String() +} + // getCutLine returns the oldest block the system must remain able to serve, which is the head of the chain less the // rollback window and the extra retention. // diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go index 068841dbcb..94de8bfbdf 100644 --- a/sei-db/management/gc/storage_garbage_collector_test.go +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -27,14 +27,14 @@ type mockStore struct { // snapshotStore models a store that can only be restored to a height it holds a snapshot for, such as SC or SS: it // answers with the newest snapshot at or below the cut line, or with its oldest snapshot when every snapshot is above the -// cut line. Snapshots must be in ascending order. With no completed snapshot yet it answers its last committed height. +// cut line. Snapshots must be in ascending order; a store given none has no completed snapshot and answers 0. func snapshotStore(name string, latestHeight uint64, snapshots ...uint64) *mockStore { return &mockStore{ name: name, latestHeight: latestHeight, oldestToRetain: func(cutLine uint64) uint64 { if len(snapshots) == 0 { - return latestHeight + return 0 } oldest := snapshots[0] for _, snapshot := range snapshots { @@ -208,30 +208,16 @@ func TestPruneDecisions(t *testing.T) { wantPruneBelow: ptr(80_000), }, { - name: "no snapshots yet votes the last committed height", + name: "a store with no snapshot yet is ignored", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000), contiguousStore("stateWAL", 100_000, true), }, - // sc has no completed snapshot, so it answers its committed height (100,000) and still participates — - // unlike answering 0, which would leave it out of PruneBelow. The WAL's cut line of 90,000 is lower. + // sc has no completed snapshot, so it answers 0 and stays out of the vote and of PruneBelow. Snapshot + // creation is assumed to finish within seconds, so no blocks are reserved for a write in flight. wantPruneBelow: ptr(90_000), }, - { - name: "an in-progress first snapshot binds below the cut line", - rollbackWindow: 10_000, - stores: []*mockStore{ - { - name: "sc", - latestHeight: 100_000, - // First snapshot is being written at 50,000 while the tip has moved on. - oldestToRetain: func(uint64) uint64 { return 50_000 }, - }, - contiguousStore("stateWAL", 100_000, true), - }, - wantPruneBelow: ptr(50_000), - }, { name: "a head of zero is ignored, the data behind it is not", rollbackWindow: 10_000, @@ -326,6 +312,21 @@ func TestPruneOnYoungChainWithDefaultConfig(t *testing.T) { require.False(t, wal.pruneBelowCalled.Load()) } +// TestPruneKeepsOptedOutStoreWithDuplicateName checks that answers are tracked positionally: two stores sharing a name +// must not make the collector prune the one that opted out with 0. +func TestPruneKeepsOptedOutStoreWithDuplicateName(t *testing.T) { + optOut := contiguousStore("ss", 100_000, false) + participating := snapshotStore("ss", 100_000, 80_000) + wal := contiguousStore("stateWAL", 100_000, true) + + require.NoError(t, newTestCollector(t, 10_000, 0, optOut, participating, wal).prune()) + + require.False(t, optOut.pruneBelowCalled.Load(), "a store answering 0 must never be pruned") + require.True(t, participating.pruneBelowCalled.Load()) + require.Equal(t, uint64(80_000), participating.prunedBelow.Load()) + require.Equal(t, uint64(80_000), wal.prunedBelow.Load()) +} + // TestPruneGetLastCommittedBlockError checks that a failure to read a head aborts the cycle before anything is deleted. func TestPruneGetLastCommittedBlockError(t *testing.T) { sentinel := errors.New("boom") From f623054858dfae9d57c1ce5b4f0c7095f2bedcc6 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Fri, 31 Jul 2026 11:59:36 -0700 Subject: [PATCH 09/15] Address comments --- sei-db/config/giga_config.go | 16 ++- sei-db/config/giga_config_test.go | 34 +++++ .../gc/storage_garbage_collector.go | 21 +-- .../gc/storage_garbage_collector_config.go | 58 ++++++--- .../gc/storage_garbage_collector_test.go | 122 +++++++++++++----- 5 files changed, 186 insertions(+), 65 deletions(-) create mode 100644 sei-db/config/giga_config_test.go diff --git a/sei-db/config/giga_config.go b/sei-db/config/giga_config.go index f8c0aa01b1..04b85ef144 100644 --- a/sei-db/config/giga_config.go +++ b/sei-db/config/giga_config.go @@ -14,7 +14,7 @@ import ( // callers build it via DefaultGigaStorageConfig. Nested knobs live on the store and // collector configs they already own (StateStoreConfig, ReceiptStoreConfig, // gc.StorageGarbageCollectorConfig) rather than being redeclared here — in particular -// RollbackWindow and StoreRetention have a single source of truth in +// RollbackWindow and RetentionBeyondRollbackWindow have a single source of truth in // gc.DefaultStorageGarbageCollectorConfig. type GigaStorageConfig struct { HomePath string @@ -22,20 +22,22 @@ type GigaStorageConfig struct { SSConfig StateStoreConfig ReceiptDBConfig ReceiptStoreConfig BlockDBConfig *littblock.LittBlockConfig - // PruningConfig is the garbage-collector config shared by every store under this - // Giga layout. Defaults come from gc.DefaultStorageGarbageCollectorConfig; do not - // reintroduce parallel RollbackWindow / StoreRetention fields on this struct. - PruningConfig *gc.StorageGarbageCollectorConfig + PruningConfig *gc.StorageGarbageCollectorConfig } // DefaultGigaStorageConfig returns a GigaStorageConfig whose store directories match the // layout below, and whose pruning knobs are gc.DefaultStorageGarbageCollectorConfig(): // // data/state_commit/flatkv -// data/state_store/evm/{backend} +// data/state_store/evm/{backend} (sole SS; no Cosmos SS in Giga) // data/ledger/receipt/{backend} // data/ledger/block // +// Giga opens SS via evm.NewEVMStateStore(dir, ssConfig) directly — not through +// composite.NewCompositeStateStore — so the EVM path lives on EVMDBDirectory (the +// argument callers pass as dir). DBDirectory and EVMSplit are left at their defaults: +// they only matter for the composite path, which Giga does not use. +// // Unlike the other Default*Config helpers in this package, this can fail: the block-store // default wraps littdb.DefaultConfig, which validates the directory path. func DefaultGigaStorageConfig(homePath string) (GigaStorageConfig, error) { @@ -48,7 +50,7 @@ func DefaultGigaStorageConfig(homePath string) (GigaStorageConfig, error) { flatKV.DataDir = utils.GetFlatKVPath(homePath) ssConfig := DefaultStateStoreConfig() - ssConfig.DBDirectory = utils.GetEVMStateStorePath(homePath, ssConfig.Backend) + ssConfig.EVMDBDirectory = utils.GetEVMStateStorePath(homePath, ssConfig.Backend) receiptConfig := DefaultReceiptStoreConfig() receiptConfig.DBDirectory = utils.GetReceiptStorePath(homePath, receiptConfig.Backend) diff --git a/sei-db/config/giga_config_test.go b/sei-db/config/giga_config_test.go new file mode 100644 index 0000000000..e590c95762 --- /dev/null +++ b/sei-db/config/giga_config_test.go @@ -0,0 +1,34 @@ +package config + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/common/utils" +) + +// TestDefaultGigaStorageConfigDirectories pins Giga's EVM-only SS layout: the path +// for evm.NewEVMStateStore lives on EVMDBDirectory. Giga does not use composite, so +// DBDirectory stays empty and EVMSplit stays at its default. +func TestDefaultGigaStorageConfigDirectories(t *testing.T) { + home := t.TempDir() + cfg, err := DefaultGigaStorageConfig(home) + require.NoError(t, err) + + require.Equal(t, home, cfg.HomePath) + require.False(t, cfg.SSConfig.EVMSplit) + require.Empty(t, cfg.SSConfig.DBDirectory) + require.Equal(t, utils.GetEVMStateStorePath(home, cfg.SSConfig.Backend), cfg.SSConfig.EVMDBDirectory) + + require.Equal(t, utils.GetFlatKVPath(home), cfg.FlatKVConfig.DataDir) + require.Equal(t, utils.GetReceiptStorePath(home, cfg.ReceiptDBConfig.Backend), cfg.ReceiptDBConfig.DBDirectory) + require.NotNil(t, cfg.BlockDBConfig.Litt) + require.Equal(t, []string{utils.GetBlockStorePath(home)}, cfg.BlockDBConfig.Litt.Paths) + + require.Equal(t, + filepath.Join(home, "data", "state_store", "evm", cfg.SSConfig.Backend), + cfg.SSConfig.EVMDBDirectory, + ) +} diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go index a4177cb15b..5e51353154 100644 --- a/sei-db/management/gc/storage_garbage_collector.go +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -112,7 +112,7 @@ func (s *StorageGarbageCollector) prune() error { return nil } - cutLine := getCutLine(globalLatestBlock, s.config.RollbackWindow, s.config.StoreRetention) + cutLine := getCutLine(globalLatestBlock, s.config.RollbackWindow, s.config.RetentionBeyondRollbackWindow) if cutLine == 0 { logger.Info("skipping pruning, the chain is younger than the retain window", "globalLatestBlock", globalLatestBlock) @@ -176,15 +176,20 @@ func (s *StorageGarbageCollector) describeAnswers(oldestBlockToRetain []uint64) } // getCutLine returns the oldest block the system must remain able to serve, which is the head of the chain less the -// rollback window and the extra retention. +// rollback window and the historical blocks guaranteed beyond it. // -// Returns 0 to mean "prune nothing", which is the case for a chain younger than that combined window. The comparison has -// to happen before the subtraction: these are unsigned, so subtracting past zero wraps to a cut line far above the head -// of the chain, and pruning to it would delete everything. +// Returns 0 to mean "prune nothing": either retentionBeyond is negative (infinite retention), or the chain is younger +// than the combined finite window. The young-chain comparison has to happen before the subtraction: these are +// unsigned, so subtracting past zero wraps to a cut line far above the head of the chain, and pruning to it would +// delete everything. // -// Config.Validate rejects a rollback window and retention that overflow when summed, so retainWindow below is exact. -func getCutLine(globalLatestBlock uint64, rollbackWindow uint64, retention uint64) uint64 { - retainWindow := rollbackWindow + retention +// Config.Validate rejects a finite rollback window and retention that overflow when summed, so retainWindow below is +// exact whenever retentionBeyond is non-negative. +func getCutLine(globalLatestBlock uint64, rollbackWindow uint64, retentionBeyond int64) uint64 { + if retentionBeyond < 0 { + return 0 + } + retainWindow := rollbackWindow + uint64(retentionBeyond) if globalLatestBlock <= retainWindow { return 0 } diff --git a/sei-db/management/gc/storage_garbage_collector_config.go b/sei-db/management/gc/storage_garbage_collector_config.go index 0a912ab5f7..62ea183ad6 100644 --- a/sei-db/management/gc/storage_garbage_collector_config.go +++ b/sei-db/management/gc/storage_garbage_collector_config.go @@ -5,23 +5,40 @@ import ( "time" ) +// InfiniteRetentionBeyondRollbackWindow is the conventional RetentionBeyondRollbackWindow +// value for never pruning. Any negative value is treated the same: the collector derives a +// cut line of 0 and skips every prune cycle. +// +// Unlike KeepRecent / MinRetainBlocks elsewhere in this repo (where 0 means keep forever), +// 0 here means "no historical blocks beyond the rollback window" — so infinite needs a +// negative sentinel. +const InfiniteRetentionBeyondRollbackWindow int64 = -1 + // Configures a StorageGarbageCollector. type StorageGarbageCollectorConfig struct { // The maximum number of blocks the system should be able to roll back at any point in time. // Storage garbage collector ensures that enough data is kept on disk such that the system can always - // roll back this many blocks. + // roll back this many blocks. Must be greater than 0: a zero window would let cutLine reach the + // chain head, and a store with no completed snapshot answering 0 would then leave contiguous + // stores free to drop everything below head — a snapshot landing at ~head could no longer be + // replayed forward. // // Note that the "always able to rollback" invariant may be broken after a rollback. For example, if we normally // require a rollback window of 10,000 blocks and then we rollback 5,000 blocks, we can then only rollback an // additional 5,000 blocks after that first rollback. RollbackWindow uint64 - // Additional blocks to retain beyond the rollback window, applied to every store. + // How many historical blocks beyond the rollback window the collector guarantees to keep + // servable, applied to every store. RollbackWindow is what correctness requires for rollback; + // this field is the additional history operators want for queries against old blocks and + // receipts. When non-negative, the cut line is + // head - (RollbackWindow + RetentionBeyondRollbackWindow). // - // This is retention the operator wants rather than retention correctness requires: RollbackWindow is what the - // rollback invariant depends on, while StoreRetention buys history that is served long after it is needed for - // rollback, such as queries against old blocks and receipts. The two are added together to form the cut line. + // Zero means guarantee only the rollback window (no extra history). Any negative value means + // infinite retention: never prune. Prefer InfiniteRetentionBeyondRollbackWindow (-1) as the + // conventional sentinel. This differs from KeepRecent / MinRetainBlocks, where 0 means keep + // forever. // // TODO: make this per-store if the shared value proves too blunt. Because every store is pruned to the lowest // block any single store still needs, retention bought for one store is paid for by all of them: raising this to @@ -30,7 +47,7 @@ type StorageGarbageCollectorConfig struct { // (e.g. a []ManagedStore{Store, RetentionBlocks} in place of []PrunableStore) and computing one cut line per // store. Note that this only reduces retention where a store's own answer is what bounds the minimum; the // contiguous stores must still cover the oldest snapshot anyone kept, or the snapshot becomes unreplayable. - StoreRetention uint64 + RetentionBeyondRollbackWindow int64 // How often the collector drives a prune cycle. PruneInterval time.Duration @@ -39,9 +56,9 @@ type StorageGarbageCollectorConfig struct { // Construct a default storage garbage collector config. func DefaultStorageGarbageCollectorConfig() *StorageGarbageCollectorConfig { return &StorageGarbageCollectorConfig{ - RollbackWindow: 1000, - StoreRetention: 100_000, - PruneInterval: 10 * time.Minute, + RollbackWindow: 1000, + RetentionBeyondRollbackWindow: 100_000, + PruneInterval: 10 * time.Minute, } } @@ -50,15 +67,26 @@ func (c *StorageGarbageCollectorConfig) Validate() error { if c == nil { return fmt.Errorf("config is required") } - // A zero rollback window is legal: it means the system prunes as aggressively as possible. + // RollbackWindow must leave at least one block of margin below head. With RollbackWindow == 0 and + // RetentionBeyondRollbackWindow == 0, cutLine == globalLatestBlock: contiguous stores may drop + // everything below head, and a store that answers 0 because it has no completed snapshot yet offers + // no counter-vote — a snapshot that then lands at ~head cannot be replayed forward. Infinite + // retention is any negative RetentionBeyondRollbackWindow, not a zero rollback window. + if c.RollbackWindow == 0 { + return fmt.Errorf("rollback window must be greater than 0") + } if c.PruneInterval <= 0 { return fmt.Errorf("prune interval must be greater than 0") } - // The cut line is derived by subtracting RollbackWindow + StoreRetention from the head of the chain. Reject a pair - // that overflows when summed, so that subtraction cannot wrap around to a cut line above the head. - if c.RollbackWindow+c.StoreRetention < c.RollbackWindow { - return fmt.Errorf("rollback window (%d) plus store retention (%d) overflows uint64", - c.RollbackWindow, c.StoreRetention) + // The cut line subtracts RollbackWindow + RetentionBeyondRollbackWindow from the head. Reject a + // finite pair that overflows when summed, so that subtraction cannot wrap around to a cut line + // above the head. Negatives mean infinite retention and skip this sum. + if c.RetentionBeyondRollbackWindow >= 0 { + retention := uint64(c.RetentionBeyondRollbackWindow) + if c.RollbackWindow+retention < c.RollbackWindow { + return fmt.Errorf("rollback window (%d) plus retention beyond rollback window (%d) overflows uint64", + c.RollbackWindow, c.RetentionBeyondRollbackWindow) + } } return nil } diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go index 94de8bfbdf..2491130f34 100644 --- a/sei-db/management/gc/storage_garbage_collector_test.go +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -94,15 +94,15 @@ func prunableStores(list ...*mockStore) []PrunableStore { func newTestCollector( t *testing.T, rollbackWindow uint64, - storeRetention uint64, + retentionBeyond int64, stores ...*mockStore, ) *StorageGarbageCollector { t.Helper() config := &StorageGarbageCollectorConfig{ - RollbackWindow: rollbackWindow, - StoreRetention: storeRetention, - PruneInterval: time.Minute, + RollbackWindow: rollbackWindow, + RetentionBeyondRollbackWindow: retentionBeyond, + PruneInterval: time.Minute, } // getCutLine subtracts without an overflow guard because Validate rejects a retain window that would overflow, so // test configs have to clear the same bar as real ones. @@ -114,11 +114,11 @@ func newTestCollector( // TestPruneDecisions is the decision matrix for a single prune cycle. wantPruneBelow == nil means no store may be pruned. func TestPruneDecisions(t *testing.T) { cases := []struct { - name string - rollbackWindow uint64 - storeRetention uint64 - stores []*mockStore - wantPruneBelow *uint64 + name string + rollbackWindow uint64 + retentionBeyond int64 + stores []*mockStore + wantPruneBelow *uint64 }{ { name: "one snapshotted store and a WAL", @@ -145,18 +145,19 @@ func TestPruneDecisions(t *testing.T) { wantPruneBelow: ptr(85_000), }, { - name: "zero rollback window prunes up to the head", - rollbackWindow: 0, + name: "minimum rollback window still leaves one block of margin", + rollbackWindow: 1, stores: []*mockStore{ - snapshotStore("sc", 100_000, 80_000, 100_000), + snapshotStore("sc", 100_000, 80_000, 99_999), contiguousStore("stateWAL", 100_000, true), }, - wantPruneBelow: ptr(100_000), + // Cut line 99,999. sc keeps the snapshot at the cut line; contiguous stores prune to that. + wantPruneBelow: ptr(99_999), }, { - name: "store retention deepens the cut line", - rollbackWindow: 10_000, - storeRetention: 5_000, + name: "retention beyond rollback window deepens the cut line", + rollbackWindow: 10_000, + retentionBeyond: 5_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 80_000, 90_000), contiguousStore("stateWAL", 100_000, true), @@ -165,6 +166,16 @@ func TestPruneDecisions(t *testing.T) { // extra retention sc would have needed only 90,000. wantPruneBelow: ptr(80_000), }, + { + name: "infinite retention beyond rollback window skips pruning", + rollbackWindow: 10_000, + retentionBeyond: InfiniteRetentionBeyondRollbackWindow, + stores: []*mockStore{ + snapshotStore("sc", 100_000, 80_000, 100_000), + contiguousStore("stateWAL", 100_000, true), + }, + wantPruneBelow: nil, + }, { name: "a snapshot exactly at the cut line is kept", rollbackWindow: 10_000, @@ -239,9 +250,9 @@ func TestPruneDecisions(t *testing.T) { wantPruneBelow: nil, }, { - name: "head exactly at the retain window", - rollbackWindow: 60_000, - storeRetention: 40_000, + name: "head exactly at the retain window", + rollbackWindow: 60_000, + retentionBeyond: 40_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 500, 1_000), contiguousStore("stateWAL", 100_000, true), @@ -276,12 +287,12 @@ func TestPruneDecisions(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - collector := newTestCollector(t, tc.rollbackWindow, tc.storeRetention, tc.stores...) + collector := newTestCollector(t, tc.rollbackWindow, tc.retentionBeyond, tc.stores...) require.NoError(t, collector.prune()) head, err := getGlobalLastCommittedBlock(prunableStores(tc.stores...)) require.NoError(t, err) - cutLine := getCutLine(head, tc.rollbackWindow, tc.storeRetention) + cutLine := getCutLine(head, tc.rollbackWindow, tc.retentionBeyond) for _, store := range tc.stores { if tc.wantPruneBelow == nil || store.GetOldestBlockToRetain(cutLine) == 0 { @@ -364,13 +375,15 @@ func TestGetCutLine(t *testing.T) { name string head uint64 rollbackWindow uint64 - retention uint64 + retention int64 want uint64 }{ {name: "rollback window only", head: 100_000, rollbackWindow: 10_000, want: 90_000}, - {name: "retention only", head: 100_000, retention: 10_000, want: 90_000}, + {name: "retention adds to rollback window", head: 100_000, rollbackWindow: 1, retention: 10_000, want: 89_999}, {name: "the two windows add", head: 100_000, rollbackWindow: 10_000, retention: 5_000, want: 85_000}, - {name: "no retain window at all", head: 100_000, want: 100_000}, + {name: "zero retention beyond rollback window", head: 100_000, rollbackWindow: 10_000, want: 90_000}, + {name: "infinite retention beyond rollback window", head: 100_000, rollbackWindow: 10_000, retention: InfiniteRetentionBeyondRollbackWindow, want: 0}, + {name: "any negative retention is infinite", head: 100_000, rollbackWindow: 10_000, retention: -99, want: 0}, {name: "head one above the window", head: 10_001, rollbackWindow: 10_000, want: 1}, {name: "head exactly at the window", head: 10_000, rollbackWindow: 10_000, want: 0}, {name: "head one below the window", head: 9_999, rollbackWindow: 10_000, want: 0}, @@ -425,7 +438,7 @@ func TestGetGlobalLastCommittedBlock(t *testing.T) { func TestDefaultStorageGarbageCollectorConfig(t *testing.T) { cfg := DefaultStorageGarbageCollectorConfig() require.Equal(t, uint64(1_000), cfg.RollbackWindow) - require.Equal(t, uint64(100_000), cfg.StoreRetention) + require.Equal(t, int64(100_000), cfg.RetentionBeyondRollbackWindow) require.Equal(t, 10*time.Minute, cfg.PruneInterval) require.NoError(t, cfg.Validate()) } @@ -433,21 +446,60 @@ func TestDefaultStorageGarbageCollectorConfig(t *testing.T) { func TestValidate(t *testing.T) { require.ErrorContains(t, (*StorageGarbageCollectorConfig)(nil).Validate(), "config is required") - // A zero rollback window is legal, as is a zero store retention. - require.NoError(t, (&StorageGarbageCollectorConfig{PruneInterval: time.Minute}).Validate()) + require.ErrorContains(t, (&StorageGarbageCollectorConfig{ + RollbackWindow: 0, + PruneInterval: time.Minute, + }).Validate(), "rollback window must be greater than 0") - require.ErrorContains(t, (&StorageGarbageCollectorConfig{PruneInterval: 0}).Validate(), "prune interval") - require.ErrorContains(t, (&StorageGarbageCollectorConfig{PruneInterval: -time.Second}).Validate(), "prune interval") + require.NoError(t, (&StorageGarbageCollectorConfig{ + RollbackWindow: 1, + RetentionBeyondRollbackWindow: 0, + PruneInterval: time.Minute, + }).Validate()) - // The retain window is the sum of the two, so the pair must not overflow. getCutLine subtracts it without a guard. require.NoError(t, (&StorageGarbageCollectorConfig{ - RollbackWindow: math.MaxUint64, - PruneInterval: time.Minute, + RollbackWindow: 1, + RetentionBeyondRollbackWindow: InfiniteRetentionBeyondRollbackWindow, + PruneInterval: time.Minute, + }).Validate()) + + // Any negative value means infinite retention. + require.NoError(t, (&StorageGarbageCollectorConfig{ + RollbackWindow: 1, + RetentionBeyondRollbackWindow: -2, + PruneInterval: time.Minute, }).Validate()) + require.ErrorContains(t, (&StorageGarbageCollectorConfig{ - RollbackWindow: math.MaxUint64, - StoreRetention: 1, - PruneInterval: time.Minute, + RollbackWindow: 1, + PruneInterval: 0, + }).Validate(), "prune interval") + require.ErrorContains(t, (&StorageGarbageCollectorConfig{ + RollbackWindow: 1, + PruneInterval: -time.Second, + }).Validate(), "prune interval") + + // The retain window is the sum of the two finite values, so the pair must not overflow. getCutLine + // subtracts it without a guard. Infinite retention (any negative) skips that sum. + require.NoError(t, (&StorageGarbageCollectorConfig{ + RollbackWindow: math.MaxUint64, + RetentionBeyondRollbackWindow: 0, + PruneInterval: time.Minute, + }).Validate()) + require.NoError(t, (&StorageGarbageCollectorConfig{ + RollbackWindow: math.MaxUint64, + RetentionBeyondRollbackWindow: InfiniteRetentionBeyondRollbackWindow, + PruneInterval: time.Minute, + }).Validate()) + require.NoError(t, (&StorageGarbageCollectorConfig{ + RollbackWindow: math.MaxUint64, + RetentionBeyondRollbackWindow: -99, + PruneInterval: time.Minute, + }).Validate()) + require.ErrorContains(t, (&StorageGarbageCollectorConfig{ + RollbackWindow: math.MaxUint64, + RetentionBeyondRollbackWindow: 1, + PruneInterval: time.Minute, }).Validate(), "overflows uint64") } From f386ed4f3b66b4f7577c021e37ce7c2209017674 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Fri, 31 Jul 2026 14:03:52 -0700 Subject: [PATCH 10/15] Move prune to static function --- .../gc/storage_garbage_collector.go | 31 +++--- .../gc/storage_garbage_collector_test.go | 103 ++---------------- 2 files changed, 28 insertions(+), 106 deletions(-) diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go index 5e51353154..7a55955e46 100644 --- a/sei-db/management/gc/storage_garbage_collector.go +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -26,6 +26,9 @@ var logger = seilog.NewLogger("db", "gc") // blocks by itself, it guarantees that the act of deleting old state does not prevent rollback within this // rollback window. // +// The collector itself is only a ticker that periodically calls prune. All decision logic lives in prune so it can be +// unit tested without threads or timing. +// // StorageGarbageCollector is not thread safe. type StorageGarbageCollector struct { @@ -74,9 +77,7 @@ func (s *StorageGarbageCollector) Close() error { return nil } -// run periodically drives prune cycles until the manager is stopped. All decision logic lives in prune so it can -// be unit tested without threading; the ticker path is covered by constructing a collector with a short -// config.PruneInterval. +// run periodically drives prune cycles until the manager is stopped. func (s *StorageGarbageCollector) run() { defer s.wg.Done() @@ -90,7 +91,7 @@ func (s *StorageGarbageCollector) run() { case <-s.ctx.Done(): return case <-ticker.C: - if err := s.prune(); err != nil { + if err := prune(s.config, s.stores); err != nil { logger.Error("prune cycle failed", "err", err) } } @@ -99,11 +100,11 @@ func (s *StorageGarbageCollector) run() { // prune performs a single prune cycle: it locates the head of the chain, derives the cut line from it, asks every store // how far back it must retain data to serve that cut line, and prunes every store below the lowest answer. -func (s *StorageGarbageCollector) prune() error { - if len(s.stores) == 0 { +func prune(config *StorageGarbageCollectorConfig, stores []PrunableStore) error { + if len(stores) == 0 { return nil } - globalLatestBlock, err := getGlobalLastCommittedBlock(s.stores) + globalLatestBlock, err := getGlobalLastCommittedBlock(stores) if err != nil { return err } @@ -112,7 +113,7 @@ func (s *StorageGarbageCollector) prune() error { return nil } - cutLine := getCutLine(globalLatestBlock, s.config.RollbackWindow, s.config.RetentionBeyondRollbackWindow) + cutLine := getCutLine(globalLatestBlock, config.RollbackWindow, config.RetentionBeyondRollbackWindow) if cutLine == 0 { logger.Info("skipping pruning, the chain is younger than the retain window", "globalLatestBlock", globalLatestBlock) @@ -129,8 +130,8 @@ func (s *StorageGarbageCollector) prune() error { // The answers are kept positionally rather than keyed by Name so that two stores sharing a name cannot make the // collector prune one that opted out. Names are for logs only. var pruneHeight uint64 - oldestBlockToRetain := make([]uint64, len(s.stores)) - for i, store := range s.stores { + oldestBlockToRetain := make([]uint64, len(stores)) + for i, store := range stores { oldestBlockToRetain[i] = store.GetOldestBlockToRetain(cutLine) if oldestBlockToRetain[i] == 0 { continue @@ -141,7 +142,7 @@ func (s *StorageGarbageCollector) prune() error { } if pruneHeight == 0 { logger.Info("skipping pruning, no store has data to retain", - "cutLine", cutLine, "oldestBlockToRetainByStore", s.describeAnswers(oldestBlockToRetain)) + "cutLine", cutLine, "oldestBlockToRetainByStore", describeAnswers(stores, oldestBlockToRetain)) return nil } @@ -149,9 +150,9 @@ func (s *StorageGarbageCollector) prune() error { "globalLatestBlock", globalLatestBlock, "cutLine", cutLine, "pruneHeight", pruneHeight, - "oldestBlockToRetainByStore", s.describeAnswers(oldestBlockToRetain), + "oldestBlockToRetainByStore", describeAnswers(stores, oldestBlockToRetain), ) - for i, store := range s.stores { + for i, store := range stores { if oldestBlockToRetain[i] == 0 { continue } @@ -164,9 +165,9 @@ func (s *StorageGarbageCollector) prune() error { // describeAnswers renders the per-store retain answers for a log line, in store order so duplicate names stay // distinguishable. -func (s *StorageGarbageCollector) describeAnswers(oldestBlockToRetain []uint64) string { +func describeAnswers(stores []PrunableStore, oldestBlockToRetain []uint64) string { var sb strings.Builder - for i, store := range s.stores { + for i, store := range stores { if i > 0 { sb.WriteString(" ") } diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go index 2491130f34..7695d2f4c4 100644 --- a/sei-db/management/gc/storage_garbage_collector_test.go +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -20,7 +20,7 @@ type mockStore struct { getErr error pruneErr error - // Atomically observed so the run-loop ticker tests can assert without racing the collector goroutine. + // Observed by tests after a prune cycle. pruneBelowCalled atomic.Bool prunedBelow atomic.Uint64 } @@ -81,7 +81,7 @@ func (m *mockStore) PruneBelow(blockNumber uint64) error { return m.pruneErr } -// prunableStores adapts the mocks to the interface slice the collector holds. +// prunableStores adapts the mocks to the interface slice prune takes. func prunableStores(list ...*mockStore) []PrunableStore { result := make([]PrunableStore, len(list)) for i, store := range list { @@ -90,15 +90,9 @@ func prunableStores(list ...*mockStore) []PrunableStore { return result } -// newTestCollector builds a collector without starting its run loop, so prune can be driven one cycle at a time. -func newTestCollector( - t *testing.T, - rollbackWindow uint64, - retentionBeyond int64, - stores ...*mockStore, -) *StorageGarbageCollector { +// testConfig builds a validated config for calling prune directly. +func testConfig(t *testing.T, rollbackWindow uint64, retentionBeyond int64) *StorageGarbageCollectorConfig { t.Helper() - config := &StorageGarbageCollectorConfig{ RollbackWindow: rollbackWindow, RetentionBeyondRollbackWindow: retentionBeyond, @@ -107,8 +101,7 @@ func newTestCollector( // getCutLine subtracts without an overflow guard because Validate rejects a retain window that would overflow, so // test configs have to clear the same bar as real ones. require.NoError(t, config.Validate()) - - return &StorageGarbageCollector{config: config, stores: prunableStores(stores...)} + return config } // TestPruneDecisions is the decision matrix for a single prune cycle. wantPruneBelow == nil means no store may be pruned. @@ -287,10 +280,10 @@ func TestPruneDecisions(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - collector := newTestCollector(t, tc.rollbackWindow, tc.retentionBeyond, tc.stores...) - require.NoError(t, collector.prune()) + stores := prunableStores(tc.stores...) + require.NoError(t, prune(testConfig(t, tc.rollbackWindow, tc.retentionBeyond), stores)) - head, err := getGlobalLastCommittedBlock(prunableStores(tc.stores...)) + head, err := getGlobalLastCommittedBlock(stores) require.NoError(t, err) cutLine := getCutLine(head, tc.rollbackWindow, tc.retentionBeyond) @@ -313,11 +306,7 @@ func TestPruneOnYoungChainWithDefaultConfig(t *testing.T) { sc := snapshotStore("sc", 100, 50, 100) wal := contiguousStore("stateWAL", 100, true) - collector := &StorageGarbageCollector{ - config: DefaultStorageGarbageCollectorConfig(), - stores: prunableStores(sc, wal), - } - require.NoError(t, collector.prune()) + require.NoError(t, prune(DefaultStorageGarbageCollectorConfig(), prunableStores(sc, wal))) require.False(t, sc.pruneBelowCalled.Load()) require.False(t, wal.pruneBelowCalled.Load()) @@ -330,7 +319,7 @@ func TestPruneKeepsOptedOutStoreWithDuplicateName(t *testing.T) { participating := snapshotStore("ss", 100_000, 80_000) wal := contiguousStore("stateWAL", 100_000, true) - require.NoError(t, newTestCollector(t, 10_000, 0, optOut, participating, wal).prune()) + require.NoError(t, prune(testConfig(t, 10_000, 0), prunableStores(optOut, participating, wal))) require.False(t, optOut.pruneBelowCalled.Load(), "a store answering 0 must never be pruned") require.True(t, participating.pruneBelowCalled.Load()) @@ -345,7 +334,7 @@ func TestPruneGetLastCommittedBlockError(t *testing.T) { broken := contiguousStore("brokenStore", 100_000, true) broken.getErr = sentinel - err := newTestCollector(t, 10_000, 0, sc, broken).prune() + err := prune(testConfig(t, 10_000, 0), prunableStores(sc, broken)) require.ErrorIs(t, err, sentinel) require.ErrorContains(t, err, "brokenStore") require.False(t, sc.pruneBelowCalled.Load()) @@ -361,7 +350,7 @@ func TestPruneBelowErrorStopsRemainingStores(t *testing.T) { second := snapshotStore("second", 100_000, 80_000) wal := contiguousStore("stateWAL", 100_000, true) - err := newTestCollector(t, 10_000, 0, first, second, wal).prune() + err := prune(testConfig(t, 10_000, 0), prunableStores(first, second, wal)) require.ErrorIs(t, err, sentinel) require.ErrorContains(t, err, "first") require.ErrorContains(t, err, "80000") // the prune height, i.e. the lowest need across the stores @@ -542,74 +531,6 @@ func TestCloseAfterContextCancelled(t *testing.T) { require.NoError(t, sm.Close()) } -// startTestCollectorWithInterval builds a running collector with a short PruneInterval so the ticker path can be -// exercised without waiting on the production default. -func startTestCollectorWithInterval( - t *testing.T, - interval time.Duration, - stores []PrunableStore, -) *StorageGarbageCollector { - t.Helper() - - config := &StorageGarbageCollectorConfig{ - RollbackWindow: 10_000, - PruneInterval: interval, - } - require.NoError(t, config.Validate()) - - s := &StorageGarbageCollector{ - config: config, - stores: stores, - ctx: context.Background(), - stopCh: make(chan struct{}), - } - s.wg.Add(1) - go s.run() - t.Cleanup(func() { require.NoError(t, s.Close()) }) - return s -} - -// TestRunFiresPruneCycle checks that the ticker path actually invokes prune. -func TestRunFiresPruneCycle(t *testing.T) { - sc := snapshotStore("sc", 100_000, 80_000) - wal := contiguousStore("stateWAL", 100_000, true) - - _ = startTestCollectorWithInterval(t, 5*time.Millisecond, prunableStores(sc, wal)) - - require.Eventually(t, func() bool { - return sc.pruneBelowCalled.Load() && wal.pruneBelowCalled.Load() - }, time.Second, 5*time.Millisecond) - require.Equal(t, uint64(80_000), sc.prunedBelow.Load()) - require.Equal(t, uint64(80_000), wal.prunedBelow.Load()) -} - -// countingStore counts how many times GetLastCommittedBlock is called so a failing prune cycle can be observed -// without depending on log output. -type countingStore struct { - *mockStore - calls atomic.Uint64 -} - -func (c *countingStore) GetLastCommittedBlock() (uint64, error) { - c.calls.Add(1) - return c.mockStore.GetLastCommittedBlock() -} - -// TestRunContinuesAfterPruneError checks that a failed prune cycle does not kill the run loop: the ticker keeps -// firing and prune is invoked again. -func TestRunContinuesAfterPruneError(t *testing.T) { - broken := &countingStore{ - mockStore: contiguousStore("broken", 100_000, true), - } - broken.getErr = errors.New("boom") - - _ = startTestCollectorWithInterval(t, 5*time.Millisecond, []PrunableStore{broken}) - - require.Eventually(t, func() bool { - return broken.calls.Load() >= 2 - }, time.Second, 5*time.Millisecond) -} - func ptr(v uint64) *uint64 { return &v } From 0c0002203152edc9f5c3a61dec54bb2a3ef22f35 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Fri, 31 Jul 2026 16:07:54 -0700 Subject: [PATCH 11/15] Fix comments and logic for inifinite retention --- sei-db/config/giga_config.go | 5 +- sei-db/management/gc/api.go | 59 ++-- .../gc/storage_garbage_collector.go | 156 +++++------ .../gc/storage_garbage_collector_config.go | 76 +---- .../gc/storage_garbage_collector_test.go | 262 ++++++++---------- 5 files changed, 239 insertions(+), 319 deletions(-) diff --git a/sei-db/config/giga_config.go b/sei-db/config/giga_config.go index 04b85ef144..84044e3689 100644 --- a/sei-db/config/giga_config.go +++ b/sei-db/config/giga_config.go @@ -14,8 +14,9 @@ import ( // callers build it via DefaultGigaStorageConfig. Nested knobs live on the store and // collector configs they already own (StateStoreConfig, ReceiptStoreConfig, // gc.StorageGarbageCollectorConfig) rather than being redeclared here — in particular -// RollbackWindow and RetentionBeyondRollbackWindow have a single source of truth in -// gc.DefaultStorageGarbageCollectorConfig. +// RollbackWindow has a single source of truth in +// gc.DefaultStorageGarbageCollectorConfig; per-store extras live on each +// PrunableStore via GetRetentionWindow. type GigaStorageConfig struct { HomePath string FlatKVConfig *flatkvConfig.Config diff --git a/sei-db/management/gc/api.go b/sei-db/management/gc/api.go index 3845be5153..ca5f839166 100644 --- a/sei-db/management/gc/api.go +++ b/sei-db/management/gc/api.go @@ -1,32 +1,55 @@ package gc +// InfiniteRetentionWindow is the conventional GetRetentionWindow value meaning "never +// prune this store." Any negative value is treated the same: the store does not vote +// and is never passed to PruneBelow. +// +// Note: elsewhere in this repo (KeepRecent / MinRetainBlocks) 0 often means "keep +// forever." Here 0 means "no extra beyond the shared RollbackWindow," so infinite +// retention needs a negative sentinel. +const InfiniteRetentionWindow int64 = -1 + // PrunableStore is a store whose old data may be dropped by the StorageGarbageCollector. type PrunableStore interface { - // Name Returns the name of the store. + // Name identifies the store (logging / errors). Duplicate names are allowed; + // the collector keys answers by index, not by name. Name() string - // PruneBelow tells the store that it may drop snapshots/data for all blocks below a specified number. - // Store can drop data asynchronously in the background. + // PruneBelow may drop data for all blocks below blockNumber. + // The store may perform the deletion asynchronously. PruneBelow(blockNumber uint64) error - // GetOldestBlockToRetain returns the oldest block this store must keep in order to remain able to serve cutLine, - // or 0 when the store opts out of pruning entirely (e.g. infinite retention or disabled). + // GetRetentionWindow is how many extra blocks beyond the shared RollbackWindow + // this store needs to keep servable. Cut line (head = min non-zero GetLastestBlock): + // + // retention < 0 → store ignored (cutLine treated as 0; conventionally -1) + // retention >= 0 → cutLine = head - RollbackWindow - retention // - // The collector ignores a store that answers 0 when choosing the prune height and does not call PruneBelow on it, - // so that other stores can still be pruned. + // RollbackWindow is shared so every store stays consistent under rollback. + // Retention is optional history on top — kept so the store can still serve queries + // after a rollback that consumes the entire rollback window. + // + // Contract by store kind: + // - Contiguous (blockDB, receiptDB, state WAL): -1 / 0 / positive as configured. + // - SC: always 0 (only needs the shared rollback window for snapshots). + // - SS: always 0 (SS prunes its own version history via KeepRecent; the collector + // only drives SS snapshot pruning for the shared rollback window). + GetRetentionWindow() int64 + + // GetPruningBoundry returns the oldest block this store must keep to remain able to + // serve cutLine, or 0 to opt out of this cycle (not participate in calculating min prune height). // - // A store that retains a contiguous range of blocks (blockDB, receiptDB, the state WAL) can be restored to any - // block it holds, so it should simply return the cutLine unchanged. + // Contiguous stores can restore to any height they hold → return cutLine. // - // A store with checkpoints can only be restored to a height it has a snapshot for, so it answers the newest - // completed snapshot at or below cutLine, or its oldest completed snapshot when every snapshot is above the cut - // line, since in that case none of them can be dropped. With no completed snapshot at all it answers 0. + // Snapshot stores can restore only at snapshot heights → return the newest completed + // snapshot ≤ cutLine, or the oldest completed snapshot if every snapshot is above + // cutLine (none can be dropped yet). No completed snapshot → 0. // - // Snapshot creation is assumed to complete within seconds, so a snapshot write in flight is not modeled: a store - // need not reserve blocks for a snapshot that has not landed yet. A store whose snapshots take long enough for the - // cut line to advance past them would need to report the in-progress height instead, and that is out of scope here. - GetOldestBlockToRetain(cutLine uint64) uint64 + // Assumption: snapshot creation finishes quickly enough that an in-flight write need + // not be reserved; modeling long-running snapshot writes is out of scope. + GetPruningBoundry(cutLine uint64) uint64 - // GetLastCommittedBlock returns the highest block this store has ingested. - GetLastCommittedBlock() (uint64, error) + // GetLastestBlock returns the highest block this store has ingested. + // 0 means "no data / uninitialized" and is ignored when computing the global head. + GetLastestBlock() (uint64, error) } diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go index 7a55955e46..cb11f2ca9d 100644 --- a/sei-db/management/gc/storage_garbage_collector.go +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -12,40 +12,35 @@ import ( var logger = seilog.NewLogger("db", "gc") -// StorageGarbageCollector manages deletion of stored data across a set of stores. +// StorageGarbageCollector periodically prunes a set of PrunableStores. // -// Each cycle locates the head of the chain, derives the cut line the system must remain able to serve, asks every store -// how far back it must retain data to serve that cut line, and prunes every store below the lowest answer. Deleting to -// the lowest answer rather than to each store's own is what keeps the stores mutually usable: a snapshot is only -// restorable if the contiguous stores still hold the blocks that follow it. +// All stores share config.RollbackWindow. Each store may request extra history via +// GetRetentionWindow (see that method). SC/SS return 0 — the collector only drives +// their snapshot pruning for the shared window; SS version-history pruning is separate. // -// The StorageGarbageCollector performs state deletion while maintaining the following invariant: -// "If it's possible to roll back at least config.RollbackWindow blocks, then any state deletion operation -// will retain the system's ability to roll back at least config.RollbackWindow blocks." That is to say, -// while the StorageGarbageCollector cannot unilaterally guarantee the ability to roll back config.RollbackWindow -// blocks by itself, it guarantees that the act of deleting old state does not prevent rollback within this -// rollback window. +// Each cycle: +// 1. head = min non-zero GetLastestBlock across stores +// 2. per store: cutLine = head - RollbackWindow - GetRetentionWindow +// (skipped when cutLine == 0: infinite retention, or head still inside the window) +// 3. ask GetPruningBoundry(cutLine); 0 = opt out for this cycle +// 4. pruneHeight = min of non-zero answers; PruneBelow(pruneHeight) on every store +// that answered non-zero // -// The collector itself is only a ticker that periodically calls prune. All decision logic lives in prune so it can be -// unit tested without threads or timing. +// Step 4 uses the shared min (not each store's own boundry) so a snapshot remains +// restorable: contiguous stores must still hold the blocks that follow it. // -// StorageGarbageCollector is not thread safe. +// Invariant: if RollbackWindow blocks of rollback were possible before a prune, that +// prune does not take the ability away. Full headroom is not promised after a rollback +// has already consumed part of the window. +// +// The type is a ticker around prune; decision logic lives in prune for unit testing. +// Not safe for concurrent use (Close must be called exactly once). type StorageGarbageCollector struct { - - // Configuration for this storage garbage collector. config *StorageGarbageCollectorConfig - - // The stores this collector prunes. stores []PrunableStore - - // Cancelled to signal the run loop to stop. - ctx context.Context - - // Closed by Close to signal the run loop to stop. + ctx context.Context stopCh chan struct{} - - // Tracks the run loop goroutine so Close can wait for it to exit. - wg sync.WaitGroup + wg sync.WaitGroup } func NewStorageGarbageCollector( @@ -63,21 +58,18 @@ func NewStorageGarbageCollector( ctx: ctx, stopCh: make(chan struct{}), } - s.wg.Add(1) go s.run() - return s, nil } -// Close stops the storage garbage collector and waits for its run loop to exit. It must be called exactly once. +// Close stops the run loop and waits for it to exit. Must be called exactly once. func (s *StorageGarbageCollector) Close() error { close(s.stopCh) s.wg.Wait() return nil } -// run periodically drives prune cycles until the manager is stopped. func (s *StorageGarbageCollector) run() { defer s.wg.Done() @@ -98,62 +90,56 @@ func (s *StorageGarbageCollector) run() { } } -// prune performs a single prune cycle: it locates the head of the chain, derives the cut line from it, asks every store -// how far back it must retain data to serve that cut line, and prunes every store below the lowest answer. +// prune runs one prune cycle. See StorageGarbageCollector for the decision rules. func prune(config *StorageGarbageCollectorConfig, stores []PrunableStore) error { if len(stores) == 0 { return nil } - globalLatestBlock, err := getGlobalLastCommittedBlock(stores) + + globalLatestBlock, err := getGlobalLastestBlock(stores) if err != nil { return err } if globalLatestBlock == 0 { - logger.Info("skipping pruning, no store has committed a block") + logger.Info("skipping pruning: no store has a latest block") return nil } - cutLine := getCutLine(globalLatestBlock, config.RollbackWindow, config.RetentionBeyondRollbackWindow) - if cutLine == 0 { - logger.Info("skipping pruning, the chain is younger than the retain window", - "globalLatestBlock", globalLatestBlock) - return nil - } - - // Every participating store must keep back to its own answer, so the system may only prune below the lowest of - // them. Pruning below a higher answer would delete blocks that a lower-answering store just reported it still - // needs, which is exactly how deletion would break the rollback invariant. - // - // A store that answers 0 is left out of both the prune-height vote and PruneBelow: it holds no snapshot yet, is - // disabled, or is intentionally retaining forever (see GetOldestBlockToRetain). - // - // The answers are kept positionally rather than keyed by Name so that two stores sharing a name cannot make the - // collector prune one that opted out. Names are for logs only. + // Boundries are positional so duplicate Name() values cannot mis-attribute an opt-out. + pruningBoundries := make([]uint64, len(stores)) var pruneHeight uint64 - oldestBlockToRetain := make([]uint64, len(stores)) for i, store := range stores { - oldestBlockToRetain[i] = store.GetOldestBlockToRetain(cutLine) - if oldestBlockToRetain[i] == 0 { + retention := store.GetRetentionWindow() + cutLine := getCutLine(globalLatestBlock, config.RollbackWindow, retention) + if cutLine == 0 { + // Infinite retention, or head still inside this store's retain window. + continue + } + + pruningBoundries[i] = store.GetPruningBoundry(cutLine) + if pruningBoundries[i] == 0 { + // Opt-out (e.g. no completed snapshot yet). continue } - if pruneHeight == 0 || oldestBlockToRetain[i] < pruneHeight { - pruneHeight = oldestBlockToRetain[i] + if pruneHeight == 0 || pruningBoundries[i] < pruneHeight { + pruneHeight = pruningBoundries[i] } } if pruneHeight == 0 { - logger.Info("skipping pruning, no store has data to retain", - "cutLine", cutLine, "oldestBlockToRetainByStore", describeAnswers(stores, oldestBlockToRetain)) + logger.Info("skipping pruning: no store reported a pruning boundry", + "globalLatestBlock", globalLatestBlock, + "pruningBoundryByStore", describeAnswers(stores, pruningBoundries), + ) return nil } - logger.Info("pruning storage", + logger.Info("pruning stores", "globalLatestBlock", globalLatestBlock, - "cutLine", cutLine, "pruneHeight", pruneHeight, - "oldestBlockToRetainByStore", describeAnswers(stores, oldestBlockToRetain), + "pruningBoundryByStore", describeAnswers(stores, pruningBoundries), ) for i, store := range stores { - if oldestBlockToRetain[i] == 0 { + if pruningBoundries[i] == 0 { continue } if err := store.PruneBelow(pruneHeight); err != nil { @@ -163,55 +149,41 @@ func prune(config *StorageGarbageCollectorConfig, stores []PrunableStore) error return nil } -// describeAnswers renders the per-store retain answers for a log line, in store order so duplicate names stay -// distinguishable. -func describeAnswers(stores []PrunableStore, oldestBlockToRetain []uint64) string { +func describeAnswers(stores []PrunableStore, pruningBoundries []uint64) string { var sb strings.Builder for i, store := range stores { if i > 0 { sb.WriteString(" ") } - fmt.Fprintf(&sb, "%s=%d", store.Name(), oldestBlockToRetain[i]) + fmt.Fprintf(&sb, "%s=%d", store.Name(), pruningBoundries[i]) } return sb.String() } -// getCutLine returns the oldest block the system must remain able to serve, which is the head of the chain less the -// rollback window and the historical blocks guaranteed beyond it. -// -// Returns 0 to mean "prune nothing": either retentionBeyond is negative (infinite retention), or the chain is younger -// than the combined finite window. The young-chain comparison has to happen before the subtraction: these are -// unsigned, so subtracting past zero wraps to a cut line far above the head of the chain, and pruning to it would -// delete everything. -// -// Config.Validate rejects a finite rollback window and retention that overflow when summed, so retainWindow below is -// exact whenever retentionBeyond is non-negative. -func getCutLine(globalLatestBlock uint64, rollbackWindow uint64, retentionBeyond int64) uint64 { - if retentionBeyond < 0 { +// getCutLine returns head - RollbackWindow - retention for retention >= 0. +// Returns 0 when retention < 0, or when head is still inside that combined window +// (unsigned subtraction must not wrap). +func getCutLine(globalLatestBlock uint64, rollbackWindow uint64, retention int64) uint64 { + if retention < 0 { return 0 } - retainWindow := rollbackWindow + uint64(retentionBeyond) - if globalLatestBlock <= retainWindow { + totalRetainWindow := rollbackWindow + uint64(retention) + if globalLatestBlock <= totalRetainWindow { return 0 } - return globalLatestBlock - retainWindow + return globalLatestBlock - totalRetainWindow } -// getGlobalLastCommittedBlock reads the head height of every store and returns the smallest. -// -// The smallest head is the head of the chain as far as the collector can tell, because it bounds what the system can -// actually serve. Measuring the cut line from a store that has run ahead of the others would let the collector prune a -// lagging store past blocks that store still needs in order to serve a rollback. -// -// Heads of 0 are skipped: a store that has ingested nothing would otherwise hold the head of the whole system at 0 and -// stall pruning everywhere. Skipping it does not put its data at risk, because whatever it holds, its -// GetOldestBlockToRetain answer still bounds the prune height. Returns 0 when no store has committed a block. -func getGlobalLastCommittedBlock(stores []PrunableStore) (uint64, error) { +// getGlobalLastestBlock returns the smallest non-zero GetLastestBlock among stores. +// Using the min keeps a lagging store from being pruned past blocks it still needs. +// Heads of 0 are skipped so an uninitialized store does not stall pruning. +// Returns 0 when no store has data. +func getGlobalLastestBlock(stores []PrunableStore) (uint64, error) { var blockNum uint64 for _, store := range stores { - storeHeight, err := store.GetLastCommittedBlock() + storeHeight, err := store.GetLastestBlock() if err != nil { - return 0, fmt.Errorf("failed to read last committed block from %s: %w", store.Name(), err) + return 0, fmt.Errorf("failed to read lastest block from %s: %w", store.Name(), err) } if storeHeight == 0 { continue diff --git a/sei-db/management/gc/storage_garbage_collector_config.go b/sei-db/management/gc/storage_garbage_collector_config.go index 62ea183ad6..829a3fc9b2 100644 --- a/sei-db/management/gc/storage_garbage_collector_config.go +++ b/sei-db/management/gc/storage_garbage_collector_config.go @@ -5,88 +5,42 @@ import ( "time" ) -// InfiniteRetentionBeyondRollbackWindow is the conventional RetentionBeyondRollbackWindow -// value for never pruning. Any negative value is treated the same: the collector derives a -// cut line of 0 and skips every prune cycle. -// -// Unlike KeepRecent / MinRetainBlocks elsewhere in this repo (where 0 means keep forever), -// 0 here means "no historical blocks beyond the rollback window" — so infinite needs a -// negative sentinel. -const InfiniteRetentionBeyondRollbackWindow int64 = -1 - -// Configures a StorageGarbageCollector. +// StorageGarbageCollectorConfig configures a StorageGarbageCollector. type StorageGarbageCollectorConfig struct { - - // The maximum number of blocks the system should be able to roll back at any point in time. - // Storage garbage collector ensures that enough data is kept on disk such that the system can always - // roll back this many blocks. Must be greater than 0: a zero window would let cutLine reach the - // chain head, and a store with no completed snapshot answering 0 would then leave contiguous - // stores free to drop everything below head — a snapshot landing at ~head could no longer be - // replayed forward. + // RollbackWindow is how many blocks behind head the system must remain able to + // roll back to. Shared by every managed store so they stay mutually consistent; + // per-store extras beyond this window use PrunableStore.GetRetentionWindow. // - // Note that the "always able to rollback" invariant may be broken after a rollback. For example, if we normally - // require a rollback window of 10,000 blocks and then we rollback 5,000 blocks, we can then only rollback an - // additional 5,000 blocks after that first rollback. - RollbackWindow uint64 - - // How many historical blocks beyond the rollback window the collector guarantees to keep - // servable, applied to every store. RollbackWindow is what correctness requires for rollback; - // this field is the additional history operators want for queries against old blocks and - // receipts. When non-negative, the cut line is - // head - (RollbackWindow + RetentionBeyondRollbackWindow). + // Must be > 0. With RollbackWindow == 0, cutLine can equal head: contiguous stores + // may drop everything below head, and a store answering 0 (no snapshot yet) offers + // no counter-vote — a snapshot that then lands near head cannot be replayed forward. // - // Zero means guarantee only the rollback window (no extra history). Any negative value means - // infinite retention: never prune. Prefer InfiniteRetentionBeyondRollbackWindow (-1) as the - // conventional sentinel. This differs from KeepRecent / MinRetainBlocks, where 0 means keep - // forever. - // - // TODO: make this per-store if the shared value proves too blunt. Because every store is pruned to the lowest - // block any single store still needs, retention bought for one store is paid for by all of them: raising this to - // keep more block and receipt history also holds that much extra state in the snapshotted stores, which is far - // more expensive per block. Separating them means giving the collector a retention value alongside each store - // (e.g. a []ManagedStore{Store, RetentionBlocks} in place of []PrunableStore) and computing one cut line per - // store. Note that this only reduces retention where a store's own answer is what bounds the minimum; the - // contiguous stores must still cover the oldest snapshot anyone kept, or the snapshot becomes unreplayable. - RetentionBeyondRollbackWindow int64 + // After a rollback that consumes part of the window, full headroom is not promised + // (e.g. window 10_000, roll back 5_000 → only ~5_000 of headroom remain). + RollbackWindow uint64 - // How often the collector drives a prune cycle. + // PruneInterval is how often the collector runs a prune cycle. Must be > 0. PruneInterval time.Duration } -// Construct a default storage garbage collector config. +// DefaultStorageGarbageCollectorConfig returns the default collector config. func DefaultStorageGarbageCollectorConfig() *StorageGarbageCollectorConfig { return &StorageGarbageCollectorConfig{ - RollbackWindow: 1000, - RetentionBeyondRollbackWindow: 100_000, - PruneInterval: 10 * time.Minute, + RollbackWindow: 1000, + PruneInterval: 10 * time.Minute, } } -// Validate the storage garbage collector's config. +// Validate checks that required fields are set to usable values. func (c *StorageGarbageCollectorConfig) Validate() error { if c == nil { return fmt.Errorf("config is required") } - // RollbackWindow must leave at least one block of margin below head. With RollbackWindow == 0 and - // RetentionBeyondRollbackWindow == 0, cutLine == globalLatestBlock: contiguous stores may drop - // everything below head, and a store that answers 0 because it has no completed snapshot yet offers - // no counter-vote — a snapshot that then lands at ~head cannot be replayed forward. Infinite - // retention is any negative RetentionBeyondRollbackWindow, not a zero rollback window. if c.RollbackWindow == 0 { return fmt.Errorf("rollback window must be greater than 0") } if c.PruneInterval <= 0 { return fmt.Errorf("prune interval must be greater than 0") } - // The cut line subtracts RollbackWindow + RetentionBeyondRollbackWindow from the head. Reject a - // finite pair that overflows when summed, so that subtraction cannot wrap around to a cut line - // above the head. Negatives mean infinite retention and skip this sum. - if c.RetentionBeyondRollbackWindow >= 0 { - retention := uint64(c.RetentionBeyondRollbackWindow) - if c.RollbackWindow+retention < c.RollbackWindow { - return fmt.Errorf("rollback window (%d) plus retention beyond rollback window (%d) overflows uint64", - c.RollbackWindow, c.RetentionBeyondRollbackWindow) - } - } return nil } diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go index 7695d2f4c4..10b0e7a04b 100644 --- a/sei-db/management/gc/storage_garbage_collector_test.go +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -3,7 +3,6 @@ package gc import ( "context" "errors" - "math" "sync/atomic" "testing" "time" @@ -11,28 +10,29 @@ import ( "github.com/stretchr/testify/require" ) -// mockStore is a PrunableStore with canned answers. Build one with snapshotStore or contiguousStore. +// mockStore is a PrunableStore with canned answers. Build via snapshotStore or contiguousStore. type mockStore struct { name string latestHeight uint64 - // oldestToRetain answers GetOldestBlockToRetain for a given cut line. - oldestToRetain func(cutLine uint64) uint64 - getErr error - pruneErr error + // retentionWindow is returned by GetRetentionWindow (0 = shared RollbackWindow only). + retentionWindow int64 + pruningBoundry func(cutLine uint64) uint64 + getErr error + pruneErr error - // Observed by tests after a prune cycle. pruneBelowCalled atomic.Bool prunedBelow atomic.Uint64 } -// snapshotStore models a store that can only be restored to a height it holds a snapshot for, such as SC or SS: it -// answers with the newest snapshot at or below the cut line, or with its oldest snapshot when every snapshot is above the -// cut line. Snapshots must be in ascending order; a store given none has no completed snapshot and answers 0. +// snapshotStore models SC/SS (retention 0). GetPruningBoundry returns the newest snapshot +// ≤ cutLine, or the oldest snapshot if all are above cutLine. Empty snapshots → 0. +// Snapshots must be ascending. func snapshotStore(name string, latestHeight uint64, snapshots ...uint64) *mockStore { return &mockStore{ - name: name, - latestHeight: latestHeight, - oldestToRetain: func(cutLine uint64) uint64 { + name: name, + latestHeight: latestHeight, + retentionWindow: 0, + pruningBoundry: func(cutLine uint64) uint64 { if len(snapshots) == 0 { return 0 } @@ -48,13 +48,15 @@ func snapshotStore(name string, latestHeight uint64, snapshots ...uint64) *mockS } } -// contiguousStore models a store that retains every block in a range, such as blockDB, receiptDB or the state WAL. It can -// be restored to any block it holds, so it needs nothing below the cut line. +// contiguousStore models blockDB / receiptDB / WAL (retention 0 by default). +// hasData=false → GetPruningBoundry returns 0 (opt out). Otherwise returns cutLine. +// Use withRetentionWindow for extras or InfiniteRetentionWindow. func contiguousStore(name string, latestHeight uint64, hasData bool) *mockStore { return &mockStore{ - name: name, - latestHeight: latestHeight, - oldestToRetain: func(cutLine uint64) uint64 { + name: name, + latestHeight: latestHeight, + retentionWindow: 0, + pruningBoundry: func(cutLine uint64) uint64 { if !hasData { return 0 } @@ -63,16 +65,25 @@ func contiguousStore(name string, latestHeight uint64, hasData bool) *mockStore } } +func withRetentionWindow(store *mockStore, retention int64) *mockStore { + store.retentionWindow = retention + return store +} + func (m *mockStore) Name() string { return m.name } -func (m *mockStore) GetLastCommittedBlock() (uint64, error) { +func (m *mockStore) GetRetentionWindow() int64 { + return m.retentionWindow +} + +func (m *mockStore) GetLastestBlock() (uint64, error) { return m.latestHeight, m.getErr } -func (m *mockStore) GetOldestBlockToRetain(cutLine uint64) uint64 { - return m.oldestToRetain(cutLine) +func (m *mockStore) GetPruningBoundry(cutLine uint64) uint64 { + return m.pruningBoundry(cutLine) } func (m *mockStore) PruneBelow(blockNumber uint64) error { @@ -81,7 +92,6 @@ func (m *mockStore) PruneBelow(blockNumber uint64) error { return m.pruneErr } -// prunableStores adapts the mocks to the interface slice prune takes. func prunableStores(list ...*mockStore) []PrunableStore { result := make([]PrunableStore, len(list)) for i, store := range list { @@ -90,43 +100,36 @@ func prunableStores(list ...*mockStore) []PrunableStore { return result } -// testConfig builds a validated config for calling prune directly. -func testConfig(t *testing.T, rollbackWindow uint64, retentionBeyond int64) *StorageGarbageCollectorConfig { +func testConfig(t *testing.T, rollbackWindow uint64) *StorageGarbageCollectorConfig { t.Helper() config := &StorageGarbageCollectorConfig{ - RollbackWindow: rollbackWindow, - RetentionBeyondRollbackWindow: retentionBeyond, - PruneInterval: time.Minute, + RollbackWindow: rollbackWindow, + PruneInterval: time.Minute, } - // getCutLine subtracts without an overflow guard because Validate rejects a retain window that would overflow, so - // test configs have to clear the same bar as real ones. require.NoError(t, config.Validate()) return config } -// TestPruneDecisions is the decision matrix for a single prune cycle. wantPruneBelow == nil means no store may be pruned. +// TestPruneDecisions covers one prune cycle. wantPruneBelow == nil means no store is pruned. func TestPruneDecisions(t *testing.T) { cases := []struct { - name string - rollbackWindow uint64 - retentionBeyond int64 - stores []*mockStore - wantPruneBelow *uint64 + name string + rollbackWindow uint64 + stores []*mockStore + wantPruneBelow *uint64 }{ { - name: "one snapshotted store and a WAL", + name: "SC and WAL both retention 0: min boundry wins", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 80_000, 85_000, 92_000), contiguousStore("stateWAL", 100_000, true), }, - // Cut line 90,000. The WAL needs only 90,000, but sc must keep the 85,000 snapshot a rollback to the cut - // line would start from, so everything is pruned to 85,000. Pruning to the higher answer would delete - // that snapshot. + // cutLine 90_000; sc keeps snapshot 85_000; WAL answers 90_000 → min 85_000. wantPruneBelow: ptr(85_000), }, { - name: "the lowest need across many stores wins", + name: "lowest boundry across many stores wins", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 80_000, 85_000, 92_000), @@ -134,43 +137,59 @@ func TestPruneDecisions(t *testing.T) { snapshotStore("flatKV", 100_000, 50_000, 90_000, 99_000), contiguousStore("stateWAL", 100_000, true), }, - // Needs are 85,000, 88,000, 90,000 and 90,000. + // sc 85_000, ss 88_000, flatKV 90_000, WAL 90_000 → min 85_000. wantPruneBelow: ptr(85_000), }, { - name: "minimum rollback window still leaves one block of margin", + name: "RollbackWindow of 1 still leaves one block of margin", rollbackWindow: 1, stores: []*mockStore{ snapshotStore("sc", 100_000, 80_000, 99_999), contiguousStore("stateWAL", 100_000, true), }, - // Cut line 99,999. sc keeps the snapshot at the cut line; contiguous stores prune to that. wantPruneBelow: ptr(99_999), }, { - name: "retention beyond rollback window deepens the cut line", - rollbackWindow: 10_000, - retentionBeyond: 5_000, + name: "positive contiguous retention deepens that store's cut line", + rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 80_000, 90_000), - contiguousStore("stateWAL", 100_000, true), + withRetentionWindow(contiguousStore("stateWAL", 100_000, true), 5_000), }, - // Cut line 85,000 rather than 90,000, so the 80,000 snapshot is now the newest at or below it. Without the - // extra retention sc would have needed only 90,000. - wantPruneBelow: ptr(80_000), + // sc cutLine 90_000 → 90_000; WAL cutLine 85_000 → 85_000 → min 85_000. + wantPruneBelow: ptr(85_000), }, { - name: "infinite retention beyond rollback window skips pruning", - rollbackWindow: 10_000, - retentionBeyond: InfiniteRetentionBeyondRollbackWindow, + name: "SS retention 0 behaves like SC", + rollbackWindow: 10_000, stores: []*mockStore{ - snapshotStore("sc", 100_000, 80_000, 100_000), + snapshotStore("ss", 100_000, 80_000, 90_000), contiguousStore("stateWAL", 100_000, true), }, + wantPruneBelow: ptr(90_000), + }, + { + name: "infinite retention on every store skips pruning", + rollbackWindow: 10_000, + stores: []*mockStore{ + withRetentionWindow(contiguousStore("blockDB", 100_000, true), InfiniteRetentionWindow), + withRetentionWindow(contiguousStore("receiptDB", 100_000, true), InfiniteRetentionWindow), + }, wantPruneBelow: nil, }, { - name: "a snapshot exactly at the cut line is kept", + name: "infinite retention on one store leaves others free to prune", + rollbackWindow: 10_000, + stores: []*mockStore{ + withRetentionWindow(contiguousStore("archiveWAL", 100_000, true), InfiniteRetentionWindow), + snapshotStore("sc", 100_000, 80_000), + contiguousStore("stateWAL", 100_000, true), + }, + // archiveWAL skipped; sc 80_000; stateWAL 90_000 → min 80_000. + wantPruneBelow: ptr(80_000), + }, + { + name: "snapshot exactly at the cut line is kept", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 50_000, 90_000), @@ -179,62 +198,56 @@ func TestPruneDecisions(t *testing.T) { wantPruneBelow: ptr(90_000), }, { - name: "every snapshot above the cut line pins the store to its oldest", + name: "all snapshots above cut line: store still votes its oldest", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 95_000, 97_000), contiguousStore("stateWAL", 100_000, true), }, - // sc cannot drop either snapshot and answers 95,000, so the WAL's 90,000 is the lowest. + // sc answers 95_000; WAL answers 90_000 → shared prune 90_000 (sc snapshots untouched). wantPruneBelow: ptr(90_000), }, { - name: "a lagging store pulls the cut line down with it", + name: "lagging store lowers the global head", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("lagging", 50_000, 30_000, 50_000), contiguousStore("stateWAL", 100_000, true), }, - // The head is the lowest of the two, 50,000, putting the cut line at 40,000 and making 30,000 the newest - // snapshot at or below it. Measuring from the WAL's head instead would put the cut line at 90,000, and the - // 30,000 snapshot needed to roll back to 40,000 would be deleted. + // head 50_000 → cutLine 40_000; lagging answers 30_000; WAL 40_000 → min 30_000. wantPruneBelow: ptr(30_000), }, { - name: "a store answering 0 is ignored", + name: "store answering 0 is ignored", rollbackWindow: 10_000, stores: []*mockStore{ contiguousStore("optOut", 100_000, false), snapshotStore("sc", 100_000, 80_000), contiguousStore("stateWAL", 100_000, true), }, - // optOut answers 0 (e.g. infinite retention) and is left out of the vote and of PruneBelow. wantPruneBelow: ptr(80_000), }, { - name: "a store with no snapshot yet is ignored", + name: "store with no snapshot yet is ignored", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000), contiguousStore("stateWAL", 100_000, true), }, - // sc has no completed snapshot, so it answers 0 and stays out of the vote and of PruneBelow. Snapshot - // creation is assumed to finish within seconds, so no blocks are reserved for a write in flight. wantPruneBelow: ptr(90_000), }, { - name: "a head of zero is ignored, the data behind it is not", + name: "zero head ignored for global head; store still votes", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("stalled", 0, 50_000), contiguousStore("stateWAL", 100_000, true), }, - // The stalled store is left out of the head, so the cut line comes from the WAL at 90,000. It still - // reports that it needs its 50,000 snapshot, and that is the lowest answer. + // head from WAL 100_000; stalled still answers 50_000 → min 50_000. wantPruneBelow: ptr(50_000), }, { - name: "chain younger than the retain window", + name: "head inside retain window: no prune", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 5_000, 1_000, 2_000), @@ -243,17 +256,17 @@ func TestPruneDecisions(t *testing.T) { wantPruneBelow: nil, }, { - name: "head exactly at the retain window", - rollbackWindow: 60_000, - retentionBeyond: 40_000, + name: "head inside one store's window skips only that store", + rollbackWindow: 60_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 500, 1_000), - contiguousStore("stateWAL", 100_000, true), + withRetentionWindow(contiguousStore("stateWAL", 100_000, true), 40_000), }, - wantPruneBelow: nil, + // WAL cutLine 0 (skipped); sc cutLine 40_000 → 1_000. + wantPruneBelow: ptr(1_000), }, { - name: "no store has committed a block", + name: "no store has a latest block", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 0), @@ -267,7 +280,6 @@ func TestPruneDecisions(t *testing.T) { stores: []*mockStore{ contiguousStore("optOut", 100_000, false), }, - // Every store answered 0, so there is no prune height. wantPruneBelow: nil, }, { @@ -281,14 +293,20 @@ func TestPruneDecisions(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { stores := prunableStores(tc.stores...) - require.NoError(t, prune(testConfig(t, tc.rollbackWindow, tc.retentionBeyond), stores)) + require.NoError(t, prune(testConfig(t, tc.rollbackWindow), stores)) - head, err := getGlobalLastCommittedBlock(stores) + head, err := getGlobalLastestBlock(stores) require.NoError(t, err) - cutLine := getCutLine(head, tc.rollbackWindow, tc.retentionBeyond) for _, store := range tc.stores { - if tc.wantPruneBelow == nil || store.GetOldestBlockToRetain(cutLine) == 0 { + shouldPrune := false + if tc.wantPruneBelow != nil && store.GetRetentionWindow() >= 0 { + cutLine := getCutLine(head, tc.rollbackWindow, store.GetRetentionWindow()) + if cutLine > 0 && store.GetPruningBoundry(cutLine) != 0 { + shouldPrune = true + } + } + if !shouldPrune { require.Falsef(t, store.pruneBelowCalled.Load(), "%s should not be pruned", store.name) continue } @@ -299,50 +317,44 @@ func TestPruneDecisions(t *testing.T) { } } -// TestPruneOnYoungChainWithDefaultConfig guards the unsigned subtraction in getCutLine using the real defaults, where the -// retain window is 101,000 blocks. Every node spends its early life here, and a wrapped cut line would land far above the -// head of the chain and delete everything. -func TestPruneOnYoungChainWithDefaultConfig(t *testing.T) { +func TestPruneOnYoungChainWithDeepContiguousRetention(t *testing.T) { + // head 100 << WAL retain window (rollback 1_000 + retention 100_000) → both cutLines 0. sc := snapshotStore("sc", 100, 50, 100) - wal := contiguousStore("stateWAL", 100, true) + wal := withRetentionWindow(contiguousStore("stateWAL", 100, true), 100_000) - require.NoError(t, prune(DefaultStorageGarbageCollectorConfig(), prunableStores(sc, wal))) + require.NoError(t, prune(testConfig(t, 1_000), prunableStores(sc, wal))) require.False(t, sc.pruneBelowCalled.Load()) require.False(t, wal.pruneBelowCalled.Load()) } -// TestPruneKeepsOptedOutStoreWithDuplicateName checks that answers are tracked positionally: two stores sharing a name -// must not make the collector prune the one that opted out with 0. func TestPruneKeepsOptedOutStoreWithDuplicateName(t *testing.T) { + // Answers are keyed by index, so a duplicate Name() must not prune the opt-out store. optOut := contiguousStore("ss", 100_000, false) participating := snapshotStore("ss", 100_000, 80_000) wal := contiguousStore("stateWAL", 100_000, true) - require.NoError(t, prune(testConfig(t, 10_000, 0), prunableStores(optOut, participating, wal))) + require.NoError(t, prune(testConfig(t, 10_000), prunableStores(optOut, participating, wal))) - require.False(t, optOut.pruneBelowCalled.Load(), "a store answering 0 must never be pruned") + require.False(t, optOut.pruneBelowCalled.Load(), "store answering 0 must not be pruned") require.True(t, participating.pruneBelowCalled.Load()) require.Equal(t, uint64(80_000), participating.prunedBelow.Load()) require.Equal(t, uint64(80_000), wal.prunedBelow.Load()) } -// TestPruneGetLastCommittedBlockError checks that a failure to read a head aborts the cycle before anything is deleted. -func TestPruneGetLastCommittedBlockError(t *testing.T) { +func TestPruneGetLastestBlockError(t *testing.T) { sentinel := errors.New("boom") sc := snapshotStore("sc", 100_000, 80_000) broken := contiguousStore("brokenStore", 100_000, true) broken.getErr = sentinel - err := prune(testConfig(t, 10_000, 0), prunableStores(sc, broken)) + err := prune(testConfig(t, 10_000), prunableStores(sc, broken)) require.ErrorIs(t, err, sentinel) require.ErrorContains(t, err, "brokenStore") require.False(t, sc.pruneBelowCalled.Load()) require.False(t, broken.pruneBelowCalled.Load()) } -// TestPruneBelowErrorStopsRemainingStores checks that the first failed deletion aborts the cycle, leaving later stores -// untouched. func TestPruneBelowErrorStopsRemainingStores(t *testing.T) { sentinel := errors.New("boom") first := snapshotStore("first", 100_000, 80_000) @@ -350,10 +362,10 @@ func TestPruneBelowErrorStopsRemainingStores(t *testing.T) { second := snapshotStore("second", 100_000, 80_000) wal := contiguousStore("stateWAL", 100_000, true) - err := prune(testConfig(t, 10_000, 0), prunableStores(first, second, wal)) + err := prune(testConfig(t, 10_000), prunableStores(first, second, wal)) require.ErrorIs(t, err, sentinel) require.ErrorContains(t, err, "first") - require.ErrorContains(t, err, "80000") // the prune height, i.e. the lowest need across the stores + require.ErrorContains(t, err, "80000") require.True(t, first.pruneBelowCalled.Load()) require.False(t, second.pruneBelowCalled.Load()) require.False(t, wal.pruneBelowCalled.Load()) @@ -370,13 +382,12 @@ func TestGetCutLine(t *testing.T) { {name: "rollback window only", head: 100_000, rollbackWindow: 10_000, want: 90_000}, {name: "retention adds to rollback window", head: 100_000, rollbackWindow: 1, retention: 10_000, want: 89_999}, {name: "the two windows add", head: 100_000, rollbackWindow: 10_000, retention: 5_000, want: 85_000}, - {name: "zero retention beyond rollback window", head: 100_000, rollbackWindow: 10_000, want: 90_000}, - {name: "infinite retention beyond rollback window", head: 100_000, rollbackWindow: 10_000, retention: InfiniteRetentionBeyondRollbackWindow, want: 0}, + {name: "zero retention", head: 100_000, rollbackWindow: 10_000, want: 90_000}, + {name: "infinite retention", head: 100_000, rollbackWindow: 10_000, retention: InfiniteRetentionWindow, want: 0}, {name: "any negative retention is infinite", head: 100_000, rollbackWindow: 10_000, retention: -99, want: 0}, {name: "head one above the window", head: 10_001, rollbackWindow: 10_000, want: 1}, {name: "head exactly at the window", head: 10_000, rollbackWindow: 10_000, want: 0}, {name: "head one below the window", head: 9_999, rollbackWindow: 10_000, want: 0}, - // The default retain window against a young chain: the subtraction must not wrap. {name: "head far below the window", head: 100, rollbackWindow: 1_000, retention: 100_000, want: 0}, {name: "head at genesis", head: 0, rollbackWindow: 10_000, want: 0}, } @@ -388,7 +399,7 @@ func TestGetCutLine(t *testing.T) { } } -func TestGetGlobalLastCommittedBlock(t *testing.T) { +func TestGetGlobalLastestBlock(t *testing.T) { storesWithHeads := func(heads ...uint64) []PrunableStore { list := make([]*mockStore, len(heads)) for i, head := range heads { @@ -407,7 +418,6 @@ func TestGetGlobalLastCommittedBlock(t *testing.T) { {name: "smallest first", heads: []uint64{80, 100}, want: 80}, {name: "smallest last", heads: []uint64{100, 80}, want: 80}, {name: "smallest in the middle", heads: []uint64{100, 50, 90}, want: 50}, - // An uninitialized store must not drag the head of the chain down with it. {name: "leading zero ignored", heads: []uint64{0, 100}, want: 100}, {name: "trailing zero ignored", heads: []uint64{100, 0}, want: 100}, {name: "zero among many ignored", heads: []uint64{100, 0, 80}, want: 80}, @@ -417,7 +427,7 @@ func TestGetGlobalLastCommittedBlock(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - head, err := getGlobalLastCommittedBlock(storesWithHeads(tc.heads...)) + head, err := getGlobalLastestBlock(storesWithHeads(tc.heads...)) require.NoError(t, err) require.Equal(t, tc.want, head) }) @@ -427,7 +437,6 @@ func TestGetGlobalLastCommittedBlock(t *testing.T) { func TestDefaultStorageGarbageCollectorConfig(t *testing.T) { cfg := DefaultStorageGarbageCollectorConfig() require.Equal(t, uint64(1_000), cfg.RollbackWindow) - require.Equal(t, int64(100_000), cfg.RetentionBeyondRollbackWindow) require.Equal(t, 10*time.Minute, cfg.PruneInterval) require.NoError(t, cfg.Validate()) } @@ -441,22 +450,8 @@ func TestValidate(t *testing.T) { }).Validate(), "rollback window must be greater than 0") require.NoError(t, (&StorageGarbageCollectorConfig{ - RollbackWindow: 1, - RetentionBeyondRollbackWindow: 0, - PruneInterval: time.Minute, - }).Validate()) - - require.NoError(t, (&StorageGarbageCollectorConfig{ - RollbackWindow: 1, - RetentionBeyondRollbackWindow: InfiniteRetentionBeyondRollbackWindow, - PruneInterval: time.Minute, - }).Validate()) - - // Any negative value means infinite retention. - require.NoError(t, (&StorageGarbageCollectorConfig{ - RollbackWindow: 1, - RetentionBeyondRollbackWindow: -2, - PruneInterval: time.Minute, + RollbackWindow: 1, + PruneInterval: time.Minute, }).Validate()) require.ErrorContains(t, (&StorageGarbageCollectorConfig{ @@ -467,29 +462,6 @@ func TestValidate(t *testing.T) { RollbackWindow: 1, PruneInterval: -time.Second, }).Validate(), "prune interval") - - // The retain window is the sum of the two finite values, so the pair must not overflow. getCutLine - // subtracts it without a guard. Infinite retention (any negative) skips that sum. - require.NoError(t, (&StorageGarbageCollectorConfig{ - RollbackWindow: math.MaxUint64, - RetentionBeyondRollbackWindow: 0, - PruneInterval: time.Minute, - }).Validate()) - require.NoError(t, (&StorageGarbageCollectorConfig{ - RollbackWindow: math.MaxUint64, - RetentionBeyondRollbackWindow: InfiniteRetentionBeyondRollbackWindow, - PruneInterval: time.Minute, - }).Validate()) - require.NoError(t, (&StorageGarbageCollectorConfig{ - RollbackWindow: math.MaxUint64, - RetentionBeyondRollbackWindow: -99, - PruneInterval: time.Minute, - }).Validate()) - require.ErrorContains(t, (&StorageGarbageCollectorConfig{ - RollbackWindow: math.MaxUint64, - RetentionBeyondRollbackWindow: 1, - PruneInterval: time.Minute, - }).Validate(), "overflows uint64") } func TestNewStorageGarbageCollectorInvalidConfig(t *testing.T) { @@ -517,7 +489,6 @@ func TestNewStorageGarbageCollectorConstructAndClose(t *testing.T) { require.NoError(t, err) require.NotNil(t, sm) - // Close must return without hanging. require.NoError(t, sm.Close()) } @@ -526,7 +497,6 @@ func TestCloseAfterContextCancelled(t *testing.T) { sm, err := NewStorageGarbageCollector(ctx, DefaultStorageGarbageCollectorConfig(), nil) require.NoError(t, err) - // Cancelling the context lets the run loop exit on its own, so Close must still return. cancel() require.NoError(t, sm.Close()) } From 3560b154a16b7cd0a3fa9557e4ceb65addbe8c47 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Fri, 31 Jul 2026 16:11:10 -0700 Subject: [PATCH 12/15] Switch to continue on error prune --- .../management/gc/storage_garbage_collector.go | 8 ++++++-- .../gc/storage_garbage_collector_test.go | 17 +++++++++++------ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go index cb11f2ca9d..af738e8687 100644 --- a/sei-db/management/gc/storage_garbage_collector.go +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -2,6 +2,7 @@ package gc import ( "context" + "errors" "fmt" "strings" "sync" @@ -28,6 +29,8 @@ var logger = seilog.NewLogger("db", "gc") // // Step 4 uses the shared min (not each store's own boundry) so a snapshot remains // restorable: contiguous stores must still hold the blocks that follow it. +// PruneBelow failures are joined and do not skip later stores — permission-to-drop +// is not transactional, and one unhealthy store must not block pruning of others. // // Invariant: if RollbackWindow blocks of rollback were possible before a prune, that // prune does not take the ability away. Full headroom is not promised after a rollback @@ -138,15 +141,16 @@ func prune(config *StorageGarbageCollectorConfig, stores []PrunableStore) error "pruneHeight", pruneHeight, "pruningBoundryByStore", describeAnswers(stores, pruningBoundries), ) + var pruneErrs error for i, store := range stores { if pruningBoundries[i] == 0 { continue } if err := store.PruneBelow(pruneHeight); err != nil { - return fmt.Errorf("failed to prune %s below %d: %w", store.Name(), pruneHeight, err) + pruneErrs = errors.Join(pruneErrs, fmt.Errorf("failed to prune %s below %d: %w", store.Name(), pruneHeight, err)) } } - return nil + return pruneErrs } func describeAnswers(stores []PrunableStore, pruningBoundries []uint64) string { diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go index 10b0e7a04b..84a1f9eff7 100644 --- a/sei-db/management/gc/storage_garbage_collector_test.go +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -355,20 +355,25 @@ func TestPruneGetLastestBlockError(t *testing.T) { require.False(t, broken.pruneBelowCalled.Load()) } -func TestPruneBelowErrorStopsRemainingStores(t *testing.T) { - sentinel := errors.New("boom") +func TestPruneBelowErrorContinuesRemainingStores(t *testing.T) { + firstErr := errors.New("boom1") + secondErr := errors.New("boom2") first := snapshotStore("first", 100_000, 80_000) - first.pruneErr = sentinel + first.pruneErr = firstErr second := snapshotStore("second", 100_000, 80_000) + second.pruneErr = secondErr wal := contiguousStore("stateWAL", 100_000, true) err := prune(testConfig(t, 10_000), prunableStores(first, second, wal)) - require.ErrorIs(t, err, sentinel) + require.ErrorIs(t, err, firstErr) + require.ErrorIs(t, err, secondErr) require.ErrorContains(t, err, "first") + require.ErrorContains(t, err, "second") require.ErrorContains(t, err, "80000") require.True(t, first.pruneBelowCalled.Load()) - require.False(t, second.pruneBelowCalled.Load()) - require.False(t, wal.pruneBelowCalled.Load()) + require.True(t, second.pruneBelowCalled.Load()) + require.True(t, wal.pruneBelowCalled.Load()) + require.Equal(t, uint64(80_000), wal.prunedBelow.Load()) } func TestGetCutLine(t *testing.T) { From d2975596c10c9094f49fe2c2cf665215a04500c5 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Fri, 31 Jul 2026 16:20:27 -0700 Subject: [PATCH 13/15] Allow 0 rollbackWindow --- sei-db/management/gc/api.go | 4 +- .../gc/storage_garbage_collector.go | 28 ++++++------ .../gc/storage_garbage_collector_config.go | 12 ++--- .../gc/storage_garbage_collector_test.go | 45 ++++++++++++++----- 4 files changed, 55 insertions(+), 34 deletions(-) diff --git a/sei-db/management/gc/api.go b/sei-db/management/gc/api.go index ca5f839166..aaaad6fabb 100644 --- a/sei-db/management/gc/api.go +++ b/sei-db/management/gc/api.go @@ -36,7 +36,7 @@ type PrunableStore interface { // only drives SS snapshot pruning for the shared rollback window). GetRetentionWindow() int64 - // GetPruningBoundry returns the oldest block this store must keep to remain able to + // GetPruningBoundary returns the oldest block this store must keep to remain able to // serve cutLine, or 0 to opt out of this cycle (not participate in calculating min prune height). // // Contiguous stores can restore to any height they hold → return cutLine. @@ -47,7 +47,7 @@ type PrunableStore interface { // // Assumption: snapshot creation finishes quickly enough that an in-flight write need // not be reserved; modeling long-running snapshot writes is out of scope. - GetPruningBoundry(cutLine uint64) uint64 + GetPruningBoundary(cutLine uint64) uint64 // GetLastestBlock returns the highest block this store has ingested. // 0 means "no data / uninitialized" and is ignored when computing the global head. diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go index af738e8687..ab14e259f9 100644 --- a/sei-db/management/gc/storage_garbage_collector.go +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -23,11 +23,11 @@ var logger = seilog.NewLogger("db", "gc") // 1. head = min non-zero GetLastestBlock across stores // 2. per store: cutLine = head - RollbackWindow - GetRetentionWindow // (skipped when cutLine == 0: infinite retention, or head still inside the window) -// 3. ask GetPruningBoundry(cutLine); 0 = opt out for this cycle +// 3. ask GetPruningBoundary(cutLine); 0 = opt out for this cycle // 4. pruneHeight = min of non-zero answers; PruneBelow(pruneHeight) on every store // that answered non-zero // -// Step 4 uses the shared min (not each store's own boundry) so a snapshot remains +// Step 4 uses the shared min (not each store's own boundary) so a snapshot remains // restorable: contiguous stores must still hold the blocks that follow it. // PruneBelow failures are joined and do not skip later stores — permission-to-drop // is not transactional, and one unhealthy store must not block pruning of others. @@ -108,8 +108,8 @@ func prune(config *StorageGarbageCollectorConfig, stores []PrunableStore) error return nil } - // Boundries are positional so duplicate Name() values cannot mis-attribute an opt-out. - pruningBoundries := make([]uint64, len(stores)) + // Boundaries are positional so duplicate Name() values cannot mis-attribute an opt-out. + pruningBoundaries := make([]uint64, len(stores)) var pruneHeight uint64 for i, store := range stores { retention := store.GetRetentionWindow() @@ -119,19 +119,19 @@ func prune(config *StorageGarbageCollectorConfig, stores []PrunableStore) error continue } - pruningBoundries[i] = store.GetPruningBoundry(cutLine) - if pruningBoundries[i] == 0 { + pruningBoundaries[i] = store.GetPruningBoundary(cutLine) + if pruningBoundaries[i] == 0 { // Opt-out (e.g. no completed snapshot yet). continue } - if pruneHeight == 0 || pruningBoundries[i] < pruneHeight { - pruneHeight = pruningBoundries[i] + if pruneHeight == 0 || pruningBoundaries[i] < pruneHeight { + pruneHeight = pruningBoundaries[i] } } if pruneHeight == 0 { - logger.Info("skipping pruning: no store reported a pruning boundry", + logger.Info("skipping pruning: no store reported a pruning boundary", "globalLatestBlock", globalLatestBlock, - "pruningBoundryByStore", describeAnswers(stores, pruningBoundries), + "pruningBoundaryByStore", describeAnswers(stores, pruningBoundaries), ) return nil } @@ -139,11 +139,11 @@ func prune(config *StorageGarbageCollectorConfig, stores []PrunableStore) error logger.Info("pruning stores", "globalLatestBlock", globalLatestBlock, "pruneHeight", pruneHeight, - "pruningBoundryByStore", describeAnswers(stores, pruningBoundries), + "pruningBoundaryByStore", describeAnswers(stores, pruningBoundaries), ) var pruneErrs error for i, store := range stores { - if pruningBoundries[i] == 0 { + if pruningBoundaries[i] == 0 { continue } if err := store.PruneBelow(pruneHeight); err != nil { @@ -153,13 +153,13 @@ func prune(config *StorageGarbageCollectorConfig, stores []PrunableStore) error return pruneErrs } -func describeAnswers(stores []PrunableStore, pruningBoundries []uint64) string { +func describeAnswers(stores []PrunableStore, pruningBoundaries []uint64) string { var sb strings.Builder for i, store := range stores { if i > 0 { sb.WriteString(" ") } - fmt.Fprintf(&sb, "%s=%d", store.Name(), pruningBoundries[i]) + fmt.Fprintf(&sb, "%s=%d", store.Name(), pruningBoundaries[i]) } return sb.String() } diff --git a/sei-db/management/gc/storage_garbage_collector_config.go b/sei-db/management/gc/storage_garbage_collector_config.go index 829a3fc9b2..eb0a2bac51 100644 --- a/sei-db/management/gc/storage_garbage_collector_config.go +++ b/sei-db/management/gc/storage_garbage_collector_config.go @@ -11,9 +11,12 @@ type StorageGarbageCollectorConfig struct { // roll back to. Shared by every managed store so they stay mutually consistent; // per-store extras beyond this window use PrunableStore.GetRetentionWindow. // - // Must be > 0. With RollbackWindow == 0, cutLine can equal head: contiguous stores - // may drop everything below head, and a store answering 0 (no snapshot yet) offers - // no counter-vote — a snapshot that then lands near head cannot be replayed forward. + // 0 is allowed: cutLine then equals head for any store with retention 0. That is + // safe only when every participating store still leaves enough history on its own + // (e.g. positive GetRetentionWindow, or snapshot votes that keep contiguous stores + // above genesis). With retention-0 contiguous stores and a snapshot store that + // answers 0 (no snapshot yet), pruneHeight can reach head and drop everything + // below it — a snapshot landing near head then cannot be replayed forward. // // After a rollback that consumes part of the window, full headroom is not promised // (e.g. window 10_000, roll back 5_000 → only ~5_000 of headroom remain). @@ -36,9 +39,6 @@ func (c *StorageGarbageCollectorConfig) Validate() error { if c == nil { return fmt.Errorf("config is required") } - if c.RollbackWindow == 0 { - return fmt.Errorf("rollback window must be greater than 0") - } if c.PruneInterval <= 0 { return fmt.Errorf("prune interval must be greater than 0") } diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go index 84a1f9eff7..d7be1cf3e8 100644 --- a/sei-db/management/gc/storage_garbage_collector_test.go +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -16,7 +16,7 @@ type mockStore struct { latestHeight uint64 // retentionWindow is returned by GetRetentionWindow (0 = shared RollbackWindow only). retentionWindow int64 - pruningBoundry func(cutLine uint64) uint64 + pruningBoundary func(cutLine uint64) uint64 getErr error pruneErr error @@ -24,7 +24,7 @@ type mockStore struct { prunedBelow atomic.Uint64 } -// snapshotStore models SC/SS (retention 0). GetPruningBoundry returns the newest snapshot +// snapshotStore models SC/SS (retention 0). GetPruningBoundary returns the newest snapshot // ≤ cutLine, or the oldest snapshot if all are above cutLine. Empty snapshots → 0. // Snapshots must be ascending. func snapshotStore(name string, latestHeight uint64, snapshots ...uint64) *mockStore { @@ -32,7 +32,7 @@ func snapshotStore(name string, latestHeight uint64, snapshots ...uint64) *mockS name: name, latestHeight: latestHeight, retentionWindow: 0, - pruningBoundry: func(cutLine uint64) uint64 { + pruningBoundary: func(cutLine uint64) uint64 { if len(snapshots) == 0 { return 0 } @@ -49,14 +49,14 @@ func snapshotStore(name string, latestHeight uint64, snapshots ...uint64) *mockS } // contiguousStore models blockDB / receiptDB / WAL (retention 0 by default). -// hasData=false → GetPruningBoundry returns 0 (opt out). Otherwise returns cutLine. +// hasData=false → GetPruningBoundary returns 0 (opt out). Otherwise returns cutLine. // Use withRetentionWindow for extras or InfiniteRetentionWindow. func contiguousStore(name string, latestHeight uint64, hasData bool) *mockStore { return &mockStore{ name: name, latestHeight: latestHeight, retentionWindow: 0, - pruningBoundry: func(cutLine uint64) uint64 { + pruningBoundary: func(cutLine uint64) uint64 { if !hasData { return 0 } @@ -82,8 +82,8 @@ func (m *mockStore) GetLastestBlock() (uint64, error) { return m.latestHeight, m.getErr } -func (m *mockStore) GetPruningBoundry(cutLine uint64) uint64 { - return m.pruningBoundry(cutLine) +func (m *mockStore) GetPruningBoundary(cutLine uint64) uint64 { + return m.pruningBoundary(cutLine) } func (m *mockStore) PruneBelow(blockNumber uint64) error { @@ -119,7 +119,7 @@ func TestPruneDecisions(t *testing.T) { wantPruneBelow *uint64 }{ { - name: "SC and WAL both retention 0: min boundry wins", + name: "SC and WAL both retention 0: min boundary wins", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 80_000, 85_000, 92_000), @@ -129,7 +129,7 @@ func TestPruneDecisions(t *testing.T) { wantPruneBelow: ptr(85_000), }, { - name: "lowest boundry across many stores wins", + name: "lowest boundary across many stores wins", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 80_000, 85_000, 92_000), @@ -149,6 +149,25 @@ func TestPruneDecisions(t *testing.T) { }, wantPruneBelow: ptr(99_999), }, + { + name: "RollbackWindow 0: cutLine equals head", + rollbackWindow: 0, + stores: []*mockStore{ + snapshotStore("sc", 100_000, 80_000, 90_000), + contiguousStore("stateWAL", 100_000, true), + }, + // cutLine 100_000; sc 90_000; WAL 100_000 → min 90_000. + wantPruneBelow: ptr(90_000), + }, + { + name: "RollbackWindow 0 with no snapshot vote prunes contiguous to head", + rollbackWindow: 0, + stores: []*mockStore{ + snapshotStore("sc", 100_000), + contiguousStore("stateWAL", 100_000, true), + }, + wantPruneBelow: ptr(100_000), + }, { name: "positive contiguous retention deepens that store's cut line", rollbackWindow: 10_000, @@ -302,7 +321,7 @@ func TestPruneDecisions(t *testing.T) { shouldPrune := false if tc.wantPruneBelow != nil && store.GetRetentionWindow() >= 0 { cutLine := getCutLine(head, tc.rollbackWindow, store.GetRetentionWindow()) - if cutLine > 0 && store.GetPruningBoundry(cutLine) != 0 { + if cutLine > 0 && store.GetPruningBoundary(cutLine) != 0 { shouldPrune = true } } @@ -385,6 +404,8 @@ func TestGetCutLine(t *testing.T) { want uint64 }{ {name: "rollback window only", head: 100_000, rollbackWindow: 10_000, want: 90_000}, + {name: "zero rollback window", head: 100_000, rollbackWindow: 0, want: 100_000}, + {name: "zero rollback with retention", head: 100_000, rollbackWindow: 0, retention: 10_000, want: 90_000}, {name: "retention adds to rollback window", head: 100_000, rollbackWindow: 1, retention: 10_000, want: 89_999}, {name: "the two windows add", head: 100_000, rollbackWindow: 10_000, retention: 5_000, want: 85_000}, {name: "zero retention", head: 100_000, rollbackWindow: 10_000, want: 90_000}, @@ -449,10 +470,10 @@ func TestDefaultStorageGarbageCollectorConfig(t *testing.T) { func TestValidate(t *testing.T) { require.ErrorContains(t, (*StorageGarbageCollectorConfig)(nil).Validate(), "config is required") - require.ErrorContains(t, (&StorageGarbageCollectorConfig{ + require.NoError(t, (&StorageGarbageCollectorConfig{ RollbackWindow: 0, PruneInterval: time.Minute, - }).Validate(), "rollback window must be greater than 0") + }).Validate()) require.NoError(t, (&StorageGarbageCollectorConfig{ RollbackWindow: 1, From ef7d7fe7cb5e87b26f6ece59a010c5b60c6a614f Mon Sep 17 00:00:00 2001 From: YimingZang Date: Fri, 31 Jul 2026 16:48:51 -0700 Subject: [PATCH 14/15] Address comments --- sei-db/management/gc/api.go | 11 +++++-- .../gc/storage_garbage_collector.go | 30 ++++++++++++------- .../gc/storage_garbage_collector_test.go | 13 ++++---- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/sei-db/management/gc/api.go b/sei-db/management/gc/api.go index aaaad6fabb..a4d8a8ecaf 100644 --- a/sei-db/management/gc/api.go +++ b/sei-db/management/gc/api.go @@ -20,7 +20,7 @@ type PrunableStore interface { PruneBelow(blockNumber uint64) error // GetRetentionWindow is how many extra blocks beyond the shared RollbackWindow - // this store needs to keep servable. Cut line (head = min non-zero GetLastestBlock): + // this store needs to keep servable. Cut line (head = min non-zero GetLatestBlock): // // retention < 0 → store ignored (cutLine treated as 0; conventionally -1) // retention >= 0 → cutLine = head - RollbackWindow - retention @@ -29,6 +29,11 @@ type PrunableStore interface { // Retention is optional history on top — kept so the store can still serve queries // after a rollback that consumes the entire rollback window. // + // Because the collector prunes every participating store to the *lowest* + // GetPruningBoundary, a large retention on one store effectively deepens pruning + // for the others (see StorageGarbageCollector). Read this as an input to that + // shared min, not as an independently applied per-store policy. + // // Contract by store kind: // - Contiguous (blockDB, receiptDB, state WAL): -1 / 0 / positive as configured. // - SC: always 0 (only needs the shared rollback window for snapshots). @@ -49,7 +54,7 @@ type PrunableStore interface { // not be reserved; modeling long-running snapshot writes is out of scope. GetPruningBoundary(cutLine uint64) uint64 - // GetLastestBlock returns the highest block this store has ingested. + // GetLatestBlock returns the highest block this store has ingested. // 0 means "no data / uninitialized" and is ignored when computing the global head. - GetLastestBlock() (uint64, error) + GetLatestBlock() (uint64, error) } diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go index ab14e259f9..ee15d75e40 100644 --- a/sei-db/management/gc/storage_garbage_collector.go +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -20,15 +20,21 @@ var logger = seilog.NewLogger("db", "gc") // their snapshot pruning for the shared window; SS version-history pruning is separate. // // Each cycle: -// 1. head = min non-zero GetLastestBlock across stores +// 1. head = min non-zero GetLatestBlock across stores // 2. per store: cutLine = head - RollbackWindow - GetRetentionWindow // (skipped when cutLine == 0: infinite retention, or head still inside the window) // 3. ask GetPruningBoundary(cutLine); 0 = opt out for this cycle // 4. pruneHeight = min of non-zero answers; PruneBelow(pruneHeight) on every store // that answered non-zero // -// Step 4 uses the shared min (not each store's own boundary) so a snapshot remains -// restorable: contiguous stores must still hold the blocks that follow it. +// Step 4 uses the shared min (not each store's own boundary) so a retained snapshot +// stays restorable: contiguous stores must still hold the blocks that follow it. +// A side effect is that one store's deep GetRetentionWindow is imposed on every +// participating store — e.g. receiptDB retention 100_000 also keeps SC/SS snapshots +// that far back even though those stores report retention 0. That uniform prune is +// an intentional tradeoff for Giga (one effective retention across stores), not +// independent per-store retention. +// // PruneBelow failures are joined and do not skip later stores — permission-to-drop // is not transactional, and one unhealthy store must not block pruning of others. // @@ -99,7 +105,7 @@ func prune(config *StorageGarbageCollectorConfig, stores []PrunableStore) error return nil } - globalLatestBlock, err := getGlobalLastestBlock(stores) + globalLatestBlock, err := getGlobalLatestBlock(stores) if err != nil { return err } @@ -165,29 +171,33 @@ func describeAnswers(stores []PrunableStore, pruningBoundaries []uint64) string } // getCutLine returns head - RollbackWindow - retention for retention >= 0. -// Returns 0 when retention < 0, or when head is still inside that combined window -// (unsigned subtraction must not wrap). +// Returns 0 when retention < 0, when the combined window overflows uint64, or when +// head is still inside that window (unsigned subtraction must not wrap). func getCutLine(globalLatestBlock uint64, rollbackWindow uint64, retention int64) uint64 { if retention < 0 { return 0 } totalRetainWindow := rollbackWindow + uint64(retention) + if totalRetainWindow < rollbackWindow { + // Addition wrapped; treat as an unsatisfiable retain window (skip pruning). + return 0 + } if globalLatestBlock <= totalRetainWindow { return 0 } return globalLatestBlock - totalRetainWindow } -// getGlobalLastestBlock returns the smallest non-zero GetLastestBlock among stores. +// getGlobalLatestBlock returns the smallest non-zero GetLatestBlock among stores. // Using the min keeps a lagging store from being pruned past blocks it still needs. // Heads of 0 are skipped so an uninitialized store does not stall pruning. // Returns 0 when no store has data. -func getGlobalLastestBlock(stores []PrunableStore) (uint64, error) { +func getGlobalLatestBlock(stores []PrunableStore) (uint64, error) { var blockNum uint64 for _, store := range stores { - storeHeight, err := store.GetLastestBlock() + storeHeight, err := store.GetLatestBlock() if err != nil { - return 0, fmt.Errorf("failed to read lastest block from %s: %w", store.Name(), err) + return 0, fmt.Errorf("failed to read latest block from %s: %w", store.Name(), err) } if storeHeight == 0 { continue diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go index d7be1cf3e8..ce5de57d58 100644 --- a/sei-db/management/gc/storage_garbage_collector_test.go +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -3,6 +3,7 @@ package gc import ( "context" "errors" + "math" "sync/atomic" "testing" "time" @@ -78,7 +79,7 @@ func (m *mockStore) GetRetentionWindow() int64 { return m.retentionWindow } -func (m *mockStore) GetLastestBlock() (uint64, error) { +func (m *mockStore) GetLatestBlock() (uint64, error) { return m.latestHeight, m.getErr } @@ -314,7 +315,7 @@ func TestPruneDecisions(t *testing.T) { stores := prunableStores(tc.stores...) require.NoError(t, prune(testConfig(t, tc.rollbackWindow), stores)) - head, err := getGlobalLastestBlock(stores) + head, err := getGlobalLatestBlock(stores) require.NoError(t, err) for _, store := range tc.stores { @@ -361,7 +362,7 @@ func TestPruneKeepsOptedOutStoreWithDuplicateName(t *testing.T) { require.Equal(t, uint64(80_000), wal.prunedBelow.Load()) } -func TestPruneGetLastestBlockError(t *testing.T) { +func TestPruneGetLatestBlockError(t *testing.T) { sentinel := errors.New("boom") sc := snapshotStore("sc", 100_000, 80_000) broken := contiguousStore("brokenStore", 100_000, true) @@ -416,6 +417,8 @@ func TestGetCutLine(t *testing.T) { {name: "head one below the window", head: 9_999, rollbackWindow: 10_000, want: 0}, {name: "head far below the window", head: 100, rollbackWindow: 1_000, retention: 100_000, want: 0}, {name: "head at genesis", head: 0, rollbackWindow: 10_000, want: 0}, + {name: "rollback plus retention overflows uint64", head: math.MaxUint64, rollbackWindow: math.MaxUint64, retention: 1, want: 0}, + {name: "max rollback alone does not overflow", head: math.MaxUint64, rollbackWindow: math.MaxUint64, want: 0}, } for _, tc := range cases { @@ -425,7 +428,7 @@ func TestGetCutLine(t *testing.T) { } } -func TestGetGlobalLastestBlock(t *testing.T) { +func TestGetGlobalLatestBlock(t *testing.T) { storesWithHeads := func(heads ...uint64) []PrunableStore { list := make([]*mockStore, len(heads)) for i, head := range heads { @@ -453,7 +456,7 @@ func TestGetGlobalLastestBlock(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - head, err := getGlobalLastestBlock(storesWithHeads(tc.heads...)) + head, err := getGlobalLatestBlock(storesWithHeads(tc.heads...)) require.NoError(t, err) require.Equal(t, tc.want, head) }) From 6cfcc2c1933636a1f0875a3657a3755356329d67 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Fri, 31 Jul 2026 17:53:01 -0700 Subject: [PATCH 15/15] Address comments --- sei-db/management/gc/api.go | 6 +- .../gc/storage_garbage_collector.go | 17 +++- .../gc/storage_garbage_collector_test.go | 84 ++++++++++++++++--- 3 files changed, 92 insertions(+), 15 deletions(-) diff --git a/sei-db/management/gc/api.go b/sei-db/management/gc/api.go index a4d8a8ecaf..86ea4a933a 100644 --- a/sei-db/management/gc/api.go +++ b/sei-db/management/gc/api.go @@ -55,6 +55,10 @@ type PrunableStore interface { GetPruningBoundary(cutLine uint64) uint64 // GetLatestBlock returns the highest block this store has ingested. - // 0 means "no data / uninitialized" and is ignored when computing the global head. + // + // 0 means "nothing ingested yet" and is ignored when computing the global head, so a store + // still filling does not stall pruning for the rest of the fleet. It does not mean + // "disabled": a store disabled for this node is not instantiated and never reaches the + // collector. See StorageGarbageCollector for why skipping a 0 head is safe. GetLatestBlock() (uint64, error) } diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go index ee15d75e40..74463dc738 100644 --- a/sei-db/management/gc/storage_garbage_collector.go +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -38,6 +38,18 @@ var logger = seilog.NewLogger("db", "gc") // PruneBelow failures are joined and do not skip later stores — permission-to-drop // is not transactional, and one unhealthy store must not block pruning of others. // +// Precondition on stores: every store passed in is live. A store disabled for this node is +// not instantiated and so never reaches the collector, so the set holds no permanently-empty +// member. A store that is present but has ingested nothing reports head 0; that head is +// ignored when computing the global head and the store opts out via GetPruningBoundary, so +// one store still filling does not stall pruning fleet-wide. +// +// That skip is only safe while no store bootstraps from another store's history: today each +// replays its own changelog and owns that changelog's retention. If a store is ever fed from +// another store's WAL, it could sit at head 0 while that WAL is pruned past what it needs, +// and this rule has to change (distinguish "empty and still bootstrapping" from "not +// participating" rather than skipping both). +// // Invariant: if RollbackWindow blocks of rollback were possible before a prune, that // prune does not take the ability away. Full headroom is not promised after a rollback // has already consumed part of the window. @@ -190,8 +202,11 @@ func getCutLine(globalLatestBlock uint64, rollbackWindow uint64, retention int64 // getGlobalLatestBlock returns the smallest non-zero GetLatestBlock among stores. // Using the min keeps a lagging store from being pruned past blocks it still needs. -// Heads of 0 are skipped so an uninitialized store does not stall pruning. // Returns 0 when no store has data. +// +// Heads of 0 are skipped rather than treated as "unknown". Note this excludes from the min +// exactly the store that holds nothing, so the min-head argument does not protect that +// store; what protects it is the store-set precondition on StorageGarbageCollector. func getGlobalLatestBlock(stores []PrunableStore) (uint64, error) { var blockNum uint64 for _, store := range stores { diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go index ce5de57d58..8bfe42d327 100644 --- a/sei-db/management/gc/storage_garbage_collector_test.go +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -23,6 +23,7 @@ type mockStore struct { pruneBelowCalled atomic.Bool prunedBelow atomic.Uint64 + pruneBelowCalls atomic.Uint64 } // snapshotStore models SC/SS (retention 0). GetPruningBoundary returns the newest snapshot @@ -90,6 +91,7 @@ func (m *mockStore) GetPruningBoundary(cutLine uint64) uint64 { func (m *mockStore) PruneBelow(blockNumber uint64) error { m.pruneBelowCalled.Store(true) m.prunedBelow.Store(blockNumber) + m.pruneBelowCalls.Add(1) return m.pruneErr } @@ -112,12 +114,16 @@ func testConfig(t *testing.T, rollbackWindow uint64) *StorageGarbageCollectorCon } // TestPruneDecisions covers one prune cycle. wantPruneBelow == nil means no store is pruned. +// +// wantPruned is hardcoded per store rather than derived from getCutLine / GetPruningBoundary: deriving it from the +// helpers under test would let a sign error or off-by-one shift the expectation in lockstep with the bug. func TestPruneDecisions(t *testing.T) { cases := []struct { name string rollbackWindow uint64 stores []*mockStore wantPruneBelow *uint64 + wantPruned []bool }{ { name: "SC and WAL both retention 0: min boundary wins", @@ -128,6 +134,7 @@ func TestPruneDecisions(t *testing.T) { }, // cutLine 90_000; sc keeps snapshot 85_000; WAL answers 90_000 → min 85_000. wantPruneBelow: ptr(85_000), + wantPruned: []bool{true, true}, }, { name: "lowest boundary across many stores wins", @@ -140,6 +147,7 @@ func TestPruneDecisions(t *testing.T) { }, // sc 85_000, ss 88_000, flatKV 90_000, WAL 90_000 → min 85_000. wantPruneBelow: ptr(85_000), + wantPruned: []bool{true, true, true, true}, }, { name: "RollbackWindow of 1 still leaves one block of margin", @@ -149,6 +157,7 @@ func TestPruneDecisions(t *testing.T) { contiguousStore("stateWAL", 100_000, true), }, wantPruneBelow: ptr(99_999), + wantPruned: []bool{true, true}, }, { name: "RollbackWindow 0: cutLine equals head", @@ -159,6 +168,7 @@ func TestPruneDecisions(t *testing.T) { }, // cutLine 100_000; sc 90_000; WAL 100_000 → min 90_000. wantPruneBelow: ptr(90_000), + wantPruned: []bool{true, true}, }, { name: "RollbackWindow 0 with no snapshot vote prunes contiguous to head", @@ -168,6 +178,7 @@ func TestPruneDecisions(t *testing.T) { contiguousStore("stateWAL", 100_000, true), }, wantPruneBelow: ptr(100_000), + wantPruned: []bool{false, true}, }, { name: "positive contiguous retention deepens that store's cut line", @@ -178,6 +189,7 @@ func TestPruneDecisions(t *testing.T) { }, // sc cutLine 90_000 → 90_000; WAL cutLine 85_000 → 85_000 → min 85_000. wantPruneBelow: ptr(85_000), + wantPruned: []bool{true, true}, }, { name: "SS retention 0 behaves like SC", @@ -187,6 +199,7 @@ func TestPruneDecisions(t *testing.T) { contiguousStore("stateWAL", 100_000, true), }, wantPruneBelow: ptr(90_000), + wantPruned: []bool{true, true}, }, { name: "infinite retention on every store skips pruning", @@ -196,6 +209,7 @@ func TestPruneDecisions(t *testing.T) { withRetentionWindow(contiguousStore("receiptDB", 100_000, true), InfiniteRetentionWindow), }, wantPruneBelow: nil, + wantPruned: []bool{false, false}, }, { name: "infinite retention on one store leaves others free to prune", @@ -207,6 +221,7 @@ func TestPruneDecisions(t *testing.T) { }, // archiveWAL skipped; sc 80_000; stateWAL 90_000 → min 80_000. wantPruneBelow: ptr(80_000), + wantPruned: []bool{false, true, true}, }, { name: "snapshot exactly at the cut line is kept", @@ -216,6 +231,7 @@ func TestPruneDecisions(t *testing.T) { contiguousStore("stateWAL", 100_000, true), }, wantPruneBelow: ptr(90_000), + wantPruned: []bool{true, true}, }, { name: "all snapshots above cut line: store still votes its oldest", @@ -226,6 +242,7 @@ func TestPruneDecisions(t *testing.T) { }, // sc answers 95_000; WAL answers 90_000 → shared prune 90_000 (sc snapshots untouched). wantPruneBelow: ptr(90_000), + wantPruned: []bool{true, true}, }, { name: "lagging store lowers the global head", @@ -236,6 +253,7 @@ func TestPruneDecisions(t *testing.T) { }, // head 50_000 → cutLine 40_000; lagging answers 30_000; WAL 40_000 → min 30_000. wantPruneBelow: ptr(30_000), + wantPruned: []bool{true, true}, }, { name: "store answering 0 is ignored", @@ -246,6 +264,7 @@ func TestPruneDecisions(t *testing.T) { contiguousStore("stateWAL", 100_000, true), }, wantPruneBelow: ptr(80_000), + wantPruned: []bool{false, true, true}, }, { name: "store with no snapshot yet is ignored", @@ -255,6 +274,7 @@ func TestPruneDecisions(t *testing.T) { contiguousStore("stateWAL", 100_000, true), }, wantPruneBelow: ptr(90_000), + wantPruned: []bool{false, true}, }, { name: "zero head ignored for global head; store still votes", @@ -265,6 +285,7 @@ func TestPruneDecisions(t *testing.T) { }, // head from WAL 100_000; stalled still answers 50_000 → min 50_000. wantPruneBelow: ptr(50_000), + wantPruned: []bool{true, true}, }, { name: "head inside retain window: no prune", @@ -274,6 +295,7 @@ func TestPruneDecisions(t *testing.T) { contiguousStore("stateWAL", 5_000, true), }, wantPruneBelow: nil, + wantPruned: []bool{false, false}, }, { name: "head inside one store's window skips only that store", @@ -284,6 +306,7 @@ func TestPruneDecisions(t *testing.T) { }, // WAL cutLine 0 (skipped); sc cutLine 40_000 → 1_000. wantPruneBelow: ptr(1_000), + wantPruned: []bool{true, false}, }, { name: "no store has a latest block", @@ -293,6 +316,7 @@ func TestPruneDecisions(t *testing.T) { contiguousStore("stateWAL", 0, false), }, wantPruneBelow: nil, + wantPruned: []bool{false, false}, }, { name: "only an opt-out store has data", @@ -301,35 +325,29 @@ func TestPruneDecisions(t *testing.T) { contiguousStore("optOut", 100_000, false), }, wantPruneBelow: nil, + wantPruned: []bool{false}, }, { name: "no stores at all", rollbackWindow: 10_000, stores: nil, wantPruneBelow: nil, + wantPruned: nil, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - stores := prunableStores(tc.stores...) - require.NoError(t, prune(testConfig(t, tc.rollbackWindow), stores)) + require.Len(t, tc.wantPruned, len(tc.stores), "wantPruned must have one entry per store") - head, err := getGlobalLatestBlock(stores) - require.NoError(t, err) + require.NoError(t, prune(testConfig(t, tc.rollbackWindow), prunableStores(tc.stores...))) - for _, store := range tc.stores { - shouldPrune := false - if tc.wantPruneBelow != nil && store.GetRetentionWindow() >= 0 { - cutLine := getCutLine(head, tc.rollbackWindow, store.GetRetentionWindow()) - if cutLine > 0 && store.GetPruningBoundary(cutLine) != 0 { - shouldPrune = true - } - } - if !shouldPrune { + for i, store := range tc.stores { + if !tc.wantPruned[i] { require.Falsef(t, store.pruneBelowCalled.Load(), "%s should not be pruned", store.name) continue } + require.NotNil(t, tc.wantPruneBelow, "wantPruned expects a prune height") require.Truef(t, store.pruneBelowCalled.Load(), "%s should be pruned", store.name) require.Equalf(t, *tc.wantPruneBelow, store.prunedBelow.Load(), "%s prune height", store.name) } @@ -530,6 +548,46 @@ func TestCloseAfterContextCancelled(t *testing.T) { require.NoError(t, sm.Close()) } +// startCollector runs a real collector on a short PruneInterval so the ticker branch in run() is exercised. +func startCollector(t *testing.T, interval time.Duration, stores ...*mockStore) { + t.Helper() + + config := &StorageGarbageCollectorConfig{RollbackWindow: 10_000, PruneInterval: interval} + require.NoError(t, config.Validate()) + + sm, err := NewStorageGarbageCollector(context.Background(), config, prunableStores(stores...)) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, sm.Close()) }) +} + +// TestRunTickerDrivesPruneCycles covers run()'s ticker branch: a cycle fires on its own and prunes to the expected +// height. +func TestRunTickerDrivesPruneCycles(t *testing.T) { + sc := snapshotStore("sc", 100_000, 80_000) + wal := contiguousStore("stateWAL", 100_000, true) + + startCollector(t, 10*time.Millisecond, sc, wal) + + require.Eventually(t, func() bool { + return sc.pruneBelowCalled.Load() && wal.pruneBelowCalled.Load() + }, 2*time.Second, 5*time.Millisecond, "ticker should drive a prune cycle") + require.Equal(t, uint64(80_000), sc.prunedBelow.Load()) + require.Equal(t, uint64(80_000), wal.prunedBelow.Load()) +} + +// TestRunSurvivesPruneError covers run()'s logger.Error branch: a PruneBelow failure is logged and the loop keeps +// ticking rather than exiting after the first error. +func TestRunSurvivesPruneError(t *testing.T) { + broken := snapshotStore("broken", 100_000, 80_000) + broken.pruneErr = errors.New("boom") + + startCollector(t, 10*time.Millisecond, broken) + + require.Eventually(t, func() bool { + return broken.pruneBelowCalls.Load() > 1 + }, 2*time.Second, 5*time.Millisecond, "a failed prune must not kill the run loop") +} + func ptr(v uint64) *uint64 { return &v }