From c1bf7418a17a84712484f3e3ef77a75831da6457 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 13 Jul 2026 16:38:46 -0700 Subject: [PATCH] Trust restore stamp when source is unresolvable Snapshot providers rotate old objects out of their buckets. A failed HTTP resolve produced an empty snapshot identity that invalidated an otherwise-valid stamp, sending the restore path into the non-empty target error on every pod restart. - Skip the restore when a stamp matches the same source, target, checksum, and compression but the source can no longer be resolved. - Stop comparing ToolVersion in StampMatches so image upgrades don't invalidate existing stamps. Co-Authored-By: Claude Fable 5 --- cmd/stream-download/main.go | 17 ++++++++ cmd/stream-download/main_test.go | 75 ++++++++++++++++++++++++++++++++ internal/restore/restore.go | 15 ++++++- 3 files changed, 105 insertions(+), 2 deletions(-) diff --git a/cmd/stream-download/main.go b/cmd/stream-download/main.go index c9c57bf..0344c08 100644 --- a/cmd/stream-download/main.go +++ b/cmd/stream-download/main.go @@ -78,6 +78,23 @@ func run(env map[string]string, stdout, stderr io.Writer) error { logger.Info("restore_already_complete", logx.Fields{"target": target}) return nil } + // Snapshot providers rotate old objects out of their buckets. If the source + // can no longer be resolved, a prior completed restore of the same source is + // still valid — trust the stamp instead of demanding a wipe. + if stamp.SnapshotID == "" { + if prior, ok := restore.ReadStamp(stampPath); ok { + prior.SnapshotID, prior.ToolVersion = "", "" + want := stamp + want.ToolVersion = "" + if prior == want { + logger.Info("restore_already_complete", logx.Fields{ + "target": target, + "message": "source no longer resolvable; trusting existing restore stamp", + }) + return nil + } + } + } if err := restore.CleanupInterruptedPublish(target); err != nil { return err } diff --git a/cmd/stream-download/main_test.go b/cmd/stream-download/main_test.go index b89cc70..0bbe404 100644 --- a/cmd/stream-download/main_test.go +++ b/cmd/stream-download/main_test.go @@ -609,3 +609,78 @@ func gzipTar(t *testing.T, name, contents string) []byte { _ = gw.Close() return gz.Bytes() } + +func TestRunTrustsStampWhenSourceRotated(t *testing.T) { + archive := gzipTar(t, "chaindata/file.txt", "hello") + gone := false + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if gone { + http.NotFound(w, r) + return + } + w.Header().Set("ETag", `"snap"`) + if r.Method == http.MethodHead { + return + } + _, _ = w.Write(archive) + })) + defer srv.Close() + + dir := filepath.Join(t.TempDir(), "data") + env := map[string]string{ + "RESTORE_SNAPSHOT": "true", + "DIR": dir, + "SCRATCH_DIR": filepath.Join(t.TempDir(), "scratch"), + "SNAPSHOT_URL": srv.URL + "/snapshot.tar.gz", + "COMPRESSION": "gzip", + "REQUIRE_MOUNTPOINT": "false", + } + if err := run(env, os.Stdout, os.Stderr); err != nil { + t.Fatalf("first run error: %v", err) + } + stamp, ok := restore.ReadStamp(filepath.Join(dir, restore.StampFileName)) + if !ok || stamp.SnapshotID == "" { + t.Fatalf("first run stamp = %+v, want resolved snapshot id", stamp) + } + + gone = true + if err := run(env, os.Stdout, os.Stderr); err != nil { + t.Fatalf("second run error: %v", err) + } + got, err := os.ReadFile(filepath.Join(dir, "chaindata/file.txt")) + if err != nil || string(got) != "hello" { + t.Fatalf("restored data disturbed: %q, %v", got, err) + } +} + +func TestRunSkipsRestoreAcrossToolVersions(t *testing.T) { + archive := gzipTar(t, "chaindata/file.txt", "hello") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", `"snap"`) + if r.Method == http.MethodHead { + return + } + _, _ = w.Write(archive) + })) + defer srv.Close() + + dir := filepath.Join(t.TempDir(), "data") + env := map[string]string{ + "RESTORE_SNAPSHOT": "true", + "DIR": dir, + "SCRATCH_DIR": filepath.Join(t.TempDir(), "scratch"), + "SNAPSHOT_URL": srv.URL + "/snapshot.tar.gz", + "COMPRESSION": "gzip", + "REQUIRE_MOUNTPOINT": "false", + } + if err := run(env, os.Stdout, os.Stderr); err != nil { + t.Fatalf("first run error: %v", err) + } + + prev := toolVersion + toolVersion = prev + "-next" + defer func() { toolVersion = prev }() + if err := run(env, os.Stdout, os.Stderr); err != nil { + t.Fatalf("run after version bump error: %v", err) + } +} diff --git a/internal/restore/restore.go b/internal/restore/restore.go index 186a72f..3e42ef6 100644 --- a/internal/restore/restore.go +++ b/internal/restore/restore.go @@ -83,15 +83,26 @@ func CleanupInterruptedPublish(target string) error { return ClearPublishMarker(target) } -func StampMatches(path string, want Stamp) bool { +func ReadStamp(path string) (Stamp, bool) { data, err := os.ReadFile(path) if err != nil { - return false + return Stamp{}, false } var got Stamp if err := json.Unmarshal(data, &got); err != nil { + return Stamp{}, false + } + return got, true +} + +func StampMatches(path string, want Stamp) bool { + got, ok := ReadStamp(path) + if !ok { return false } + // The restored data does not depend on which tool version wrote it. + got.ToolVersion = "" + want.ToolVersion = "" return got == want }