From 887a1aad54487bc4d744104869af5d29163487a7 Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 19 Jun 2026 12:11:16 +0200 Subject: [PATCH 1/3] fix(cli): chunk storage delete requests --- apps/cli-go/internal/storage/rm/rm.go | 17 +++- apps/cli-go/internal/storage/rm/rm_test.go | 110 +++++++++++++++++++++ apps/cli-go/pkg/storage/api.go | 2 + 3 files changed, 127 insertions(+), 2 deletions(-) diff --git a/apps/cli-go/internal/storage/rm/rm.go b/apps/cli-go/internal/storage/rm/rm.go index 56e55a7974..8254f93164 100644 --- a/apps/cli-go/internal/storage/rm/rm.go +++ b/apps/cli-go/internal/storage/rm/rm.go @@ -71,7 +71,7 @@ func Run(ctx context.Context, paths []string, recursive bool, fsys afero.Fs) err } // Always try deleting first in case the paths resolve to extensionless files fmt.Fprintln(os.Stderr, "Deleting objects:", prefixes) - removed, err := api.DeleteObjects(ctx, bucket, prefixes) + removed, err := deleteObjects(ctx, api, bucket, prefixes) if err != nil { return err } @@ -124,7 +124,7 @@ func RemoveStoragePathAll(ctx context.Context, api storage.StorageAPI, bucket, p } if len(files) > 0 { fmt.Fprintln(os.Stderr, "Deleting objects:", files) - if _, err := api.DeleteObjects(ctx, bucket, files); err != nil { + if _, err := deleteObjects(ctx, api, bucket, files); err != nil { return err } } @@ -141,3 +141,16 @@ func RemoveStoragePathAll(ctx context.Context, api storage.StorageAPI, bucket, p } return nil } + +func deleteObjects(ctx context.Context, api storage.StorageAPI, bucket string, prefixes []string) ([]storage.DeleteObjectsResponse, error) { + var removed []storage.DeleteObjectsResponse + for start := 0; start < len(prefixes); start += storage.DELETE_OBJECTS_LIMIT { + end := min(start+storage.DELETE_OBJECTS_LIMIT, len(prefixes)) + objects, err := api.DeleteObjects(ctx, bucket, prefixes[start:end]) + if err != nil { + return nil, err + } + removed = append(removed, objects...) + } + return removed, nil +} diff --git a/apps/cli-go/internal/storage/rm/rm_test.go b/apps/cli-go/internal/storage/rm/rm_test.go index cc02418cfc..6cc1e5ff01 100644 --- a/apps/cli-go/internal/storage/rm/rm_test.go +++ b/apps/cli-go/internal/storage/rm/rm_test.go @@ -2,6 +2,7 @@ package rm import ( "context" + "fmt" "net/http" "testing" @@ -113,6 +114,35 @@ func TestStorageRM(t *testing.T) { assert.Empty(t, apitest.ListUnmatchedRequests()) }) + t.Run("chunks explicit object deletes by storage api cap", func(t *testing.T) { + t.Cleanup(fstest.MockStdin(t, "y")) + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + flags.ProjectRef + "/api-keys"). + Reply(http.StatusOK). + JSON(apiKeys) + prefixes := numberedStorageFiles(1001) + gock.New("https://" + utils.GetSupabaseHost(flags.ProjectRef)). + Delete("/storage/v1/object/private"). + JSON(storage.DeleteObjectsRequest{Prefixes: prefixes[:1000]}). + Reply(http.StatusOK). + JSON(deleteObjectsResponse("private", prefixes[:1000])) + gock.New("https://" + utils.GetSupabaseHost(flags.ProjectRef)). + Delete("/storage/v1/object/private"). + JSON(storage.DeleteObjectsRequest{Prefixes: prefixes[1000:]}). + Reply(http.StatusOK). + JSON(deleteObjectsResponse("private", prefixes[1000:])) + // Run test + paths := storageURLs("private", prefixes) + err := Run(context.Background(), paths, false, fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + t.Run("removes buckets and directories", func(t *testing.T) { t.Cleanup(fstest.MockStdin(t, "y")) // Setup in-memory fs @@ -262,6 +292,42 @@ func TestRemoveAll(t *testing.T) { assert.Empty(t, apitest.ListUnmatchedRequests()) }) + t.Run("chunks recursive object deletes by storage api cap", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + prefixes := numberedStorageFiles(1001) + for page := 0; page <= len(prefixes)/storage.PAGE_LIMIT; page++ { + start := page * storage.PAGE_LIMIT + end := min(start+storage.PAGE_LIMIT, len(prefixes)) + gock.New("http://127.0.0.1"). + Post("/storage/v1/object/list/private"). + JSON(storage.ListObjectsQuery{ + Prefix: "tmp/", + Search: "", + Limit: storage.PAGE_LIMIT, + Offset: start, + }). + Reply(http.StatusOK). + JSON(objectResponses(prefixes[start:end])) + } + files := prefixedStorageFiles("tmp/", prefixes) + gock.New("http://127.0.0.1"). + Delete("/storage/v1/object/private"). + JSON(storage.DeleteObjectsRequest{Prefixes: files[:1000]}). + Reply(http.StatusOK). + JSON(deleteObjectsResponse("private", files[:1000])) + gock.New("http://127.0.0.1"). + Delete("/storage/v1/object/private"). + JSON(storage.DeleteObjectsRequest{Prefixes: files[1000:]}). + Reply(http.StatusOK). + JSON(deleteObjectsResponse("private", files[1000:])) + // Run test + err := RemoveStoragePathAll(context.Background(), mockApi, "private", "tmp/") + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + t.Run("removes empty bucket", func(t *testing.T) { // Setup mock api defer gock.OffAll() @@ -324,3 +390,47 @@ func TestRemoveAll(t *testing.T) { assert.Empty(t, apitest.ListUnmatchedRequests()) }) } + +func numberedStorageFiles(count int) []string { + files := make([]string, count) + for i := range files { + files[i] = fmt.Sprintf("file-%04d.txt", i) + } + return files +} + +func storageURLs(bucket string, prefixes []string) []string { + paths := make([]string, len(prefixes)) + for i, prefix := range prefixes { + paths[i] = fmt.Sprintf("ss:///%s/%s", bucket, prefix) + } + return paths +} + +func prefixedStorageFiles(prefix string, files []string) []string { + paths := make([]string, len(files)) + for i, file := range files { + paths[i] = prefix + file + } + return paths +} + +func objectResponses(files []string) []storage.ObjectResponse { + objects := make([]storage.ObjectResponse, len(files)) + for i, file := range files { + objects[i] = mockFile + objects[i].Name = file + } + return objects +} + +func deleteObjectsResponse(bucket string, prefixes []string) []storage.DeleteObjectsResponse { + objects := make([]storage.DeleteObjectsResponse, len(prefixes)) + for i, prefix := range prefixes { + objects[i] = storage.DeleteObjectsResponse{ + BucketId: bucket, + Name: prefix, + } + } + return objects +} diff --git a/apps/cli-go/pkg/storage/api.go b/apps/cli-go/pkg/storage/api.go index e7765eb9f0..a2af3588e3 100644 --- a/apps/cli-go/pkg/storage/api.go +++ b/apps/cli-go/pkg/storage/api.go @@ -7,3 +7,5 @@ type StorageAPI struct { } const PAGE_LIMIT = 100 + +const DELETE_OBJECTS_LIMIT = 1000 From 575be6dd2b31cb4e47f026121fe4d8b507db6662 Mon Sep 17 00:00:00 2001 From: avallete Date: Sat, 20 Jun 2026 11:48:08 +0200 Subject: [PATCH 2/3] fix(cli): keep logflare off ghcr mirror --- apps/cli-go/internal/utils/docker.go | 3 +++ apps/cli-go/internal/utils/docker_test.go | 30 +++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/apps/cli-go/internal/utils/docker.go b/apps/cli-go/internal/utils/docker.go index 9736fa651d..27d61fca31 100644 --- a/apps/cli-go/internal/utils/docker.go +++ b/apps/cli-go/internal/utils/docker.go @@ -197,6 +197,9 @@ func GetRegistryImageUrl(imageName string) string { if registry == "docker.io" { return imageName } + if registry == "ghcr.io" && strings.HasPrefix(imageName, "supabase/logflare:") { + return imageName + } // Configure mirror registry parts := strings.Split(imageName, "/") imageName = parts[len(parts)-1] diff --git a/apps/cli-go/internal/utils/docker_test.go b/apps/cli-go/internal/utils/docker_test.go index 0692045590..e1771d8363 100644 --- a/apps/cli-go/internal/utils/docker_test.go +++ b/apps/cli-go/internal/utils/docker_test.go @@ -24,6 +24,36 @@ const ( imageId = "test-image" ) +func TestGetRegistryImageUrl(t *testing.T) { + t.Cleanup(func() { + viper.Set("INTERNAL_IMAGE_REGISTRY", "docker.io") + }) + + t.Run("mirrors supabase images to the configured registry", func(t *testing.T) { + viper.Set("INTERNAL_IMAGE_REGISTRY", "ghcr.io") + + actual := GetRegistryImageUrl("supabase/postgres:17.4.1.054") + + assert.Equal(t, "ghcr.io/supabase/postgres:17.4.1.054", actual) + }) + + t.Run("keeps logflare on docker hub when missing from ghcr", func(t *testing.T) { + viper.Set("INTERNAL_IMAGE_REGISTRY", "ghcr.io") + + actual := GetRegistryImageUrl("supabase/logflare:1.45.0") + + assert.Equal(t, "supabase/logflare:1.45.0", actual) + }) + + t.Run("mirrors logflare to ecr", func(t *testing.T) { + viper.Set("INTERNAL_IMAGE_REGISTRY", "public.ecr.aws") + + actual := GetRegistryImageUrl("supabase/logflare:1.45.0") + + assert.Equal(t, "public.ecr.aws/supabase/logflare:1.45.0", actual) + }) +} + func TestPullImage(t *testing.T) { viper.Set("INTERNAL_IMAGE_REGISTRY", "docker.io") From a18a226e2198616deca94a7a954ac869f6f8841c Mon Sep 17 00:00:00 2001 From: avallete Date: Sat, 20 Jun 2026 11:54:08 +0200 Subject: [PATCH 3/3] test(cli): remove constant storage rm helper parameter --- apps/cli-go/internal/storage/rm/rm_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/cli-go/internal/storage/rm/rm_test.go b/apps/cli-go/internal/storage/rm/rm_test.go index 6cc1e5ff01..ebc184f3e1 100644 --- a/apps/cli-go/internal/storage/rm/rm_test.go +++ b/apps/cli-go/internal/storage/rm/rm_test.go @@ -129,12 +129,12 @@ func TestStorageRM(t *testing.T) { Delete("/storage/v1/object/private"). JSON(storage.DeleteObjectsRequest{Prefixes: prefixes[:1000]}). Reply(http.StatusOK). - JSON(deleteObjectsResponse("private", prefixes[:1000])) + JSON(deleteObjectsResponse(prefixes[:1000])) gock.New("https://" + utils.GetSupabaseHost(flags.ProjectRef)). Delete("/storage/v1/object/private"). JSON(storage.DeleteObjectsRequest{Prefixes: prefixes[1000:]}). Reply(http.StatusOK). - JSON(deleteObjectsResponse("private", prefixes[1000:])) + JSON(deleteObjectsResponse(prefixes[1000:])) // Run test paths := storageURLs("private", prefixes) err := Run(context.Background(), paths, false, fsys) @@ -315,12 +315,12 @@ func TestRemoveAll(t *testing.T) { Delete("/storage/v1/object/private"). JSON(storage.DeleteObjectsRequest{Prefixes: files[:1000]}). Reply(http.StatusOK). - JSON(deleteObjectsResponse("private", files[:1000])) + JSON(deleteObjectsResponse(files[:1000])) gock.New("http://127.0.0.1"). Delete("/storage/v1/object/private"). JSON(storage.DeleteObjectsRequest{Prefixes: files[1000:]}). Reply(http.StatusOK). - JSON(deleteObjectsResponse("private", files[1000:])) + JSON(deleteObjectsResponse(files[1000:])) // Run test err := RemoveStoragePathAll(context.Background(), mockApi, "private", "tmp/") // Check error @@ -424,11 +424,11 @@ func objectResponses(files []string) []storage.ObjectResponse { return objects } -func deleteObjectsResponse(bucket string, prefixes []string) []storage.DeleteObjectsResponse { +func deleteObjectsResponse(prefixes []string) []storage.DeleteObjectsResponse { objects := make([]storage.DeleteObjectsResponse, len(prefixes)) for i, prefix := range prefixes { objects[i] = storage.DeleteObjectsResponse{ - BucketId: bucket, + BucketId: "private", Name: prefix, } }