fix(cli): chunk storage delete requests to respect API cap (#1814)

apps/cli-go/internal/storage/rm/rm.go

@@ -71,7 +71,7 @@ func Run(ctx context.Context, paths []string, recursive bool, fsys afero.Fs) err
 		}
 		// Always try deleting first in case the paths resolve to extensionless files
 		fmt.Fprintln(os.Stderr, "Deleting objects:", prefixes)
-		removed, err := api.DeleteObjects(ctx, bucket, prefixes)
+		removed, err := deleteObjects(ctx, api, bucket, prefixes)
 		if err != nil {
 			return err
 		}
@@ -124,7 +124,7 @@ func RemoveStoragePathAll(ctx context.Context, api storage.StorageAPI, bucket, p
 		}
 		if len(files) > 0 {
 			fmt.Fprintln(os.Stderr, "Deleting objects:", files)
-			if _, err := api.DeleteObjects(ctx, bucket, files); err != nil {
+			if _, err := deleteObjects(ctx, api, bucket, files); err != nil {
 				return err
 			}
 		}
@@ -141,3 +141,16 @@ func RemoveStoragePathAll(ctx context.Context, api storage.StorageAPI, bucket, p
 	}
 	return nil
 }
+
+func deleteObjects(ctx context.Context, api storage.StorageAPI, bucket string, prefixes []string) ([]storage.DeleteObjectsResponse, error) {
+	var removed []storage.DeleteObjectsResponse
+	for start := 0; start < len(prefixes); start += storage.DELETE_OBJECTS_LIMIT {
+		end := min(start+storage.DELETE_OBJECTS_LIMIT, len(prefixes))
+		objects, err := api.DeleteObjects(ctx, bucket, prefixes[start:end])
+		if err != nil {
+			return nil, err
+		}
+		removed = append(removed, objects...)
+	}
+	return removed, nil
+}

apps/cli-go/internal/storage/rm/rm_test.go

@@ -2,6 +2,7 @@ package rm
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"testing"
 
@@ -113,6 +114,35 @@ func TestStorageRM(t *testing.T) {
 		assert.Empty(t, apitest.ListUnmatchedRequests())
 	})
 
+	t.Run("chunks explicit object deletes by storage api cap", func(t *testing.T) {
+		t.Cleanup(fstest.MockStdin(t, "y"))
+		// Setup in-memory fs
+		fsys := afero.NewMemMapFs()
+		// Setup mock api
+		defer gock.OffAll()
+		gock.New(utils.DefaultApiHost).
+			Get("/v1/projects/" + flags.ProjectRef + "/api-keys").
+			Reply(http.StatusOK).
+			JSON(apiKeys)
+		prefixes := numberedStorageFiles(1001)
+		gock.New("https://" + utils.GetSupabaseHost(flags.ProjectRef)).
+			Delete("/storage/v1/object/private").
+			JSON(storage.DeleteObjectsRequest{Prefixes: prefixes[:1000]}).
+			Reply(http.StatusOK).
+			JSON(deleteObjectsResponse(prefixes[:1000]))
+		gock.New("https://" + utils.GetSupabaseHost(flags.ProjectRef)).
+			Delete("/storage/v1/object/private").
+			JSON(storage.DeleteObjectsRequest{Prefixes: prefixes[1000:]}).
+			Reply(http.StatusOK).
+			JSON(deleteObjectsResponse(prefixes[1000:]))
+		// Run test
+		paths := storageURLs("private", prefixes)
+		err := Run(context.Background(), paths, false, fsys)
+		// Check error
+		assert.NoError(t, err)
+		assert.Empty(t, apitest.ListUnmatchedRequests())
+	})
+
 	t.Run("removes buckets and directories", func(t *testing.T) {
 		t.Cleanup(fstest.MockStdin(t, "y"))
 		// Setup in-memory fs
@@ -262,6 +292,42 @@ func TestRemoveAll(t *testing.T) {
 		assert.Empty(t, apitest.ListUnmatchedRequests())
 	})
 
+	t.Run("chunks recursive object deletes by storage api cap", func(t *testing.T) {
+		// Setup mock api
+		defer gock.OffAll()
+		prefixes := numberedStorageFiles(1001)
+		for page := 0; page <= len(prefixes)/storage.PAGE_LIMIT; page++ {
+			start := page * storage.PAGE_LIMIT
+			end := min(start+storage.PAGE_LIMIT, len(prefixes))
+			gock.New("http://127.0.0.1").
+				Post("/storage/v1/object/list/private").
+				JSON(storage.ListObjectsQuery{
+					Prefix: "tmp/",
+					Search: "",
+					Limit:  storage.PAGE_LIMIT,
+					Offset: start,
+				}).
+				Reply(http.StatusOK).
+				JSON(objectResponses(prefixes[start:end]))
+		}
+		files := prefixedStorageFiles("tmp/", prefixes)
+		gock.New("http://127.0.0.1").
+			Delete("/storage/v1/object/private").
+			JSON(storage.DeleteObjectsRequest{Prefixes: files[:1000]}).
+			Reply(http.StatusOK).
+			JSON(deleteObjectsResponse(files[:1000]))
+		gock.New("http://127.0.0.1").
+			Delete("/storage/v1/object/private").
+			JSON(storage.DeleteObjectsRequest{Prefixes: files[1000:]}).
+			Reply(http.StatusOK).
+			JSON(deleteObjectsResponse(files[1000:]))
+		// Run test
+		err := RemoveStoragePathAll(context.Background(), mockApi, "private", "tmp/")
+		// Check error
+		assert.NoError(t, err)
+		assert.Empty(t, apitest.ListUnmatchedRequests())
+	})
+
 	t.Run("removes empty bucket", func(t *testing.T) {
 		// Setup mock api
 		defer gock.OffAll()
@@ -324,3 +390,47 @@ func TestRemoveAll(t *testing.T) {
 		assert.Empty(t, apitest.ListUnmatchedRequests())
 	})
 }
+
+func numberedStorageFiles(count int) []string {
+	files := make([]string, count)
+	for i := range files {
+		files[i] = fmt.Sprintf("file-%04d.txt", i)
+	}
+	return files
+}
+
+func storageURLs(bucket string, prefixes []string) []string {
+	paths := make([]string, len(prefixes))
+	for i, prefix := range prefixes {
+		paths[i] = fmt.Sprintf("ss:///%s/%s", bucket, prefix)
+	}
+	return paths
+}
+
+func prefixedStorageFiles(prefix string, files []string) []string {
+	paths := make([]string, len(files))
+	for i, file := range files {
+		paths[i] = prefix + file
+	}
+	return paths
+}
+
+func objectResponses(files []string) []storage.ObjectResponse {
+	objects := make([]storage.ObjectResponse, len(files))
+	for i, file := range files {
+		objects[i] = mockFile
+		objects[i].Name = file
+	}
+	return objects
+}
+
+func deleteObjectsResponse(prefixes []string) []storage.DeleteObjectsResponse {
+	objects := make([]storage.DeleteObjectsResponse, len(prefixes))
+	for i, prefix := range prefixes {
+		objects[i] = storage.DeleteObjectsResponse{
+			BucketId: "private",
+			Name:     prefix,
+		}
+	}
+	return objects
+}

apps/cli-go/internal/utils/docker.go

@@ -197,6 +197,9 @@ func GetRegistryImageUrl(imageName string) string {
 	if registry == "docker.io" {
 		return imageName
 	}
+	if registry == "ghcr.io" && strings.HasPrefix(imageName, "supabase/logflare:") {
+		return imageName
+	}
 	// Configure mirror registry
 	parts := strings.Split(imageName, "/")
 	imageName = parts[len(parts)-1]

apps/cli-go/internal/utils/docker_test.go

@@ -24,6 +24,36 @@ const (
 	imageId     = "test-image"
 )
 
+func TestGetRegistryImageUrl(t *testing.T) {
+	t.Cleanup(func() {
+		viper.Set("INTERNAL_IMAGE_REGISTRY", "docker.io")
+	})
+
+	t.Run("mirrors supabase images to the configured registry", func(t *testing.T) {
+		viper.Set("INTERNAL_IMAGE_REGISTRY", "ghcr.io")
+
+		actual := GetRegistryImageUrl("supabase/postgres:17.4.1.054")
+
+		assert.Equal(t, "ghcr.io/supabase/postgres:17.4.1.054", actual)
+	})
+
+	t.Run("keeps logflare on docker hub when missing from ghcr", func(t *testing.T) {
+		viper.Set("INTERNAL_IMAGE_REGISTRY", "ghcr.io")
+
+		actual := GetRegistryImageUrl("supabase/logflare:1.45.0")
+
+		assert.Equal(t, "supabase/logflare:1.45.0", actual)
+	})
+
+	t.Run("mirrors logflare to ecr", func(t *testing.T) {
+		viper.Set("INTERNAL_IMAGE_REGISTRY", "public.ecr.aws")
+
+		actual := GetRegistryImageUrl("supabase/logflare:1.45.0")
+
+		assert.Equal(t, "public.ecr.aws/supabase/logflare:1.45.0", actual)
+	})
+}
+
 func TestPullImage(t *testing.T) {
 	viper.Set("INTERNAL_IMAGE_REGISTRY", "docker.io")
 

apps/cli-go/pkg/storage/api.go

@@ -7,3 +7,5 @@ type StorageAPI struct {
 }
 
 const PAGE_LIMIT = 100
+
+const DELETE_OBJECTS_LIMIT = 1000