diff --git a/apps/cli-go/internal/storage/rm/rm.go b/apps/cli-go/internal/storage/rm/rm.go index 56e55a7974..8254f93164 100644 --- a/apps/cli-go/internal/storage/rm/rm.go +++ b/apps/cli-go/internal/storage/rm/rm.go @@ -71,7 +71,7 @@ func Run(ctx context.Context, paths []string, recursive bool, fsys afero.Fs) err } // Always try deleting first in case the paths resolve to extensionless files fmt.Fprintln(os.Stderr, "Deleting objects:", prefixes) - removed, err := api.DeleteObjects(ctx, bucket, prefixes) + removed, err := deleteObjects(ctx, api, bucket, prefixes) if err != nil { return err } @@ -124,7 +124,7 @@ func RemoveStoragePathAll(ctx context.Context, api storage.StorageAPI, bucket, p } if len(files) > 0 { fmt.Fprintln(os.Stderr, "Deleting objects:", files) - if _, err := api.DeleteObjects(ctx, bucket, files); err != nil { + if _, err := deleteObjects(ctx, api, bucket, files); err != nil { return err } } @@ -141,3 +141,16 @@ func RemoveStoragePathAll(ctx context.Context, api storage.StorageAPI, bucket, p } return nil } + +func deleteObjects(ctx context.Context, api storage.StorageAPI, bucket string, prefixes []string) ([]storage.DeleteObjectsResponse, error) { + var removed []storage.DeleteObjectsResponse + for start := 0; start < len(prefixes); start += storage.DELETE_OBJECTS_LIMIT { + end := min(start+storage.DELETE_OBJECTS_LIMIT, len(prefixes)) + objects, err := api.DeleteObjects(ctx, bucket, prefixes[start:end]) + if err != nil { + return nil, err + } + removed = append(removed, objects...) + } + return removed, nil +} diff --git a/apps/cli-go/internal/storage/rm/rm_test.go b/apps/cli-go/internal/storage/rm/rm_test.go index cc02418cfc..ebc184f3e1 100644 --- a/apps/cli-go/internal/storage/rm/rm_test.go +++ b/apps/cli-go/internal/storage/rm/rm_test.go @@ -2,6 +2,7 @@ package rm import ( "context" + "fmt" "net/http" "testing" @@ -113,6 +114,35 @@ func TestStorageRM(t *testing.T) { assert.Empty(t, apitest.ListUnmatchedRequests()) }) + t.Run("chunks explicit object deletes by storage api cap", func(t *testing.T) { + t.Cleanup(fstest.MockStdin(t, "y")) + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + flags.ProjectRef + "/api-keys"). + Reply(http.StatusOK). + JSON(apiKeys) + prefixes := numberedStorageFiles(1001) + gock.New("https://" + utils.GetSupabaseHost(flags.ProjectRef)). + Delete("/storage/v1/object/private"). + JSON(storage.DeleteObjectsRequest{Prefixes: prefixes[:1000]}). + Reply(http.StatusOK). + JSON(deleteObjectsResponse(prefixes[:1000])) + gock.New("https://" + utils.GetSupabaseHost(flags.ProjectRef)). + Delete("/storage/v1/object/private"). + JSON(storage.DeleteObjectsRequest{Prefixes: prefixes[1000:]}). + Reply(http.StatusOK). + JSON(deleteObjectsResponse(prefixes[1000:])) + // Run test + paths := storageURLs("private", prefixes) + err := Run(context.Background(), paths, false, fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + t.Run("removes buckets and directories", func(t *testing.T) { t.Cleanup(fstest.MockStdin(t, "y")) // Setup in-memory fs @@ -262,6 +292,42 @@ func TestRemoveAll(t *testing.T) { assert.Empty(t, apitest.ListUnmatchedRequests()) }) + t.Run("chunks recursive object deletes by storage api cap", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + prefixes := numberedStorageFiles(1001) + for page := 0; page <= len(prefixes)/storage.PAGE_LIMIT; page++ { + start := page * storage.PAGE_LIMIT + end := min(start+storage.PAGE_LIMIT, len(prefixes)) + gock.New("http://127.0.0.1"). + Post("/storage/v1/object/list/private"). + JSON(storage.ListObjectsQuery{ + Prefix: "tmp/", + Search: "", + Limit: storage.PAGE_LIMIT, + Offset: start, + }). + Reply(http.StatusOK). + JSON(objectResponses(prefixes[start:end])) + } + files := prefixedStorageFiles("tmp/", prefixes) + gock.New("http://127.0.0.1"). + Delete("/storage/v1/object/private"). + JSON(storage.DeleteObjectsRequest{Prefixes: files[:1000]}). + Reply(http.StatusOK). + JSON(deleteObjectsResponse(files[:1000])) + gock.New("http://127.0.0.1"). + Delete("/storage/v1/object/private"). + JSON(storage.DeleteObjectsRequest{Prefixes: files[1000:]}). + Reply(http.StatusOK). + JSON(deleteObjectsResponse(files[1000:])) + // Run test + err := RemoveStoragePathAll(context.Background(), mockApi, "private", "tmp/") + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + t.Run("removes empty bucket", func(t *testing.T) { // Setup mock api defer gock.OffAll() @@ -324,3 +390,47 @@ func TestRemoveAll(t *testing.T) { assert.Empty(t, apitest.ListUnmatchedRequests()) }) } + +func numberedStorageFiles(count int) []string { + files := make([]string, count) + for i := range files { + files[i] = fmt.Sprintf("file-%04d.txt", i) + } + return files +} + +func storageURLs(bucket string, prefixes []string) []string { + paths := make([]string, len(prefixes)) + for i, prefix := range prefixes { + paths[i] = fmt.Sprintf("ss:///%s/%s", bucket, prefix) + } + return paths +} + +func prefixedStorageFiles(prefix string, files []string) []string { + paths := make([]string, len(files)) + for i, file := range files { + paths[i] = prefix + file + } + return paths +} + +func objectResponses(files []string) []storage.ObjectResponse { + objects := make([]storage.ObjectResponse, len(files)) + for i, file := range files { + objects[i] = mockFile + objects[i].Name = file + } + return objects +} + +func deleteObjectsResponse(prefixes []string) []storage.DeleteObjectsResponse { + objects := make([]storage.DeleteObjectsResponse, len(prefixes)) + for i, prefix := range prefixes { + objects[i] = storage.DeleteObjectsResponse{ + BucketId: "private", + Name: prefix, + } + } + return objects +} diff --git a/apps/cli-go/internal/utils/docker.go b/apps/cli-go/internal/utils/docker.go index 9736fa651d..27d61fca31 100644 --- a/apps/cli-go/internal/utils/docker.go +++ b/apps/cli-go/internal/utils/docker.go @@ -197,6 +197,9 @@ func GetRegistryImageUrl(imageName string) string { if registry == "docker.io" { return imageName } + if registry == "ghcr.io" && strings.HasPrefix(imageName, "supabase/logflare:") { + return imageName + } // Configure mirror registry parts := strings.Split(imageName, "/") imageName = parts[len(parts)-1] diff --git a/apps/cli-go/internal/utils/docker_test.go b/apps/cli-go/internal/utils/docker_test.go index 0692045590..e1771d8363 100644 --- a/apps/cli-go/internal/utils/docker_test.go +++ b/apps/cli-go/internal/utils/docker_test.go @@ -24,6 +24,36 @@ const ( imageId = "test-image" ) +func TestGetRegistryImageUrl(t *testing.T) { + t.Cleanup(func() { + viper.Set("INTERNAL_IMAGE_REGISTRY", "docker.io") + }) + + t.Run("mirrors supabase images to the configured registry", func(t *testing.T) { + viper.Set("INTERNAL_IMAGE_REGISTRY", "ghcr.io") + + actual := GetRegistryImageUrl("supabase/postgres:17.4.1.054") + + assert.Equal(t, "ghcr.io/supabase/postgres:17.4.1.054", actual) + }) + + t.Run("keeps logflare on docker hub when missing from ghcr", func(t *testing.T) { + viper.Set("INTERNAL_IMAGE_REGISTRY", "ghcr.io") + + actual := GetRegistryImageUrl("supabase/logflare:1.45.0") + + assert.Equal(t, "supabase/logflare:1.45.0", actual) + }) + + t.Run("mirrors logflare to ecr", func(t *testing.T) { + viper.Set("INTERNAL_IMAGE_REGISTRY", "public.ecr.aws") + + actual := GetRegistryImageUrl("supabase/logflare:1.45.0") + + assert.Equal(t, "public.ecr.aws/supabase/logflare:1.45.0", actual) + }) +} + func TestPullImage(t *testing.T) { viper.Set("INTERNAL_IMAGE_REGISTRY", "docker.io") diff --git a/apps/cli-go/pkg/storage/api.go b/apps/cli-go/pkg/storage/api.go index e7765eb9f0..a2af3588e3 100644 --- a/apps/cli-go/pkg/storage/api.go +++ b/apps/cli-go/pkg/storage/api.go @@ -7,3 +7,5 @@ type StorageAPI struct { } const PAGE_LIMIT = 100 + +const DELETE_OBJECTS_LIMIT = 1000