fix: ListResources failing for projects with large number of resources

proto/gen/rill/runtime/v1/runtime.swagger.yaml

@@ -3525,6 +3525,17 @@ paths:
           in: query
           required: false
           type: boolean
+        - name: pageSize
+          description: Maximum number of resources to return. If zero, returns all resources.
+          in: query
+          required: false
+          type: integer
+          format: int64
+        - name: pageToken
+          description: Page token returned by a previous ListResources call.
+          in: query
+          required: false
+          type: string
       tags:
         - RuntimeService
   /v1/instances/{instanceId}/resources/-/watch:
@@ -6025,6 +6036,8 @@ definitions:
         items:
           type: object
           $ref: '#/definitions/v1Resource'
+      nextPageToken:
+        type: string
   v1ListTablesResponse:
     type: object
     properties:

proto/rill/runtime/v1/api.proto

@@ -844,10 +844,15 @@ message ListResourcesRequest {
   string path = 3;
   // Skip security checks
   bool skip_security_checks = 4;
+  // Maximum number of resources to return. If zero, returns all resources.
+  uint32 page_size = 5 [(validate.rules).uint32 = {ignore_empty: true, lte: 10000}];
+  // Page token returned by a previous ListResources call.
+  string page_token = 6;
 }
 
 message ListResourcesResponse {
   repeated Resource resources = 1;
+  string next_page_token = 2;
 }
 
 message WatchResourcesRequest {

runtime/client/client.go

@@ -6,10 +6,12 @@ import (
 	"net/url"
 
 	runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1"
+	"github.com/rilldata/rill/runtime/pkg/pagination"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/protobuf/proto"
 )
 
 // Retry policy for requests to the runtime.
@@ -70,6 +72,30 @@ func New(runtimeHost, bearerToken string) (*Client, error) {
 	}, nil
 }
 
+// ListResources retrieves all pages of resources.
+func (c *Client) ListResources(ctx context.Context, req *runtimev1.ListResourcesRequest, opts ...grpc.CallOption) (*runtimev1.ListResourcesResponse, error) {
+	pageReq := proto.Clone(req).(*runtimev1.ListResourcesRequest)
+	pageSize := pageReq.PageSize
+	if pageSize == 0 {
+		pageSize = 100
+	}
+
+	resources, err := pagination.CollectAll(ctx, func(ctx context.Context, pageSize uint32, token string) ([]*runtimev1.Resource, string, error) {
+		pageReq.PageSize = pageSize
+		pageReq.PageToken = token
+		page, err := c.RuntimeServiceClient.ListResources(ctx, pageReq, opts...)
+		if err != nil {
+			return nil, "", err
+		}
+		return page.Resources, page.NextPageToken, nil
+	}, pageSize)
+	if err != nil {
+		return nil, err
+	}
+
+	return &runtimev1.ListResourcesResponse{Resources: resources}, nil
+}
+
 // Close closes the client connection.
 func (c *Client) Close() error {
 	return c.conn.Close()

runtime/server/controller.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"slices"
+	"sort"
 	"strings"
 	"time"
 
@@ -32,6 +33,7 @@ func (s *Server) ListResources(ctx context.Context, req *runtimev1.ListResources
 		attribute.String("args.instance_id", req.InstanceId),
 		attribute.String("args.kind", req.Kind),
 		attribute.Bool("args.skip_security_checks", req.SkipSecurityChecks),
+		attribute.Int64("args.page_size", int64(req.PageSize)),
 	)
 
 	claims := auth.GetClaims(ctx, req.InstanceId)
@@ -49,44 +51,64 @@ func (s *Server) ListResources(ctx context.Context, req *runtimev1.ListResources
 		return nil, err
 	}
 
+	if req.SkipSecurityChecks {
+		if !claims.Can(runtime.ReadInstance) {
+			return nil, ErrForbidden
+		}
+	} else {
+		i := 0
+		for i < len(rs) {
+			r, access, err := s.runtime.ApplySecurityPolicy(ctx, req.InstanceId, claims, rs[i])
+			if err != nil {
+				return nil, mapGRPCErrorWithFallback(err, codes.InvalidArgument)
+			}
+			if !access {
+				rs[i] = rs[len(rs)-1]
+				rs[len(rs)-1] = nil
+				rs = rs[:len(rs)-1]
+				continue
+			}
+			rs[i] = r
+			i++
+		}
+	}
+
 	slices.SortFunc(rs, func(a, b *runtimev1.Resource) int {
 		an := a.Meta.Name
 		bn := b.Meta.Name
-		if an.Kind < bn.Kind {
-			return -1
-		}
-		if an.Kind > bn.Kind {
-			return 1
+		if an.Kind != bn.Kind {
+			return strings.Compare(an.Kind, bn.Kind)
 		}
 		return strings.Compare(an.Name, bn.Name)
 	})
 
-	if req.SkipSecurityChecks {
-		if !claims.Can(runtime.ReadInstance) {
-			return nil, ErrForbidden
-		}
+	if req.PageSize == 0 {
 		return &runtimev1.ListResourcesResponse{Resources: rs}, nil
 	}
 
-	i := 0
-	for i < len(rs) {
-		r := rs[i]
-		r, access, err := s.runtime.ApplySecurityPolicy(ctx, req.InstanceId, claims, r)
-		if err != nil {
-			return nil, mapGRPCErrorWithFallback(err, codes.InvalidArgument)
-		}
-		if !access {
-			// Remove from the slice
-			rs[i] = rs[len(rs)-1]
-			rs[len(rs)-1] = nil
-			rs = rs[:len(rs)-1]
-			continue
+	var afterKind, afterName string
+	if req.PageToken != "" {
+		if err := pagination.UnmarshalPageToken(req.PageToken, &afterKind, &afterName); err != nil {
+			return nil, status.Errorf(codes.InvalidArgument, "failed to parse page token: %v", err)
 		}
-		rs[i] = r
-		i++
 	}
 
-	return &runtimev1.ListResourcesResponse{Resources: rs}, nil
+	start := sort.Search(len(rs), func(i int) bool {
+		n := rs[i].Meta.Name
+		return n.Kind > afterKind || (n.Kind == afterKind && n.Name > afterName)
+	})
+	end := min(start+int(req.PageSize), len(rs))
+
+	var nextPageToken string
+	if end < len(rs) {
+		last := rs[end-1].Meta.Name
+		nextPageToken = pagination.MarshalPageToken(last.Kind, last.Name)
+	}
+
+	return &runtimev1.ListResourcesResponse{
+		Resources:     rs[start:end],
+		NextPageToken: nextPageToken,
+	}, nil
 }
 
 // WatchResources implements runtimev1.RuntimeServiceServer

web-common/src/proto/gen/rill/runtime/v1/api_pb.ts

@@ -2912,6 +2912,20 @@ export class ListResourcesRequest extends Message$1<ListResourcesRequest> {
    */
   skipSecurityChecks = false;
 
+  /**
+   * Maximum number of resources to return. If zero, returns all resources.
+   *
+   * @generated from field: uint32 page_size = 5;
+   */
+  pageSize = 0;
+
+  /**
+   * Page token returned by a previous ListResources call.
+   *
+   * @generated from field: string page_token = 6;
+   */
+  pageToken = "";
+
   constructor(data?: PartialMessage<ListResourcesRequest>) {
     super();
     proto3.util.initPartial(data, this);
@@ -2924,6 +2938,8 @@ export class ListResourcesRequest extends Message$1<ListResourcesRequest> {
     { no: 2, name: "kind", kind: "scalar", T: 9 /* ScalarType.STRING */ },
     { no: 3, name: "path", kind: "scalar", T: 9 /* ScalarType.STRING */ },
     { no: 4, name: "skip_security_checks", kind: "scalar", T: 8 /* ScalarType.BOOL */ },
+    { no: 5, name: "page_size", kind: "scalar", T: 13 /* ScalarType.UINT32 */ },
+    { no: 6, name: "page_token", kind: "scalar", T: 9 /* ScalarType.STRING */ },
   ]);
 
   static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): ListResourcesRequest {
@@ -2952,6 +2968,11 @@ export class ListResourcesResponse extends Message$1<ListResourcesResponse> {
    */
   resources: Resource[] = [];
 
+  /**
+   * @generated from field: string next_page_token = 2;
+   */
+  nextPageToken = "";
+
   constructor(data?: PartialMessage<ListResourcesResponse>) {
     super();
     proto3.util.initPartial(data, this);
@@ -2961,6 +2982,7 @@ export class ListResourcesResponse extends Message$1<ListResourcesResponse> {
   static readonly typeName = "rill.runtime.v1.ListResourcesResponse";
   static readonly fields: FieldList = proto3.util.newFieldList(() => [
     { no: 1, name: "resources", kind: "message", T: Resource, repeated: true },
+    { no: 2, name: "next_page_token", kind: "scalar", T: 9 /* ScalarType.STRING */ },
   ]);
 
   static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): ListResourcesResponse {