From 2db17075549134937ddec5c4019454127f5a932c Mon Sep 17 00:00:00 2001 From: Anshul Khandelwal <12948312+k-anshul@users.noreply.github.com> Date: Wed, 22 Jul 2026 20:36:38 +0530 Subject: [PATCH 1/4] fix: ListResources failing for projects with large number of resources --- proto/gen/rill/runtime/v1/api.pb.go | 49 ++++++++-- proto/gen/rill/runtime/v1/api.pb.validate.go | 19 ++++ .../gen/rill/runtime/v1/runtime.swagger.yaml | 13 +++ proto/rill/runtime/v1/api.proto | 5 ++ runtime/client/client.go | 22 +++++ runtime/server/controller.go | 73 +++++++++------ runtime/server/controller_test.go | 56 ++++++++++++ .../src/proto/gen/rill/runtime/v1/api_pb.ts | 22 +++++ .../runtime-client/v2/gen/runtime-service.ts | 90 +++++++++++++++++++ 9 files changed, 316 insertions(+), 33 deletions(-) diff --git a/proto/gen/rill/runtime/v1/api.pb.go b/proto/gen/rill/runtime/v1/api.pb.go index c833e9ce4228..ef0666651def 100644 --- a/proto/gen/rill/runtime/v1/api.pb.go +++ b/proto/gen/rill/runtime/v1/api.pb.go @@ -3750,6 +3750,10 @@ type ListResourcesRequest struct { Path string `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"` // Skip security checks SkipSecurityChecks bool `protobuf:"varint,4,opt,name=skip_security_checks,json=skipSecurityChecks,proto3" json:"skip_security_checks,omitempty"` + // Maximum number of resources to return. If zero, returns all resources. + PageSize uint32 `protobuf:"varint,5,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` + // Page token returned by a previous ListResources call. + PageToken string `protobuf:"bytes,6,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"` } func (x *ListResourcesRequest) Reset() { @@ -3812,12 +3816,27 @@ func (x *ListResourcesRequest) GetSkipSecurityChecks() bool { return false } +func (x *ListResourcesRequest) GetPageSize() uint32 { + if x != nil { + return x.PageSize + } + return 0 +} + +func (x *ListResourcesRequest) GetPageToken() string { + if x != nil { + return x.PageToken + } + return "" +} + type ListResourcesResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Resources []*Resource `protobuf:"bytes,1,rep,name=resources,proto3" json:"resources,omitempty"` + Resources []*Resource `protobuf:"bytes,1,rep,name=resources,proto3" json:"resources,omitempty"` + NextPageToken string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` } func (x *ListResourcesResponse) Reset() { @@ -3859,6 +3878,13 @@ func (x *ListResourcesResponse) GetResources() []*Resource { return nil } +func (x *ListResourcesResponse) GetNextPageToken() string { + if x != nil { + return x.NextPageToken + } + return "" +} + type WatchResourcesRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -9258,7 +9284,7 @@ var file_rill_runtime_v1_api_proto_rawDesc = []byte{ 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x67, 0x52, - 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x91, 0x01, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, + 0x03, 0x6c, 0x6f, 0x67, 0x22, 0xd9, 0x01, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, @@ -9267,12 +9293,19 @@ var file_rill_runtime_v1_api_proto_rawDesc = []byte{ 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x30, 0x0a, 0x14, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x73, 0x6b, 0x69, 0x70, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, - 0x74, 0x79, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x22, 0x50, 0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x37, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, - 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, - 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x22, 0x64, 0x0a, 0x15, 0x57, 0x61, + 0x74, 0x79, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x12, 0x27, 0x0a, 0x09, 0x70, 0x61, 0x67, 0x65, + 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x42, 0x0a, 0xfa, 0x42, 0x07, + 0x2a, 0x05, 0x18, 0x90, 0x4e, 0x40, 0x01, 0x52, 0x08, 0x70, 0x61, 0x67, 0x65, 0x53, 0x69, 0x7a, + 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x22, 0x78, 0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x09, 0x72, 0x65, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x72, + 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, + 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6e, 0x65, 0x78, + 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x64, 0x0a, 0x15, 0x57, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, diff --git a/proto/gen/rill/runtime/v1/api.pb.validate.go b/proto/gen/rill/runtime/v1/api.pb.validate.go index 6a0a1e54aa0f..67d3071cfe13 100644 --- a/proto/gen/rill/runtime/v1/api.pb.validate.go +++ b/proto/gen/rill/runtime/v1/api.pb.validate.go @@ -7072,6 +7072,23 @@ func (m *ListResourcesRequest) validate(all bool) error { // no validation rules for SkipSecurityChecks + if m.GetPageSize() != 0 { + + if m.GetPageSize() > 10000 { + err := ListResourcesRequestValidationError{ + field: "PageSize", + reason: "value must be less than or equal to 10000", + } + if !all { + return err + } + errors = append(errors, err) + } + + } + + // no validation rules for PageToken + if len(errors) > 0 { return ListResourcesRequestMultiError(errors) } @@ -7208,6 +7225,8 @@ func (m *ListResourcesResponse) validate(all bool) error { } + // no validation rules for NextPageToken + if len(errors) > 0 { return ListResourcesResponseMultiError(errors) } diff --git a/proto/gen/rill/runtime/v1/runtime.swagger.yaml b/proto/gen/rill/runtime/v1/runtime.swagger.yaml index 717e0d82e12c..41ce4e2e95c1 100644 --- a/proto/gen/rill/runtime/v1/runtime.swagger.yaml +++ b/proto/gen/rill/runtime/v1/runtime.swagger.yaml @@ -3525,6 +3525,17 @@ paths: in: query required: false type: boolean + - name: pageSize + description: Maximum number of resources to return. If zero, returns all resources. + in: query + required: false + type: integer + format: int64 + - name: pageToken + description: Page token returned by a previous ListResources call. + in: query + required: false + type: string tags: - RuntimeService /v1/instances/{instanceId}/resources/-/watch: @@ -6025,6 +6036,8 @@ definitions: items: type: object $ref: '#/definitions/v1Resource' + nextPageToken: + type: string v1ListTablesResponse: type: object properties: diff --git a/proto/rill/runtime/v1/api.proto b/proto/rill/runtime/v1/api.proto index 5a7e1daf9a5b..8c0575a1520f 100644 --- a/proto/rill/runtime/v1/api.proto +++ b/proto/rill/runtime/v1/api.proto @@ -844,10 +844,15 @@ message ListResourcesRequest { string path = 3; // Skip security checks bool skip_security_checks = 4; + // Maximum number of resources to return. If zero, returns all resources. + uint32 page_size = 5 [(validate.rules).uint32 = {ignore_empty: true, lte: 10000}]; + // Page token returned by a previous ListResources call. + string page_token = 6; } message ListResourcesResponse { repeated Resource resources = 1; + string next_page_token = 2; } message WatchResourcesRequest { diff --git a/runtime/client/client.go b/runtime/client/client.go index 1d7f068373b9..a3717d693518 100644 --- a/runtime/client/client.go +++ b/runtime/client/client.go @@ -10,6 +10,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/proto" ) // Retry policy for requests to the runtime. @@ -70,6 +71,27 @@ func New(runtimeHost, bearerToken string) (*Client, error) { }, nil } +// ListResources retrieves all pages of resources. +func (c *Client) ListResources(ctx context.Context, req *runtimev1.ListResourcesRequest, opts ...grpc.CallOption) (*runtimev1.ListResourcesResponse, error) { + pageReq := proto.Clone(req).(*runtimev1.ListResourcesRequest) + if pageReq.PageSize == 0 { + pageReq.PageSize = 100 + } + + res := &runtimev1.ListResourcesResponse{} + for { + page, err := c.RuntimeServiceClient.ListResources(ctx, pageReq, opts...) + if err != nil { + return nil, err + } + res.Resources = append(res.Resources, page.Resources...) + if page.NextPageToken == "" { + return res, nil + } + pageReq.PageToken = page.NextPageToken + } +} + // Close closes the client connection. func (c *Client) Close() error { return c.conn.Close() diff --git a/runtime/server/controller.go b/runtime/server/controller.go index 1da447092dce..0f3f46036eab 100644 --- a/runtime/server/controller.go +++ b/runtime/server/controller.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "slices" + "sort" "strings" "time" @@ -32,6 +33,7 @@ func (s *Server) ListResources(ctx context.Context, req *runtimev1.ListResources attribute.String("args.instance_id", req.InstanceId), attribute.String("args.kind", req.Kind), attribute.Bool("args.skip_security_checks", req.SkipSecurityChecks), + attribute.Int64("args.page_size", int64(req.PageSize)), ) claims := auth.GetClaims(ctx, req.InstanceId) @@ -49,44 +51,65 @@ func (s *Server) ListResources(ctx context.Context, req *runtimev1.ListResources return nil, err } + if req.SkipSecurityChecks { + if !claims.Can(runtime.ReadInstance) { + return nil, ErrForbidden + } + } else { + i := 0 + for i < len(rs) { + r, access, err := s.runtime.ApplySecurityPolicy(ctx, req.InstanceId, claims, rs[i]) + if err != nil { + return nil, mapGRPCErrorWithFallback(err, codes.InvalidArgument) + } + if !access { + rs[i] = rs[len(rs)-1] + rs[len(rs)-1] = nil + rs = rs[:len(rs)-1] + continue + } + rs[i] = r + i++ + } + } + slices.SortFunc(rs, func(a, b *runtimev1.Resource) int { an := a.Meta.Name bn := b.Meta.Name - if an.Kind < bn.Kind { - return -1 - } - if an.Kind > bn.Kind { - return 1 + if an.Kind != bn.Kind { + return strings.Compare(an.Kind, bn.Kind) } return strings.Compare(an.Name, bn.Name) }) - if req.SkipSecurityChecks { - if !claims.Can(runtime.ReadInstance) { - return nil, ErrForbidden - } + if req.PageSize == 0 { return &runtimev1.ListResourcesResponse{Resources: rs}, nil } - i := 0 - for i < len(rs) { - r := rs[i] - r, access, err := s.runtime.ApplySecurityPolicy(ctx, req.InstanceId, claims, r) - if err != nil { - return nil, mapGRPCErrorWithFallback(err, codes.InvalidArgument) - } - if !access { - // Remove from the slice - rs[i] = rs[len(rs)-1] - rs[len(rs)-1] = nil - rs = rs[:len(rs)-1] - continue + var afterKind, afterName string + if req.PageToken != "" { + if err := pagination.UnmarshalPageToken(req.PageToken, &afterKind, &afterName); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to parse page token: %v", err) } - rs[i] = r - i++ } - return &runtimev1.ListResourcesResponse{Resources: rs}, nil + start := sort.Search(len(rs), func(i int) bool { + n := rs[i].Meta.Name + return n.Kind > afterKind || (n.Kind == afterKind && n.Name > afterName) + }) + pageSize := min(int(req.PageSize), 10000) + end := min(start+pageSize, len(rs)) + + var nextPageToken string + if end < len(rs) { + last := rs[end-1].Meta.Name + nextPageToken = pagination.MarshalPageToken(last.Kind, last.Name) + } + + return &runtimev1.ListResourcesResponse{ + Resources: rs[start:end], + NextPageToken: nextPageToken, + }, nil } // WatchResources implements runtimev1.RuntimeServiceServer diff --git a/runtime/server/controller_test.go b/runtime/server/controller_test.go index 4586e1d018d2..7e9164795776 100644 --- a/runtime/server/controller_test.go +++ b/runtime/server/controller_test.go @@ -2,6 +2,7 @@ package server_test import ( "context" + "fmt" "testing" runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" @@ -15,6 +16,61 @@ import ( "go.uber.org/zap" ) +func TestListResourcesPagination(t *testing.T) { + rt, instanceID := testruntime.NewInstance(t) + files := make(map[string]string) + for i := range 101 { + files[fmt.Sprintf("theme-%03d.yaml", i)] = "type: theme\n" + } + testruntime.PutFiles(t, rt, instanceID, files) + testruntime.ReconcileParserAndWait(t, rt, instanceID) + + ctrl, err := rt.Controller(context.Background(), instanceID) + require.NoError(t, err) + want, err := ctrl.List(context.Background(), "", "", false) + require.NoError(t, err) + + srv, err := server.NewServer(context.Background(), &server.Options{}, rt, zap.NewNop(), ratelimit.NewNoop(), activity.NewNoopClient()) + require.NoError(t, err) + + unpaginated, err := srv.ListResources(testCtx(), &runtimev1.ListResourcesRequest{ + InstanceId: instanceID, + SkipSecurityChecks: true, + }) + require.NoError(t, err) + require.Len(t, unpaginated.Resources, len(want)) + require.Empty(t, unpaginated.NextPageToken) + + var got []*runtimev1.Resource + var pageToken string + for { + res, err := srv.ListResources(testCtx(), &runtimev1.ListResourcesRequest{ + InstanceId: instanceID, + SkipSecurityChecks: true, + PageSize: 100, + PageToken: pageToken, + }) + require.NoError(t, err) + require.LessOrEqual(t, len(res.Resources), 100) + if pageToken == "" { + require.Len(t, res.Resources, 100) + require.NotEmpty(t, res.NextPageToken) + } + got = append(got, res.Resources...) + if res.NextPageToken == "" { + break + } + pageToken = res.NextPageToken + } + + require.Len(t, got, len(want)) + for i := 1; i < len(got); i++ { + previous := got[i-1].Meta.Name + current := got[i].Meta.Name + require.True(t, previous.Kind < current.Kind || (previous.Kind == current.Kind && previous.Name < current.Name)) + } +} + func TestCreateTriggerAll(t *testing.T) { rt, instanceID := testruntime.NewInstance(t) diff --git a/web-common/src/proto/gen/rill/runtime/v1/api_pb.ts b/web-common/src/proto/gen/rill/runtime/v1/api_pb.ts index f95b89fe8d21..19361f15ec86 100644 --- a/web-common/src/proto/gen/rill/runtime/v1/api_pb.ts +++ b/web-common/src/proto/gen/rill/runtime/v1/api_pb.ts @@ -2912,6 +2912,20 @@ export class ListResourcesRequest extends Message$1 { */ skipSecurityChecks = false; + /** + * Maximum number of resources to return. If zero, returns all resources. + * + * @generated from field: uint32 page_size = 5; + */ + pageSize = 0; + + /** + * Page token returned by a previous ListResources call. + * + * @generated from field: string page_token = 6; + */ + pageToken = ""; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -2924,6 +2938,8 @@ export class ListResourcesRequest extends Message$1 { { no: 2, name: "kind", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 3, name: "path", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 4, name: "skip_security_checks", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 5, name: "page_size", kind: "scalar", T: 13 /* ScalarType.UINT32 */ }, + { no: 6, name: "page_token", kind: "scalar", T: 9 /* ScalarType.STRING */ }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): ListResourcesRequest { @@ -2952,6 +2968,11 @@ export class ListResourcesResponse extends Message$1 { */ resources: Resource[] = []; + /** + * @generated from field: string next_page_token = 2; + */ + nextPageToken = ""; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -2961,6 +2982,7 @@ export class ListResourcesResponse extends Message$1 { static readonly typeName = "rill.runtime.v1.ListResourcesResponse"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ { no: 1, name: "resources", kind: "message", T: Resource, repeated: true }, + { no: 2, name: "next_page_token", kind: "scalar", T: 9 /* ScalarType.STRING */ }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): ListResourcesResponse { diff --git a/web-common/src/runtime-client/v2/gen/runtime-service.ts b/web-common/src/runtime-client/v2/gen/runtime-service.ts index 9628ce4013fc..5095a85060e0 100644 --- a/web-common/src/runtime-client/v2/gen/runtime-service.ts +++ b/web-common/src/runtime-client/v2/gen/runtime-service.ts @@ -2295,6 +2295,96 @@ export function createRuntimeServiceListInstancesInfinite< return createInfiniteQuery(queryOptions, queryClient); } +export function getRuntimeServiceListResourcesInfiniteQueryOptions< + TData = InfiniteData, +>( + client: RuntimeClient, + request: Omit< + PartialMessage, + "instanceId" | "pageToken" + >, + options?: { + query?: Partial< + CreateInfiniteQueryOptions< + V1ListResourcesResponse, + ConnectError, + TData, + V1ListResourcesResponse, + QueryKey, + string | undefined + > + >; + }, +): CreateInfiniteQueryOptions< + V1ListResourcesResponse, + ConnectError, + TData, + V1ListResourcesResponse, + QueryKey, + string | undefined +> & { queryKey: QueryKey } { + const queryKey = [ + ...getRuntimeServiceListResourcesQueryKey(client.instanceId, request), + "infinite", + ] as QueryKey; + return { + queryKey, + queryFn: ({ pageParam, signal }) => + runtimeServiceListResources( + client, + { ...request, pageToken: pageParam } as Omit< + PartialMessage, + "instanceId" + >, + { signal }, + ), + initialPageParam: undefined as string | undefined, + getNextPageParam: (lastPage) => + ((lastPage as Record)?.nextPageToken as + | string + | undefined) || undefined, + enabled: !!client.instanceId, + ...options?.query, + } as CreateInfiniteQueryOptions< + V1ListResourcesResponse, + ConnectError, + TData, + V1ListResourcesResponse, + QueryKey, + string | undefined + > & { queryKey: QueryKey }; +} + +export function createRuntimeServiceListResourcesInfinite< + TData = InfiniteData, +>( + client: RuntimeClient, + request: Omit< + PartialMessage, + "instanceId" | "pageToken" + >, + options?: { + query?: Partial< + CreateInfiniteQueryOptions< + V1ListResourcesResponse, + ConnectError, + TData, + V1ListResourcesResponse, + QueryKey, + string | undefined + > + >; + }, + queryClient?: QueryClient, +): CreateInfiniteQueryResult { + const queryOptions = getRuntimeServiceListResourcesInfiniteQueryOptions( + client, + request, + options, + ); + return createInfiniteQuery(queryOptions, queryClient); +} + export function getRuntimeServiceGetModelPartitionsInfiniteQueryOptions< TData = InfiniteData, >( From 4b9662f4705b9da1c3b1d2cfc60dd1dd85870551 Mon Sep 17 00:00:00 2001 From: Anshul Khandelwal <12948312+k-anshul@users.noreply.github.com> Date: Wed, 22 Jul 2026 20:37:29 +0530 Subject: [PATCH 2/4] fix: ListResources failing for projects with large number of resources --- runtime/server/controller_test.go | 56 ------------------------------- 1 file changed, 56 deletions(-) diff --git a/runtime/server/controller_test.go b/runtime/server/controller_test.go index 7e9164795776..4586e1d018d2 100644 --- a/runtime/server/controller_test.go +++ b/runtime/server/controller_test.go @@ -2,7 +2,6 @@ package server_test import ( "context" - "fmt" "testing" runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" @@ -16,61 +15,6 @@ import ( "go.uber.org/zap" ) -func TestListResourcesPagination(t *testing.T) { - rt, instanceID := testruntime.NewInstance(t) - files := make(map[string]string) - for i := range 101 { - files[fmt.Sprintf("theme-%03d.yaml", i)] = "type: theme\n" - } - testruntime.PutFiles(t, rt, instanceID, files) - testruntime.ReconcileParserAndWait(t, rt, instanceID) - - ctrl, err := rt.Controller(context.Background(), instanceID) - require.NoError(t, err) - want, err := ctrl.List(context.Background(), "", "", false) - require.NoError(t, err) - - srv, err := server.NewServer(context.Background(), &server.Options{}, rt, zap.NewNop(), ratelimit.NewNoop(), activity.NewNoopClient()) - require.NoError(t, err) - - unpaginated, err := srv.ListResources(testCtx(), &runtimev1.ListResourcesRequest{ - InstanceId: instanceID, - SkipSecurityChecks: true, - }) - require.NoError(t, err) - require.Len(t, unpaginated.Resources, len(want)) - require.Empty(t, unpaginated.NextPageToken) - - var got []*runtimev1.Resource - var pageToken string - for { - res, err := srv.ListResources(testCtx(), &runtimev1.ListResourcesRequest{ - InstanceId: instanceID, - SkipSecurityChecks: true, - PageSize: 100, - PageToken: pageToken, - }) - require.NoError(t, err) - require.LessOrEqual(t, len(res.Resources), 100) - if pageToken == "" { - require.Len(t, res.Resources, 100) - require.NotEmpty(t, res.NextPageToken) - } - got = append(got, res.Resources...) - if res.NextPageToken == "" { - break - } - pageToken = res.NextPageToken - } - - require.Len(t, got, len(want)) - for i := 1; i < len(got); i++ { - previous := got[i-1].Meta.Name - current := got[i].Meta.Name - require.True(t, previous.Kind < current.Kind || (previous.Kind == current.Kind && previous.Name < current.Name)) - } -} - func TestCreateTriggerAll(t *testing.T) { rt, instanceID := testruntime.NewInstance(t) From 0e6647c6a999fb8e3c9fc71ca7b7a65c5a6abe21 Mon Sep 17 00:00:00 2001 From: Anshul Khandelwal <12948312+k-anshul@users.noreply.github.com> Date: Thu, 23 Jul 2026 10:34:09 +0530 Subject: [PATCH 3/4] review --- runtime/client/client.go | 24 ++++++++++++++---------- runtime/server/controller.go | 3 +-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/runtime/client/client.go b/runtime/client/client.go index a3717d693518..96035504c574 100644 --- a/runtime/client/client.go +++ b/runtime/client/client.go @@ -6,6 +6,7 @@ import ( "net/url" runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" + "github.com/rilldata/rill/runtime/pkg/pagination" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -74,22 +75,25 @@ func New(runtimeHost, bearerToken string) (*Client, error) { // ListResources retrieves all pages of resources. func (c *Client) ListResources(ctx context.Context, req *runtimev1.ListResourcesRequest, opts ...grpc.CallOption) (*runtimev1.ListResourcesResponse, error) { pageReq := proto.Clone(req).(*runtimev1.ListResourcesRequest) - if pageReq.PageSize == 0 { - pageReq.PageSize = 100 + pageSize := pageReq.PageSize + if pageSize == 0 { + pageSize = 100 } - res := &runtimev1.ListResourcesResponse{} - for { + resources, err := pagination.CollectAll(ctx, func(ctx context.Context, pageSize uint32, token string) ([]*runtimev1.Resource, string, error) { + pageReq.PageSize = pageSize + pageReq.PageToken = token page, err := c.RuntimeServiceClient.ListResources(ctx, pageReq, opts...) if err != nil { - return nil, err + return nil, "", err } - res.Resources = append(res.Resources, page.Resources...) - if page.NextPageToken == "" { - return res, nil - } - pageReq.PageToken = page.NextPageToken + return page.Resources, page.NextPageToken, nil + }, pageSize) + if err != nil { + return nil, err } + + return &runtimev1.ListResourcesResponse{Resources: resources}, nil } // Close closes the client connection. diff --git a/runtime/server/controller.go b/runtime/server/controller.go index 0f3f46036eab..923e2ebfbae6 100644 --- a/runtime/server/controller.go +++ b/runtime/server/controller.go @@ -97,8 +97,7 @@ func (s *Server) ListResources(ctx context.Context, req *runtimev1.ListResources n := rs[i].Meta.Name return n.Kind > afterKind || (n.Kind == afterKind && n.Name > afterName) }) - pageSize := min(int(req.PageSize), 10000) - end := min(start+pageSize, len(rs)) + end := min(start+int(req.PageSize), len(rs)) var nextPageToken string if end < len(rs) { From 306da997695d2dbce403a79bf9190bf3514bcc33 Mon Sep 17 00:00:00 2001 From: Anshul Khandelwal <12948312+k-anshul@users.noreply.github.com> Date: Thu, 23 Jul 2026 12:26:09 +0530 Subject: [PATCH 4/4] also sort the final result --- runtime/client/client.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/runtime/client/client.go b/runtime/client/client.go index 96035504c574..99b9ffc83f2b 100644 --- a/runtime/client/client.go +++ b/runtime/client/client.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "net/url" + "slices" + "strings" runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime/pkg/pagination" @@ -93,6 +95,15 @@ func (c *Client) ListResources(ctx context.Context, req *runtimev1.ListResources return nil, err } + slices.SortFunc(resources, func(a, b *runtimev1.Resource) int { + an := a.Meta.Name + bn := b.Meta.Name + if an.Kind != bn.Kind { + return strings.Compare(an.Kind, bn.Kind) + } + return strings.Compare(an.Name, bn.Name) + }) + return &runtimev1.ListResourcesResponse{Resources: resources}, nil }