diff --git a/proto/gen/rill/runtime/v1/api.pb.go b/proto/gen/rill/runtime/v1/api.pb.go index c833e9ce4228..ef0666651def 100644 --- a/proto/gen/rill/runtime/v1/api.pb.go +++ b/proto/gen/rill/runtime/v1/api.pb.go @@ -3750,6 +3750,10 @@ type ListResourcesRequest struct { Path string `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"` // Skip security checks SkipSecurityChecks bool `protobuf:"varint,4,opt,name=skip_security_checks,json=skipSecurityChecks,proto3" json:"skip_security_checks,omitempty"` + // Maximum number of resources to return. If zero, returns all resources. + PageSize uint32 `protobuf:"varint,5,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` + // Page token returned by a previous ListResources call. + PageToken string `protobuf:"bytes,6,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"` } func (x *ListResourcesRequest) Reset() { @@ -3812,12 +3816,27 @@ func (x *ListResourcesRequest) GetSkipSecurityChecks() bool { return false } +func (x *ListResourcesRequest) GetPageSize() uint32 { + if x != nil { + return x.PageSize + } + return 0 +} + +func (x *ListResourcesRequest) GetPageToken() string { + if x != nil { + return x.PageToken + } + return "" +} + type ListResourcesResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Resources []*Resource `protobuf:"bytes,1,rep,name=resources,proto3" json:"resources,omitempty"` + Resources []*Resource `protobuf:"bytes,1,rep,name=resources,proto3" json:"resources,omitempty"` + NextPageToken string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` } func (x *ListResourcesResponse) Reset() { @@ -3859,6 +3878,13 @@ func (x *ListResourcesResponse) GetResources() []*Resource { return nil } +func (x *ListResourcesResponse) GetNextPageToken() string { + if x != nil { + return x.NextPageToken + } + return "" +} + type WatchResourcesRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -9258,7 +9284,7 @@ var file_rill_runtime_v1_api_proto_rawDesc = []byte{ 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x67, 0x52, - 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x91, 0x01, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, + 0x03, 0x6c, 0x6f, 0x67, 0x22, 0xd9, 0x01, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, @@ -9267,12 +9293,19 @@ var file_rill_runtime_v1_api_proto_rawDesc = []byte{ 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x30, 0x0a, 0x14, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x73, 0x6b, 0x69, 0x70, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, - 0x74, 0x79, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x22, 0x50, 0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x37, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, - 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, - 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x22, 0x64, 0x0a, 0x15, 0x57, 0x61, + 0x74, 0x79, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x12, 0x27, 0x0a, 0x09, 0x70, 0x61, 0x67, 0x65, + 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x42, 0x0a, 0xfa, 0x42, 0x07, + 0x2a, 0x05, 0x18, 0x90, 0x4e, 0x40, 0x01, 0x52, 0x08, 0x70, 0x61, 0x67, 0x65, 0x53, 0x69, 0x7a, + 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x22, 0x78, 0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x09, 0x72, 0x65, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x72, + 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, + 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6e, 0x65, 0x78, + 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x64, 0x0a, 0x15, 0x57, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, diff --git a/proto/gen/rill/runtime/v1/api.pb.validate.go b/proto/gen/rill/runtime/v1/api.pb.validate.go index 6a0a1e54aa0f..67d3071cfe13 100644 --- a/proto/gen/rill/runtime/v1/api.pb.validate.go +++ b/proto/gen/rill/runtime/v1/api.pb.validate.go @@ -7072,6 +7072,23 @@ func (m *ListResourcesRequest) validate(all bool) error { // no validation rules for SkipSecurityChecks + if m.GetPageSize() != 0 { + + if m.GetPageSize() > 10000 { + err := ListResourcesRequestValidationError{ + field: "PageSize", + reason: "value must be less than or equal to 10000", + } + if !all { + return err + } + errors = append(errors, err) + } + + } + + // no validation rules for PageToken + if len(errors) > 0 { return ListResourcesRequestMultiError(errors) } @@ -7208,6 +7225,8 @@ func (m *ListResourcesResponse) validate(all bool) error { } + // no validation rules for NextPageToken + if len(errors) > 0 { return ListResourcesResponseMultiError(errors) } diff --git a/proto/gen/rill/runtime/v1/runtime.swagger.yaml b/proto/gen/rill/runtime/v1/runtime.swagger.yaml index 717e0d82e12c..41ce4e2e95c1 100644 --- a/proto/gen/rill/runtime/v1/runtime.swagger.yaml +++ b/proto/gen/rill/runtime/v1/runtime.swagger.yaml @@ -3525,6 +3525,17 @@ paths: in: query required: false type: boolean + - name: pageSize + description: Maximum number of resources to return. If zero, returns all resources. + in: query + required: false + type: integer + format: int64 + - name: pageToken + description: Page token returned by a previous ListResources call. + in: query + required: false + type: string tags: - RuntimeService /v1/instances/{instanceId}/resources/-/watch: @@ -6025,6 +6036,8 @@ definitions: items: type: object $ref: '#/definitions/v1Resource' + nextPageToken: + type: string v1ListTablesResponse: type: object properties: diff --git a/proto/rill/runtime/v1/api.proto b/proto/rill/runtime/v1/api.proto index 5a7e1daf9a5b..8c0575a1520f 100644 --- a/proto/rill/runtime/v1/api.proto +++ b/proto/rill/runtime/v1/api.proto @@ -844,10 +844,15 @@ message ListResourcesRequest { string path = 3; // Skip security checks bool skip_security_checks = 4; + // Maximum number of resources to return. If zero, returns all resources. + uint32 page_size = 5 [(validate.rules).uint32 = {ignore_empty: true, lte: 10000}]; + // Page token returned by a previous ListResources call. + string page_token = 6; } message ListResourcesResponse { repeated Resource resources = 1; + string next_page_token = 2; } message WatchResourcesRequest { diff --git a/runtime/client/client.go b/runtime/client/client.go index 1d7f068373b9..99b9ffc83f2b 100644 --- a/runtime/client/client.go +++ b/runtime/client/client.go @@ -4,12 +4,16 @@ import ( "context" "fmt" "net/url" + "slices" + "strings" runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" + "github.com/rilldata/rill/runtime/pkg/pagination" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/proto" ) // Retry policy for requests to the runtime. @@ -70,6 +74,39 @@ func New(runtimeHost, bearerToken string) (*Client, error) { }, nil } +// ListResources retrieves all pages of resources. +func (c *Client) ListResources(ctx context.Context, req *runtimev1.ListResourcesRequest, opts ...grpc.CallOption) (*runtimev1.ListResourcesResponse, error) { + pageReq := proto.Clone(req).(*runtimev1.ListResourcesRequest) + pageSize := pageReq.PageSize + if pageSize == 0 { + pageSize = 100 + } + + resources, err := pagination.CollectAll(ctx, func(ctx context.Context, pageSize uint32, token string) ([]*runtimev1.Resource, string, error) { + pageReq.PageSize = pageSize + pageReq.PageToken = token + page, err := c.RuntimeServiceClient.ListResources(ctx, pageReq, opts...) + if err != nil { + return nil, "", err + } + return page.Resources, page.NextPageToken, nil + }, pageSize) + if err != nil { + return nil, err + } + + slices.SortFunc(resources, func(a, b *runtimev1.Resource) int { + an := a.Meta.Name + bn := b.Meta.Name + if an.Kind != bn.Kind { + return strings.Compare(an.Kind, bn.Kind) + } + return strings.Compare(an.Name, bn.Name) + }) + + return &runtimev1.ListResourcesResponse{Resources: resources}, nil +} + // Close closes the client connection. func (c *Client) Close() error { return c.conn.Close() diff --git a/runtime/server/controller.go b/runtime/server/controller.go index 1da447092dce..923e2ebfbae6 100644 --- a/runtime/server/controller.go +++ b/runtime/server/controller.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "slices" + "sort" "strings" "time" @@ -32,6 +33,7 @@ func (s *Server) ListResources(ctx context.Context, req *runtimev1.ListResources attribute.String("args.instance_id", req.InstanceId), attribute.String("args.kind", req.Kind), attribute.Bool("args.skip_security_checks", req.SkipSecurityChecks), + attribute.Int64("args.page_size", int64(req.PageSize)), ) claims := auth.GetClaims(ctx, req.InstanceId) @@ -49,44 +51,64 @@ func (s *Server) ListResources(ctx context.Context, req *runtimev1.ListResources return nil, err } + if req.SkipSecurityChecks { + if !claims.Can(runtime.ReadInstance) { + return nil, ErrForbidden + } + } else { + i := 0 + for i < len(rs) { + r, access, err := s.runtime.ApplySecurityPolicy(ctx, req.InstanceId, claims, rs[i]) + if err != nil { + return nil, mapGRPCErrorWithFallback(err, codes.InvalidArgument) + } + if !access { + rs[i] = rs[len(rs)-1] + rs[len(rs)-1] = nil + rs = rs[:len(rs)-1] + continue + } + rs[i] = r + i++ + } + } + slices.SortFunc(rs, func(a, b *runtimev1.Resource) int { an := a.Meta.Name bn := b.Meta.Name - if an.Kind < bn.Kind { - return -1 - } - if an.Kind > bn.Kind { - return 1 + if an.Kind != bn.Kind { + return strings.Compare(an.Kind, bn.Kind) } return strings.Compare(an.Name, bn.Name) }) - if req.SkipSecurityChecks { - if !claims.Can(runtime.ReadInstance) { - return nil, ErrForbidden - } + if req.PageSize == 0 { return &runtimev1.ListResourcesResponse{Resources: rs}, nil } - i := 0 - for i < len(rs) { - r := rs[i] - r, access, err := s.runtime.ApplySecurityPolicy(ctx, req.InstanceId, claims, r) - if err != nil { - return nil, mapGRPCErrorWithFallback(err, codes.InvalidArgument) - } - if !access { - // Remove from the slice - rs[i] = rs[len(rs)-1] - rs[len(rs)-1] = nil - rs = rs[:len(rs)-1] - continue + var afterKind, afterName string + if req.PageToken != "" { + if err := pagination.UnmarshalPageToken(req.PageToken, &afterKind, &afterName); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to parse page token: %v", err) } - rs[i] = r - i++ } - return &runtimev1.ListResourcesResponse{Resources: rs}, nil + start := sort.Search(len(rs), func(i int) bool { + n := rs[i].Meta.Name + return n.Kind > afterKind || (n.Kind == afterKind && n.Name > afterName) + }) + end := min(start+int(req.PageSize), len(rs)) + + var nextPageToken string + if end < len(rs) { + last := rs[end-1].Meta.Name + nextPageToken = pagination.MarshalPageToken(last.Kind, last.Name) + } + + return &runtimev1.ListResourcesResponse{ + Resources: rs[start:end], + NextPageToken: nextPageToken, + }, nil } // WatchResources implements runtimev1.RuntimeServiceServer diff --git a/web-common/src/proto/gen/rill/runtime/v1/api_pb.ts b/web-common/src/proto/gen/rill/runtime/v1/api_pb.ts index f95b89fe8d21..19361f15ec86 100644 --- a/web-common/src/proto/gen/rill/runtime/v1/api_pb.ts +++ b/web-common/src/proto/gen/rill/runtime/v1/api_pb.ts @@ -2912,6 +2912,20 @@ export class ListResourcesRequest extends Message$1 { */ skipSecurityChecks = false; + /** + * Maximum number of resources to return. If zero, returns all resources. + * + * @generated from field: uint32 page_size = 5; + */ + pageSize = 0; + + /** + * Page token returned by a previous ListResources call. + * + * @generated from field: string page_token = 6; + */ + pageToken = ""; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -2924,6 +2938,8 @@ export class ListResourcesRequest extends Message$1 { { no: 2, name: "kind", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 3, name: "path", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 4, name: "skip_security_checks", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 5, name: "page_size", kind: "scalar", T: 13 /* ScalarType.UINT32 */ }, + { no: 6, name: "page_token", kind: "scalar", T: 9 /* ScalarType.STRING */ }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): ListResourcesRequest { @@ -2952,6 +2968,11 @@ export class ListResourcesResponse extends Message$1 { */ resources: Resource[] = []; + /** + * @generated from field: string next_page_token = 2; + */ + nextPageToken = ""; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -2961,6 +2982,7 @@ export class ListResourcesResponse extends Message$1 { static readonly typeName = "rill.runtime.v1.ListResourcesResponse"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ { no: 1, name: "resources", kind: "message", T: Resource, repeated: true }, + { no: 2, name: "next_page_token", kind: "scalar", T: 9 /* ScalarType.STRING */ }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): ListResourcesResponse { diff --git a/web-common/src/runtime-client/v2/gen/runtime-service.ts b/web-common/src/runtime-client/v2/gen/runtime-service.ts index 9628ce4013fc..5095a85060e0 100644 --- a/web-common/src/runtime-client/v2/gen/runtime-service.ts +++ b/web-common/src/runtime-client/v2/gen/runtime-service.ts @@ -2295,6 +2295,96 @@ export function createRuntimeServiceListInstancesInfinite< return createInfiniteQuery(queryOptions, queryClient); } +export function getRuntimeServiceListResourcesInfiniteQueryOptions< + TData = InfiniteData, +>( + client: RuntimeClient, + request: Omit< + PartialMessage, + "instanceId" | "pageToken" + >, + options?: { + query?: Partial< + CreateInfiniteQueryOptions< + V1ListResourcesResponse, + ConnectError, + TData, + V1ListResourcesResponse, + QueryKey, + string | undefined + > + >; + }, +): CreateInfiniteQueryOptions< + V1ListResourcesResponse, + ConnectError, + TData, + V1ListResourcesResponse, + QueryKey, + string | undefined +> & { queryKey: QueryKey } { + const queryKey = [ + ...getRuntimeServiceListResourcesQueryKey(client.instanceId, request), + "infinite", + ] as QueryKey; + return { + queryKey, + queryFn: ({ pageParam, signal }) => + runtimeServiceListResources( + client, + { ...request, pageToken: pageParam } as Omit< + PartialMessage, + "instanceId" + >, + { signal }, + ), + initialPageParam: undefined as string | undefined, + getNextPageParam: (lastPage) => + ((lastPage as Record)?.nextPageToken as + | string + | undefined) || undefined, + enabled: !!client.instanceId, + ...options?.query, + } as CreateInfiniteQueryOptions< + V1ListResourcesResponse, + ConnectError, + TData, + V1ListResourcesResponse, + QueryKey, + string | undefined + > & { queryKey: QueryKey }; +} + +export function createRuntimeServiceListResourcesInfinite< + TData = InfiniteData, +>( + client: RuntimeClient, + request: Omit< + PartialMessage, + "instanceId" | "pageToken" + >, + options?: { + query?: Partial< + CreateInfiniteQueryOptions< + V1ListResourcesResponse, + ConnectError, + TData, + V1ListResourcesResponse, + QueryKey, + string | undefined + > + >; + }, + queryClient?: QueryClient, +): CreateInfiniteQueryResult { + const queryOptions = getRuntimeServiceListResourcesInfiniteQueryOptions( + client, + request, + options, + ); + return createInfiniteQuery(queryOptions, queryClient); +} + export function getRuntimeServiceGetModelPartitionsInfiniteQueryOptions< TData = InfiniteData, >(