fix: don't fail the grid when relationship resolution exceeds the query value limit

src/routes/(console)/project-[region]-[project]/databases/database-[database]/collection-[collection]/+page.ts

@@ -2,7 +2,7 @@ import { Dependencies, SPREADSHEET_PAGE_LIMIT } from '$lib/constants';
 import { getLimit, getPage, getQuery, getView, pageToOffset, View } from '$lib/helpers/load';
 import type { PageLoad } from './$types';
 import { queries, queryParamToMap } from '$lib/components/filters';
-import { buildGridQueries, extractSortFromQueries } from '$database/store';
+import { buildGridQueries, extractSortFromQueries, loadGridRows } from '$database/store';
 import { getCollectionService } from '$database/(entity)';
 
 export const load: PageLoad = async ({ params, depends, url, route, parent }) => {
@@ -22,17 +22,28 @@ export const load: PageLoad = async ({ params, depends, url, route, parent }) =>
     const currentSort = extractSortFromQueries(parsedQueries);
     const collectionSdk = getCollectionService(params.region, params.project, database.type);
 
+    const documentsPage = await loadGridRows(
+        collection,
+        (includeRelationships) =>
+            buildGridQueries(limit, offset, parsedQueries, collection, includeRelationships),
+        async (queries) => {
+            const response = await collectionSdk.listDocuments({
+                databaseId: params.database,
+                collectionId: params.collection,
+                queries
+            });
+
+            return { total: response.total, rows: response.documents };
+        }
+    );
+
     return {
         offset,
         limit,
         view,
         query,
         currentSort,
         parsedQueries,
-        documents: await collectionSdk.listDocuments({
-            databaseId: params.database,
-            collectionId: params.collection,
-            queries: buildGridQueries(limit, offset, parsedQueries, collection)
-        })
+        documents: { total: documentsPage.total, documents: documentsPage.rows }
     };
 };

src/routes/(console)/project-[region]-[project]/databases/database-[database]/store.ts

@@ -3,7 +3,8 @@ import type { Column } from '$lib/helpers/types';
 import { IconCloudUpload, IconCog } from '@appwrite.io/pink-icons-svelte';
 import { resolveRoute, withPath } from '$lib/stores/navigation';
 import type { Page } from '@sveltejs/kit';
-import { type Models, Query } from '@appwrite.io/console';
+import { AppwriteException, type Models, Query } from '@appwrite.io/console';
+import { chunks } from '$lib/helpers/array';
 import type { Entity, Field } from '$database/(entity)';
 import { isRelationship } from '$database/table-[table]/rows/store';
 import type { TagValue } from '$lib/components/filters/store';
@@ -178,7 +179,8 @@ export function buildGridQueries(
     limit: number,
     offset: number,
     parsedQueries: Map<TagValue, string>,
-    table: Entity
+    table: Entity,
+    includeRelationships: boolean = true
 ) {
     const hasOrderQuery = Array.from(parsedQueries.values()).some(
         (q) => q.includes('orderAsc') || q.includes('orderDesc')
@@ -191,7 +193,75 @@ export function buildGridQueries(
         queryArray.push(Query.orderDesc(''));
     }
 
-    queryArray.push(...parsedQueries.values(), ...buildWildcardEntitiesQuery(table));
+    queryArray.push(
+        ...parsedQueries.values(),
+        ...(includeRelationships ? buildWildcardEntitiesQuery(table) : [Query.select(['*'])])
+    );
 
     return queryArray;
 }
+
+const RELATIONSHIP_CHUNK_SIZE = 10;
+
+function isTooManyQueryValues(error: unknown): boolean {
+    return (
+        error instanceof AppwriteException && /greater than \d+ values/i.test(error.message ?? '')
+    );
+}
+
+type EntityRows<T> = { total: number; rows: T[] };
+
+/**
+ * Loads a page of the grid, falling back to smaller relationship batches when
+ * the API trips its own 500 value limit resolving them for the whole page.
+ */
+export async function loadGridRows<T extends { $id: string }>(
+    entity: Entity,
+    buildQueries: (includeRelationships: boolean) => string[],
+    fetchRows: (queries: string[]) => Promise<EntityRows<T>>
+): Promise<EntityRows<T>> {
+    try {
+        return await fetchRows(buildQueries(true));
+    } catch (error) {
+        if (!isTooManyQueryValues(error)) throw error;
+
+        const page = await fetchRows(buildQueries(false));
+
+        return {
+            total: page.total,
+            rows: await populateRelationships(entity, page.rows, fetchRows)
+        };
+    }
+}
+
+async function populateRelationships<T extends { $id: string }>(
+    entity: Entity,
+    rows: T[],
+    fetchRows: (queries: string[]) => Promise<EntityRows<T>>
+): Promise<T[]> {
+    const relationshipQueries = buildWildcardEntitiesQuery(entity);
+
+    if (relationshipQueries.length <= 1 || rows.length === 0) return rows;
+
+    const populated = new Map<string, T>();
+
+    await Promise.all(
+        chunks(rows, RELATIONSHIP_CHUNK_SIZE).map(async (chunk) => {
+            const rowIds = chunk.map((row) => row.$id);
+
+            try {
+                const response = await fetchRows([
+                    Query.equal('$id', rowIds),
+                    Query.limit(rowIds.length),
+                    ...relationshipQueries
+                ]);
+
+                response.rows.forEach((row) => populated.set(row.$id, row));
+            } catch {
+                // keep the unpopulated rows for this chunk!
+            }
+        })
+    );
+
+    return rows.map((row) => populated.get(row.$id) ?? row);
+}

src/routes/(console)/project-[region]-[project]/databases/database-[database]/table-[table]/+page.ts

@@ -3,7 +3,7 @@ import { getLimit, getPage, getQuery, getView, pageToOffset, View } from '$lib/h
 import { sdk } from '$lib/stores/sdk';
 import type { PageLoad } from './$types';
 import { queries, queryParamToMap } from '$lib/components/filters';
-import { buildGridQueries, extractSortFromQueries } from '$database/store';
+import { buildGridQueries, extractSortFromQueries, loadGridRows } from '$database/store';
 
 export const load: PageLoad = async ({ params, depends, url, route, parent }) => {
     const { table } = await parent();
@@ -28,10 +28,16 @@ export const load: PageLoad = async ({ params, depends, url, route, parent }) =>
         query,
         currentSort,
         parsedQueries,
-        rows: await sdk.forProject(params.region, params.project).tablesDB.listRows({
-            databaseId: params.database,
-            tableId: params.table,
-            queries: buildGridQueries(limit, offset, parsedQueries, table)
-        })
+        rows: await loadGridRows(
+            table,
+            (includeRelationships) =>
+                buildGridQueries(limit, offset, parsedQueries, table, includeRelationships),
+            (queries) =>
+                sdk.forProject(params.region, params.project).tablesDB.listRows({
+                    databaseId: params.database,
+                    tableId: params.table,
+                    queries
+                })
+        )
     };
 };

src/routes/(console)/project-[region]-[project]/databases/database-[database]/table-[table]/spreadsheet.svelte

@@ -98,6 +98,7 @@
         expandTabs,
         type Columns,
         buildWildcardEntitiesQuery,
+        loadGridRows,
         type SortState,
         randomDataModalState,
         spreadsheetLoading,
@@ -722,22 +723,26 @@
         }
 
         const parsedQueries = data.parsedQueries;
-        const filterQueries = parsedQueries.size ? data.parsedQueries.values() : [];
+        // materialized: the fallback in `loadGridRows` builds the queries more than once.
+        const filterQueries = parsedQueries.size ? Array.from(data.parsedQueries.values()) : [];
 
         $paginatedRowsLoading = true;
-        const loadedRows = await sdk
-            .forProject(page.params.region, page.params.project)
-            .tablesDB.listRows({
-                databaseId,
-                tableId,
-                queries: [
-                    getCorrectOrderQuery(),
-                    Query.limit(SPREADSHEET_PAGE_LIMIT),
-                    Query.offset(pageToOffset(pageNumber, SPREADSHEET_PAGE_LIMIT)),
-                    ...filterQueries /* filter queries */,
-                    ...buildWildcardEntitiesQuery(table)
-                ]
-            });
+        const loadedRows = await loadGridRows(
+            table,
+            (includeRelationships) => [
+                getCorrectOrderQuery(),
+                Query.limit(SPREADSHEET_PAGE_LIMIT),
+                Query.offset(pageToOffset(pageNumber, SPREADSHEET_PAGE_LIMIT)),
+                ...filterQueries /* filter queries */,
+                ...(includeRelationships
+                    ? buildWildcardEntitiesQuery(table)
+                    : [Query.select(['*'])])
+            ],
+            (queries) =>
+                sdk
+                    .forProject(page.params.region, page.params.project)
+                    .tablesDB.listRows({ databaseId, tableId, queries })
+        );
 
         paginatedRows.setPage(pageNumber, loadedRows.rows);
         $paginatedRowsLoading = false;
@@ -753,18 +758,21 @@
             paginatedRows.setMaxPage(targetPageNum);
             $paginatedRowsLoading = true;
 
-            const loadedRows = await sdk
-                .forProject(page.params.region, page.params.project)
-                .tablesDB.listRows({
-                    databaseId,
-                    tableId,
-                    queries: [
-                        getCorrectOrderQuery(),
-                        Query.limit(SPREADSHEET_PAGE_LIMIT),
-                        Query.offset(pageToOffset(targetPageNum, SPREADSHEET_PAGE_LIMIT)),
-                        ...buildWildcardEntitiesQuery(table)
-                    ]
-                });
+            const loadedRows = await loadGridRows(
+                table,
+                (includeRelationships) => [
+                    getCorrectOrderQuery(),
+                    Query.limit(SPREADSHEET_PAGE_LIMIT),
+                    Query.offset(pageToOffset(targetPageNum, SPREADSHEET_PAGE_LIMIT)),
+                    ...(includeRelationships
+                        ? buildWildcardEntitiesQuery(table)
+                        : [Query.select(['*'])])
+                ],
+                (queries) =>
+                    sdk
+                        .forProject(page.params.region, page.params.project)
+                        .tablesDB.listRows({ databaseId, tableId, queries })
+            );
 
             paginatedRows.setPage(targetPageNum, loadedRows.rows);
             $paginatedRowsLoading = false;