[pull] master from cube-js:master

packages/cubejs-api-gateway/openspec.yml

@@ -390,8 +390,53 @@ components:
           type: "object"
         timeDimensions:
           type: "object"
-    V1LoadResultData:
+    V1LoadResultDataRow:
+      type: "array"
+      description: "Row-oriented (default) data format - an array of rows, each row is an object keyed by member name."
+      items:
+        type: "object"
+    V1LoadResultDataCompact:
       type: "object"
+      description: "Compact data format - a single object with the members list and a dataset of primitive arrays. Returned when `responseFormat=compact` is requested."
+      required:
+        - members
+        - dataset
+      properties:
+        members:
+          type: "array"
+          description: "Ordered list of member names that correspond to each cell position in `dataset` rows."
+          items:
+            type: "string"
+        dataset:
+          type: "array"
+          description: "Array of rows, where each row is an array of primitive values (null, boolean, number, string) aligned with `members`."
+          items:
+            type: "array"
+            items: {}
+    V1LoadResultDataColumnar:
+      type: "object"
+      description: "Columnar data format - members list paired with one primitive array per column. Returned when `responseFormat=columnar` is requested."
+      required:
+        - members
+        - columns
+      properties:
+        members:
+          type: "array"
+          description: "Ordered list of member names. Element `i` of `columns` holds the values for `members[i]` across all rows."
+          items:
+            type: "string"
+        columns:
+          type: "array"
+          description: "One array per member, in the same order as `members`. Each inner array contains the primitive value of that member for every row (null, boolean, number, string)."
+          items:
+            type: "array"
+            items: {}
+    V1LoadResultData:
+      description: "Result `data` payload. Either an array of objects keyed by member name (default); a compact `{ members, dataset }` object when `responseFormat=compact`; or a columnar `{ members, columns }` object when `responseFormat=columnar`."
+      oneOf:
+        - $ref: "#/components/schemas/V1LoadResultDataRow"
+        - $ref: "#/components/schemas/V1LoadResultDataCompact"
+        - $ref: "#/components/schemas/V1LoadResultDataColumnar"
     V1LoadResult:
       type: "object"
       required:
@@ -403,9 +448,7 @@ components:
         annotation:
           $ref: "#/components/schemas/V1LoadResultAnnotation"
         data:
-          type: "array"
-          items:
-            $ref: "#/components/schemas/V1LoadResultData"
+          $ref: "#/components/schemas/V1LoadResultData"
         refreshKeyValues:
           type: "array"
           items:
@@ -552,6 +595,13 @@ components:
             $ref: "#/components/schemas/V1LoadRequestJoinHint"
         timezone:
           type: "string"
+        responseFormat:
+          type: "string"
+          description: "Output format of the result `data` payload. `default` returns row-oriented data (`V1LoadResultDataRow`); `compact` returns a `{ members, dataset }` object with rows of primitive arrays (`V1LoadResultDataCompact`); `columnar` returns a `{ members, columns }` object with one primitive array per member (`V1LoadResultDataColumnar`)."
+          enum:
+            - default
+            - compact
+            - columnar
     V1LoadRequest:
       type: "object"
       properties:

packages/cubejs-api-gateway/src/query.js

@@ -192,7 +192,7 @@ const querySchema = Joi.object().keys({
   cacheMode: Joi.valid('stale-if-slow', 'stale-while-revalidate', 'must-revalidate', 'no-cache'),
   cache: Joi.valid('stale-if-slow', 'stale-while-revalidate', 'must-revalidate', 'no-cache'),
   ungrouped: Joi.boolean(),
-  responseFormat: Joi.valid('default', 'compact'),
+  responseFormat: Joi.valid('default', 'compact', 'columnar'),
   subqueryJoins: Joi.array().items(subqueryJoin),
   joinHints: Joi.array().items(joinHint),
   maskedMembers: Joi.array().items(Joi.string()),
@@ -334,6 +334,7 @@ const normalizeQuery = (query, persistent, cacheMode) => {
   if (error) {
     throw new UserError(`Invalid query format: ${error.message || error.toString()}`);
   }
+
   const validQuery = query.measures?.length ||
     query.dimensions?.length ||
     query.timeDimensions?.filter(td => !!td.granularity).length;

packages/cubejs-api-gateway/src/types/enums.ts

@@ -19,7 +19,8 @@ enum QueryType {
  */
 enum ResultType {
   DEFAULT = 'default',
-  COMPACT = 'compact'
+  COMPACT = 'compact',
+  COLUMNAR = 'columnar'
 }
 
 /**

packages/cubejs-api-gateway/src/types/strings.ts

@@ -16,7 +16,8 @@ type RequestType =
  */
 type ResultType =
   'default' |
-  'compact';
+  'compact' |
+  'columnar';
 
 /**
  * API type data type.

packages/cubejs-api-gateway/test/normalize-query.test.ts

@@ -0,0 +1,21 @@
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { normalizeQuery } from '../src/query';
+
+const baseQuery = {
+  measures: ['Foo.count'],
+  timezone: 'UTC',
+};
+
+describe('responseFormat validation', () => {
+  test.each(['default', 'compact', 'columnar'])(
+    'accepts responseFormat=%s',
+    (responseFormat) => {
+      const result = normalizeQuery({ ...baseQuery, responseFormat }, false);
+      expect(result.responseFormat).toBe(responseFormat);
+    }
+  );
+
+  test('rejects unknown responseFormat', () => {
+    expect(() => normalizeQuery({ ...baseQuery, responseFormat: 'arrow' }, false)).toThrow(/Invalid query format/);
+  });
+});

packages/cubejs-backend-native/src/orchestrator.rs

@@ -154,6 +154,10 @@ impl ValueObject for ResultWrapper {
                 members: _members,
                 dataset,
             } => Ok(dataset.len()),
+            TransformedData::Columnar {
+                members: _members,
+                columns,
+            } => Ok(columns.first().map(|c| c.len()).unwrap_or(0)),
             TransformedData::Vanilla(dataset) => Ok(dataset.len()),
         }
     }
@@ -183,6 +187,30 @@ impl ValueObject for ResultWrapper {
 
                 row.get(member_index).unwrap_or(&DBResponsePrimitive::Null)
             }
+            TransformedData::Columnar { members, columns } => {
+                let Some(member_index) = members.iter().position(|m| m == field_name) else {
+                    return Err(CubeError::user(format!(
+                        "Field name '{}' not found in members",
+                        field_name
+                    )));
+                };
+
+                let Some(column) = columns.get(member_index) else {
+                    return Err(CubeError::user(format!(
+                        "Unexpected response from Cube, missing column for '{}'",
+                        field_name
+                    )));
+                };
+
+                let Some(value) = column.get(index) else {
+                    return Err(CubeError::user(format!(
+                        "Unexpected response from Cube, can't get {} row",
+                        index
+                    )));
+                };
+
+                value
+            }
             TransformedData::Vanilla(dataset) => {
                 let Some(row) = dataset.get(index) else {
                     return Err(CubeError::user(format!(

packages/cubejs-client-core/src/index.ts

@@ -167,7 +167,7 @@ function mutexPromise<T>(promise: Promise<T>): Promise<T | null> {
     });
 }
 
-export type ResponseFormat = 'compact' | 'default' | undefined;
+export type ResponseFormat = 'compact' | 'columnar' | 'default' | undefined;
 
 export type CubeApiOptions = {
   /**
@@ -183,7 +183,7 @@ export type CubeApiOptions = {
   pollInterval?: number;
   credentials?: TransportOptions['credentials'];
   parseDateMeasures?: boolean;
-  resType?: 'default' | 'compact';
+  resType?: 'default' | 'compact' | 'columnar';
   castNumerics?: boolean;
   /**
    * How many network errors would be retried before returning to users. Default to 0.
@@ -470,12 +470,12 @@ class CubeApi {
    */
   private patchQueryInternal(query: DeeplyReadonly<Query>, responseFormat: ResponseFormat): DeeplyReadonly<Query> {
     if (
-      responseFormat === 'compact' &&
-      query.responseFormat !== 'compact'
+      (responseFormat === 'compact' || responseFormat === 'columnar') &&
+      query.responseFormat !== responseFormat
     ) {
       return {
         ...query,
-        responseFormat: 'compact',
+        responseFormat,
       };
     } else {
       return query;
@@ -528,6 +528,21 @@ class CubeApi {
           });
           response.results[j].data = data;
         });
+      } else if (response.results[0].query.responseFormat &&
+        response.results[0].query.responseFormat === 'columnar') {
+        response.results.forEach((result, j) => {
+          const { columns, members } = result.data as unknown as { columns: any[][]; members: string[] };
+          const rowCount = columns[0]?.length ?? 0;
+          const data: Record<string, any>[] = [];
+          for (let i = 0; i < rowCount; i++) {
+            const row: Record<string, any> = {};
+            members.forEach((m, k) => {
+              row[m] = columns[k][i];
+            });
+            data.push(row);
+          }
+          response.results[j].data = data;
+        });
       }
     }
 
@@ -609,12 +624,12 @@ class CubeApi {
       ...options
     };
 
-    if (responseFormat === 'compact') {
+    if (responseFormat === 'compact' || responseFormat === 'columnar') {
       if (Array.isArray(query)) {
-        const patched = query.map((q) => this.patchQueryInternal(q, 'compact'));
+        const patched = query.map((q) => this.patchQueryInternal(q, responseFormat));
         return [patched as unknown as QueryType, options];
       } else {
-        const patched = this.patchQueryInternal(query as DeeplyReadonly<Query>, 'compact');
+        const patched = this.patchQueryInternal(query as DeeplyReadonly<Query>, responseFormat);
         return [patched as QueryType, options];
       }
     }

packages/cubejs-client-core/src/types.ts

@@ -149,7 +149,7 @@ export interface Query {
   // @deprecated
   renewQuery?: boolean;
   ungrouped?: boolean;
-  responseFormat?: 'compact' | 'default';
+  responseFormat?: 'compact' | 'columnar' | 'default';
   total?: boolean;
 }