From 1f749cf5c37d5f8dbe70fb6d785f66da02408507 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 30 Apr 2026 15:06:43 +0200 Subject: [PATCH] feat: Support columnar format in REST-API /v1/load (#10775) ntroduces a true column-oriented result format ({ members, columns }) alongside the existing default (rows of objects) and compact (rows of arrays). Columnar groups primitives by member, which compresses better on the wire and lets consumers feed each member to a single typed buffer without re-parsing object keys per row. --- packages/cubejs-api-gateway/openspec.yml | 58 ++- packages/cubejs-api-gateway/src/query.js | 3 +- .../cubejs-api-gateway/src/types/enums.ts | 3 +- .../cubejs-api-gateway/src/types/strings.ts | 3 +- .../test/normalize-query.test.ts | 21 + .../cubejs-backend-native/src/orchestrator.rs | 28 ++ packages/cubejs-client-core/src/index.ts | 31 +- packages/cubejs-client-core/src/types.ts | 2 +- .../birdbox-postgresql-cubestore.test.ts.snap | 428 +++++++++++------- .../birdbox-postgresql.test.ts.snap | 78 ++++ .../__snapshots__/cli-postgresql.test.ts.snap | 78 ++++ .../cubejs-testing/test/abstract-test-case.ts | 143 ++++-- rust/cubeorchestrator/Cargo.lock | 256 +++++++++++ rust/cubeorchestrator/Cargo.toml | 7 + rust/cubeorchestrator/benches/transform.rs | 149 ++++++ .../src/query_result_transform.rs | 214 +++++++++ rust/cubeorchestrator/src/transport.rs | 1 + .../cubeclient/.openapi-generator/FILES | 4 + .../cubeclient/.openapi-generator/VERSION | 2 +- .../cubeclient/src/apis/default_api.rs | 13 +- rust/cubesql/cubeclient/src/models/mod.rs | 10 +- .../src/models/v1_cube_meta_dimension.rs | 1 + .../models/v1_cube_meta_dimension_order.rs | 11 +- .../src/models/v1_cube_meta_measure.rs | 1 + .../cubeclient/src/models/v1_load_request.rs | 2 +- .../src/models/v1_load_request_query.rs | 20 + .../cubeclient/src/models/v1_load_response.rs | 24 +- .../cubeclient/src/models/v1_load_result.rs | 32 +- .../src/models/v1_load_result_data.rs | 29 ++ .../models/v1_load_result_data_columnar.rs | 33 ++ .../src/models/v1_load_result_data_compact.rs | 33 ++ .../src/models/v1_load_result_data_row.rs | 15 + rust/cubesql/cubesql/src/compile/builder.rs | 1 + .../cubesql/src/compile/engine/df/wrapper.rs | 1 + rust/cubesql/cubesql/src/transport/service.rs | 2 +- 35 files changed, 1476 insertions(+), 261 deletions(-) create mode 100644 packages/cubejs-api-gateway/test/normalize-query.test.ts create mode 100644 rust/cubeorchestrator/benches/transform.rs create mode 100644 rust/cubesql/cubeclient/src/models/v1_load_result_data.rs create mode 100644 rust/cubesql/cubeclient/src/models/v1_load_result_data_columnar.rs create mode 100644 rust/cubesql/cubeclient/src/models/v1_load_result_data_compact.rs create mode 100644 rust/cubesql/cubeclient/src/models/v1_load_result_data_row.rs diff --git a/packages/cubejs-api-gateway/openspec.yml b/packages/cubejs-api-gateway/openspec.yml index 8be11c1a162be..b7623ccfd81f7 100644 --- a/packages/cubejs-api-gateway/openspec.yml +++ b/packages/cubejs-api-gateway/openspec.yml @@ -390,8 +390,53 @@ components: type: "object" timeDimensions: type: "object" - V1LoadResultData: + V1LoadResultDataRow: + type: "array" + description: "Row-oriented (default) data format - an array of rows, each row is an object keyed by member name." + items: + type: "object" + V1LoadResultDataCompact: type: "object" + description: "Compact data format - a single object with the members list and a dataset of primitive arrays. Returned when `responseFormat=compact` is requested." + required: + - members + - dataset + properties: + members: + type: "array" + description: "Ordered list of member names that correspond to each cell position in `dataset` rows." + items: + type: "string" + dataset: + type: "array" + description: "Array of rows, where each row is an array of primitive values (null, boolean, number, string) aligned with `members`." + items: + type: "array" + items: {} + V1LoadResultDataColumnar: + type: "object" + description: "Columnar data format - members list paired with one primitive array per column. Returned when `responseFormat=columnar` is requested." + required: + - members + - columns + properties: + members: + type: "array" + description: "Ordered list of member names. Element `i` of `columns` holds the values for `members[i]` across all rows." + items: + type: "string" + columns: + type: "array" + description: "One array per member, in the same order as `members`. Each inner array contains the primitive value of that member for every row (null, boolean, number, string)." + items: + type: "array" + items: {} + V1LoadResultData: + description: "Result `data` payload. Either an array of objects keyed by member name (default); a compact `{ members, dataset }` object when `responseFormat=compact`; or a columnar `{ members, columns }` object when `responseFormat=columnar`." + oneOf: + - $ref: "#/components/schemas/V1LoadResultDataRow" + - $ref: "#/components/schemas/V1LoadResultDataCompact" + - $ref: "#/components/schemas/V1LoadResultDataColumnar" V1LoadResult: type: "object" required: @@ -403,9 +448,7 @@ components: annotation: $ref: "#/components/schemas/V1LoadResultAnnotation" data: - type: "array" - items: - $ref: "#/components/schemas/V1LoadResultData" + $ref: "#/components/schemas/V1LoadResultData" refreshKeyValues: type: "array" items: @@ -552,6 +595,13 @@ components: $ref: "#/components/schemas/V1LoadRequestJoinHint" timezone: type: "string" + responseFormat: + type: "string" + description: "Output format of the result `data` payload. `default` returns row-oriented data (`V1LoadResultDataRow`); `compact` returns a `{ members, dataset }` object with rows of primitive arrays (`V1LoadResultDataCompact`); `columnar` returns a `{ members, columns }` object with one primitive array per member (`V1LoadResultDataColumnar`)." + enum: + - default + - compact + - columnar V1LoadRequest: type: "object" properties: diff --git a/packages/cubejs-api-gateway/src/query.js b/packages/cubejs-api-gateway/src/query.js index 543c83cc0d2c7..af9f52c798b36 100644 --- a/packages/cubejs-api-gateway/src/query.js +++ b/packages/cubejs-api-gateway/src/query.js @@ -192,7 +192,7 @@ const querySchema = Joi.object().keys({ cacheMode: Joi.valid('stale-if-slow', 'stale-while-revalidate', 'must-revalidate', 'no-cache'), cache: Joi.valid('stale-if-slow', 'stale-while-revalidate', 'must-revalidate', 'no-cache'), ungrouped: Joi.boolean(), - responseFormat: Joi.valid('default', 'compact'), + responseFormat: Joi.valid('default', 'compact', 'columnar'), subqueryJoins: Joi.array().items(subqueryJoin), joinHints: Joi.array().items(joinHint), maskedMembers: Joi.array().items(Joi.string()), @@ -334,6 +334,7 @@ const normalizeQuery = (query, persistent, cacheMode) => { if (error) { throw new UserError(`Invalid query format: ${error.message || error.toString()}`); } + const validQuery = query.measures?.length || query.dimensions?.length || query.timeDimensions?.filter(td => !!td.granularity).length; diff --git a/packages/cubejs-api-gateway/src/types/enums.ts b/packages/cubejs-api-gateway/src/types/enums.ts index f6ef248b5f023..809c66f8deb07 100644 --- a/packages/cubejs-api-gateway/src/types/enums.ts +++ b/packages/cubejs-api-gateway/src/types/enums.ts @@ -19,7 +19,8 @@ enum QueryType { */ enum ResultType { DEFAULT = 'default', - COMPACT = 'compact' + COMPACT = 'compact', + COLUMNAR = 'columnar' } /** diff --git a/packages/cubejs-api-gateway/src/types/strings.ts b/packages/cubejs-api-gateway/src/types/strings.ts index b8c24670d8dfb..081e7937680c9 100644 --- a/packages/cubejs-api-gateway/src/types/strings.ts +++ b/packages/cubejs-api-gateway/src/types/strings.ts @@ -16,7 +16,8 @@ type RequestType = */ type ResultType = 'default' | - 'compact'; + 'compact' | + 'columnar'; /** * API type data type. diff --git a/packages/cubejs-api-gateway/test/normalize-query.test.ts b/packages/cubejs-api-gateway/test/normalize-query.test.ts new file mode 100644 index 0000000000000..32f92cf763323 --- /dev/null +++ b/packages/cubejs-api-gateway/test/normalize-query.test.ts @@ -0,0 +1,21 @@ +// eslint-disable-next-line import/no-extraneous-dependencies +import { normalizeQuery } from '../src/query'; + +const baseQuery = { + measures: ['Foo.count'], + timezone: 'UTC', +}; + +describe('responseFormat validation', () => { + test.each(['default', 'compact', 'columnar'])( + 'accepts responseFormat=%s', + (responseFormat) => { + const result = normalizeQuery({ ...baseQuery, responseFormat }, false); + expect(result.responseFormat).toBe(responseFormat); + } + ); + + test('rejects unknown responseFormat', () => { + expect(() => normalizeQuery({ ...baseQuery, responseFormat: 'arrow' }, false)).toThrow(/Invalid query format/); + }); +}); diff --git a/packages/cubejs-backend-native/src/orchestrator.rs b/packages/cubejs-backend-native/src/orchestrator.rs index 8fd9b91d5a795..baa52f19ba73f 100644 --- a/packages/cubejs-backend-native/src/orchestrator.rs +++ b/packages/cubejs-backend-native/src/orchestrator.rs @@ -154,6 +154,10 @@ impl ValueObject for ResultWrapper { members: _members, dataset, } => Ok(dataset.len()), + TransformedData::Columnar { + members: _members, + columns, + } => Ok(columns.first().map(|c| c.len()).unwrap_or(0)), TransformedData::Vanilla(dataset) => Ok(dataset.len()), } } @@ -183,6 +187,30 @@ impl ValueObject for ResultWrapper { row.get(member_index).unwrap_or(&DBResponsePrimitive::Null) } + TransformedData::Columnar { members, columns } => { + let Some(member_index) = members.iter().position(|m| m == field_name) else { + return Err(CubeError::user(format!( + "Field name '{}' not found in members", + field_name + ))); + }; + + let Some(column) = columns.get(member_index) else { + return Err(CubeError::user(format!( + "Unexpected response from Cube, missing column for '{}'", + field_name + ))); + }; + + let Some(value) = column.get(index) else { + return Err(CubeError::user(format!( + "Unexpected response from Cube, can't get {} row", + index + ))); + }; + + value + } TransformedData::Vanilla(dataset) => { let Some(row) = dataset.get(index) else { return Err(CubeError::user(format!( diff --git a/packages/cubejs-client-core/src/index.ts b/packages/cubejs-client-core/src/index.ts index a27eec6d1fbef..a7bec9fa84ecd 100644 --- a/packages/cubejs-client-core/src/index.ts +++ b/packages/cubejs-client-core/src/index.ts @@ -167,7 +167,7 @@ function mutexPromise(promise: Promise): Promise { }); } -export type ResponseFormat = 'compact' | 'default' | undefined; +export type ResponseFormat = 'compact' | 'columnar' | 'default' | undefined; export type CubeApiOptions = { /** @@ -183,7 +183,7 @@ export type CubeApiOptions = { pollInterval?: number; credentials?: TransportOptions['credentials']; parseDateMeasures?: boolean; - resType?: 'default' | 'compact'; + resType?: 'default' | 'compact' | 'columnar'; castNumerics?: boolean; /** * How many network errors would be retried before returning to users. Default to 0. @@ -470,12 +470,12 @@ class CubeApi { */ private patchQueryInternal(query: DeeplyReadonly, responseFormat: ResponseFormat): DeeplyReadonly { if ( - responseFormat === 'compact' && - query.responseFormat !== 'compact' + (responseFormat === 'compact' || responseFormat === 'columnar') && + query.responseFormat !== responseFormat ) { return { ...query, - responseFormat: 'compact', + responseFormat, }; } else { return query; @@ -528,6 +528,21 @@ class CubeApi { }); response.results[j].data = data; }); + } else if (response.results[0].query.responseFormat && + response.results[0].query.responseFormat === 'columnar') { + response.results.forEach((result, j) => { + const { columns, members } = result.data as unknown as { columns: any[][]; members: string[] }; + const rowCount = columns[0]?.length ?? 0; + const data: Record[] = []; + for (let i = 0; i < rowCount; i++) { + const row: Record = {}; + members.forEach((m, k) => { + row[m] = columns[k][i]; + }); + data.push(row); + } + response.results[j].data = data; + }); } } @@ -609,12 +624,12 @@ class CubeApi { ...options }; - if (responseFormat === 'compact') { + if (responseFormat === 'compact' || responseFormat === 'columnar') { if (Array.isArray(query)) { - const patched = query.map((q) => this.patchQueryInternal(q, 'compact')); + const patched = query.map((q) => this.patchQueryInternal(q, responseFormat)); return [patched as unknown as QueryType, options]; } else { - const patched = this.patchQueryInternal(query as DeeplyReadonly, 'compact'); + const patched = this.patchQueryInternal(query as DeeplyReadonly, responseFormat); return [patched as QueryType, options]; } } diff --git a/packages/cubejs-client-core/src/types.ts b/packages/cubejs-client-core/src/types.ts index 76d505f163eff..8f53e43c387d5 100644 --- a/packages/cubejs-client-core/src/types.ts +++ b/packages/cubejs-client-core/src/types.ts @@ -149,7 +149,7 @@ export interface Query { // @deprecated renewQuery?: boolean; ungrouped?: boolean; - responseFormat?: 'compact' | 'default'; + responseFormat?: 'compact' | 'columnar' | 'default'; total?: boolean; } diff --git a/packages/cubejs-testing/test/__snapshots__/birdbox-postgresql-cubestore.test.ts.snap b/packages/cubejs-testing/test/__snapshots__/birdbox-postgresql-cubestore.test.ts.snap index acd21eb4d8737..e716c0b260dbb 100644 --- a/packages/cubejs-testing/test/__snapshots__/birdbox-postgresql-cubestore.test.ts.snap +++ b/packages/cubejs-testing/test/__snapshots__/birdbox-postgresql-cubestore.test.ts.snap @@ -16,6 +16,63 @@ Array [ ] `; +exports[`postgresql-cubestore HTTP Transport #3 Events.count with Events.type order by Events.type DESC, Events.count: #3 Events.count with Events.type order by Events.type DESC, Events.count 1`] = ` +Array [ + Object { + "Events.count": "92", + "Events.type": "WatchEvent", + }, + Object { + "Events.count": "2", + "Events.type": "ReleaseEvent", + }, + Object { + "Events.count": "513", + "Events.type": "PushEvent", + }, + Object { + "Events.count": "21", + "Events.type": "PullRequestReviewCommentEvent", + }, + Object { + "Events.count": "32", + "Events.type": "PullRequestEvent", + }, + Object { + "Events.count": "1", + "Events.type": "MemberEvent", + }, + Object { + "Events.count": "57", + "Events.type": "IssuesEvent", + }, + Object { + "Events.count": "104", + "Events.type": "IssueCommentEvent", + }, + Object { + "Events.count": "21", + "Events.type": "GollumEvent", + }, + Object { + "Events.count": "21", + "Events.type": "ForkEvent", + }, + Object { + "Events.count": "14", + "Events.type": "DeleteEvent", + }, + Object { + "Events.count": "120", + "Events.type": "CreateEvent", + }, + Object { + "Events.count": "1", + "Events.type": "CommitCommentEvent", + }, +] +`; + exports[`postgresql-cubestore HTTP Transport Dbt orders count: Dbt orders count 1`] = ` Array [ Object { @@ -24,6 +81,95 @@ Array [ ] `; +exports[`postgresql-cubestore HTTP Transport Different column data types: Different column data types 1`] = ` +Array [ + Object { + "unusualDataTypes.array": Array [ + 1, + 2, + 3, + ], + "unusualDataTypes.bit_column": "11111111", + "unusualDataTypes.boolean_column": true, + "unusualDataTypes.cidr_column": "192.168.0.0/24", + "unusualDataTypes.id": 1, + "unusualDataTypes.inet_column": "192.168.0.1", + "unusualDataTypes.json": Object { + "key": "value1", + "number": 42, + }, + "unusualDataTypes.jsonb": Object { + "key": "value1", + "number": 42, + }, + "unusualDataTypes.mac_address": "11:22:33:44:55:66", + "unusualDataTypes.point_column": Object { + "x": 1, + "y": 1, + }, + "unusualDataTypes.status": "new", + "unusualDataTypes.text_column": "Hello, world!", + "unusualDataTypes.xml_column": "data", + }, + Object { + "unusualDataTypes.array": Array [ + 4, + 5, + 6, + ], + "unusualDataTypes.bit_column": "00000001", + "unusualDataTypes.boolean_column": false, + "unusualDataTypes.cidr_column": "192.168.0.0/24", + "unusualDataTypes.id": 2, + "unusualDataTypes.inet_column": "192.168.0.2", + "unusualDataTypes.json": Object { + "key": "value2", + "number": 84, + }, + "unusualDataTypes.jsonb": Object { + "key": "value2", + "number": 84, + }, + "unusualDataTypes.mac_address": "00:11:22:33:44:55", + "unusualDataTypes.point_column": Object { + "x": 2, + "y": 2, + }, + "unusualDataTypes.status": "new", + "unusualDataTypes.text_column": "Goodbye, world!", + "unusualDataTypes.xml_column": "more data", + }, + Object { + "unusualDataTypes.array": Array [ + 7, + 8, + 9, + ], + "unusualDataTypes.bit_column": "11110000", + "unusualDataTypes.boolean_column": true, + "unusualDataTypes.cidr_column": "192.168.0.0/24", + "unusualDataTypes.id": 3, + "unusualDataTypes.inet_column": "192.168.0.3", + "unusualDataTypes.json": Object { + "key": "value3", + "number": 168, + }, + "unusualDataTypes.jsonb": Object { + "key": "value3", + "number": 168, + }, + "unusualDataTypes.mac_address": "22:33:44:55:66:77", + "unusualDataTypes.point_column": Object { + "x": 3, + "y": 3, + }, + "unusualDataTypes.status": "processed", + "unusualDataTypes.text_column": "PostgreSQL is awesome!", + "unusualDataTypes.xml_column": "even more data", + }, +] +`; + exports[`postgresql-cubestore WS Transport #1 Orders.totalAmount: #1 Orders.totalAmount 1`] = ` Array [ Object { @@ -40,6 +186,63 @@ Array [ ] `; +exports[`postgresql-cubestore WS Transport #3 Events.count with Events.type order by Events.type DESC, Events.count: #3 Events.count with Events.type order by Events.type DESC, Events.count 1`] = ` +Array [ + Object { + "Events.count": "92", + "Events.type": "WatchEvent", + }, + Object { + "Events.count": "2", + "Events.type": "ReleaseEvent", + }, + Object { + "Events.count": "513", + "Events.type": "PushEvent", + }, + Object { + "Events.count": "21", + "Events.type": "PullRequestReviewCommentEvent", + }, + Object { + "Events.count": "32", + "Events.type": "PullRequestEvent", + }, + Object { + "Events.count": "1", + "Events.type": "MemberEvent", + }, + Object { + "Events.count": "57", + "Events.type": "IssuesEvent", + }, + Object { + "Events.count": "104", + "Events.type": "IssueCommentEvent", + }, + Object { + "Events.count": "21", + "Events.type": "GollumEvent", + }, + Object { + "Events.count": "21", + "Events.type": "ForkEvent", + }, + Object { + "Events.count": "14", + "Events.type": "DeleteEvent", + }, + Object { + "Events.count": "120", + "Events.type": "CreateEvent", + }, + Object { + "Events.count": "1", + "Events.type": "CommitCommentEvent", + }, +] +`; + exports[`postgresql-cubestore filters contains #1 Orders.status.contains: ["e"]: #1 Orders.status.contains: ["e"] 1`] = ` Array [ Object { @@ -152,7 +355,7 @@ Array [ ] `; -exports[`postgresql-cubestore responseFormat http+responseFormat=compact option#1+2: result-type 1`] = ` +exports[`postgresql-cubestore responseFormat http+responseFormat=columnar option#1+2: result-type 1`] = ` Array [ Object { "Orders.status": "processed", @@ -165,7 +368,7 @@ Array [ ] `; -exports[`postgresql-cubestore responseFormat http+responseFormat=compact option#1: result-type 1`] = ` +exports[`postgresql-cubestore responseFormat http+responseFormat=columnar option#1: result-type 1`] = ` Array [ Object { "Orders.status": "processed", @@ -178,7 +381,7 @@ Array [ ] `; -exports[`postgresql-cubestore responseFormat http+responseFormat=compact option#2: result-type 1`] = ` +exports[`postgresql-cubestore responseFormat http+responseFormat=columnar option#2: result-type 1`] = ` Array [ Object { "Orders.status": "processed", @@ -191,7 +394,7 @@ Array [ ] `; -exports[`postgresql-cubestore responseFormat http+responseFormat=default: result-type 1`] = ` +exports[`postgresql-cubestore responseFormat http+responseFormat=compact option#1+2: result-type 1`] = ` Array [ Object { "Orders.status": "processed", @@ -204,7 +407,7 @@ Array [ ] `; -exports[`postgresql-cubestore responseFormat ws+responseFormat=compact option#1+2: result-type 1`] = ` +exports[`postgresql-cubestore responseFormat http+responseFormat=compact option#1: result-type 1`] = ` Array [ Object { "Orders.status": "processed", @@ -217,7 +420,7 @@ Array [ ] `; -exports[`postgresql-cubestore responseFormat ws+responseFormat=compact option#1: result-type 1`] = ` +exports[`postgresql-cubestore responseFormat http+responseFormat=compact option#2: result-type 1`] = ` Array [ Object { "Orders.status": "processed", @@ -230,7 +433,7 @@ Array [ ] `; -exports[`postgresql-cubestore responseFormat ws+responseFormat=compact option#2: result-type 1`] = ` +exports[`postgresql-cubestore responseFormat http+responseFormat=default: result-type 1`] = ` Array [ Object { "Orders.status": "processed", @@ -243,7 +446,7 @@ Array [ ] `; -exports[`postgresql-cubestore responseFormat ws+responseFormat=default: result-type 1`] = ` +exports[`postgresql-cubestore responseFormat ws+responseFormat=columnar option#1+2: result-type 1`] = ` Array [ Object { "Orders.status": "processed", @@ -256,205 +459,80 @@ Array [ ] `; -exports[`postgresql-cubestore HTTP Transport #3 Events.count with Events.type order by Events.type DESC, Events.count: #3 Events.count with Events.type order by Events.type DESC, Events.count 1`] = ` +exports[`postgresql-cubestore responseFormat ws+responseFormat=columnar option#1: result-type 1`] = ` Array [ Object { - "Events.count": "92", - "Events.type": "WatchEvent", - }, - Object { - "Events.count": "2", - "Events.type": "ReleaseEvent", - }, - Object { - "Events.count": "513", - "Events.type": "PushEvent", - }, - Object { - "Events.count": "21", - "Events.type": "PullRequestReviewCommentEvent", - }, - Object { - "Events.count": "32", - "Events.type": "PullRequestEvent", - }, - Object { - "Events.count": "1", - "Events.type": "MemberEvent", - }, - Object { - "Events.count": "57", - "Events.type": "IssuesEvent", - }, - Object { - "Events.count": "104", - "Events.type": "IssueCommentEvent", - }, - Object { - "Events.count": "21", - "Events.type": "GollumEvent", - }, - Object { - "Events.count": "21", - "Events.type": "ForkEvent", - }, - Object { - "Events.count": "14", - "Events.type": "DeleteEvent", - }, - Object { - "Events.count": "120", - "Events.type": "CreateEvent", + "Orders.status": "processed", + "Orders.totalAmount": "800", }, Object { - "Events.count": "1", - "Events.type": "CommitCommentEvent", + "Orders.status": "shipped", + "Orders.totalAmount": "600", }, ] `; -exports[`postgresql-cubestore HTTP Transport Different column data types: Different column data types 1`] = ` +exports[`postgresql-cubestore responseFormat ws+responseFormat=columnar option#2: result-type 1`] = ` Array [ Object { - "unusualDataTypes.array": Array [ - 1, - 2, - 3, - ], - "unusualDataTypes.bit_column": "11111111", - "unusualDataTypes.boolean_column": true, - "unusualDataTypes.cidr_column": "192.168.0.0/24", - "unusualDataTypes.id": 1, - "unusualDataTypes.inet_column": "192.168.0.1", - "unusualDataTypes.json": Object { - "key": "value1", - "number": 42, - }, - "unusualDataTypes.jsonb": Object { - "key": "value1", - "number": 42, - }, - "unusualDataTypes.mac_address": "11:22:33:44:55:66", - "unusualDataTypes.point_column": Object { - "x": 1, - "y": 1, - }, - "unusualDataTypes.status": "new", - "unusualDataTypes.text_column": "Hello, world!", - "unusualDataTypes.xml_column": "data", - }, - Object { - "unusualDataTypes.array": Array [ - 4, - 5, - 6, - ], - "unusualDataTypes.bit_column": "00000001", - "unusualDataTypes.boolean_column": false, - "unusualDataTypes.cidr_column": "192.168.0.0/24", - "unusualDataTypes.id": 2, - "unusualDataTypes.inet_column": "192.168.0.2", - "unusualDataTypes.json": Object { - "key": "value2", - "number": 84, - }, - "unusualDataTypes.jsonb": Object { - "key": "value2", - "number": 84, - }, - "unusualDataTypes.mac_address": "00:11:22:33:44:55", - "unusualDataTypes.point_column": Object { - "x": 2, - "y": 2, - }, - "unusualDataTypes.status": "new", - "unusualDataTypes.text_column": "Goodbye, world!", - "unusualDataTypes.xml_column": "more data", + "Orders.status": "processed", + "Orders.totalAmount": "800", }, Object { - "unusualDataTypes.array": Array [ - 7, - 8, - 9, - ], - "unusualDataTypes.bit_column": "11110000", - "unusualDataTypes.boolean_column": true, - "unusualDataTypes.cidr_column": "192.168.0.0/24", - "unusualDataTypes.id": 3, - "unusualDataTypes.inet_column": "192.168.0.3", - "unusualDataTypes.json": Object { - "key": "value3", - "number": 168, - }, - "unusualDataTypes.jsonb": Object { - "key": "value3", - "number": 168, - }, - "unusualDataTypes.mac_address": "22:33:44:55:66:77", - "unusualDataTypes.point_column": Object { - "x": 3, - "y": 3, - }, - "unusualDataTypes.status": "processed", - "unusualDataTypes.text_column": "PostgreSQL is awesome!", - "unusualDataTypes.xml_column": "even more data", + "Orders.status": "shipped", + "Orders.totalAmount": "600", }, ] `; -exports[`postgresql-cubestore WS Transport #3 Events.count with Events.type order by Events.type DESC, Events.count: #3 Events.count with Events.type order by Events.type DESC, Events.count 1`] = ` +exports[`postgresql-cubestore responseFormat ws+responseFormat=compact option#1+2: result-type 1`] = ` Array [ Object { - "Events.count": "92", - "Events.type": "WatchEvent", - }, - Object { - "Events.count": "2", - "Events.type": "ReleaseEvent", - }, - Object { - "Events.count": "513", - "Events.type": "PushEvent", - }, - Object { - "Events.count": "21", - "Events.type": "PullRequestReviewCommentEvent", - }, - Object { - "Events.count": "32", - "Events.type": "PullRequestEvent", - }, - Object { - "Events.count": "1", - "Events.type": "MemberEvent", + "Orders.status": "processed", + "Orders.totalAmount": "800", }, Object { - "Events.count": "57", - "Events.type": "IssuesEvent", + "Orders.status": "shipped", + "Orders.totalAmount": "600", }, +] +`; + +exports[`postgresql-cubestore responseFormat ws+responseFormat=compact option#1: result-type 1`] = ` +Array [ Object { - "Events.count": "104", - "Events.type": "IssueCommentEvent", + "Orders.status": "processed", + "Orders.totalAmount": "800", }, Object { - "Events.count": "21", - "Events.type": "GollumEvent", + "Orders.status": "shipped", + "Orders.totalAmount": "600", }, +] +`; + +exports[`postgresql-cubestore responseFormat ws+responseFormat=compact option#2: result-type 1`] = ` +Array [ Object { - "Events.count": "21", - "Events.type": "ForkEvent", + "Orders.status": "processed", + "Orders.totalAmount": "800", }, Object { - "Events.count": "14", - "Events.type": "DeleteEvent", + "Orders.status": "shipped", + "Orders.totalAmount": "600", }, +] +`; + +exports[`postgresql-cubestore responseFormat ws+responseFormat=default: result-type 1`] = ` +Array [ Object { - "Events.count": "120", - "Events.type": "CreateEvent", + "Orders.status": "processed", + "Orders.totalAmount": "800", }, Object { - "Events.count": "1", - "Events.type": "CommitCommentEvent", + "Orders.status": "shipped", + "Orders.totalAmount": "600", }, ] `; diff --git a/packages/cubejs-testing/test/__snapshots__/birdbox-postgresql.test.ts.snap b/packages/cubejs-testing/test/__snapshots__/birdbox-postgresql.test.ts.snap index 3cb010729d78e..8ce5e3d76c39c 100644 --- a/packages/cubejs-testing/test/__snapshots__/birdbox-postgresql.test.ts.snap +++ b/packages/cubejs-testing/test/__snapshots__/birdbox-postgresql.test.ts.snap @@ -355,6 +355,45 @@ Array [ ] `; +exports[`postgresql responseFormat http+responseFormat=columnar option#1+2: result-type 1`] = ` +Array [ + Object { + "Orders.status": "processed", + "Orders.totalAmount": "800", + }, + Object { + "Orders.status": "shipped", + "Orders.totalAmount": "600", + }, +] +`; + +exports[`postgresql responseFormat http+responseFormat=columnar option#1: result-type 1`] = ` +Array [ + Object { + "Orders.status": "processed", + "Orders.totalAmount": "800", + }, + Object { + "Orders.status": "shipped", + "Orders.totalAmount": "600", + }, +] +`; + +exports[`postgresql responseFormat http+responseFormat=columnar option#2: result-type 1`] = ` +Array [ + Object { + "Orders.status": "processed", + "Orders.totalAmount": "800", + }, + Object { + "Orders.status": "shipped", + "Orders.totalAmount": "600", + }, +] +`; + exports[`postgresql responseFormat http+responseFormat=compact option#1+2: result-type 1`] = ` Array [ Object { @@ -407,6 +446,45 @@ Array [ ] `; +exports[`postgresql responseFormat ws+responseFormat=columnar option#1+2: result-type 1`] = ` +Array [ + Object { + "Orders.status": "processed", + "Orders.totalAmount": "800", + }, + Object { + "Orders.status": "shipped", + "Orders.totalAmount": "600", + }, +] +`; + +exports[`postgresql responseFormat ws+responseFormat=columnar option#1: result-type 1`] = ` +Array [ + Object { + "Orders.status": "processed", + "Orders.totalAmount": "800", + }, + Object { + "Orders.status": "shipped", + "Orders.totalAmount": "600", + }, +] +`; + +exports[`postgresql responseFormat ws+responseFormat=columnar option#2: result-type 1`] = ` +Array [ + Object { + "Orders.status": "processed", + "Orders.totalAmount": "800", + }, + Object { + "Orders.status": "shipped", + "Orders.totalAmount": "600", + }, +] +`; + exports[`postgresql responseFormat ws+responseFormat=compact option#1+2: result-type 1`] = ` Array [ Object { diff --git a/packages/cubejs-testing/test/__snapshots__/cli-postgresql.test.ts.snap b/packages/cubejs-testing/test/__snapshots__/cli-postgresql.test.ts.snap index 3cb010729d78e..8ce5e3d76c39c 100644 --- a/packages/cubejs-testing/test/__snapshots__/cli-postgresql.test.ts.snap +++ b/packages/cubejs-testing/test/__snapshots__/cli-postgresql.test.ts.snap @@ -355,6 +355,45 @@ Array [ ] `; +exports[`postgresql responseFormat http+responseFormat=columnar option#1+2: result-type 1`] = ` +Array [ + Object { + "Orders.status": "processed", + "Orders.totalAmount": "800", + }, + Object { + "Orders.status": "shipped", + "Orders.totalAmount": "600", + }, +] +`; + +exports[`postgresql responseFormat http+responseFormat=columnar option#1: result-type 1`] = ` +Array [ + Object { + "Orders.status": "processed", + "Orders.totalAmount": "800", + }, + Object { + "Orders.status": "shipped", + "Orders.totalAmount": "600", + }, +] +`; + +exports[`postgresql responseFormat http+responseFormat=columnar option#2: result-type 1`] = ` +Array [ + Object { + "Orders.status": "processed", + "Orders.totalAmount": "800", + }, + Object { + "Orders.status": "shipped", + "Orders.totalAmount": "600", + }, +] +`; + exports[`postgresql responseFormat http+responseFormat=compact option#1+2: result-type 1`] = ` Array [ Object { @@ -407,6 +446,45 @@ Array [ ] `; +exports[`postgresql responseFormat ws+responseFormat=columnar option#1+2: result-type 1`] = ` +Array [ + Object { + "Orders.status": "processed", + "Orders.totalAmount": "800", + }, + Object { + "Orders.status": "shipped", + "Orders.totalAmount": "600", + }, +] +`; + +exports[`postgresql responseFormat ws+responseFormat=columnar option#1: result-type 1`] = ` +Array [ + Object { + "Orders.status": "processed", + "Orders.totalAmount": "800", + }, + Object { + "Orders.status": "shipped", + "Orders.totalAmount": "600", + }, +] +`; + +exports[`postgresql responseFormat ws+responseFormat=columnar option#2: result-type 1`] = ` +Array [ + Object { + "Orders.status": "processed", + "Orders.totalAmount": "800", + }, + Object { + "Orders.status": "shipped", + "Orders.totalAmount": "600", + }, +] +`; + exports[`postgresql responseFormat ws+responseFormat=compact option#1+2: result-type 1`] = ` Array [ Object { diff --git a/packages/cubejs-testing/test/abstract-test-case.ts b/packages/cubejs-testing/test/abstract-test-case.ts index cc2bc4c02a357..910018b8012ea 100644 --- a/packages/cubejs-testing/test/abstract-test-case.ts +++ b/packages/cubejs-testing/test/abstract-test-case.ts @@ -176,34 +176,9 @@ export function createBirdBoxTestCase( describe('responseFormat', () => { const responses: unknown[] = []; - let transport: WebSocketTransport; - let http: CubeApi; - let ws: CubeApi; - - beforeAll(async () => { - try { - transport = new WebSocketTransport({ - apiUrl: birdbox.configuration.apiUrl, - }); - http = cubejs(async () => 'test', { - apiUrl: birdbox.configuration.apiUrl, - }); - ws = cubejs(async () => 'test', { - apiUrl: birdbox.configuration.apiUrl, - transport, - }); - } catch (e) { - console.log(e); - process.exit(1); - } - }); - - afterAll(async () => { - await transport.close(); - }); test('http+responseFormat=default', async () => { - const response = await http.load({ + const response = await httpClient.load({ dimensions: ['Orders.status'], measures: ['Orders.totalAmount'], limit: 2, @@ -213,7 +188,7 @@ export function createBirdBoxTestCase( }); test('http+responseFormat=compact option#1', async () => { - const response = await http.load({ + const response = await httpClient.load({ dimensions: ['Orders.status'], measures: ['Orders.totalAmount'], limit: 2, @@ -224,7 +199,7 @@ export function createBirdBoxTestCase( }); test('http+responseFormat=compact option#2', async () => { - const response = await http.load( + const response = await httpClient.load( { dimensions: ['Orders.status'], measures: ['Orders.totalAmount'], @@ -239,7 +214,7 @@ export function createBirdBoxTestCase( }); test('http+responseFormat=compact option#1+2', async () => { - const response = await http.load( + const response = await httpClient.load( { dimensions: ['Orders.status'], measures: ['Orders.totalAmount'], @@ -255,7 +230,7 @@ export function createBirdBoxTestCase( }); test('ws+responseFormat=default', async () => { - const response = await ws.load({ + const response = await wsClient.load({ dimensions: ['Orders.status'], measures: ['Orders.totalAmount'], limit: 2, @@ -265,7 +240,7 @@ export function createBirdBoxTestCase( }); test('ws+responseFormat=compact option#1', async () => { - const response = await ws.load({ + const response = await wsClient.load({ dimensions: ['Orders.status'], measures: ['Orders.totalAmount'], limit: 2, @@ -276,7 +251,7 @@ export function createBirdBoxTestCase( }); test('ws+responseFormat=compact option#2', async () => { - const response = await ws.load( + const response = await wsClient.load( { dimensions: ['Orders.status'], measures: ['Orders.totalAmount'], @@ -291,7 +266,7 @@ export function createBirdBoxTestCase( }); test('ws+responseFormat=compact option#1+2', async () => { - const response = await ws.load( + const response = await wsClient.load( { dimensions: ['Orders.status'], measures: ['Orders.totalAmount'], @@ -306,21 +281,95 @@ export function createBirdBoxTestCase( expect(response.rawData()).toMatchSnapshot('result-type'); }); + test('http+responseFormat=columnar option#1', async () => { + const response = await httpClient.load({ + dimensions: ['Orders.status'], + measures: ['Orders.totalAmount'], + limit: 2, + responseFormat: 'columnar', + }); + responses.push(response); + expect(response.rawData()).toMatchSnapshot('result-type'); + }); + + test('http+responseFormat=columnar option#2', async () => { + const response = await httpClient.load( + { + dimensions: ['Orders.status'], + measures: ['Orders.totalAmount'], + limit: 2, + }, + undefined, + undefined, + 'columnar', + ); + responses.push(response); + expect(response.rawData()).toMatchSnapshot('result-type'); + }); + + test('http+responseFormat=columnar option#1+2', async () => { + const response = await httpClient.load( + { + dimensions: ['Orders.status'], + measures: ['Orders.totalAmount'], + limit: 2, + responseFormat: 'columnar', + }, + undefined, + undefined, + 'columnar', + ); + responses.push(response); + expect(response.rawData()).toMatchSnapshot('result-type'); + }); + + test('ws+responseFormat=columnar option#1', async () => { + const response = await wsClient.load({ + dimensions: ['Orders.status'], + measures: ['Orders.totalAmount'], + limit: 2, + responseFormat: 'columnar', + }); + responses.push(response); + expect(response.rawData()).toMatchSnapshot('result-type'); + }); + + test('ws+responseFormat=columnar option#2', async () => { + const response = await wsClient.load( + { + dimensions: ['Orders.status'], + measures: ['Orders.totalAmount'], + limit: 2, + }, + undefined, + undefined, + 'columnar', + ); + responses.push(response); + expect(response.rawData()).toMatchSnapshot('result-type'); + }); + + test('ws+responseFormat=columnar option#1+2', async () => { + const response = await wsClient.load( + { + dimensions: ['Orders.status'], + measures: ['Orders.totalAmount'], + limit: 2, + responseFormat: 'columnar', + }, + undefined, + undefined, + 'columnar', + ); + responses.push(response); + expect(response.rawData()).toMatchSnapshot('result-type'); + }); + test('responses', () => { - // @ts-ignore - expect(responses[0].rawData()).toEqual(responses[1].rawData()); - // @ts-ignore - expect(responses[0].rawData()).toEqual(responses[2].rawData()); - // @ts-ignore - expect(responses[0].rawData()).toEqual(responses[3].rawData()); - // @ts-ignore - expect(responses[0].rawData()).toEqual(responses[4].rawData()); - // @ts-ignore - expect(responses[0].rawData()).toEqual(responses[5].rawData()); - // @ts-ignore - expect(responses[0].rawData()).toEqual(responses[6].rawData()); - // @ts-ignore - expect(responses[0].rawData()).toEqual(responses[7].rawData()); + for (let i = 1; i < responses.length; i++) { + // @ts-ignore + expect(responses[0].rawData()).toEqual(responses[i].rawData()); + } }); }); diff --git a/rust/cubeorchestrator/Cargo.lock b/rust/cubeorchestrator/Cargo.lock index 3bc48b61ed7d8..5c786d4d76f05 100644 --- a/rust/cubeorchestrator/Cargo.lock +++ b/rust/cubeorchestrator/Cargo.lock @@ -17,6 +17,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -32,6 +41,18 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + [[package]] name = "anyhow" version = "1.0.93" @@ -71,6 +92,12 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.94" @@ -98,18 +125,110 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "num-traits", + "oorandom", + "page_size", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "cubeorchestrator" version = "0.1.0" dependencies = [ "anyhow", "chrono", + "criterion", "cubeshared", "indexmap", "itertools", @@ -153,6 +272,17 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7" +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -306,6 +436,22 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -330,6 +476,21 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "regex" +version = "1.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + [[package]] name = "rustc-demangle" version = "0.1.20" @@ -351,6 +512,15 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "semver" version = "1.0.23" @@ -423,6 +593,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tokio" version = "1.37.0" @@ -439,6 +619,16 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasm-bindgen" version = "0.2.97" @@ -494,6 +684,37 @@ version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.52.0" @@ -503,6 +724,21 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.52.4" @@ -559,3 +795,23 @@ name = "windows_x86_64_msvc" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/rust/cubeorchestrator/Cargo.toml b/rust/cubeorchestrator/Cargo.toml index f1c400bb7bfaa..600766a8d2531 100644 --- a/rust/cubeorchestrator/Cargo.toml +++ b/rust/cubeorchestrator/Cargo.toml @@ -16,3 +16,10 @@ indexmap = { version = "2.0", features = ["serde"] } version = "=1" default-features = false features = ["napi-1", "napi-4", "napi-6", "futures"] + +[dev-dependencies] +criterion = { version = "0.8", default-features = false, features = ["html_reports"] } + +[[bench]] +name = "transform" +harness = false diff --git a/rust/cubeorchestrator/benches/transform.rs b/rust/cubeorchestrator/benches/transform.rs new file mode 100644 index 0000000000000..3067475e75612 --- /dev/null +++ b/rust/cubeorchestrator/benches/transform.rs @@ -0,0 +1,149 @@ +use std::collections::HashMap; +use std::hint::black_box; + +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use cubeorchestrator::query_message_parser::QueryResult; +use cubeorchestrator::query_result_transform::{DBResponsePrimitive, TransformedData}; +use cubeorchestrator::transport::{ + ConfigItem, JsRawData, MemberOrMemberExpression, NormalizedQuery, QueryType, ResultType, + TransformDataRequest, +}; +use indexmap::IndexMap; + +const DIMENSIONS: &[(&str, &str)] = &[ + ("Sales.country", "sales__country"), + ("Sales.city", "sales__city"), + ("Sales.region", "sales__region"), + ("Sales.product", "sales__product"), + ("Sales.category", "sales__category"), + ("Sales.segment", "sales__segment"), +]; + +const MEASURES: &[(&str, &str)] = &[ + ("Sales.revenue", "sales__revenue"), + ("Sales.profit", "sales__profit"), + ("Sales.discount", "sales__discount"), + ("Sales.count", "sales__count"), +]; + +fn config_item(member_type: &str) -> ConfigItem { + ConfigItem { + title: None, + short_title: None, + description: None, + member_type: Some(member_type.to_string()), + format: None, + currency: None, + meta: None, + drill_members: None, + drill_members_grouped: None, + granularities: None, + granularity: None, + } +} + +fn build_request(res_type: Option) -> TransformDataRequest { + let mut alias_to_member_name_map = HashMap::new(); + let mut annotation = HashMap::new(); + + for (member, alias) in DIMENSIONS { + alias_to_member_name_map.insert((*alias).to_string(), (*member).to_string()); + annotation.insert((*member).to_string(), config_item("string")); + } + for (member, alias) in MEASURES { + alias_to_member_name_map.insert((*alias).to_string(), (*member).to_string()); + annotation.insert((*member).to_string(), config_item("number")); + } + + let dimensions = DIMENSIONS + .iter() + .map(|(m, _)| MemberOrMemberExpression::Member((*m).to_string())) + .collect(); + let measures = MEASURES + .iter() + .map(|(m, _)| MemberOrMemberExpression::Member((*m).to_string())) + .collect(); + + let query = NormalizedQuery { + measures: Some(measures), + dimensions: Some(dimensions), + time_dimensions: None, + segments: None, + limit: None, + offset: None, + total: None, + total_query: None, + timezone: Some("UTC".to_string()), + renew_query: None, + ungrouped: None, + response_format: None, + filters: None, + row_limit: None, + order: None, + query_type: Some(QueryType::RegularQuery), + }; + + TransformDataRequest { + alias_to_member_name_map, + annotation, + query, + query_type: Some(QueryType::RegularQuery), + res_type, + } +} + +fn build_dataset(row_count: usize) -> JsRawData { + let dim_count = DIMENSIONS.len(); + let total_cols = dim_count + MEASURES.len(); + let mut rows = Vec::with_capacity(row_count); + + for i in 0..row_count { + let mut row = IndexMap::with_capacity(total_cols); + for (j, (_, alias)) in DIMENSIONS.iter().enumerate() { + row.insert( + (*alias).to_string(), + DBResponsePrimitive::String(format!("dim_{}_{}", j, i % 1000)), + ); + } + for (j, (_, alias)) in MEASURES.iter().enumerate() { + row.insert( + (*alias).to_string(), + DBResponsePrimitive::Number(((i * (j + 1)) as f64) * 0.5), + ); + } + rows.push(row); + } + + rows +} + +fn bench_transform(c: &mut Criterion) { + let mut group = c.benchmark_group("TransformedData::transform"); + + for &row_count in &[1_000usize, 10_000, 50_000, 100_000] { + let raw = + QueryResult::from_js_raw_data(build_dataset(row_count)).expect("from_js_raw_data"); + + group.throughput(Throughput::Elements(row_count as u64)); + + for (label, res_type) in [ + ("compact", Some(ResultType::Compact)), + ("columnar", Some(ResultType::Columnar)), + ("vanilla", None), + ] { + let request = build_request(res_type); + group.bench_with_input(BenchmarkId::new(label, row_count), &row_count, |b, _| { + b.iter(|| { + let result = TransformedData::transform(black_box(&request), black_box(&raw)) + .expect("transform"); + black_box(result); + }); + }); + } + } + + group.finish(); +} + +criterion_group!(benches, bench_transform); +criterion_main!(benches); diff --git a/rust/cubeorchestrator/src/query_result_transform.rs b/rust/cubeorchestrator/src/query_result_transform.rs index 56ff3aed74cab..ee89bf2976420 100644 --- a/rust/cubeorchestrator/src/query_result_transform.rs +++ b/rust/cubeorchestrator/src/query_result_transform.rs @@ -322,6 +322,27 @@ pub fn get_members( Ok((members_map, members_arr)) } +pub fn transpose_to_columns( + members: &[String], + dataset: Vec>, +) -> Vec> { + let row_count = dataset.len(); + let col_count = members.len(); + + let mut columns: Vec> = (0..col_count) + .map(|_| Vec::with_capacity(row_count)) + .collect(); + + for row in dataset { + let mut row_iter = row.into_iter(); + for col in columns.iter_mut().take(col_count) { + col.push(row_iter.next().unwrap_or(DBResponsePrimitive::Null)); + } + } + + columns +} + /// Convert DB response object to the compact output format. pub fn get_compact_row( members_to_alias_map: &IndexMap, @@ -573,6 +594,10 @@ pub enum TransformedData { members: Vec, dataset: Vec>, }, + Columnar { + members: Vec, + columns: Vec>, + }, Vanilla(Vec>), } @@ -615,6 +640,25 @@ impl TransformedData { .collect::>>()?; Ok(TransformedData::Compact { members, dataset }) } + Some(ResultType::Columnar) => { + let dataset: Vec> = cube_store_result + .rows + .iter() + .map(|row| { + get_compact_row( + &members_to_alias_map, + annotation, + query_type, + &members, + query.time_dimensions.as_ref(), + row, + &cube_store_result.columns_pos, + ) + }) + .collect::>>()?; + let columns = transpose_to_columns(&members, dataset); + Ok(TransformedData::Columnar { members, columns }) + } _ => { let dataset: Vec<_> = cube_store_result .rows @@ -1582,6 +1626,65 @@ mod tests { Ok(()) } + ( + TransformedData::Columnar { + members: left_members, + columns: left_columns, + }, + TransformedData::Columnar { + members: right_members, + columns: right_columns, + }, + ) => { + let mut left_sorted_members = left_members.clone(); + let mut right_sorted_members = right_members.clone(); + left_sorted_members.sort(); + right_sorted_members.sort(); + + if left_sorted_members != right_sorted_members { + return Err(TestError("Members do not match after sorting".to_string())); + } + + if left_columns.len() != right_columns.len() { + return Err(TestError( + "Column counts do not match between Columnar results".to_string(), + )); + } + + let mut member_index_map = HashMap::new(); + for (i, member) in left_members.iter().enumerate() { + if let Some(right_index) = right_members.iter().position(|x| x == member) { + member_index_map.insert(i, right_index); + } else { + return Err(TestError("Member not found in right object".to_string())); + } + } + + for (left_idx, left_column) in left_columns.iter().enumerate() { + let right_idx = *member_index_map.get(&left_idx).unwrap(); + let right_column = &right_columns[right_idx]; + if left_column.len() != right_column.len() { + return Err(TestError(format!( + "Column {} (member {}) row counts differ: {} != {}", + left_idx, + left_members[left_idx], + left_column.len(), + right_column.len() + ))); + } + for (row, left_value) in left_column.iter().enumerate() { + let right_value = &right_column[row]; + if left_value != right_value { + return Err(TestError(format!( + "Columnar value at row {} for member '{}' differs: {} != {}", + row, left_members[left_idx], left_value, right_value + ))); + } + } + } + + Ok(()) + } (TransformedData::Vanilla(left_dataset), TransformedData::Vanilla(right_dataset)) => { if left_dataset.len() != right_dataset.len() { return Err(TestError( @@ -2856,4 +2959,115 @@ mod tests { } } } + + /// Run the same fixture through both `Compact` and `Columnar` transforms and + /// assert that the columnar columns are the column-major transpose of the + /// compact dataset. This pins the contract: same data, different orientation. + fn assert_columnar_matches_compact(fixture: &str) -> Result<()> { + let mut test_data = TEST_SUITE_DATA.get(fixture).unwrap().clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + + test_data.request.res_type = Some(ResultType::Compact); + let compact = TransformedData::transform(&test_data.request, &raw_data)?; + let (compact_members, compact_dataset) = match compact { + TransformedData::Compact { members, dataset } => (members, dataset), + _ => panic!("expected Compact"), + }; + + test_data.request.res_type = Some(ResultType::Columnar); + let columnar = TransformedData::transform(&test_data.request, &raw_data)?; + let (columnar_members, columnar_columns) = match columnar { + TransformedData::Columnar { members, columns } => (members, columns), + _ => panic!("expected Columnar"), + }; + + assert_eq!( + compact_members, columnar_members, + "members must match across formats" + ); + assert_eq!( + columnar_columns.len(), + compact_members.len(), + "one column per member" + ); + for (col_idx, column) in columnar_columns.iter().enumerate() { + assert_eq!( + column.len(), + compact_dataset.len(), + "column {} length must equal row count", + col_idx + ); + for (row_idx, expected_row) in compact_dataset.iter().enumerate() { + assert_eq!( + &column[row_idx], &expected_row[col_idx], + "value at column {} row {} must match compact dataset", + col_idx, row_idx + ); + } + } + Ok(()) + } + + #[test] + fn test_regular_discount_by_city_columnar() -> Result<()> { + assert_columnar_matches_compact("regular_discount_by_city") + } + + #[test] + fn test_regular_profit_by_postal_code_columnar() -> Result<()> { + assert_columnar_matches_compact("regular_profit_by_postal_code") + } + + #[test] + fn test_compare_date_range_count_by_order_date_columnar() -> Result<()> { + assert_columnar_matches_compact("compare_date_range_count_by_order_date") + } + + #[test] + fn test_blending_query_multiple_granularities_columnar() -> Result<()> { + assert_columnar_matches_compact("blending_query_multiple_granularities") + } + + #[test] + fn test_transpose_to_columns_basic() { + let members = vec!["a".to_string(), "b".to_string()]; + let dataset = vec![ + vec![ + DBResponsePrimitive::Number(1.0), + DBResponsePrimitive::String("x".to_string()), + ], + vec![ + DBResponsePrimitive::Number(2.0), + DBResponsePrimitive::String("y".to_string()), + ], + ]; + let columns = transpose_to_columns(&members, dataset); + assert_eq!(columns.len(), 2); + assert_eq!( + columns[0], + vec![ + DBResponsePrimitive::Number(1.0), + DBResponsePrimitive::Number(2.0), + ] + ); + assert_eq!( + columns[1], + vec![ + DBResponsePrimitive::String("x".to_string()), + DBResponsePrimitive::String("y".to_string()), + ] + ); + } + + #[test] + fn test_transpose_to_columns_pads_short_rows_with_null() { + let members = vec!["a".to_string(), "b".to_string(), "c".to_string()]; + let dataset = vec![vec![ + DBResponsePrimitive::Number(1.0), + DBResponsePrimitive::String("x".to_string()), + ]]; + let columns = transpose_to_columns(&members, dataset); + assert_eq!(columns.len(), 3); + assert_eq!(columns[2], vec![DBResponsePrimitive::Null]); + } } diff --git a/rust/cubeorchestrator/src/transport.rs b/rust/cubeorchestrator/src/transport.rs index 77cc3c77b77dc..a52eb1793a89f 100644 --- a/rust/cubeorchestrator/src/transport.rs +++ b/rust/cubeorchestrator/src/transport.rs @@ -9,6 +9,7 @@ use std::{collections::HashMap, fmt::Display}; pub enum ResultType { Default, Compact, + Columnar, } #[derive(Default, Debug, Clone, Serialize, Deserialize)] diff --git a/rust/cubesql/cubeclient/.openapi-generator/FILES b/rust/cubesql/cubeclient/.openapi-generator/FILES index bc0167db69881..20de347b629a8 100644 --- a/rust/cubesql/cubeclient/.openapi-generator/FILES +++ b/rust/cubesql/cubeclient/.openapi-generator/FILES @@ -6,6 +6,7 @@ src/models/v1_cube_meta_custom_numeric_format.rs src/models/v1_cube_meta_custom_time_format.rs src/models/v1_cube_meta_dimension.rs src/models/v1_cube_meta_dimension_granularity.rs +src/models/v1_cube_meta_dimension_order.rs src/models/v1_cube_meta_folder.rs src/models/v1_cube_meta_format.rs src/models/v1_cube_meta_format_description.rs @@ -29,4 +30,7 @@ src/models/v1_load_request_query_time_dimension.rs src/models/v1_load_response.rs src/models/v1_load_result.rs src/models/v1_load_result_annotation.rs +src/models/v1_load_result_data.rs +src/models/v1_load_result_data_columnar.rs +src/models/v1_load_result_data_compact.rs src/models/v1_meta_response.rs diff --git a/rust/cubesql/cubeclient/.openapi-generator/VERSION b/rust/cubesql/cubeclient/.openapi-generator/VERSION index 6328c5424a4a6..a29ba3d5cecb3 100644 --- a/rust/cubesql/cubeclient/.openapi-generator/VERSION +++ b/rust/cubesql/cubeclient/.openapi-generator/VERSION @@ -1 +1 @@ -7.17.0 +7.21.0 diff --git a/rust/cubesql/cubeclient/src/apis/default_api.rs b/rust/cubesql/cubeclient/src/apis/default_api.rs index ccb54a893a49b..15aa6f3661603 100644 --- a/rust/cubesql/cubeclient/src/apis/default_api.rs +++ b/rust/cubesql/cubeclient/src/apis/default_api.rs @@ -1,6 +1,6 @@ use log::{debug, error}; use reqwest; -use serde::{Deserialize, Serialize}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; use uuid::Uuid; use super::{configuration, Error}; @@ -24,10 +24,13 @@ pub enum MetaV1Error { UnknownValue(serde_json::Value), } -pub async fn load_v1( +pub async fn load_v1( configuration: &configuration::Configuration, v1_load_request: Option, -) -> Result> { +) -> Result, Error> +where + D: DeserializeOwned, +{ let local_var_client = &configuration.client; let request_id = Uuid::new_v4().to_string(); @@ -61,7 +64,7 @@ pub async fn load_v1( if !local_var_status.is_client_error() && !local_var_status.is_server_error() { let response_ok = - serde_json::from_str::(&local_var_content) + serde_json::from_str::>(&local_var_content) .map_err(Error::from); if response_ok.is_ok() { return response_ok; @@ -235,7 +238,7 @@ mod tests { let mut configuration = Configuration::new(client); configuration.base_path = server.uri(); - let resp = load_v1(&configuration, None).await; + let resp = load_v1::(&configuration, None).await; match resp { Ok(_) => {} Err(e) => panic!("must be successful, {:?}", e), diff --git a/rust/cubesql/cubeclient/src/models/mod.rs b/rust/cubesql/cubeclient/src/models/mod.rs index 0d8bbb6166306..c90a33c7d3770 100644 --- a/rust/cubesql/cubeclient/src/models/mod.rs +++ b/rust/cubesql/cubeclient/src/models/mod.rs @@ -11,8 +11,8 @@ pub use self::v1_cube_meta_custom_time_format::Type as V1CubeMetaCustomTimeForma pub mod v1_cube_meta_dimension; pub use self::v1_cube_meta_dimension::V1CubeMetaDimension; pub mod v1_cube_meta_dimension_granularity; -pub mod v1_cube_meta_dimension_order; pub use self::v1_cube_meta_dimension_granularity::V1CubeMetaDimensionGranularity; +pub mod v1_cube_meta_dimension_order; pub use self::v1_cube_meta_dimension_order::V1CubeMetaDimensionOrder; pub mod v1_cube_meta_folder; pub use self::v1_cube_meta_folder::V1CubeMetaFolder; @@ -67,5 +67,13 @@ pub mod v1_load_result; pub use self::v1_load_result::V1LoadResult; pub mod v1_load_result_annotation; pub use self::v1_load_result_annotation::V1LoadResultAnnotation; +pub mod v1_load_result_data; +pub use self::v1_load_result_data::V1LoadResultData; +pub mod v1_load_result_data_columnar; +pub use self::v1_load_result_data_columnar::V1LoadResultDataColumnar; +pub mod v1_load_result_data_compact; +pub use self::v1_load_result_data_compact::V1LoadResultDataCompact; +pub mod v1_load_result_data_row; +pub use self::v1_load_result_data_row::V1LoadResultDataRow; pub mod v1_meta_response; pub use self::v1_meta_response::V1MetaResponse; diff --git a/rust/cubesql/cubeclient/src/models/v1_cube_meta_dimension.rs b/rust/cubesql/cubeclient/src/models/v1_cube_meta_dimension.rs index 55d0be8ac4a24..e7b544612b9dc 100644 --- a/rust/cubesql/cubeclient/src/models/v1_cube_meta_dimension.rs +++ b/rust/cubesql/cubeclient/src/models/v1_cube_meta_dimension.rs @@ -34,6 +34,7 @@ pub struct V1CubeMetaDimension { pub format: Option>, #[serde(rename = "formatDescription", skip_serializing_if = "Option::is_none")] pub format_description: Option>, + /// ISO 4217 currency code in uppercase (3 characters, e.g. USD, EUR) #[serde(rename = "currency", skip_serializing_if = "Option::is_none")] pub currency: Option, #[serde(rename = "order", skip_serializing_if = "Option::is_none")] diff --git a/rust/cubesql/cubeclient/src/models/v1_cube_meta_dimension_order.rs b/rust/cubesql/cubeclient/src/models/v1_cube_meta_dimension_order.rs index 5cb5e3cd815ed..655fc449eb41c 100644 --- a/rust/cubesql/cubeclient/src/models/v1_cube_meta_dimension_order.rs +++ b/rust/cubesql/cubeclient/src/models/v1_cube_meta_dimension_order.rs @@ -8,9 +8,9 @@ * Generated by: https://openapi-generator.tech */ +use crate::models; use serde::{Deserialize, Serialize}; -/// Default sort order for a dimension #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] pub enum V1CubeMetaDimensionOrder { #[serde(rename = "asc")] @@ -19,6 +19,15 @@ pub enum V1CubeMetaDimensionOrder { Desc, } +impl std::fmt::Display for V1CubeMetaDimensionOrder { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::Asc => write!(f, "asc"), + Self::Desc => write!(f, "desc"), + } + } +} + impl Default for V1CubeMetaDimensionOrder { fn default() -> V1CubeMetaDimensionOrder { Self::Asc diff --git a/rust/cubesql/cubeclient/src/models/v1_cube_meta_measure.rs b/rust/cubesql/cubeclient/src/models/v1_cube_meta_measure.rs index 449b07cd8e9ce..b9e5167ee8e7e 100644 --- a/rust/cubesql/cubeclient/src/models/v1_cube_meta_measure.rs +++ b/rust/cubesql/cubeclient/src/models/v1_cube_meta_measure.rs @@ -31,6 +31,7 @@ pub struct V1CubeMetaMeasure { pub format: Option>, #[serde(rename = "formatDescription", skip_serializing_if = "Option::is_none")] pub format_description: Option>, + /// ISO 4217 currency code in uppercase (3 characters, e.g. USD, EUR) #[serde(rename = "currency", skip_serializing_if = "Option::is_none")] pub currency: Option, /// When measure is defined in View, it keeps the original path: Cube.measure diff --git a/rust/cubesql/cubeclient/src/models/v1_load_request.rs b/rust/cubesql/cubeclient/src/models/v1_load_request.rs index 8363498c4fa63..ec494ca1d20ed 100644 --- a/rust/cubesql/cubeclient/src/models/v1_load_request.rs +++ b/rust/cubesql/cubeclient/src/models/v1_load_request.rs @@ -18,7 +18,7 @@ pub struct V1LoadRequest { #[serde(rename = "cache", skip_serializing_if = "Option::is_none")] pub cache: Option, #[serde(rename = "query", skip_serializing_if = "Option::is_none")] - pub query: Option, + pub query: Option>, } impl V1LoadRequest { diff --git a/rust/cubesql/cubeclient/src/models/v1_load_request_query.rs b/rust/cubesql/cubeclient/src/models/v1_load_request_query.rs index 3cf2d9c6500e5..043f4f776aa65 100644 --- a/rust/cubesql/cubeclient/src/models/v1_load_request_query.rs +++ b/rust/cubesql/cubeclient/src/models/v1_load_request_query.rs @@ -37,6 +37,9 @@ pub struct V1LoadRequestQuery { pub join_hints: Option>>, #[serde(rename = "timezone", skip_serializing_if = "Option::is_none")] pub timezone: Option, + /// Output format of the result `data` payload. `default` returns row-oriented data (`V1LoadResultDataRow`); `compact` returns a `{ members, dataset }` object with rows of primitive arrays (`V1LoadResultDataCompact`); `columnar` returns a `{ members, columns }` object with one primitive array per member (`V1LoadResultDataColumnar`). + #[serde(rename = "responseFormat", skip_serializing_if = "Option::is_none")] + pub response_format: Option, } impl V1LoadRequestQuery { @@ -54,6 +57,23 @@ impl V1LoadRequestQuery { subquery_joins: None, join_hints: None, timezone: None, + response_format: None, } } } +/// Output format of the result `data` payload. `default` returns row-oriented data (`V1LoadResultDataRow`); `compact` returns a `{ members, dataset }` object with rows of primitive arrays (`V1LoadResultDataCompact`); `columnar` returns a `{ members, columns }` object with one primitive array per member (`V1LoadResultDataColumnar`). +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub enum ResponseFormat { + #[serde(rename = "default")] + Default, + #[serde(rename = "compact")] + Compact, + #[serde(rename = "columnar")] + Columnar, +} + +impl Default for ResponseFormat { + fn default() -> ResponseFormat { + Self::Default + } +} diff --git a/rust/cubesql/cubeclient/src/models/v1_load_response.rs b/rust/cubesql/cubeclient/src/models/v1_load_response.rs index 21b52b844b68a..4f992522269d0 100644 --- a/rust/cubesql/cubeclient/src/models/v1_load_response.rs +++ b/rust/cubesql/cubeclient/src/models/v1_load_response.rs @@ -6,13 +6,16 @@ * The version of the OpenAPI document: 1.0.0 * * Generated by: https://openapi-generator.tech + * + * Hand-maintained: generic over the result `data` payload type. See + * `v1_load_result.rs` for the rationale. */ use crate::models; use serde::{Deserialize, Serialize}; -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct V1LoadResponse { +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct V1LoadResponse { #[serde(rename = "pivotQuery", skip_serializing_if = "Option::is_none")] pub pivot_query: Option, #[serde(rename = "slowQuery", skip_serializing_if = "Option::is_none")] @@ -20,11 +23,22 @@ pub struct V1LoadResponse { #[serde(rename = "queryType", skip_serializing_if = "Option::is_none")] pub query_type: Option, #[serde(rename = "results")] - pub results: Vec, + pub results: Vec>, +} + +impl Default for V1LoadResponse { + fn default() -> Self { + Self { + pivot_query: None, + slow_query: None, + query_type: None, + results: Vec::new(), + } + } } -impl V1LoadResponse { - pub fn new(results: Vec) -> V1LoadResponse { +impl V1LoadResponse { + pub fn new(results: Vec>) -> V1LoadResponse { V1LoadResponse { pivot_query: None, slow_query: None, diff --git a/rust/cubesql/cubeclient/src/models/v1_load_result.rs b/rust/cubesql/cubeclient/src/models/v1_load_result.rs index 54aa48ffcad12..5911c19a820a7 100644 --- a/rust/cubesql/cubeclient/src/models/v1_load_result.rs +++ b/rust/cubesql/cubeclient/src/models/v1_load_result.rs @@ -6,30 +6,46 @@ * The version of the OpenAPI document: 1.0.0 * * Generated by: https://openapi-generator.tech + * + * Hand-maintained: this file is generic over the `data` payload type so that + * callers who know the wire format (e.g. row vs compact vs columnar) can + * deserialize directly into the concrete type and skip the buffered re-parse + * that `#[serde(untagged)]` on `V1LoadResultData` would otherwise impose. + * Default `D` is `V1LoadResultDataRow` — the format that `load_v1` requests + * by default. */ use crate::models; use serde::{Deserialize, Serialize}; -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct V1LoadResult { +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct V1LoadResult { #[serde(rename = "dataSource", skip_serializing_if = "Option::is_none")] pub data_source: Option, #[serde(rename = "annotation")] pub annotation: Box, #[serde(rename = "data")] - pub data: Vec, + pub data: D, #[serde(rename = "refreshKeyValues", skip_serializing_if = "Option::is_none")] pub refresh_key_values: Option>, #[serde(rename = "lastRefreshTime", skip_serializing_if = "Option::is_none")] pub last_refresh_time: Option, } -impl V1LoadResult { - pub fn new( - annotation: models::V1LoadResultAnnotation, - data: Vec, - ) -> V1LoadResult { +impl Default for V1LoadResult { + fn default() -> Self { + Self { + data_source: None, + annotation: Box::new(models::V1LoadResultAnnotation::default()), + data: D::default(), + refresh_key_values: None, + last_refresh_time: None, + } + } +} + +impl V1LoadResult { + pub fn new(annotation: models::V1LoadResultAnnotation, data: D) -> V1LoadResult { V1LoadResult { data_source: None, annotation: Box::new(annotation), diff --git a/rust/cubesql/cubeclient/src/models/v1_load_result_data.rs b/rust/cubesql/cubeclient/src/models/v1_load_result_data.rs new file mode 100644 index 0000000000000..ab33c7ab8d8bb --- /dev/null +++ b/rust/cubesql/cubeclient/src/models/v1_load_result_data.rs @@ -0,0 +1,29 @@ +/* + * Cube.js + * + * Cube.js Swagger Schema + * + * The version of the OpenAPI document: 1.0.0 + * + * Generated by: https://openapi-generator.tech + */ + +use crate::models; +use serde::{Deserialize, Serialize}; + +/// V1LoadResultData : Result `data` payload. Either an array of objects keyed by member name (default); a compact `{ members, dataset }` object when `responseFormat=compact`; or a columnar `{ members, columns }` object when `responseFormat=columnar`. +/// Result `data` payload. Either an array of objects keyed by member name (default); a compact `{ members, dataset }` object when `responseFormat=compact`; or a columnar `{ members, columns }` object when `responseFormat=columnar`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum V1LoadResultData { + /// Row-oriented (default) data format - an array of rows, each row is an object keyed by member name. + V1LoadResultDataRow(Vec), + V1LoadResultDataCompact(Box), + V1LoadResultDataColumnar(Box), +} + +impl Default for V1LoadResultData { + fn default() -> Self { + Self::V1LoadResultDataRow(Default::default()) + } +} diff --git a/rust/cubesql/cubeclient/src/models/v1_load_result_data_columnar.rs b/rust/cubesql/cubeclient/src/models/v1_load_result_data_columnar.rs new file mode 100644 index 0000000000000..a3b642619f0a3 --- /dev/null +++ b/rust/cubesql/cubeclient/src/models/v1_load_result_data_columnar.rs @@ -0,0 +1,33 @@ +/* + * Cube.js + * + * Cube.js Swagger Schema + * + * The version of the OpenAPI document: 1.0.0 + * + * Generated by: https://openapi-generator.tech + */ + +use crate::models; +use serde::{Deserialize, Serialize}; + +/// V1LoadResultDataColumnar : Columnar data format - members list paired with one primitive array per column. Returned when `responseFormat=columnar` is requested. +#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] +pub struct V1LoadResultDataColumnar { + /// Ordered list of member names. Element `i` of `columns` holds the values for `members[i]` across all rows. + #[serde(rename = "members")] + pub members: Vec, + /// One array per member, in the same order as `members`. Each inner array contains the primitive value of that member for every row (null, boolean, number, string). + #[serde(rename = "columns")] + pub columns: Vec>, +} + +impl V1LoadResultDataColumnar { + /// Columnar data format - members list paired with one primitive array per column. Returned when `responseFormat=columnar` is requested. + pub fn new( + members: Vec, + columns: Vec>, + ) -> V1LoadResultDataColumnar { + V1LoadResultDataColumnar { members, columns } + } +} diff --git a/rust/cubesql/cubeclient/src/models/v1_load_result_data_compact.rs b/rust/cubesql/cubeclient/src/models/v1_load_result_data_compact.rs new file mode 100644 index 0000000000000..f0cfb19c48390 --- /dev/null +++ b/rust/cubesql/cubeclient/src/models/v1_load_result_data_compact.rs @@ -0,0 +1,33 @@ +/* + * Cube.js + * + * Cube.js Swagger Schema + * + * The version of the OpenAPI document: 1.0.0 + * + * Generated by: https://openapi-generator.tech + */ + +use crate::models; +use serde::{Deserialize, Serialize}; + +/// V1LoadResultDataCompact : Compact data format - a single object with the members list and a dataset of primitive arrays. Returned when `responseFormat=compact` is requested. +#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] +pub struct V1LoadResultDataCompact { + /// Ordered list of member names that correspond to each cell position in `dataset` rows. + #[serde(rename = "members")] + pub members: Vec, + /// Array of rows, where each row is an array of primitive values (null, boolean, number, string) aligned with `members`. + #[serde(rename = "dataset")] + pub dataset: Vec>, +} + +impl V1LoadResultDataCompact { + /// Compact data format - a single object with the members list and a dataset of primitive arrays. Returned when `responseFormat=compact` is requested. + pub fn new( + members: Vec, + dataset: Vec>, + ) -> V1LoadResultDataCompact { + V1LoadResultDataCompact { members, dataset } + } +} diff --git a/rust/cubesql/cubeclient/src/models/v1_load_result_data_row.rs b/rust/cubesql/cubeclient/src/models/v1_load_result_data_row.rs new file mode 100644 index 0000000000000..92a03f7b6e292 --- /dev/null +++ b/rust/cubesql/cubeclient/src/models/v1_load_result_data_row.rs @@ -0,0 +1,15 @@ +/* + * Cube.js + * + * Cube.js Swagger Schema + * + * The version of the OpenAPI document: 1.0.0 + * + * Hand-maintained: openapi-generator does not emit a Rust type for top-level + * array schemas, so we mirror the `V1LoadResultDataRow` schema here as a type + * alias. This is the default payload type used by `load_v1` — it's a streamed + * `Vec` that avoids the buffered re-parse cost of the + * untagged `V1LoadResultData` enum on the row-oriented hot path. + */ + +pub type V1LoadResultDataRow = Vec; diff --git a/rust/cubesql/cubesql/src/compile/builder.rs b/rust/cubesql/cubesql/src/compile/builder.rs index 799947c688092..b14dfad12099b 100644 --- a/rust/cubesql/cubesql/src/compile/builder.rs +++ b/rust/cubesql/cubesql/src/compile/builder.rs @@ -154,6 +154,7 @@ impl QueryBuilder { subquery_joins: None, join_hints: None, timezone: None, + response_format: None, }, meta: self.meta, } diff --git a/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs b/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs index b7ab363d0691b..9085596a9031f 100644 --- a/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs +++ b/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs @@ -3538,6 +3538,7 @@ impl WrappedSelectNode { .then_some(prepared_join_subqueries), join_hints: load_request.join_hints.clone(), + response_format: None, }; // TODO time dimensions, filters, segments diff --git a/rust/cubesql/cubesql/src/transport/service.rs b/rust/cubesql/cubesql/src/transport/service.rs index 2ebda4676d736..0b720c0ff11f6 100644 --- a/rust/cubesql/cubesql/src/transport/service.rs +++ b/rust/cubesql/cubesql/src/transport/service.rs @@ -312,7 +312,7 @@ impl TransportService for HttpTransport { // TODO: support meta_fields for HTTP let request = TransportLoadRequest { - query: Some(query), + query: Some(Box::new(query)), query_type: Some("multi".to_string()), cache: cache_mode, };