diff --git a/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts b/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts index 5d616e756eb44..2516c4a74c5ed 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts @@ -175,6 +175,10 @@ export class DatabricksQuery extends BaseQuery { templates.functions.LTRIM = 'LTRIM({{ args|reverse|join(", ") }})'; templates.functions.RTRIM = 'RTRIM({{ args|reverse|join(", ") }})'; templates.functions.DATEDIFF = 'DATEDIFF({{ date_part }}, DATE_TRUNC(\'{{ date_part }}\', {{ args[1] }}), DATE_TRUNC(\'{{ date_part }}\', {{ args[2] }}))'; + // DATEADD is being rewritten to DATE_ADD. The unquoted multi-unit form is used + // because the ANSI form, INTERVAL '2' HOUR, only spans YEAR to SECOND, while sub-day + // intervals are reported in milliseconds + templates.functions.DATE_ADD = '({{ args[0] }} + INTERVAL {{ interval }} {{ date_part }})'; templates.functions.LEAST = 'LEAST({{ args_concat }})'; templates.functions.GREATEST = 'GREATEST({{ args_concat }})'; templates.functions.TRUNC = 'CASE WHEN ({{ args[0] }}) >= 0 THEN FLOOR({{ args_concat }}) ELSE CEIL({{ args_concat }}) END'; diff --git a/packages/cubejs-duckdb-driver/src/DuckDBQuery.ts b/packages/cubejs-duckdb-driver/src/DuckDBQuery.ts index 11a0d4626ed0d..c7023098ccd55 100644 --- a/packages/cubejs-duckdb-driver/src/DuckDBQuery.ts +++ b/packages/cubejs-duckdb-driver/src/DuckDBQuery.ts @@ -67,6 +67,8 @@ export class DuckDBQuery extends BaseQuery { templates.functions.LEAST = 'LEAST({{ args_concat }})'; templates.functions.GREATEST = 'GREATEST({{ args_concat }})'; templates.functions.STRING_AGG = 'STRING_AGG({% if distinct %}DISTINCT {% endif %}{{ args[0] }}, COALESCE({{ args[1] }}, \'\'))'; + // DATEADD is being rewritten to DATE_ADD + templates.functions.DATE_ADD = '({{ args[0] }} + \'{{ interval }} {{ date_part }}\'::interval)'; delete templates.functions.WIDTH_BUCKET; templates.expressions.like = '{{ expr }} {% if negated %}NOT {% endif %}LIKE {{ pattern }}{% if default_escape %} ESCAPE \'\\\'{% endif %}'; templates.expressions.ilike = '{{ expr }} {% if negated %}NOT {% endif %}ILIKE {{ pattern }}{% if default_escape %} ESCAPE \'\\\'{% endif %}'; diff --git a/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts b/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts index 9c2187bc60878..6a988b54e92f8 100644 --- a/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts @@ -265,6 +265,9 @@ export class ClickHouseQuery extends BaseQuery { templates.functions.DATETRUNC = 'DATE_TRUNC({{ args_concat }})'; templates.functions.UTCTIMESTAMP = 'now(\'UTC\')'; templates.functions.STRING_AGG = 'arrayStringConcat(group{% if distinct %}Uniq{% endif %}Array({{ args[0] }}), {{ args[1] }})'; + // DATEADD is being rewritten to DATE_ADD. The operator form is used instead of + // addDate(), which only exists since ClickHouse 23.9 + templates.functions.DATE_ADD = '({{ args[0] }} + INTERVAL {{ interval }} {{ date_part }})'; // TODO: Introduce additional filter in jinja? or parseDateTimeBestEffort? // https://github.com/ClickHouse/ClickHouse/issues/19351 templates.expressions.timestamp_literal = 'parseDateTimeBestEffort(\'{{ value }}\')'; diff --git a/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts b/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts index d462acc766136..b33cea8be43cb 100644 --- a/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts @@ -271,6 +271,8 @@ export class MssqlQuery extends BaseQuery { templates.functions.UTCTIMESTAMP = 'GETUTCDATE()'; // MSSQL ROUND requires 2 arguments: ROUND(number, length) templates.functions.ROUND = 'ROUND({{ args_concat }}{% if args | length < 2 %}, 0{% endif %})'; + // DATEADD is being rewritten to DATE_ADD + templates.functions.DATE_ADD = 'DATEADD({{ date_part }}, {{ interval }}, {{ args[0] }})'; // NOTE: MSSQL does not support DISTINCT clause. No workaround is available delete templates.functions.STRING_AGG; // PERCENTILE_CONT works but requires PARTITION BY diff --git a/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts b/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts index 243e47b7a9f83..bf0c2c01e0011 100644 --- a/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts @@ -187,6 +187,11 @@ export class MysqlQuery extends BaseQuery { const templates = super.sqlTemplates(); templates.functions.STRING_AGG = 'GROUP_CONCAT({% if distinct %}DISTINCT {% endif %}{{ args[0] }} SEPARATOR {{ args[1] }})'; templates.functions.UTCTIMESTAMP = 'UTC_TIMESTAMP()'; + // DATEADD is being rewritten to DATE_ADD, which reports sub-day intervals in + // milliseconds. MySQL has no MILLISECOND unit, so those are scaled to microseconds + templates.functions.DATE_ADD = 'DATE_ADD({{ args[0] }}, INTERVAL ' + + '{% if date_part == "MILLISECOND" %}{{ interval }}000 MICROSECOND' + + '{% else %}{{ interval }} {{ date_part }}{% endif %})'; // PERCENTILE_CONT works but requires PARTITION BY delete templates.functions.PERCENTILECONT; delete templates.functions.WIDTH_BUCKET; diff --git a/packages/cubejs-schema-compiler/src/adapter/PostgresQuery.ts b/packages/cubejs-schema-compiler/src/adapter/PostgresQuery.ts index 5ebd05f05e8f1..4ebed37d312dc 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PostgresQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PostgresQuery.ts @@ -88,7 +88,7 @@ export class PostgresQuery extends BaseQuery { templates.functions.NOW = 'NOW({{ args_concat }})'; templates.functions.UTCTIMESTAMP = '(NOW() AT TIME ZONE \'UTC\')'; // DATEADD is being rewritten to DATE_ADD - // templates.functions.DATEADD = '({{ args[2] }} + \'{{ interval }} {{ date_part }}\'::interval)'; + templates.functions.DATE_ADD = '({{ args[0] }} + \'{{ interval }} {{ date_part }}\'::interval)'; // TODO: is DATEDIFF expr worth documenting? templates.functions.DATEDIFF = 'CASE WHEN LOWER(\'{{ date_part }}\') IN (\'year\', \'quarter\', \'month\') THEN (EXTRACT(YEAR FROM AGE(DATE_TRUNC(\'{{ date_part }}\', {{ args[2] }}), DATE_TRUNC(\'{{ date_part }}\', {{ args[1] }}))) * 12 + EXTRACT(MONTH FROM AGE(DATE_TRUNC(\'{{ date_part }}\', {{ args[2] }}), DATE_TRUNC(\'{{ date_part }}\', {{ args[1] }})))) / CASE LOWER(\'{{ date_part }}\') WHEN \'year\' THEN 12 WHEN \'quarter\' THEN 3 WHEN \'month\' THEN 1 END ELSE EXTRACT(EPOCH FROM DATE_TRUNC(\'{{ date_part }}\', {{ args[2] }}) - DATE_TRUNC(\'{{ date_part }}\', {{ args[1] }})) / EXTRACT(EPOCH FROM \'1 {{ date_part }}\'::interval) END::bigint'; templates.expressions.interval = 'INTERVAL \'{{ interval }}\''; diff --git a/packages/cubejs-schema-compiler/src/adapter/PrestodbQuery.ts b/packages/cubejs-schema-compiler/src/adapter/PrestodbQuery.ts index c8cac69163058..b81d5b114b416 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PrestodbQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PrestodbQuery.ts @@ -163,6 +163,8 @@ export class PrestodbQuery extends BaseQuery { templates.functions.DATETRUNC = 'DATE_TRUNC({{ args_concat }})'; templates.functions.DATEPART = 'DATE_PART({{ args_concat }})'; templates.functions.DATEDIFF = 'DATE_DIFF(\'{{ date_part }}\', {{ args[1] }}, {{ args[2] }})'; + // DATEADD is being rewritten to DATE_ADD + templates.functions.DATE_ADD = 'DATE_ADD(\'{{ date_part }}\', {{ interval }}, {{ args[0] }})'; templates.functions.CURRENTDATE = 'CURRENT_DATE'; templates.functions.UTCTIMESTAMP = 'CAST(NOW() AT TIME ZONE \'UTC\' AS TIMESTAMP)'; templates.functions.TRUNC = 'TRUNCATE({{ args_concat }})'; diff --git a/packages/cubejs-schema-compiler/src/adapter/RedshiftQuery.ts b/packages/cubejs-schema-compiler/src/adapter/RedshiftQuery.ts index e1063e0dd6597..2c7cb9b44f727 100644 --- a/packages/cubejs-schema-compiler/src/adapter/RedshiftQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/RedshiftQuery.ts @@ -88,6 +88,8 @@ export class RedshiftQuery extends PostgresQuery { // nodes, unlike NOW(), which is a leader node–only function. templates.functions.UTCTIMESTAMP = 'GETDATE()'; templates.functions.DATEDIFF = 'DATEDIFF({{ date_part }}, {{ args[1] }}, {{ args[2] }})'; + // DATEADD is being rewritten to DATE_ADD + templates.functions.DATE_ADD = 'DATEADD({{ date_part }}, {{ interval }}, {{ args[0] }})'; templates.functions.STRING_AGG = 'LISTAGG({% if distinct %}DISTINCT {% endif %}{{ args_concat }})'; templates.statements.time_series_select = 'SELECT dates.f::timestamp date_from, dates.t::timestamp date_to \n' + 'FROM (\n' + diff --git a/packages/cubejs-schema-compiler/src/adapter/SnowflakeQuery.ts b/packages/cubejs-schema-compiler/src/adapter/SnowflakeQuery.ts index 88ccca3a6b391..6955afecbdf01 100644 --- a/packages/cubejs-schema-compiler/src/adapter/SnowflakeQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/SnowflakeQuery.ts @@ -114,6 +114,8 @@ export class SnowflakeQuery extends BaseQuery { templates.functions.CHARACTERLENGTH = 'LENGTH({{ args[0] }})'; templates.functions.BTRIM = 'TRIM({{ args_concat }})'; templates.functions.STRING_AGG = 'LISTAGG({% if distinct %}DISTINCT {% endif %}{{ args_concat }})'; + // DATEADD is being rewritten to DATE_ADD + templates.functions.DATE_ADD = 'DATEADD({{ date_part }}, {{ interval }}, {{ args[0] }})'; templates.expressions.extract = 'EXTRACT({{ date_part }} FROM {{ expr }})'; // Snowflake `/` is decimal division even for integer operands (output scale // is dividend scale + 6), while this template must keep PostgreSQL integer diff --git a/rust/cubesql/cubesql/src/compile/test/test_wrapper.rs b/rust/cubesql/cubesql/src/compile/test/test_wrapper.rs index e202347037baa..97a290e842b9f 100644 --- a/rust/cubesql/cubesql/src/compile/test/test_wrapper.rs +++ b/rust/cubesql/cubesql/src/compile/test/test_wrapper.rs @@ -2853,3 +2853,148 @@ async fn test_wrapper_only_system_fields() { displayable(physical_plan.as_ref()).indent() ); } + +/// A per-group aggregate in a CTE, date-filtered and counted on the outside, must be +/// pushed down whole: `DATEADD` in the outer filter is rewritten to `DATE_ADD`, so the +/// filter and the aggregate above it only push down when the data source has a +/// `functions/DATE_ADD` template. +#[tokio::test] +async fn test_wrapper_cte_aggregate_then_date_filter() { + if !Rewriter::sql_push_down_enabled() { + return; + } + init_testing_logger(); + + let query_plan = convert_select_to_query_plan( + r#" + WITH first_orders AS ( + SELECT customer_gender, MIN(order_date) AS first_order_at + FROM KibanaSampleDataEcommerce + WHERE has_subscription = true + GROUP BY 1 + ) + SELECT COUNT(DISTINCT customer_gender) AS customers + FROM first_orders + WHERE first_order_at >= DATEADD('month', -12, CURRENT_DATE()) + AND first_order_at < CURRENT_DATE() + "# + .to_string(), + DatabaseProtocol::PostgreSQL, + ) + .await; + + let logical_plan = query_plan.as_logical_plan(); + let sql = logical_plan.find_cube_scan_wrapped_sql().wrapped_sql.sql; + assert!( + sql.contains("COUNT(DISTINCT"), + "outer aggregate is pushed down: {}", + sql + ); + assert!( + sql.contains("DATE_ADD"), + "outer date filter is pushed down: {}", + sql + ); + + let _physical_plan = query_plan.as_physical_plan().await.unwrap(); +} + +/// `DATEADD` is rewritten to `date_add`, and the dialect template renders it from the +/// `date_part` and `interval` variables rather than from the arguments. The rewrite maps +/// every unit onto one of three parts - sub-day units become `MILLISECOND`, `day` and +/// `week` become `DAY`, and `month`, `quarter` and `year` become `MONTH` - so each dialect +/// has to render all three. These are the real templates from the query classes; the ones +/// used elsewhere in these tests take `args_concat` and would not catch a wrong unit. +#[tokio::test] +async fn test_wrapper_date_add_dialect_templates() { + if !Rewriter::sql_push_down_enabled() { + return; + } + init_testing_logger(); + + let dialects = [ + ( + // PostgresQuery, RedshiftQuery inherits it, DuckDBQuery repeats it + "({{ args[0] }} + '{{ interval }} {{ date_part }}'::interval)", + [ + "(CURRENT_DATE() + '7200000 MILLISECOND'::interval)", + "(CURRENT_DATE() + '14 DAY'::interval)", + "(CURRENT_DATE() + '24 MONTH'::interval)", + ], + ), + ( + // SnowflakeQuery, MssqlQuery, RedshiftQuery + "DATEADD({{ date_part }}, {{ interval }}, {{ args[0] }})", + [ + "DATEADD(MILLISECOND, 7200000, CURRENT_DATE())", + "DATEADD(DAY, 14, CURRENT_DATE())", + "DATEADD(MONTH, 24, CURRENT_DATE())", + ], + ), + ( + // MysqlQuery: MySQL has no MILLISECOND unit, so those become microseconds + "DATE_ADD({{ args[0] }}, INTERVAL {% if date_part == \"MILLISECOND\" %}\ + {{ interval }}000 MICROSECOND{% else %}{{ interval }} {{ date_part }}{% endif %})", + [ + "DATE_ADD(CURRENT_DATE(), INTERVAL 7200000000 MICROSECOND)", + "DATE_ADD(CURRENT_DATE(), INTERVAL 14 DAY)", + "DATE_ADD(CURRENT_DATE(), INTERVAL 24 MONTH)", + ], + ), + ( + // ClickHouseQuery, DatabricksQuery + "({{ args[0] }} + INTERVAL {{ interval }} {{ date_part }})", + [ + "(CURRENT_DATE() + INTERVAL 7200000 MILLISECOND)", + "(CURRENT_DATE() + INTERVAL 14 DAY)", + "(CURRENT_DATE() + INTERVAL 24 MONTH)", + ], + ), + ( + // PrestodbQuery, TrinoQuery and AthenaQuery inherit it + "DATE_ADD('{{ date_part }}', {{ interval }}, {{ args[0] }})", + [ + "DATE_ADD('MILLISECOND', 7200000, CURRENT_DATE())", + "DATE_ADD('DAY', 14, CURRENT_DATE())", + "DATE_ADD('MONTH', 24, CURRENT_DATE())", + ], + ), + ]; + + for (template, expected) in dialects { + for (unit, expected) in ["hour", "week", "year"].iter().zip(expected) { + // A filter over a per-group aggregate cannot become a Cube filter, so the + // whole expression has to be rendered by the template + let sql = convert_select_to_query_plan_customized( + format!( + r#" + WITH first_orders AS ( + SELECT customer_gender, MIN(order_date) AS first_order_at + FROM KibanaSampleDataEcommerce + GROUP BY 1 + ) + SELECT COUNT(DISTINCT customer_gender) AS customers + FROM first_orders + WHERE first_order_at > DATEADD('{unit}', 2, CURRENT_DATE()) + "# + ), + DatabaseProtocol::PostgreSQL, + vec![("functions/DATE_ADD".to_string(), template.to_string())], + ) + .await + .as_logical_plan() + .find_cube_scan_wrapped_sql() + .wrapped_sql + .sql; + + assert!( + sql.contains(expected), + "`{}` renders as `{}` with template `{}`, got: {}", + unit, + expected, + template, + sql + ); + } + } +}