diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/__init__.py b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/__init__.py new file mode 100644 index 00000000000..b177e5d4932 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/__init__.py @@ -0,0 +1,3 @@ +import os + +PACKAGE_PATH = os.path.dirname(__file__) diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/dbt_project.yml b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/dbt_project.yml new file mode 100644 index 00000000000..8952ba41bc2 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/dbt_project.yml @@ -0,0 +1,6 @@ +name: dbt_sqlserver +version: 1.0 + +config-version: 2 + +macro-paths: ["macros"] diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/apply_grants.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/apply_grants.sql new file mode 100644 index 00000000000..fbeb53d8fab --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/apply_grants.sql @@ -0,0 +1,71 @@ +{% macro sqlserver__apply_grants(relation, grant_config, should_revoke=True) %} + {#-- If grant_config is {} or None, this is a no-op --#} + {% if grant_config %} + {% if should_revoke %} + {#-- We think previous grants may have carried over --#} + {#-- Show current grants and calculate diffs --#} + {% set current_grants_table = run_query(get_show_grant_sql(relation)) %} + {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %} + {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %} + {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %} + {% if not (needs_granting or needs_revoking) %} + {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}} + {% endif %} + {% else %} + {#-- We don't think there's any chance of previous grants having carried over. --#} + {#-- Jump straight to granting what the user has configured. --#} + {% set needs_revoking = {} %} + {% set needs_granting = grant_config %} + {% endif %} + {% if needs_granting or needs_revoking %} + {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %} + + {% if config.get('auto_provision_aad_principals', False) %} + {% set provision_statement_list = get_dcl_statement_list(relation, needs_granting, get_provision_sql) %} + {% else %} + {% set provision_statement_list = [] %} + {% endif %} + + {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %} + {% set dcl_statement_list = revoke_statement_list + provision_statement_list + grant_statement_list %} + {% if dcl_statement_list %} + {{ call_dcl_statements(dcl_statement_list) }} + {% endif %} + {% endif %} + {% endif %} +{% endmacro %} + +{% macro sqlserver__get_show_grant_sql(relation) %} + select + GRANTEE as grantee, + PRIVILEGE_TYPE as privilege_type + from INFORMATION_SCHEMA.TABLE_PRIVILEGES {{ information_schema_hints() }} + where TABLE_CATALOG = '{{ relation.database }}' + and TABLE_SCHEMA = '{{ relation.schema }}' + and TABLE_NAME = '{{ relation.identifier }}' +{% endmacro %} + +{%- macro sqlserver__get_grant_sql(relation, privilege, grantees) -%} + {%- set grantees_safe = [] -%} + {%- for grantee in grantees -%} + {%- set grantee_safe = adapter.quote(grantee) -%} + {%- do grantees_safe.append(grantee_safe) -%} + {%- endfor -%} + grant {{ privilege }} on {{ relation }} to {{ grantees_safe | join(', ') }} +{%- endmacro -%} + +{%- macro sqlserver__get_revoke_sql(relation, privilege, grantees) -%} + {%- set grantees_safe = [] -%} + {%- for grantee in grantees -%} + {%- set grantee_safe = adapter.quote(grantee) -%} + {%- do grantees_safe.append(grantee_safe) -%} + {%- endfor -%} + revoke {{ privilege }} on {{ relation }} from {{ grantees_safe | join(', ') }} +{%- endmacro -%} + +{% macro get_provision_sql(relation, privilege, grantees) %} + {% for grantee in grantees %} + if not exists(select name from sys.database_principals where name = '{{ grantee }}') + create user {{ adapter.quote(grantee) }} from external provider; + {% endfor %} +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/catalog.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/catalog.sql new file mode 100644 index 00000000000..55d07ab765f --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/catalog.sql @@ -0,0 +1,321 @@ +{% macro sqlserver__get_catalog(information_schemas, schemas) -%} + {% set query_label = get_query_options() %} + {%- call statement('catalog', fetch_result=True) -%} + {{ get_use_database_sql(information_schemas.database) }} + with + principals as ( + select + name as principal_name, + principal_id as principal_id + from + sys.database_principals {{ information_schema_hints() }} + ), + + schemas as ( + select + name as schema_name, + schema_id as schema_id, + principal_id as principal_id + from + sys.schemas {{ information_schema_hints() }} + ), + + tables as ( + select + t.object_id, + t.name as table_name, + t.schema_id as schema_id, + t.principal_id as principal_id, + 'BASE TABLE' as table_type, + cast(ep.value as nvarchar(max)) as table_comment + from + sys.tables as t {{ information_schema_hints() }} + left join sys.extended_properties as ep {{ information_schema_hints() }} + on ep.class = 1 + and ep.major_id = t.object_id + and ep.minor_id = 0 + and ep.name = N'MS_Description' + ), + + tables_with_metadata as ( + select + object_id, + table_name, + schema_name, + coalesce(tables.principal_id, schemas.principal_id) as owner_principal_id, + table_type, + table_comment + from + tables + join schemas on tables.schema_id = schemas.schema_id + ), + + views as ( + select + v.object_id, + v.name as table_name, + v.schema_id as schema_id, + v.principal_id as principal_id, + 'VIEW' as table_type, + cast(ep.value as nvarchar(max)) as table_comment + from + sys.views as v {{ information_schema_hints() }} + left join sys.extended_properties as ep {{ information_schema_hints() }} + on ep.class = 1 + and ep.major_id = v.object_id + and ep.minor_id = 0 + and ep.name = N'MS_Description' + ), + + views_with_metadata as ( + select + object_id, + table_name, + schema_name, + coalesce(views.principal_id, schemas.principal_id) as owner_principal_id, + table_type, + table_comment + from + views + join schemas on views.schema_id = schemas.schema_id + ), + + tables_and_views as ( + select + object_id, + table_name, + schema_name, + principal_name, + table_type, + table_comment + from + tables_with_metadata + join principals on tables_with_metadata.owner_principal_id = principals.principal_id + union all + select + object_id, + table_name, + schema_name, + principal_name, + table_type, + table_comment + from + views_with_metadata + join principals on views_with_metadata.owner_principal_id = principals.principal_id + ), + + cols as ( + select + c.object_id, + c.name as column_name, + c.column_id as column_index, + t.name as column_type, + cast(ep.value as nvarchar(max)) as column_comment + from sys.columns as c {{ information_schema_hints() }} + left join sys.types as t {{ information_schema_hints() }} + on c.user_type_id = t.user_type_id + left join sys.extended_properties as ep {{ information_schema_hints() }} + on ep.class = 1 + and ep.major_id = c.object_id + and ep.minor_id = c.column_id + and ep.name = N'MS_Description' + ) + + select + DB_NAME() as table_database, + tv.schema_name as table_schema, + tv.table_name, + tv.table_type, + tv.table_comment, + tv.principal_name as table_owner, + cols.column_name, + cols.column_index, + cols.column_type, + cols.column_comment + from tables_and_views tv + join cols on tv.object_id = cols.object_id + where ({%- for schema in schemas -%} + upper(tv.schema_name) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} + {%- endfor -%}) + + order by column_index + {{ query_label }} + + {%- endcall -%} + + {{ return(load_result('catalog').table) }} + +{%- endmacro %} + +{% macro sqlserver__get_catalog_relations(information_schema, relations) -%} + {% set query_label = get_query_options() %} + {%- set distinct_databases = relations | map(attribute='database') | unique | list -%} + + {%- if distinct_databases | length == 1 -%} + {%- call statement('catalog', fetch_result=True) -%} + {{ get_use_database_sql(distinct_databases[0]) }} + with + principals as ( + select + name as principal_name, + principal_id as principal_id + from + sys.database_principals {{ information_schema_hints() }} + ), + + schemas as ( + select + name as schema_name, + schema_id as schema_id, + principal_id as principal_id + from + sys.schemas {{ information_schema_hints() }} + ), + + tables as ( + select + t.object_id, + t.name as table_name, + t.schema_id as schema_id, + t.principal_id as principal_id, + 'BASE TABLE' as table_type, + cast(ep.value as nvarchar(max)) as table_comment + from + sys.tables as t {{ information_schema_hints() }} + left join sys.extended_properties as ep {{ information_schema_hints() }} + on ep.class = 1 + and ep.major_id = t.object_id + and ep.minor_id = 0 + and ep.name = N'MS_Description' + ), + + tables_with_metadata as ( + select + object_id, + table_name, + schema_name, + coalesce(tables.principal_id, schemas.principal_id) as owner_principal_id, + table_type, + table_comment + from + tables + join schemas on tables.schema_id = schemas.schema_id + ), + + views as ( + select + v.object_id, + v.name as table_name, + v.schema_id as schema_id, + v.principal_id as principal_id, + 'VIEW' as table_type, + cast(ep.value as nvarchar(max)) as table_comment + from + sys.views as v {{ information_schema_hints() }} + left join sys.extended_properties as ep {{ information_schema_hints() }} + on ep.class = 1 + and ep.major_id = v.object_id + and ep.minor_id = 0 + and ep.name = N'MS_Description' + ), + + views_with_metadata as ( + select + object_id, + table_name, + schema_name, + coalesce(views.principal_id, schemas.principal_id) as owner_principal_id, + table_type, + table_comment + from + views + join schemas on views.schema_id = schemas.schema_id + ), + + tables_and_views as ( + select + object_id, + table_name, + schema_name, + principal_name, + table_type, + table_comment + from + tables_with_metadata + join principals on tables_with_metadata.owner_principal_id = principals.principal_id + union all + select + object_id, + table_name, + schema_name, + principal_name, + table_type, + table_comment + from + views_with_metadata + join principals on views_with_metadata.owner_principal_id = principals.principal_id + ), + + cols as ( + select + c.object_id, + c.name as column_name, + c.column_id as column_index, + t.name as column_type, + cast(ep.value as nvarchar(max)) as column_comment + from sys.columns as c {{ information_schema_hints() }} + left join sys.types as t {{ information_schema_hints() }} + on c.user_type_id = t.user_type_id + left join sys.extended_properties as ep {{ information_schema_hints() }} + on ep.class = 1 + and ep.major_id = c.object_id + and ep.minor_id = c.column_id + and ep.name = N'MS_Description' + ) + + select + DB_NAME() as table_database, + tv.schema_name as table_schema, + tv.table_name, + tv.table_type, + tv.table_comment, + tv.principal_name as table_owner, + cols.column_name, + cols.column_index, + cols.column_type, + cols.column_comment + from tables_and_views tv + join cols on tv.object_id = cols.object_id + where ( + {%- for relation in relations -%} + {% if relation.schema and relation.identifier %} + ( + upper(tv.schema_name) = upper('{{ relation.schema }}') + and upper(tv.table_name) = upper('{{ relation.identifier }}') + ) + {% elif relation.schema %} + ( + upper(tv.schema_name) = upper('{{ relation.schema }}') + ) + {% else %} + {% do exceptions.raise_compiler_error( + '`get_catalog_relations` requires a list of relations, each with a schema' + ) %} + {% endif %} + + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) + + order by column_index + {{ query_label }} + + {%- endcall -%} + {{ return(load_result('catalog').table) }} + {% else %} + {% do exceptions.raise_compiler_error( + '`get_catalog_relations` can catalog one database at a time' + ) %} + {% endif %} + +{%- endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/columns.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/columns.sql new file mode 100644 index 00000000000..d43ad5818cf --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/columns.sql @@ -0,0 +1,114 @@ +{% macro sqlserver__select_starts_with_cte(select_sql) %} + {#-- Strip comments first so a leading comment does not hide the CTE --#} + {%- set select_sql_stripped = modules.re.sub('(?s)/\\*.*?\\*/|--[^\n]*\n', '', select_sql) -%} + {{ return(select_sql_stripped.strip().lower().startswith('with')) }} +{% endmacro %} + +{% macro sqlserver__get_empty_subquery_sql(select_sql, select_sql_header=none) %} + {% if sqlserver__select_starts_with_cte(select_sql) %} + {{ select_sql }} + {% else -%} + select * from ( + {{ select_sql }} + ) dbt_sbq_tmp + where 1 = 0 + {%- endif -%} + +{% endmacro %} + +{% macro sqlserver__get_columns_in_query(select_sql) %} + {% set query_label = get_query_options() %} + {% if sqlserver__select_starts_with_cte(select_sql) %} + {#-- A query starting with a CTE cannot be wrapped in a subquery; describe its result set instead of executing it (dbt-msft/dbt-sqlserver#698) --#} + {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%} + exec sp_describe_first_result_set @tsql = N'{{ escape_single_quotes(select_sql) }}' + {% endcall %} + {{ return(load_result('get_columns_in_query').table.columns['name'].values() | list) }} + {% else %} + {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%} + select TOP 0 * from ( + {{ select_sql }} + ) as __dbt_sbq + where 0 = 1 + {{ query_label }} + {% endcall %} + {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }} + {% endif %} +{% endmacro %} + +{% macro sqlserver__alter_column_type(relation, column_name, new_column_type) %} + + {% set prefer_single = config.get('prefer_single_alter_column', false) %} + + {% if prefer_single and relation.type == 'table' %} + {% set alter_sql %} + alter {{ relation.type }} {{ relation }} + alter column "{{ column_name }}" {{ new_column_type }}; + {%- endset %} + {% do run_query(alter_sql) %} + + {% else %} + {%- set tmp_column = column_name + "__dbt_alter" -%} + + {% set add_column %} + alter {{ relation.type }} {{ relation }} + add "{{ tmp_column }}" {{ new_column_type }}; + {%- endset %} + {% set update_column %} + update {{ relation }} set "{{ tmp_column }}" = "{{ column_name }}"; + {%- endset %} + {% set drop_column %} + alter {{ relation.type }} {{ relation }} + drop column "{{ column_name }}"; + {%- endset %} + {% set rename_column %} + exec sp_rename '{{ escape_single_quotes(relation.include(database=False)) }}.{{ escape_single_quotes(adapter.quote(tmp_column)) }}', '{{ escape_single_quotes(column_name) }}', 'column' + {%- endset %} + + {% do run_query(add_column) %} + {% do run_query(update_column) %} + {% do run_query(drop_column) %} + {% do run_query(rename_column) %} + {% endif %} + +{% endmacro %} + + +{% macro sqlserver__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %} + {% call statement('add_drop_columns') -%} + {% if add_columns %} + alter {{ relation.type }} {{ relation }} + add {% for column in add_columns %}"{{ column.name }}" {{ column.data_type }}{{ ', ' if not loop.last }}{% endfor %}; + {% endif %} + + {% if remove_columns %} + alter {{ relation.type }} {{ relation }} + drop column {% for column in remove_columns %}"{{ column.name }}"{{ ',' if not loop.last }}{% endfor %}; + {% endif %} + {%- endcall -%} +{% endmacro %} + +{% macro sqlserver__get_columns_in_relation(relation) -%} + {% set query_label = get_query_options() %} + {% call statement('get_columns_in_relation', fetch_result=True) %} + {{ get_use_database_sql(relation.database) }} + select + c.name collate database_default as column_name, + t.name as data_type, + case + when (t.name in ('nchar', 'nvarchar', 'sysname') and c.max_length <> -1) then c.max_length / 2 + else c.max_length + end as character_maximum_length, + c.precision as numeric_precision, + c.scale as numeric_scale + from sys.columns c {{ information_schema_hints() }} + inner join sys.types t {{ information_schema_hints() }} + on c.user_type_id = t.user_type_id + where c.object_id = object_id('{{ 'tempdb..' ~ relation.include(database=false, schema=false) if '#' in relation.identifier else relation }}') + order by c.column_id + {{ query_label }} + + {% endcall %} + {% set table = load_result('get_columns_in_relation').table %} + {{ return(sql_convert_columns_in_relation(table)) }} +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/indexes.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/indexes.sql new file mode 100644 index 00000000000..373e9db7395 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/indexes.sql @@ -0,0 +1,7 @@ +{% macro sqlserver__get_create_index_sql(relation, index_dict) -%} + {{ exceptions.raise_compiler_error( + "Custom `indexes:` config is not implemented yet in dbt Core v2's SQL Server " + "adapter (tracked as a follow-up issue). Remove the indexes config from " + ~ relation ~ " to build without it." + ) }} +{%- endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/metadata.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/metadata.sql new file mode 100644 index 00000000000..75ff593e168 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/metadata.sql @@ -0,0 +1,209 @@ +{% macro get_query_options(parse_options=False) %} + {{ log (config.get('query_tag','dbt-sqlserver'))}} + {#- Escape single quotes so a query_tag like "it's" can't break out of the LABEL literal. -#} + {%- set query_label = escape_single_quotes(config.get('query_tag','dbt-sqlserver')) -%} + {%- set query_options = config.get('query_options', {}) -%} + {%- set query_options_raw = config.get('query_options_raw', []) -%} + + {%- set options_list = ["LABEL = '" ~ query_label ~ "'"] -%} + + {%- if parse_options -%} + {%- set valid_options = [ + 'HASH GROUP', 'ORDER GROUP', + 'CONCAT UNION', 'HASH UNION', 'MERGE UNION', + 'LOOP JOIN', 'MERGE JOIN', 'HASH JOIN', + 'DISABLE_OPTIMIZED_PLAN_FORCING', + 'EXPAND VIEWS', + 'FAST', + 'FORCE ORDER', + 'FORCE EXTERNALPUSHDOWN', 'DISABLE EXTERNALPUSHDOWN', + 'FORCE SCALEOUTEXECUTION', 'DISABLE SCALEOUTEXECUTION', + 'IGNORE_NONCLUSTERED_COLUMNSTORE_INDEX', + 'KEEP PLAN', + 'KEEPFIXED PLAN', + 'MAX_GRANT_PERCENT', + 'MIN_GRANT_PERCENT', + 'MAXDOP', + 'MAXRECURSION', + 'NO_PERFORMANCE_SPOOL', + 'OPTIMIZE FOR UNKNOWN', + 'QUERYTRACEON', + 'RECOMPILE', + 'ROBUST PLAN', + ] -%} + {#- SQL Server uses `OPTION (X = N)` for grant-percent hints, not `OPTION (X N)`. -#} + {%- set equals_syntax_options = ['MAX_GRANT_PERCENT', 'MIN_GRANT_PERCENT'] -%} + + {%- for key, value in query_options.items() -%} + {%- if key | upper not in valid_options -%} + {{ exceptions.raise_compiler_error("Invalid query option: '" ~ key ~ "'. Use query_options_raw for non-standard hints. Allowed: " ~ valid_options | join(', ')) }} + {%- endif -%} + + {%- if value is none -%} + {%- do options_list.append(key | upper) -%} + {%- else -%} + {%- if value is not number -%} + {{ exceptions.raise_compiler_error("Query option '" ~ key ~ "' value must be a number, got: '" ~ value ~ "'") }} + {%- endif -%} + {%- set separator = ' = ' if key | upper in equals_syntax_options else ' ' -%} + {#- Render the value verbatim: ints become "1", floats become "12.5". + MAX_GRANT_PERCENT / MIN_GRANT_PERCENT accept decimals 0.0–100.0; integer-only + options will surface a clear SQL Server parse error on invalid decimals. -#} + {%- do options_list.append(key | upper ~ separator ~ value) -%} + {%- endif -%} + {%- endfor -%} + + {#- query_options_raw bypasses the allowlist; users opt in to writing valid SQL Server syntax themselves. + Shape-check only: a plain string would be iterated character-by-character into garbage. -#} + {%- if query_options_raw is string or query_options_raw is mapping -%} + {{ exceptions.raise_compiler_error("query_options_raw must be a list of strings, got: '" ~ query_options_raw ~ "'") }} + {%- endif -%} + {%- for raw in query_options_raw -%} + {%- do options_list.append(raw) -%} + {%- endfor -%} + {%- endif -%} + + OPTION ({{ options_list | join(', ') }}); +{% endmacro %} + +{#- DEPRECATED: backward-compat alias for the pre-1.10 macro. + + Calls to `{{ apply_label() }}` from user macros still resolve and emit + a LABEL-only OPTION clause — but apply_label() is no longer the + extension point. Adapter macros now call get_query_options() instead, + so overriding apply_label() in a project's macros directory will have + no effect on adapter-emitted SQL. + + To customise the OPTION clause emitted by adapter macros (table, + incremental, snapshot, unit_test), override get_query_options instead. -#} +{% macro apply_label() %} + {{ log (config.get('query_tag','dbt-sqlserver'))}} + {%- set query_label = escape_single_quotes(config.get('query_tag','dbt-sqlserver')) -%} + OPTION (LABEL = '{{query_label}}'); +{% endmacro %} + +{#- Guard for materializations and incremental strategies that cannot emit OPTION clauses. + Raises a compiler error if the user has configured query_options/query_options_raw. -#} +{% macro raise_if_query_options_set(context_label) %} + {%- if config.get('query_options') or config.get('query_options_raw') -%} + {{ exceptions.raise_compiler_error( + "query_options/query_options_raw is not supported on " ~ context_label + ~ ". Remove the config or switch to a supported materialization (table, incremental delete+insert, snapshot, unit_test)." + ) }} + {%- endif -%} +{% endmacro %} + +{% macro default__information_schema_hints() %}{% endmacro %} +{% macro sqlserver__information_schema_hints() %}with (nolock){% endmacro %} + +{% macro information_schema_hints() %} + {{ return(adapter.dispatch('information_schema_hints')()) }} +{% endmacro %} + +{% macro sqlserver__information_schema_name(database) -%} + information_schema +{%- endmacro %} + +{% macro get_use_database_sql(database) %} + {{ return(adapter.dispatch('get_use_database_sql', 'dbt')(database)) }} +{% endmacro %} + +{%- macro sqlserver__get_use_database_sql(database) -%} + USE {{ adapter.quote(database | replace('"', '')) }}; +{%- endmacro -%} + +{% macro sqlserver__list_schemas(database) %} + {% call statement('list_schemas', fetch_result=True, auto_begin=False) -%} + {{ get_use_database_sql(database) }} + select name as [schema] + from sys.schemas {{ information_schema_hints() }} {{ get_query_options() }} + {% endcall %} + {{ return(load_result('list_schemas').table) }} +{% endmacro %} + +{% macro sqlserver__check_schema_exists(information_schema, schema) -%} + {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) -%} + SELECT count(*) as schema_exist FROM sys.schemas WHERE name = '{{ schema }}' {{ get_query_options() }} + {%- endcall %} + {{ return(load_result('check_schema_exists').table) }} +{% endmacro %} + +{% macro sqlserver__list_relations_without_caching(schema_relation) -%} + {% call statement('list_relations_without_caching', fetch_result=True) -%} + {{ get_use_database_sql(schema_relation.database) }} + declare @schema_id int = schema_id('{{ schema_relation.schema }}'); + select + DB_NAME() as [database], + t.name as [name], + '{{ schema_relation.schema }}' as [schema], + 'table' as table_type + from sys.tables as t {{ information_schema_hints() }} + where t.schema_id = @schema_id + union all + select + DB_NAME() as [database], + v.name as [name], + '{{ schema_relation.schema }}' as [schema], + 'view' as table_type + from sys.views as v {{ information_schema_hints() }} + where v.schema_id = @schema_id + {{ get_query_options() }} + {% endcall %} + {{ return(load_result('list_relations_without_caching').table) }} +{% endmacro %} + +{% macro sqlserver__get_relation_without_caching(schema_relation) -%} + {% call statement('get_relation_without_caching', fetch_result=True) -%} + {{ get_use_database_sql(schema_relation.database) }} + declare @schema_id int = schema_id('{{ schema_relation.schema }}'); + select + DB_NAME() as [database], + t.name as [name], + '{{ schema_relation.schema }}' as [schema], + 'table' as table_type + from sys.tables as t {{ information_schema_hints() }} + where t.schema_id = @schema_id and t.name = '{{ schema_relation.identifier }}' + union all + select + DB_NAME() as [database], + v.name as [name], + '{{ schema_relation.schema }}' as [schema], + 'view' as table_type + from sys.views as v {{ information_schema_hints() }} + where v.schema_id = @schema_id and v.name = '{{ schema_relation.identifier }}' + {{ get_query_options() }} + {% endcall %} + {{ return(load_result('get_relation_without_caching').table) }} +{% endmacro %} + +{% macro get_view_definition_sql(relation) %} + {{ return(adapter.dispatch('get_view_definition_sql')(relation)) }} +{% endmacro %} + +{% macro sqlserver__get_view_definition_sql(relation) -%} + {%- set object_name = "quotename('" ~ relation.schema ~ "') + '.' + quotename('" ~ relation.identifier ~ "')" -%} + {{ get_use_database_sql(relation.database) }} + select object_definition(object_id({{ object_name }}, 'V')) as definition + where object_id({{ object_name }}, 'V') is not null +{% endmacro %} + +{% macro sqlserver__get_relation_last_modified(information_schema, relations) -%} + {%- call statement('last_modified', fetch_result=True) -%} + select + o.name as [identifier] + , s.name as [schema] + , o.modify_date as last_modified + , current_timestamp as snapshotted_at + from sys.objects o {{ information_schema_hints() }} + inner join sys.schemas s {{ information_schema_hints() }} on o.schema_id = s.schema_id and [type] = 'U' + where ( + {%- for relation in relations -%} + (upper(s.name) = upper('{{ relation.schema }}') and + upper(o.name) = upper('{{ relation.identifier }}')){%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) + {{ get_query_options() }} + {%- endcall -%} + {{ return(load_result('last_modified')) }} + +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/persist_docs.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/persist_docs.sql new file mode 100644 index 00000000000..5095eddf197 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/persist_docs.sql @@ -0,0 +1,137 @@ +{% macro sqlserver__alter_relation_comment(relation, relation_comment) -%} + {%- set escaped_comment = (relation_comment or '') | replace("'", "''") -%} + {%- set relation_name = relation.identifier | replace("'", "''") -%} + {%- set schema_name = relation.schema | replace("'", "''") -%} + + declare @relation_schema sysname = N'{{ schema_name }}'; + declare @relation_name sysname = N'{{ relation_name }}'; + declare @relation_comment nvarchar(3750) = N'{{ escaped_comment }}'; + declare @relation_type nvarchar(128); + + select + @relation_type = + case + when obj.[type] = 'V' then N'VIEW' + when obj.[type] = 'U' then N'TABLE' + end + from sys.objects as obj {{ information_schema_hints() }} + inner join sys.schemas as sch {{ information_schema_hints() }} + on sch.schema_id = obj.schema_id + where sch.name = @relation_schema + and obj.name = @relation_name + and obj.[type] in ('U', 'V'); + + if @relation_type is not null + begin + if exists ( + select 1 + from sys.extended_properties as ep {{ information_schema_hints() }} + inner join sys.objects as obj {{ information_schema_hints() }} + on obj.object_id = ep.major_id + inner join sys.schemas as sch {{ information_schema_hints() }} + on sch.schema_id = obj.schema_id + where ep.class = 1 + and ep.minor_id = 0 + and ep.name = N'MS_Description' + and sch.name = @relation_schema + and obj.name = @relation_name + and obj.[type] in ('U', 'V') + ) + begin + exec sys.sp_updateextendedproperty + @name = N'MS_Description', + @value = @relation_comment, + @level0type = N'SCHEMA', + @level0name = @relation_schema, + @level1type = @relation_type, + @level1name = @relation_name; + end + else + begin + exec sys.sp_addextendedproperty + @name = N'MS_Description', + @value = @relation_comment, + @level0type = N'SCHEMA', + @level0name = @relation_schema, + @level1type = @relation_type, + @level1name = @relation_name; + end + end; +{%- endmacro %} + + +{% macro sqlserver__alter_column_comment(relation, column_dict) -%} + {%- set relation_name = relation.identifier | replace("'", "''") -%} + {%- set schema_name = relation.schema | replace("'", "''") -%} + + {%- for column_name, column_config in column_dict.items() %} + {%- set escaped_column_name = column_name | replace("'", "''") -%} + {%- set escaped_comment = (column_config.get('description') or '') | replace("'", "''") -%} + + declare @schema_{{ loop.index }} sysname = N'{{ schema_name }}'; + declare @relation_{{ loop.index }} sysname = N'{{ relation_name }}'; + declare @column_{{ loop.index }} sysname = N'{{ escaped_column_name }}'; + declare @comment_{{ loop.index }} nvarchar(3750) = N'{{ escaped_comment }}'; + declare @relation_type_{{ loop.index }} nvarchar(128); + + select + @relation_type_{{ loop.index }} = + case + when obj.[type] = 'V' then N'VIEW' + when obj.[type] = 'U' then N'TABLE' + end + from sys.objects as obj {{ information_schema_hints() }} + inner join sys.schemas as sch {{ information_schema_hints() }} + on sch.schema_id = obj.schema_id + inner join sys.columns as col {{ information_schema_hints() }} + on col.object_id = obj.object_id + where sch.name = @schema_{{ loop.index }} + and obj.name = @relation_{{ loop.index }} + and col.name = @column_{{ loop.index }} + and obj.[type] in ('U', 'V'); + + if @relation_type_{{ loop.index }} is not null + begin + if exists ( + select 1 + from sys.extended_properties as ep {{ information_schema_hints() }} + inner join sys.objects as obj {{ information_schema_hints() }} + on obj.object_id = ep.major_id + inner join sys.schemas as sch {{ information_schema_hints() }} + on sch.schema_id = obj.schema_id + inner join sys.columns as col {{ information_schema_hints() }} + on col.object_id = ep.major_id + and col.column_id = ep.minor_id + where ep.class = 1 + and ep.name = N'MS_Description' + and sch.name = @schema_{{ loop.index }} + and obj.name = @relation_{{ loop.index }} + and col.name = @column_{{ loop.index }} + and obj.[type] in ('U', 'V') + ) + begin + exec sys.sp_updateextendedproperty + @name = N'MS_Description', + @value = @comment_{{ loop.index }}, + @level0type = N'SCHEMA', + @level0name = @schema_{{ loop.index }}, + @level1type = @relation_type_{{ loop.index }}, + @level1name = @relation_{{ loop.index }}, + @level2type = N'COLUMN', + @level2name = @column_{{ loop.index }}; + end + else + begin + exec sys.sp_addextendedproperty + @name = N'MS_Description', + @value = @comment_{{ loop.index }}, + @level0type = N'SCHEMA', + @level0name = @schema_{{ loop.index }}, + @level1type = @relation_type_{{ loop.index }}, + @level1name = @relation_{{ loop.index }}, + @level2type = N'COLUMN', + @level2name = @column_{{ loop.index }}; + end + end; + {%- endfor %} +{%- endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/relation.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/relation.sql new file mode 100644 index 00000000000..1ac67422d74 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/relation.sql @@ -0,0 +1,62 @@ +{% macro sqlserver__make_temp_relation(base_relation, suffix='__dbt_temp') %} + {%- set temp_identifier = base_relation.identifier ~ suffix -%} + {%- set temp_relation = base_relation.incorporate( + path={"identifier": temp_identifier}) -%} + + {{ return(temp_relation) }} +{% endmacro %} + +{% macro sqlserver__get_drop_sql(relation) -%} + {% if relation.type == 'view' -%} + {#- auto_begin=false, matching dbt-adapters' own relations/drop.sql: a + read-only lookup must not open the ambient transaction. This is the + first statement of the temp-relation build (create.sql -> + adapter.drop_relation), so with the default auto_begin=True it held + that build's sys.sysschobjs X locks until the materialization + committed, deadlocking a concurrent worker (error 1205). -#} + {% call statement('find_references', fetch_result=true, auto_begin=false) %} + {{ get_use_database_sql(relation.database) }} + select + sch.name as schema_name, + obj.name as view_name + from sys.sql_expression_dependencies refs {{ information_schema_hints() }} + inner join sys.objects obj {{ information_schema_hints() }} + on refs.referencing_id = obj.object_id + inner join sys.schemas sch {{ information_schema_hints() }} + on obj.schema_id = sch.schema_id + where refs.referenced_database_name = '{{ relation.database }}' + and refs.referenced_schema_name = '{{ relation.schema }}' + and refs.referenced_entity_name = '{{ relation.identifier }}' + and obj.type = 'V' + {{ get_query_options() }} + {% endcall %} + {% set references = load_result('find_references')['data'] %} + {% for reference in references -%} + -- dropping referenced view {{ reference[0] }}.{{ reference[1] }} + {% do adapter.drop_relation + (api.Relation.create( + identifier = reference[1], schema = reference[0], database = relation.database, type='view' + ))%} + {% endfor %} + {% elif relation.type == 'table'%} + {% set object_id_type = 'U' %} + {%- else -%} + {{ exceptions.raise_not_implemented('Invalid relation being dropped: ' ~ relation) }} + {% endif %} + {{ get_use_database_sql(relation.database) }} + EXEC('DROP {{ relation.type }} IF EXISTS {{ relation.include(database=False) }};'); +{% endmacro %} + +{% macro sqlserver__rename_relation(from_relation, to_relation) -%} + {% call statement('rename_relation') -%} + {{ get_use_database_sql(from_relation.database) }} + {#- @objname takes a quoted, qualified name; @newname must stay bare -#} + EXEC sp_rename '{{ escape_single_quotes(from_relation.include(database=False)) }}', '{{ escape_single_quotes(to_relation.identifier) }}' + {%- endcall %} +{% endmacro %} + +{% macro sqlserver__truncate_relation(relation) -%} + {% call statement('truncate_relation') -%} + truncate table {{ relation }} + {%- endcall %} +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/schema.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/schema.sql new file mode 100644 index 00000000000..5dba7b854bd --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/schema.sql @@ -0,0 +1,42 @@ +{% macro sqlserver__create_schema(relation) -%} + {% call statement('create_schema') -%} + {{ get_use_database_sql(relation.database) }} + IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.schema }}') + BEGIN + EXEC('CREATE SCHEMA {{ adapter.quote(relation.schema) }}') + END + {% endcall %} +{% endmacro %} + +{% macro sqlserver__create_schema_with_authorization(relation, schema_authorization) -%} + {% call statement('create_schema') -%} + {{ get_use_database_sql(relation.database) }} + IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.schema }}') + BEGIN + EXEC('CREATE SCHEMA {{ adapter.quote(relation.schema) }} AUTHORIZATION {{ adapter.quote(schema_authorization) }}') + END + {% endcall %} +{% endmacro %} + +{% macro sqlserver__drop_schema(relation) -%} + {%- set relations_in_schema = list_relations_without_caching(relation) %} + + {% for row in relations_in_schema %} + {%- set schema_relation = api.Relation.create(database=relation.database, + schema=relation.schema, + identifier=row[1], + type=row[3] + ) -%} + {% do adapter.drop_relation(schema_relation) %} + {%- endfor %} + + {% call statement('drop_schema') -%} + {{ get_use_database_sql(relation.database) }} + EXEC('DROP SCHEMA IF EXISTS {{ relation.schema }}') + {% endcall %} +{% endmacro %} + +{% macro sqlserver__drop_schema_named(schema_name) %} + {% set schema_relation = api.Relation.create(schema=schema_name, database=target.database) %} + {{ adapter.drop_schema(schema_relation) }} +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/show.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/show.sql new file mode 100644 index 00000000000..14337a733c3 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/show.sql @@ -0,0 +1,12 @@ +{% macro sqlserver__get_limit_sql(sql, limit) %} + {%- if limit == -1 or limit is none -%} + {{ sql }} + {#- Special processing if the last non-blank line starts with order by -#} + {%- elif sql.strip().splitlines()[-1].strip().lower().startswith('order by') -%} + {{ sql }} + offset 0 rows fetch first {{ limit }} rows only + {%- else -%} + {{ sql }} + order by (select null) offset 0 rows fetch first {{ limit }} rows only + {%- endif -%} +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/validate_sql.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/validate_sql.sql new file mode 100644 index 00000000000..aa73794b158 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/adapters/validate_sql.sql @@ -0,0 +1,6 @@ +{% macro sqlserver__validate_sql(sql) -%} + {% call statement('validate_sql') -%} + {{ sql }} + {% endcall %} + {{ return(load_result('validate_sql')) }} +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/hooks.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/hooks.sql new file mode 100644 index 00000000000..c8d50336c74 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/hooks.sql @@ -0,0 +1,19 @@ +{% macro run_hooks(hooks, inside_transaction=True) %} + {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} + {% if not inside_transaction and loop.first %} + {% call statement(auto_begin=inside_transaction) %} + {#- guarded, not a bare COMMIT: nothing guarantees a transaction is + open here. A bare one appeared safe only because some earlier + statement had auto-begun one (find_references, until relation.sql + stopped doing that); with @@TRANCOUNT = 0 it raises Msg 3902. -#} + if @@trancount > 0 commit; + {% endcall %} + {% endif %} + {% set rendered = render(hook.get('sql')) | trim %} + {% if (rendered | length) > 0 %} + {% call statement(auto_begin=inside_transaction) %} + {{ rendered }} + {% endcall %} + {% endif %} + {% endfor %} +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/incremental/incremental.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/incremental/incremental.sql new file mode 100644 index 00000000000..61c577b1b89 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/incremental/incremental.sql @@ -0,0 +1,141 @@ +{% materialization incremental, adapter='sqlserver' -%} + + -- relations + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='table') -%} + {%- set temp_relation = make_temp_relation(target_relation)-%} + {%- set intermediate_relation = make_intermediate_relation(target_relation)-%} + {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + + -- configs + {%- set unique_key = config.get('unique_key') -%} + {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%} + {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%} + {%- set full_refresh_build = config.get('full_refresh_build', 'heap_then_index') -%} + {%- if full_refresh_build == 'prebuilt' -%} + {{ exceptions.raise_compiler_error( + "full_refresh_build='prebuilt' is not implemented yet in dbt Core v2's SQL Server " + "adapter (tracked as a follow-up issue). Use the default 'heap_then_index'." + ) }} + {%- elif full_refresh_build != 'heap_then_index' -%} + {{ exceptions.raise_compiler_error( + "Invalid full_refresh_build '" ~ full_refresh_build ~ "'. Only 'heap_then_index' (default) is supported." + ) }} + {%- endif -%} + + -- the temp_ and backup_ relations should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation. This has to happen before + -- BEGIN, in a separate transaction + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%} + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparison later on + {% set grant_config = config.get('grants') %} + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + {% set to_drop = [] %} + {% set need_swap = false %} + {#- true only where the statement('main') batch below carries create_table_as + DDL, i.e. the fresh-create / full-refresh branches. The incremental + branch's strategy DML stays transactional, so it leaves this false. -#} + {% set build_sql_is_create_table_as = false %} + + {% if existing_relation is none %} + {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %} + {% set build_sql_is_create_table_as = true %} + {% elif full_refresh_mode %} + {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %} + {% set build_sql_is_create_table_as = true %} + {% set need_swap = true %} + {% else %} + + {#- The temp build is all catalog DDL (CREATE OR ALTER VIEW / SELECT * + INTO / DROP VIEW) and must not share the ambient transaction with the + strategy DML: held to commit, its sysschobjs X keylocks deadlock a + second worker. Nothing opens a transaction here - run_query never + auto-begins, and find_references (relation.sql) no longer does either - + so each statement autocommits and drops its catalog locks as it + finishes. The strategy DML below still runs transactionally, via + statement('main')'s default auto_begin through to adapter.commit(). -#} + {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %} + + {% set contract_config = config.get('contract') %} + {% if not contract_config or not contract_config.enforced %} + {% set expansion_max_rows = config.get('column_type_expansion_max_rows', 1000000) %} + {% do adapter.expand_target_column_types( + from_relation=temp_relation, + to_relation=target_relation, + max_rows=expansion_max_rows) %} + {% endif %} + {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#} + {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %} + {% if not dest_columns %} + {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %} + {% endif %} + + {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#} + {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} + {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %} + {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %} + {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %} + {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %} + + {% do to_drop.append(temp_relation) %} + {% endif %} + + {% if build_sql_is_create_table_as %} + {#- This batch is create_table_as catalog DDL, so letting statement() open + the ambient transaction would hold its sysschobjs X keylocks until + adapter.commit() and deadlock a second worker. -#} + {% call statement("main", auto_begin=False) %} + {{ build_sql }} + {% endcall %} + {#- Reopen the ambient transaction the batch above declined to start, so + the swap and the tail (grants/persist_docs/indexes/post-hooks) keep + their semantics and adapter.commit() below has a matching BEGIN + rather than raising Msg 3902. -#} + {% do adapter.commit_if_open() %} + {% do adapter.begin_if_closed() %} + {{ build_model_constraints(target_relation) }} + {% else %} + {% call statement("main") %} + {{ build_sql }} + {% endcall %} + {% endif %} + + {% if need_swap %} + {% do adapter.rename_relation(target_relation, backup_relation) %} + {% do adapter.rename_relation(intermediate_relation, target_relation) %} + {% do to_drop.append(backup_relation) %} + {% endif %} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {% if build_sql_is_create_table_as %} + {% do create_indexes(target_relation) %} + {% endif %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + -- `COMMIT` happens here + {% do adapter.commit() %} + + {% for rel in to_drop %} + {% do adapter.drop_relation(rel) %} + {% endfor %} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{%- endmaterialization %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/incremental/incremental_strategies.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/incremental/incremental_strategies.sql new file mode 100644 index 00000000000..2adb1f34387 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/incremental/incremental_strategies.sql @@ -0,0 +1,11 @@ +{% macro sqlserver__get_incremental_default_sql(arg_dict) %} + + {% if arg_dict["unique_key"] %} + -- Merge strategy: emits a MERGE statement via get_incremental_merge_sql + {% do return(get_incremental_merge_sql(arg_dict)) %} + {% else %} + -- Incremental Append will insert data into target table. + {% do return(get_incremental_append_sql(arg_dict)) %} + {% endif %} + +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/incremental/merge.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/incremental/merge.sql new file mode 100644 index 00000000000..731a267ba80 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/incremental/merge.sql @@ -0,0 +1,95 @@ +{% macro sqlserver__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) %} + {{ default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) }} + {{ get_query_options(parse_options=True) }} +{% endmacro %} + +{% macro sqlserver__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) %} + {{ default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) }}; +{% endmacro %} + +{% macro sqlserver__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) %} + + {% set query_label = get_query_options(parse_options=True) %} + {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} + + {% if unique_key %} + {% if unique_key is sequence and unique_key is not string %} + SET NOCOUNT ON; + delete from {{ target }} + where exists ( + select null + from {{ source }} + where + {% for key in unique_key %} + {{ source }}.{{ key }} = {{ target }}.{{ key }} + {{ "and " if not loop.last }} + {% endfor %} + ) + {% if incremental_predicates %} + {% for predicate in incremental_predicates %} + and {{ predicate }} + {% endfor %} + {% endif %} + {{ query_label }} + SET NOCOUNT OFF; + {% else %} + SET NOCOUNT ON; + delete from {{ target }} + where ( + {{ unique_key }}) in ( + select ({{ unique_key }}) + from {{ source }} + ) + {%- if incremental_predicates %} + {% for predicate in incremental_predicates %} + and {{ predicate }} + {% endfor %} + {%- endif -%} + {{ query_label }} + SET NOCOUNT OFF; + {% endif %} + {% endif %} + + insert into {{ target }} ({{ dest_cols_csv }}) + ( + select {{ dest_cols_csv }} + from {{ source }} + ){{ query_label }} +{% endmacro %} + +{% macro sqlserver__get_incremental_microbatch_sql(arg_dict) %} + {%- set target = arg_dict["target_relation"] -%} + {%- set source = arg_dict["temp_relation"] -%} + {%- set dest_columns = arg_dict["dest_columns"] -%} + {%- set incremental_predicates = [] if arg_dict.get('incremental_predicates') is none else arg_dict.get('incremental_predicates') -%} + {%- set query_label = get_query_options(parse_options=True) -%} + + {#-- Add additional incremental_predicates to filter for batch --#} + {% if model.config.get("__dbt_internal_microbatch_event_time_start") -%} + {{ log("incremental append event start time > DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " >= cast('" ~ model.config.__dbt_internal_microbatch_event_time_start ~ "' as datetimeoffset)") }} + {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " >= cast('" ~ model.config.__dbt_internal_microbatch_event_time_start ~ "' as datetimeoffset)") %} + {% endif %} + {% if model.config.__dbt_internal_microbatch_event_time_end -%} + {{ log("incremental append event end time < DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " < cast('" ~ model.config.__dbt_internal_microbatch_event_time_end ~ "' as datetimeoffset)") }} + {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " < cast('" ~ model.config.__dbt_internal_microbatch_event_time_end ~ "' as datetimeoffset)") %} + {% endif %} + {% do arg_dict.update({'incremental_predicates': incremental_predicates}) %} + + SET NOCOUNT ON; + delete DBT_INTERNAL_TARGET from {{ target }} AS DBT_INTERNAL_TARGET + where ( + {% for predicate in incremental_predicates %} + {%- if not loop.first %}and {% endif -%} {{ predicate }} + {% endfor %} + ) + {{ query_label }} + SET NOCOUNT OFF; + + {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} + insert into {{ target }} ({{ dest_cols_csv }}) + ( + select {{ dest_cols_csv }} + from {{ source }} + ) + {{ query_label }} +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/table/columns_spec_ddl.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/table/columns_spec_ddl.sql new file mode 100644 index 00000000000..e545dbadff6 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/table/columns_spec_ddl.sql @@ -0,0 +1,30 @@ +{% macro build_columns_constraints(relation) %} + {{ return(adapter.dispatch('build_columns_constraints', 'dbt')(relation)) }} +{% endmacro %} + +{% macro sqlserver__build_columns_constraints(relation) %} + {# loop through user_provided_columns to create DDL with data types and constraints #} + {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%} + ( + {% for c in raw_column_constraints -%} + {{ c }}{{ "," if not loop.last }} + {% endfor %} + ) +{% endmacro %} + +{% macro build_model_constraints(relation) %} + {{ return(adapter.dispatch('build_model_constraints', 'dbt')(relation)) }} +{% endmacro %} + +{% macro sqlserver__build_model_constraints(relation) %} + {# loop through user_provided_columns to create DDL with data types and constraints #} + {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%} + {% for c in raw_model_constraints -%} + {% set alter_table_script %} + alter table {{ relation.include(database=False) }} {{c}}; + {%endset%} + {% call statement('alter_table_add_constraint') -%} + {{alter_table_script}} + {%- endcall %} + {% endfor -%} +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/table/create_table_as.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/table/create_table_as.sql new file mode 100644 index 00000000000..8f645ab409c --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/table/create_table_as.sql @@ -0,0 +1,95 @@ +{% macro sqlserver__create_clustered_columnstore_index(relation) -%} + {#- cci_name embeds the schema, so it must be quoted as an identifier + (raw only in the string comparison below) -- issue #409 -#} + {%- set cci_name = (relation.schema ~ '_' ~ relation.identifier ~ '_cci') | replace(".", "") | replace(" ", "") -%} + {%- set relation_name = relation.include(database=False) -%} + {{ get_use_database_sql(relation.database) }} + if EXISTS ( + SELECT * + FROM sys.indexes {{ information_schema_hints() }} + WHERE name = '{{ escape_single_quotes(cci_name) }}' + AND object_id=object_id('{{ escape_single_quotes(relation_name) }}') + ) + DROP index {{ relation_name }}.{{ adapter.quote(cci_name) }} + CREATE CLUSTERED COLUMNSTORE INDEX {{ adapter.quote(cci_name) }} + ON {{ relation_name }} +{% endmacro %} + +{% macro sqlserver__create_table_as(temporary, relation, sql) -%} + {%- set query_label = get_query_options(parse_options=True) -%} + {%- set full_refresh_build = config.get('full_refresh_build', 'heap_then_index') -%} + {%- if full_refresh_build == 'prebuilt' -%} + {{ exceptions.raise_compiler_error( + "full_refresh_build='prebuilt' is not implemented yet in dbt Core v2's " + "SQL Server adapter (tracked as a follow-up issue). Use the default 'heap_then_index'." + ) }} + {%- elif full_refresh_build != 'heap_then_index' -%} + {{ exceptions.raise_compiler_error( + "Invalid full_refresh_build '" ~ full_refresh_build ~ "'. Only 'heap_then_index' (default) is supported." + ) }} + {%- endif -%} + {%- set tmp_relation = relation.incorporate(path={"identifier": relation.identifier ~ '__dbt_tmp_vw'}, type='view') -%} + + {#- Now that the incremental temp build commits standalone (see + incremental.sql), a crash can leave a throwaway table behind and the + SELECT * INTO below would hit Msg 2714. Drop it first, but only for + adapter-generated throwaways: `temporary` covers the incremental + __dbt_temp build, the suffix covers the full-refresh / table-refresh + __dbt_tmp intermediate. Suffix match is exact, never substring, so a + user model named stg__dbt_tmp_x is untouched. + Never guard a fresh-create of the real target: dbt has decided that + table does not exist, so 2714 must still surface rather than silently + destroying an object dbt does not know about. -#} + {%- set _ident = relation.identifier -%} + {%- set build_into_temp = temporary or _ident.endswith('__dbt_tmp') or _ident.endswith('__dbt_tmp_vw') -%} + + {%- do adapter.drop_relation(tmp_relation) -%} + {{ get_use_database_sql(relation.database) }} + {{ get_create_view_as_sql(tmp_relation, sql) }} + + {%- set table_name -%} + {{ relation }} + {%- endset -%} + + + {%- set contract_config = config.get('contract') -%} + {%- set query -%} + {% if contract_config.enforced and (not temporary) %} + CREATE TABLE {{table_name}} + {{ get_assert_columns_equivalent(sql) }} + {{ build_columns_constraints(relation) }} + {% set listColumns %} + {% for column in model['columns'] %} + {{ adapter.quote(column) }}{{ ", " if not loop.last }} + {% endfor %} + {%endset%} + INSERT INTO {{relation}} WITH (TABLOCK) ({{listColumns}}) + SELECT {{listColumns}} FROM {{tmp_relation}} {{ query_label }} + + {% else %} + {%- if build_into_temp -%} + IF OBJECT_ID('{{ escape_single_quotes(relation.include(database=False)) }}', 'U') IS NOT NULL + EXEC('DROP TABLE {{ relation }}'); + {%- endif -%} + SELECT * INTO {{ table_name }} FROM {{ tmp_relation }} {{ query_label }} + {% endif %} + {%- endset -%} + + EXEC('{{- escape_single_quotes(query) -}}') + + {# For some reason drop_relation is not firing. This solves the issue for now. #} + EXEC('DROP VIEW IF EXISTS {{ tmp_relation.include(database=False) }}') + + + + {% set as_columnstore = config.get('as_columnstore', default=true) %} + {% if not temporary and as_columnstore -%} + {#- + add columnstore index + this creates with dbt_temp as its coming from a temporary relation before renaming + could alter relation to drop the dbt_temp portion if needed + -#} + {{ sqlserver__create_clustered_columnstore_index(relation) }} + {% endif %} + +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/table/table.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/table/table.sql new file mode 100644 index 00000000000..ce452d67616 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/table/table.sql @@ -0,0 +1,89 @@ +{% materialization table, adapter='sqlserver' %} + + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='table') %} + {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} + -- the intermediate_relation should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} + /* + See ../view/view.sql for more information about this relation. + */ + {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + -- as above, the backup_relation should not already exist + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparison later on + {% set grant_config = config.get('grants') %} + + {%- set table_refresh_method = config.get('table_refresh_method', 'rename') -%} + {%- if table_refresh_method == 'dml' -%} + {{ exceptions.raise_compiler_error( + "table_refresh_method='dml' is not implemented yet in dbt Core v2's SQL Server " + "adapter (tracked as a follow-up issue). Use the default 'rename'." + ) }} + {%- elif table_refresh_method != 'rename' -%} + {{ exceptions.raise_compiler_error( + "Invalid table_refresh_method '" ~ table_refresh_method ~ "'. Only 'rename' (default) is supported." + ) }} + {%- endif -%} + {%- set full_refresh_build = config.get('full_refresh_build', 'heap_then_index') -%} + {%- if full_refresh_build == 'prebuilt' -%} + {{ exceptions.raise_compiler_error( + "full_refresh_build='prebuilt' is not implemented yet in dbt Core v2's SQL Server " + "adapter (tracked as a follow-up issue). Use the default 'heap_then_index'." + ) }} + {%- elif full_refresh_build != 'heap_then_index' -%} + {{ exceptions.raise_compiler_error( + "Invalid full_refresh_build '" ~ full_refresh_build ~ "'. Only 'heap_then_index' (default) is supported." + ) }} + {%- endif -%} + + -- drop the temp relations if they exist already in the database + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% call statement('main') -%} + {{ get_create_table_as_sql(False, intermediate_relation, sql) }} + {%- endcall %} + + -- cleanup + {% if existing_relation is not none %} + /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped + since the variable was first set. */ + {% set existing_relation = load_cached_relation(existing_relation) %} + {% if existing_relation is not none %} + {{ adapter.rename_relation(existing_relation, backup_relation) }} + {% endif %} + {% endif %} + + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + + {% do create_indexes(target_relation) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {{ build_model_constraints(target_relation) }} + + -- `COMMIT` happens here + {{ adapter.commit() }} + + -- finally, drop the existing/backup relation after the commit + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} +{% endmaterialization %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/unit_test/get_fixture_sql.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/unit_test/get_fixture_sql.sql new file mode 100644 index 00000000000..d1cf5183721 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/unit_test/get_fixture_sql.sql @@ -0,0 +1,101 @@ +{# + dbt-core does not dispatch "get_fixture_sql" or "get_expected_sql", so this file + shadows the implementations from the dbt global project + (macros/unit_test_sql/get_fixture_sql.sql) - adapter package macros take + precedence over the global project. Keep in sync with dbt-core when upgrading. + + To ease a future transition if dbt-core adds adapter.dispatch for these macros, + the public wrappers below are intentionally thin. They delegate to the + SQL Server-specific implementations (sqlserver__get_fixture_sql and + sqlserver__get_expected_sql), which contain the copied/adapted upstream logic. + + When upstream dispatches these macros, the public wrappers can be deleted and + the sqlserver__ implementations kept as the dispatched handlers. + + Changes from upstream (see dbt-msft/dbt-sqlserver#698): + - sqlserver__get_fixture_sql: the empty-rows branch emits "select top 0" + instead of "limit 0", which is not valid T-SQL. + - sqlserver__get_expected_sql: the empty-rows branch emits a "select top 0" + of typed nulls instead of "select * from dbt_internal_unit_test_actual limit 0". + Besides the invalid "limit", sqlserver__get_unit_test_sql wraps the expected + SQL in its own view, where that CTE name is out of scope. +#} + +{% macro get_fixture_sql(rows, column_name_to_data_types) %} + {{ return(sqlserver__get_fixture_sql(rows, column_name_to_data_types)) }} +{% endmacro %} + + +{% macro sqlserver__get_fixture_sql(rows, column_name_to_data_types) %} +-- Fixture for {{ model.name }} +{% set default_row = {} %} + +{%- if not column_name_to_data_types -%} +{#-- Use defer_relation IFF it is available in the manifest and 'this' is missing from the database --#} +{%- set this_or_defer_relation = defer_relation if (defer_relation and not load_relation(this)) else this -%} +{%- set columns_in_relation = adapter.get_columns_in_relation(this_or_defer_relation) -%} + +{%- set column_name_to_data_types = {} -%} +{%- set column_name_to_quoted = {} -%} +{%- for column in columns_in_relation -%} + +{#-- This needs to be a case-insensitive comparison --#} +{%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%} +{%- do column_name_to_quoted.update({column.name|lower: column.quoted}) -%} +{%- endfor -%} +{%- endif -%} + +{%- if not column_name_to_data_types -%} + {{ exceptions.raise_compiler_error("Not able to get columns for unit test '" ~ model.name ~ "' from relation " ~ this ~ " because the relation doesn't exist") }} +{%- endif -%} + +{%- for column_name, column_type in column_name_to_data_types.items() -%} + {%- do default_row.update({column_name: (safe_cast("null", column_type) | trim )}) -%} +{%- endfor -%} + +{{ validate_fixture_rows(rows, row_number) }} + +{%- for row in rows -%} +{%- set formatted_row = format_row(row, column_name_to_data_types) -%} +{%- set default_row_copy = default_row.copy() -%} +{%- do default_row_copy.update(formatted_row) -%} +select +{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name_to_quoted[column_name] }}{% if not loop.last -%}, {%- endif %} +{%- endfor %} +{%- if not loop.last %} +union all +{% endif %} +{%- endfor -%} + +{%- if (rows | length) == 0 -%} + select top 0 + {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name_to_quoted[column_name] }}{% if not loop.last -%},{%- endif %} + {%- endfor %} +{%- endif -%} +{% endmacro %} + + +{% macro get_expected_sql(rows, column_name_to_data_types, column_name_to_quoted) %} + {{ return(sqlserver__get_expected_sql(rows, column_name_to_data_types, column_name_to_quoted)) }} +{% endmacro %} + + +{% macro sqlserver__get_expected_sql(rows, column_name_to_data_types, column_name_to_quoted) %} + +{%- if (rows | length) == 0 -%} + select top 0 + {%- for column_name, column_type in column_name_to_data_types.items() %} {{ safe_cast("null", column_type) | trim }} as {{ column_name_to_quoted[column_name] }}{% if not loop.last -%},{%- endif %} + {%- endfor %} +{%- else -%} +{%- for row in rows -%} +{%- set formatted_row = format_row(row, column_name_to_data_types) -%} +select +{%- for column_name, column_value in formatted_row.items() %} {{ column_value }} as {{ column_name_to_quoted[column_name] }}{% if not loop.last -%}, {%- endif %} +{%- endfor %} +{%- if not loop.last %} +union all +{% endif %} +{%- endfor -%} +{%- endif -%} + +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/view/create_view_as.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/view/create_view_as.sql new file mode 100644 index 00000000000..3ea79c48852 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/view/create_view_as.sql @@ -0,0 +1,30 @@ +{% macro sqlserver__create_view_as(relation, sql) -%} + {#- Only guard against user-configured view materializations; this macro is also + called for intermediate temp views during table/snapshot materializations, + where query_options is intended for the *outer* DML and shouldn't trip a guard here. -#} + {%- if config.get('materialized') == 'view' -%} + {{ raise_if_query_options_set('view materializations (SQL Server does not accept OPTION on CREATE VIEW)') }} + {%- endif -%} + + {%- if config.get('sql_header') -%} + {{ exceptions.raise_compiler_error( + "sql_header is not supported on SQL Server. " + "CREATE VIEW must be the first statement in a batch, so sql_header cannot run in the same query. " + "Use pre_hooks for pre-model SQL or query_options for session-level SET options (e.g. query_options={'NOCOUNT': 'ON'})." + ) }} + {%- endif -%} + + {{ get_use_database_sql(relation.database) }} + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced %} + {{ get_assert_columns_equivalent(sql) }} + {%- endif %} + + {% set query %} + CREATE OR ALTER VIEW {{ relation.include(database=False) }} AS {{ sql }}; + {% endset %} + + {{ get_use_database_sql(relation.database) }} + EXEC('{{- escape_single_quotes(query) -}}') + +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/view/view.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/view/view.sql new file mode 100644 index 00000000000..3eea0c2d992 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/models/view/view.sql @@ -0,0 +1,109 @@ +{%- materialization view, adapter='sqlserver' -%} + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='view') -%} + {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} + + -- the intermediate_relation should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} + /* + This relation (probably) doesn't exist yet. If it does exist, it's a leftover from + a previous run, and we're going to try to drop it immediately. At the end of this + materialization, we're going to rename the "existing_relation" to this identifier, + and then we're going to drop it. In order to make sure we run the correct one of: + - drop view ... + - drop table ... + + We need to set the type of this relation to be the type of the existing_relation, if it exists, + or else "view" as a sane default if it does not. Note that if the existing_relation does not + exist, then there is nothing to move out of the way and subsequentally drop. In that case, + this relation will be effectively unused. + */ + {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + -- as above, the backup_relation should not already exist + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparison later on + {% set grant_config = config.get('grants') %} + {% set preserved_grants = {} %} + {% set should_skip_view_update = false %} + {% set build_sql = none %} + + {% if existing_relation is not none and existing_relation.type != 'view' %} + {% set current_grants_table = run_query(get_show_grant_sql(existing_relation)) %} + {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %} + {% set preserved_grants = {} %} + {% for privilege, grantees in diff_of_two_dicts(current_grants_dict, grant_config).items() %} + {% if privilege | lower in ['select', 'insert', 'update', 'delete'] %} + {% do preserved_grants.update({privilege: grantees}) %} + {% endif %} + {% endfor %} + {% set build_sql = get_create_view_as_sql(intermediate_relation, sql) %} + {% elif existing_relation is not none and existing_relation.type == 'view' %} + {% set current_view_definition_table = run_query(get_view_definition_sql(existing_relation)) %} + {% if current_view_definition_table is not none and current_view_definition_table.rows | length > 0 %} + {% set normalized_relation = target_relation.include(database=False) | lower | replace('\n', '') | replace('\r', '') | replace('\t', '') | replace(' ', '') | replace(';', '') %} + {% set normalized_sql = sql | lower | replace('\n', '') | replace('\r', '') | replace('\t', '') | replace(' ', '') | replace(';', '') %} + {% set normalized_definition = current_view_definition_table.rows[0][0] | lower | replace('\n', '') | replace('\r', '') | replace('\t', '') | replace(' ', '') | replace(';', '') %} + {% set should_skip_view_update = normalized_definition.endswith(normalized_sql) %} + {% endif %} + {% if should_skip_view_update %} + {% set build_sql = 'declare @dbt_sqlserver_noop int;' %} + {% else %} + {% set build_sql = get_create_view_as_sql(target_relation, sql) %} + {% endif %} + {% else %} + {% set build_sql = get_create_view_as_sql(target_relation, sql) %} + {% endif %} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- drop the temp relations if they exist already in the database + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + {% if existing_relation is not none and existing_relation.type != 'view' %} + -- build model + {% call statement('main') -%} + {{ build_sql }} + {%- endcall %} + + -- cleanup + -- move the existing relation out of the way + {% set existing_relation = load_cached_relation(existing_relation) %} + {% if existing_relation is not none %} + {{ adapter.rename_relation(existing_relation, backup_relation) }} + {% endif %} + + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + {% else %} + -- build model + {% call statement('main') -%} + {{ build_sql }} + {%- endcall %} + {% endif %} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% if preserved_grants %} + {% do apply_grants(target_relation, preserved_grants, should_revoke=False) %} + {% endif %} + + {% do persist_docs(target_relation, model) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {{ adapter.commit() }} + + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{%- endmaterialization -%} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/seeds/helpers.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/seeds/helpers.sql new file mode 100644 index 00000000000..46b59f0a1c2 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/seeds/helpers.sql @@ -0,0 +1,60 @@ +{% macro sqlserver__get_binding_char() %} + {{ return('?') }} +{% endmacro %} + +{% macro sqlserver__get_batch_size() %} + {{ return(400) }} +{% endmacro %} + +{% macro calc_batch_size(num_columns) %} + {# + SQL Server allows for a max of 2098 parameters in a single statement. + Check if the max_batch_size fits with the number of columns, otherwise + reduce the batch size so it fits. + #} + {% set max_batch_size = get_batch_size() %} + {% set calculated_batch = (2098 / num_columns)|int %} + {% set batch_size = [max_batch_size, calculated_batch] | min %} + + {{ return(batch_size) }} +{% endmacro %} + +{% macro sqlserver__load_csv_rows(model, agate_table) %} + {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %} + {% set batch_size = calc_batch_size(agate_table.column_names|length) %} + {% set statements = [] %} + + {{ log("Inserting batches of " ~ batch_size ~ " records") }} + + {% for chunk in agate_table.rows | batch(batch_size) %} + {% set bindings = [] %} + {% set values_clause = [] %} + + {% for row in chunk %} + {% set row_values = [] %} + {% for column in agate_table.column_names %} + {%- set val = row[loop.index0] -%} + {%- if val is none -%} + {%- do row_values.append("null") -%} + {%- else -%} + {%- do row_values.append(get_binding_char()) -%} + {%- do bindings.append(val) -%} + {%- endif -%} + {% endfor %} + {% do values_clause.append("(" ~ row_values | join(", ") ~ ")") %} + {% endfor %} + + {% set sql %} + insert into {{ this.render() }} ({{ cols_sql }}) values {{ values_clause | join(", ") }} + {% endset %} + + {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %} + + {% if loop.index0 == 0 %} + {% do statements.append(sql) %} + {% endif %} + {% endfor %} + + {# Return SQL so we can render it out into the compiled files #} + {{ return(statements[0]) }} +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/snapshots/helpers.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/snapshots/helpers.sql new file mode 100644 index 00000000000..0f1e908bebf --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/snapshots/helpers.sql @@ -0,0 +1,189 @@ +{% macro sqlserver__create_columns(relation, columns) %} + {% set column_list %} + {% for column_entry in columns %} + {{column_entry.name}} {{column_entry.data_type}}{{ ", " if not loop.last }} + {% endfor %} + {% endset %} + + {% set alter_sql %} + ALTER TABLE {{ relation }} + ADD {{ column_list }} + {% endset %} + + {% set results = run_query(alter_sql) %} + +{% endmacro %} + +{% macro build_snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) %} + {% set temp_relation = make_temp_relation(target_relation) %} + {{ adapter.drop_relation(temp_relation) }} + + {% set select = snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) %} + + {% set tmp_tble_vw_relation = temp_relation.incorporate(path={"identifier": temp_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} + -- Dropping temp view relation if it exists + {{ adapter.drop_relation(tmp_tble_vw_relation) }} + + {% call statement('build_snapshot_staging_relation') %} + {{ get_create_table_as_sql(True, temp_relation, select) }} + {% endcall %} + + -- Dropping temp view relation if it exists + {{ adapter.drop_relation(tmp_tble_vw_relation) }} + + {% do return(temp_relation) %} +{% endmacro %} + + +{% macro sqlserver__post_snapshot(staging_relation) %} + -- Clean up the snapshot temp table + {% do drop_relation_if_exists(staging_relation) %} +{% endmacro %} + +{% macro sqlserver__get_true_sql() %} + {{ return('1=1') }} +{% endmacro %} + +{% macro sqlserver__build_snapshot_table(strategy, relation) %} + {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} + select *, + {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}, + {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, + {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, + {{ get_dbt_valid_to_current(strategy, columns) }} + {%- if strategy.hard_deletes == 'new_record' -%} + , 'False' as {{ columns.dbt_is_deleted }} + {% endif -%} + from ( + select * from {{ relation }} + ) sbq + +{% endmacro %} + +{% macro sqlserver__snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) -%} + + {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} + + with snapshot_query as ( + select * from {{ temp_snapshot_relation }} + ), + snapshotted_data as ( + select *, + {{ unique_key_fields(strategy.unique_key) }} + from {{ target_relation }} + where + {% if config.get('dbt_valid_to_current') %} + {# Check for either dbt_valid_to_current OR null, in order to correctly update records with nulls #} + ( {{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or {{ columns.dbt_valid_to }} is null) + {% else %} + {{ columns.dbt_valid_to }} is null + {% endif %} + {%- if strategy.hard_deletes == 'new_record' -%} + and {{ columns.dbt_is_deleted }} = 'False' + {% endif -%} + ), + insertions_source_data as ( + select *, + {{ unique_key_fields(strategy.unique_key) }}, + {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, + {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, + {{ get_dbt_valid_to_current(strategy, columns) }}, + {{ strategy.scd_id }} as {{ columns.dbt_scd_id }} + from snapshot_query + ), + updates_source_data as ( + select *, + {{ unique_key_fields(strategy.unique_key) }}, + {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, + {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, + {{ strategy.updated_at }} as {{ columns.dbt_valid_to }} + from snapshot_query + ), + {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %} + deletes_source_data as ( + select *, {{ unique_key_fields(strategy.unique_key) }} + from snapshot_query + ), + {% endif %} + insertions as ( + select 'insert' as dbt_change_type, source_data.* + {%- if strategy.hard_deletes == 'new_record' -%} + ,'False' as {{ columns.dbt_is_deleted }} + {%- endif %} + from insertions_source_data as source_data + left outer join snapshotted_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} + where {{ unique_key_is_null(strategy.unique_key, "snapshotted_data") }} + or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and ({{ strategy.row_changed }})) + ), + updates as ( + select 'update' as dbt_change_type, source_data.*, + snapshotted_data.{{ columns.dbt_scd_id }} + {%- if strategy.hard_deletes == 'new_record' -%} + , snapshotted_data.{{ columns.dbt_is_deleted }} + {%- endif %} + from updates_source_data as source_data + join snapshotted_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} + where ({{ strategy.row_changed }}) + ) + {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %} + , + deletes as ( + select 'delete' as dbt_change_type, + source_data.*, + {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }}, + {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }}, + {{ snapshot_get_time() }} as {{ columns.dbt_valid_to }}, + snapshotted_data.{{ columns.dbt_scd_id }} + {%- if strategy.hard_deletes == 'new_record' -%} + , snapshotted_data.{{ columns.dbt_is_deleted }} + {%- endif %} + from snapshotted_data + left join deletes_source_data as source_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} + where {{ unique_key_is_null(strategy.unique_key, "source_data") }} + ) + {%- endif %} + {%- if strategy.hard_deletes == 'new_record' %} + {%set source_query = "select * from "~temp_snapshot_relation%} + {% set source_sql_cols = get_column_schema_from_query(source_query) %} + , + deletion_records as ( + + select + 'insert' as dbt_change_type, + {%- for col in source_sql_cols -%} + snapshotted_data.{{ adapter.quote(col.column) }}, + {% endfor -%} + {%- if strategy.unique_key | is_list -%} + {%- for key in strategy.unique_key -%} + snapshotted_data.{{ key }} as dbt_unique_key_{{ loop.index }}, + {% endfor -%} + {%- else -%} + snapshotted_data.dbt_unique_key as dbt_unique_key, + {% endif -%} + {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }}, + {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }}, + snapshotted_data.{{ columns.dbt_valid_to }} as {{ columns.dbt_valid_to }}, + snapshotted_data.{{ columns.dbt_scd_id }}, + 'True' as {{ columns.dbt_is_deleted }} + from snapshotted_data + left join deletes_source_data as source_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} + where {{ unique_key_is_null(strategy.unique_key, "source_data") }} + ) + {%- endif %} + select * from insertions + union all + select * from updates + {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %} + union all + select * from deletes + {%- endif %} + {%- if strategy.hard_deletes == 'new_record' %} + union all + select * from deletion_records + {%- endif %} + +{%- endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/snapshots/snapshot.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/snapshots/snapshot.sql new file mode 100644 index 00000000000..3401ec9a458 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/snapshots/snapshot.sql @@ -0,0 +1,126 @@ +{% materialization snapshot, adapter='sqlserver' %} + + {%- set config = model['config'] -%} + {%- set target_table = model.get('alias', model.get('name')) -%} + {%- set strategy_name = config.get('strategy') -%} + {%- set unique_key = config.get('unique_key') %} + -- grab current tables grants config for comparison later on + {%- set grant_config = config.get('grants') -%} + + {% set target_relation_exists, target_relation = get_or_create_relation( + database=model.database, + schema=model.schema, + identifier=target_table, + type='table') -%} + + {%- if not target_relation.is_table -%} + {% do exceptions.relation_wrong_type(target_relation, 'table') %} + {%- endif -%} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + {% set strategy_macro = strategy_dispatch(strategy_name) %} + {% set strategy = strategy_macro(model, "snapshotted_data", "source_data", config, target_relation_exists) %} + + {% set temp_snapshot_relation_exists, temp_snapshot_relation = get_or_create_relation( + database=model.database, + schema=model.schema, + identifier=target_table+"_snapshot_staging_temp_view", + type='view') -%} + + -- Create a temporary view to manage if user SQl uses CTE + {% set temp_snapshot_relation_sql = model['compiled_code'] %} + {{ adapter.drop_relation(temp_snapshot_relation) }} + + {% call statement('create temp_snapshot_relation') -%} + {{ get_create_view_as_sql(temp_snapshot_relation, temp_snapshot_relation_sql) }} + {%- endcall %} + + {% if not target_relation_exists %} + + {% set build_sql = build_snapshot_table(strategy, temp_snapshot_relation) %} + {% set build_or_select_sql = build_sql %} + + -- naming a temp relation + {% set tmp_relation_view = target_relation.incorporate(path={"identifier": target_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} + -- SQL server adapter uses temp relation because of lack of CTE support for CTE in CTAS, Insert + -- drop temp relation if exists + {{ adapter.drop_relation(tmp_relation_view) }} + {% set final_sql = get_create_table_as_sql(False, target_relation, build_sql) %} + {{ adapter.drop_relation(tmp_relation_view) }} + + {% else %} + + {% set columns = get_snapshot_table_column_names() %} + {% set meta = config.get("snapshot_meta_column_names") %} + {% if meta %} + {% if meta.dbt_valid_from %}{% do columns.update({"dbt_valid_from": meta.dbt_valid_from}) %}{% endif %} + {% if meta.dbt_valid_to %}{% do columns.update({"dbt_valid_to": meta.dbt_valid_to}) %}{% endif %} + {% if meta.dbt_scd_id %}{% do columns.update({"dbt_scd_id": meta.dbt_scd_id}) %}{% endif %} + {% if meta.dbt_updated_at %}{% do columns.update({"dbt_updated_at": meta.dbt_updated_at}) %}{% endif %} + {% if meta.dbt_is_deleted %}{% do columns.update({"dbt_is_deleted": meta.dbt_is_deleted}) %}{% endif %} + {% endif %} + {{ adapter.valid_snapshot_target(target_relation, columns) }} + {% set build_or_select_sql = snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) %} + {% set staging_table = build_snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) %} + -- this may no-op if the database does not require column expansion + {% set expansion_max_rows = config.get('column_type_expansion_max_rows', 1000000) %} + {% do adapter.expand_target_column_types(from_relation=staging_table, + to_relation=target_relation, + max_rows=expansion_max_rows) %} + + {% set remove_columns = ['dbt_change_type', 'DBT_CHANGE_TYPE', 'dbt_unique_key', 'DBT_UNIQUE_KEY'] %} + {% if unique_key | is_list %} + {% for key in strategy.unique_key %} + {{ remove_columns.append('dbt_unique_key_' + loop.index|string) }} + {{ remove_columns.append('DBT_UNIQUE_KEY_' + loop.index|string) }} + {% endfor %} + {% endif %} + {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation) + | rejectattr('name', 'in', remove_columns) + | list %} + {% if missing_columns|length > 0 %} + {{log("Missing columns length is: "~ missing_columns|length)}} + {% do create_columns(target_relation, missing_columns) %} + {% endif %} + {% set source_columns = adapter.get_columns_in_relation(staging_table) + | rejectattr('name', 'in', remove_columns) + | list %} + {% set quoted_source_columns = [] %} + {% for column in source_columns %} + {% do quoted_source_columns.append(adapter.quote(column.name)) %} + {% endfor %} + {% set final_sql = snapshot_merge_sql( + target = target_relation, + source = staging_table, + insert_cols = quoted_source_columns + ) + %} + {% endif %} + {{ check_time_data_types(build_or_select_sql) }} + {% call statement('main') %} + {{ final_sql }} + {% endcall %} + + {{ adapter.drop_relation(temp_snapshot_relation) }} + {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {% if not target_relation_exists %} + {% do create_indexes(target_relation) %} + {% endif %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + {{ adapter.commit() }} + + {% if staging_table is defined %} + {% do post_snapshot(staging_table) %} + {% endif %} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + {{ return({'relations': [target_relation]}) }} + +{% endmaterialization %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/snapshots/snapshot_merge.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/snapshots/snapshot_merge.sql new file mode 100644 index 00000000000..3b659f04b0c --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/snapshots/snapshot_merge.sql @@ -0,0 +1,30 @@ +{% macro sqlserver__snapshot_merge_sql(target, source, insert_cols) %} + + {%- set insert_cols_csv = insert_cols | join(', ') -%} + {%- set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() -%} + {%- set target_table = target.include(database=False) -%} + {%- set source_table = source.include(database=False) -%} + {% set target_columns_list = [] %} + {% for column in insert_cols %} + {% set target_columns_list = target_columns_list.append("DBT_INTERNAL_SOURCE."+column) %} + {% endfor %} + {%- set target_columns = target_columns_list | join(', ') -%} + + update DBT_INTERNAL_DEST + set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }} + from {{ target_table }} as DBT_INTERNAL_DEST + inner join {{ source_table }} as DBT_INTERNAL_SOURCE + on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }} + where DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete') + {% if config.get("dbt_valid_to_current") %} + and (DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null) + {% else %} + and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null + {% endif %} + {{ get_query_options(parse_options=True) }} + + insert into {{ target_table }} ({{ insert_cols_csv }}) + select {{target_columns}} from {{ source_table }} as DBT_INTERNAL_SOURCE + where DBT_INTERNAL_SOURCE.dbt_change_type = 'insert' + {{ get_query_options(parse_options=True) }} +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/snapshots/strategies.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/snapshots/strategies.sql new file mode 100644 index 00000000000..6a316c6f5f9 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/snapshots/strategies.sql @@ -0,0 +1,5 @@ +{% macro sqlserver__snapshot_hash_arguments(args) %} + CONVERT(VARCHAR(32), HashBytes('MD5', {% for arg in args %} + coalesce(cast({{ arg }} as varchar(8000)), '') {% if not loop.last %} + '|' + {% endif %} + {% endfor %}), 2) +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/tests/helpers.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/tests/helpers.sql new file mode 100644 index 00000000000..bc47757958f --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/materializations/tests/helpers.sql @@ -0,0 +1,80 @@ +{% macro sqlserver__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%} + + -- Create target schema if it does not + {{ get_use_database_sql(target.database) }} + IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ target.schema }}') + BEGIN + EXEC('CREATE SCHEMA {{ adapter.quote(target.schema) }}') + END + + {% set testview_name = "testview_" ~ local_md5(main_sql) ~ "_" ~ (range(1300, 19000) | random) %} + {% set testview %} + {{ adapter.quote(target.schema) }}.{{ adapter.quote(testview_name) }} + {% endset %} + + {% set sql = main_sql.replace("'", "''")%} + EXEC('create view {{testview}} as {{ sql }};') + select + {{ "top (" ~ limit ~ ')' if limit != none }} + {{ fail_calc }} as failures, + case when {{ fail_calc }} {{ warn_if }} + then 'true' else 'false' end as should_warn, + case when {{ fail_calc }} {{ error_if }} + then 'true' else 'false' end as should_error + from ( + select * from {{testview}} + ) dbt_internal_test; + + EXEC('drop view {{testview}};') + +{%- endmacro %} + +{% macro sqlserver__get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%} + + {{ get_use_database_sql(target.database) }} + IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ target.schema }}') + BEGIN + EXEC('CREATE SCHEMA {{ adapter.quote(target.schema) }}') + END + + {% set test_view_name = "testview_" ~ local_md5(main_sql) ~ "_" ~ (range(1300, 19000) | random) %} + {% set test_view %} + {{ adapter.quote(target.schema) }}.{{ adapter.quote(test_view_name) }} + {% endset %} + {% set test_sql = main_sql.replace("'", "''")%} + EXEC('create view {{test_view}} as {{ test_sql }};') + + {% set expected_view_name = "expectedview_" ~ local_md5(expected_fixture_sql) ~ "_" ~ (range(1300, 19000) | random) %} + {% set expected_view %} + {{ adapter.quote(target.schema) }}.{{ adapter.quote(expected_view_name) }} + {% endset %} + {% set expected_sql = expected_fixture_sql.replace("'", "''")%} + EXEC('create view {{expected_view}} as {{ expected_sql }};') + + -- Build actual result given inputs + {% set unittest_sql %} + with dbt_internal_unit_test_actual as ( + select + {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal("actual") }} as {{ adapter.quote("actual_or_expected") }} + from + {{ test_view }} + ), + -- Build expected result + dbt_internal_unit_test_expected as ( + select + {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal("expected") }} as {{ adapter.quote("actual_or_expected") }} + from + {{ expected_view }} + ) + -- Union actual and expected results + select * from dbt_internal_unit_test_actual + union all + select * from dbt_internal_unit_test_expected + {% endset %} + + EXEC('{{- escape_single_quotes(unittest_sql) -}}') + + EXEC('drop view {{test_view}};') + EXEC('drop view {{expected_view}};') + +{%- endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/any_value.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/any_value.sql new file mode 100644 index 00000000000..6dcf8ec2689 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/any_value.sql @@ -0,0 +1,5 @@ +{% macro sqlserver__any_value(expression) -%} + + min({{ expression }}) + +{%- endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/array_construct.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/array_construct.sql new file mode 100644 index 00000000000..5088c9acda8 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/array_construct.sql @@ -0,0 +1,3 @@ +{% macro sqlserver__array_construct(inputs, data_type) -%} + JSON_ARRAY({{ inputs|join(' , ') }}) +{%- endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/cast_bool_to_text.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/cast_bool_to_text.sql new file mode 100644 index 00000000000..9771afbf16e --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/cast_bool_to_text.sql @@ -0,0 +1,7 @@ +{% macro sqlserver__cast_bool_to_text(field) %} + case {{ field }} + when 1 then 'true' + when 0 then 'false' + else null + end +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/concat.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/concat.sql new file mode 100644 index 00000000000..1b7c17559a7 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/concat.sql @@ -0,0 +1,7 @@ +{% macro sqlserver__concat(fields) -%} + {%- if fields|length < 2 -%} + {{ fields[0] }} + {%- else -%} + concat({{ fields|join(', ') }}) + {%- endif -%} +{%- endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/date_trunc.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/date_trunc.sql new file mode 100644 index 00000000000..85b4ce326a5 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/date_trunc.sql @@ -0,0 +1,3 @@ +{% macro sqlserver__date_trunc(datepart, date) %} + CAST(DATEADD({{datepart}}, DATEDIFF({{datepart}}, 0, {{date}}), 0) AS DATE) +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/dateadd.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/dateadd.sql new file mode 100644 index 00000000000..f3b24fa6f5d --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/dateadd.sql @@ -0,0 +1,9 @@ +{% macro sqlserver__dateadd(datepart, interval, from_date_or_timestamp) %} + + dateadd( + {{ datepart }}, + {{ interval }}, + cast({{ from_date_or_timestamp }} as datetime2(6)) + ) + +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/get_tables_by_pattern.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/get_tables_by_pattern.sql new file mode 100644 index 00000000000..75d6b500b14 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/get_tables_by_pattern.sql @@ -0,0 +1,12 @@ +{% macro sqlserver__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + + select distinct + table_schema as {{ adapter.quote('table_schema') }}, + table_name as {{ adapter.quote('table_name') }}, + {{ dbt_utils.get_table_types_sql() }} + from {{ database }}.INFORMATION_SCHEMA.TABLES + where table_schema like '{{ schema_pattern }}' + and table_name like '{{ table_pattern }}' + and table_name not like '{{ exclude }}' + +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/hash.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/hash.sql new file mode 100644 index 00000000000..d965f81f0d6 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/hash.sql @@ -0,0 +1,3 @@ +{% macro sqlserver__hash(field) %} + lower(convert(varchar(50), hashbytes('md5', coalesce(convert(varchar(8000), {{field}}), '')), 2)) +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/last_day.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/last_day.sql new file mode 100644 index 00000000000..c523d944087 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/last_day.sql @@ -0,0 +1,13 @@ +{% macro sqlserver__last_day(date, datepart) -%} + + {%- if datepart == 'quarter' -%} + CAST(DATEADD(QUARTER, DATEDIFF(QUARTER, 0, {{ date }}) + 1, -1) AS DATE) + {%- elif datepart == 'month' -%} + EOMONTH ( {{ date }}) + {%- elif datepart == 'year' -%} + CAST(DATEADD(YEAR, DATEDIFF(year, 0, {{ date }}) + 1, -1) AS DATE) + {%- else -%} + {{dbt_utils.default_last_day(date, datepart)}} + {%- endif -%} + +{%- endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/length.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/length.sql new file mode 100644 index 00000000000..ee9431ace31 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/length.sql @@ -0,0 +1,5 @@ +{% macro sqlserver__length(expression) %} + + len( {{ expression }} ) + +{%- endmacro -%} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/listagg.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/listagg.sql new file mode 100644 index 00000000000..4d6ab215f3c --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/listagg.sql @@ -0,0 +1,8 @@ +{% macro sqlserver__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} + + string_agg({{ measure }}, {{ delimiter_text }}) + {%- if order_by_clause != None %} + within group ({{ order_by_clause }}) + {%- endif %} + +{%- endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/position.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/position.sql new file mode 100644 index 00000000000..bd3f657750e --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/position.sql @@ -0,0 +1,8 @@ +{% macro sqlserver__position(substring_text, string_text) %} + + CHARINDEX( + {{ substring_text }}, + {{ string_text }} + ) + +{%- endmacro -%} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/safe_cast.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/safe_cast.sql new file mode 100644 index 00000000000..4ae065a7952 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/safe_cast.sql @@ -0,0 +1,3 @@ +{% macro sqlserver__safe_cast(field, type) %} + try_cast({{field}} as {{type}}) +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/split_part.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/split_part.sql new file mode 100644 index 00000000000..2f67cf32ac3 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/split_part.sql @@ -0,0 +1,17 @@ +{# + For more information on how this XML trick works with splitting strings, see https://www.sqlservertips.com/sqlservertip/1771/splitting-delimited-strings-using-xml-in-sql-server/ + On Azure SQL and SQL Server 2019, we can use the string_split function instead of the XML trick. + But since we don't know which version of SQL Server the user is using, we'll stick with the XML trick in this adapter. + However, since the XML data type is not supported in Synapse, it has to be overridden in that adapter. + + To adjust for negative part numbers, aka 'from the end of the split', we take the position and subtract from last to get the specific part. + Since the input is '-1' for the last, '-2' for second last, we add 1 to the part number to get the correct position. +#} + +{% macro sqlserver__split_part(string_text, delimiter_text, part_number) %} + {% if part_number >= 0 %} + LTRIM(CAST((''+REPLACE({{ string_text }},{{ delimiter_text }} ,'')+'') AS XML).value('(/X)[{{ part_number }}]', 'VARCHAR(128)')) + {% else %} + LTRIM(CAST((''+REPLACE({{ string_text }},{{ delimiter_text }} ,'')+'') AS XML).value('(/X)[position() = last(){{ part_number }}+1][1]', 'VARCHAR(128)')) + {% endif %} +{% endmacro %} diff --git a/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/timestamps.sql b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/timestamps.sql new file mode 100644 index 00000000000..31795764e94 --- /dev/null +++ b/crates/dbt-loader/src/dbt_macro_assets/dbt-sqlserver/macros/utils/timestamps.sql @@ -0,0 +1,8 @@ +{% macro sqlserver__current_timestamp() -%} + CAST(SYSDATETIME() AS DATETIME2(6)) +{%- endmacro %} + +{% macro sqlserver__snapshot_string_as_time(timestamp) -%} + {%- set result = "CONVERT(DATETIME2(6), '" ~ timestamp ~ "')" -%} + {{ return(result) }} +{%- endmacro %} diff --git a/crates/dbt-loader/tests/materializations/mod.rs b/crates/dbt-loader/tests/materializations/mod.rs index 89b7385446e..86ad5e8b84c 100644 --- a/crates/dbt-loader/tests/materializations/mod.rs +++ b/crates/dbt-loader/tests/materializations/mod.rs @@ -1,3 +1,4 @@ mod incremental; mod materialized_view; +mod table; mod view; diff --git a/crates/dbt-loader/tests/materializations/table.rs b/crates/dbt-loader/tests/materializations/table.rs new file mode 100644 index 00000000000..c500f99fc74 --- /dev/null +++ b/crates/dbt-loader/tests/materializations/table.rs @@ -0,0 +1,178 @@ +use std::collections::BTreeMap; +use std::sync::Arc; + +use dbt_adapter::relation::RelationObject; +use dbt_adapter_core::AdapterType; +use dbt_jinja_utils::mock_object::MockJinjaObject; +use dbt_schemas::dbt_types::RelationType; +use minijinja::Value; + +use crate::macro_test_harness::MacroTestHarness; + +mod sqlserver { + use super::*; + const ADAPTER: AdapterType = AdapterType::SqlServer; + + fn build_harness() -> MacroTestHarness { + let harness = MacroTestHarness::for_adapter(ADAPTER) + .load_all_macros() + .with_stub_functions() + .build() + .expect("harness should build"); + + let mock = harness.mock(); + mock.on("quote", |args| { + Ok(args.first().cloned().unwrap_or(Value::UNDEFINED)) + }); + mock.on("rename_relation", |_| Ok(Value::UNDEFINED)); + mock.on("drop_relation", |_| Ok(Value::UNDEFINED)); + mock.on("commit", |_| Ok(Value::UNDEFINED)); + mock.on("render_raw_model_constraints", |_| { + Ok(Value::from(Vec::::new())) + }); + + harness + } + + /// A `config` mock that always answers `contract`/`indexes` with a + /// no-op-safe default (empty), with room for a test to override specific + /// keys. `config.get(key, default=...)` uses a keyword arg for `default` + /// on some call sites (e.g. dbt-adapters' own `create_indexes`), which + /// this harness's generic positional fallback doesn't unwrap correctly - + /// so any key a macro iterates or branches on needs its own explicit arm + /// here rather than relying on the passthrough default. + fn config_mock(overrides: BTreeMap<&'static str, Value>) -> Arc { + let mock = Arc::new(MockJinjaObject::new()); + mock.on("get", move |args| { + let key = args.first().and_then(|v| v.as_str()).unwrap_or(""); + if let Some(value) = overrides.get(key) { + return Ok(value.clone()); + } + match key { + "contract" => Ok(Value::from_serialize(BTreeMap::from([( + "enforced".to_string(), + Value::from(false), + )]))), + "indexes" => Ok(Value::from(Vec::::new())), + // Positional `config.get(key, default)` calls (table_refresh_method, + // full_refresh_build, ...) land their default in args[1]; keyword + // `default=` calls (dbt-adapters' own create_indexes) don't, which is + // why indexes/contract get their own arms above instead of relying + // on this. + _ => Ok(args.get(1).cloned().unwrap_or(Value::UNDEFINED)), + } + }); + mock.on("persist_column_docs", |_| Ok(Value::from(false))); + mock.on("persist_relation_docs", |_| Ok(Value::from(false))); + mock + } + + fn render_table( + harness: &MacroTestHarness, + ctx: BTreeMap, + ) -> dbt_common::FsResult { + harness.render("{{ materialization_table_sqlserver() }}", ctx) + } + + #[test] + fn no_existing_relation_renames_intermediate_into_target() { + let harness = build_harness(); + harness.mock().on("get_relation", |_| Ok(Value::from(()))); + + let ctx = harness + .materialization_context("my_table", "SELECT id FROM source_table") + .relation_type(RelationType::Table) + .config(Value::from_dyn_object(config_mock(BTreeMap::new()))) + .build(); + render_table(&harness, ctx) + .unwrap_or_else(|e| panic!("table materialization failed: {e:?}")); + + // Intermediate -> target only; no existing/backup relation to swap out. + assert_eq!( + harness + .mock() + .observed_calls() + .to("rename_relation") + .count(), + 1, + "expected only the intermediate->target rename" + ); + } + + #[test] + fn existing_table_renamed_to_backup_before_swap() { + let harness = build_harness(); + let existing = harness.relation( + "TEST_DB", + "TEST_SCHEMA", + "my_table", + Some(RelationType::Table), + ); + harness.mock().on("get_relation", move |_| { + Ok(RelationObject::new(Arc::clone(&existing)).into_value()) + }); + + let ctx = harness + .materialization_context("my_table", "SELECT id FROM source_table") + .relation_type(RelationType::Table) + .config(Value::from_dyn_object(config_mock(BTreeMap::new()))) + .build(); + render_table(&harness, ctx) + .unwrap_or_else(|e| panic!("table materialization failed: {e:?}")); + + // Two renames: existing -> backup, then intermediate -> target. + assert_eq!( + harness + .mock() + .observed_calls() + .to("rename_relation") + .count(), + 2, + "expected existing->backup and intermediate->target renames" + ); + } + + #[test] + fn full_refresh_build_prebuilt_raises_compiler_error() { + let harness = build_harness(); + harness.mock().on("get_relation", |_| Ok(Value::from(()))); + + let overrides = BTreeMap::from([("full_refresh_build", Value::from("prebuilt"))]); + let ctx = harness + .materialization_context("my_table", "SELECT id FROM source_table") + .relation_type(RelationType::Table) + .config(Value::from_dyn_object(config_mock(overrides))) + .build(); + + let result = render_table(&harness, ctx); + assert!( + result.is_err(), + "full_refresh_build='prebuilt' should raise a compiler error, got: {result:?}" + ); + } + + #[test] + fn indexes_config_raises_compiler_error() { + let harness = build_harness(); + harness.mock().on("get_relation", |_| Ok(Value::from(()))); + + let overrides = BTreeMap::from([( + "indexes", + Value::from_serialize(vec![BTreeMap::from([( + "columns".to_string(), + Value::from(vec![Value::from("id")]), + )])]), + )]); + let ctx = harness + .materialization_context("my_table", "SELECT id FROM source_table") + .relation_type(RelationType::Table) + .config(Value::from_dyn_object(config_mock(overrides))) + .build(); + + let result = render_table(&harness, ctx); + assert!( + result.is_err(), + "a configured `indexes:` should raise a compiler error rather than silently no-op, got: {result:?}" + ); + } +}