diff --git a/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_debug.png b/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_debug.png deleted file mode 100644 index c6371d6f417..00000000000 Binary files a/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_debug.png and /dev/null differ diff --git a/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_docs_generate.png b/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_docs_generate.png deleted file mode 100644 index a7f9646556e..00000000000 Binary files a/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_docs_generate.png and /dev/null differ diff --git a/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_docs_serve.png b/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_docs_serve.png deleted file mode 100644 index 332a1d391e2..00000000000 Binary files a/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_docs_serve.png and /dev/null differ diff --git a/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_init_database_name.png b/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_init_database_name.png deleted file mode 100644 index 26daeff834d..00000000000 Binary files a/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_init_database_name.png and /dev/null differ diff --git a/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_init_project_name.png b/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_init_project_name.png deleted file mode 100644 index 140e8841e70..00000000000 Binary files a/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_init_project_name.png and /dev/null differ diff --git a/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_run.png b/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_run.png deleted file mode 100644 index 544201bd40a..00000000000 Binary files a/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_run.png and /dev/null differ diff --git a/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_test.png b/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_test.png deleted file mode 100644 index 5c8f8711842..00000000000 Binary files a/quickstarts/elt/images/getting-started-with-airbyte-dbt/dbt_test.png and /dev/null differ diff --git a/quickstarts/elt/images/getting-started-with-airbyte-dbt/raw_data_vantage_dbeaver.png b/quickstarts/elt/images/getting-started-with-airbyte-dbt/raw_data_vantage_dbeaver.png deleted file mode 100644 index 79f2f94efb3..00000000000 Binary files a/quickstarts/elt/images/getting-started-with-airbyte-dbt/raw_data_vantage_dbeaver.png and /dev/null differ diff --git a/quickstarts/elt/images/terraform-airbyte-provider/airbyteconnection.png b/quickstarts/elt/images/terraform-airbyte-provider/airbyteconnection.png index 035f4707595..0892038fbf2 100644 Binary files a/quickstarts/elt/images/terraform-airbyte-provider/airbyteconnection.png and b/quickstarts/elt/images/terraform-airbyte-provider/airbyteconnection.png differ diff --git a/quickstarts/elt/images/terraform-airbyte-provider/terraformapply.png b/quickstarts/elt/images/terraform-airbyte-provider/terraformapply.png index bc63600db81..6dbca3e7574 100644 Binary files a/quickstarts/elt/images/terraform-airbyte-provider/terraformapply.png and b/quickstarts/elt/images/terraform-airbyte-provider/terraformapply.png differ diff --git a/quickstarts/elt/images/terraform-airbyte-provider/terraforminit.png b/quickstarts/elt/images/terraform-airbyte-provider/terraforminit.png index 6b33ba1d26c..934b5f5bf63 100644 Binary files a/quickstarts/elt/images/terraform-airbyte-provider/terraforminit.png and b/quickstarts/elt/images/terraform-airbyte-provider/terraforminit.png differ diff --git a/quickstarts/elt/images/terraform-airbyte-provider/terraformplan.png b/quickstarts/elt/images/terraform-airbyte-provider/terraformplan.png index 6b2009f4483..fd11c56ea44 100644 Binary files a/quickstarts/elt/images/terraform-airbyte-provider/terraformplan.png and b/quickstarts/elt/images/terraform-airbyte-provider/terraformplan.png differ diff --git a/quickstarts/elt/images/terraform-airbyte-provider/terraformvalidate.png b/quickstarts/elt/images/terraform-airbyte-provider/terraformvalidate.png new file mode 100644 index 00000000000..568c0b244f6 Binary files /dev/null and b/quickstarts/elt/images/terraform-airbyte-provider/terraformvalidate.png differ diff --git a/quickstarts/manage-data/terraform-airbyte-provider.md b/quickstarts/manage-data/terraform-airbyte-provider.md index 11181e9d5f4..da5defe68a1 100644 --- a/quickstarts/manage-data/terraform-airbyte-provider.md +++ b/quickstarts/manage-data/terraform-airbyte-provider.md @@ -1,49 +1,54 @@ --- sidebar_position: 9 -author: Janeth Graziani -email: Janeth.graziani@teradata.com -page_last_update: February 28, 2024 +author: Janeth Graziani, Daniel Herrera +email: developer.relations@teradata.com +page_last_update: July 10, 2026 description: Use Terraform to manage Teradata data pipelines in Airbyte using Terraform. -keywords: [Terraform, Airbyte, Teradata Vantage, data engineering, ELT, automation, data integration, CI/CD, version control] +keywords: [Terraform, Airbyte, Teradata, data engineering, ELT, automation, data integration, CI/CD, version control] --- import YouTubeVideo from '../_partials/terraform-video.mdx'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Manage ELT pipelines as code with Terraform and Airbyte on Teradata Vantage +# Manage ELT pipelines as code with Terraform and Airbyte on Teradata ### Overview -This quickstart explains how to use Terraform to manage Airbyte data pipelines as code. Instead of manual configurations through the WebUI, we'll use code to create and manage Airbyte resources. The provided example illustrates a basic ELT pipeline from Google Sheets to Teradata Vantage using Airbyte's Terraform provider. +This quickstart explains how to use Terraform to manage Airbyte data pipelines as code. Instead of manual configurations through the WebUI, we'll use code to create and manage Airbyte resources. The provided example illustrates a basic ELT pipeline from Google Sheets to Teradata using Airbyte's Terraform provider. -The Airbyte Terraform provider is available for users on Airbyte Cloud, OSS & Self-Managed Enterprise. +The Airbyte Terraform provider is available for users on Airbyte Cloud, OSS, and Self-Managed Enterprise. **This guide covers Airbyte Cloud setup. For OSS or Self-Managed Enterprise deployments, refer to [Airbyte's documentation](https://docs.airbyte.com/terraform-integration).** -Watch this concise explanation of how this integration works: +Watch this concise explanation of how this integration works (for the specific code refer to the examples below, the provider has been updated since the video was published): ### Introduction [Terraform](https://www.terraform.io) is a leading open-source tool in the Infrastructure as Code (IaC) space. It enables the automated provisioning and management of infrastructure, cloud platforms, and services via configuration files, instead of manual setup. Terraform uses plugins, known as Terraform providers, to communicate with infrastructure hosts, cloud providers, APIs, and SaaS platforms. -Airbyte, the data integration platform, has a Terraform provider that communicates directly with [Airbyte's API](https://reference.airbyte.com/reference/start). This allows data engineers to manage Airbyte configurations, enforce version control, and apply good data engineering practices within their ELT pipelines. +Airbyte, the data integration platform, has a Terraform provider that communicates directly with [Airbyte's API](https://reference.airbyte.com/reference/start). This allows data engineers to manage Airbyte configurations, enforce version control, and apply good data engineering practices within our ELT pipelines. ### Prerequisites -* [Airbyte Cloud Account](https://airbyte.com/connectors/teradata-vantage). Start with a 14-day free trial that begins after the first successful sync. -- Generate an Airbyte API Key by logging into the [developer portal](https://portal.airbyte.com). -* Teradata Vantage Instance. You will need a database `Host`, `Username`, and `Password` for Airbyte’s Terraform configuration. -- [Create a free Teradata instance on ClearScape Analytics Experience](../get-access-to-vantage/clearscape-analytics-experience/getting-started-with-csae.md) +* [Airbyte Cloud Account](https://airbyte.com/connectors/teradata-vantage). Start with a 30-day free trial that begins after the first successful sync. + - Log into [Airbyte Cloud ETL](https://airbyte.com/signin). + - [Obtain an Airbyte Client ID and Client Secret](https://docs.airbyte.com/platform/using-airbyte/configuring-api-access) -* Source Data. For demonstration purposes we will use a [sample Google Sheets,](https://docs.google.com/spreadsheets/d/1XNBYUw3p7xG6ptfwjChqZ-dNXbTuVwPi7ToQfYKgJIE/edit#gid=0). Make a copy of it into a personal Google worspace. +* Teradata Instance. You will need a database `host`, `username`, and `password` for Airbyte's Terraform configuration. + - [Create a free Teradata instance on Teradata Trial](https://www.teradata.com/try) -* [Google Cloud Platform API enabled for your personal or organizational account](https://support.google.com/googleapi/answer/6158841?hl=en]=). You’ll need to authenticate your Google account via OAuth or via Service Account Key Authenticator. In this example, we use [Service Account Key Authenticator](https://cloud.google.com/iam/docs/keys-create-delete). +* Source Data. For demonstration purposes, we will use a [sample Google Sheets](https://docs.google.com/spreadsheets/d/1XNBYUw3p7xG6ptfwjChqZ-dNXbTuVwPi7ToQfYKgJIE/edit#gid=0). + - Open the shared spreadsheet link. + - Click **File** → **Make a copy**. + - Save the copy to your Google Drive. + - Note the spreadsheet URL: `https://docs.google.com/spreadsheets/d/spreadsheetid/edit`. + +* You will need a service account key from Google API Service. Follow the instructions from [Airbyte Documentation](https://docs.airbyte.com/integrations/sources/google-sheets#set-up-the-service-account-key) ### Install Terraform -* Apply the respective commands to install Terraform on your Operating System. Find additional options on the [Terraform site](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli). +* Apply the respective commands to install Terraform on your operating system. Find additional options on the [Terraform documentation](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli). -```mdx-code-block First, install the HashiCorp tap, a repository of all [Homebrew](https://brew.sh) packages. @@ -69,11 +74,10 @@ Airbyte, the data integration platform, has a Terraform provider that communicat ``` -``` -### Environment preparation +### Environment Preparation -Prepare the environment by creating a directory for the Terraform configuration and initialize two files: `main.tf` and `variables.tf`. +Prepare the environment by creating a directory for the Terraform configuration and initializing two files: `main.tf` and `variables.tf`. ``` bash mkdir terraform_airbyte @@ -81,146 +85,199 @@ cd terraform_airbyte touch main.tf variables.tf ``` -### Define a data pipeline -Define the data source, destination and connection within the `main.tf` file. Open the newly created `main.tf` file in Visual Studio Code or any preferred code editor. +### Define a Data Pipeline -- If using Visual Studio Code, install [HashiCorp Terraform Extensions](https://marketplace.visualstudio.com/items?itemName=HashiCorp.terraform) to add autocompletion and syntax highlighting. You can also add [Terraform by Anton Kuliko](https://marketplace.visualstudio.com/items?itemName=4ops.terraform) for configuration language support. +Define the data source, destination, and connection within the `main.tf` file. Open the newly created `main.tf` file in Visual Studio Code or any preferred code editor. + +- If using Visual Studio Code, install [HashiCorp Terraform Extensions](https://marketplace.visualstudio.com/items?itemName=HashiCorp.terraform) to add autocompletion and syntax highlighting. ![Terraform Extensions on Visual Studio Code](../elt/images/terraform-airbyte-provider/extensions.png) -Populate the main.tf file with the template provided. +Populate the `main.tf` file with the template provided: ``` bash # Provider Configuration terraform { required_providers { airbyte = { source = "airbytehq/airbyte" - version = "0.4.1" // Latest Version https://registry.terraform.io/providers/airbytehq/airbyte/latest + version = "0.13.0" } } } provider "airbyte" { - // If running on Airbyte Cloud, generate & save the API key from https://portal.airbyte.com - bearer_auth = var.api_key -} -# Google Sheets Source Configuration -resource "airbyte_source_google_sheets" "my_source_gsheets" { - configuration = { - source_type = "google-sheets" - credentials = { - service_account_key_authentication = { - service_account_info = var.google_private_key - } - } - names_conversion = true, - spreadsheet_id = var.spreadsheet_id - } - name = "Google Sheets" - workspace_id = var.workspace_id + // Use client credentials so the provider refreshes access tokens automatically. + client_id = var.airbyte_client_id + client_secret = var.airbyte_client_secret + token_url = var.airbyte_token_url } + # Teradata Vantage Destination Configuration # For optional parameters visit https://registry.terraform.io/providers/airbytehq/airbyte/latest/docs/resources/destination_teradata resource "airbyte_destination_teradata" "my_destination_teradata" { configuration = { - host = var.host - password = var.password - schema = "airbyte_td_two" - ssl = false + host = var.host + schema = "airbyte_td_two" + ssl = false ssl_mode = { allow = {} } - username = var.username + logmech = { + td2 = { + username = var.username + password = var.password + } + } } - name = "Teradata" - workspace_id = var.workspace_id + name = "Teradata" + workspace_id = var.workspace_id } # Connection Configuration resource "airbyte_connection" "googlesheets_teradata" { - name = "Google Sheets - Teradata" - source_id = airbyte_source_google_sheets.my_source_gsheets.source_id + name = "Google Sheets - Teradata" + source_id = airbyte_source_google_sheets.my_source_google_sheets.source_id destination_id = airbyte_destination_teradata.my_destination_teradata.destination_id - schedule = { - schedule_type = "cron" // "manual" - cron_expression = "0 15 * * * ?" # This sets the data sync to run every 15 minutes of the hour + + schedule = { + schedule_type = "cron" + cron_expression = "0 */15 * * * ?" # every 15 minutes + } +} +# Google Sheets Source Configuration +resource "airbyte_source_google_sheets" "my_source_google_sheets" { + configuration = { + spreadsheet_id = var.google_sheets_spreadsheet_id + credentials = { + service_account_key_authentication = { + service_account_info = var.google_service_account_info + } } } + name = "Google Sheets Source" + workspace_id = var.workspace_id +} ``` -Note that this example uses a cron expression to schedule the data transfer to run every 15 minutes past the hour. +Note that this example uses a cron expression to schedule the data transfer to run every 15 minutes. -In our `main.tf` file we reference variables which are held in the `variables.tf` file, including the API key, workspace ID, Google Sheet id, Google private key and Teradata Vantage credentials. Copy the following template into the `variables.tf` file and populate with the appropriate configuration values in the `default` attribute. +In our `main.tf` file, we reference variables that are held in the `variables.tf` file, including the API key, workspace ID, Google Sheets ID, Google private key, and Teradata credentials. We will populate sensitive credentials to a `terraform.tfvars` file that we will not commit to version control. -### Configuring the variables.tf file +### Configuring the variables.tf File ``` bash -#log in to https://portal.airbyte.com generate, save and populate the variable with an API key -variable "api_key" { - type = string - default = "" +# Create these in Airbyte UI: User settings -> Applications. +variable "airbyte_client_id" { + type = string + sensitive = true + description = "Airbyte application client ID" } -#workspace_id is found in the url to the Airbyte Cloud account https://cloud.airbyte.com/workspaces//settings/dbt-cloud -variable "workspace_id" { - type = string - default = "" -} -#Google spreadsheet id and Google private key -variable "spreadsheet_id" { - type = string - default = "" +variable "airbyte_client_secret" { + type = string + sensitive = true + description = "Airbyte application client secret" } -variable "google_private_key" { - type = string - default = "" + +variable "airbyte_token_url" { + type = string + description = "OAuth token endpoint used by the Airbyte provider" + default = "https://api.airbyte.com/v1/applications/token" } + +#workspace_id is found in the URL to the Airbyte Cloud account https://cloud.airbyte.com/workspaces//settings/dbt-cloud +variable "workspace_id" { + type = string +} + # Teradata Vantage connection credentials variable "host" { type = string - default = "" - } +} variable "username" { type = string - default = "demo_user" - } - variable "password" { +} +variable "password" { type = string - default = "" - } + sensitive = true +} + +variable "google_sheets_spreadsheet_id" { + type = string + description = "Google Sheets URL to read from" +} + +variable "google_service_account_info" { + type = string + sensitive = true + description = "Service account JSON key as a single string" +} ``` +### Sample Terraform .tfvars File + +We will need a `terraform.tfvars` file with the following structure: + +```bash +airbyte_client_id = "your-airbyte-client-id-here" +airbyte_client_secret = "your-airbyte-client-secret-here" +workspace_id = "your-workspace-id-here" + +# Teradata Vantage connection credentials +host = "your-teradata-host-here" +username = "your-teradata-username-here" +password = "your-teradata-password-here" + +# Google Sheets configuration +google_sheets_spreadsheet_id = "https://docs.google.com/spreadsheets/d/your-spreadsheet-id-here" +google_service_account_info = < -* Sample data: The sample data [Jaffle Shop Dataset](https://docs.google.com/spreadsheets/d/1-R4F3q8J9KDnFRWpiT3Ysp1RlOoUu3PeQR7xDeLxFts/edit#gid=42273685) can be found in Google Sheets. -* Reference dbt project repository: [Jaffle Project with Airbyte.](https://github.com/Teradata/airbyte-dbt-jaffle) -* Python 3.7, 3.8, 3.9, 3.10 or 3.11 installed. - -## Sample Data Loading -* Follow the steps in the [Airbyte tutorial](./use-airbyte-to-load-data-from-external-sources-to-teradata.md). Make sure you load data from the [Jaffle Shop spreadsheet](https://docs.google.com/spreadsheets/d/1-R4F3q8J9KDnFRWpiT3Ysp1RlOoUu3PeQR7xDeLxFts/edit#gid=42273685) and not the default dataset referenced by the Airbyte tutorial. Also, set the `Default Schema` in the Teradata destination to `airbyte_jaffle_shop`. - -:::note -When you configure a Teradata destination in Airbyte, it will ask for a `Default Schema`. Set the `Default Schema` to `airbyte_jaffle_shop`. -::: - -## Clone the project -Clone the tutorial repository and change the directory to the project directory: - -```bash -git clone https://github.com/Teradata/airbyte-dbt-jaffle -cd airbyte-dbt-jaffle -``` - -## Install dbt -* Create a new python environment to manage dbt and its dependencies. Activate the environment: - - ```bash - python3 -m venv env - source env/bin/activate - ``` - - - :::note - You can activate the virtual environment in Windows by executing the corresponding batch file `./myenv/Scripts/activate`. - ::: - -* Install `dbt-teradata` module and its dependencies. The core dbt module is included as a dependency so you don't have to install it separately: - - ```bash - pip install dbt-teradata - ``` - -## Configure dbt -* Initialize a dbt project. - - ```bash - dbt init - ``` - - - The dbt project wizard will ask you for a project name and database management system to use in the project. In this demo, we define the project name as `dbt_airbyte_demo`. Since we are using the dbt-teradata connector, the only database management system available is Teradata. - - ![Project name prompt](../elt/images/getting-started-with-airbyte-dbt/dbt_init_project_name.png) - - ![Database name prompt](../elt/images/getting-started-with-airbyte-dbt/dbt_init_database_name.png) - -* Configure the `profiles.yml` file located in the `$HOME/.dbt` directory. If the `profiles.yml` file is not present, you can create a new one. -* Adjust `server`, `username`, `password` to match your Teradata instance's `HOST`, `Username`, `Password` respectively. -* In this configuration, `schema` stands for the database that contains the sample data, in our case that is the default schema that we defined in Airbyte `airbyte_jaffle_shop`. - - ``` yaml , id="dbt_first_config", role="emits-gtm-events" - dbt_airbyte_demo: - target: dev - outputs: - dev: - type: teradata - server: - schema: airbyte_jaffle_shop - username: - password: - tmode: ANSI - ``` - -* Once the `profiles.yml` file is ready, we can validate the setup. Go to the dbt project folder and run the command: - - ``` bash - dbt debug - ``` - - If the debug command returned errors, you likely have an issue with the content of `profiles.yml`. If the setup is correct, you will get message `All checks passed!` - - ![dbt debug output](../elt/images/getting-started-with-airbyte-dbt/dbt_debug.png) - -## The Jaffle Shop dbt project - -`jaffle_shop` is a fictional restaurant that takes orders online. The data of this business consists of tables for `customers`, `orders` and `payments`that follow the entity relations diagram below: - -![](../images/dbt1.svg) - -The data in the source system is normalized. A dimensional model based on the same data, more suitable for analytics tools, is presented below: - -![](../images/dbt2.svg) - -### dbt transformations - -:::note -The complete dbt project encompassing the transformations detailed below is located at [Jaffle Project with Airbyte](https://github.com/Teradata/airbyte-dbt-jaffle). -::: - -The reference dbt project performs two types of transformations. - -* First, it transforms the raw data (in JSON format), loaded from Google Sheets via Airbyte, into staging views. At this stage the data is normalized. -* Next, it transforms the normalized views into a dimensional model ready for analytics. - -The following diagram shows the transformation steps in Teradata Vantage using dbt: - -![](../images/dita.svg) - - -As in all dbt projects, the folder `models` contains the data models that the project materializes as tables, or views, according to the corresponding configurations at the project, or individual model level. - -The models can be organized into different folders according to their purpose in the organization of the data warehouse/lake. Common folder layouts include a folder for `staging`, a folder for `core`, and a folder for `marts`. This structure can be simplified without affecting the workings of dbt. - -### Staging models -In the original [dbt Jaffle Shop tutorial](https://github.com/dbt-labs/jaffle_shop-dev) the project's data is loaded from csv files located in the `./data` folder through dbt's `seed` command. The `seed` command is commonly used to load data from tables, however, this command is not designed to perform data loading. - -In this demo we are assuming a more typical setup in which a tool designed for data loading, Airbyte, was used to load data into the datawarehouse/lake. -Data loaded through Airbyte though is represented as raw JSON strings. From these raw data we are creating normalized staging views. We perform this task through the following staging models. - -* The `stg_customers` model creates the normalized staging view for `customers` from the `_airbyte_raw_customers` table. -* The `stg_orders` model creates the normalized view for `orders` from the `_airbyte_raw_orders` table -* The `stg_payments` model creates the normalized view for `payments` from the `_airbyte_raw_payments` table. - -:::note -As the method of extracting JSON strings remains consistent across all staging models, we will provide a detailed explanation for the transformations using just one of these models as an example. -::: - -Below an example of transforming raw JSON data into a view through the `stg_orders.sql` model : -```sql -WITH source AS ( - SELECT * FROM {{ source('airbyte_jaffle_shop', '_airbyte_raw_orders')}} -), - -flattened_json_data AS ( - SELECT - _airbyte_data.JSONExtractValue('$.id') AS order_id, - _airbyte_data.JSONExtractValue('$.user_id') AS customer_id, - _airbyte_data.JSONExtractValue('$.order_date') AS order_date, - _airbyte_data.JSONExtractValue('$.status') AS status - FROM source -) - - -SELECT * FROM flattened_json_data -``` - -* In this model the source is defined as the raw table `_airbyte_raw_orders`. -* This raw table columns contains both metadata, and the actual ingested data. The data column is called `_airbyte_data`. -* This column is of Teradata JSON type. This type supports the method JSONExtractValue for retrieving scalar values from the JSON object. -* In this model we are retrieving each of the attributes of interest and adding meaningful aliases in order to materialize a view. - -### Dimensional models (marts) -Building a Dimensional Model is a two-step process: - -* First, we take the normalized views in `stg_orders`, `stg_customers`, `stg_payments` and build denormalized intermediate join tables `customer_orders`, `order_payments`, `customer_payments`. You will find the definitions of these tables in `./models/marts/core/intermediate`. -* In the second step, we create the `dim_customers` and `fct_orders` models. These constitute the dimensional model tables that we want to expose to our BI tool. You will find the definitions of these tables in `./models/marts/core`. - -### Executing transformations -For executing the transformations defined in the dbt project we run: - -```bash -dbt run -``` - -You will get the status of each model as given below: - -![dbt run output](../elt/images/getting-started-with-airbyte-dbt/dbt_run.png) - -### Test data -To ensure that the data in the dimensional model is correct, dbt allows us to define and execute tests against the data. - -The tests are defined in `./models/marts/core/schema.yml` and `./models/staging/schema.yml`. Each column can have multiple tests configured under the `tests` key. - -* For example, we expect that `fct_orders.order_id` column will contain unique, non-null values. - -To validate that the data in the produced tables satisfies the test conditions run: - -``` bash -dbt test -``` - -If the data in the models satisfies all the test cases, the result of this command will be as below: - -![dbt test output](../elt/images/getting-started-with-airbyte-dbt/dbt_test.png) - -### Generate documentation - -Our model consists of just a few tables. In a scenario with more sources of data, and a more complex dimensional model, documenting the data lineage and what is the purpose of each of the intermediate models is very important. - -Generating this type of documentation with dbt is very straight forward. - -``` bash -dbt docs generate -``` - -This will produce html files in the `./target` directory. - -You can start your own server to browse the documentation. The following command will start a server and open up a browser tab with the docs' landing page: - -``` bash -dbt docs serve -``` - -#### Lineage graph - -![dbt lineage graph](../elt/images/getting-started-with-airbyte-dbt/dbt_docs_serve.png) - -## Summary - -This tutorial demonstrated how to use dbt to transform raw JSON data loaded through Airbyte into dimensional model in Teradata Vantage. The sample project takes raw JSON data loaded in Teradata Vantage, creates normalized views and finally produces a dimensional data mart. We used dbt to transform JSON into Normalized views and multiple dbt commands to create models (`dbt run`), test the data (`dbt test`), and generate and serve model documentation (`dbt docs generate`, `dbt docs serve`). - - -## Further reading -* [dbt documentation](https://docs.getdbt.com/docs) -* [dbt-teradata plugin documentation](https://github.com/Teradata/dbt-teradata) - -import CommunityLinkPartial from '../_partials/community_link.mdx'; - -