├── .gitignore ├── .python-version ├── README.md ├── app.py ├── code_crawler.py ├── context-sources ├── cheat-sheet.md ├── discourse.csv └── docs │ ├── cheat-sheet.md │ ├── community │ ├── contribute.md │ ├── contributing │ │ ├── contributing-coding.md │ │ ├── contributing-online-community.md │ │ ├── contributing-realtime-events.md │ │ └── contributing-writing.md │ ├── join.md │ └── resources │ │ ├── code-of-conduct.md │ │ ├── contributor-license-agreements.md │ │ ├── forum-guidelines.md │ │ ├── maintaining-a-channel.md │ │ ├── organizing-inclusive-events.md │ │ ├── oss-expectations.md │ │ ├── oss-projects.md │ │ ├── slack-rules-of-the-road.md │ │ ├── speaking-at-a-meetup.md │ │ ├── vendor-guidelines.md │ │ └── viewpoint.md │ ├── dbt-cli │ └── cli-overview.md │ ├── docs │ ├── about-setup.md │ ├── about │ │ └── overview.md │ ├── build │ │ ├── analyses.md │ │ ├── custom-aliases.md │ │ ├── custom-databases.md │ │ ├── custom-schemas.md │ │ ├── custom-target-names.md │ │ ├── environment-variables.md │ │ ├── exposures.md │ │ ├── hooks-operations.md │ │ ├── incremental-models.md │ │ ├── jinja-macros.md │ │ ├── materializations.md │ │ ├── metrics.md │ │ ├── models.md │ │ ├── packages.md │ │ ├── project-variables.md │ │ ├── projects.md │ │ ├── python-models.md │ │ ├── seeds.md │ │ ├── snapshots.md │ │ ├── sources.md │ │ ├── sql-models.md │ │ └── tests.md │ ├── building-a-dbt-project │ │ └── dont-nest-your-curlies.md │ ├── cloud │ │ ├── about-cloud-setup.md │ │ ├── about-cloud │ │ │ ├── about-cloud-ide.md │ │ │ ├── architecture.md │ │ │ ├── assets │ │ │ │ └── dbt-cloud-advanced-architecture-guide.pdf │ │ │ ├── dbt-cloud-features.md │ │ │ ├── regions-ip-addresses.md │ │ │ └── tenancy.md │ │ ├── connect-data-platform │ │ │ ├── about-connections.md │ │ │ ├── connect-apache-spark.md │ │ │ ├── connect-databricks.md │ │ │ ├── connect-redshift-postgresql-alloydb.md │ │ │ ├── connect-snowflake.md │ │ │ ├── connect-your-database.md │ │ │ └── connnect-bigquery.md │ │ ├── dbt-cloud-tips.md │ │ ├── develop-in-the-cloud.md │ │ ├── git │ │ │ ├── authenticate-azure.md │ │ │ ├── connect-azure-devops.md │ │ │ ├── connect-github.md │ │ │ ├── connect-gitlab.md │ │ │ ├── import-a-project-by-git-url.md │ │ │ └── setup-azure.md │ │ ├── manage-access │ │ │ ├── about-access.md │ │ │ ├── audit-log.md │ │ │ ├── cloud-seats-and-users.md │ │ │ ├── enterprise-permissions.md │ │ │ ├── licenses-and-groups.md │ │ │ ├── self-service-permissions.md │ │ │ ├── set-up-bigquery-oauth.md │ │ │ ├── set-up-snowflake-oauth.md │ │ │ ├── set-up-sso-azure-active-directory.md │ │ │ ├── set-up-sso-google-workspace.md │ │ │ ├── set-up-sso-okta.md │ │ │ ├── set-up-sso-saml-2.0.md │ │ │ └── sso-overview.md │ │ └── privatelink │ │ │ ├── about-privatelink.md │ │ │ ├── databricks-privatelink.md │ │ │ ├── redshift-privatelink.md │ │ │ └── snowflake-privatelink.md │ ├── collaborate │ │ ├── cloud-build-and-view-your-docs.md │ │ ├── documentation.md │ │ ├── environments.md │ │ ├── environments │ │ │ ├── dbt-cloud-environments.md │ │ │ ├── dbt-core-environments.md │ │ │ └── environments-in-dbt.md │ │ ├── git-version-control.md │ │ ├── git │ │ │ ├── managed-repository.md │ │ │ ├── merge-conflicts.md │ │ │ ├── pr-template.md │ │ │ └── version-control-basics.md │ │ └── publish │ │ │ ├── about-publishing-models.md │ │ │ ├── model-access.md │ │ │ ├── model-contracts.md │ │ │ └── model-versions.md │ ├── core │ │ ├── about-the-cli.md │ │ ├── connection-profiles.md │ │ ├── docker-install.md │ │ ├── homebrew-install.md │ │ ├── installation-overview.md │ │ ├── pip-install.md │ │ └── source-install.md │ ├── dbt-cloud-apis │ │ ├── access-metadata-api.md │ │ ├── admin-cloud-api.md │ │ ├── apis-overview.md │ │ ├── metadata-api.md │ │ ├── metadata-querying.md │ │ ├── metadata-use-case-guides.md │ │ ├── migrating-to-v2.md │ │ ├── schema-metadata-exposure.mdx │ │ ├── schema-metadata-exposures.mdx │ │ ├── schema-metadata-metric.mdx │ │ ├── schema-metadata-metrics.mdx │ │ ├── schema-metadata-model.mdx │ │ ├── schema-metadata-modelByEnv.mdx │ │ ├── schema-metadata-models.mdx │ │ ├── schema-metadata-seed.mdx │ │ ├── schema-metadata-seeds.mdx │ │ ├── schema-metadata-snapshots.mdx │ │ ├── schema-metadata-source.mdx │ │ ├── schema-metadata-sources.mdx │ │ ├── schema-metadata-test.mdx │ │ ├── schema-metadata-tests.mdx │ │ ├── schema.jsx │ │ ├── service-tokens.md │ │ └── user-tokens.md │ ├── dbt-support.md │ ├── dbt-versions │ │ ├── core-versions.md │ │ ├── product-lifecycles.md │ │ ├── release-notes.md │ │ ├── release-notes │ │ │ ├── 10-Mar-2023 │ │ │ │ ├── 1.0-deprecation.md │ │ │ │ └── apiv2-limit.md │ │ │ ├── 11-Feb-2023 │ │ │ │ ├── feb-ide-updates.md │ │ │ │ └── no-partial-parse-config.md │ │ │ ├── 12-Jan-2023 │ │ │ │ └── ide-updates.md │ │ │ ├── 23-Dec-2022 │ │ │ │ ├── default-thread-value.md │ │ │ │ ├── new-jobs-default-as-off.md │ │ │ │ └── private-packages-clone-git-token.md │ │ │ ├── 24-Nov-2022 │ │ │ │ ├── dbt-databricks-unity-catalog-support.md │ │ │ │ └── ide-features-ide-deprecation.md │ │ │ ├── 25-Oct-2022 │ │ │ │ ├── cloud-integration-azure.md │ │ │ │ └── new-ide-launch.md │ │ │ ├── 26-Sept-2022 │ │ │ │ ├── liststeps-endpoint-deprecation.md │ │ │ │ └── metadata-api-data-retention-limits.md │ │ │ ├── 27-Aug-2022 │ │ │ │ ├── ide-improvement-beta.md │ │ │ │ └── support-redshift-ra3.md │ │ │ ├── 28-July-2022 │ │ │ │ └── render-lineage-feature.md │ │ │ ├── 29-May-2022 │ │ │ │ └── gitlab-auth.md │ │ │ ├── 30-April-2022 │ │ │ │ ├── audit-log.md │ │ │ │ ├── credentials-saved.md │ │ │ │ ├── email-verification.md │ │ │ │ └── scheduler-improvements.md │ │ │ ├── 31-March-2022 │ │ │ │ ├── ide-timeout-message.md │ │ │ │ └── prep-and-waiting-time.md │ │ │ ├── 32-February-2022 │ │ │ │ ├── DAG-updates-more.md │ │ │ │ └── service-tokens-more.md │ │ │ ├── 33-January-2022 │ │ │ │ ├── IDE-autocomplete-more.md │ │ │ │ └── model-timing-more.md │ │ │ ├── 34-dbt-cloud-changelog-2021.md │ │ │ └── 35-dbt-cloud-changelog-2019-2020.md │ │ └── upgrade-core-in-cloud.md │ ├── deploy │ │ ├── about-state.md │ │ ├── airgapped.md │ │ ├── artifacts.md │ │ ├── cloud-ci-job.md │ │ ├── dashboard-status-tiles.md │ │ ├── dbt-cloud-job.md │ │ ├── deployment-overview.md │ │ ├── deployment-tools.md │ │ ├── job-commands.md │ │ ├── job-notifications.md │ │ ├── job-triggers.md │ │ ├── source-freshness.md │ │ └── webhooks.md │ ├── introduction.md │ ├── quickstarts │ │ ├── dbt-cloud │ │ │ ├── bigquery-qs.md │ │ │ ├── databricks-qs.md │ │ │ ├── redshift-qs.md │ │ │ └── snowflake-qs.md │ │ ├── dbt-core │ │ │ └── dbt-core-qs.md │ │ └── overview.md │ ├── running-a-dbt-project │ │ ├── run-your-dbt-projects.md │ │ └── using-the-dbt-ide.md │ ├── supported-data-platforms.md │ └── use-dbt-semantic-layer │ │ ├── avail-sl-integrations.md │ │ ├── dbt-semantic-layer.md │ │ ├── quickstart-semantic-layer.md │ │ └── set-dbt-semantic-layer.md │ ├── faqs │ ├── API │ │ └── rotate-token.md │ ├── Accounts │ │ ├── change-billing.md │ │ ├── change-users-license.md │ │ ├── cloud-upgrade-instructions.md │ │ ├── configurable-snapshot-path.md │ │ ├── dbt-specific-jinja.md │ │ ├── delete-users.md │ │ ├── find-user-id.md │ │ ├── git-account-in-use.md │ │ ├── payment-accepted.md │ │ ├── slack.md │ │ └── transfer-account.md │ ├── Core │ │ ├── install-pip-best-practices.md │ │ ├── install-pip-os-prereqs.md │ │ └── install-python-compatibility.md │ ├── Docs │ │ ├── document-all-columns.md │ │ ├── document-other-resources.md │ │ ├── documenting-macros.md │ │ ├── long-descriptions.md │ │ ├── modify-owner-column.md │ │ └── sharing-documentation.md │ ├── Environments │ │ ├── beta-release.md │ │ ├── custom-branch-settings.md │ │ ├── delete-environment-job.md │ │ ├── diff-database-environment.md │ │ ├── profile-env-vars.md │ │ ├── profile-name.md │ │ └── target-names.md │ ├── Git │ │ ├── gitignore.md │ │ ├── gitlab-authentication.md │ │ ├── gitlab-selfhosted.md │ │ ├── google-cloud-repo.md │ │ ├── managed-repo.md │ │ └── run-on-pull.md │ ├── Jinja │ │ ├── jinja-whitespace.md │ │ ├── quoting-column-names.md │ │ └── which-jinja-docs.md │ ├── Models │ │ ├── available-configurations.md │ │ ├── available-materializations.md │ │ ├── configurable-model-path.md │ │ ├── create-a-schema.md │ │ ├── create-dependencies.md │ │ ├── insert-records.md │ │ ├── model-custom-schemas.md │ │ ├── reference-models-in-another-project.md │ │ ├── removing-deleted-models.md │ │ ├── run-downtime.md │ │ ├── source-quotes.md │ │ ├── specifying-column-types.md │ │ ├── sql-dialect.md │ │ └── unique-model-names.md │ ├── Project │ │ ├── add-a-seed.md │ │ ├── consolidate-projects.md │ │ ├── dbt-source-freshness.md │ │ ├── debugging-jinja.md │ │ ├── define-a-column-type.md │ │ ├── delete-a-project.md │ │ ├── docs-for-multiple-projects.md │ │ ├── example-projects.md │ │ ├── exclude-table-from-freshness.md │ │ ├── extract-load-support.md │ │ ├── multiple-resource-yml-files.md │ │ ├── properties-not-in-config.md │ │ ├── resource-yml-name.md │ │ ├── schema-yml-name.md │ │ ├── separate-profile.md │ │ ├── source-has-bad-name.md │ │ ├── source-in-different-database.md │ │ ├── structure-a-project.md │ │ ├── which-materialization.md │ │ ├── which-schema.md │ │ ├── why-not-write-dml.md │ │ ├── why-so-many-macros.md │ │ ├── why-version-2.md │ │ └── yaml-file-extension.md │ ├── Runs │ │ ├── checking-logs.md │ │ ├── failed-prod-run.md │ │ ├── failed-tests.md │ │ ├── run-downstream-of-seed.md │ │ ├── run-one-model.md │ │ ├── run-one-snapshot.md │ │ ├── running-models-downstream-of-source.md │ │ └── snapshot-frequency.md │ ├── Seeds │ │ ├── build-one-seed.md │ │ ├── full-refresh-seed.md │ │ ├── leading-zeros-in-seed.md │ │ ├── load-raw-data-with-seed.md │ │ ├── seed-custom-schemas.md │ │ ├── seed-datatypes.md │ │ └── seed-hooks.md │ ├── Snapshots │ │ ├── snapshot-hooks.md │ │ ├── snapshot-schema-changes.md │ │ ├── snapshot-target-schema.md │ │ └── snapshotting-freshness-for-one-source.md │ ├── Tests │ │ ├── available-tests.md │ │ ├── configurable-data-path.md │ │ ├── configurable-data-test-path.md │ │ ├── custom-test-thresholds.md │ │ ├── multiple-test-files.m │ │ ├── recommended-tests.md │ │ ├── test-one-model.md │ │ ├── testing-seeds.md │ │ ├── testing-sources.md │ │ ├── uniqueness-two-columns.md │ │ └── when-to-test.md │ ├── Troubleshooting │ │ ├── access-gdrive-credential.md │ │ ├── access_token_error.md │ │ ├── dispatch-could-not-find-package.md │ │ ├── git-revlist-error.md │ │ ├── gitignore.md │ │ ├── gitlab-authentication.md │ │ ├── nonetype-ide-error.md │ │ ├── partial-parsing-error.md │ │ ├── runtime-error-could-not-find-profile.md │ │ ├── runtime-packages.yml.md │ │ ├── sql-errors.md │ │ └── unused-model-configurations.md │ └── Warehouse │ │ ├── bq-impersonate-service-account-setup.md │ │ ├── bq-impersonate-service-account-why.md │ │ ├── connecting-to-two-dbs-not-allowed.md │ │ ├── database-privileges.md │ │ ├── loading-data.md │ │ └── sample-profiles.md │ ├── guides │ ├── advanced │ │ ├── creating-new-materializations.md │ │ └── using-jinja.md │ ├── best-practices │ │ ├── custom-generic-tests.md │ │ ├── debugging-errors.md │ │ ├── environment-setup │ │ │ ├── 1-env-guide-overview.md │ │ │ ├── 2-one-deployment-environment.md │ │ │ └── 3-many-deployment-environments.md │ │ ├── how-we-structure │ │ │ ├── 1-guide-overview.md │ │ │ ├── 2-staging.md │ │ │ ├── 3-intermediate.md │ │ │ ├── 4-marts.md │ │ │ └── 5-the-rest-of-the-project.md │ │ └── materializations │ │ │ ├── materializations-guide-1-guide-overview.md │ │ │ ├── materializations-guide-2-available-materializations.md │ │ │ ├── materializations-guide-3-configuring-materializations.md │ │ │ ├── materializations-guide-4-incremental-models.md │ │ │ ├── materializations-guide-5-best-practices.md │ │ │ ├── materializations-guide-6-examining-builds.md │ │ │ └── materializations-guide-7-conclusion.md │ ├── dbt-ecosystem │ │ ├── adapter-development │ │ │ ├── 1-what-are-adapters.md │ │ │ ├── 2-prerequisites-for-a-new-adapter.md │ │ │ ├── 3-building-a-new-adapter.md │ │ │ ├── 4-testing-a-new-adapter.md │ │ │ ├── 5-documenting-a-new-adapter.md │ │ │ ├── 6-promoting-a-new-adapter.md │ │ │ ├── 7-verifying-a-new-adapter.md │ │ │ └── adapter-development │ │ ├── databricks-guides │ │ │ ├── dbt-unity-catalog-best-practices.md │ │ │ └── how-to-set-up-your-databricks-dbt-project.md │ │ ├── dbt-python-snowpark │ │ │ ├── 1-overview-dbt-python-snowpark.md │ │ │ ├── 10-python-transformations.md │ │ │ ├── 11-machine-learning-prep.md │ │ │ ├── 12-machine-learning-training-testing.md │ │ │ ├── 13-testing.md │ │ │ ├── 14-documentation.md │ │ │ ├── 15-deployment.md │ │ │ ├── 2-snowflake-configuration.md │ │ │ ├── 3-connect-to-data-source.md │ │ │ ├── 4-configure-dbt.md │ │ │ ├── 5-development-schema-name.md │ │ │ ├── 6-foundational-structure.md │ │ │ ├── 7-folder-structure.md │ │ │ ├── 8-sources-and-staging.md │ │ │ └── 9-sql-transformations.md │ │ └── sl-partner-integration-guide.md │ ├── legacy │ │ ├── best-practices.md │ │ ├── building-packages.md │ │ ├── creating-date-partitioned-tables.md │ │ ├── debugging-schema-names.md │ │ ├── getting-help.md │ │ └── videos.md │ ├── migration │ │ ├── tools │ │ │ ├── migrating-from-spark-to-databricks.md │ │ │ ├── migrating-from-stored-procedures │ │ │ │ ├── 1-migrating-from-stored-procedures.md │ │ │ │ ├── 2-mapping-inserts.md │ │ │ │ ├── 3-mapping-updates.md │ │ │ │ ├── 4-mapping-deletes.md │ │ │ │ ├── 5-mapping-merges.md │ │ │ │ └── 6-migrating-from-stored-procedures-conclusion.md │ │ │ └── refactoring-legacy-sql.md │ │ └── versions │ │ │ ├── 02-upgrading-to-v1.5.md │ │ │ ├── 03-upgrading-to-dbt-utils-v1.0.md │ │ │ ├── 04-upgrading-to-v1.4.md │ │ │ ├── 05-upgrading-to-v1.3.md │ │ │ ├── 06-upgrading-to-v1.2.md │ │ │ ├── 07-upgrading-to-v1.1.md │ │ │ ├── 08-upgrading-to-v1.0.md │ │ │ ├── 09-upgrading-to-v0.21.md │ │ │ ├── 10-upgrading-to-v0.20.md │ │ │ └── 11-Older versions │ │ │ ├── upgrading-to-0-11-0.md │ │ │ ├── upgrading-to-0-12-0.md │ │ │ ├── upgrading-to-0-13-0.md │ │ │ ├── upgrading-to-0-14-0.md │ │ │ ├── upgrading-to-0-14-1.md │ │ │ ├── upgrading-to-0-15-0.md │ │ │ ├── upgrading-to-0-16-0.md │ │ │ ├── upgrading-to-0-17-0.md │ │ │ ├── upgrading-to-0-18-0.md │ │ │ └── upgrading-to-0-19-0.md │ └── orchestration │ │ ├── airflow-and-dbt-cloud │ │ ├── 1-airflow-and-dbt-cloud.md │ │ ├── 2-setting-up-airflow-and-dbt-cloud.md │ │ ├── 3-running-airflow-and-dbt-cloud.md │ │ └── 4-airflow-and-dbt-cloud-faqs.md │ │ ├── custom-cicd-pipelines │ │ ├── 1-cicd-background.md │ │ ├── 2-lint-on-push.md │ │ ├── 3-dbt-cloud-job-on-merge.md │ │ ├── 4-dbt-cloud-job-on-pr.md │ │ └── 5-something-to-consider.md │ │ └── webhooks │ │ ├── serverless-datadog.md │ │ ├── serverless-pagerduty.md │ │ ├── zapier-ms-teams.md │ │ ├── zapier-new-cloud-job.md │ │ ├── zapier-refresh-mode-report.md │ │ └── zapier-slack.md │ ├── reference │ ├── advanced-config-usage.md │ ├── analysis-properties.md │ ├── artifacts │ │ ├── catalog-json.md │ │ ├── dbt-artifacts.md │ │ ├── manifest-json.md │ │ ├── other-artifacts.md │ │ ├── run-results-json.md │ │ └── sources-json.md │ ├── commands │ │ ├── build.md │ │ ├── clean.md │ │ ├── cmd-docs.md │ │ ├── compile.md │ │ ├── debug.md │ │ ├── deps.md │ │ ├── init.md │ │ ├── list.md │ │ ├── parse.md │ │ ├── rpc.md │ │ ├── run-operation.md │ │ ├── run.md │ │ ├── seed.md │ │ ├── snapshot.md │ │ ├── source.md │ │ └── test.md │ ├── config-datatypes.yml │ ├── configs-and-properties.md │ ├── dbt-classes.md │ ├── dbt-commands.md │ ├── dbt-jinja-functions │ │ ├── adapter.md │ │ ├── as_bool.md │ │ ├── as_native.md │ │ ├── as_number.md │ │ ├── as_text.md │ │ ├── builtins.md │ │ ├── config.md │ │ ├── cross-database-macros.md │ │ ├── dbt-project-yml-context.md │ │ ├── dbt_version.md │ │ ├── debug-method.md │ │ ├── dispatch.md │ │ ├── doc.md │ │ ├── env_var.md │ │ ├── exceptions.md │ │ ├── execute.md │ │ ├── flags.md │ │ ├── fromjson.md │ │ ├── fromyaml.md │ │ ├── graph.md │ │ ├── invocation_id.md │ │ ├── local-md5.md │ │ ├── log.md │ │ ├── model.md │ │ ├── modules.md │ │ ├── on-run-end-context.md │ │ ├── print.md │ │ ├── profiles-yml-context.md │ │ ├── project_name.md │ │ ├── ref.md │ │ ├── return.md │ │ ├── run_query.md │ │ ├── run_started_at.md │ │ ├── schema.md │ │ ├── schemas.md │ │ ├── selected_resources.md │ │ ├── set.md │ │ ├── source.md │ │ ├── statement-blocks.md │ │ ├── target.md │ │ ├── this.md │ │ ├── tojson.md │ │ ├── toyaml.md │ │ ├── var.md │ │ └── zip.md │ ├── dbt_project.yml.md │ ├── dbtignore.md │ ├── events-logging.md │ ├── exit-codes.md │ ├── exposure-properties.md │ ├── global-cli-flags.md │ ├── global-configs.md │ ├── macro-properties.md │ ├── model-configs.md │ ├── model-properties.md │ ├── node-selection │ │ ├── defer.md │ │ ├── exclude.md │ │ ├── graph-operators.md │ │ ├── methods.md │ │ ├── putting-it-together.md │ │ ├── set-operators.md │ │ ├── state-comparison-caveats.md │ │ ├── syntax.md │ │ ├── test-selection-examples.md │ │ └── yaml-selectors.md │ ├── parsing.md │ ├── profiles.yml.md │ ├── project-configs │ │ ├── analysis-paths.md │ │ ├── asset-paths.md │ │ ├── clean-targets.md │ │ ├── config-version.md │ │ ├── dispatch-config.md │ │ ├── docs-paths.md │ │ ├── log-path.md │ │ ├── macro-paths.md │ │ ├── model-paths.md │ │ ├── name.md │ │ ├── on-run-start-on-run-end.md │ │ ├── packages-install-path.md │ │ ├── profile.md │ │ ├── query-comment.md │ │ ├── quoting.md │ │ ├── require-dbt-version.md │ │ ├── seed-paths.md │ │ ├── snapshot-paths.md │ │ ├── target-path.md │ │ ├── test-paths.md │ │ └── version.md │ ├── resource-configs │ │ ├── alias.md │ │ ├── azuresynapse-configs.md │ │ ├── bigquery-configs.md │ │ ├── check_cols.md │ │ ├── clickhouse-configs.md │ │ ├── column_types.md │ │ ├── contract.md │ │ ├── database.md │ │ ├── docs.md │ │ ├── doris-configs.md │ │ ├── enabled.md │ │ ├── fail_calc.md │ │ ├── fal-configs.md │ │ ├── firebolt-configs.md │ │ ├── full_refresh.md │ │ ├── glue-configs.md │ │ ├── grants.md │ │ ├── greenplum-configs.md │ │ ├── hive-configs.md │ │ ├── impala-configs.md │ │ ├── invalidate_hard_deletes.md │ │ ├── limit.md │ │ ├── materialize-configs.md │ │ ├── materialized.md │ │ ├── meta.md │ │ ├── mindsdb-configs.md │ │ ├── mssql-configs.md │ │ ├── no-configs.md │ │ ├── persist_docs.md │ │ ├── plus-prefix.md │ │ ├── postgres-configs.md │ │ ├── pre-hook-post-hook.md │ │ ├── quote_columns.md │ │ ├── redshift-configs.md │ │ ├── resource-path.md │ │ ├── schema.md │ │ ├── severity.md │ │ ├── singlestore-configs.md │ │ ├── snapshot_name.md │ │ ├── snowflake-configs.md │ │ ├── spark-configs.md │ │ ├── sql_header.md │ │ ├── store_failures.md │ │ ├── strategy.md │ │ ├── tags.md │ │ ├── target_database.md │ │ ├── target_schema.md │ │ ├── teradata-configs.md │ │ ├── unique_key.md │ │ ├── updated_at.md │ │ ├── vertica-configs.md │ │ └── where.md │ ├── resource-properties │ │ ├── argument-type.md │ │ ├── columns.md │ │ ├── config.md │ │ ├── constraints.md │ │ ├── database.md │ │ ├── description.md │ │ ├── external.md │ │ ├── freshness.md │ │ ├── identifier.md │ │ ├── loader.md │ │ ├── model_name.md │ │ ├── name.md │ │ ├── overrides.md │ │ ├── quote.md │ │ ├── quoting.md │ │ ├── schema.md │ │ └── tests.md │ ├── seed-configs.md │ ├── seed-properties.md │ ├── snapshot-configs.md │ ├── snapshot-properties.md │ ├── snowflake-permissions.md │ ├── source-configs.md │ ├── source-properties.md │ ├── test-configs.md │ └── warehouse-setups │ │ ├── alloydb-setup.md │ │ ├── athena-setup.md │ │ ├── azuresynapse-setup.md │ │ ├── bigquery-setup.md │ │ ├── clickhouse-setup.md │ │ ├── databend-setup.md │ │ ├── databricks-setup.md │ │ ├── decodable-setup.md │ │ ├── doris-setup.md │ │ ├── dremio-setup.md │ │ ├── duckdb-setup.md │ │ ├── exasol-setup.md │ │ ├── fal-setup.md │ │ ├── firebolt-setup.md │ │ ├── glue-setup.md │ │ ├── greenplum-setup.md │ │ ├── hive-setup.md │ │ ├── ibmdb2-setup.md │ │ ├── impala-setup.md │ │ ├── infer-setup.md │ │ ├── iomete-setup.md │ │ ├── layer-setup.md │ │ ├── materialize-setup.md │ │ ├── mindsdb-setup.md │ │ ├── mssql-setup.md │ │ ├── mysql-setup.md │ │ ├── oracle-setup.md │ │ ├── postgres-setup.md │ │ ├── redshift-setup.md │ │ ├── rockset-setup.md │ │ ├── singlestore-setup.md │ │ ├── snowflake-setup.md │ │ ├── spark-setup.md │ │ ├── sqlite-setup.md │ │ ├── teradata-setup.md │ │ ├── tidb-setup.md │ │ ├── trino-setup.md │ │ └── vertica-setup.md │ ├── sql-reference │ ├── aggregate-functions │ │ ├── sql-array-agg.md │ │ ├── sql-avg.md │ │ ├── sql-count.md │ │ ├── sql-max.md │ │ ├── sql-min.md │ │ ├── sql-round.md │ │ └── sql-sum.md │ ├── clauses │ │ ├── sql-having.md │ │ ├── sql-limit.md │ │ ├── sql-order-by.md │ │ └── sql-where.md │ ├── data-type │ │ ├── sql-data-types.md │ │ └── sql-strings.md │ ├── date-functions │ │ ├── sql-date-trunc.md │ │ ├── sql-dateadd.md │ │ ├── sql-datediff.md │ │ └── sql-datepart.md │ ├── joins │ │ ├── sql-cross-join.md │ │ ├── sql-full-outer-join.md │ │ ├── sql-inner-join.md │ │ ├── sql-left-join.md │ │ ├── sql-right-join.md │ │ └── sql-self-join.md │ ├── operators │ │ ├── sql-and.md │ │ ├── sql-any-all.md │ │ ├── sql-between.md │ │ ├── sql-ilike.md │ │ ├── sql-in.md │ │ ├── sql-like.md │ │ ├── sql-not.md │ │ └── sql-or.md │ ├── other │ │ ├── sql-cast.md │ │ └── sql-comments.md │ ├── statements │ │ ├── sql-case-statement.md │ │ ├── sql-distinct.md │ │ ├── sql-from.md │ │ ├── sql-group-by.md │ │ └── sql-select.md │ ├── string-functions │ │ ├── sql-concat.md │ │ ├── sql-lower.md │ │ ├── sql-trim.md │ │ └── sql-upper.md │ └── window-functions │ │ ├── sql-rank.md │ │ └── sql-row-number.md │ └── terms │ ├── cte.md │ ├── dag.md │ ├── data-catalog.md │ ├── data-extraction.md │ ├── data-lake.md │ ├── data-lineage.md │ ├── data-warehouse.md │ ├── data-wrangling.md │ ├── dataframe.md │ ├── ddl.md │ ├── dimensional-modeling.md │ ├── dml.md │ ├── dry.md │ ├── edw.md │ ├── elt.md │ ├── etl.md │ ├── grain.md │ ├── idempotent.md │ ├── json.md │ ├── materialization.md │ ├── monotonically-increasing.md │ ├── primary-key.md │ ├── relational-database.md │ ├── reverse-etl.md │ ├── subquery.md │ ├── surrogate-key.md │ ├── table.md │ └── view.md ├── csv_crawler.py ├── document_crawler.py ├── markdown_crawler.py ├── question_answerer.py ├── requirements.txt ├── test-docs ├── discourse.csv └── guides │ └── best-practices │ ├── custom-generic-tests.md │ ├── debugging-errors.md │ ├── how-we-structure │ ├── 1-guide-overview.md │ ├── 2-staging.md │ ├── 3-intermediate.md │ ├── 4-marts.md │ └── 5-the-rest-of-the-project.md │ └── materializations │ ├── materializations-guide-1-guide-overview.md │ ├── materializations-guide-2-available-materializations.md │ ├── materializations-guide-3-configuring-materializations.md │ ├── materializations-guide-4-incremental-models.md │ ├── materializations-guide-5-best-practices.md │ ├── materializations-guide-6-examining-builds.md │ └── materializations-guide-7-conclusion.md ├── test_code_crawler.py ├── test_csv_crawler.py ├── test_markdown_crawler.py ├── test_question_answerer.py ├── test_vector_store.py ├── vector_store.py └── vercel.json /.gitignore: -------------------------------------------------------------------------------- 1 | .venv 2 | .env 3 | __pycache__ 4 | .pytest_cache 5 | .vscode 6 | qa-db 7 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.10.10 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ***Archival Notice*** 2 | This repository has been archived. 3 | 4 | As a result all of its historical issues and PRs have been closed. 5 | 6 | Please *do not clone* this repo without understanding the risk in doing so: 7 | - It may have unaddressed security vulnerabilities 8 | - It may have unaddressed bugs 9 | 10 |
11 | Click for historical readme 12 | 13 | # dbot 14 | An LLM-powered chatbot with the added context of the dbt knowledge base. 15 | 16 | ## An experiment 17 | This is currently a basic CLI prototype that has been supplied with the knowledge of the dbt Developer Hub (docs, guides, etc) and uses this to add context to questions asked by users. 18 | 19 | ## How does it work? 20 | It stores embeddings of the material on the Developer Hub (chunked by markdown heading sections) in a vector database, and finds documents similar to the question that is asked. It then prepends as much of that context onto the prompt as it can fit in ChatGPT's context window. 21 | 22 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from fastapi import FastAPI, Request 4 | from slack_bolt import App 5 | from slack_bolt.adapter.fastapi import SlackRequestHandler 6 | 7 | from question_answerer import QuestionAnswerer 8 | 9 | qa = QuestionAnswerer() 10 | api = FastAPI() 11 | app = App( 12 | token=os.environ.get("SLACK_BOT_TOKEN"), 13 | signing_secret=os.environ.get("SLACK_SIGNING_SECRET"), 14 | ) 15 | app_handler = SlackRequestHandler(app) 16 | 17 | 18 | @app.event("app_mention") 19 | def handle_app_mentions(body, say, logger): 20 | logger.info(body) 21 | bot_id = app.client.auth_test()["user_id"] 22 | text = body["event"]["text"].replace(f"<@{bot_id}>", "").strip() 23 | answer = qa.answer_question(text) 24 | say(text=answer, unfurl_links=False, unfurl_media=False) 25 | 26 | 27 | @app.event("message") 28 | def handle_message(): 29 | pass 30 | 31 | 32 | @api.get("/") 33 | def hello(): 34 | return "Welcome, I'm dbt's question answering bot." 35 | 36 | 37 | @api.get("/answer") 38 | def get_answer(question: str = "What is dbt?"): 39 | return {"answer": qa.answer_question(question)} 40 | 41 | 42 | @api.post("/slack/events") 43 | async def endpoint(req: Request): 44 | return await app_handler.handle(req) 45 | -------------------------------------------------------------------------------- /context-sources/docs/docs/build/custom-target-names.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Custom target names" 3 | id: "custom-target-names" 4 | description: "You can define a custom target name for any dbt Cloud job to correspond to settings in your dbt project." 5 | 6 | --- 7 | 8 | ## dbt Cloud Scheduler 9 | 10 | You can define a custom target name for any dbt Cloud job to correspond to settings in your dbt project. This is helpful if you have logic in your dbt project that behaves differently depending on the specified target, for example: 11 | 12 | ```sql 13 | select * 14 | from a_big_table 15 | 16 | -- limit the amount of data queried in dev 17 | {% if target.name != 'prod' %} 18 | where created_at > date_trunc('month', current_date) 19 | {% endif %} 20 | ``` 21 | 22 | To set a custom target name for a job in dbt Cloud, configure the **Target Name** field for your job in the Job Settings page. 23 | 24 | 25 | 26 | ## dbt Cloud IDE 27 | When developing in dbt Cloud, you can set a custom target name in your development credentials. Go to your account (from the gear menu in the top right hand corner), select the project under **Credentials**, and update the target name. 28 | 29 | 30 | -------------------------------------------------------------------------------- /context-sources/docs/docs/cloud/git/authenticate-azure.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Authenticate with Azure DevOps" 3 | id: "authenticate-azure" 4 | description: "dbt Cloud developers need to authenticate with Azure DevOps." 5 | sidebar_label: "Authenticate with Azure DevOps" 6 | --- 7 | 8 | 9 | If you use the dbt Cloud IDE to collaborate on your team's Azure DevOps dbt repo, you need to [link your dbt Cloud profile to Azure DevOps](#link-your-dbt-cloud-profile-to-azure-devops), which provides an extra layer of authentication. 10 | 11 | ## Link your dbt Cloud profile to Azure DevOps 12 | 13 | Connect your dbt Cloud profile to Azure DevOps using OAuth: 14 | 15 | 1. Click the gear icon at the top right and select **Profile settings**. 16 | 2. Click **Linked Accounts**. 17 | 3. Next to Azure DevOps, click **Link**. 18 | 19 | 20 | 4. Once you're redirected to Azure DevOps, sign into your account. 21 | 5. When you see the permission request screen from Azure DevOps App, click **Accept**. 22 | 23 | 24 | You will be directed back to dbt Cloud, and your profile should be linked. You are now ready to develop in dbt Cloud! 25 | -------------------------------------------------------------------------------- /context-sources/docs/docs/cloud/git/connect-azure-devops.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Connect to Azure DevOps" 3 | id: "connect-azure-devops" 4 | --- 5 | 6 | 7 | 8 | 9 | ## About Azure DevOps and dbt Cloud 10 | 11 | Connect your Azure DevOps cloud account in dbt Cloud to unlock new product experiences: 12 | 13 | - Import new Azure DevOps repos with a couple clicks during dbt Cloud project setup. 14 | - Clone repos using HTTPS rather than SSH 15 | - Enforce user authorization with OAuth 2.0. 16 | - Carry Azure DevOps user repository permissions (read / write access) through to dbt Cloud IDE's git actions. 17 | - Trigger Continuous integration (CI) builds when pull requests are opened in Azure DevOps. 18 | 19 | 20 | To connect Azure DevOps in dbt Cloud: 21 | 22 | 1. An account admin needs to [set up an Active Directory application and add it to dbt Cloud](/docs/cloud/git/setup-azure). 23 | 2. dbt Cloud developers need to [personally authenticate with Azure DevOps](/docs/cloud/git/authenticate-azure) from dbt Cloud. 24 | 25 | 26 | -------------------------------------------------------------------------------- /context-sources/docs/docs/collaborate/environments/environments-in-dbt.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Environments in dbt" 3 | id: "environments-in-dbt" 4 | --- 5 | 6 | In software engineering, environments are used to enable engineers to develop and test code without impacting the users of their software. 7 | 8 | “Production” (or _prod_) refers to the environment that end users interact with, while “development” (or _dev_) is the environment that engineers work in. This means that engineers can work iteratively when writing and testing new code in _development_, and once they are confident in these changes, deploy their code to _production_. 9 | 10 | In traditional software engineering, different environments often use completely separate architecture. For example, the dev and prod versions of a website may use different servers and databases. 11 | 12 | Data warehouses can also be designed to have separate environments – the _production_ environment refers to the relations (for example, schemas, tables, and views) that your end users query (often through a BI tool). 13 | 14 | 15 | ## Related docs 16 | - [About dbt Core versions](/docs/dbt-versions/core) 17 | - [Set Environment variables in dbt Cloud](/docs/build/environment-variables#special-environment-variables) 18 | - [Use Environment variables in jinja](/reference/dbt-jinja-functions/env_var) 19 | -------------------------------------------------------------------------------- /context-sources/docs/docs/collaborate/git/managed-repository.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Managed repository" 3 | id: "managed-repository" 4 | --- 5 | 6 | If you do not already have a git repository for your dbt project, you can let dbt Cloud manage a repository for you. Managed repositories are a great way to trial dbt without needing to create a new repository. 7 | 8 | To set up a project with a managed repository: 9 | 10 | 1. From your Account settings in dbt Cloud, select the project you want to set up with a managed repository. If the project already has a repository set up, you need to edit the repository settings and disconnect the existing repository. 11 | 2. Click **Edit** for the project. 12 | 3. Under Repository, click **Configure repository**. 13 | 4. Select **Managed**. 14 | 5. Enter a name for the repository. For example, "analytics" or "dbt-models." 15 | 6. Click **Create**. 16 | 17 | 18 | dbt Cloud will host and manage this repository for you. If in the future you choose to host this repository yourself, you can contact support to have the contents of your repo transferred to you. 19 | 20 | ** We do not recommend using a managed repository in a production environment. You will not be able to use git features like pull requests which are part of our recommended version control best practices. 21 | -------------------------------------------------------------------------------- /context-sources/docs/docs/collaborate/publish/about-publishing-models.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "About publishing models" 3 | id: about-publishing-models 4 | description: "Information about new features related to publishing models" 5 | --- 6 | 7 | :::info 8 | 9 | Publishing models contains features new in dbt Core v1.5, currently in beta. To see the documentation related to publishing models, make sure to [select v1.5 or higher](/docs/collaborate/publish/about-publishing-models?version=1.5). 10 | 11 | ::: -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-cloud-apis/admin-cloud-api.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "dbt Cloud Administrative API" 3 | id: "admin-cloud-api" 4 | --- 5 | 6 | The dbt Cloud Administrative API is enabled by default for _Team_ and _Enterprise_ plans. It can be used to: 7 | 8 | - Download artifacts after a job has completed 9 | - Kick off a job run from an orchestration tool 10 | - Manage your dbt Cloud account 11 | - and more 12 | 13 | You can use [dbt Cloud Administrative API v2 reference documentation](/dbt-cloud/api-v2) to help you access the API. 14 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-cloud-apis/apis-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "APIs Overview" 3 | id: "overview" 4 | --- 5 | 6 | ## Overview 7 | 8 | Accounts on the _Team_ and _Enterprise_ plans can query the dbt Cloud APIs. 9 | 10 | dbt Cloud provides two APIs: 11 | 12 | - The [dbt Cloud Administrative API](/docs/dbt-cloud-apis/admin-cloud-api) can be used to administrate a dbt Cloud account. 13 | - The [dbt Metadata API](docs/dbt-cloud-apis/metadata-api) can be used to fetch metadata related to the state and health of your dbt project. 14 | 15 | If you want to learn more about webhooks, refer to [Webhooks for your jobs](/docs/deploy/webhooks). 16 | 17 | ## How to Access the APIs 18 | 19 | dbt Cloud supports two types of API Tokens: [user tokens](/docs/dbt-cloud-apis/user-tokens) and [service account tokens](/docs/dbt-cloud-apis/service-tokens). Requests to the dbt Cloud APIs can be authorized using these tokens. 20 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-cloud-apis/schema-metadata-modelByEnv.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Model By Environment" 3 | id: "metadata-schema-modelByEnv" 4 | --- 5 | 6 | import { ArgsTable, SchemaTable } from "./schema"; 7 | 8 | 9 | 10 | This model by environment object allows you to query information about a particular model based on `environmentId`. 11 | 12 | The [example query](#example-query) illustrates a few fields you can query in this `modelByEnvironment` object. Refer to [Fields](#fields) to see the entire schema, which provides all possible fields you can query. 13 | 14 | ### Arguments 15 | 16 | When querying for `modelByEnvironment`, you can use the following arguments. 17 | 18 | 19 | 20 | 21 | ### Example Query 22 | 23 | You can use the `environment_id` and `model_unique_id` to return the model and its execution time for the last 10 times it was run, regardless of which job ran it! 24 | 25 | ```graphql 26 | { 27 | modelByEnvironment( 28 | environmentId: 1, 29 | uniqueId:"model.jaffle_shop.dim_user", 30 | lastRunCount:10, 31 | withCatalog: false 32 | ){ 33 | uniqueId 34 | jobId, 35 | runId, 36 | executionTime 37 | } 38 | } 39 | ``` 40 | 41 | ### Fields 42 | When querying for `modelByEnvironment`, you can use the following fields. 43 | 44 | 45 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-cloud-apis/schema-metadata-seed.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Seed" 3 | id: "metadata-schema-seed" 4 | --- 5 | 6 | import { ArgsTable, SchemaTable } from "./schema"; 7 | 8 | The seed object allows you to query information about a particular seed in a given job. 9 | 10 | ### Arguments 11 | 12 | When querying for a `seed`, the following arguments are available. Note that if you do not include a runId, it will default to the most recent run of the specified job: 13 | 14 | 15 | 16 | Below we show some illustrative example queries and outline the schema of this seed object. 17 | 18 | ### Example Queries 19 | 20 | #### Seed information 21 | 22 | The query below pulls relevant information about a given seed. For example, we could see the load time. 23 | 24 | ```graphql 25 | { 26 | seed(jobId: 123, uniqueId: "seed.jaffle_shop.raw_customers") { 27 | database 28 | schema 29 | uniqueId 30 | name 31 | status 32 | error 33 | } 34 | } 35 | ``` 36 | 37 | ### Fields 38 | 39 | When querying for a `seed`, the following fields are available: 40 | 41 | 42 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-cloud-apis/schema-metadata-seeds.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Seeds" 3 | id: "metadata-schema-seeds" 4 | --- 5 | 6 | import { ArgsTable, SchemaTable } from "./schema"; 7 | 8 | The seeds object allows you to query information about a all seeds in a given job. 9 | 10 | ### Arguments 11 | 12 | When querying for `seeds`, the following arguments are available. Note that if you do not include a runId, it will default to the most recent run of the specified job: 13 | 14 | 15 | 16 | Below we show some illustrative example queries and outline the schema of this seeds object. 17 | 18 | ### Example Queries 19 | #### Seeds information 20 | 21 | The query below pulls relevant information about all seeds in a given job. For example, we could see the load times. 22 | 23 | ```graphql 24 | { 25 | seeds(jobId: 123) { 26 | uniqueId 27 | name 28 | executionTime 29 | status 30 | } 31 | } 32 | ``` 33 | 34 | ### Fields 35 | 36 | When querying for `seeds`, the following fields are available: 37 | 38 | 39 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-cloud-apis/schema-metadata-source.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Source" 3 | id: "metadata-schema-source" 4 | --- 5 | 6 | import { ArgsTable, SchemaTable } from "./schema"; 7 | 8 | The source object allows you to query information about a particular source in a given job. 9 | 10 | ### Arguments 11 | 12 | When querying for a `source`, the following arguments are available. Note that if you do not include a runId, it will default to the most recent run of the specified job: 13 | 14 | 15 | 16 | Below we show some illustrative example queries and outline the schema of this source object. 17 | 18 | ### Example Queries 19 | 20 | #### Source information 21 | 22 | The query below pulls relevant information about a given source. For example, we could see the load time and the state (“pass”, “fail”, “error”) of that source. 23 | 24 | ```graphql 25 | { 26 | source(jobId: 123, uniqueId: "source.jaffle_shop.snowplow.event") { 27 | uniqueId 28 | sourceName 29 | name 30 | state 31 | maxLoadedAt 32 | criteria { 33 | warnAfter { 34 | period 35 | count 36 | } 37 | errorAfter { 38 | period 39 | count 40 | } 41 | } 42 | maxLoadedAtTimeAgoInS 43 | } 44 | } 45 | ``` 46 | 47 | ### Fields 48 | 49 | When querying for a `source`, the following fields are available: 50 | 51 | 52 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-cloud-apis/schema-metadata-test.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Test" 3 | id: "metadata-schema-test" 4 | --- 5 | 6 | import { ArgsTable, SchemaTable } from "./schema"; 7 | 8 | The test object allows you to query information about a particular test. 9 | 10 | ### Arguments 11 | 12 | When querying for a `test`, the following arguments are available. Note that if you do not include a runId, it will default to the most recent run of the specified job: 13 | 14 | 15 | 16 | Below we show some illustrative example queries and outline the schema (all possible fields you can query) of this test object. 17 | 18 | ### Example Queries 19 | #### Test result 20 | 21 | The example query below outputs information about a test, including the state of the test result. This can be one of, in order of severity, "error", "fail", "warn", "pass." 22 | 23 | ```graphql 24 | { 25 | test(jobId: 123, uniqueId: "test.internal_analytics.not_null_metrics_id") { 26 | runId 27 | accountId 28 | projectId 29 | uniqueId 30 | name 31 | columnName 32 | state 33 | } 34 | } 35 | ``` 36 | 37 | ### Fields 38 | When querying for a `test`, the following fields are available: 39 | 40 | 41 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-cloud-apis/schema-metadata-tests.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Tests" 3 | id: "metadata-schema-tests" 4 | --- 5 | 6 | import { ArgsTable, SchemaTable } from "./schema"; 7 | 8 | The tests object allows you to query information about all tests in a given job. 9 | 10 | 11 | ### Arguments 12 | 13 | When querying for `tests`, the following arguments are available. Note that if you do not include a runId, it will default to the most recent run of the specified job: 14 | 15 | 16 | 17 | Below we show some illustrative example queries and outline the schema (all possible fields you can query) of this tests object. 18 | 19 | ### Example Queries 20 | #### Tests result 21 | 22 | The example query below finds all tests in this job, and includes information about those tests. 23 | 24 | ```graphql 25 | { 26 | tests(jobId: 123) { 27 | runId 28 | accountId 29 | projectId 30 | uniqueId 31 | name 32 | columnName 33 | state 34 | } 35 | } 36 | ``` 37 | 38 | ### Fields 39 | When querying for `tests`, the following fields are available: 40 | 41 | 42 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-cloud-apis/user-tokens.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "User tokens" 3 | id: "user-tokens" 4 | --- 5 | 6 | ## User API tokens 7 | 8 | Each dbt Cloud user with a [Developer license](cloud-seats-and-users) is 9 | issued an API token. This token can be used to execute queries against 10 | the dbt Cloud API on the user's behalf. User API tokens inherit the 11 | permissions of the user the that they were created for. 12 | 13 | You can find your User API token in the Profile page under the `API Access` 14 | label. 15 | 16 | 17 | 18 | ## FAQs 19 | 20 | 21 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "About dbt Cloud Release Notes" 3 | id: "dbt-cloud-release-notes" 4 | description: "Release notes for dbt Cloud" 5 | --- 6 | 7 | dbt provides release notes for dbt Cloud so you can see recent and historical changes. Generally, you'll see release notes for these changes: 8 | 9 | * New products and features (new) 10 | * Performance improvements and feature enhancements (enhancement) 11 | * Bug fixes (fix) 12 | 13 | Release notes will be grouped by month. For customers using dbt Virtual Private Cloud, you will see a tag for changes available in your versioned release. 14 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/10-Mar-2023/1.0-deprecation.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "dbt Cloud requires dbt version 1.0 or later" 3 | id: "1.0-deprecation" 4 | description: "Mar 2023: dbt Cloud now requires dbt version 1.0 or later and non-compliant jobs or environments were automatically upgraded to v1.4" 5 | sidebar_label: "Deprecation: dbt Cloud requires dbt v1.0 or later." 6 | tags: [Mar-1-2023] 7 | --- 8 | 9 | 10 | dbt Cloud now requires dbt version 1.0 or later. As of March 1, 2023, we removed all instances of older dbt versions from dbt Cloud. 11 | 12 | Any environments or jobs configured with a dbt version lower than 1.0 were automatically updated to dbt v1.4, which is the latest minor version available on dbt Cloud. 13 | 14 | For more info on dbt versions, releases, and dbt Cloud support timeline, refer to [About dbt Core versions](/docs/dbt-versions/core#latest-releases). 15 | 16 | Refer to some additional info and resources to help you upgrade your dbt version: 17 | 18 | - [How to upgrade dbt without fear](https://docs.getdbt.com/blog/upgrade-dbt-without-fear) 19 | - [Upgrade Q&A on breaking changes](/docs/dbt-versions/upgrade-core-in-cloud#upgrading-legacy-versions-under-10) 20 | - [Version migration guides](/guides/migration/versions) 21 | 22 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/10-Mar-2023/apiv2-limit.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "API requests have a maximum limit of `100`" 3 | id: apiv2-limit" 4 | description: "Mar 2023: In order to ease pressure on our API, we have implemented a maximum limit of `100` for all API requests to our `list` endpoints. This limit is applicable to multi-tenant instances only." 5 | sidebar_label: "Update: API requests have a maximum limit of `100`" 6 | tags: [Mar-6-2023, API] 7 | --- 8 | 9 | 10 | To make the API more scalable and reliable, we've implemented a maximum limit of `100` for all API requests to our `list` endpoints. If API requests exceed the maximum limit parameter of `100`, a user will receive an API error message. 11 | 12 | This maximum limit applies to [multi-tenant instances](/docs/cloud/about-cloud/regions-ip-addresses) only, and _does not_ apply to single tenant instances. 13 | 14 | Refer to the [Pagination](https://docs.getdbt.com/dbt-cloud/api-v2#section/Pagination) section for more information on this change. -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/11-Feb-2023/no-partial-parse-config.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Disable partial parsing in dbt Cloud job commands" 3 | id: "no-partial-parse-config" 4 | description: "You can now disable partial parsing in dbt Cloud job commands." 5 | sidebar_label: "New: Disable partial parsing in job commands" 6 | tags: [Feb-15-2023] 7 | --- 8 | 9 | You can now use the `--no-partial-parse` flag to disable partial parsing in your dbt Cloud job commands.  10 | 11 | Previously, the [`--no-partial-parse` global config](/reference/global-configs#partial-parsing) was only available in dbt Core. For more information, refer to [partial parsing](/reference/parsing#partial-parsing). 12 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/23-Dec-2022/default-thread-value.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Threads default value changed to 4" 3 | id: "default-thread-value" 4 | description: "Threads now default to 4 users' profile." 5 | sidebar_label: "Enhancement: Threads default value changed to 4 " 6 | tags: [Dec-12-2022] 7 | --- 8 | 9 | Threads help parallelize node execution in the dbt directed acyclic graph [(DAG)](https://docs.getdbt.com/terms/dag). 10 | 11 | Previously, the thread value defaulted to 1, which can increase the runtime of your project. To help reduce the runtime of your project, the default value for threads in user profiles is now set to 4 threads. 12 | 13 | You can supply a custom thread count if you'd prefer more or less parallelization. 14 | 15 | For more information, read [Understanding threads](/docs/core/connection-profiles#understanding-threads). 16 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/23-Dec-2022/new-jobs-default-as-off.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Creating a new job no longer triggers a run by default" 3 | id: "new-jobs-default-as-off.md" 4 | description: "You need to click **Run on schedule** before a job will be scheduled to run" 5 | sidebar_label: "Improvement: New jobs no longer run by default" 6 | tags: [Dec-05-2022] 7 | --- 8 | 9 | To help save compute time, new jobs will no longer be triggered to run by default. When you create a new job in dbt Cloud, you can trigger the job to run by selecting **Run on schedule** and completing the desired schedule and timing information. 10 | 11 | For more information, refer to [Job triggers](/docs/deploy/job-triggers). 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/23-Dec-2022/private-packages-clone-git-token.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Private packages must be cloned using access tokens provided by environment variables" 3 | description: "Private GitHub packages must be cloned using access tokens provided by environment variables." 4 | sidebar_label: "Deprecation: Private packages must be cloned using access tokens" 5 | tags: [Dec-23-2022] 6 | --- 7 | 8 | The supported method for cloning private GitHub packages is the [git token method](/docs/build/packages#git-token-method), where an appropriate access token is passed into the package repository URL with an environment variable. 9 | 10 | A small number of people have been able to clone private packages using dbt's native GitHub application without explicitly providing an access token. This functionality is being deprecated as it’s limited in flexibility. 11 | 12 | If you have been using a package hosted in a private repository on GitHub, you must start passing an access token into the URL. 13 | 14 | An example of passing an access token: 15 | 16 | 17 | 18 | ```yaml 19 | 20 | packages: 21 | - git: "https://{{env_var('DBT_ENV_SECRET_GIT_CREDENTIAL')}}@github.com/dbt-labs/awesome_repo.git" 22 | 23 | ``` 24 | 25 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/24-Nov-2022/dbt-databricks-unity-catalog-support.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "The dbt Cloud + Databricks experience is getting even better" 3 | id: "cloud-databricks-unitycatalog" 4 | description: "The dbt-databricks adapter is now available for use with dbt Cloud" 5 | sidebar_label: "Enhancement: Support for Databricks Unity Catalog with dbt-databricks" 6 | tags: [Nov-17-2022, v1.1.66.15] 7 | --- 8 | 9 | dbt Cloud is the easiest and most reliable way to develop and deploy a dbt project. It helps remove complexity while also giving you more features and better performance. A simpler Databricks connection experience with support for Databricks’ Unity Catalog and better modeling defaults is now available for your use. 10 | 11 | For all the Databricks customers already using dbt Cloud with the dbt-spark adapter, you can now [migrate](https://docs.getdbt.com/guides/migration/tools/migrating-from-spark-to-databricks#migration) your connection to the [dbt-databricks adapter](https://docs.getdbt.com/reference/warehouse-setups/databricks-setup) to get the benefits. [Databricks](https://www.databricks.com/blog/2022/11/17/introducing-native-high-performance-integration-dbt-cloud.html) is committed to maintaining and improving the adapter, so this integrated experience will continue to provide the best of dbt and Databricks. 12 | 13 | Check out our [live blog post](https://www.getdbt.com/blog/dbt-cloud-databricks-experience/) to learn more. 14 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/25-Oct-2022/cloud-integration-azure.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Announcing dbt Cloud’s native integration with Azure DevOps" 3 | id: "cloud-integration-azure" 4 | description: "dbt Cloud native integration with Azure DevOps" 5 | sidebar_label: "Improvement: Native integration with Azure DevOps" 6 | tags: [Oct-11-2022, v1.1.64] 7 | --- 8 | 9 | dbt Cloud now offers a native integration with Azure DevOps for dbt Cloud customers on the enterprise plan. We built this integration to remove friction, increase security, and unlock net new product experiences for our customers. [Setting up the Azure DevOps integration](/docs/cloud/git/connect-azure-devops) in dbt Cloud provides: 10 | - easy dbt project set up, 11 | - an improved security posture, 12 | - repo permissions enforcement in dbt Cloud IDE, and 13 | - dbt Cloud Slim CI. 14 | 15 | Check out our [live blog post](https://www.getdbt.com/blog/dbt-cloud-integration-azure-devops/) to learn more! 16 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/25-Oct-2022/new-ide-launch.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Enhancement: New Cloud IDE launch" 3 | id: "new-ide-launch" 4 | description: "Oct 2022: Introducing a new dbt Cloud IDE that's faster, has performance upgrades, ergonomics improvements, and other delightful enhancements." 5 | sidebar_label: "Snappier, faster, and new Cloud IDE" 6 | tags: [Oct-18-2022, IDE] 7 | --- 8 | 9 | ## Introducing a snappier, improved, and powerful Cloud IDE 10 | 11 | The new version of the Cloud IDE makes it easy for you to build data models without thinking much about environment setup and configuration. 12 | 13 | The new Cloud IDE includes performance upgrades, ergonomics improvements, and some delightful enhancements! 14 | 15 | Some of the improvements include: 16 | 17 | - Improved Cloud IDE startup time (starting the IDE), interaction time (saving and committing), and reliability. 18 | - Better organization and navigation with features like drag and drop of files, breadcrumb, build button drop-down, and more. 19 | - You can use new features like auto-format your file, auto-complete model names, and git diff view to see your changes before making a pull request. 20 | 21 | Read more about the new [Cloud IDE features](/docs/cloud/develop-in-the-cloud#cloud-ide-features) and check out [New and improved Cloud IDE](https://www.getdbt.com/blog/new-improved-cloud-ide/) blog for more info! 22 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/26-Sept-2022/liststeps-endpoint-deprecation.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "List Steps API endpoint deprecation warning" 3 | id: "liststeps-endpoint-deprecation.md" 4 | description: "List Steps API deprecation" 5 | sidebar_label: "Deprecation: List Steps API endpoint" 6 | tags: [Sept-15-2022] 7 | --- 8 | 9 | On October 14th, 2022 dbt Labs is deprecating the [List Steps](https://docs.getdbt.com/dbt-cloud/api-v2#tag/Runs/operation/listSteps) API endpoint. From October 14th, any GET requests to this endpoint will fail. Please prepare to stop using the List Steps endpoint as soon as possible. 10 | 11 | dbt Labs will continue to maintain the [Get Run](https://docs.getdbt.com/dbt-cloud/api-v2#tag/Runs/operation/getRunById) endpoint, which is a viable alternative depending on the use case. 12 | 13 | You can fetch run steps for an individual run with a GET request to the following URL, replacing `YOUR_ACCESS_URL` with the [appropriate Access URL](/docs/cloud/about-cloud/regions-ip-addresses) for your region and plan: 14 | 15 | `https://YOUR_ACCESS_URL/api/v2/accounts/{accountId}/runs/{runId}/?include_related=["run_steps"]` 16 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/26-Sept-2022/metadata-api-data-retention-limits.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Query the previous three months of data using the metadata API" 3 | id: "metadata-api-data-retention-limits.md" 4 | description: "Metadata API data retention limits" 5 | sidebar_label: "Fix: Metadata API data retention limits" 6 | tags: [Sept-29-2022] 7 | --- 8 | 9 | In order to make the metadata API more scalable and improve its latency, we’ve implemented data retention limits. The metadata API can now query data from the previous three months. For example, if today was March 1, you could query data back to January 1st. 10 | 11 | For more information, see [Metadata API](/docs/dbt-cloud-apis/metadata-api) 12 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/27-Aug-2022/support-redshift-ra3.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Enhancement: Support for cross-database sources on Redshift RA3 instances" 3 | id: "support-redshift-ra3.md" 4 | description: "Adding support for cross-database queries for RA3" 5 | sidebar_label: "Enhancement: Support for RA3" 6 | tags: [Aug-31-2022, 1.1.61.5] 7 | 8 | --- 9 | 10 | Cross-database queries for RA3 instances are now supported by dbt Cloud projects using a Redshift connection. 11 | 12 | With cross-database queries, you can seamlessly query data from any database in the cluster, regardless of which database you are connected to with dbt. 13 | 14 | The [connection configuration](https://docs.getdbt.com/reference/warehouse-profiles/redshift-profile) `ra3_node` has been defaulted to `true`. This allows users to: 15 | 16 | - benefit from the full RA3 nodes’ capabilities, 17 | - generate appropriate dbt documentation. 18 | 19 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/28-July-2022/render-lineage-feature.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Enhancement: Large DAG feature" 3 | id: "render-lineage-feature" 4 | description: "Jul 2022: Use the Render Lineage button to visualize large DAGs" 5 | sidebar_label: "Enhancement: Large DAG feature" 6 | tags: [July-5-2022, v1.1.56, IDE] 7 | 8 | --- 9 | 10 | You can now select **Render Lineage** to visualize large DAGs. 11 | 12 | Large DAGs can take a long time (10 or more seconds, if not minutes) to render and can cause browsers to crash. 13 | 14 | The new button prevents large DAGs from rendering automatically. Instead, you can select **Render Lineage** to load the visualization. This should affect about 15% of the DAGs. 15 | 16 | 17 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/29-May-2022/gitlab-auth.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Refresh expired access tokens in the IDE when using GitLab" 3 | id: "gitlab-auth" 4 | description: "Adding support for expiring OAuth access tokens." 5 | sidebar_label: "Enhancement: Refreshing GitLab OAuth Access" 6 | tags: [May-19-2022, v1.1.52] 7 | --- 8 | 9 | On May 22, GitLab changed how they treat [OAuth access tokens that don't expire](https://docs.gitlab.com/ee/update/deprecations.html#oauth-tokens-without-expiration). We updated our IDE logic to handle OAuth token expiration more gracefully. Now, the first time your token expires after 2 hours of consecutive IDE usage, you will have to re-authenticate in GitLab to refresh your expired OAuth access token. We will handle subsequent refreshes for you if you provide the authorization when you re-authenticate. 10 | 11 | This additional security layer in the IDE is available only to the dbt Cloud enterprise plan. 12 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/30-April-2022/audit-log.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Audit log" 3 | id: "audit-log" 4 | description: "Review actions performed by people in your organization." 5 | sidebar_label: "New: Audit log" 6 | 7 | tags: [April-26-2022] 8 | --- 9 | 10 | 11 | To review actions performed by people in your organization, dbt provides logs of audited user and system events. The dbt Cloud audit log lists events triggered in your organization within the last 90 days. 12 | 13 | The audit log includes details such as who performed the action, what the action was, and when it was performed. For more details, review [the audit log for dbt Cloud Enterprise](/docs/cloud/manage-access/audit-log) documentation. 14 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/30-April-2022/credentials-saved.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Credentials no longer accidentally wiped when editing an environment" 3 | id: "credentials-saved" 4 | description: "Credentials are now saved when editing an environment." 5 | sidebar_label: "Fix: Credentials saved" 6 | tags: [April-29-2022, v1.1.51] 7 | --- 8 | 9 | We resolved a bug where when updating unencrypted fields (e.g. threads, schema name) in an environment setting would cause secret fields (e.g. password, keypair, credential details) to be deleted from that environment. Now users can freely update environment settings without fear of unintentionally wiping credentials. 10 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/30-April-2022/email-verification.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Email verification" 3 | description: "dbt Cloud users created via SAML JIT will now confirm identity via email to activate their account." 4 | sidebar_label: "New: Email verification" 5 | sidebar_position: 1 6 | 7 | tags: [April-26-2022, v1.1.48] 8 | --- 9 | 10 | To enhance the security of user creation, dbt Cloud users created using SAML Just-in-Time (JIT) will now confirm identity via email to activate their account. Using email to confirm identity ensures the user still has access to the same email address they use to login via SAML. 11 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/30-April-2022/scheduler-improvements.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Scheduler performance improvements" 3 | id: "scheduler-improvements" 4 | description: "scheduler, which has much faster prep times, especially at the top of the hour." 5 | sidebar_label: "Improvement: Scheduler performance" 6 | tags: [April-19-2022] 7 | --- 8 | 9 | We rolled out our new distributed scheduler, which has much faster prep times, especially at the top of the hour. We share more about our work and improvements in our [product news blog post](https://www.getdbt.com/blog/a-good-problem-to-have/). 10 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/31-March-2022/ide-timeout-message.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Spotty internet issues no longer cause a session time out message" 3 | id: "ide-timeout-message" 4 | description: "Mar 2022: We fixed an issue where a spotty internet connection could cause the “IDE session timed out” message to appear unexpectedly. People using a VPN were most likely to see this issue." 5 | sidebar_label: "Fix: Session time out" 6 | tags: [v1.1.47, March-10-2022, IDE] 7 | --- 8 | 9 | We fixed an issue where a spotty internet connection could cause the “IDE session timed out” message to appear unexpectedly. People using a VPN were most likely to see this issue. 10 | 11 | We updated the health check logic so it now excludes client-side connectivity issues from the IDE session check. If you lose your internet connection, we no longer update the health-check state. Now, losing internet connectivity will no longer cause this unexpected message. 12 | 13 | 14 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/31-March-2022/prep-and-waiting-time.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Dividing queue time into waiting and prep time" 3 | id: "prep-and-waiting-time" 4 | description: "dbt Cloud now shows waiting time and prep time for a run." 5 | sidebar_label: "Enhancement: Waiting time and prep time" 6 | 7 | tags: [v1.1.46, March-02-2022] 8 | --- 9 | 10 | dbt Cloud now shows "waiting time" and "prep time" for a run, which used to be expressed in aggregate as "queue time". Waiting time captures the time dbt Cloud waits to run your job if there isn't an available run slot or if a previous run of the same job is still running. Prep time represents the time it takes dbt Cloud to ready your job to run in your cloud data warehouse. 11 | 12 | 13 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/32-February-2022/DAG-updates-more.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "DAG updates and performance improvements" 3 | description: "Feb 2022: Clicking a node in the DAG opens a model or config file in new tab in the IDE." 4 | id: "DAG-updates-more" 5 | sidebar_label: "DAG updates and more" 6 | tags: [v1.1.44, February-02-2022, IDE] 7 | --- 8 | 9 | Love the DAG in the IDE as much as we do? Now when you click on a node in the DAG, the model or config file will open as a new tab in the IDE, so you can directly view or edit the code. We'll continue to ship better developer ergonomic functionality throughout the year. 10 | 11 | #### Performance improvements and enhancements 12 | 13 | * Updated recommended dbt commands in the IDE to include dbt Core v1.0 commands, such as "build" and the "--select" argument. -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/32-February-2022/service-tokens-more.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Service tokens and bug fixes" 3 | is: "service-tokens-more" 4 | description: "Service tokens can now be assigned granular permissions to enforce least privilege access." 5 | sidebar_label: "Service tokens and more" 6 | tags: [v1.1.45, February-16-2022] 7 | --- 8 | 9 | Service tokens can now be assigned granular permissions to enforce least privilege access. If you're on Enterprise, you can assign any enterprise permission set to newly issued service tokens. If you're on Teams, you can assign the Job Admin permission set to newly issued service tokens. We highly recommend you re-issue service tokens with these new permissions to increase your security posture! See docs [here](https://docs.getdbt.com/docs/dbt-cloud-apis/service-tokens#permissions-for-service-account-tokens). 10 | 11 | #### New products and features 12 | 13 | - We are joining the [GitHub secret scanning partner program](https://docs.github.com/en/developers/overview/secret-scanning-partner-program) to better secure your token against accidental public exposure and potential fraudulent usage. 14 | 15 | #### Bug fixes 16 | 17 | - Credentials are no longer accidentally deleted when a user updates an environment setting. 18 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/33-January-2022/IDE-autocomplete-more.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Autocomplete snippets for SQL and YAML files in IDE" 3 | id: "ide-timeout-message" 4 | description: "Jan 2022: Autocomplete snippets for SQL and YAML files in IDE" 5 | sidebar_label: "Autocomplete in IDE and more" 6 | tags: [v1.1.43, January-19-2022, IDE] 7 | --- 8 | 9 | Some noteworthy improvements include autocomplete snippets for sql and yaml files in the IDE, which are available for use now! We also added a [new metric layer page](https://docs.getdbt.com/docs/dbt-cloud/using-dbt-cloud/cloud-metrics-layer) to docs.getdbt.com to help you begin thinking about the metrics layer in dbt Cloud. 10 | 11 | #### Performance improvements and enhancements 12 | 13 | * Branch names now default to "main" instead of "master" in new managed and unmanaged Git repositories. 14 | * Update IDE autocomplete snippets. 15 | -------------------------------------------------------------------------------- /context-sources/docs/docs/dbt-versions/release-notes/33-January-2022/model-timing-more.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Model timing for Multi-tenant Team and Enterprise accounts" 3 | id: "model-timing-more" 4 | description: "Model timing is now available for Multi-tenant Team and Enterprise accounts, plus performance improvements." 5 | sidebar_label: "Model timing and more" 6 | 7 | tags: [v1.1.42, January-05-2022] 8 | --- 9 | #### New products and features 10 | 11 | We started the new year with a gift! Multi-tenant Team and Enterprise accounts can now use the new [Model timing](https://docs.getdbt.com/docs/dbt-cloud/using-dbt-cloud/cloud-model-timing-tab) tab in dbt Cloud. You can use this tab to further explore long-running models to see if they need refactoring or rescheduling. 12 | 13 | #### Performance improvements and enhancements 14 | 15 | * We added client-side naming validation for file or folder creation. 16 | -------------------------------------------------------------------------------- /context-sources/docs/docs/deploy/airgapped.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: airgapped-deployment 3 | title: Airgapped (Beta) 4 | --- 5 | 6 | :::info Airgapped 7 | 8 | This section provides a high level summary of the airgapped deployment type for dbt Cloud. This deployment type is currently in Beta and may not be supported in the long term. 9 | If you’re interested in learning more about airgapped deployments for dbt Cloud, contact us at sales@getdbt.com. 10 | 11 | ::: 12 | 13 | The airgapped deployment is similar to an on-premise installation in that the dbt Cloud instance will live in your network, and is subject to your security procedures, technologies, and controls. An airgapped install allows you to run dbt Cloud without any external network dependencies and is ideal for organizations that have strict rules around installing software from the cloud. 14 | 15 | The installation process for airgapped is a bit different. Instead of downloading and installing images during installation time, you will download all of the necessary configuration and Docker images before starting the installation process, and manage uploading these images yourself. This means that you can remove all external network dependencies and run this application in a very secure environment. 16 | 17 | For more information about the dbt Cloud Airgapped deployment see the below. 18 | 19 | - [Customer Managed Network Architecture](/docs/cloud/about-cloud/architecture) 20 | -------------------------------------------------------------------------------- /context-sources/docs/docs/deploy/job-notifications.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Job notifications" 3 | id: "job-notifications" 4 | description: "Set up notifications in dbt Cloud to receive Email or Slack alerts for job run status." 5 | --- 6 | 7 | 8 | Setting up notifications in dbt Cloud will allow you to receive alerts via Email or a chosen Slack channel when a job run succeeds, fails, or is cancelled. 9 | 10 | ### Email 11 | 12 | There are two options for setting up email notifications. As a **user**, you can set up email notifications for yourself under your Profile. As an **admin**, you can set up notifications on behalf of your team members. 13 | 14 | 1. Click the gear in the top right and select **Notification settings**. 15 | 16 | 2. **As a user:** Select **Edit** and select the type of Notification (Succeeds, Fails, or Is Cancelled) for each Job for which you would like to be notified, or 17 | 18 | **As an admin:** Select one or more users you'd like to set notifications for. If you only see your own name, then you might not have admin privileges. Select **Edit** and select the type of Notification (Succeeds, Fails, or Is Cancelled) for each Job for which they will be notified. 19 | 20 | 3. Click **Save**. 21 | 22 | 23 | ### Slack 24 | 25 | 26 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Accounts/change-billing.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How can I update my billing information? 3 | description: "Instructions on how to update your billing info" 4 | sidebar_label: 'How to update billing' 5 | id: change-billing 6 | --- 7 | 8 | 9 | If you want to change your account's credit card details, select the gear menu in the upper right corner of dbt Cloud. Go to Account Settings → Billing → Payment Information. In the upper right corner of Payment Information, click **Edit** to enter the new credit card details. Only the _account owner_ can make this change. 10 | 11 | To change your billing name or location address, send our Support team a message at support@getdbt.com with the newly updated information, and we can make that change for you! 12 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Accounts/change-users-license.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I change a user license type to read-only in dbt Cloud? 3 | description: "Changing a user license type to read-only in dbt Cloud" 4 | sidebar_label: 'How to change a user license type to read-only' 5 | id: change-user-license 6 | 7 | --- 8 | 9 | To change the license type for a user from `developer` to `read-only` in dbt Cloud, you must be an account owner or have admin privileges. You might make this change to free up a billable seat but retain the user’s access to view the information in the dbt Cloud account. 10 | 11 | 1. From dbt Cloud, click the gear icon at the top right and select **Account Settings**. 12 | 13 | 14 | 15 | 2. In **Account Settings**, select **Users** under **Teams**. 16 | 3. Select the user you want to remove, and click **Edit** in the bottom of their profile. 17 | 4. For the **License** option, choose **Read-only** (from **Developer**), and click **Save**. 18 | 19 | 20 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Accounts/configurable-snapshot-path.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Can I store my snapshots in a directory other than the `snapshot` directory in my project? 3 | description: "You can see how to change snapshots directory in your project" 4 | sidebar_label: 'Store snapshot in other directory' 5 | id: configurable-snapshot-path 6 | --- 7 | By default, dbt expects your snapshot files to be located in the `snapshots` subdirectory of your project. 8 | 9 | To change this, update the [snapshot-paths](reference/project-configs/snapshot-paths.md) configuration in your `dbt_project.yml` 10 | file, like so: 11 | 12 | 13 | 14 | ```yml 15 | snapshot-paths: ["snapshots"] 16 | ``` 17 | 18 | 19 | 20 | Note that you cannot co-locate snapshots and models in the same directory. 21 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Accounts/dbt-specific-jinja.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: What parts of Jinja are dbt-specific? 3 | description: "You can check which Jinja functions are dbt-specific" 4 | sidebar_label: 'dbt-specific Jinja' 5 | id: dbt-specific-jinja 6 | --- 7 | 8 | There are certain expressions that are specific to dbt — these are documented in the [Jinja function reference](/reference/dbt-jinja-functions) section of these docs. Further, docs blocks, snapshots, and materializations are custom Jinja _blocks_ that exist only in dbt. 9 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Accounts/find-user-id.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Where can I find my user id? 3 | description: "Instructions on where to find your user id" 4 | sidebar_label: 'Where can I find my user ID' 5 | id: find-user-id 6 | --- 7 | 8 | Knowing your dbt Cloud user ID can help with actions related to [rotating your API token](/docs/faqs/API/rotate-token), interacting with support, and more. 9 | 10 | To find your user ID in dbt Cloud, read the following steps: 11 | 12 | 1. Go to **Account Settings**, **Team**, and then **Users**, 13 | 2. Select your user,
14 | 3. In the address bar, the number after `/users` is your user ID. 15 | 4. Copy that number or save it somewhere safe.
16 | 17 | For example, if the URL is `https://cloud.getdbt.com/settings/accounts/12345/users/67891` — the user ID is `67891`

-------------------------------------------------------------------------------- /context-sources/docs/faqs/Accounts/git-account-in-use.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Why am I getting an "account in use" error? 3 | description: "What to do when you receive a Git account in use error" 4 | sidebar_label: 'Git account in use' 5 | id: git-account-in-use 6 | --- 7 | 8 | If you're receiving an 'Account in use' error when trying to integrate GitHub in your Profile page, this is because the Git integration is a 1-to-1 integration, so you can only have your Git account linked to one dbt Cloud user account. 9 | 10 | Here are some steps to take to get you unstuck: 11 | 12 | * Log in to the dbt Cloud account integrated with your Git account. Go to your user profile and click on Integrations to remove the link. 13 | 14 | If you don't remember which dbt Cloud account is integrated, please email dbt Support at support@getdbt.com and we'll do our best to disassociate the integration for you. 15 | 16 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Accounts/payment-accepted.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Can I pay via invoice? 3 | description: "What payments types are accepted" 4 | sidebar_label: 'Can I pay invoice' 5 | id: payment-accepted 6 | --- 7 | 8 | Presently for Team plans, self-service dbt Cloud payments must be made via credit card and by default, they will be billed monthly based on the number of active developer seats. 9 | 10 | We don't have any plans to do invoicing for self-service teams in the near future, but we *do* currently support invoices for companies on the **dbt Cloud Enterprise plan.** Feel free to [contact us](https://www.getdbt.com/contact) to build your Enterprise pricing. 11 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Accounts/slack.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I set up Slack notifications? 3 | description: "Instructions on how to set up slack notifications" 4 | sidebar_label: 'How to set up Slack' 5 | id: slack 6 | --- 7 | 8 | 9 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Docs/document-all-columns.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Do I need to add a yaml entry for column for it to appear in the docs site? 3 | description: "All columns appear in your docs site" 4 | sidebar_label: 'Types of columns included in doc site' 5 | id: document-all-columns 6 | --- 7 | Fortunately, no! 8 | 9 | dbt will introspect your warehouse to generate a list of columns in each relation, and match it with the list of columns in your `.yml` files. As such, any undocumented columns will still appear in your documentation! 10 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Docs/document-other-resources.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Can I document things other than models, like sources, seeds, and snapshots? 3 | description: "Using the description key to document resources" 4 | sidebar_label: 'Document other resources' 5 | id: document-other-resources 6 | --- 7 | 8 | Yes! You can document almost everything in your project using the `description:` key. Check out the reference docs on [descriptions](description) for more info! 9 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Docs/documenting-macros.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I document macros? 3 | description: "You can use a schema file to document macros" 4 | sidebar_label: 'Document macros' 5 | id: documenting-macros 6 | --- 7 | 8 | The `macros:` key is new in 0.16.0. 9 | 10 | To document macros, use a [schema file](macro-properties) and nest the configurations under a `macros:` key 11 | 12 | ## Example 13 | 14 | 15 | 16 | ```yml 17 | version: 2 18 | 19 | macros: 20 | - name: cents_to_dollars 21 | description: A macro to convert cents to dollars 22 | arguments: 23 | - name: column_name 24 | type: string 25 | description: The name of the column you want to convert 26 | - name: precision 27 | type: integer 28 | description: Number of decimal places. Defaults to 2. 29 | ``` 30 | 31 | 32 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Docs/long-descriptions.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I write long-form explanations in my descriptions? 3 | description: "Write long descriptions in your documentation" 4 | sidebar_label: 'Write long descriptions' 5 | id: long-descriptions 6 | --- 7 | If you need more than a sentence to explain a model, you can: 8 | 1. Split your description over multiple lines using `>`. Interior line breaks are removed and Markdown can be used. This method is recommended for simple, single-paragraph descriptions: 9 | ```yml 10 | version: 2 11 | 12 | models: 13 | - name: customers 14 | description: > 15 | Lorem ipsum **dolor** sit amet, consectetur adipisicing elit, sed do eiusmod 16 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 17 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 18 | consequat. 19 | ``` 20 | 21 | 2. Split your description over multiple lines using `|`. Interior line breaks are maintained and Markdown can be used. This method is recommended for more complex descriptions: 22 | ```yml 23 | version: 2 24 | 25 | models: 26 | - name: customers 27 | description: | 28 | ### Lorem ipsum 29 | 30 | * dolor sit amet, consectetur adipisicing elit, sed do eiusmod 31 | * tempor incididunt ut labore et dolore magna aliqua. 32 | ``` 33 | 34 | 3. Use a [docs block](/documentation#using-docs-blocks) to write the description in a separate Markdown file. 35 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Docs/modify-owner-column.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I populate the owner column in the generated docs? 3 | description: "Modify owner column" 4 | sidebar_label: 'Can I populate owner column in docs?' 5 | id: modify-owner-column 6 | --- 7 | 8 | 9 | Due to the nature of the field, you won't be able to change the owner column in your generated documentation. 10 | 11 | The _owner_ field in `dbt-docs` is pulled from database metdata (`catalog.json`), meaning the owner of that table in the database. With the exception of exposures, it's not pulled from an `owner` field set within dbt. 12 | 13 | Generally, dbt's database user owns the tables created in the database. Source tables are usually owned by the service responsible for ingesting/loading them. 14 | 15 | If you set `meta.owner`, you should now be seeing that field appear under **meta** (pulled from dbt), but still not under the top-level **owner** field. 16 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Docs/sharing-documentation.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I share my documentation with my team members? 3 | description: "Use read-only seats to share documentation" 4 | sidebar_label: 'Share documentation with teammates' 5 | id: sharing-documentation 6 | --- 7 | 8 | If you're using dbt Cloud to deploy your project, and have the [Team plan](https://www.getdbt.com/pricing/), you can have up to 5 read-only users, who will be able access the documentation for your project. 9 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Environments/beta-release.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: What is a beta release? 3 | description: "How to try out beta features" 4 | sidebar_label: 'What is a beta release?' 5 | id: beta-release 6 | --- 7 | This is a chance to try out brand-new functionality. You get to start planning for use cases that the next minor version will unlock. We get to hear from you about unexpected behavior and nasty bugs, so that the release candidate has more polish and fewer surprises. 8 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Environments/diff-database-environment.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Can I set a different connection at the environment level? 3 | description: "Separate projects for different environments workaround" 4 | sidebar_label: 'Set different database connections at environment level' 5 | id: diff-database-environment 6 | --- 7 | 8 | Although you're unable to set a different connection at the environment level, there is a workaround where you can have separate projects for their different environments and link them to different hostnames, while still connecting to the same repo. 9 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Environments/profile-env-vars.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Can I use environment variables in my profile? 3 | description: "Use env_var in your profile" 4 | sidebar_label: 'Use env_var in your profile' 5 | id: profile-env-vars 6 | --- 7 | Yes! Check out the docs on [environment variables](env_var) for more information. 8 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Environments/profile-name.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: What should I name my profile? 3 | description: "Use company name for profile name" 4 | sidebar_label: 'Naming your profile' 5 | id: profile-name 6 | --- 7 | We typically use a company name for a profile name, and then use targets to differentiate between `dev` and `prod`. Check out the docs on [environments in dbt Core](/docs/collaborate/environments/dbt-core-environments) for more information. 8 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Environments/target-names.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: What should I name my target? 3 | description: "Using targets to clarify development and production runs" 4 | sidebar_label: 'Naming your target' 5 | id: target-names 6 | --- 7 | 8 | We typically use targets to differentiate between development and production runs of dbt, naming the targets `dev` and `prod`, respectively. Check out the docs on [managing environments in dbt Core](/docs/collaborate/environments/dbt-core-environments) for more information. 9 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Git/gitignore.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Why can't I checkout a branch or create a new branch? 3 | description: "Add or fill in gitignore file" 4 | sidebar_label: 'Unable to checkout or create branch' 5 | id: gitignore 6 | --- 7 | 8 | If you're finding yourself unable to revert changes, check out a branch or click commit - this is usually do to your project missing a [.gitignore](https://github.com/dbt-labs/dbt-starter-project/blob/main/.gitignore) file OR your gitignore file doesn't contain the necessary content inside the folder. 9 | 10 | This is what causes that 'commit' git action button to display. No worries though - to fix this, you'll need to complete the following steps in order: 11 | 12 | 1. In the Cloud IDE, add the missing .gitignore file or contents to your project. You'll want to make sure the .gitignore file includes the following: 13 | 14 | ```shell 15 | target/ 16 | dbt_modules/ 17 | dbt_packages/ 18 | logs/ 19 | ``` 20 | 21 | 2. Once you've added that, make sure to save and commit. 22 | 23 | 3. Navigate to the same branch in your remote repository (which can be accessed directly through your git provider's web interface) and delete the logs, target, and dbt_modules/dbt_packages folders. 24 | 25 | 4. Go back into the Cloud IDE and reclone your repository. This can be done by clicking on the green "ready" in the bottom right corner of the IDE (next to the command bar), and then clicking the orange "reclone repo" button in the pop up. 26 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Git/gitlab-selfhosted.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Can self-hosted GitLab instances only be connected via dbt Cloud Enterprise plans? 3 | description: "Self hosted GitLab instances plan availability" 4 | sidebar_label: 'Self-hosted GitLab instances available for dbt Cloud Enterprise' 5 | id: gitlab-selfhosted 6 | --- 7 | 8 | Presently yes, this is only available to Enterprise users. This is because of the way you have to set up the GitLab app redirect URL for auth, which can only be customized if you're a user on an Enterprise plan. 9 | 10 | Check out our [pricing page](https://www.getdbt.com/pricing/) for more information or feel free to [contact us](https://www.getdbt.com/contact) to build your Enterprise pricing. 11 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Git/google-cloud-repo.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How can I connect dbt to a Google Source repository? 3 | description: "Use SSH URL method to connect to Google Source Repo" 4 | sidebar_label: 'Connect dbt to a Google Source repository' 5 | id: google-cloud-repo 6 | 7 | --- 8 | 9 | Although we don't officially support Google Cloud as a git repository, the below workaround using the SSH URL method should help you to connect: 10 | 11 | - First: "import" your Repository into dbt Cloud using the SSH URL provided to you by GCP. That will look something like: 12 | `ssh://drew@fishtownanalytics.com@source.developers.google.com:2022/p/dbt-integration-tests/r/drew-debug` 13 | 14 | - After importing the repo, you should see a public key generated by dbt Cloud for the repository. You'll want to copy that public key into a new SSH Key for your user here: [https://source.cloud.google.com/user/ssh_keys](https://source.cloud.google.com/user/ssh_keys) 15 | 16 | - After saving this SSH key, dbt Cloud should be able to read and write to this repo. 17 | 18 | If you've tried the workaround above and are still having issues connecting - reach out to the Support team at support@getdbt.com and we'll be happy to help! 19 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Git/managed-repo.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How can we move our project from a managed repository, to a self-hosted repository? 3 | description: "Email Support to request your managed repository" 4 | sidebar_label: 'How to request your managed repository zipfile' 5 | id: managed-repo 6 | --- 7 | 8 | dbt Labs can send your managed repository through a ZIP file in its current state for you to push up to a git provider. After that, you'd just need to switch over to the [repo in your project](/docs/cloud/git/import-a-project-by-git-url) to point to the new repository. 9 | 10 | When you're ready to do this, [contact the dbt Labs Support team](mailto:support@getdbt.com) with your request and your managed repo URL, which you can find by navigating to your project setting. To find project settings, click the gear icon in the upper right, select **Account settings**, click **Projects**, and then select your project. Under **Repository** in the project details page, you can find your managed repo URL. 11 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Git/run-on-pull.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Why is Run on Pull request grayed out? 3 | description: "Use the GitHub auth method to enable Rull on Pull request" 4 | sidebar_label: 'Run on Pull request grayed out' 5 | id: run-on-pull 6 | --- 7 | 8 | If you're unable to enable Run on Pull requests, you'll want to make sure your existing repo was not added via the Deploy Key auth method. 9 | 10 | If it was added via a deploy key method, you'll want to use the [GitHub auth method](https://docs.getdbt.com/docs/cloud-installing-the-github-application) to enable CI in dbt Cloud. 11 | 12 | To go ahead and enable 'Run on Pull requests', you'll want to remove dbt Cloud from the Apps & Integration on GitHub and re-integrate it again via the GitHub app method. 13 | 14 | If you've tried the workaround above and are still experiencing this behavior - reach out to the Support team at support@getdbt.com and we'll be happy to help! 15 | 16 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Jinja/jinja-whitespace.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: My compiled SQL has a lot of spaces and new lines, how can I get rid of it? 3 | description: "Managing whitespace control" 4 | sidebar_label: 'Compiled sql has a lot of white space' 5 | id: jinja-whitespace 6 | --- 7 | 8 | This is known as "whitespace control". 9 | 10 | Use a minus sign (`-`, e.g. `{{- ... -}}`, `{%- ... %}`, `{#- ... -#}`) at the start or end of a block to strip whitespace before or after the block (more docs [here](https://jinja.palletsprojects.com/page/templates/#whitespace-control)). Check out the [tutorial on using Jinja](using-jinja#use-whitespace-control-to-tidy-up-compiled-code) for an example. 11 | 12 | Take caution: it's easy to fall down a rabbit hole when it comes to whitespace control! 13 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Jinja/quoting-column-names.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Why do I need to quote column names in Jinja? 3 | description: "Use quotes to pass string" 4 | sidebar_label: 'Why quote column names in Jinja' 5 | id: quoting-column-names 6 | --- 7 | 8 | In the [macro example](jinja-macros#macros) we passed the column name `amount` quotes: 9 | 10 | ```sql 11 | {{ cents_to_dollars('amount') }} as amount_usd 12 | ``` 13 | 14 | We have to use quotes to pass the _string_ `'amount'` to the macro. 15 | 16 | Without the quotes, the Jinja parser will look for a variable named `amount`. Since this doesn't exist, it will compile to nothing. 17 | 18 | Quoting in Jinja can take a while to get used to! The rule is that you're within a Jinja expression or statement (i.e. within `{% ... %}` or `{{ ... }}`), you'll need to use quotes for any arguments that are strings. 19 | 20 | Single and double quotes are equivalent in Jinja – just make sure you match them appropriately. 21 | 22 | And if you do need to pass a variable as an argument, make sure you [don't nest your curlies](dont-nest-your-curlies) 23 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Jinja/which-jinja-docs.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Which docs should I use when writing Jinja or creating a macro? 3 | description: "Useful Jinja docs" 4 | sidebar_label: 'Useful Jinja docs' 5 | id: which-jinja-docs 6 | --- 7 | 8 | If you are stuck with a Jinja issue, it can get confusing where to check for more information. We recommend you check (in order): 9 | 10 | 1. [Jinja's Template Designer Docs](https://jinja.palletsprojects.com/page/templates/): This is the best reference for most of the Jinja you'll use 11 | 2. [Our Jinja function reference](/reference/dbt-jinja-functions): This documents any additional functionality we've added to Jinja in dbt. 12 | 3. [Agate's table docs](https://agate.readthedocs.io/page/api/table.html): If you're operating on the result of a query, dbt will pass it back to you as an agate table. This means that the methods you call on the belong to the Agate library rather than Jinja or dbt. 13 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Models/available-configurations.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: What model configurations exist? 3 | description: "Learning about model configurations" 4 | sidebar_label: 'Model configurations' 5 | id: available-configurations 6 | --- 7 | You can also configure: 8 | 9 | * [tags](resource-configs/tags) to support easy categorization and graph selection 10 | * [custom schemas](resource-configs/schema) to split your models across multiple schemas 11 | * [aliases](resource-configs/alias) if your / name should differ from the filename 12 | * Snippets of SQL to run at the start or end of a model, known as [hooks](hooks-operations) 13 | * Warehouse-specific configurations for performance (e.g. `sort` and `dist` keys on Redshift, `partitions` on BigQuery) 14 | 15 | Check out the docs on [model configurations](model-configs) to learn more. 16 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Models/available-materializations.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: What materializations are available in dbt? 3 | description: "dbt uses 4 materializations" 4 | sidebar_label: 'Materializations available' 5 | id: available-materializations 6 | --- 7 | 8 | dbt ships with four materializations: `view`, `table`, `incremental` and `ephemeral`. Check out the documentation on [materializations](materializations) for more information on each of these options. 9 | 10 | You can also create your own [custom materializations](creating-new-materializations), if required however this is an advanced feature of dbt. 11 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Models/configurable-model-path.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Can I store my models in a directory other than the `models` directory in my project? 3 | description: "How to name models directory" 4 | sidebar_label: 'How to name models directory' 5 | id: configurable-model-path 6 | 7 | --- 8 | 9 | 10 | 11 | - **v1.0.0:** The config 'source-path' has been deprecated in favor of [`model-paths`](model-paths). 12 | 13 | 14 | 15 | By default, dbt expects your seed files to be located in the `models` subdirectory of your project. 16 | 17 | To change this, update the [model-paths](reference/project-configs/model-paths.md) configuration in your `dbt_project.yml` 18 | file, like so: 19 | 20 | 21 | 22 | ```yml 23 | model-paths: ["transformations"] 24 | ``` 25 | 26 | 27 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Models/create-a-schema.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Do I need to create my target schema before running dbt? 3 | description: "Target schema is checked or created for you before running dbt" 4 | sidebar_label: 'Target schema before running dbt' 5 | id: create-a-schema 6 | 7 | --- 8 | Nope! dbt will check if the schema exists when it runs. If the schema does not exist, dbt will create it for you. 9 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Models/insert-records.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: If models can only be `select` statements, how do I insert records? 3 | description: "Incremental models to insert records" 4 | sidebar_label: 'How to insert records' 5 | id: insert-records 6 | 7 | --- 8 | 9 | For those coming from an (Extract Transform Load) paradigm, there's often a desire to write transformations as `insert` and `update` statements. In comparison, dbt will wrap your `select` query in a `create table as` statement, which can feel counter-productive. 10 | 11 | * If you wish to use `insert` statements for performance reasons (i.e. to reduce data that is processed), consider [incremental models](/docs/build/incremental-models) 12 | * If you wish to use `insert` statements since your source data is constantly changing (e.g. to create "Type 2 Slowly Changing Dimensions"), consider [snapshotting your source data](/docs/build/sources#snapshotting-source-data-freshness), and building models on top of your snaphots. 13 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Models/model-custom-schemas.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Can I build my models in a schema other than my target schema or split my models across multiple schemas? 3 | description: "You can build models outside target schema" 4 | sidebar_label: 'How to build models in schema(s) outside target schema' 5 | id: model-custom-schemas 6 | 7 | --- 8 | 9 | Yes! Use the [schema](reference/resource-configs/schema.md) configuration in your `dbt_project.yml` file, or using a `config` block: 10 | 11 | 12 | 13 | ```yml 14 | 15 | name: jaffle_shop 16 | ... 17 | 18 | models: 19 | jaffle_shop: 20 | marketing: 21 | schema: marketing # seeds in the `models/mapping/ subdirectory will use the marketing schema 22 | ``` 23 | 24 | 25 | 26 | 27 | 28 | ```sql 29 | {{ 30 | config( 31 | schema='core' 32 | ) 33 | }} 34 | ``` 35 | 36 | 37 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Models/reference-models-in-another-project.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How can I reference models or macros in another project? 3 | description: "Use packages to add another project to your dbt project" 4 | sidebar_label: 'Reference models or macros in another project' 5 | id: reference-models-in-another-project 6 | 7 | --- 8 | 9 | You can use [packages](/docs/build/packages) to add another project to your dbt 10 | project, including other projects you've created. Check out the [docs](/docs/build/packages) 11 | for more information! 12 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Models/removing-deleted-models.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I remove deleted models from my data warehouse? 3 | description: "Remove deleted models directly in your data warehouse" 4 | sidebar_label: 'Removing deleted models from your warehouse' 5 | id: removing-deleted-models 6 | 7 | --- 8 | 9 | If you delete a model from your dbt project, dbt does not automatically drop the relation from your schema. This means that you can end up with extra objects in schemas that dbt creates, which can be confusing to other users. 10 | 11 | (This can also happen when you switch a model from being a or , to ephemeral) 12 | 13 | When you remove models from your dbt project, you should manually drop the related relations from your schema. 14 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Models/run-downtime.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: If I rerun dbt, will there be any downtime as models are rebuilt? 3 | description: "No downtime when re-building models in dbt " 4 | sidebar_label: 'Re-building models in dbt atomically' 5 | id: run-downtime 6 | 7 | --- 8 | Nope! The SQL that dbt generates behind the scenes ensures that any relations are replaced atomically (i.e. your business users won't experience any downtime). 9 | 10 | The implementation of this varies on each warehouse, check out the [logs](/docs/faqs/Runs/checking-logs) to see the SQL dbt is executing. 11 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Models/source-quotes.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: I need to use quotes to select from my source, what should I do? 3 | description: "Use quoting property to quote values" 4 | sidebar_label: 'How to quote values' 5 | id: source-quotes 6 | 7 | --- 8 | 9 | This is reasonably common on Snowflake in particular. 10 | 11 | By default, dbt will not quote the database, schema, or identifier for the source tables that you've specified. 12 | 13 | To force dbt to quote one of these values, use the [`quoting` property](resource-properties/quoting): 14 | 15 | 16 | 17 | ```yaml 18 | version: 2 19 | 20 | sources: 21 | - name: jaffle_shop 22 | database: raw 23 | quoting: 24 | database: true 25 | schema: true 26 | identifier: true 27 | 28 | tables: 29 | - name: order_items 30 | - name: orders 31 | # This overrides the `jaffle_shop` quoting config 32 | quoting: 33 | identifier: false 34 | ``` 35 | 36 | 37 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Models/unique-model-names.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Do model names need to be unique? 3 | description: "Unique model names to build dependencies" 4 | sidebar_label: 'Model names need to be unique' 5 | id: unique-model-names 6 | 7 | --- 8 | 9 | Yes! To build dependencies between models, you need to use the `ref` function. The `ref` function only takes one argument — the model name (i.e. the filename). As a result, these model names need to be unique, _even if they are in distinct folders_. 10 | 11 | Often, this question comes up because users want to give two models the same name in their warehouse, splitting them across separate schemas (e.g. `stripe.users` and `app.users`). Checkout the docs on [custom aliases](/docs/build/custom-aliases) and [custom schemas](/docs/build/custom-schemas) to achieve this. 12 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/add-a-seed.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Add a seed file 3 | id: add-a-seed 4 | description: Learn how to add a seed file to your project 5 | --- 6 | 7 | 1. Add a seed file: 8 | 9 | 10 | 11 | ```text 12 | country_code,country_name 13 | US,United States 14 | CA,Canada 15 | GB,United Kingdom 16 | ... 17 | ``` 18 | 19 | 20 | 21 | 2. Run `dbt seed` 22 | 3. Ref the model in a downstream model 23 | 24 | 25 | 26 | ```sql 27 | select * from {{ ref('country_codes') }} 28 | ``` 29 | 30 | 31 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/dbt-source-freshness.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Are the results of freshness stored anywhere? 3 | description: "How to access Source Freshness results" 4 | sidebar_label: 'Accessing Source Freshness results' 5 | id: dbt-source-freshness 6 | 7 | --- 8 | Yes! 9 | 10 | The `dbt source freshness` command will output a pass/warning/error status for each selected in the freshness snapshot. 11 | 12 | Additionally, dbt will write the freshness results to a file in the `target/` directory called `sources.json` by default. You can also override this destination, use the `-o` flag to the `dbt source freshness` command. 13 | 14 | After enabling source freshness within a job, configure [Artifacts](/docs/deploy/artifacts) in your **Project Details** page, which you can find by clicking the gear icon and then selecting **Account settings**. You can see the current status for source freshness by clicking **View Sources** in the job page. 15 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/debugging-jinja.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I debug my Jinja? 3 | description: "Using target folder or log function to debug Jinja" 4 | sidebar_label: 'Debugging Jinja' 5 | id: debugging-jinja 6 | 7 | --- 8 | 9 | You should get familiar with checking the compiled SQL in `target/compiled//` and the logs in `logs/dbt.log` to see what dbt is running behind the scenes. 10 | 11 | You can also use the [log](log) function to debug Jinja by printing objects to the command line. 12 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/delete-a-project.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I delete a project in dbt Cloud? 3 | description: "Deleting a project in dbt Cloud" 4 | sidebar_label: 'How to delete a project' 5 | id: delete-a-project 6 | 7 | --- 8 | To delete a project in dbt Cloud, you must be the account owner or have admin privileges. 9 | 10 | 1. From dbt Cloud, click the gear icon at the top right corner and select **Account Settings**. 11 | 12 | 13 | 14 | 2. In **Account Settings**, select **Projects**. Click the project you want to delete from the **Projects** page. 15 | 3. Click the edit icon in the lower right-hand corner of the **Project Details**. A **Delete** option will appear on the left side of the same details view. 16 | 4. Select **Delete**. Confirm the action to immediately delete the user without additional password prompts. There will be no account password prompt, and the project is deleted immediately after confirmation. Once a project is deleted, this action cannot be undone. 17 | 18 | 19 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/docs-for-multiple-projects.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Can I render docs for multiple projects? 3 | description: "Using packages to render docs for multiple projects" 4 | sidebar_label: 'Render docs for multiple projects' 5 | id: docs-for-multiple-projects 6 | 7 | --- 8 | 9 | Yes! To do this, you'll need to create a "super project" that lists each project as a dependent [package](/docs/build/packages) in a `packages.yml` file. Then run `dbt deps` to install the projects as packages, prior to running `dbt docs generate`. 10 | 11 | If you are going down the route of multiple projects, be sure to check out our advice [1](https://discourse.getdbt.com/t/should-i-have-an-organisation-wide-project-a-monorepo-or-should-each-work-flow-have-their-own/666) [2](https://discourse.getdbt.com/t/how-to-configure-your-dbt-repository-one-or-many/2121) on the topic. 12 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/exclude-table-from-freshness.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I exclude a table from a freshness snapshot? 3 | description: "Use null to exclude a table from a freshness snapshot" 4 | sidebar_label: 'Exclude a table from freshness snapshot ' 5 | id: exclude-table-from-freshness 6 | 7 | --- 8 | 9 | Some tables in a data source may be updated infrequently. If you've set a `freshness` property at the source level, this is likely to fail checks. 10 | 11 | To work around this, you can set the table's freshness to null (`freshness: null`) to "unset" the freshness for a particular table: 12 | 13 | 14 | 15 | ```yaml 16 | 17 | version: 2 18 | 19 | sources: 20 | - name: jaffle_shop 21 | database: raw 22 | 23 | freshness: 24 | warn_after: {count: 12, period: hour} 25 | error_after: {count: 24, period: hour} 26 | 27 | loaded_at_field: _etl_loaded_at 28 | 29 | tables: 30 | - name: orders 31 | - name: product_skus 32 | freshness: null # do not check freshness for this table 33 | ``` 34 | 35 | 36 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/extract-load-support.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Does dbt offer extract and load functionality? 3 | description: "dbt is a transformation tool" 4 | sidebar_label: 'Does dbt offer extract and load functionality?' 5 | id: transformation-tool 6 | --- 7 | dbt is a transformation tool. It is _not_ designed for extract or load functionality, and dbt Labs strongly recommends against using dbt in this way. Support is not provided for extract or load functionality. 8 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/multiple-resource-yml-files.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Should I use separate files to declare resource properties, or one large file? 3 | description: "Structuring your files and project" 4 | sidebar_label: 'Ways to structure your project/files' 5 | id: multiple-resource-yml-files 6 | 7 | --- 8 | It's up to you: 9 | - Some folks find it useful to have one file per model (or source / snapshot / seed etc) 10 | - Some find it useful to have one per directory, documenting and testing multiple models in one file 11 | 12 | Choose what works for your team. We have more recommendations in our guide on [structuring dbt project](https://docs.getdbt.com/guides/best-practices/how-we-structure/1-guide-overview). 13 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/properties-not-in-config.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Can I add tests and descriptions in a config block? 3 | description: "Types of properties defined in config blocks" 4 | sidebar_label: 'Types of properties defined in config blocks' 5 | id: properties-not-in-config 6 | 7 | --- 8 | 9 | dbt has the ability to define node configs in `.yml` files, in addition to `config()` blocks and `dbt_project.yml`. But the reverse isn't always true: there are some things in `.yml` files that can _only_ be defined there. 10 | 11 | Certain properties are special, because: 12 | - They have a unique Jinja rendering context 13 | - They create new project resources 14 | - They don't make sense as hierarchical configuration 15 | - They're older properties that haven't yet been redefined as configs 16 | 17 | These properties are: 18 | - [`description`](resource-properties/description) 19 | - [`tests`](resource-properties/tests) 20 | - [`docs`](/reference/resource-configs/docs) 21 | - `columns` 22 | - [`quote`](resource-properties/quote) 23 | - [`source` properties](source-properties) (e.g. `loaded_at_field`, `freshness`) 24 | - [`exposure` properties](exposure-properties) (e.g. `type`, `maturity`) 25 | - [`macro` properties](macro-properties) (e.g. `arguments`) 26 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/resource-yml-name.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: If I can name these files whatever I'd like, what should I name them? 3 | description: "Recommendations on how to name your files" 4 | sidebar_label: 'How to name your files' 5 | id: resource-yml-name 6 | 7 | --- 8 | It's up to you! Here's a few options: 9 | - Default to the existing terminology: `schema.yml` (though this does make it hard to find the right file over time) 10 | - Use the same name as your directory (assuming you're using sensible names for your directories) 11 | - If you test and document one model (or seed, snapshot, macro etc.) per file, you can give it the same name as the model (or seed, snapshot, macro etc.) 12 | 13 | Choose what works for your team. We have more recommendations in our guide on [structuring dbt projects](https://docs.getdbt.com/guides/best-practices/how-we-structure/1-guide-overview). 14 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/schema-yml-name.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Does my `.yml` file containing tests and descriptions need to be named `schema.yml`? 3 | description: "Naming your Tests and Descriptions files" 4 | sidebar_label: 'How to name your tests and descriptions files' 5 | id: schema-yml-name 6 | 7 | --- 8 | No! You can name this file whatever you want (including `whatever_you_want.yml`), so long as: 9 | * The file is in your `models/` directory¹ 10 | * The file has `.yml` extension 11 | 12 | Check out the [docs](configs-and-properties) for more information. 13 | 14 | ¹If you're declaring properties for seeds, snapshots, or macros, you can also place this file in the related directory — `seeds/`, `snapshots/` and `macros/` respectively. 15 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/separate-profile.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Why are profiles stored outside of my project? 3 | description: "Profiles stored outside version control" 4 | sidebar_label: 'Profiles stored outside project' 5 | id: separate-profile 6 | 7 | --- 8 | 9 | Profiles are stored separately to dbt projects to avoid checking credentials into version control. Database credentials are extremely sensitive information and should **never be checked into version control**. 10 | 11 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/source-has-bad-name.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: What if my source is in a poorly named schema or table? 3 | description: "Use schema and identifier properities to define names" 4 | sidebar_label: 'Source is in a poorly-named scheme or table' 5 | id: source-has-bad-name 6 | 7 | --- 8 | 9 | By default, dbt will use the `name:` parameters to construct the source reference. 10 | 11 | If these names are a little less-than-perfect, use the [schema](resource-properties/schema) and [identifier](identifier) properties to define the names as per the database, and use your `name:` property for the name that makes sense! 12 | 13 | 14 | 15 | ```yml 16 | version: 2 17 | 18 | sources: 19 | - name: jaffle_shop 20 | schema: postgres_backend_public_schema 21 | database: raw 22 | tables: 23 | - name: orders 24 | identifier: api_orders 25 | 26 | 27 | ``` 28 | 29 | 30 | 31 | 32 | In a downstream model: 33 | ```sql 34 | select * from {{ source('jaffle_shop', 'orders') }} 35 | ``` 36 | 37 | Will get compiled to: 38 | ```sql 39 | select * from raw.postgres_backend_public_schema.api_orders 40 | ``` 41 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/source-in-different-database.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: What if my source is in a different database to my target database? 3 | description: "Use database property to define source in a diff database" 4 | sidebar_label: 'Source is in a different database to target database' 5 | id: source-in-different-database 6 | 7 | --- 8 | 9 | Use the [`database` property](resource-properties/database) to define the database that the source is in. 10 | 11 | 12 | 13 | ```yml 14 | version: 2 15 | 16 | sources: 17 | - name: jaffle_shop 18 | database: raw 19 | tables: 20 | - name: orders 21 | - name: customers 22 | 23 | ``` 24 | 25 | 26 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/structure-a-project.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How should I structure my project? 3 | description: "Recommendations on how to structure project" 4 | sidebar_label: 'How to structure project' 5 | id: structure-a-project 6 | 7 | --- 8 | 9 | There's no one best way to structure a project! Every organization is unique. 10 | 11 | If you're just getting started, check out how we (dbt Labs) [structure our dbt projects](https://docs.getdbt.com/guides/best-practices/how-we-structure/1-guide-overview). 12 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/which-materialization.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Which materialization should I use for my model? 3 | description: "Recommendations on materializations to use for models" 4 | sidebar_label: 'What materializations to use' 5 | id: which-materialization 6 | 7 | --- 8 | Start out with views, and then change models to tables when required for performance reasons (i.e. downstream queries have slowed). 9 | 10 | Check out the [docs on materializations](materializations) for advice on when to use each . 11 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/which-schema.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How did dbt choose which schema to build my models in? 3 | description: "You can change your target schema in your Environment Settings." 4 | sidebar_label: 'dbt builds models in your target schema' 5 | id: which-schema 6 | 7 | --- 8 | By default, dbt builds models in your target schema. To change your target schema: 9 | * If you're developing in **dbt Cloud**, these are set for each user when you first use a development environment. 10 | * If you're developing with the **dbt CLI**, this is the `schema:` parameter in your `profiles.yml` file. 11 | 12 | If you wish to split your models across multiple schemas, check out the docs on [using custom schemas](/docs/build/custom-schemas). 13 | 14 | Note: on BigQuery, `dataset` is used interchangeably with `schema`. 15 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/why-so-many-macros.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Why does my dbt output have so many macros in it? 3 | description: "Your dbt project includes many macros." 4 | sidebar_label: 'dbt project has a lot of macros' 5 | id: why-so-many-macros 6 | 7 | --- 8 | 9 | The output of a dbt run counts over 100 macros in your project! 10 | 11 | ```shell 12 | $ dbt run 13 | Running with dbt=0.17.0 14 | Found 1 model, 0 tests, 0 snapshots, 0 analyses, 138 macros, 0 operations, 0 seed files, 0 sources 15 | ``` 16 | 17 | This is because dbt ships with its own project, which also includes macros! You can learn more about this [here](https://discourse.getdbt.com/t/did-you-know-dbt-ships-with-its-own-project/764). 18 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/why-version-2.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Why do model and source yml files always start with `version: 2`?" 3 | description: ".yml file structure more extensible with version 2." 4 | sidebar_label: 'Why does yml file start with version 2' 5 | id: why-version-2 6 | 7 | --- 8 | 9 | 10 | 11 | Once upon a time, the structure of these `.yml` files was very different (s/o to anyone who was using dbt back then!). Adding `version: 2` allowed us to make this structure more extensible. 12 | 13 | Resource yml files do not currently require this config. We only support `version: 2` if it's specified. Although we do not expect to update yml files to `version: 3` soon, having this config will make it easier for us to introduce new structures in the future 14 | 15 | 16 | 17 | 18 | 19 | Once upon a time, the structure of these `.yml` files was very different (s/o to anyone who was using dbt back then!). Adding `version: 2` allowed us to make this structure more extensible. 20 | 21 | Currently, Version 2 is the only supported version for these files. We kept `version:` around as a required key so that in the future, if we need to introduce a new structure for these files, we'll be able to do this more easily. 22 | 23 | 24 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Project/yaml-file-extension.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Can I use a yaml file extension? 3 | description: "dbt will only search for files with a `.yml` file extension" 4 | sidebar_label: '.yml file extension search' 5 | id: yaml-file-extension 6 | 7 | --- 8 | 9 | No. At present, dbt will only search for files with a `.yml` file extension. In a future release of dbt, dbt will also search for files with a `.yaml` file extension. 10 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Runs/checking-logs.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How can I see the SQL that dbt is running? 3 | description: "Review logs to check the sql dbt is running" 4 | sidebar_label: 'Reviewing sql that dbt runs' 5 | id: checking-logs 6 | 7 | --- 8 | 9 | To check out the SQL that dbt is running, you can look in: 10 | 11 | * dbt Cloud: 12 | * Within the run output, click on a model name, and then select "Details" 13 | * dbt CLI: 14 | * The `target/compiled/` directory for compiled `select` statements 15 | * The `target/run/` directory for compiled `create` statements 16 | * The `logs/dbt.log` file for verbose logging. 17 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Runs/failed-prod-run.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: What happens if one of my runs fails? 3 | description: "Set up notifications to debug failed runs" 4 | sidebar_label: 'Notifications to debug failed runs' 5 | id: failed-prod-run 6 | 7 | --- 8 | 9 | If you're using dbt Cloud, we recommend setting up email and Slack notifications (`Account Settings > Notifications`) for any failed runs. Then, debug these runs the same way you would debug any runs in development. 10 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Runs/failed-tests.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: One of my tests failed, how can I debug it? 3 | description: "You can debug failed tests by finding the SQL" 4 | sidebar_label: 'Debug failed tests' 5 | id: failed-tests 6 | 7 | --- 8 | To debug a failing test, find the SQL that dbt ran by: 9 | 10 | * dbt Cloud: 11 | * Within the test output, click on the failed test, and then select "Details" 12 | 13 | * dbt CLI: 14 | * Open the file path returned as part of the error message. 15 | * Navigate to the `target/compiled/schema_tests` directory for all compiled test queries 16 | 17 | Copy the SQL into a query editor (in dbt Cloud, you can paste it into a new `Statement`), and run the query to find the records that failed. 18 | 19 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Runs/run-downstream-of-seed.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I run models downstream of a seed? 3 | description: "You run models downstread using model selection syntax" 4 | sidebar_label: 'Run models downstream of a seed' 5 | id: run-downstream-of-seed 6 | 7 | --- 8 | 9 | You can run models downstream of a seed using the [model selection syntax](node-selection/syntax), and treating the seed like a model. 10 | 11 | For example, the following would run all models downstream of a seed named `country_codes`: 12 | 13 | ```shell 14 | $ dbt run --select country_codes+ 15 | ``` 16 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Runs/run-one-model.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I run one model at a time? 3 | description: "Use select flags to run one model at a time" 4 | sidebar_label: 'Run one model at a time' 5 | id: run-one-model 6 | 7 | --- 8 | 9 | To run one model, use the `--select` flag (or `-s` flag), followed by the name of the model: 10 | 11 | ```shell 12 | $ dbt run --select customers 13 | ``` 14 | 15 | Check out the [model selection syntax documentation](node-selection/syntax) for more operators and examples. 16 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Runs/run-one-snapshot.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I run one snapshot at a time? 3 | description: "Use select flags to run one snapshot at a time" 4 | sidebar_label: 'Run one snapshot at a time' 5 | id: run-one-snapshot 6 | 7 | --- 8 | 9 | To run one snapshot, use the `--select` flag, followed by the name of the snapshot: 10 | 11 | ```shell 12 | $ dbt snapshot --select order_snapshot 13 | ``` 14 | 15 | Check out the [model selection syntax documentation](node-selection/syntax) for more operators and examples. 16 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Runs/running-models-downstream-of-source.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I run models downstream of one source? 3 | description: "Use source selector to run models downstream of a source" 4 | sidebar_label: 'Run models downstream of one source' 5 | id: running-model-downstream-of-source 6 | 7 | --- 8 | To run models downstream of a source, use the `source:` selector: 9 | 10 | ```shell 11 | $ dbt run --select source:jaffle_shop+ 12 | ``` 13 | (You can also use the `-s` shorthand here instead of `--select`) 14 | 15 | To run models downstream of one source : 16 | 17 | ```shell 18 | $ dbt run --select source:jaffle_shop.orders+ 19 | ``` 20 | 21 | Check out the [model selection syntax](node-selection/syntax) for more examples! 22 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Runs/snapshot-frequency.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How often should I run the snapshot command? 3 | description: "Snapshot command intended to run on hourly/daily schedule " 4 | sidebar_label: 'Snapshot command schedule' 5 | id: snapshot-frequency 6 | 7 | --- 8 | 9 | Snapshots are a batch-based approach to [change data capture](https://en.wikipedia.org/wiki/Change_data_capture). The `dbt snapshot` command must be run on a schedule to ensure that changes to tables are actually recorded! While individual use-cases may vary, snapshots are intended to be run between hourly and daily. If you find yourself snapshotting more frequently than that, consider if there isn't a more appropriate way to capture changes in your source data tables. 10 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Seeds/build-one-seed.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I build one seed at a time? 3 | description: "Use select flag to build one seed at a time" 4 | sidebar_label: "Build one seed at a time" 5 | id: build-one-seed 6 | --- 7 | 8 | As of v0.16.0, you can use a `--select` option with the `dbt seed` command, like so: 9 | 10 | ```shell 11 | 12 | $ dbt seed --select country_codes 13 | 14 | ``` 15 | 16 | There is also an `--exclude` option. 17 | 18 | Check out more in the [model selection syntax](node-selection/syntax) documentation. 19 | 20 | Prior to v0.16.0, there was no way to build one seed at a time. 21 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Seeds/leading-zeros-in-seed.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I preserve leading zeros in a seed? 3 | description: "Use column types to include leading zeros in seed" 4 | sidebar_label: 'Include leading zeroes in your seed file' 5 | id: leading-zeros-in-seed 6 | 7 | --- 8 | 9 | If you need to preserve leading zeros (for example in a zipcode or mobile number): 10 | 11 | 1. v0.16.0 onwards: Include leading zeros in your seed file, and use the `column_types` [configuration](reference/resource-configs/column_types.md) with a varchar datatype of the correct length. 12 | 2. Prior to v0.16.0: Use a downstream model to pad the leading zeros using SQL, for example: `lpad(zipcode, 5, '0')` 13 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Seeds/load-raw-data-with-seed.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Can I use seeds to load raw data? 3 | description: "Use seeds to load business specific logic" 4 | sidebar_label: 'Seed data files requirements' 5 | id: load-raw-data-with-seed 6 | 7 | --- 8 | 9 | Seeds should **not** be used to load raw data (for example, large CSV exports from a production database). 10 | 11 | Since seeds are version controlled, they are best suited to files that contain business-specific logic, for example a list of country codes or user IDs of employees. 12 | 13 | Loading CSVs using dbt's seed functionality is not performant for large files. Consider using a different tool to load these CSVs into your . 14 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Seeds/seed-custom-schemas.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Can I build my seeds in a schema other than my target schema or can I split my seeds across multiple schemas? 3 | description: "Use schema config in your dbt_project.yml file" 4 | sidebar_label: 'Build seeds in a schema outside target schema' 5 | id: seed-custom-schemas 6 | 7 | --- 8 | 9 | Yes! Use the [schema](reference/resource-configs/schema.md) configuration in your `dbt_project.yml` file. 10 | 11 | 12 | 13 | ```yml 14 | 15 | name: jaffle_shop 16 | ... 17 | 18 | seeds: 19 | jaffle_shop: 20 | schema: mappings # all seeds in this project will use the schema "mappings" by default 21 | marketing: 22 | schema: marketing # seeds in the "seeds/marketing/" subdirectory will use the schema "marketing" 23 | ``` 24 | 25 | 26 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Seeds/seed-datatypes.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I set a datatype for a column in my seed? 3 | description: "Use column_types to set a datatype" 4 | sidebar_label: 'Set a datatype for a column in seed' 5 | id: seed-datatypes 6 | 7 | --- 8 | dbt will infer the datatype for each column based on the data in your CSV. 9 | 10 | You can also explicitly set a datatype using the `column_types` [configuration](reference/resource-configs/column_types.md) like so: 11 | 12 | 13 | 14 | ```yml 15 | seeds: 16 | jaffle_shop: # you must include the project name 17 | warehouse_locations: 18 | +column_types: 19 | zipcode: varchar(5) 20 | ``` 21 | 22 | 23 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Seeds/seed-hooks.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Do hooks run with seeds? 3 | description: "Run hooks with seeds and configure them in project.yml file" 4 | sidebar_label: "Use hooks to run with seeds" 5 | id: seed-hooks 6 | 7 | --- 8 | 9 | Yes! The following hooks are available: 10 | 11 | - [pre-hooks & post-hooks](reference/resource-configs/pre-hook-post-hook.md) 12 | - [on-run-start & on-run-end hooks](reference/project-configs/on-run-start-on-run-end.md) 13 | 14 | Configure these in your `dbt_project.yml` file. 15 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Snapshots/snapshot-hooks.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Do hooks run with snapshots? 3 | description: "Run hooks with snapshots" 4 | sidebar_label: 'Use hooks to run with snapshots' 5 | id: snapshot-hooks 6 | 7 | --- 8 | 9 | Yes! The following hooks are available for snapshots: 10 | - [pre-hooks](/reference/resource-configs/pre-hook-post-hook/) 11 | - [post-hooks](/reference/resource-configs/pre-hook-post-hook/) 12 | - [on-run-start](/reference/project-configs/on-run-start-on-run-end/) 13 | - [on-run-end](/reference/project-configs/on-run-start-on-run-end/) 14 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Snapshots/snapshot-schema-changes.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: What happens if I add new columns to my snapshot query? 3 | description: "Reconcile changes when adding new columns in snapshot query" 4 | sidebar_label: 'Snapshot column changes' 5 | id: snapshot-schema-changes 6 | 7 | --- 8 | When the columns of your source query changes, dbt will attempt to reconcile this change in the destination snapshot . dbt does this by: 9 | 10 | 1. Creating new columns from the source query in the destination table 11 | 2. Expanding the size of string types where necessary (eg. `varchar`s on Redshift) 12 | 13 | dbt _will not_ delete columns in the destination snapshot table if they are removed from the source query. It will also not change the type of a column beyond expanding the size of varchar columns. That is, if a `string` column is changed to a `date` column in the snapshot source query, dbt will not attempt to change the type of the column in the destination table. 14 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Snapshots/snapshotting-freshness-for-one-source.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I snapshot freshness for one source only? 3 | description: "Use select flag to snapshot for specific sources" 4 | sidebar_label: 'Snapshot freshness for specific sources.' 5 | id: snapshotting-freshness-for-one-source 6 | 7 | --- 8 | 9 | 10 | Use the `--select` flag to snapshot freshness for specific sources. Eg: 11 | 12 | ```shell 13 | # Snapshot freshness for all Jaffle Shop tables: 14 | $ dbt source freshness --select source:jaffle_shop 15 | 16 | # Snapshot freshness for a particular source : 17 | $ dbt source freshness --select source:jaffle_shop.orders 18 | 19 | # Snapshot freshness for multiple particular source tables: 20 | $ dbt source freshness --select source:jaffle_shop.orders source:jaffle_shop.customers 21 | ``` 22 | 23 | See the [`source freshness` command reference](commands/source) for more information. 24 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Tests/available-tests.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: What tests are available for me to use in dbt? 3 | description: "Types of tests to use in dbt" 4 | sidebar_label: 'Available test to use in dbt' 5 | id: available-tests 6 | 7 | --- 8 | Out of the box, dbt ships with the following tests: 9 | 10 | * `unique` 11 | * `not_null` 12 | * `accepted_values` 13 | * `relationships` (i.e. referential integrity) 14 | 15 | You can also write your own [custom schema tests](/docs/build/tests). 16 | 17 | Some additional custom schema tests have been open-sourced in the [dbt-utils package](https://github.com/dbt-labs/dbt-utils/tree/0.2.4/#schema-tests), check out the docs on [packages](/docs/build/packages) to learn how to make these tests available in your project. 18 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Tests/configurable-data-path.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Can I store my seeds in a directory other than the `seeds` directory in my project? 3 | description: "Where to store seeds in a directory" 4 | sidebar_label: 'How to name seeds directory' 5 | id: configurable-data-path 6 | 7 | --- 8 | 9 | 10 | 11 | - **v1.0.0:** The config 'data-paths' has been deprecated in favor of [`seed-paths`](seed-paths). 12 | 13 | 14 | 15 | By default, dbt expects your seed files to be located in the `seeds` subdirectory 16 | of your project. 17 | 18 | To change this, update the [seed-paths](reference/project-configs/seed-paths.md) configuration in your `dbt_project.yml` 19 | file, like so: 20 | 21 | 22 | 23 | ```yml 24 | seed-paths: ["custom_seeds"] 25 | ``` 26 | 27 | 28 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Tests/configurable-data-test-path.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Can I store my tests in a directory other than the `tests` directory in my project? 3 | description: "Where to store tests in a directory" 4 | sidebar_label: 'How to name tests directory' 5 | id: configurable-data-test-path 6 | 7 | --- 8 | By default, dbt expects your singular test files to be located in the `tests` subdirectory of your project, and generic test definitions to be located in `tests/generic` or `macros`. 9 | 10 | To change this, update the [test-paths](reference/project-configs/test-paths.md) configuration in your `dbt_project.yml` 11 | file, like so: 12 | 13 | 14 | 15 | ```yml 16 | test-paths: ["my_cool_tests"] 17 | ``` 18 | 19 | 20 | 21 | Then, you can define generic tests in `my_cool_tests/generic/`, and singular tests everywhere else in `my_cool_tests/`. 22 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Tests/custom-test-thresholds.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Can I set test failure thresholds? 3 | description: "Use configs to set custom failure thresholds in tests" 4 | sidebar_label: 'How to set failure thresholds in test' 5 | id: custom-test-thresholds 6 | 7 | --- 8 | 9 | As of `v0.20.0`, you can use the `error_if` and `warn_if` configs to set custom failure thresholds in your tests. For more details, see [reference](reference/resource-configs/severity) for more information. 10 | 11 | For dbt `v0.19.0` and earlier, you could try these possible solutions: 12 | 13 | * Setting the [severity](resource-properties/tests#severity) to `warn`, or: 14 | * Writing a [custom generic test](custom-generic-tests) that accepts a threshold argument ([example](https://discourse.getdbt.com/t/creating-an-error-threshold-for-schema-tests/966)) 15 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Tests/multiple-test-files.m: -------------------------------------------------------------------------------- 1 | --- 2 | title: Do all my tests go in one file? 3 | description: "Structuring files for tests" 4 | sidebar_label: 'How to structure files for tests' 5 | id: multiple-test-files 6 | 7 | --- 8 | No! You can use as many files as you want! Some folks find it useful to have one file per model, we tend to have one per directory. 9 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Tests/recommended-tests.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: What tests should I add to my project? 3 | description: "Recommended tests for project" 4 | sidebar_label: 'Recommended tests for project' 5 | id: recommended-tests 6 | 7 | --- 8 | We recommend that every model has a test on a , that is, a column that is `unique` and `not_null`. 9 | 10 | We also recommend that you test any assumptions on your source data. For example, if you believe that your payments can only be one of three payment methods, you should test that assumption regularly — a new payment method may introduce logic errors in your SQL. 11 | 12 | In advanced dbt projects, we recommend using [sources](/docs/build/sources) and running these source data-integrity tests against the sources rather than models. 13 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Tests/test-one-model.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I test one model at a time? 3 | description: "Use select flag to test one model at a time" 4 | sidebar_label: 'Testing one model at a time' 5 | id: test-one-model 6 | 7 | --- 8 | 9 | Running tests on one model looks very similar to running a model: use the `--select` flag (or `-s` flag), followed by the name of the model: 10 | 11 | ```shell 12 | dbt test --select customers 13 | ``` 14 | 15 | Check out the [model selection syntax documentation](node-selection/syntax) for full syntax, and [test selection examples](test-selection-examples) in particular. 16 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Tests/testing-seeds.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I test and document seeds? 3 | description: "Use a schema file to test and document seeds" 4 | sidebar_label: 'Test and document seeds' 5 | id: testing-seeds 6 | 7 | --- 8 | 9 | The `seeds:` key is new in 0.16.0. Prior to this, use a `models:` key instead. 10 | 11 | To test and document seeds, use a [schema file](configs-and-properties) and nest the configurations under a `seeds:` key 12 | 13 | ## Example 14 | 15 | 16 | 17 | ```yml 18 | version: 2 19 | 20 | seeds: 21 | - name: country_codes 22 | description: A mapping of two letter country codes to country names 23 | columns: 24 | - name: country_code 25 | tests: 26 | - unique 27 | - not_null 28 | - name: country_name 29 | tests: 30 | - unique 31 | - not_null 32 | ``` 33 | 34 | 35 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Tests/testing-sources.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I run tests on just my sources? 3 | description: "Use the select source command to test sources" 4 | sidebar_label: 'Run tests on all sources' 5 | id: testing-sources 6 | 7 | --- 8 | 9 | To run tests on all sources, use the following command: 10 | 11 | ```shell 12 | $ dbt test --select source:* 13 | ``` 14 | 15 | (You can also use the `-s` shorthand here instead of `--select`) 16 | 17 | To run tests on one source (and all of its tables): 18 | 19 | ```shell 20 | $ dbt test --select source:jaffle_shop 21 | ``` 22 | 23 | And, to run tests on one source only: 24 | 25 | ```shell 26 | $ dbt test --select source:jaffle_shop.orders 27 | ``` 28 | 29 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Tests/when-to-test.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: When should I run my tests? 3 | description: "Run tests in production or when writing new code" 4 | sidebar_label: 'When to run tests' 5 | id: when-to-test 6 | 7 | --- 8 | 9 | You should run your tests whenever you are writing new code (to ensure you haven't broken any existing models by changing SQL), and whenever you run your transformations in production (to ensure that your assumptions about your source data are still valid). 10 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Troubleshooting/access_token_error.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: I'm receiving an `access_token` error when trying to run queries in the IDE. 3 | description: "Reauthenticate warehouse when seeing `access_token` error" 4 | sidebar_label: '`access_token` error in the IDE' 5 | id: access_token_error 6 | 7 | --- 8 | 9 | If you're seeing the below error when you try to run queries in the IDE, have no fear - we'll do our best to get you unstuck with the following steps! 10 | 11 | `access_token` 12 | 13 | In order to resolve the issue, you'll want to: 14 | 15 | 1. Go to you Profile (upper right corner of dbt Cloud) 16 | 2. Click on the correct warehouse connection under "Credentials" 17 | 3. Click "Connect Snowflake Account" green button in the "Development Credentials" section. This drives you through reauthentication through the SSO flow. 18 | 19 | If you've tried the step above and are still experiencing this behavior - reach out to the Support team at support@getdbt.com and we'll be happy to help! 20 | 21 | 22 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Troubleshooting/dispatch-could-not-find-package.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "[Error] Could not find my_project package" 3 | description: "Macros missing from packages" 4 | sidebar_label: 'Could not find package error' 5 | id: dispatch-could-not-find-package 6 | 7 | --- 8 | 9 | If a package name is included in the `search_order` of a project-level `dispatch` config, dbt expects that package to contain macros which are viable candidates for dispatching. If an included package does not contain _any_ macros, dbt will raise an error like: 10 | 11 | ```shell 12 | Compilation Error 13 | In dispatch: Could not find package 'my_project' 14 | ``` 15 | 16 | This does not mean the package or root project is missing—it means that any macros from it are missing, and so it is missing from the search spaces available to `dispatch`. 17 | 18 | If you've tried the step above and are still experiencing this behavior - reach out to the Support team at support@getdbt.com and we'll be happy to help! 19 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Troubleshooting/nonetype-ide-error.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: I'm receiving a NoneType object has no attribute error in the IDE? 3 | description: "Copy SSH key to your warehouse" 4 | sidebar_label: 'NoneType error in the IDE' 5 | id: nonetype-ide-error 6 | 7 | --- 8 | 9 | If you're unable to access the IDE due to the below error message, we'll do our best to get you unstuck with the below steps! 10 | 11 | ```shell 12 | NoneType object has no attribute 13 | enumerate_fields' 14 | ``` 15 | 16 | Usually this errors indicates that you tried connecting your database via [SSH tunnel](https://docs.getdbt.com/docs/dbt-cloud/cloud-configuring-dbt-cloud/connecting-your-database#connecting-via-an-ssh-tunnel). If you're seeing this error, double-check you have supplied the following items: 17 | 18 | - the hostname 19 | - username 20 | - port of bastion server 21 | 22 | If you've tried the step above and are still experiencing this behavior - reach out to the Support team at support@getdbt.com and we'll be happy to help! 23 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Troubleshooting/partial-parsing-error.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "I'm getting a \"Partial parsing enabled: 1 files deleted, 0 files added, 2 files changed\" compilation error in dbt Cloud?" 3 | description: "Delete partial_parse file" 4 | sidebar_label: 'partial_parse error in the IDE' 5 | id: partial-parsing-error 6 | 7 | --- 8 | 9 | If you're receiving this error, try deleting the `target/partial_parse.msgpack` file from your project and refresh your IDE. 10 | 11 | If you've tried the workaround above and are still experiencing this behavior - reach out to the Support team at support@getdbt.com and we'll be happy to help! 12 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Troubleshooting/runtime-packages.yml.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Why am I receiving a Runtime Error in my packages? 3 | description: "Update dbt_utils package in packages.yml file" 4 | sidebar_label: 'Runtime error in packages.yml file' 5 | id: runtime-packages.yml 6 | 7 | --- 8 | 9 | If you're receiving the runtime error below in your packages.yml folder, it may be due to an old version of your dbt_utils package that isn't compatible with your current dbt Cloud version. 10 | 11 | ```shell 12 | Running with dbt=xxx 13 | Runtime Error 14 | Failed to read package: Runtime Error 15 | Invalid config version: 1, expected 2 16 | Error encountered in dbt_utils/dbt_project.yml 17 | ``` 18 | 19 | Try updating the old version of the dbt_utils package in your packages.yml to the latest version found in the [dbt hub](https://hub.getdbt.com/dbt-labs/dbt_utils/latest/): 20 | 21 | ```shell 22 | packages: 23 | - package: dbt-labs/dbt_utils 24 | 25 | version: xxx 26 | ``` 27 | 28 | If you've tried the workaround above and are still experiencing this behavior - reach out to the Support team at support@getdbt.com and we'll be happy to help! 29 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Troubleshooting/sql-errors.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: What happens if the SQL in my query is bad or I get a database error? 3 | description: "Use error message and logs to debug" 4 | sidebar_label: 'How to debug SQL or database error' 5 | id: sql-errors 6 | 7 | --- 8 | 9 | 10 | If there's a mistake in your SQL, dbt will return the error that your database returns. 11 | 12 | ```shell 13 | $ dbt run --select customers 14 | Running with dbt=0.15.0 15 | Found 3 models, 9 tests, 0 snapshots, 0 analyses, 133 macros, 0 operations, 0 seed files, 0 sources 16 | 17 | 14:04:12 | Concurrency: 1 threads (target='dev') 18 | 14:04:12 | 19 | 14:04:12 | 1 of 1 START view model dbt_alice.customers.......................... [RUN] 20 | 14:04:13 | 1 of 1 ERROR creating view model dbt_alice.customers................. [ERROR in 0.81s] 21 | 14:04:13 | 22 | 14:04:13 | Finished running 1 view model in 1.68s. 23 | 24 | Completed with 1 error and 0 warnings: 25 | 26 | Database Error in model customers (models/customers.sql) 27 | Syntax error: Expected ")" but got identifier `your-info-12345` at [13:15] 28 | compiled SQL at target/run/jaffle_shop/customers.sql 29 | 30 | Done. PASS=0 WARN=0 ERROR=1 SKIP=0 TOTAL=1 31 | ``` 32 | 33 | Any models downstream of this model will also be skipped. Use the error message and the [compiled SQL](/faqs/Runs/checking-logs) to debug any errors. 34 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Troubleshooting/unused-model-configurations.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: I got an "unused model configurations" error message, what does this mean? 3 | description: "Model configurations error message" 4 | sidebar_label: 'unused model configurations error' 5 | id: unused-model-configurations 6 | 7 | --- 8 | You might have forgotten to nest your configurations under your project name, or you might be trying to apply configurations to a directory that doesn't exist. 9 | 10 | Check out this [article](https://discourse.getdbt.com/t/faq-i-got-an-unused-model-configurations-error-message-what-does-this-mean/112) to understand more. 11 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Warehouse/bq-impersonate-service-account-setup.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How can I set up the right permissions in BigQuery? 3 | description: "Use service account to set up permissions in BigQuery" 4 | sidebar_label: 'Setting up permissions in BigQuery"' 5 | id: bq-impersonate-service-account-setup 6 | 7 | --- 8 | 9 | To use this functionality, first create the service account you want to 10 | impersonate. Then grant users that you want to be able to impersonate 11 | this service account the `roles/iam.serviceAccountTokenCreator` role on 12 | the service account resource. Then, you also need to grant the service 13 | account the same role on itself. This allows it to create short-lived 14 | tokens identifying itself, and allows your human users (or other service 15 | accounts) to do the same. More information on this scenario is available 16 | [here](https://cloud.google.com/iam/docs/understanding-service-accounts#directly_impersonating_a_service_account). 17 | 18 | Once you've granted the appropriate permissions, you'll need to enable 19 | the [IAM Service Account Credentials API](https://console.cloud.google.com/apis/library/iamcredentials.googleapis.com). 20 | Enabling the API and granting the role are eventually consistent operations, 21 | taking up to 7 minutes to fully complete, but usually fully propagating within 60 22 | seconds. Give it a few minutes, then add the `impersonate_service_account` 23 | option to your BigQuery profile configuration. 24 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Warehouse/bq-impersonate-service-account-why.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Why would I want to impersonate a service account? 3 | description: "Build models using dedicated service account" 4 | sidebar_label: 'Why impersonate a service account' 5 | id: bq-impersonate-service-account-why 6 | 7 | --- 8 | 9 | You may want your models to be built using a dedicated service account that has 10 | elevated access to read or write data to the specified project or dataset. 11 | Typically, this requires you to create a service account key for running under 12 | development or on your CI server. By specifing the email address of the service 13 | account you want to build models as, you can use [Application Default Credentials](https://cloud.google.com/sdk/gcloud/reference/auth/application-default) or the 14 | service's configured service account (when running in GCP) to assume the identity 15 | of the service account with elevated permissions. 16 | 17 | This allows you to reap the advantages of using federated identity for developers 18 | (via ADC) without needing to grant individual access to read and write data 19 | directly, and without needing to create separate service account and keys for 20 | each user. It also allows you to completely eliminate the need for service 21 | account keys in CI as long as your CI is running on GCP (Cloud Build, Jenkins, 22 | GitLab/Github Runners, etc). 23 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Warehouse/database-privileges.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: What privileges does my database user need to use dbt? 3 | description: "Database privileges to use dbt" 4 | sidebar_label: 'Database privileges to use dbt' 5 | id: database-privileges 6 | 7 | --- 8 | Your user will need to be able to: 9 | * `select` from raw data in your warehouse (i.e. data to be transformed) 10 | * `create` schemas, and therefore create tables/views within that 11 | schema¹ 12 | * read system views to generate documentation (i.e. views in 13 | `information_schema`) 14 | 15 | On Postgres, Redshift, and Snowflake, use a series of `grants` to ensure that 16 | your user has the correct privileges. 17 | 18 | On BigQuery, use the "BigQuery User" role to assign these privileges. 19 | 20 | --- 21 | ¹Alternatively, a separate user can create a schema for the dbt user, and then grant the user privileges to create within this schema. We generally recommend granting your dbt user the ability to create schemas, as it is less complicated to implement. 22 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Warehouse/loading-data.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How do I load data into my warehouse? 3 | description: "Recommendations on tools to load data to warehouse" 4 | sidebar_label: 'Recommendations on tools to get data into your warehouse' 5 | id: loading-data 6 | 7 | --- 8 | dbt assumes that you already have a copy of your data, in your . We recommend you use an off-the-shelf tool like [Stitch](https://www.stitchdata.com/) or [Fivetran](https://fivetran.com/) to get data into your warehouse. 9 | 10 | **Can dbt be used to load data?** 11 | 12 | No, dbt does not extract or load data. It focuses on the transformation step only. 13 | -------------------------------------------------------------------------------- /context-sources/docs/faqs/Warehouse/sample-profiles.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: What should my profiles.yml file look like for my warehouse? 3 | description: "Go to Profile Set up to see the structure of a profile" 4 | sidebar_label: 'Structure of profile for each warehouse' 5 | id: sample-profiles 6 | 7 | --- 8 | 9 | The structure of a profile looks different on each warehouse. Check out the [Supported Data Platforms](supported-data-platforms) page, and navigate to the `Profile Setup` section for your warehouse. 10 | -------------------------------------------------------------------------------- /context-sources/docs/guides/best-practices/materializations/materializations-guide-7-conclusion.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Conclusion" 3 | id: materializations-guide-7-conclusion 4 | slug: 7-conclusion 5 | description: Learn how to utilize materializations in dbt. 6 | displayText: Materializations best practices 7 | hoverSnippet: Learn how to utilize materializations in dbt. 8 | --- 9 | 10 | You're now following best practices in your project, and have optimized the materializations of your DAG. You’re equipped with the 3 main materializations that cover almost any analytics engineering situation! 11 | 12 | There are more configs and materializations available, as well as specific materializations for certain platforms and adapters — and like everything with dbt, materializations are extensible, meaning you can create your own [custom materializations](guides/advanced/creating-new-materializations) for your needs. So this is just the beginning of what you can do with these powerful configurations. 13 | 14 | For the vast majority of users and companies though, tables, views, and incremental models will handle everything you can throw at them. Develop your intuition and expertise for these materializations, and you’ll be well on your way to tackling advanced analytics engineering problems. 15 | -------------------------------------------------------------------------------- /context-sources/docs/guides/dbt-ecosystem/adapter-development/adapter-development: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /context-sources/docs/guides/legacy/videos.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Videos 🎥" 3 | id: "videos" 4 | --- 5 | 6 | Check out some cool videos about using and deploying dbt! 7 | 8 | ## dbt tutorial (February, 2017) 9 | 10 | 11 | 12 | ## dbt docs demo with GitLab (September, 2018) 13 | 14 | -------------------------------------------------------------------------------- /context-sources/docs/guides/migration/tools/migrating-from-stored-procedures/6-migrating-from-stored-procedures-conclusion.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Putting it all together 3 | id: 6-migrating-from-stored-procedures-conclusion 4 | --- 5 | 6 | The techniques shared above are useful ways to get started converting the individual DML statements that are often found in stored procedures. Using these types of patterns, legacy procedural code can be rapidly transitioned to dbt models that are much more readable, maintainable, and benefit from software engineering best practices like DRY principles. Additionally, once transformations are rewritten as dbt models, it becomes much easier to test the transformations to ensure that the data being used downstream is high-quality and trustworthy. 7 | -------------------------------------------------------------------------------- /context-sources/docs/guides/migration/versions/11-Older versions/upgrading-to-0-12-0.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Upgrading to 0.12.0" 3 | id: "upgrading-to-0-12-0" 4 | --- 5 | 6 | ## End of support 7 | 8 | Support for the `repositories:` block in `dbt_project.yml` (deprecated in 0.10.0) was removed. 9 | In order to install packages in your dbt project, you must use [a `packages.yml` file](/docs/build/packages#how-do-i-add-a-package-to-my-project). 10 | -------------------------------------------------------------------------------- /context-sources/docs/guides/orchestration/custom-cicd-pipelines/5-something-to-consider.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Something to Consider 3 | id: 5-something-to-consider 4 | --- 5 | 6 | Running dbt Cloud jobs through a CI/CD pipeline is a form of job orchestration. If you also run jobs using dbt Cloud’s built in scheduler, you now have 2 orchestration tools running jobs. The risk with this is that you could run into conflicts - you can imagine a case where you are triggering a pipeline on certain actions and running scheduled jobs in dbt Cloud, you would probably run into job clashes. The more tools you have, the more you have to make sure everything talks to each other. 7 | 8 | That being said, if **the only reason you want to use pipelines is for adding a lint check or run on merge**, you might decide the pros outweigh the cons, and as such you want to go with a hybrid approach. Just keep in mind that if two processes try and run the same job at the same time, dbt Cloud will queue the jobs and run one after the other. It’s a balancing act but can be accomplished with diligence to ensure you’re orchestrating jobs in a manner that does not conflict. -------------------------------------------------------------------------------- /context-sources/docs/reference/advanced-config-usage.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Advanced configuration usage 3 | sidebar_label: Advanced usage 4 | --- 5 | ## Alternative config block syntax 6 | 7 | Some configurations may contain characters (e.g. dashes) that cannot be parsed as a jinja argument. For example, the following would return an error: 8 | 9 | ```sql 10 | {{ config( 11 | post-hook="grant select on {{ this }} to role reporter", 12 | materialized='table' 13 | ) }} 14 | 15 | select ... 16 | ``` 17 | 18 | While dbt provides an alias for any core configurations (e.g. you should use `pre_hook` instead of `pre-hook` in a config block), your dbt project may contain custom configurations without aliases. 19 | 20 | If you want to specify these configurations inside of a model, use the altenative config block syntax: 21 | 22 | 23 | 24 | 25 | ```sql 26 | {{ 27 | config({ 28 | "post-hook": "grant select on {{ this }} to role reporter", 29 | "materialized": "table" 30 | }) 31 | }} 32 | 33 | 34 | select ... 35 | ``` 36 | 37 | 38 | 39 | 43 | -------------------------------------------------------------------------------- /context-sources/docs/reference/analysis-properties.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Analysis properties 3 | --- 4 | 5 | We recommend you define analysis properties in your `analyses/` directory, which is illustrated in the [`analysis-paths`](analysis-paths) configuration. 6 | 7 | You can name these files `whatever_you_want.yml`, and nest them arbitrarily deeply in subfolders within the `analyses/` or `models/` directory. 8 | 9 | 10 | 11 | ```yml 12 | version: 2 13 | 14 | analyses: 15 | - name: # required 16 | [description](description): 17 | [docs](/reference/resource-configs/docs): 18 | show: true | false 19 | config: 20 | [tags](resource-configs/tags): | [] 21 | columns: 22 | - name: 23 | [description](description): 24 | - name: ... # declare properties of additional columns 25 | 26 | - name: ... # declare properties of additional analyses 27 | 28 | ``` 29 | 30 | 31 | 32 | 33 | 34 | 35 | * `v0.16.0`: The ability to declare analysis properties was introduced. 36 | 37 | 38 | -------------------------------------------------------------------------------- /context-sources/docs/reference/artifacts/other-artifacts.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Other Artifacts 3 | --- 4 | 5 | ### index.html 6 | 7 | **Produced by:** [`docs generate`](commands/cmd-docs) 8 | 9 | This file is the skeleton of the [auto-generated dbt documentation website](documentation). The contents of the site are populated by the [manifest](manifest-json) and [catalog](catalog-json). 10 | 11 | Note: the source code for `index.json` comes from the [dbt-docs repo](https://github.com/dbt-labs/dbt-docs). Head over there if you want to make a bug report, suggestion, or contribution relating to the documentation site. 12 | 13 | ### partial_parse.msgpack 14 | 15 | **Produced by:** [manifest commands](manifest-json) + [`parse`](parse) 16 | 17 | This file is used to store a compressed representation of files dbt has parsed. If you have [partial parsing](parsing#partial-parsing) enabled, dbt will use this file to identify the files that have changed and avoid re-parsing the rest. 18 | 19 | ### graph.gpickle 20 | 21 | **Produced by:** commands supporting [node selection](node-selection/syntax) 22 | 23 | Stores the networkx representation of the dbt resource DAG. 24 | -------------------------------------------------------------------------------- /context-sources/docs/reference/commands/clean.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "clean" 3 | id: "clean" 4 | --- 5 | 6 | 7 | 8 | - **v1.0.0:** `dbt_modules` has been replaced by `dbt_packages` by default for the [clean-target](clean-targets) for packages. 9 | 10 | 11 | 12 | `dbt clean` is a utility function that deletes all folders specified in the `clean-targets` list specified in `dbt_project.yml`. You can use this to delete the `dbt_packages` and `target` directories. 13 | 14 | To avoid complex permissions issues and potentially deleting crucial aspects of the remote file system without access to fix them, this command does not work when interfacing with the RPC server that powers the dbt Cloud IDE. Instead, when working in dbt Cloud, the `dbt deps` command cleans before it installs packages automatically. The `target` folder can be manually deleted from the sidebar file tree if needed. 15 | -------------------------------------------------------------------------------- /context-sources/docs/reference/commands/compile.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "compile" 3 | id: "compile" 4 | --- 5 | 6 | `dbt compile` generates executable SQL from source `model`, `test`, and `analysis` files. You can find these compiled SQL files in the `target/` directory of your dbt project. 7 | 8 | The `compile` command is useful for: 9 | 10 | 1. Visually inspecting the compiled output of model files. This is useful for validating complex jinja logic or macro usage. 11 | 2. Manually running compiled SQL. While debugging a model or schema test, it's often useful to execute the underlying `select` statement to find the source of the bug. 12 | 3. Compiling `analysis` files. Read more about analysis files [here](/docs/build/analyses). 13 | 14 | It is _not_ a pre-requisite of `dbt run`. 15 | -------------------------------------------------------------------------------- /context-sources/docs/reference/commands/debug.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "debug" 3 | id: "debug" 4 | --- 5 | 6 | `dbt debug` is a utility function to test the database connection and show information for debugging purposes. Not to be confused with the `--debug` option which increases verbosity. 7 | 8 | The `--config-dir` option to `dbt debug` will show the configured location for the `profiles.yml` file and exit: 9 | 10 | ```text 11 | $ dbt debug --config-dir 12 | To view your profiles.yml file, run: 13 | 14 | open /Users/alice/.dbt 15 | ``` 16 | -------------------------------------------------------------------------------- /context-sources/docs/reference/commands/run-operation.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "run-operation" 3 | id: "run-operation" 4 | --- 5 | 6 | ### Overview 7 | 8 | The `dbt run-operation` command is used to invoke a macro. For usage information, consult the docs on [operations](hooks-operations#operations). 9 | 10 | ### Usage 11 | ``` 12 | $ dbt run-operation {macro} --args '{args}' 13 | {macro} Specify the macro to invoke. dbt will call this macro 14 | with the supplied arguments and then exit 15 | --args ARGS Supply arguments to the macro. This dictionary will be 16 | mapped to the keyword arguments defined in the 17 | selected macro. This argument should be a YAML string, 18 | eg. '{my_variable: my_value}' 19 | ``` 20 | ### Command line examples 21 | 22 | Example 1: 23 | 24 | `$ dbt run-operation grant_select --args '{role: reporter}'` 25 | 26 | Example 2: 27 | 28 | `$ dbt run-operation clean_stale_models --args '{days: 7, dry_run: True}'` 29 | -------------------------------------------------------------------------------- /context-sources/docs/reference/commands/seed.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "seed" 3 | id: "seed" 4 | --- 5 | 6 | 7 | 8 | - **v1.0.0:** The default config for this command will now be `seed-paths` instead of `data-paths`. 9 | 10 | 11 | 12 | 13 | The `dbt seed` command will load `csv` files located in the `seed-paths` directory of your dbt project into your . 14 | 15 | 16 | ### Selecting seeds to run 17 | 18 | Added in v0.16.0 19 | 20 | Specific seeds can be run using the `--select` flag to `dbt seed`. Example: 21 | 22 | ``` 23 | $ dbt seed --select country_codes 24 | Found 2 models, 3 tests, 0 archives, 0 analyses, 53 macros, 0 operations, 2 seed files 25 | 26 | 14:46:15 | Concurrency: 1 threads (target='dev') 27 | 14:46:15 | 28 | 14:46:15 | 1 of 1 START seed file analytics.country_codes........................... [RUN] 29 | 14:46:15 | 1 of 1 OK loaded seed file analytics.country_codes....................... [INSERT 3 in 0.01s] 30 | 14:46:16 | 31 | 14:46:16 | Finished running 1 seed in 0.14s. 32 | 33 | ``` 34 | -------------------------------------------------------------------------------- /context-sources/docs/reference/commands/snapshot.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "snapshot" 3 | id: "snapshot" 4 | --- 5 | 6 | The `dbt snapshot` command executes the [Snapshots](snapshots) defined in your project. 7 | 8 | dbt will looks for Snapshots in the `snapshot-paths` paths defined in your `dbt_project.yml` file. By default, the `snapshot-paths` path is `snapshots/`. 9 | 10 | **Usage:** 11 | ``` 12 | $ dbt snapshot --help 13 | usage: dbt snapshot [-h] [--profiles-dir PROFILES_DIR] 14 | [--profile PROFILE] [--target TARGET] 15 | [--vars VARS] [--bypass-cache] 16 | [--threads THREADS] 17 | [--select SELECTOR [SELECTOR ...]] 18 | [--exclude EXCLUDE [EXCLUDE ...]] 19 | 20 | optional arguments: 21 | --select SELECTOR [SELECTOR ...] 22 | Specify the snapshots to include in the run. 23 | --exclude EXCLUDE [EXCLUDE ...] 24 | Specify the snapshots to exclude in the run. 25 | ``` -------------------------------------------------------------------------------- /context-sources/docs/reference/commands/test.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "test" 3 | id: "test" 4 | --- 5 | 6 | `dbt test` runs tests defined on models, sources, snapshots, and seeds. It expects that you have already created those resources through the appropriate commands. 7 | 8 | The tests to run can be selected using the `--select` flag discussed [here](node-selection/syntax). 9 | 10 | ```bash 11 | # run tests for one_specific_model 12 | dbt test --select one_specific_model 13 | 14 | # run tests for all models in package 15 | dbt test --select some_package.* 16 | 17 | # run only tests defined singularly 18 | dbt test --select test_type:singular 19 | 20 | # run only tests defined generically 21 | dbt test --select test_type:generic 22 | 23 | # run singular tests limited to one_specific_model 24 | dbt test --select one_specific_model,test_type:singular 25 | 26 | # run generic tests limited to one_specific_model 27 | dbt test --select one_specific_model,test_type:generic 28 | ``` 29 | 30 | For more information on writing tests, see the [Testing Documentation](/docs/build/tests). 31 | -------------------------------------------------------------------------------- /context-sources/docs/reference/config-datatypes.yml: -------------------------------------------------------------------------------- 1 | 2 | - name: string 3 | 4 | - name: [string] 5 | description: An array of strings 6 | 7 | - name: version 8 | description: A valid semantic version number, for example, 0.0.1 9 | 10 | - name: directorypath 11 | description: A valid path to a directory, relative to the `dbt_project.yml` file. For example, `seeds` would point to a subdirectory named `seeds` 12 | 13 | - name: [directorypath] 14 | description: An array of directory paths 15 | 16 | - name: boolean 17 | render_as: "true | false" # where `render_as` is not specified, it should be rendered in a dbt_project.yml as the name 18 | 19 | - name: version-range 20 | description: A valid version range, e.g. `0.1.0`, or `>=0.1.0` or `>=0.1.0<0.2.0` 21 | 22 | - name: sql-statement 23 | description: A SQL statement that can be executed against a warehouse. 24 | 25 | - name: [sql-statements] 26 | description: An array of sql-statements 27 | 28 | - name: column_name 29 | description: The identifier of a column that can be selected from a relation. 30 | 31 | - name: column_name_or_expression # this is for like the loaded_at field, but I'll fill it in when we get there. Is it bad that this is snake_case whereas others are kebab-case? 32 | 33 | - name: datatype 34 | description: A valid datatype according for a data warehouse, e.g. `varchar` 35 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/as_bool.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "as_bool" 3 | id: "as_bool" 4 | --- 5 | 6 | The `as_bool` Jinja filter will coerce Jinja-compiled output into a boolean 7 | value (`True` or `False`), or return an error if it cannot be represented 8 | as a bool. 9 | 10 | ### Usage: 11 | 12 | In the example below, the `as_bool` filter is used to coerce a Jinja 13 | expression to enable or disable a set of models based on the `target`. 14 | 15 | 16 | 17 | ```yml 18 | models: 19 | my_project: 20 | for_export: 21 | enabled: "{{ (target.name == 'prod') | as_bool }}" 22 | ``` 23 | 24 | 25 | 26 | 27 | 28 | * `v0.17.1`: Native rendering is disabled by default. The `as_bool` filter was 29 | introduced. 30 | 31 | 32 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/as_native.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "as_native" 3 | id: "as_native" 4 | --- 5 | 6 | The `as_native` Jinja filter will coerce Jinja-compiled output into its 7 | Python native representation according to [`ast.literal_eval`](https://docs.python.org/3/library/ast.html#ast.literal_eval). 8 | The result can be any Python native type (set, list, tuple, dict, etc). 9 | 10 | To render boolean and numeric values, it is recommended to use [`as_bool`](as_bool) 11 | and [`as_number`](as_number) instead. 12 | 13 | :::danger Proceed with caution 14 | Unlike `as_bool` and `as_number`, `as_native` will return a rendered value 15 | regardless of the input type. Ensure that your inputs match expectations. 16 | ::: 17 | 18 | 19 | 20 | * `v0.17.1`: Native rendering is disabled by default. The `as_native` filter was 21 | introduced. 22 | 23 | 24 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/as_number.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "as_number" 3 | id: "as_number" 4 | --- 5 | 6 | The `as_number` Jinja filter will coerce Jinja-compiled output into a numeric 7 | value (integer or float), or return an error if it cannot be represented as 8 | a number. 9 | 10 | ### Usage 11 | 12 | In the example below, the `as_number` filter is used to coerce an environment 13 | variables into a numeric value to dynamically control the connection port. 14 | 15 | 16 | 17 | ```yml 18 | my_profile: 19 | outputs: 20 | dev: 21 | type: postgres 22 | port: "{{ env_var('PGPORT') | as_number }}" 23 | ``` 24 | 25 | 26 | 27 | 28 | 29 | * `v0.17.1`: Native rendering is disabled by default. The `as_number` filter was 30 | introduced. 31 | 32 | 33 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/dbt-project-yml-context.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "dbt_project.yml Context" 3 | id: "dbt-project-yml-context" 4 | --- 5 | 6 | 7 | 8 | The compilation context of the `dbt_project.yml` file is well-defined as 9 | of dbt v0.16.0 10 | 11 | 12 | 13 | The following context variables and methods are available when configuring 14 | resources in the `dbt_project.yml` file. This applies to the `models:`, `seeds:`, 15 | and `snapshots:` keys in the `dbt_project.yml` file. 16 | 17 | **Available context variables:** 18 | - [target](target) 19 | - [env_var](env_var) 20 | - [vars](var) (_Note: only variables defined with `--vars` are available_) 21 | - [builtins](builtins) 22 | - [dbt_version](dbt_version) 23 | 24 | 25 | ### Example configuration 26 | 27 | 28 | 29 | ```yml 30 | name: my_project 31 | version: 1.0.0 32 | 33 | # Configure the models in models/facts/ to be materialized as views 34 | # in development and tables in production/CI contexts 35 | 36 | models: 37 | my_project: 38 | facts: 39 | +materialized: "{{ 'view' if target.name == 'dev' else 'table' }}" 40 | ``` 41 | 42 | 43 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/dbt_version.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "dbt_version" 3 | id: "dbt_version" 4 | --- 5 | 6 | New in 0.16.0 7 | 8 | The `dbt_version` variable returns the installed version of dbt that is 9 | currently running. It can be used for debugging or auditing purposes. 10 | 11 | ### Example Usage 12 | 13 | 14 | 15 | ```sql 16 | {% macro get_version() %} 17 | 18 | {% do log("The installed version of dbt is: " ~ dbt_version, info=true) %} 19 | 20 | {% endmacro %} 21 | ``` 22 | 23 | 24 | 25 | 26 | ``` 27 | $ dbt run-operation get_version 28 | The installed version of dbt is 0.16.0 29 | ``` 30 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/doc.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "doc" 3 | id: "doc" 4 | --- 5 | 6 | The `doc` function is used to reference docs blocks in the description field of schema.yml files. It is analogous to the `ref` function. For more information, consult the [Documentation guide](documentation). 7 | 8 | Usage: 9 | 10 | 11 | 12 | ```jinja2 13 | 14 | {% docs orders %} 15 | 16 | # docs 17 | - go 18 | - here 19 | 20 | {% enddocs %} 21 | ``` 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | ```yaml 30 | 31 | version: 2 32 | models: 33 | - name: orders 34 | description: "{{ doc('orders') }}" 35 | ``` 36 | 37 | 38 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/exceptions.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "exceptions" 3 | id: "exceptions" 4 | --- 5 | 6 | The `exceptions` namespace can be used to raise warnings and errors in dbt userspace. 7 | 8 | ## raise_compiler_error 9 | 10 | The `exceptions.raise_compiler_error` method will raise a compiler error with the provided message. This is typically only useful in macros or materializations when invalid arguments are provided by the calling model. Note that throwing an exception will cause a model to fail, so please use this variable with care! 11 | 12 | __Example usage__: 13 | 14 | 15 | 16 | ```sql 17 | {% if number < 0 or number > 100 %} 18 | {{ exceptions.raise_compiler_error("Invalid `number`. Got: " ~ number) }} 19 | {% endif %} 20 | ``` 21 | 22 | 23 | 24 | ## warn 25 | 26 | The `exceptions.warn` method will raise a compiler warning with the provided message, but any model will still be successful and be treated as a PASS. If the `--warn-error` flag is provided to dbt, then this warning will be elevated to an exception, which is raised. 27 | 28 | __Example usage__: 29 | 30 | 31 | 32 | ```sql 33 | {% if number < 0 or number > 100 %} 34 | {% do exceptions.warn("Invalid `number`. Got: " ~ number) %} 35 | {% endif %} 36 | ``` 37 | 38 | 39 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/fromjson.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "fromjson" 3 | id: "fromjson" 4 | --- 5 | 6 | The `fromjson` context method can be used to deserialize a json string into a Python object primitive, eg. a `dict` or `list`. 7 | 8 | __Args__: 9 | * `string`: The json string to deserialize (required) 10 | * `default`: A default value to return if the `string` argument cannot be deserialized (optional) 11 | 12 | ### Usage: 13 | ``` 14 | {% set my_json_str = '{"abc": 123}' %} 15 | {% set my_dict = fromjson(my_json_str) %} 16 | 17 | {% do log(my_dict['abc']) %} 18 | ``` 19 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/fromyaml.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "fromyaml" 3 | id: "fromyaml" 4 | --- 5 | 6 | The `fromyaml` context method can be used to deserialize a yaml string into a Python object primitive, eg. a `dict` or `list`. 7 | 8 | __Args__: 9 | * `string`: The yaml string to deserialize (required) 10 | * `default`: A default value to return if the `string` argument cannot be deserialized (optional) 11 | 12 | ### Usage: 13 | ``` 14 | {% set my_yml_str -%} 15 | 16 | dogs: 17 | - good 18 | - bad 19 | 20 | {%- endset %} 21 | 22 | {% set my_dict = fromyaml(my_yml_str) %} 23 | 24 | {% do log(my_dict['dogs'], info=true) %} 25 | -- ["good", "bad"] 26 | 27 | {% do my_dict['dogs'].pop() %} 28 | {% do log(my_dict['dogs'], info=true) %} 29 | -- ["good"] 30 | ``` 31 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/invocation_id.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "invocation_id" 3 | id: "invocation_id" 4 | --- 5 | 6 | The `invocation_id` outputs a UUID generated for this dbt command. This value is 7 | useful when auditing or analyzing dbt invocation metadata. 8 | 9 | 10 | 11 | - Support for `query-comment` is new in v0.16.1 12 | - Automatic inclusion of `invocation_id` in both dbt artifacts and BigQuery jobs is new in v0.19.0 13 | 14 | 15 | 16 | If available, the `invocation_id` is: 17 | - available in the compilation context of [`query-comment`](query-comment) 18 | - included in the `info` dictionary in dbt [events and logs](events-logging#info) 19 | - included in the `metadata` dictionary in [dbt artifacts](dbt-artifacts#common-metadata) 20 | - included as a label in all BigQuery jobs that dbt originates 21 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/local-md5.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "local_md5" 3 | id: "local_md5" 4 | --- 5 | 6 | The `local_md5` context variable calculates an [MD5 hash](https://en.wikipedia.org/wiki/MD5) of the given string. The string `local_md5` emphasizes that the hash is calculated _locally_, in the dbt-Jinja context. This variable is typically useful for advanced use cases. For example, when you generate unique identifiers within custom materialization or operational logic, you can either avoid collisions between temporary relations or identify changes by comparing checksums. 7 | 8 | It is different than the `md5` SQL function, supported by many SQL dialects, which runs remotely in the data platform. You want to always use SQL hashing functions when generating surrogate keys. 9 | 10 | Usage: 11 | ```sql 12 | -- source 13 | {%- set value_hash = local_md5("hello world") -%} 14 | '{{ value_hash }}' 15 | 16 | -- compiled 17 | '5eb63bbbe01eeed093cb22bb8f5acdc3' 18 | ``` 19 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/log.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "log" 3 | id: "log" 4 | --- 5 | 6 | __Args__: 7 | 8 | * `msg`: The message (string) to log 9 | * `info`: If False, write to the log file. If True, write to both the log file and stdout (default=False) 10 | 11 | Logs a line to either the log file or stdout. 12 | 13 | ([Source on Github](https://github.com/dbt-labs/dbt-core/blob/HEAD/core/dbt/context/base.py#L432)) 14 | 15 | ```sql 16 | 17 | {% macro some_macro(arg1, arg2) %} 18 | 19 | {{ log("Running some_macro: " ~ arg1 ~ ", " ~ arg2) }} 20 | 21 | {% endmacro %} 22 | ``` 23 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/model.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "model" 3 | id: "model" 4 | --- 5 | 6 | `model` is the dbt [graph object](graph) (or node) for the current model. It can be used to: 7 | - Access `config` settings, say, in a post-hook 8 | - Access the path to the model 9 | 10 | For example: 11 | ```jinja 12 | {% if model.config.materialization = 'view' %} 13 | {{ log(model.name ~ " is a view.", info=True) }} 14 | {% endif %} 15 | ``` 16 | 17 | To view the contents of `model` for a given model, use [log()](/reference/dbt-jinja-functions/log) 18 | to print the full contents: 19 | 20 | ```jinja 21 | {{ log(model, info=True) }} 22 | ``` 23 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/print.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "print" 3 | id: "print" 4 | --- 5 | 6 | ## About print 7 | 8 | Use the `print()` function when you want to print messages to both the log file and standard output (stdout). 9 | 10 | When used in conjunction with the `QUIET` global config, which suppresses non-error logs, you will only see error logs and the print messages in stdout. For more information, see [Global configs](/reference/global-configs). 11 | 12 | ## Example 13 | 14 | ```sql 15 | {% macro some_macro(arg1, arg2) %} 16 | {{ print("Running some_macro: " ~ arg1 ~ ", " ~ arg2) }} 17 | {% endmacro %} 18 | ``` 19 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/profiles-yml-context.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "profiles.yml Context" 3 | id: "profiles-yml-context" 4 | --- 5 | 6 | The following context variables and methods are available when configuring 7 | resources in the `profiles.yml` file. 8 | 9 | **Available context variables:** 10 | - [env_var](env_var) 11 | - [vars](var) (_Note: only variables defined with `--vars` are available_) 12 | 13 | ### Example usage 14 | 15 | 16 | 17 | ```yml 18 | jaffle_shop: 19 | target: dev 20 | outputs: 21 | dev: 22 | type: redshift 23 | host: "{{ env_var('DBT_HOST') }}" 24 | user: "{{ env_var('DBT_USER') }}" 25 | password: "{{ env_var('DBT_PASS') }}" 26 | port: 5439 27 | dbname: analytics 28 | schema: dbt_dbanin 29 | threads: 4 30 | ``` 31 | 32 | 33 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/project_name.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "project_name" 3 | id: "project_name" 4 | --- 5 | 6 | New in 0.16.0 7 | 8 | The `project_name` context variable returns the `name` for the root-level project 9 | which is being run by dbt. This variable can be used to defer execution to a 10 | root-level project macro if one exists. 11 | 12 | ### Example Usage 13 | 14 | 15 | 16 | ```sql 17 | /* 18 | This macro vacuums tables in a Redshift database. If a macro exists in the 19 | root-level project called `get_tables_to_vacuum`, this macro will call _that_ 20 | macro to find the tables to vacuum. If the macro is not defined in the root 21 | project, this macro will use a default implementation instead. 22 | */ 23 | 24 | {% macro vacuum_tables() %} 25 | 26 | {% set root_project = context[project_name] %} 27 | {% if root_project.get_tables_to_vacuum %} 28 | {% set tables = root_project.get_tables_to_vacuum() %} 29 | {% else %} 30 | {% set tables = redshift.get_tables_to_vacuum() %} 31 | {% endif %} 32 | 33 | {% for table in tables %} 34 | {% do redshift.vacuum_table(table) %} 35 | {% endfor %} 36 | 37 | {% endmacro %} 38 | ``` 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/return.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "return" 3 | id: "return" 4 | --- 5 | 6 | __Args__: 7 | 8 | * `data`: The data to return to the caller 9 | 10 | The `return` function can be used in macros to return data to the caller. The type of the data (dict, list, int, etc) will be preserved through the `return` call. 11 | 12 | 13 | 14 | ```sql 15 | {% macro get_data() %} 16 | 17 | {{ return([1,2,3]) }} 18 | 19 | {% endmacro %} 20 | ``` 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | ```sql 29 | 30 | select 31 | -- getdata() returns a list! 32 | {% for i in get_data() %} 33 | {{ i }} 34 | {% if not loop.last %},{% endif %} 35 | {% endfor %} 36 | ``` 37 | 38 | 39 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/run_started_at.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "run_started_at" 3 | id: "run_started_at" 4 | --- 5 | 6 | `run_started_at` outputs the timestamp that this run started, e.g. `2017-04-21 01:23:45.678`. 7 | 8 | The `run_started_at` variable is a Python `datetime` object. As of 0.9.1, the timezone of this variable 9 | defaults to UTC. 10 | 11 | 12 | 13 | ```sql 14 | select 15 | '{{ run_started_at.strftime("%Y-%m-%d") }}' as date_day 16 | 17 | from ... 18 | ``` 19 | 20 | 21 | 22 | To modify the timezone of this variable, use the the `pytz` module: 23 | 24 | 25 | 26 | ```sql 27 | select 28 | '{{ run_started_at.astimezone(modules.pytz.timezone("America/New_York")) }}' as run_started_est 29 | 30 | from ... 31 | ``` 32 | 33 | 34 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/schema.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "schema" 3 | id: "schema" 4 | --- 5 | 6 | The schema that the model is configured to be materialized in. This is typically the same as `model['schema']`. 7 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/schemas.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "schemas" 3 | id: "schemas" 4 | --- 5 | 6 | `schemas` is a variable available in an `on-run-end` hook, representing a list of schemas that dbt built objects in on this run. 7 | 8 | If you do not use [custom schemas](/docs/build/custom-schemas), `schemas` will evaluate to your target schema, e.g. `['dbt_alice']`. If you use custom schemas, it will include these as well, e.g. `['dbt_alice', 'dbt_alice_marketing', 'dbt_alice_finance']`. 9 | 10 | The `schemas` variable is useful for granting privileges to all schemas that dbt builds relations in, like so (note this is Redshift specific syntax): 11 | 12 | 13 | 14 | 15 | ```yaml 16 | ... 17 | 18 | on-run-end: 19 | - "{% for schema in schemas%}grant usage on schema {{ schema }} to group reporter;{% endfor%}" 20 | - "{% for schema in schemas %}grant select on all tables in schema {{ schema }} to group reporter;{% endfor%}" 21 | - "{% for schema in schemas %}alter default privileges in schema {{ schema }} grant select on tables to group reporter;{% endfor %}" 22 | ``` 23 | 24 | 25 | 26 | 27 | 28 | :::info Want more in-depth instructions on the recommended way to grant privileges? 29 | 30 | We've written a full discourse article [here](https://discourse.getdbt.com/t/the-exact-grant-statements-we-use-in-a-dbt-project/430) 31 | 32 | ::: 33 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/source.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "source" 3 | id: "source" 4 | --- 5 | ```sql 6 | select * from {{ source(source_name, table_name) }} 7 | ``` 8 | 9 | ## Definition 10 | 11 | This function: 12 | - Returns a [Relation](dbt-classes#relation) for a [source](/docs/build/sources) 13 | - Creates dependencies between a source and the current model, which is useful for documentation and model selection 14 | - Compiles to the full object name in the database 15 | 16 | ## Related guides 17 | - [Using sources](/docs/build/sources) 18 | 19 | ## Arguments 20 | * `source_name`: The `name:` defined under a `sources:` key 21 | * `table_name`: The `name:` defined under a `tables:` key 22 | 23 | ## Example 24 | 25 | Consider a source defined like so: 26 | 27 | 28 | 29 | ```yaml 30 | version: 2 31 | 32 | sources: 33 | - name: jaffle_shop # this is the source_name 34 | database: raw 35 | 36 | tables: 37 | - name: customers # this is the table_name 38 | - name: orders 39 | ``` 40 | 41 | 42 | 43 | Select from the source in a model: 44 | 45 | 46 | 47 | ```sql 48 | select 49 | ... 50 | 51 | from {{ source('jaffle_shop', 'customers') }} 52 | 53 | left join {{ source('jaffle_shop', 'orders') }} using (customer_id) 54 | 55 | ``` 56 | 57 | 58 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/this.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "this" 3 | id: "this" 4 | --- 5 | 6 | `this` is the database representation of the current model. It is useful when: 7 | - Defining a `where` statement within [incremental models](/docs/build/incremental-models) 8 | - Using [pre or post hooks](pre-hook-post-hook) 9 | 10 | `this` is a [Relation](dbt-classes#relation), and as such, properties such as `{{ this.database }}` and `{{ this.schema }}` compile as expected. 11 | 12 | `this` can be thought of as equivalent to `ref('')`, and is a neat way to avoid circular dependencies. 13 | 14 | ## Examples 15 | 16 | 17 | 18 | 19 | 20 | ### Grant permissions on a model in a post-hook 21 | 22 | 23 | 24 | ```yaml 25 | models: 26 | project-name: 27 | +post-hook: 28 | - "grant select on {{ this }} to db_reader" 29 | ``` 30 | 31 | 32 | 33 | 34 | 35 | 36 | ### Configuring incremental models 37 | 38 | 39 | 40 | ```sql 41 | {{ config(materialized='incremental') }} 42 | 43 | select 44 | *, 45 | my_slow_function(my_column) 46 | 47 | from raw_app_data.events 48 | 49 | {% if is_incremental() %} 50 | where event_time > (select max(event_time) from {{ this }}) 51 | {% endif %} 52 | ``` 53 | 54 | 55 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/tojson.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "tojson" 3 | id: "tojson" 4 | --- 5 | 6 | The `tojson` context method can be used to serialize a Python object primitive, eg. a `dict` or `list` to a JSON string. 7 | 8 | __Args__: 9 | * `value`: The value to serialize to json (required) 10 | * `default`: A default value to return if the `value` argument cannot be serialized (optional) 11 | 12 | ### Usage: 13 | ``` 14 | {% set my_dict = {"abc": 123} %} 15 | {% set my_json_string = tojson(my_dict) %} 16 | 17 | {% do log(my_json_string) %} 18 | ``` 19 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbt-jinja-functions/toyaml.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "toyaml" 3 | id: "toyaml" 4 | --- 5 | 6 | The `toyaml` context method can be used to serialize a Python object primitive, eg. a `dict` or `list` to a yaml string. 7 | 8 | __Args__: 9 | * `value`: The value to serialize to yaml (required) 10 | * `default`: A default value to return if the `value` argument cannot be serialized (optional) 11 | 12 | ### Usage: 13 | ``` 14 | {% set my_dict = {"abc": 123} %} 15 | {% set my_yaml_string = toyaml(my_dict) %} 16 | 17 | {% do log(my_yaml_string) %} 18 | ``` 19 | -------------------------------------------------------------------------------- /context-sources/docs/reference/dbtignore.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: .dbtignore 3 | --- 4 | 5 | You can create a `.dbtignore` file in the root of your [dbt project](projects) to specify files that should be **entirely** ignored by dbt. The file behaves like a [`.gitignore` file, using the same syntax](https://git-scm.com/docs/gitignore). Files and subdirectories matching the pattern will not be read, parsed, or otherwise detected by dbt—as if they didn't exist. 6 | 7 | **Examples** 8 | 9 | 10 | 11 | ```md 12 | # .dbtignore 13 | 14 | # ignore individual .py files 15 | not-a-dbt-model.py 16 | another-non-dbt-model.py 17 | 18 | # ignore all .py files 19 | **.py 20 | 21 | # ignore all .py files with "codegen" in the filename 22 | *codegen*.py 23 | ``` 24 | 25 | 26 | -------------------------------------------------------------------------------- /context-sources/docs/reference/exit-codes.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Exit codes" 3 | id: "exit-codes" 4 | --- 5 | 6 | When dbt exits, it will return an exit code of either 0, 1, or 2. 7 | 8 | | Exit Code | Condition | 9 | | --------- | --------- | 10 | | 0 | The dbt invocation completed without error. | 11 | | 1 | The dbt invocation completed with at least one handled error (eg. model syntax error, bad permissions, etc). The run was completed, but some models may have been skipped. | 12 | | 2 | The dbt invocation completed with an unhandled error (eg. ctrl-c, network interruption, etc). | 13 | 14 | While these exit codes may change in the future, a zero exit code will always imply success whereas a nonzero exit code will always imply failure. 15 | -------------------------------------------------------------------------------- /context-sources/docs/reference/global-cli-flags.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Global CLI flags" 3 | id: "global-cli-flags" 4 | --- 5 | 6 | dbt commands, such as `run` or `test`, support their own command-specific CLI flags. In addition, the dbt command itself supports "global" flags applicable to *all* subcommands. 7 | 8 | As of v1.0, the vast majority of global CLI flags map to [global configs](global-configs), which can also be configured via environment variables or in the `profiles.yml`. 9 | 10 | The `--version` and `--record-timing-info` flags remain available to the CLI only. 11 | 12 | ## Version 13 | 14 | The `--version` flag returns information about the currently installed version of dbt. 15 | 16 | 17 | 18 | ```text 19 | $ dbt --version 20 | installed version: 0.15.1 21 | latest version: 0.15.1 22 | 23 | Up to date! 24 | ``` 25 | 26 | 27 | 28 | ## Record timing info 29 | 30 | The `-r` or `--record-timing-info` flag saves performance profiling information to a file. This file can be visualized with `snakeviz` to understand the performance characteristics of a dbt invocation 31 | 32 | 33 | 34 | ```text 35 | $ dbt -r timing.txt run 36 | ... 37 | 38 | $ snakeviz timing.txt 39 | ``` 40 | 41 | 42 | -------------------------------------------------------------------------------- /context-sources/docs/reference/macro-properties.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Macro properties 3 | --- 4 | 5 | Macro properties can be declared in `.yml` files. 6 | 7 | You can name these files `whatever_you_want.yml`, and nest them arbitrarily deeply in subfolders. 8 | 9 | 10 | 11 | ```yml 12 | version: 2 13 | 14 | macros: 15 | - name: 16 | [description](description): 17 | [docs](/reference/resource-configs/docs): 18 | show: true | false 19 | arguments: 20 | - name: 21 | [type](argument-type): 22 | [description](description): 23 | - ... # declare properties of additional arguments 24 | 25 | - name: ... # declare properties of additional macros 26 | 27 | ``` 28 | 29 | 30 | 31 | 32 | 33 | * `v0.16.0`: The ability to declare macro properties was introduced. 34 | 35 | 36 | -------------------------------------------------------------------------------- /context-sources/docs/reference/model-properties.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Model properties 3 | --- 4 | 5 | Models properties can be declared in `.yml` files in your `models/` directory (as defined by the [`model-paths` config](model-paths)). 6 | 7 | You can name these files `whatever_you_want.yml`, and nest them arbitrarily deeply in subfolders within the `models/` directory. 8 | 9 | 10 | 11 | ```yml 12 | version: 2 13 | 14 | models: 15 | - [name](model_name): 16 | [description](description): 17 | [docs](/reference/resource-configs/docs): 18 | show: true | false 19 | [config](resource-properties/config): 20 | [](model-configs): 21 | [tests](resource-properties/tests): 22 | - 23 | - ... # declare additional tests 24 | columns: 25 | - name: # required 26 | [description](description): 27 | [meta](meta): {} 28 | [quote](quote): true | false 29 | [tests](resource-properties/tests): 30 | - 31 | - ... # declare additional tests 32 | [tags](resource-configs/tags): [] 33 | 34 | - name: ... # declare properties of additional columns 35 | 36 | ``` 37 | 38 | 39 | 40 | 44 | -------------------------------------------------------------------------------- /context-sources/docs/reference/node-selection/exclude.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Exclude" 3 | --- 4 | 5 | ### Excluding models 6 | dbt provides an `--exclude` flag with the same semantics as `--select`. Models specified with the `--exclude` flag will be removed from the set of models selected with `--select`. 7 | 8 | ```bash 9 | $ dbt run --select my_package.*+ --exclude my_package.a_big_model+ # select all models in my_package and their children except a_big_model and its children 10 | ``` 11 | 12 | Exclude a specific resource by its name or lineage: 13 | 14 | ```bash 15 | # test 16 | $ dbt test --exclude not_null_orders_order_id # test all models except the not_null_orders_order_id test 17 | $ dbt test --exclude orders # test all models except tests associated with the orders model 18 | 19 | # seed 20 | $ dbt seed --exclude account_parent_mappings # load all seeds except account_parent_mappings 21 | 22 | # snapshot 23 | $ dbt snapshot --exclude snap_order_statuses # execute all snapshots except snap_order_statuses 24 | ``` 25 | -------------------------------------------------------------------------------- /context-sources/docs/reference/node-selection/set-operators.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Set operators" 3 | --- 4 | 5 | ### Unions 6 | Providing multiple space-delineated arguments to the `--select`, `--exclude`, or `--selector` flags selects 7 | the union of them all. If a resource is included in at least one selector, it will be 8 | included in the final set. 9 | 10 | Run snowplow_sessions, all ancestors of snowplow_sessions, fct_orders, and all ancestors of fct_orders: 11 | 12 | 13 | ```bash 14 | $ dbt run --select +snowplow_sessions +fct_orders 15 | ``` 16 | 17 | ### Intersections 18 | 19 | If you separate multiple arguments for `--select`, `--exclude`, and `--selector` with commas and no whitespace in between, dbt will select only resources that satisfy _all_ arguments. 20 | 21 | Run all the common ancestors of snowplow_sessions and fct_orders: 22 | 23 | 24 | ```bash 25 | $ dbt run --select +snowplow_sessions,+fct_orders 26 | ``` 27 | 28 | 29 | Run all the common descendents of stg_invoices and stg_accounts: 30 | 31 | 32 | ```bash 33 | $ dbt run --select stg_invoices+,stg_accounts+ 34 | ``` 35 | 36 | 37 | Run models that are in the marts/finance subdirectory *and* tagged nightly: 38 | 39 | 40 | ```bash 41 | $ dbt run --select marts.finance,tag:nightly 42 | ``` 43 | -------------------------------------------------------------------------------- /context-sources/docs/reference/project-configs/analysis-paths.md: -------------------------------------------------------------------------------- 1 | --- 2 | datatype: [directorypath] 3 | default_value: [] 4 | --- 5 | 6 | 7 | 8 | ```yml 9 | analysis-paths: [directorypath] 10 | ``` 11 | 12 | 13 | 14 | ## Definition 15 | Specify a custom list of directories where [analyses](analyses) are located. 16 | 17 | ## Default 18 | Without specifying this config, dbt will not compile any `.sql` files as analyses. 19 | 20 | However, the [`dbt init` command](init) populates this value as `analyses` ([source](https://github.com/dbt-labs/dbt-starter-project/blob/HEAD/dbt_project.yml#L15)) 21 | 22 | ## Examples 23 | ### Use a subdirectory named `analyses` 24 | This is the value populated by the [`dbt init` command](init). 25 | 26 | 27 | 28 | ```yml 29 | analysis-paths: ["analyses"] 30 | ``` 31 | 32 | 33 | 34 | ### Use a subdirectory named `custom_analyses` 35 | 36 | 37 | 38 | ```yml 39 | analysis-paths: ["custom_analyses"] 40 | ``` 41 | 42 | 43 | -------------------------------------------------------------------------------- /context-sources/docs/reference/project-configs/asset-paths.md: -------------------------------------------------------------------------------- 1 | --- 2 | datatype: [directorypath] 3 | default_value: [] 4 | --- 5 | 6 | 7 | 8 | ```yml 9 | asset-paths: [directorypath] 10 | ``` 11 | 12 | 13 | 14 | ## Definition 15 | Optionally specify a custom list of directories to copy to the `target` directory as part of the `docs generate` command. This is useful for rendering images in your repository in your project documentation. 16 | 17 | 18 | 19 | * `v0.18.0`: This configuration was introduced — see the [migration guide](/guides/migration/versions) for more details. 20 | 21 | 22 | 23 | ## Default 24 | By default, dbt will not copy any additional files as part of docs generate, i.e. `asset-paths: []` 25 | 26 | ## Examples 27 | ### Compile files in the `assets` subdirectory as part of `docs generate` 28 | 29 | 30 | 31 | ```yml 32 | asset-paths: ["assets"] 33 | ``` 34 | 35 | 36 | 37 | Any files included in this directory will be copied to the `target/` directory as part of `dbt docs generate`, making them accessible as images in your project documentation. 38 | 39 | Check out the full writeup on including images in your descriptions [here](/reference/resource-properties/description/#include-an-image-from-your-repo-in-your-descriptions). 40 | -------------------------------------------------------------------------------- /context-sources/docs/reference/project-configs/config-version.md: -------------------------------------------------------------------------------- 1 | --- 2 | datatype: integer 3 | --- 4 | 5 | 6 | 7 | ```yml 8 | config-version: 2 9 | ``` 10 | 11 | 12 | 13 | ## Definition 14 | Specify your `dbt_project.yml` as using the v2 structure. 15 | 16 | 17 | 18 | * `v0.17.0`: This configuration was introduced — see the [migration guide](/guides/migration/versions) for more details. 19 | * `v1.5.0`: This configuration was made optional. 20 | 21 | 22 | 23 | ## Default 24 | Without this configuration, dbt will assume your `dbt_project.yml` uses the version 1 syntax, which was deprecated in dbt v0.19.0. 25 | -------------------------------------------------------------------------------- /context-sources/docs/reference/project-configs/dispatch-config.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: dispatch (config) 3 | datatype: list 4 | required: False 5 | --- 6 | 7 | 8 | 9 | ```yml 10 | dispatch: 11 | - macro_namespace: packagename 12 | search_order: [packagename] 13 | - macro_namespace: packagename 14 | search_order: [packagename] 15 | ``` 16 | 17 | 18 | 19 | ## Definition 20 | 21 | Optionally override the [dispatch](dispatch) search locations for macros in certain namespaces. If not specified, `dispatch` will look in your root project _first_, by default, and then look for implementations in the package named by `macro_namespace`. 22 | 23 | ## Examples 24 | 25 | 26 | I want to "shim" the `dbt_utils` package with the `spark_utils` compatibility package. 27 | 28 | 29 | 30 | ```yml 31 | dispatch: 32 | - macro_namespace: dbt_utils 33 | search_order: ['spark_utils', 'dbt_utils'] 34 | ``` 35 | 36 | 37 | 38 | I've reimplemented certain macros from the `dbt_utils` package in my root project (`'my_root_project'`), and I want my versions to take precedence. Otherwise, fall back to the versions in `dbt_utils`. 39 | 40 | _Note: This is the default behavior. You may optionally choose to express that search order explicitly as:_ 41 | 42 | 43 | 44 | ```yml 45 | dispatch: 46 | - macro_namespace: dbt_utils 47 | search_order: ['my_root_project', 'dbt_utils'] 48 | ``` 49 | 50 | 51 | -------------------------------------------------------------------------------- /context-sources/docs/reference/project-configs/docs-paths.md: -------------------------------------------------------------------------------- 1 | --- 2 | datatype: [directorypath] 3 | default_value: [] 4 | --- 5 | 6 | 7 | 8 | ```yml 9 | docs-paths: [directorypath] 10 | ``` 11 | 12 | 13 | 14 | ## Definition 15 | Optionally specify a custom list of directories where [docs blocks](documentation#docs-blocks) are located. 16 | 17 | 18 | ## Default 19 | By default, dbt will search in all resource paths for docs blocks (i.e. the combined list of [model-paths](model-paths), [seed-paths](seed-paths), [analysis-paths](analysis-paths), [macro-paths](macro-paths) and [snapshot-paths](snapshot-paths)). If this option is configured, dbt will _only_ look in the specified directory for docs blocks. 20 | 21 | 22 | ## Examples 23 | :::info 24 | We typically omit this configuration as we prefer dbt's default behavior. 25 | ::: 26 | 27 | ### Use a subdirectory named `docs` for docs blocks 28 | 29 | 30 | 31 | ```yml 32 | docs-paths: ["docs"] 33 | ``` 34 | 35 | 36 | -------------------------------------------------------------------------------- /context-sources/docs/reference/project-configs/log-path.md: -------------------------------------------------------------------------------- 1 | --- 2 | datatype: directorypath 3 | default_value: logs 4 | --- 5 | 6 | 7 | ```yml 8 | log-path: directorypath 9 | ``` 10 | 11 | 12 | 13 | ## Definition 14 | Optionally specify a custom directory where dbt will write logs. 15 | 16 | ## Default 17 | By default, dbt will write to the `logs` directory, i.e. `log-path: logs` 18 | 19 | 20 | 21 | ## Configuration 22 | 23 | In the manner of a ["global" config](global-configs), the log path can be set in three places: 24 | 1. `--log-path` CLI flag 25 | 2. `DBT_LOG_PATH` environment variable 26 | 3. `log-path` in `dbt_project.yml` 27 | 28 | The precedence order is: CLI flag > env var > `dbt_project.yml` 29 | 30 | 31 | 32 | ## Examples 33 | ### Write logs to a subdirectory named `dbt_logs` instead of `logs` 34 | 35 | 36 | 37 | ```yml 38 | log-path: dbt_logs 39 | ``` 40 | 41 | 42 | -------------------------------------------------------------------------------- /context-sources/docs/reference/project-configs/macro-paths.md: -------------------------------------------------------------------------------- 1 | --- 2 | datatype: directorypath 3 | default_value: [macros] 4 | --- 5 | 6 | 7 | 8 | ```yml 9 | macro-paths: [directorypath] 10 | ``` 11 | 12 | 13 | 14 | ## Definition 15 | Optionally specify a custom list of directories where [macros](jinja-macros#macros) are located. Note that you cannot co-locate models and macros. 16 | 17 | ## Default 18 | By default, dbt will search for macros in a directory named `macros`, i.e. `macro-paths: ["macros"]` 19 | 20 | ## Examples 21 | ### Use a subdirectory named `custom_macros` instead of `macros` 22 | 23 | 24 | 25 | ```yml 26 | macro-paths: ["custom_macros"] 27 | ``` 28 | 29 | 30 | -------------------------------------------------------------------------------- /context-sources/docs/reference/project-configs/model-paths.md: -------------------------------------------------------------------------------- 1 | --- 2 | datatype: [directorypath] 3 | default_value: [models] 4 | --- 5 | 6 | 7 | - **v1.0.0:** The config `source-paths` has been deprecated in favor of `model-paths`. 8 | 9 | 10 | 11 | 12 | 13 | ```yml 14 | model-paths: [directorypath] 15 | ``` 16 | 17 | 18 | 19 | ## Definition 20 | Optionally specify a custom list of directories where [models](/docs/build/models) and [sources](/docs/build/sources) are located. 21 | 22 | ## Default 23 | By default, dbt will search for models and sources in the `models` directory, i.e. `model-paths: ["models"]` 24 | 25 | ## Examples 26 | ### Use a subdirectory named `transformations` instead of `models` 27 | 28 | 29 | 30 | ```yml 31 | model-paths: ["transformations"] 32 | ``` 33 | 34 | 35 | -------------------------------------------------------------------------------- /context-sources/docs/reference/project-configs/name.md: -------------------------------------------------------------------------------- 1 | --- 2 | datatype: string 3 | required: True 4 | --- 5 | 6 | 7 | 8 | ```yml 9 | name: string 10 | ``` 11 | 12 | 13 | 14 | ## Definition 15 | **Required configuration** 16 | 17 | The name of a dbt project. Must be letters, digits and underscores only, and cannot start with a digit. 18 | 19 | ## Recommendation 20 | Often an organization has one dbt project, so it is sensible to name a project with your organization's name, in `snake_case`. For example: 21 | * `name: acme` 22 | * `name: jaffle_shop` 23 | * `name: evilcorp` 24 | 25 | 26 | ## Troubleshooting 27 | ### Invalid project name 28 | 29 | ``` 30 | Encountered an error while reading the project: 31 | ERROR: Runtime Error 32 | at path ['name']: 'jaffle-shop' does not match '^[^\\d\\W]\\w*$' 33 | Runtime Error 34 | Could not run dbt 35 | ``` 36 | 37 | This project has: 38 | 39 | 40 | 41 | ```yml 42 | name: jaffle-shop 43 | ``` 44 | 45 | 46 | 47 | In this case, change your project name to be `snake_case` instead: 48 | 49 | 50 | 51 | ```yml 52 | name: jaffle_shop 53 | ``` 54 | 55 | 56 | -------------------------------------------------------------------------------- /context-sources/docs/reference/project-configs/packages-install-path.md: -------------------------------------------------------------------------------- 1 | --- 2 | datatype: directorypath 3 | default_value: dbt_packages 4 | --- 5 | 6 | 7 | 8 | - **v1.0.0:** The default config has changed from `modules-path` to `packages-install-path` with a new default value of `dbt_packages`. 9 | 10 | 11 | 12 | 13 | 14 | ```yml 15 | packages-install-path: directorypath 16 | ``` 17 | 18 | 19 | 20 | ## Definition 21 | Optionally specify a custom directory where [packages](/docs/build/packages) are installed when you run the `dbt deps` [command](deps). Note that this directory is usually git-ignored. 22 | 23 | ## Default 24 | By default, dbt will install packages in the `dbt_packages` directory, i.e. `packages-install-path: dbt_packages` 25 | 26 | ## Examples 27 | ### Install packages in a subdirectory named `packages` instead of `dbt_packages` 28 | 29 | 30 | 31 | ```yml 32 | packages-install-path: packages 33 | ``` 34 | 35 | 36 | -------------------------------------------------------------------------------- /context-sources/docs/reference/project-configs/profile.md: -------------------------------------------------------------------------------- 1 | --- 2 | datatype: string 3 | --- 4 | 5 | 6 | ```yml 7 | profile: string 8 | ``` 9 | 10 | 11 | 12 | ## Definition 13 | The profile your dbt project should use to connect to your . 14 | * If you are developing in dbt Cloud: This configuration is not applicable 15 | * If you are developing locally: This configuration is required, unless a command-line option (i.e. `--profile`) is supplied. 16 | 17 | ## Related guides 18 | * [Connecting to your warehouse using the command line](/docs/core/connection-profiles#connecting-to-your-warehouse-using-the-command-line) 19 | 20 | ## Recommendation 21 | Often an organization has only one , so it is sensible to use your organization's name as a profile name, in `snake_case`. For example: 22 | * `profile: acme` 23 | * `profile: jaffle_shop` 24 | 25 | It is also reasonable to include the name of your warehouse technology in your profile name, particularly if you have multiple warehouses. For example: 26 | * `profile: acme_snowflake` 27 | * `profile: jaffle_shop_bigquery` 28 | * `profile: jaffle_shop_redshift` 29 | -------------------------------------------------------------------------------- /context-sources/docs/reference/project-configs/seed-paths.md: -------------------------------------------------------------------------------- 1 | --- 2 | datatype: [directorypath] 3 | default_value: [data] 4 | --- 5 | 6 | 7 | 8 | - **v1.0.0:** The config `data-paths` has been deprecated in favor of `seed-paths`. 9 | 10 | 11 | 12 | 13 | 14 | ```yml 15 | seed-paths: [directorypath] 16 | ``` 17 | 18 | 19 | 20 | ## Definition 21 | Optionally specify a custom list of directories where [seed](/docs/build/seeds) files are located. 22 | 23 | ## Default 24 | 25 | By default, dbt expects seeds to be located in the `seeds` directory, i.e. `seed-paths: ["seeds"]` 26 | 27 | ## Examples 28 | ### Use a subdirectory named `custom_seeds` instead of `seeds` 29 | 30 | 31 | 32 | ```yml 33 | seed-paths: ["custom_seeds"] 34 | ``` 35 | 36 | 37 | 38 | ### Co-locate your models and seeds in the `models` directory 39 | Note: this works because dbt is looking for different file types for seeds (`.csv` files) and models (`.sql` files). 40 | 41 | 42 | 43 | ```yml 44 | seed-paths: ["models"] 45 | model-paths: ["models"] 46 | ``` 47 | 48 | 49 | 50 | ### Split your seeds across two directories 51 | Note: We recommend that you instead use two subdirectories within the `seeds/` directory to achieve a similar effect. 52 | 53 | 54 | 55 | ```yml 56 | seed-paths: ["seeds", "custom_seeds"] 57 | ``` 58 | 59 | 60 | -------------------------------------------------------------------------------- /context-sources/docs/reference/project-configs/snapshot-paths.md: -------------------------------------------------------------------------------- 1 | --- 2 | datatype: [directorypath] 3 | default_value: [snapshots] 4 | --- 5 | 6 | 7 | ```yml 8 | snapshot-paths: [directorypath] 9 | ``` 10 | 11 | 12 | 13 | ## Definition 14 | Optionally specify a custom list of directories where [snapshots](snapshots) are located. Note that you cannot co-locate models and snapshots. 15 | 16 | 17 | 18 | * `v0.14.0`: Snapshots were introduced 19 | 20 | 21 | 22 | ## Default 23 | By default, dbt will search for snapshots in the `snapshots` directory, i.e. `snapshot-paths: ["snapshots"]` 24 | 25 | ## Examples 26 | ### Use a subdirectory named `archives` instead of `snapshots` 27 | 28 | 29 | 30 | ```yml 31 | snapshot-paths: ["archives"] 32 | ``` 33 | 34 | 35 | -------------------------------------------------------------------------------- /context-sources/docs/reference/project-configs/target-path.md: -------------------------------------------------------------------------------- 1 | --- 2 | datatype: directorypath 3 | default_value: target 4 | --- 5 | 6 | 7 | ```yml 8 | target-path: directorypath 9 | ``` 10 | 11 | 12 | 13 | # Definition 14 | Optionally specify a custom directory where compiled files (e.g. compiled models and tests) will be written when you run the `dbt run`, `dbt compile`, or `dbt test` command. 15 | 16 | 17 | 18 | ## Default 19 | By default, dbt will write compiled files to the `target` directory, i.e. `target-path: target` 20 | 21 | 22 | 23 | ## Configuration 24 | 25 | In the manner of a ["global" config](global-configs), the target path can be set in three places: 26 | 1. `--target-path` CLI flag 27 | 2. `DBT_TARGET_PATH` environment variable 28 | 3. `target-path` in `dbt_project.yml` 29 | 30 | The precedence order is: CLI flag > env var > `dbt_project.yml` 31 | 32 | 33 | 34 | ## Examples 35 | ### Use a subdirectory named `compiled` for compiled files 36 | 37 | 38 | 39 | ```yml 40 | target-path: "compiled" 41 | ``` 42 | 43 | 44 | -------------------------------------------------------------------------------- /context-sources/docs/reference/project-configs/test-paths.md: -------------------------------------------------------------------------------- 1 | --- 2 | datatype: [directorypath] 3 | default_value: [test] 4 | --- 5 | 6 | 7 | 8 | * `v1.0.0`: Generic tests can be defined in the `tests/generic` subfolder, in addition to the `macros/` directory 9 | 10 | 11 | 12 | 13 | 14 | ```yml 15 | test-paths: [directorypath] 16 | ``` 17 | 18 | 19 | 20 | ## Definition 21 | 22 | Optionally specify a custom list of directories where [singular tests](/docs/build/tests) are located. 23 | 24 | 25 | ## Default 26 | Without specifying this config, dbt will search for tests in the `tests` directory, i.e. `test-paths: ["tests"]`. Specifically, it will look for `.sql` files containing: 27 | - Generic test definitions in the `tests/generic` subdirectory 28 | - Singular tests (all other files) 29 | 30 | ## Examples 31 | ### Use a subdirectory named `custom_tests` instead of `tests` for data tests 32 | 33 | 34 | 35 | ```yml 36 | test-paths: ["custom_tests"] 37 | ``` 38 | 39 | 40 | -------------------------------------------------------------------------------- /context-sources/docs/reference/resource-configs/alias.md: -------------------------------------------------------------------------------- 1 | --- 2 | resource_types: [models, seeds, snapshots, tests] 3 | datatype: string 4 | --- 5 | 6 | :::caution Heads up! 7 | This is a work in progress document. While this configuration applies to multiple resource types, the documentation has only been written for seeds. 8 | 9 | ::: 10 | 11 | ## Definition 12 | 13 | Optionally specify a custom alias for a [model](/docs/build/models) or [seed](/docs/build/seeds). 14 | 15 | When dbt creates a relation (/) in a database, it creates it as: `{{ database }}.{{ schema }}.{{ identifier }}`, e.g. `analytics.finance.payments` 16 | 17 | The standard behavior of dbt is: 18 | * If a custom alias is _not_ specified, the identifier of the relation is the resource name (i.e. the filename). 19 | * If a custom alias is specified, the identifier of the relation is the `{{ alias }}` value. 20 | 21 | To learn more about changing the way that dbt generates a relation's `identifier`, read [Using Aliases](/docs/build/custom-aliases). 22 | 23 | 24 | ## Usage 25 | 26 | ### Seeds 27 | Configure a seed's alias in your `dbt_project.yml` file. 28 | 29 | The seed at `seeds/country_codes.csv` will be built as a named `country_mappings`. 30 | 31 | 32 | 33 | ```yml 34 | seeds: 35 | jaffle_shop: 36 | country_codes: 37 | +alias: country_mappings 38 | 39 | ``` 40 | 41 | 42 | -------------------------------------------------------------------------------- /context-sources/docs/reference/resource-configs/invalidate_hard_deletes.md: -------------------------------------------------------------------------------- 1 | --- 2 | resource_types: [snapshots] 3 | datatype: column_name 4 | --- 5 | 6 | New in v0.19.0 7 | 8 | 9 | ```jinja2 10 | {{ 11 | config( 12 | strategy="timestamp", 13 | invalidate_hard_deletes=True 14 | ) 15 | }} 16 | 17 | ``` 18 | 19 | 20 | 21 | 22 | 23 | ```yml 24 | snapshots: 25 | [](resource-path): 26 | +strategy: timestamp 27 | +invalidate_hard_deletes: true 28 | 29 | ``` 30 | 31 | 32 | 33 | ## Description 34 | Opt-in feature to enable invalidating hard deleted records while snapshotting the query. 35 | 36 | 37 | ## Default 38 | By default the feature is disabled. 39 | 40 | ## Example 41 | 42 | 43 | 44 | ```sql 45 | {% snapshot orders_snapshot %} 46 | 47 | {{ 48 | config( 49 | target_schema='snapshots', 50 | strategy='timestamp', 51 | unique_key='id', 52 | updated_at='updated_at', 53 | invalidate_hard_deletes=True, 54 | ) 55 | }} 56 | 57 | select * from {{ source('jaffle_shop', 'orders') }} 58 | 59 | {% endsnapshot %} 60 | ``` 61 | 62 | 63 | -------------------------------------------------------------------------------- /context-sources/docs/reference/resource-configs/materialized.md: -------------------------------------------------------------------------------- 1 | :::danger 2 | 3 | These docs are a placeholder for a yet-to-be-written reference section. 4 | 5 | Please refer to the [guide on materializations](materializations) for current documentation. 6 | 7 | ::: 8 | -------------------------------------------------------------------------------- /context-sources/docs/reference/resource-configs/no-configs.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "No specifc configurations for this Adapter" 3 | id: "no-configs" 4 | --- 5 | 6 | If you were guided to this page from a data platform setup article, it most likely means: 7 | 8 | - Setting up the profile is the only action the end-user needs to take on the data platform, or 9 | - The subsequent actions the end-user needs to take are not currently documented 10 | 11 | If you'd like to contribute to data platform-specifc configuration information, refer to [Documenting a new adapter](5-documenting-a-new-adapter) -------------------------------------------------------------------------------- /context-sources/docs/reference/resource-configs/snapshot_name.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ```jinja2 4 | {% snapshot snapshot_name %} 5 | 6 | {% endsnapshot %} 7 | 8 | ``` 9 | 10 | 11 | 12 | ## Description 13 | 14 | The name of a snapshot, as defined in the `{% snapshot %}` block header. This name is used when selecting from a snapshot using the [`ref` function](ref) 15 | 16 | This name must not conflict with any other snapshot names, or any model names. 17 | 18 | The name does not need to match the file name. As a result, snapshot filenames do not need to be unique. 19 | 20 | ## Examples 21 | ### Name a snapshot `order_snapshot` 22 | 23 | 24 | 25 | ```jinja2 26 | {% snapshot orders_snapshot %} 27 | ... 28 | {% endsnapshot %} 29 | 30 | ``` 31 | 32 | 33 | 34 | 35 | To select from this snapshot in a downstream model: 36 | 37 | ```sql 38 | select * from {{ ref('orders_snapshot') }} 39 | ``` 40 | -------------------------------------------------------------------------------- /context-sources/docs/reference/resource-properties/argument-type.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: type 3 | resource_types: macro_argument 4 | datatype: argument_type 5 | --- 6 | 7 | 8 | 9 | 10 | ```yml 11 | version: 2 12 | 13 | macros: 14 | - name: 15 | arguments: 16 | - name: 17 | type: 18 | 19 | ``` 20 | 21 | 22 | 23 | ## Definition 24 | The data type of your argument. Note that this is only used for documentation purposes — there are no restrictions on the values you can use here. 25 | 26 | ## Examples 27 | ### Document a macro 28 | 29 | 30 | 31 | ```sql 32 | {% macro cents_to_dollars(column_name, scale=2) %} 33 | ({{ column_name }} / 100)::numeric(16, {{ scale }}) 34 | {% endmacro %} 35 | 36 | ``` 37 | 38 | 39 | 40 | 41 | 42 | ```yml 43 | version: 2 44 | 45 | macros: 46 | - name: cents_to_dollars 47 | arguments: 48 | - name: column_name 49 | type: column name or expression 50 | description: "The name of a column, or an expression — anything that can be `select`-ed as a column" 51 | 52 | - name: scale 53 | type: integer 54 | description: "The number of decimal places to round to. Default is 2." 55 | 56 | ``` 57 | 58 | 59 | -------------------------------------------------------------------------------- /context-sources/docs/reference/resource-properties/database.md: -------------------------------------------------------------------------------- 1 | --- 2 | resource_types: sources 3 | datatype: database_name 4 | --- 5 | 6 | 7 | 8 | ```yml 9 | version: 2 10 | 11 | sources: 12 | - name: 13 | database: 14 | tables: 15 | - name: 16 | - ... 17 | 18 | ``` 19 | 20 | 21 | 22 | ## Definition 23 | The database that your source is stored in. 24 | 25 | Note that to use this parameter, your warehouse must allow cross-database queries. 26 | 27 | :::info 28 | 29 | #### BigQuery terminology 30 | If you're using BigQuery, use the _project_ name as the `database:` property. 31 | 32 | ::: 33 | 34 | ## Default 35 | By default, dbt will search in your target database (i.e. the database that you are creating tables and views). 36 | 37 | ## Examples 38 | ### Define a source that is stored in the `raw` database 39 | 40 | 41 | 42 | ```yml 43 | version: 2 44 | 45 | sources: 46 | - name: jaffle_shop 47 | database: raw 48 | tables: 49 | - name: orders 50 | - name: customers 51 | 52 | ``` 53 | 54 | 55 | -------------------------------------------------------------------------------- /context-sources/docs/reference/resource-properties/loader.md: -------------------------------------------------------------------------------- 1 | --- 2 | resource_types: sources 3 | datatype: string 4 | --- 5 | 6 | 7 | 8 | ```yml 9 | version: 2 10 | 11 | sources: 12 | - name: 13 | database: 14 | loader: 15 | tables: 16 | - ... 17 | 18 | ``` 19 | 20 | 21 | 22 | ## Definition 23 | Describe the tool that loads this source into your warehouse. Note that this property is for documentation purposes only — dbt does not meaningfully use this. 24 | 25 | ## Examples 26 | ### Indicate which EL tool loaded data 27 | 28 | 29 | 30 | ```yml 31 | version: 2 32 | 33 | sources: 34 | - name: jaffle_shop 35 | loader: fivetran 36 | tables: 37 | - name: orders 38 | - name: customers 39 | 40 | - name: stripe 41 | loader: stitch 42 | tables: 43 | - name: payments 44 | ``` 45 | 46 | 47 | -------------------------------------------------------------------------------- /context-sources/docs/reference/resource-properties/model_name.md: -------------------------------------------------------------------------------- 1 | --- 2 | resource_types: [models] 3 | datatype: model_name 4 | required: yes 5 | --- 6 | 7 | 8 | 9 | ```yml 10 | version: 2 11 | 12 | models: 13 | - name: model_name 14 | ``` 15 | 16 | 17 | 18 | ## Definition 19 | The name of the model you are declaring properties for. Must match the _filename_ of a model. 20 | 21 | ## Default 22 | 23 | This is a **required property**, no default exists. 24 | -------------------------------------------------------------------------------- /context-sources/docs/reference/resource-properties/name.md: -------------------------------------------------------------------------------- 1 | #### name 2 | The `name` field can be used to specify both model names and column names in schema.yml files. When `name` is used to denote a model (as shown on line 4 above), dbt will look up that model by its filename. The example above will look for a model called `events.sql`, for instance. 3 | -------------------------------------------------------------------------------- /context-sources/docs/reference/resource-properties/schema.md: -------------------------------------------------------------------------------- 1 | --- 2 | resource_types: sources 3 | datatype: schema_name 4 | --- 5 | 6 | 7 | 8 | ```yml 9 | version: 2 10 | 11 | sources: 12 | - name: 13 | database: 14 | schema: 15 | tables: 16 | - name: 17 | - ... 18 | 19 | ``` 20 | 21 | 22 | 23 | ## Definition 24 | The schema name as stored in the database. 25 | 26 | This parameter is useful if you want to use a source name that differs from the schema name. 27 | 28 | :::info 29 | 30 | #### BigQuery terminology 31 | If you're using BigQuery, use the _dataset_ name as the `schema:` property. 32 | 33 | ::: 34 | 35 | ## Default 36 | By default, dbt will use the source's `name:` parameter as the schema name. 37 | 38 | ## Examples 39 | ### Use a simpler name for a source schema than the one in your database 40 | 41 | 42 | 43 | ```yml 44 | version: 2 45 | 46 | sources: 47 | - name: jaffle_shop 48 | schema: postgres_backend_public_schema 49 | tables: 50 | - name: orders 51 | 52 | ``` 53 | 54 | 55 | 56 | 57 | In a downstream model: 58 | ```sql 59 | select * from {{ source('jaffle_shop', 'orders') }} 60 | ``` 61 | 62 | Will get compiled to: 63 | ```sql 64 | select * from postgres_backend_public_schema.orders 65 | ``` 66 | -------------------------------------------------------------------------------- /context-sources/docs/reference/snowflake-permissions.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Snowflake Permissions" 3 | --- 4 | 5 | ## Example Snowflake permissions 6 | 7 | ``` 8 | -- NOTE: warehouse_name, database_name, and role_name are placeholders! 9 | -- Replace as-needed for your organization's naming convention! 10 | 11 | grant all on warehouse warehouse_name to role role_name; 12 | grant usage on database database_name to role role_name; 13 | grant create schema on database database_name to role role_name; 14 | grant usage on schema database.an_existing_schema to role role_name; 15 | grant create table on schema database.an_existing_schema to role role_name; 16 | grant create view on schema database.an_existing_schema to role role_name; 17 | grant usage on future schemas in database database_name to role role_name; 18 | grant select on future tables in database database_name to role role_name; 19 | grant select on future views in database database_name to role role_name; 20 | grant usage on all schemas in database database_name to role role_name; 21 | grant select on all tables in database database_name to role role_name; 22 | grant select on all views in database database_name to role role_name; 23 | ``` 24 | -------------------------------------------------------------------------------- /context-sources/docs/terms/materialization.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: materialization 3 | title: Materialization 4 | description: A materialization is the exact Data Definition Language (DDL) that dbt will use when creating the model’s equivalent in a data warehouse. 5 | displayText: materialization 6 | hoverSnippet: The exact Data Definition Language (DDL) that dbt will use when creating the model’s equivalent in a data warehouse. 7 | --- 8 | 9 | 10 | What does materialization mean in the context of dbt? 11 | 12 | 13 | :::important This page could use some love 14 | This term would benefit from additional depth and examples. Have knowledge to contribute? [Create a discussion in the docs.getdbt.com GitHub repository](https://github.com/dbt-labs/docs.getdbt.com/discussions) to begin the process of becoming a glossary contributor! 15 | ::: 16 | 17 | The exact Data Definition Language (DDL) that dbt will use when creating the model’s equivalent in a . It's the manner in which the data is represented, and each of those options is defined either canonically (tables, views, incremental), or bespoke. 18 | 19 | It is important to consider the downstream impacts of your materialization choice on query run times and macro capabilities. 20 | 21 | -------------------------------------------------------------------------------- /context-sources/docs/terms/monotonically-increasing.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: monotonically-increasing 3 | title: Monotonically increasing 4 | description: Monotonicity means unchanging (think monotone). A monotonically-increasing value is a value which increases at a constant rate, for example the values 1, 2, 3, 4. 5 | displayText: monotonically increasing 6 | hoverSnippet: Monotonicity means unchanging (think monotone). A monotonically-increasing value is a value which increases at a constant rate, for example the values 1, 2, 3, 4. 7 | --- 8 | 9 | Monotonicity means unchanging (think monotone). A monotonically-increasing value is a value which increases at a constant rate, for example the values `[1, 2, 3, 4]`. 10 | 11 | Monotonically-increasing values often appear in primary keys generated by production systems. In an analytics engineering context, you should avoid generating such values or assuming their existence in your models, because they make it more difficult to create an data model. Instead you should create a which is derived from the unique component(s) of a row. -------------------------------------------------------------------------------- /document_crawler.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | from abc import abstractmethod 3 | from typing import List 4 | 5 | from langchain.docstore.document import Document 6 | 7 | 8 | class DocumentCrawler: 9 | @abstractmethod 10 | def crawl_and_make_docs(self) -> List[Document]: 11 | pass 12 | 13 | @abstractmethod 14 | def split_file_into_docs(self) -> List[Document]: 15 | pass 16 | 17 | @abstractmethod 18 | def hash_string(self, string: str) -> str: 19 | """Creates an md5 hash from a string.""" 20 | return hashlib.md5(string.encode()).hexdigest() 21 | 22 | @abstractmethod 23 | def make_source_link(self, source: str) -> str: 24 | """Creates a link to the source file from the Document source property.""" 25 | pass 26 | -------------------------------------------------------------------------------- /test-docs/guides/best-practices/materializations/materializations-guide-7-conclusion.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Conclusion" 3 | id: materializations-guide-7-conclusion 4 | slug: 7-conclusion 5 | description: Learn how to utilize materializations in dbt. 6 | displayText: Materializations best practices 7 | hoverSnippet: Learn how to utilize materializations in dbt. 8 | --- 9 | 10 | You're now following best practices in your project, and have optimized the materializations of your DAG. You’re equipped with the 3 main materializations that cover almost any analytics engineering situation! 11 | 12 | There are more configs and materializations available, as well as specific materializations for certain platforms and adapters — and like everything with dbt, materializations are extensible, meaning you can create your own [custom materializations](guides/advanced/creating-new-materializations) for your needs. So this is just the beginning of what you can do with these powerful configurations. 13 | 14 | For the vast majority of users and companies though, tables, views, and incremental models will handle everything you can throw at them. Develop your intuition and expertise for these materializations, and you’ll be well on your way to tackling advanced analytics engineering problems. 15 | -------------------------------------------------------------------------------- /test_code_crawler.py: -------------------------------------------------------------------------------- 1 | from code_crawler import CodeCrawler 2 | 3 | 4 | def test_should_split_csv_into_rows_except_heading(): 5 | test_dir = "./test-docs/" 6 | crawler = CodeCrawler(test_dir) 7 | 8 | docs = crawler.crawl_and_make_docs() 9 | 10 | assert len(docs) == 0 11 | 12 | 13 | # def test_should_create_id_based_on_hashed_filename_and_content(): 14 | # test_dir = "./test-docs/" 15 | # crawler = CodeCrawler(test_dir) 16 | 17 | # docs = crawler.crawl_and_make_docs() 18 | 19 | # assert docs[0].metadata["id"] == crawler.hash_string( 20 | # f"{docs[0].metadata['source']}{docs[0].page_content}" 21 | # ) 22 | -------------------------------------------------------------------------------- /test_csv_crawler.py: -------------------------------------------------------------------------------- 1 | from csv_crawler import CSVCrawler 2 | 3 | 4 | def test_should_split_csv_into_rows_except_heading(): 5 | test_dir = "./test-docs/" 6 | crawler = CSVCrawler(test_dir) 7 | 8 | docs = crawler.crawl_and_make_docs() 9 | 10 | assert len(docs) == 2313 11 | 12 | 13 | def test_should_create_id_based_on_hashed_filename_and_content(): 14 | test_dir = "./test-docs/" 15 | crawler = CSVCrawler(test_dir) 16 | 17 | docs = crawler.crawl_and_make_docs() 18 | 19 | assert docs[0].metadata["id"] == crawler.hash_string( 20 | f"{docs[0].metadata['source']}{docs[0].page_content}" 21 | ) 22 | -------------------------------------------------------------------------------- /test_markdown_crawler.py: -------------------------------------------------------------------------------- 1 | from markdown_crawler import MarkdownCrawler 2 | 3 | 4 | def test_should_split_markdown_at_headings(): 5 | test_dir = "./test-docs/guides/best-practices/materializations/" 6 | crawler = MarkdownCrawler(test_dir) 7 | 8 | docs = crawler.crawl_and_make_docs() 9 | 10 | assert len(docs) == 21 11 | 12 | 13 | def test_should_create_id_based_on_hashed_filename_and_content(): 14 | test_dir = "./test-docs/guides/best-practices/materializations/" 15 | crawler = MarkdownCrawler(test_dir) 16 | 17 | docs = crawler.crawl_and_make_docs() 18 | 19 | assert docs[0].metadata["id"] == crawler.hash_string( 20 | f"{docs[0].metadata['source']}{docs[0].page_content}" 21 | ) 22 | -------------------------------------------------------------------------------- /test_question_answerer.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from question_answerer import QuestionAnswerer 4 | 5 | 6 | # def test_question_answerer_returns_a_string(): 7 | # qa = QuestionAnswerer("./test-docs") 8 | 9 | # answer = qa.answer_question("What is dbt?") 10 | 11 | # assert type(answer) == str 12 | 13 | 14 | # def test_question_answerer_refuses_non_strings_as_input(): 15 | # qa = QuestionAnswerer("./test-docs") 16 | 17 | # with pytest.raises(TypeError): 18 | # qa.answer_question(42) # type: ignore 19 | -------------------------------------------------------------------------------- /test_vector_store.py: -------------------------------------------------------------------------------- 1 | from langchain.docstore.document import Document 2 | 3 | from vector_store import VectorStore 4 | 5 | 6 | # def test_vector_store_returns_list_of_similar_documents_with_scores(): 7 | # vector_store = VectorStore( 8 | # name="qa-db", sources_path="./test-docs", reindex=True 9 | # ) 10 | 11 | # results = vector_store.get_similar_documents("What is dbt?") 12 | 13 | # assert type(results) == list 14 | # assert all(isinstance(document[0], Document) for document in results) 15 | # assert all(isinstance(document[1], float) for document in results) 16 | # assert all(isinstance(document[1], float) for document in results) 17 | -------------------------------------------------------------------------------- /vercel.json: -------------------------------------------------------------------------------- 1 | { 2 | "builds": [ 3 | { 4 | "src": "app.py", 5 | "use": "@vercel/python" 6 | } 7 | ], 8 | "routes": [ 9 | { 10 | "src": "/(.*)", 11 | "dest": "app.py" 12 | } 13 | ] 14 | } 15 | --------------------------------------------------------------------------------