├── .devcontainer ├── README.md └── devcontainer.json ├── .gitignore ├── CONTRIBUTING.md ├── README.md ├── airbyte_dbt_airflow_bigquery ├── .gitignore ├── README.md ├── assets │ ├── 10_dbt-lineage.png │ ├── 1_airflow-connection-menu.png │ ├── 2_airflow-connections-page.png │ ├── 3_airbyte-connection.png │ ├── 4_airbyte-dag.png │ ├── 5_airbyte-dataset.png │ ├── 6_dbt-airbyte-dag.png │ ├── 7_dbt-jaffle-shop.png │ ├── 8_dbt-others.png │ ├── 9_airflow-dags.png │ ├── ecommerce_dag.png │ └── elt_dag.png ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── analyses │ │ └── .gitkeep │ ├── dbt_project.yml │ ├── macros │ │ └── .gitkeep │ ├── models │ │ └── ecommerce │ │ │ ├── marts │ │ │ ├── product_popularity.sql │ │ │ ├── purchase_patterns.sql │ │ │ ├── schema.yml │ │ │ └── user_demographics.sql │ │ │ ├── sources │ │ │ └── faker_sources.yml │ │ │ └── staging │ │ │ ├── schema.yml │ │ │ ├── stg_products.sql │ │ │ ├── stg_purchases.sql │ │ │ └── stg_users.sql │ ├── profiles.yml │ ├── seeds │ │ ├── .gitkeep │ │ ├── raw_customers.csv │ │ ├── raw_orders.csv │ │ └── raw_payments.csv │ ├── snapshots │ │ └── .gitkeep │ └── tests │ │ └── .gitkeep ├── infra │ ├── .gitignore │ ├── README.md │ └── airbyte │ │ ├── main.tf │ │ ├── provider.tf │ │ ├── terraform.tfvars │ │ └── variables.tf ├── orchestration │ ├── .env.example │ ├── .gitignore │ ├── Dockerfile │ ├── README.md │ ├── airflow │ │ ├── config │ │ │ └── dbt_config.py │ │ ├── dags │ │ │ └── elt_dag.py │ │ └── plugins │ │ │ ├── custom_docs_plugin.py │ │ │ ├── dbt_upload_docs.py │ │ │ ├── static │ │ │ └── .gitkeep │ │ │ └── templates │ │ │ └── dbt │ │ │ └── .gitkeep │ ├── docker-compose.yaml │ └── requirements.txt └── setup.py ├── airbyte_dbt_airflow_snowflake ├── .gitignore ├── README.md ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── analyses │ │ └── .gitkeep │ ├── dbt_project.yml │ ├── macros │ │ └── .gitkeep │ ├── models │ │ ├── example │ │ │ ├── my_first_dbt_model.sql │ │ │ ├── my_second_dbt_model.sql │ │ │ └── schema.yml │ │ └── sources.yml │ ├── profiles.yml │ ├── seeds │ │ └── .gitkeep │ ├── snapshots │ │ └── .gitkeep │ └── tests │ │ └── .gitkeep ├── images │ ├── ada_snowflake_stack.png │ └── dag.png ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf ├── orchestration │ ├── .env.example │ ├── .gitignore │ ├── Dockerfile │ ├── README.md │ ├── airflow │ │ ├── dags │ │ │ └── my_elt_dag.py │ │ └── plugins │ │ │ ├── static │ │ │ └── .gitkeep │ │ │ └── templates │ │ │ └── dbt │ │ │ └── .gitkeep │ ├── docker-compose.yaml │ └── requirements.txt └── setup.py ├── airbyte_dbt_dagster ├── .gitignore ├── README.md ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── analyses │ │ └── .gitkeep │ ├── dbt_project.yml │ ├── macros │ │ └── .gitkeep │ ├── models │ │ └── example │ │ │ ├── my_first_dbt_model.sql │ │ │ ├── my_second_dbt_model.sql │ │ │ └── schema.yml │ ├── profiles.yml │ ├── seeds │ │ └── .gitkeep │ ├── snapshots │ │ └── .gitkeep │ └── tests │ │ └── .gitkeep ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── .terraform.lock.hcl │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf ├── orchestration │ ├── orchestration │ │ ├── __init__.py │ │ ├── assets.py │ │ ├── constants.py │ │ ├── definitions.py │ │ └── schedules.py │ ├── pyproject.toml │ └── setup.py └── setup.py ├── airbyte_dbt_dagster_snowflake ├── .gitignore ├── README.md ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── analyses │ │ └── .gitkeep │ ├── dbt_project.yml │ ├── macros │ │ └── .gitkeep │ ├── models │ │ ├── example │ │ │ ├── my_first_dbt_model.sql │ │ │ ├── my_second_dbt_model.sql │ │ │ └── schema.yml │ │ └── sources.yml │ ├── profiles.yml │ ├── seeds │ │ └── .gitkeep │ ├── snapshots │ │ └── .gitkeep │ └── tests │ │ └── .gitkeep ├── images │ ├── dad_snowflake_stack.png │ └── dag.svg ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf ├── orchestration │ ├── orchestration │ │ ├── __init__.py │ │ ├── assets.py │ │ ├── constants.py │ │ ├── definitions.py │ │ └── schedules.py │ ├── pyproject.toml │ └── setup.py └── setup.py ├── airbyte_dbt_prefect_bigquery ├── .gitignore ├── README.md ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── analyses │ │ └── .gitkeep │ ├── dbt_project.yml │ ├── macros │ │ └── .gitkeep │ ├── models │ │ ├── marts │ │ │ ├── product_popularity.sql │ │ │ ├── purchase_patterns.sql │ │ │ └── user_demographics.sql │ │ ├── sources │ │ │ └── faker_sources.yml │ │ └── staging │ │ │ ├── stg_products.sql │ │ │ ├── stg_purchases.sql │ │ │ └── stg_users.sql │ ├── profiles.yml │ ├── seeds │ │ └── .gitkeep │ ├── snapshots │ │ └── .gitkeep │ └── tests │ │ └── .gitkeep ├── images │ ├── dag.png │ └── pad_bigquery_stack.png ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf ├── orchestration │ └── my_elt_flow.py └── setup.py ├── airbyte_dbt_prefect_snowflake ├── .gitignore ├── README.md ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── analyses │ │ └── .gitkeep │ ├── dbt_project.yml │ ├── macros │ │ └── .gitkeep │ ├── models │ │ ├── example │ │ │ ├── my_first_dbt_model.sql │ │ │ ├── my_second_dbt_model.sql │ │ │ └── schema.yml │ │ └── sources.yml │ ├── profiles.yml │ ├── seeds │ │ └── .gitkeep │ ├── snapshots │ │ └── .gitkeep │ └── tests │ │ └── .gitkeep ├── images │ └── infrastructure.png ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf ├── orchestration │ └── my_elt_flow.py └── setup.py ├── airbyte_dbt_snowflake_looker ├── .gitignore ├── README.md ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── analyses │ │ └── .gitkeep │ ├── dbt_project.yml │ ├── macros │ │ └── .gitkeep │ ├── models │ │ ├── example │ │ │ ├── my_first_dbt_model.sql │ │ │ ├── my_second_dbt_model.sql │ │ │ └── schema.yml │ │ └── sources.yml │ ├── profiles.yml │ ├── seeds │ │ └── .gitkeep │ ├── snapshots │ │ └── .gitkeep │ └── tests │ │ └── .gitkeep ├── images │ ├── adsl_stack.png │ └── dag.svg ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf ├── orchestration │ ├── orchestration │ │ ├── __init__.py │ │ ├── assets.py │ │ ├── constants.py │ │ ├── definitions.py │ │ └── schedules.py │ ├── pyproject.toml │ └── setup.py └── setup.py ├── airbyte_lib_notebooks ├── AirbyteLib_Basic_Features_Demo.ipynb ├── AirbyteLib_CoinAPI_Demo.ipynb ├── AirbyteLib_GA4_Demo.ipynb ├── AirbyteLib_Github_Incremental_Demo.ipynb ├── PyAirbyte_Postgres_Custom_Cache_Demo.ipynb ├── PyAirbyte_Shopify_Demo.ipynb └── README.md ├── airbyte_s3_pinecone_rag ├── .env.template ├── .gitignore ├── README.md ├── assets │ └── dataflow.png ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── dbt_project.yml │ ├── models │ │ ├── marts │ │ │ └── purchase_data.sql │ │ ├── sources │ │ │ └── s3.source.yml │ │ └── staging │ │ │ └── stg_purchases.sql │ └── profiles.yml ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── .terraform.lock.hcl │ │ ├── main.tf │ │ ├── output.tf │ │ ├── provider.tf │ │ └── variables.tf ├── query.py ├── quickstart.md └── setup.py ├── api_to_warehouse ├── .gitignore ├── Readme.md ├── images │ └── infrastructure.jpg ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── .terraform.lock.hcl │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf └── setup.py ├── customer_segmentation_analytics_shopify ├── .gitignore ├── README.md ├── assets │ └── Global_Asset_Lineage (7).svg ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── analyses │ │ ├── customer_activity_analysis.py │ │ ├── purchase_pattern_segmentation_analysis.py │ │ └── rfm_segmentation_analysis.py │ ├── dbt_project.yml │ ├── models │ │ ├── marts │ │ │ ├── customer_activity.sql │ │ │ ├── purchase_pattern_segmentation.sql │ │ │ └── rfm_segmentation.sql │ │ ├── sources │ │ │ └── shopify_sources.yml │ │ └── staging │ │ │ ├── stg_customers.sql │ │ │ └── stg_transactions.sql │ └── profiles.yml ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── .terraform.lock.hcl │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf ├── orchestration │ ├── orchestration │ │ ├── __init__.py │ │ ├── assets.py │ │ ├── constants.py │ │ ├── definitions.py │ │ └── schedules.py │ ├── pyproject.toml │ └── setup.py └── setup.py ├── data_to_pinecone_llm ├── .env.template ├── .gitignore ├── .vscode │ └── quickstart.code-workspace ├── README.md ├── assets │ ├── 1-dataflow.png │ ├── 2-sources.png │ ├── 3-destinations.png │ ├── 4-connections.png │ ├── 5-sync-notion.png │ ├── 6-bigquery-dataset.png │ ├── 7-bigquery-dataset-with-view.png │ └── 8-sync-pinecone.png ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── analyses │ │ └── .gitkeep │ ├── dbt_project.yml │ ├── macros │ │ └── .gitkeep │ ├── models │ │ ├── notion.source.yml │ │ └── notion_data.sql │ ├── profiles.yml │ ├── seeds │ │ └── .gitkeep │ ├── snapshots │ │ └── .gitkeep │ └── tests │ │ └── .gitkeep ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── .terraform.lock.hcl │ │ ├── .tfvars │ │ ├── main.tf │ │ ├── output.tf │ │ ├── provider.tf │ │ └── variables.tf ├── query.py ├── quickstart.md ├── secrets │ ├── .gitignore │ └── README.md └── setup.py ├── database_snapshot ├── .gitignore ├── README.md ├── images │ └── layout.png ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── connections │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf │ │ ├── destinations │ │ ├── main.tf │ │ ├── outputs.tf │ │ ├── provider.tf │ │ └── variables.tf │ │ ├── main.tf │ │ ├── provider.tf │ │ ├── sources │ │ ├── main.tf │ │ ├── outputs.tf │ │ ├── provider.tf │ │ └── variables.tf │ │ └── variables.tf └── setup.py ├── developer_productivity_analytics_github ├── .gitignore ├── README.md ├── assets │ └── Global_Asset_Lineage (5).svg ├── dbt_project │ ├── .gitignore │ ├── dbt_project.yml │ ├── models │ │ ├── marts │ │ │ ├── avarage_time_to_merge_pr_analysis.sql │ │ │ ├── commits_over_time_per_dev_analysis.sql │ │ │ ├── dev_activity_by_day_of_week_analysis.sql │ │ │ ├── dev_collaboration_network_analysis.sql │ │ │ ├── freq_of_code_contribution_analysis.sql │ │ │ ├── no_of_code_reviews_per_dev_analysis.sql │ │ │ ├── no_of_commits_per_dev_per_repo_analysis.sql │ │ │ ├── no_of_pr_per_dev_analysis.sql │ │ │ ├── number_of_pr_open_or_closed.sql │ │ │ ├── top_collaborators_by_repo_analysis.sql │ │ │ └── track_issues_assigned_by_dev_analysis.sql │ │ ├── sources │ │ │ └── github_source.yml │ │ └── staging │ │ │ ├── stg_branches.sql │ │ │ ├── stg_collaborators.sql │ │ │ ├── stg_comments.sql │ │ │ ├── stg_commits.sql │ │ │ ├── stg_issues.sql │ │ │ ├── stg_organizations.sql │ │ │ ├── stg_pull_requests.sql │ │ │ ├── stg_repositories.sql │ │ │ ├── stg_review_comments.sql │ │ │ ├── stg_reviews.sql │ │ │ ├── stg_stargazers.sql │ │ │ ├── stg_tags.sql │ │ │ ├── stg_teams.sql │ │ │ └── stg_users.sql │ ├── profiles.yml │ ├── snapshots │ │ └── .gitkeep │ └── tests │ │ └── .gitkeep ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── .terraform.lock.hcl │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf ├── orchestration │ ├── orchestration │ │ ├── __init__.py │ │ ├── assets.py │ │ ├── constants.py │ │ ├── definitions.py │ │ └── schedules.py │ ├── pyproject.toml │ └── setup.py └── setup.py ├── ecommerce_analytics_bigquery ├── .gitignore ├── README.md ├── assets │ └── Global_Asset_Lineage.svg ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── analyses │ │ └── .gitkeep │ ├── dbt_project.yml │ ├── macros │ │ └── .gitkeep │ ├── models │ │ ├── marts │ │ │ ├── product_popularity.sql │ │ │ ├── purchase_patterns.sql │ │ │ └── user_demographics.sql │ │ ├── sources │ │ │ └── faker_sources.yml │ │ └── staging │ │ │ ├── stg_products.sql │ │ │ ├── stg_purchases.sql │ │ │ └── stg_users.sql │ ├── profiles.yml │ ├── seeds │ │ └── .gitkeep │ ├── snapshots │ │ └── .gitkeep │ └── tests │ │ └── .gitkeep ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── .terraform.lock.hcl │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf ├── orchestration │ ├── orchestration │ │ ├── __init__.py │ │ ├── assets.py │ │ ├── constants.py │ │ ├── definitions.py │ │ └── schedules.py │ ├── pyproject.toml │ └── setup.py └── setup.py ├── elt_simplified_stack ├── .gitignore ├── README.md ├── dbt_project │ ├── .gitignore │ ├── dbt_project.yml │ ├── models │ │ ├── marts │ │ │ ├── commits-per-repo.sql │ │ │ ├── pr-per-dev.sql │ │ │ └── pr-per-status.sql │ │ ├── sources │ │ │ └── github_source.yml │ │ └── staging │ │ │ ├── stg_commits.sql │ │ │ └── stg_pull_requests.sql │ ├── profiles.yml │ ├── snapshots │ │ └── .gitkeep │ └── tests │ │ └── .gitkeep ├── images │ └── infrastrcuture.png ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── .terraform.lock.hcl │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf ├── orchestration │ └── my_elt_flow.py └── setup.py ├── error_analysis_stack_sentry ├── Readme.md ├── assets │ ├── Infrastructure_layout.jpg │ └── orchestration_lineage.png ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── analyses │ │ └── .gitkeep │ ├── dbt_project.yml │ ├── macros │ │ └── .gitkeep │ ├── models │ │ ├── example │ │ │ ├── Insight_Table.sql │ │ │ └── schema.yml │ │ └── sources.yml │ ├── profiles.yml │ ├── seeds │ │ └── .gitkeep │ ├── snapshots │ │ └── .gitkeep │ └── tests │ │ └── .gitkeep ├── error_analysis_stack.egg-info │ ├── PKG-INFO │ ├── SOURCES.txt │ ├── dependency_links.txt │ ├── requires.txt │ └── top_level.txt ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── .terraform.lock.hcl │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf ├── orchestration │ ├── orchestration.egg-info │ │ ├── PKG-INFO │ │ ├── SOURCES.txt │ │ ├── dependency_links.txt │ │ ├── requires.txt │ │ └── top_level.txt │ ├── orchestration │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-310.pyc │ │ │ ├── assets.cpython-310.pyc │ │ │ ├── constants.cpython-310.pyc │ │ │ ├── definitions.cpython-310.pyc │ │ │ └── schedules.cpython-310.pyc │ │ ├── assets.py │ │ ├── constants.py │ │ ├── definitions.py │ │ └── schedules.py │ ├── pyproject.toml │ └── setup.py └── setup.py ├── github_insight_stack ├── .gitignore ├── README.md ├── architecture.png ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── analyses │ │ └── .gitkeep │ ├── dbt_project.yml │ ├── macros │ │ └── .gitkeep │ ├── models │ │ ├── Readme.md │ │ ├── sources.yml │ │ └── test-models │ │ │ ├── code_quality.sql │ │ │ ├── collaboration_patterns.sql │ │ │ └── project_health.sql │ ├── profiles.yml │ ├── seeds │ │ └── .gitkeep │ ├── snapshots │ │ └── .gitkeep │ └── tests │ │ └── .gitkeep ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── .terraform.lock.hcl │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf ├── orchestration │ ├── orchestration │ │ ├── __init__.py │ │ ├── assets.py │ │ ├── constants.py │ │ ├── definitions.py │ │ └── schedules.py │ ├── pyproject.toml │ └── setup.py └── setup.py ├── low_latency_data_availability ├── .gitignore ├── README.md ├── images │ └── infrastructure_layout.png ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf └── setup.py ├── mongodb_mysql_integration ├── .gitignore ├── README.md ├── images │ └── Infrastructure_layout.png ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf └── setup.py ├── multisource_aggregation ├── .gitignore ├── README.md ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── analyses │ │ └── .gitkeep │ ├── dbt_project.yml │ ├── macros │ │ └── .gitkeep │ ├── models │ │ ├── example │ │ │ ├── my_first_dbt_model.sql │ │ │ ├── my_second_dbt_model.sql │ │ │ └── schema.yml │ │ └── sources.yml │ ├── profiles.yml │ ├── seeds │ │ └── .gitkeep │ ├── snapshots │ │ └── .gitkeep │ └── tests │ │ └── .gitkeep ├── images │ ├── dag.svg │ └── layout.png ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── connections │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf │ │ ├── destination_warehouse │ │ ├── main.tf │ │ ├── outputs.tf │ │ ├── provider.tf │ │ └── variables.tf │ │ ├── main.tf │ │ ├── provider.tf │ │ ├── source_databases │ │ ├── main.tf │ │ ├── outputs.tf │ │ ├── provider.tf │ │ └── variables.tf │ │ └── variables.tf ├── orchestration │ ├── orchestration │ │ ├── __init__.py │ │ ├── assets.py │ │ ├── constants.py │ │ ├── definitions.py │ │ └── schedules.py │ ├── pyproject.toml │ └── setup.py └── setup.py ├── mysql_to_postgres_incremental_stack ├── .gitignore ├── README.md ├── image │ └── sample.png ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf └── setup.py ├── outdoor_activity_analytics_recreation ├── .gitignore ├── README.md ├── assets │ └── Global_Asset_Lineage (6).svg ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── analyses │ │ ├── campsite_availability_analysis.py │ │ ├── campsite_type_analysis.py │ │ ├── count_recareas_by_activity_analysis.py │ │ └── most_common_activities_in_recareas_analysis.py │ ├── dbt_project.yml │ ├── models │ │ ├── marts │ │ │ ├── campsite_availability_over_time.sql │ │ │ ├── campsite_type_counts.sql │ │ │ ├── count_recarea_by_activity_analysis.sql │ │ │ └── most_common_activities_in_recareas.sql │ │ ├── sources │ │ │ └── recreation_source.yml │ │ └── staging │ │ │ ├── stg_activities.sql │ │ │ ├── stg_campsites.sql │ │ │ ├── stg_facilities.sql │ │ │ └── stg_recreationareas.sql │ └── profiles.yml ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── .terraform.lock.hcl │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf ├── orchestration │ ├── orchestration │ │ ├── __init__.py │ │ ├── assets.py │ │ ├── constants.py │ │ ├── definitions.py │ │ └── schedules.py │ ├── pyproject.toml │ └── setup.py └── setup.py ├── postgres_data_replication ├── .gitignore ├── README.md ├── images │ └── infrastructure_layout.png ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf └── setup.py ├── postgres_snowflake_integration ├── .gitignore ├── README.md ├── images │ └── Infrastructure_layout.png ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf └── setup.py ├── postgres_to_mysql_migration ├── .gitignore ├── README.md ├── images │ └── infrastructure_layout.png ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf └── setup.py ├── pyairbyte_notebooks ├── AI ChatBot - 1.0 Launch Demo.ipynb ├── Chatoverpolygonstockdata_langchain.ipynb ├── PyAirbyte_Apify_Demo.ipynb ├── PyAirbyte_Basic_Features_Demo.ipynb ├── PyAirbyte_CoinAPI_Demo.ipynb ├── PyAirbyte_Document_Creation_RAG_with_Langchain_Demo.ipynb ├── PyAirbyte_GA4_Demo.ipynb ├── PyAirbyte_Github_Incremental_Demo.ipynb ├── PyAirbyte_Postgres_Custom_Cache_Demo.ipynb ├── PyAirbyte_Shopify_Demo.ipynb ├── PyAirbyte_Snowflake_Cortex_Github.ipynb ├── PyAirbyte_Snowflake_Custom_Cache_Demo.ipynb ├── PyAirbyte_as_an_Orchestrator_Demo.ipynb ├── RAG_using_github_pyairbyte_chroma.ipynb ├── README.md ├── rag_using_gdrive_pyairbyte_pinecone.ipynb ├── rag_using_github_pyairbyte_weaviate.ipynb ├── rag_using_gitlab_pyairbyte_qdrant.ipynb ├── rag_using_jira_pyairbyte_pinecone.ipynb ├── rag_using_s3_pyairbyte_pinecone.ipynb ├── rag_using_shopify_pyairbyte_langchain.ipynb ├── rag_with_fb_marketing_milvus_lite.ipynb ├── rag_with_pyairbyte_and_milvus_lite.ipynb ├── sentiment_analysis_airbyte_gsheets_snowflakecortex.ipynb └── using_langchain_airbyte_package.ipynb ├── satisfaction_analytics_zendesk_support ├── .gitignore ├── README.md ├── assets │ └── Global_Asset_Lineage (3).svg ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── analysis │ │ └── .gitkeep │ ├── dbt_project.yml │ ├── models │ │ ├── marts │ │ │ ├── analyze_satisfaction_score_over_time.sql │ │ │ ├── avarage_satisfaction_rating.sql │ │ │ ├── feedback_analysis_for_low_score.sql │ │ │ └── trend_analysis_by_score.sql │ │ ├── sources │ │ │ └── zendesk_support_sources.yml │ │ └── staging │ │ │ ├── stg_brands.sql │ │ │ ├── stg_groups.sql │ │ │ ├── stg_organizations.sql │ │ │ ├── stg_satisfaction_ratings.sql │ │ │ ├── stg_tags.sql │ │ │ ├── stg_ticket_audits.sql │ │ │ ├── stg_ticket_comments.sql │ │ │ ├── stg_ticket_fields.sql │ │ │ ├── stg_ticket_forms.sql │ │ │ ├── stg_ticket_metric_events.sql │ │ │ ├── stg_ticket_metrics.sql │ │ │ ├── stg_tickets.sql │ │ │ └── stg_users.sql │ ├── profiles.yml │ └── tests │ │ └── .gitkeep ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── .terraform.lock.hcl │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf ├── orchestration │ ├── orchestration │ │ ├── __init__.py │ │ ├── assets.py │ │ ├── constants.py │ │ ├── definitions.py │ │ └── schedules.py │ ├── pyproject.toml │ ├── setup.py │ ├── tmp3ks7pwhz │ │ ├── history │ │ │ ├── runs.db │ │ │ └── runs │ │ │ │ ├── ad21fadd-c131-4a7c-98a7-fa5ad3a929de.db │ │ │ │ ├── f7507115-918d-443f-ab91-a065e84fa403.db │ │ │ │ └── index.db │ │ ├── schedules │ │ │ └── schedules.db │ │ └── storage │ │ │ ├── ad21fadd-c131-4a7c-98a7-fa5ad3a929de │ │ │ └── compute_logs │ │ │ │ ├── mdvhnoik.complete │ │ │ │ ├── mdvhnoik.err │ │ │ │ ├── mdvhnoik.out │ │ │ │ ├── uzgmeijp.complete │ │ │ │ ├── uzgmeijp.err │ │ │ │ └── uzgmeijp.out │ │ │ └── f7507115-918d-443f-ab91-a065e84fa403 │ │ │ └── compute_logs │ │ │ ├── aeebjmfa.complete │ │ │ ├── aeebjmfa.err │ │ │ ├── aeebjmfa.out │ │ │ ├── zqbkkiww.complete │ │ │ ├── zqbkkiww.err │ │ │ └── zqbkkiww.out │ └── tmpb3ctnsbk │ │ ├── history │ │ ├── runs.db │ │ └── runs │ │ │ ├── 0bc4e544-546d-44df-b79c-e75413c56ecb.db │ │ │ ├── 1eac78ed-12d1-4147-9c48-79b27dd586ed.db │ │ │ └── index.db │ │ ├── schedules │ │ └── schedules.db │ │ └── storage │ │ ├── 0bc4e544-546d-44df-b79c-e75413c56ecb │ │ └── compute_logs │ │ │ ├── xozgecli.complete │ │ │ ├── xozgecli.err │ │ │ ├── xozgecli.out │ │ │ ├── yyxjctam.complete │ │ │ ├── yyxjctam.err │ │ │ └── yyxjctam.out │ │ └── 1eac78ed-12d1-4147-9c48-79b27dd586ed │ │ └── compute_logs │ │ ├── iqvvuhde.complete │ │ ├── iqvvuhde.err │ │ ├── iqvvuhde.out │ │ ├── izklbfmq.complete │ │ ├── izklbfmq.err │ │ └── izklbfmq.out └── setup.py ├── shopping_cart_analytics_shopify ├── .gitignore ├── README.md ├── assets │ └── Global_Asset_Lineage (8).svg ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── dbt_project.yml │ ├── models │ │ ├── marts │ │ │ ├── abandoned_checkout_ratio.sql │ │ │ ├── location_based_abandoned_checkouts.sql │ │ │ ├── most_abandoned_products.sql │ │ │ └── time_based.sql │ │ ├── sources │ │ │ └── shopify_source.yml │ │ └── staging │ │ │ └── stg_abandoned_checkouts.sql │ └── profiles.yml ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── .terraform.lock.hcl │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf ├── orchestration │ ├── orchestration │ │ ├── __init__.py │ │ ├── assets.py │ │ ├── constants.py │ │ ├── definitions.py │ │ └── schedules.py │ ├── pyproject.toml │ └── setup.py └── setup.py ├── ticket_volume_analytics_zendesk_support ├── .gitignore ├── README.md ├── assets │ └── Global_Asset_Lineage (4).svg ├── dbt_project │ ├── .gitignore │ ├── README.md │ ├── dbt_project.yml │ ├── models │ │ ├── marts │ │ │ ├── busier_day_of_week_analysis.sql │ │ │ ├── pattern_and_trend_analysis.sql │ │ │ ├── seasonal_analysis.sql │ │ │ ├── ticket_priority_analysis.sql │ │ │ ├── ticket_resolution_time_analysis.sql │ │ │ ├── ticket_source_analysis.sql │ │ │ └── ticket_volume_analysis.sql │ │ ├── sources │ │ │ └── zendesk_support_sources.yml │ │ └── staging │ │ │ ├── stg_schedules.sql │ │ │ ├── stg_ticket_metrics.sql │ │ │ ├── stg_tickets.sql │ │ │ └── stg_users.sql │ ├── profiles.yml │ ├── snapshots │ │ └── .gitkeep │ └── tests │ │ └── .gitkeep ├── infra │ ├── .gitignore │ └── airbyte │ │ ├── .terraform.lock.hcl │ │ ├── main.tf │ │ ├── provider.tf │ │ └── variables.tf ├── orchestration │ ├── orchestration │ │ ├── __init__.py │ │ ├── assets.py │ │ ├── constants.py │ │ ├── definitions.py │ │ └── schedules.py │ ├── pyproject.toml │ └── setup.py └── setup.py ├── vector_store_integration ├── AI_assistant_streamlit_app │ ├── .gitignore │ ├── README.md │ ├── app.py │ └── requirements.txt ├── RAG_using_PGVector.ipynb ├── RAG_using_Snowflake_Cortex.ipynb └── RAG_using_Vectara.ipynb └── weather_data_stack ├── .gitignore ├── README.md ├── assets └── Weather.png ├── dbt_project ├── .gitignore ├── README.md ├── analyses │ └── .gitkeep ├── dbt_project.yml ├── macros │ └── .gitkeep ├── models │ ├── marts │ │ └── historial_weather_trends.sql │ ├── sources │ │ └── weatherstack_source.yml │ └── staging │ │ └── stg_current_weather.sql ├── profiles.yml ├── seeds │ └── .gitkeep ├── snapshots │ └── .gitkeep └── tests │ └── .gitkeep ├── orchestration ├── orchestration │ ├── __init__.py │ ├── assets.py │ ├── constants.py │ ├── definitions.py │ └── schedules.py ├── pyproject.toml └── setup.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Environments 2 | .env 3 | .venv 4 | env/ 5 | venv/ 6 | ENV/ 7 | env.bak/ 8 | venv.bak/ 9 | 10 | #Desktop Services Store 11 | .DS_Store 12 | 13 | # PyAirbyte caches and virtual environments 14 | .cache 15 | .venv* 16 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/assets/10_dbt-lineage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/assets/10_dbt-lineage.png -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/assets/1_airflow-connection-menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/assets/1_airflow-connection-menu.png -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/assets/2_airflow-connections-page.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/assets/2_airflow-connections-page.png -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/assets/3_airbyte-connection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/assets/3_airbyte-connection.png -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/assets/4_airbyte-dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/assets/4_airbyte-dag.png -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/assets/5_airbyte-dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/assets/5_airbyte-dataset.png -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/assets/6_dbt-airbyte-dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/assets/6_dbt-airbyte-dag.png -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/assets/7_dbt-jaffle-shop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/assets/7_dbt-jaffle-shop.png -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/assets/8_dbt-others.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/assets/8_dbt-others.png -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/assets/9_airflow-dags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/assets/9_airflow-dags.png -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/assets/ecommerce_dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/assets/ecommerce_dag.png -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/assets/elt_dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/assets/elt_dag.png -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | #Desktop Services Store 7 | .DS_Store 8 | 9 | #User cookie 10 | .user.yml -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/dbt_project/analyses/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/dbt_project/analyses/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/dbt_project/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/dbt_project/macros/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/dbt_project/models/ecommerce/marts/product_popularity.sql: -------------------------------------------------------------------------------- 1 | WITH base AS ( 2 | SELECT 3 | product_id, 4 | COUNT(id) AS purchase_count 5 | FROM {{ ref('stg_purchases') }} 6 | GROUP BY 1 7 | ) 8 | 9 | SELECT 10 | p.id, 11 | p.make, 12 | p.model, 13 | b.purchase_count 14 | FROM {{ ref('stg_products') }} p 15 | LEFT JOIN base b ON p.id = b.product_id 16 | ORDER BY b.purchase_count DESC 17 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/dbt_project/models/ecommerce/marts/purchase_patterns.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | user_id, 3 | product_id, 4 | purchased_at, 5 | added_to_cart_at, 6 | TIMESTAMP_DIFF(purchased_at, added_to_cart_at, SECOND) AS time_to_purchase_seconds, 7 | returned_at 8 | FROM {{ ref('stg_purchases') }} 9 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/dbt_project/models/ecommerce/marts/user_demographics.sql: -------------------------------------------------------------------------------- 1 | WITH base AS ( 2 | SELECT 3 | id AS user_id, 4 | gender, 5 | academic_degree, 6 | nationality, 7 | age 8 | FROM {{ ref('stg_users') }} 9 | ) 10 | 11 | SELECT 12 | gender, 13 | academic_degree, 14 | nationality, 15 | AVG(age) AS average_age, 16 | COUNT(user_id) AS user_count 17 | FROM base 18 | GROUP BY 1, 2, 3 19 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/dbt_project/models/ecommerce/staging/stg_products.sql: -------------------------------------------------------------------------------- 1 | select 2 | id, 3 | year, 4 | price, 5 | model, 6 | make, 7 | created_at, 8 | updated_at, 9 | _airbyte_extracted_at 10 | from {{ source('faker', 'products') }} 11 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/dbt_project/models/ecommerce/staging/stg_purchases.sql: -------------------------------------------------------------------------------- 1 | select 2 | id, 3 | user_id, 4 | product_id, 5 | updated_at, 6 | purchased_at, 7 | returned_at, 8 | created_at, 9 | added_to_cart_at, 10 | _airbyte_extracted_at 11 | from {{ source('faker', 'purchases') }} 12 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/dbt_project/models/ecommerce/staging/stg_users.sql: -------------------------------------------------------------------------------- 1 | select 2 | id, 3 | gender, 4 | academic_degree, 5 | title, 6 | nationality, 7 | age, 8 | name, 9 | email, 10 | created_at, 11 | updated_at, 12 | _airbyte_extracted_at, 13 | from {{ source('faker', 'users') }} 14 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/dbt_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_project: 2 | outputs: 3 | dev: 4 | dataset: transformed_data 5 | job_execution_timeout_seconds: 300 6 | job_retries: 1 7 | keyfile: /opt/airflow/service_accounts/your_keyfile_path.json # Update this field with your file name, example: /opt/airflow/service_accounts/airflow-***116-83db69931a10.json 8 | location: your_dataset_location # Update this field with your dataset location, example: US 9 | method: service-account 10 | priority: interactive 11 | project: your_project_id # Update this field with your BigQuery project ID 12 | threads: 1 13 | type: bigquery 14 | target: dev 15 | 16 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/dbt_project/seeds/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/dbt_project/seeds/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/dbt_project/snapshots/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/infra/airbyte/terraform.tfvars: -------------------------------------------------------------------------------- 1 | workspace_id="" 2 | dataset_id="sample_ecommerce" 3 | project_id="" 4 | credentials_json_path = "" -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | variable "dataset_id" { 6 | type = string 7 | } 8 | 9 | variable "project_id" { 10 | type = string 11 | } 12 | 13 | variable "credentials_json_path" { 14 | type = string 15 | } 16 | 17 | 18 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/orchestration/.env.example: -------------------------------------------------------------------------------- 1 | POSTGRES_USER=airflow 2 | POSTGRES_PASSWORD=airflow 3 | AIRFLOW_UID=50000 4 | 5 | LOAD_EXAMPLES=false 6 | AIRFLOW_PROJ_DIR="./airflow" 7 | _AIRFLOW_WWW_USER_USERNAME=airflow 8 | _AIRFLOW_WWW_USER_PASSWORD=airflow 9 | AIRFLOW__CORE__TEST_CONNECTION=Enabled 10 | DBT_PROJ_DIR="../dbt_project" 11 | GCP_SERVICE_ACCOUNT_PATH="/path/to/your/service/accounts/directory" -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/orchestration/.gitignore: -------------------------------------------------------------------------------- 1 | logs 2 | __pycache__ -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/orchestration/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM apache/airflow:2.7.2-python3.11 2 | COPY requirements.txt / 3 | RUN pip install --no-cache-dir -r /requirements.txt -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/orchestration/airflow/plugins/static/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/orchestration/airflow/plugins/static/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/orchestration/airflow/plugins/templates/dbt/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_bigquery/orchestration/airflow/plugins/templates/dbt/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/orchestration/requirements.txt: -------------------------------------------------------------------------------- 1 | dbt-core~=1.6.0 2 | astronomer-cosmos~=1.1.0 3 | astronomer-cosmos[dbt-bigquery]~=1.1.0 4 | apache-airflow-providers-google~=10.9.0 5 | apache-airflow-providers-airbyte~=3.3.2 -------------------------------------------------------------------------------- /airbyte_dbt_airflow_bigquery/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="airbyte-dbt-airflow-bigquery", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-bigquery", 8 | "astronomer-cosmos[dbt-bigquery]", 9 | "apache-airflow-providers-google", 10 | "apache-airflow-providers-airbyte", 11 | "apache-airflow", 12 | ], 13 | extras_require={"dev": ["pytest"]}, 14 | ) 15 | 16 | 17 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | #Desktop Services Store 7 | .DS_Store 8 | 9 | #User cookie 10 | .user.yml -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | - dbt run 7 | - dbt test 8 | 9 | 10 | ### Resources: 11 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 12 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 13 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 14 | - Find [dbt events](https://events.getdbt.com) near you 15 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 16 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/dbt_project/analyses/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_snowflake/dbt_project/analyses/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/dbt_project/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_snowflake/dbt_project/macros/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/dbt_project/models/example/my_first_dbt_model.sql: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | Welcome to your first dbt model! 4 | Did you know that you can also configure models directly within SQL files? 5 | This will override configurations stated in dbt_project.yml 6 | 7 | Try changing "table" to "view" below 8 | */ 9 | 10 | {{ config(materialized='table') }} 11 | 12 | with source_data as ( 13 | 14 | select * from {{ source('snowflake', 'sample_table') }} 15 | 16 | ) 17 | 18 | select * 19 | from source_data 20 | 21 | /* 22 | Uncomment the line below to remove records with null `id` values 23 | */ 24 | 25 | -- where id is not null 26 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/dbt_project/models/example/my_second_dbt_model.sql: -------------------------------------------------------------------------------- 1 | 2 | -- Use the `ref` function to select from other models 3 | 4 | select * 5 | from {{ ref('my_first_dbt_model') }} 6 | where id = 1 7 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/dbt_project/models/example/schema.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2 3 | 4 | models: 5 | - name: my_first_dbt_model 6 | description: "A starter dbt model" 7 | columns: 8 | - name: id 9 | description: "The primary key for this table" 10 | tests: 11 | - unique 12 | - not_null 13 | 14 | - name: my_second_dbt_model 15 | description: "A starter dbt model" 16 | columns: 17 | - name: id 18 | description: "The primary key for this table" 19 | tests: 20 | - unique 21 | - not_null 22 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/dbt_project/models/sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: snowflake 5 | tables: 6 | - name: sample_table 7 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/dbt_project/seeds/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_snowflake/dbt_project/seeds/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_snowflake/dbt_project/snapshots/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_snowflake/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/images/ada_snowflake_stack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_snowflake/images/ada_snowflake_stack.png -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/images/dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_snowflake/images/dag.png -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/orchestration/.env.example: -------------------------------------------------------------------------------- 1 | _AIRFLOW_WWW_USER_USERNAME=airflow 2 | _AIRFLOW_WWW_USER_PASSWORD=airflow 3 | 4 | AIRFLOW__CORE__TEST_CONNECTION=Enabled 5 | 6 | AIRFLOW_PROJ_DIR="./airflow" 7 | AIRFLOW_UID=50000 8 | AIRFLOW_AIRBYTE_CONN='...connection_uri_to_airbyte...' 9 | AIRBYTE_CONN_ID='...connection_id_in_airbyte...' 10 | 11 | DBT_PROJ_DIR="../dbt_project" 12 | 13 | LOAD_EXAMPLES=false 14 | 15 | POSTGRES_USER=airflow 16 | POSTGRES_PASSWORD=airflow 17 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/orchestration/.gitignore: -------------------------------------------------------------------------------- 1 | logs 2 | __pycache__ -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/orchestration/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM apache/airflow:2.7.2-python3.11 2 | COPY requirements.txt / 3 | RUN pip install --no-cache-dir -r /requirements.txt -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/orchestration/airflow/plugins/static/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_snowflake/orchestration/airflow/plugins/static/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/orchestration/airflow/plugins/templates/dbt/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_airflow_snowflake/orchestration/airflow/plugins/templates/dbt/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/orchestration/requirements.txt: -------------------------------------------------------------------------------- 1 | dbt-core~=1.6.0 2 | dbt-snowflake 3 | apache-airflow-providers-airbyte~=3.3.2 4 | -------------------------------------------------------------------------------- /airbyte_dbt_airflow_snowflake/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="airbyte-dbt-airflow-snowflake", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-snowflake", 8 | "apache-airflow[airbyte]", 9 | "apache-airflow", 10 | ], 11 | extras_require={"dev": ["pytest"]}, 12 | ) -------------------------------------------------------------------------------- /airbyte_dbt_dagster/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | #Desktop Services Store 7 | .DS_Store 8 | 9 | #User cookie 10 | .user.yml -------------------------------------------------------------------------------- /airbyte_dbt_dagster/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | - dbt run 7 | - dbt test 8 | 9 | 10 | ### Resources: 11 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 12 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 13 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 14 | - Find [dbt events](https://events.getdbt.com) near you 15 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 16 | -------------------------------------------------------------------------------- /airbyte_dbt_dagster/dbt_project/analyses/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_dagster/dbt_project/analyses/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_dagster/dbt_project/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_dagster/dbt_project/macros/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_dagster/dbt_project/models/example/my_first_dbt_model.sql: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | Welcome to your first dbt model! 4 | Did you know that you can also configure models directly within SQL files? 5 | This will override configurations stated in dbt_project.yml 6 | 7 | Try changing "table" to "view" below 8 | */ 9 | 10 | {{ config(materialized='table') }} 11 | 12 | with source_data as ( 13 | 14 | select 1 as id 15 | union all 16 | select null as id 17 | 18 | ) 19 | 20 | select * 21 | from source_data 22 | 23 | /* 24 | Uncomment the line below to remove records with null `id` values 25 | */ 26 | 27 | -- where id is not null 28 | -------------------------------------------------------------------------------- /airbyte_dbt_dagster/dbt_project/models/example/my_second_dbt_model.sql: -------------------------------------------------------------------------------- 1 | 2 | -- Use the `ref` function to select from other models 3 | 4 | select * 5 | from {{ ref('my_first_dbt_model') }} 6 | where id = 1 7 | -------------------------------------------------------------------------------- /airbyte_dbt_dagster/dbt_project/models/example/schema.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2 3 | 4 | models: 5 | - name: my_first_dbt_model 6 | description: "A starter dbt model" 7 | columns: 8 | - name: id 9 | description: "The primary key for this table" 10 | tests: 11 | - unique 12 | - not_null 13 | 14 | - name: my_second_dbt_model 15 | description: "A starter dbt model" 16 | columns: 17 | - name: id 18 | description: "The primary key for this table" 19 | tests: 20 | - unique 21 | - not_null 22 | -------------------------------------------------------------------------------- /airbyte_dbt_dagster/dbt_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_project: 2 | outputs: 3 | dev: 4 | dataset: my_dataset 5 | job_execution_timeout_seconds: 300 6 | job_retries: 1 7 | keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH', '') }}" 8 | location: my_dataset_location 9 | method: service-account 10 | priority: interactive 11 | project: my_project_id 12 | threads: 1 13 | type: bigquery 14 | target: dev -------------------------------------------------------------------------------- /airbyte_dbt_dagster/dbt_project/seeds/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_dagster/dbt_project/seeds/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_dagster/dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_dagster/dbt_project/snapshots/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_dagster/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_dagster/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_dagster/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /airbyte_dbt_dagster/orchestration/orchestration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_dagster/orchestration/orchestration/__init__.py -------------------------------------------------------------------------------- /airbyte_dbt_dagster/orchestration/orchestration/definitions.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dagster import Definitions 4 | from dagster_dbt import DbtCliResource 5 | 6 | from .assets import dbt_project_dbt_assets, airbyte_assets 7 | from .constants import dbt_project_dir 8 | from .schedules import schedules 9 | 10 | defs = Definitions( 11 | assets=[dbt_project_dbt_assets, airbyte_assets], 12 | schedules=schedules, 13 | resources={ 14 | "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)), 15 | }, 16 | ) -------------------------------------------------------------------------------- /airbyte_dbt_dagster/orchestration/orchestration/schedules.py: -------------------------------------------------------------------------------- 1 | """ 2 | To add a daily schedule that materializes your dbt assets, uncomment the following lines. 3 | """ 4 | from dagster_dbt import build_schedule_from_dbt_selection 5 | 6 | from .assets import dbt_project_dbt_assets 7 | 8 | schedules = [ 9 | # build_schedule_from_dbt_selection( 10 | # [dbt_project_dbt_assets], 11 | # job_name="materialize_dbt_models", 12 | # cron_schedule="0 0 * * *", 13 | # dbt_select="fqn:*", 14 | # ), 15 | ] -------------------------------------------------------------------------------- /airbyte_dbt_dagster/orchestration/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.dagster] 6 | module_name = "orchestration.definitions" 7 | code_location_name = "orchestration" -------------------------------------------------------------------------------- /airbyte_dbt_dagster/orchestration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="orchestration", 5 | version="0.0.1", 6 | packages=find_packages(), 7 | install_requires=[ 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dbt-core>=1.4.0", 12 | "dbt-bigquery", 13 | ], 14 | extras_require={ 15 | "dev": [ 16 | "dagster-webserver", 17 | ] 18 | }, 19 | ) -------------------------------------------------------------------------------- /airbyte_dbt_dagster/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="airbyte-dbt-dagster", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-bigquery", 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dagster-airbyte", 12 | ], 13 | extras_require={"dev": ["dagit", "pytest"]}, 14 | ) -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | #Desktop Services Store 7 | .DS_Store 8 | 9 | #User cookie 10 | .user.yml -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | - dbt run 7 | - dbt test 8 | 9 | 10 | ### Resources: 11 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 12 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 13 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 14 | - Find [dbt events](https://events.getdbt.com) near you 15 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 16 | -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/dbt_project/analyses/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_dagster_snowflake/dbt_project/analyses/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/dbt_project/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_dagster_snowflake/dbt_project/macros/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/dbt_project/models/example/my_first_dbt_model.sql: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | Welcome to your first dbt model! 4 | Did you know that you can also configure models directly within SQL files? 5 | This will override configurations stated in dbt_project.yml 6 | 7 | Try changing "table" to "view" below 8 | */ 9 | 10 | {{ config(materialized='table') }} 11 | 12 | with source_data as ( 13 | 14 | select * from {{ source('snowflake', 'sample_table') }} 15 | 16 | ) 17 | 18 | select * 19 | from source_data 20 | 21 | /* 22 | Uncomment the line below to remove records with null `id` values 23 | */ 24 | 25 | -- where id is not null 26 | -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/dbt_project/models/example/my_second_dbt_model.sql: -------------------------------------------------------------------------------- 1 | 2 | -- Use the `ref` function to select from other models 3 | 4 | select * 5 | from {{ ref('my_first_dbt_model') }} 6 | where id = 1 7 | -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/dbt_project/models/example/schema.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2 3 | 4 | models: 5 | - name: my_first_dbt_model 6 | description: "A starter dbt model" 7 | columns: 8 | - name: id 9 | description: "The primary key for this table" 10 | tests: 11 | - unique 12 | - not_null 13 | 14 | - name: my_second_dbt_model 15 | description: "A starter dbt model" 16 | columns: 17 | - name: id 18 | description: "The primary key for this table" 19 | tests: 20 | - unique 21 | - not_null 22 | -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/dbt_project/models/sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: snowflake 5 | tables: 6 | - name: sample_table 7 | meta: 8 | dagster: 9 | asset_key: ["sample_table"] # This metadata specifies the corresponding Dagster asset for this dbt source. 10 | -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/dbt_project/seeds/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_dagster_snowflake/dbt_project/seeds/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_dagster_snowflake/dbt_project/snapshots/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_dagster_snowflake/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/images/dad_snowflake_stack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_dagster_snowflake/images/dad_snowflake_stack.png -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/orchestration/orchestration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_dagster_snowflake/orchestration/orchestration/__init__.py -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/orchestration/orchestration/definitions.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dagster import Definitions 4 | from dagster_dbt import DbtCliResource 5 | 6 | from .assets import dbt_project_dbt_assets, airbyte_assets 7 | from .constants import dbt_project_dir 8 | from .schedules import schedules 9 | 10 | defs = Definitions( 11 | assets=[dbt_project_dbt_assets, airbyte_assets], 12 | schedules=schedules, 13 | resources={ 14 | "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)), 15 | }, 16 | ) -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/orchestration/orchestration/schedules.py: -------------------------------------------------------------------------------- 1 | """ 2 | To add a daily schedule that materializes your dbt assets, uncomment the following lines. 3 | """ 4 | from dagster_dbt import build_schedule_from_dbt_selection 5 | 6 | from .assets import dbt_project_dbt_assets 7 | 8 | schedules = [ 9 | # build_schedule_from_dbt_selection( 10 | # [dbt_project_dbt_assets], 11 | # job_name="materialize_dbt_models", 12 | # cron_schedule="0 0 * * *", 13 | # dbt_select="fqn:*", 14 | # ), 15 | ] -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/orchestration/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.dagster] 6 | module_name = "orchestration.definitions" 7 | code_location_name = "orchestration" -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/orchestration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="orchestration", 5 | version="0.0.1", 6 | packages=find_packages(), 7 | install_requires=[ 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dbt-core>=1.4.0", 12 | "dbt-snowflake", 13 | ], 14 | extras_require={ 15 | "dev": [ 16 | "dagster-webserver", 17 | ] 18 | }, 19 | ) -------------------------------------------------------------------------------- /airbyte_dbt_dagster_snowflake/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="airbyte-dbt-dagster-snowflake", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-snowflake", 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dagster-airbyte", 12 | ], 13 | extras_require={"dev": ["dagit", "pytest"]}, 14 | ) -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | #Desktop Services Store 7 | .DS_Store 8 | 9 | #User cookie 10 | .user.yml -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | - dbt run 7 | - dbt test 8 | 9 | 10 | ### Resources: 11 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 12 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 13 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 14 | - Find [dbt events](https://events.getdbt.com) near you 15 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 16 | -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/dbt_project/analyses/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_prefect_bigquery/dbt_project/analyses/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/dbt_project/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_prefect_bigquery/dbt_project/macros/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/dbt_project/models/marts/product_popularity.sql: -------------------------------------------------------------------------------- 1 | WITH base AS ( 2 | SELECT 3 | product_id, 4 | COUNT(id) AS purchase_count 5 | FROM {{ ref('stg_purchases') }} 6 | GROUP BY 1 7 | ) 8 | 9 | SELECT 10 | p.id, 11 | p.make, 12 | p.model, 13 | b.purchase_count 14 | FROM {{ ref('stg_products') }} p 15 | LEFT JOIN base b ON p.id = b.product_id 16 | ORDER BY b.purchase_count DESC 17 | -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/dbt_project/models/marts/purchase_patterns.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | user_id, 3 | product_id, 4 | purchased_at, 5 | added_to_cart_at, 6 | TIMESTAMP_DIFF(purchased_at, added_to_cart_at, SECOND) AS time_to_purchase_seconds, 7 | returned_at 8 | FROM {{ ref('stg_purchases') }} 9 | -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/dbt_project/models/marts/user_demographics.sql: -------------------------------------------------------------------------------- 1 | WITH base AS ( 2 | SELECT 3 | id AS user_id, 4 | gender, 5 | academic_degree, 6 | nationality, 7 | age 8 | FROM {{ ref('stg_users') }} 9 | ) 10 | 11 | SELECT 12 | gender, 13 | academic_degree, 14 | nationality, 15 | AVG(age) AS average_age, 16 | COUNT(user_id) AS user_count 17 | FROM base 18 | GROUP BY 1, 2, 3 19 | -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/dbt_project/models/staging/stg_products.sql: -------------------------------------------------------------------------------- 1 | select 2 | id, 3 | year, 4 | price, 5 | model, 6 | make, 7 | created_at, 8 | updated_at, 9 | _airbyte_extracted_at, 10 | from {{ source('faker', 'products') }} 11 | -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/dbt_project/models/staging/stg_purchases.sql: -------------------------------------------------------------------------------- 1 | select 2 | id, 3 | user_id, 4 | product_id, 5 | updated_at, 6 | purchased_at, 7 | returned_at, 8 | created_at, 9 | added_to_cart_at, 10 | _airbyte_extracted_at, 11 | from {{ source('faker', 'purchases') }} 12 | -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/dbt_project/models/staging/stg_users.sql: -------------------------------------------------------------------------------- 1 | select 2 | id, 3 | gender, 4 | academic_degree, 5 | title, 6 | nationality, 7 | age, 8 | name, 9 | email, 10 | created_at, 11 | updated_at, 12 | _airbyte_extracted_at, 13 | from {{ source('faker', 'users') }} 14 | -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/dbt_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_project: 2 | outputs: 3 | dev: 4 | dataset: transformed_data 5 | job_execution_timeout_seconds: 300 6 | job_retries: 1 7 | keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH', '') }}" 8 | location: my_dataset_location 9 | method: service-account 10 | priority: interactive 11 | project: my_project_id 12 | threads: 1 13 | type: bigquery 14 | target: dev -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/dbt_project/seeds/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_prefect_bigquery/dbt_project/seeds/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_prefect_bigquery/dbt_project/snapshots/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_prefect_bigquery/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/images/dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_prefect_bigquery/images/dag.png -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/images/pad_bigquery_stack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_prefect_bigquery/images/pad_bigquery_stack.png -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | variable "project_id" { 6 | type = string 7 | } 8 | 9 | variable "credentials_json_path" { 10 | type = string 11 | } -------------------------------------------------------------------------------- /airbyte_dbt_prefect_bigquery/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="airbyte-dbt-prefect-bigquery", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "prefect", 8 | "prefect-airbyte", 9 | "prefect-dbt", 10 | "dbt-bigquery", 11 | ], 12 | extras_require={"dev": ["pytest"]}, 13 | ) -------------------------------------------------------------------------------- /airbyte_dbt_prefect_snowflake/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | #Desktop Services Store 7 | .DS_Store 8 | 9 | #User cookie 10 | .user.yml 11 | -------------------------------------------------------------------------------- /airbyte_dbt_prefect_snowflake/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | - dbt run 7 | - dbt test 8 | 9 | 10 | ### Resources: 11 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 12 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 13 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 14 | - Find [dbt events](https://events.getdbt.com) near you 15 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 16 | -------------------------------------------------------------------------------- /airbyte_dbt_prefect_snowflake/dbt_project/analyses/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_prefect_snowflake/dbt_project/analyses/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_prefect_snowflake/dbt_project/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_prefect_snowflake/dbt_project/macros/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_prefect_snowflake/dbt_project/models/example/my_first_dbt_model.sql: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | Welcome to your first dbt model! 4 | Did you know that you can also configure models directly within SQL files? 5 | This will override configurations stated in dbt_project.yml 6 | 7 | Try changing "table" to "view" below 8 | */ 9 | 10 | {{ config(materialized='table') }} 11 | 12 | with source_data as ( 13 | 14 | select * from {{ source('snowflake', 'sample_table') }} 15 | 16 | ) 17 | 18 | select * 19 | from source_data 20 | 21 | /* 22 | Uncomment the line below to remove records with null `id` values 23 | */ 24 | 25 | -- where id is not null 26 | -------------------------------------------------------------------------------- /airbyte_dbt_prefect_snowflake/dbt_project/models/example/my_second_dbt_model.sql: -------------------------------------------------------------------------------- 1 | 2 | -- Use the `ref` function to select from other models 3 | 4 | select * 5 | from {{ ref('my_first_dbt_model') }} 6 | where id = 1 7 | -------------------------------------------------------------------------------- /airbyte_dbt_prefect_snowflake/dbt_project/models/example/schema.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2 3 | 4 | models: 5 | - name: my_first_dbt_model 6 | description: "A starter dbt model" 7 | columns: 8 | - name: id 9 | description: "The primary key for this table" 10 | tests: 11 | - unique 12 | - not_null 13 | 14 | - name: my_second_dbt_model 15 | description: "A starter dbt model" 16 | columns: 17 | - name: id 18 | description: "The primary key for this table" 19 | tests: 20 | - unique 21 | - not_null 22 | -------------------------------------------------------------------------------- /airbyte_dbt_prefect_snowflake/dbt_project/models/sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: snowflake 5 | tables: 6 | - name: sample_table 7 | meta: 8 | dagster: 9 | asset_key: ["sample_table"] # This metadata specifies the corresponding Dagster asset for this dbt source. 10 | -------------------------------------------------------------------------------- /airbyte_dbt_prefect_snowflake/dbt_project/seeds/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_prefect_snowflake/dbt_project/seeds/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_prefect_snowflake/dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_prefect_snowflake/dbt_project/snapshots/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_prefect_snowflake/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_prefect_snowflake/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_prefect_snowflake/images/infrastructure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_prefect_snowflake/images/infrastructure.png -------------------------------------------------------------------------------- /airbyte_dbt_prefect_snowflake/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | -------------------------------------------------------------------------------- /airbyte_dbt_prefect_snowflake/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="airbyte-dbt-prefect-snowflake", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-snowflake", 8 | "prefect", 9 | "prefect-airbyte", 10 | "prefect-dbt", 11 | "dbt-core>=1.4.0", 12 | ], 13 | extras_require={"dev": ["pytest"]}, 14 | ) 15 | -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | #Desktop Services Store 7 | .DS_Store 8 | 9 | #User cookie 10 | .user.yml -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | - dbt run 7 | - dbt test 8 | 9 | 10 | ### Resources: 11 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 12 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 13 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 14 | - Find [dbt events](https://events.getdbt.com) near you 15 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 16 | -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/dbt_project/analyses/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_snowflake_looker/dbt_project/analyses/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/dbt_project/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_snowflake_looker/dbt_project/macros/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/dbt_project/models/example/my_first_dbt_model.sql: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | Welcome to your first dbt model! 4 | Did you know that you can also configure models directly within SQL files? 5 | This will override configurations stated in dbt_project.yml 6 | 7 | Try changing "table" to "view" below 8 | */ 9 | 10 | {{ config(materialized='table') }} 11 | 12 | with source_data as ( 13 | 14 | select * from {{ source('snowflake', 'sample_table') }} 15 | 16 | ) 17 | 18 | select * 19 | from source_data 20 | 21 | /* 22 | Uncomment the line below to remove records with null `id` values 23 | */ 24 | 25 | -- where id is not null 26 | -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/dbt_project/models/example/my_second_dbt_model.sql: -------------------------------------------------------------------------------- 1 | 2 | -- Use the `ref` function to select from other models 3 | 4 | select * 5 | from {{ ref('my_first_dbt_model') }} 6 | where id = 1 7 | -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/dbt_project/models/example/schema.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2 3 | 4 | models: 5 | - name: my_first_dbt_model 6 | description: "A starter dbt model" 7 | columns: 8 | - name: id 9 | description: "The primary key for this table" 10 | tests: 11 | - unique 12 | - not_null 13 | 14 | - name: my_second_dbt_model 15 | description: "A starter dbt model" 16 | columns: 17 | - name: id 18 | description: "The primary key for this table" 19 | tests: 20 | - unique 21 | - not_null 22 | -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/dbt_project/models/sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: snowflake 5 | tables: 6 | - name: sample_table 7 | meta: 8 | dagster: 9 | asset_key: ["sample_table"] # This metadata specifies the corresponding Dagster asset for this dbt source. 10 | -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/dbt_project/seeds/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_snowflake_looker/dbt_project/seeds/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_snowflake_looker/dbt_project/snapshots/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_snowflake_looker/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/images/adsl_stack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_snowflake_looker/images/adsl_stack.png -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/orchestration/orchestration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_dbt_snowflake_looker/orchestration/orchestration/__init__.py -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/orchestration/orchestration/definitions.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dagster import Definitions 4 | from dagster_dbt import DbtCliResource 5 | 6 | from .assets import dbt_project_dbt_assets, airbyte_assets 7 | from .constants import dbt_project_dir 8 | from .schedules import schedules 9 | 10 | defs = Definitions( 11 | assets=[dbt_project_dbt_assets, airbyte_assets], 12 | schedules=schedules, 13 | resources={ 14 | "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)), 15 | }, 16 | ) -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/orchestration/orchestration/schedules.py: -------------------------------------------------------------------------------- 1 | """ 2 | To add a daily schedule that materializes your dbt assets, uncomment the following lines. 3 | """ 4 | from dagster_dbt import build_schedule_from_dbt_selection 5 | 6 | from .assets import dbt_project_dbt_assets 7 | 8 | schedules = [ 9 | # build_schedule_from_dbt_selection( 10 | # [dbt_project_dbt_assets], 11 | # job_name="materialize_dbt_models", 12 | # cron_schedule="0 0 * * *", 13 | # dbt_select="fqn:*", 14 | # ), 15 | ] -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/orchestration/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.dagster] 6 | module_name = "orchestration.definitions" 7 | code_location_name = "orchestration" -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/orchestration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="orchestration", 5 | version="0.0.1", 6 | packages=find_packages(), 7 | install_requires=[ 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dbt-core>=1.4.0", 12 | "dbt-snowflake", 13 | ], 14 | extras_require={ 15 | "dev": [ 16 | "dagster-webserver", 17 | ] 18 | }, 19 | ) -------------------------------------------------------------------------------- /airbyte_dbt_snowflake_looker/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="airbyte-dbt-snowflake-looker", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-snowflake", 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dagster-airbyte", 12 | ], 13 | extras_require={"dev": ["dagit", "pytest"]}, 14 | ) -------------------------------------------------------------------------------- /airbyte_lib_notebooks/AirbyteLib_CoinAPI_Demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "This demo has been be moved. Click [here](https://github.com/airbytehq/quickstarts/blob/main/pyairbyte_notebooks/PyAirbyte_CoinAPI_Demo.ipynb) to go to the demo. " 8 | ] 9 | } 10 | ], 11 | "metadata": { 12 | "colab": { 13 | "include_colab_link": true, 14 | "provenance": [] 15 | }, 16 | "kernelspec": { 17 | "display_name": "Python 3", 18 | "name": "python3" 19 | }, 20 | "language_info": { 21 | "name": "python" 22 | } 23 | }, 24 | "nbformat": 4, 25 | "nbformat_minor": 0 26 | } 27 | -------------------------------------------------------------------------------- /airbyte_lib_notebooks/PyAirbyte_Shopify_Demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "This demo has been be moved. Click [here](https://github.com/airbytehq/quickstarts/blob/main/pyairbyte_notebooks/PyAirbyte_Shopify_Demo.ipynb) to go to the demo. " 8 | ] 9 | } 10 | ], 11 | "metadata": { 12 | "colab": { 13 | "provenance": [] 14 | }, 15 | "kernelspec": { 16 | "display_name": "Python 3", 17 | "name": "python3" 18 | }, 19 | "language_info": { 20 | "name": "python" 21 | } 22 | }, 23 | "nbformat": 4, 24 | "nbformat_minor": 0 25 | } 26 | -------------------------------------------------------------------------------- /airbyte_s3_pinecone_rag/assets/dataflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/airbyte_s3_pinecone_rag/assets/dataflow.png -------------------------------------------------------------------------------- /airbyte_s3_pinecone_rag/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | #Desktop Services Store 7 | .DS_Store 8 | 9 | #User cookie 10 | .user.yml 11 | .vscode 12 | -------------------------------------------------------------------------------- /airbyte_s3_pinecone_rag/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | - dbt run 7 | - dbt test 8 | 9 | 10 | ### Resources: 11 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 12 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 13 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 14 | - Find [dbt events](https://events.getdbt.com) near you 15 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 16 | -------------------------------------------------------------------------------- /airbyte_s3_pinecone_rag/dbt_project/models/staging/stg_purchases.sql: -------------------------------------------------------------------------------- 1 | -- models/stg_purchases.sql 2 | select 3 | id, 4 | user_id, 5 | product_id, 6 | updated_at, 7 | purchased_at, 8 | returned_at, 9 | created_at, 10 | added_to_cart_at, 11 | _airbyte_extracted_at, 12 | from {{ source('s3', 'purchases') }} 13 | -------------------------------------------------------------------------------- /airbyte_s3_pinecone_rag/dbt_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_project: 2 | outputs: 3 | dev: 4 | type: bigquery 5 | project: "{{ env_var('BIGQUERY_PROJECT_ID') }}" 6 | dataset: "{{ env_var('BIGQUERY_DATASET_ID') }}" 7 | location: "{{ env_var('BIGQUERY_DATASET_LOCATION') }}" 8 | keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH') }}" 9 | method: service-account 10 | priority: interactive 11 | job_execution_timeout_seconds: 300 12 | job_retries: 1 13 | threads: 1 14 | target: dev 15 | -------------------------------------------------------------------------------- /airbyte_s3_pinecone_rag/infra/airbyte/output.tf: -------------------------------------------------------------------------------- 1 | // These variables will be printed at the end of a successful `terraform apply`, 2 | // or by running `terraform output` at any time. 3 | 4 | output "airbyte_cloud_url" { 5 | value = local.airbyte_cloud_url 6 | } 7 | -------------------------------------------------------------------------------- /airbyte_s3_pinecone_rag/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="pyairbyte_s3_pinecone_rag", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-bigquery", 8 | "pinecone-client", 9 | "langchain", 10 | "openai==0.28.1", 11 | "tiktoken", 12 | "rich" 13 | ], 14 | extras_require={"dev": ["pytest"]}, 15 | ) 16 | -------------------------------------------------------------------------------- /api_to_warehouse/images/infrastructure.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/api_to_warehouse/images/infrastructure.jpg -------------------------------------------------------------------------------- /api_to_warehouse/infra/airbyte/provider.tf: -------------------------------------------------------------------------------- 1 | // Airbyte Terraform provider documentation: https://registry.terraform.io/providers/airbytehq/airbyte/latest/docs 2 | 3 | terraform { 4 | required_providers { 5 | airbyte = { 6 | source = "airbytehq/airbyte" 7 | version = "0.3.4" 8 | } 9 | } 10 | } 11 | 12 | provider "airbyte" { 13 | // If running locally (Airbyte OSS) with docker-compose using the airbyte-proxy, 14 | // include the actual password/username you've set up (or use the defaults below) 15 | username = "airbyte" 16 | password = "password" 17 | 18 | // if running locally (Airbyte OSS), include the server url to the airbyte-api-server 19 | server_url = "http://localhost:8006/v1" 20 | } -------------------------------------------------------------------------------- /api_to_warehouse/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "workspace_id" { 3 | type = string 4 | } 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /api_to_warehouse/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="api-to-warehouse", 5 | packages=find_packages(), 6 | install_requires=[ 7 | ], 8 | extras_require={"dev": ["pytest"]}, 9 | ) -------------------------------------------------------------------------------- /customer_segmentation_analytics_shopify/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | .user.yml 7 | -------------------------------------------------------------------------------- /customer_segmentation_analytics_shopify/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | 7 | - dbt run 8 | - dbt test 9 | 10 | ### Resources: 11 | 12 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 13 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 14 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 15 | - Find [dbt events](https://events.getdbt.com) near you 16 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 17 | -------------------------------------------------------------------------------- /customer_segmentation_analytics_shopify/dbt_project/models/marts/customer_activity.sql: -------------------------------------------------------------------------------- 1 | 2 | WITH customer_activity AS ( 3 | SELECT 4 | id, 5 | orders_count, 6 | CASE 7 | WHEN orders_count >= 10 THEN 'Highly Active' 8 | WHEN orders_count >= 5 THEN 'Moderately Active' 9 | ELSE 'Low Activity' 10 | END AS activity_level 11 | FROM transformed_data.stg_customers 12 | ) 13 | SELECT 14 | id, 15 | orders_count, 16 | activity_level 17 | FROM customer_activity 18 | -------------------------------------------------------------------------------- /customer_segmentation_analytics_shopify/dbt_project/models/marts/purchase_pattern_segmentation.sql: -------------------------------------------------------------------------------- 1 | 2 | with purchase_data as ( 3 | select 4 | id, 5 | count(*) as total_purchases, 6 | sum(total_spent) as total_spent 7 | from transformed_data.stg_customers 8 | group by id 9 | ), 10 | behavioral_segments as ( 11 | select 12 | id, 13 | case 14 | when total_purchases >= 5 and total_spent >= 500 then 'High-Value Shoppers' 15 | when total_purchases < 5 and total_spent < 100 then 'Low-Value Shoppers' 16 | else 'Regular Shoppers' 17 | end as purchase_segment 18 | from purchase_data 19 | ) 20 | select * from behavioral_segments 21 | -------------------------------------------------------------------------------- /customer_segmentation_analytics_shopify/dbt_project/models/staging/stg_customers.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('shopify', 'customers') }} 4 | -------------------------------------------------------------------------------- /customer_segmentation_analytics_shopify/dbt_project/models/staging/stg_transactions.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('shopify', 'transactions') }} 4 | -------------------------------------------------------------------------------- /customer_segmentation_analytics_shopify/dbt_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_project: 2 | outputs: 3 | dev: 4 | dataset: transformed_data 5 | job_execution_timeout_seconds: 300 6 | job_retries: 1 7 | # Use an env variable to indicate your JSON key file path 8 | keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH', '') }}" 9 | location: US 10 | method: service-account 11 | priority: interactive 12 | # Indicate your BigQuery project ID 13 | project: "{{ env_var('BIGQUERY_PROJECT_ID', '') }}" 14 | threads: 1 15 | type: bigquery 16 | target: dev -------------------------------------------------------------------------------- /customer_segmentation_analytics_shopify/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "api_password" { 2 | type = string 3 | } 4 | 5 | variable "workspace_id" { 6 | type = string 7 | } 8 | 9 | variable "dataset_id" { 10 | type = string 11 | } 12 | 13 | variable "project_id" { 14 | type = string 15 | } 16 | 17 | variable "credentials_json" { 18 | type = string 19 | } 20 | 21 | variable "shop" { 22 | type = string 23 | } 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /customer_segmentation_analytics_shopify/orchestration/orchestration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/customer_segmentation_analytics_shopify/orchestration/orchestration/__init__.py -------------------------------------------------------------------------------- /customer_segmentation_analytics_shopify/orchestration/orchestration/definitions.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dagster import Definitions 4 | from dagster_dbt import DbtCliResource 5 | 6 | from .assets import dbt_project_dbt_assets, airbyte_assets 7 | from .constants import dbt_project_dir 8 | from .schedules import schedules 9 | 10 | defs = Definitions( 11 | assets=[dbt_project_dbt_assets, airbyte_assets], 12 | schedules=schedules, 13 | resources={ 14 | "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)), 15 | }, 16 | ) 17 | -------------------------------------------------------------------------------- /customer_segmentation_analytics_shopify/orchestration/orchestration/schedules.py: -------------------------------------------------------------------------------- 1 | """ 2 | To add a daily schedule that materializes your dbt assets, uncomment the following lines. 3 | """ 4 | from dagster_dbt import build_schedule_from_dbt_selection 5 | 6 | from .assets import dbt_project_dbt_assets 7 | 8 | schedules = [ 9 | # build_schedule_from_dbt_selection( 10 | # [dbt_project_dbt_assets], 11 | # job_name="materialize_dbt_models", 12 | # cron_schedule="0 0 * * *", 13 | # dbt_select="fqn:*", 14 | # ), 15 | ] 16 | -------------------------------------------------------------------------------- /customer_segmentation_analytics_shopify/orchestration/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.dagster] 6 | module_name = "orchestration.definitions" 7 | code_location_name = "orchestration" -------------------------------------------------------------------------------- /customer_segmentation_analytics_shopify/orchestration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="orchestration", 5 | version="0.0.1", 6 | packages=find_packages(), 7 | install_requires=[ 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dbt-core>=1.4.0", 12 | "dbt-bigquery", 13 | ], 14 | extras_require={ 15 | "dev": [ 16 | "dagster-webserver", 17 | ] 18 | }, 19 | ) -------------------------------------------------------------------------------- /customer_segmentation_analytics_shopify/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="airbyte-dbt-dagster", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-bigquery", 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dagster-airbyte", 12 | ], 13 | extras_require={"dev": ["dagit", "pytest"]}, 14 | ) -------------------------------------------------------------------------------- /data_to_pinecone_llm/.vscode/quickstart.code-workspace: -------------------------------------------------------------------------------- 1 | { 2 | "folders": [ 3 | { 4 | "path": ".." 5 | }, 6 | { 7 | "path": "../.." 8 | } 9 | ], 10 | "settings": {} 11 | } -------------------------------------------------------------------------------- /data_to_pinecone_llm/assets/1-dataflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/data_to_pinecone_llm/assets/1-dataflow.png -------------------------------------------------------------------------------- /data_to_pinecone_llm/assets/2-sources.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/data_to_pinecone_llm/assets/2-sources.png -------------------------------------------------------------------------------- /data_to_pinecone_llm/assets/3-destinations.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/data_to_pinecone_llm/assets/3-destinations.png -------------------------------------------------------------------------------- /data_to_pinecone_llm/assets/4-connections.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/data_to_pinecone_llm/assets/4-connections.png -------------------------------------------------------------------------------- /data_to_pinecone_llm/assets/5-sync-notion.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/data_to_pinecone_llm/assets/5-sync-notion.png -------------------------------------------------------------------------------- /data_to_pinecone_llm/assets/6-bigquery-dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/data_to_pinecone_llm/assets/6-bigquery-dataset.png -------------------------------------------------------------------------------- /data_to_pinecone_llm/assets/7-bigquery-dataset-with-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/data_to_pinecone_llm/assets/7-bigquery-dataset-with-view.png -------------------------------------------------------------------------------- /data_to_pinecone_llm/assets/8-sync-pinecone.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/data_to_pinecone_llm/assets/8-sync-pinecone.png -------------------------------------------------------------------------------- /data_to_pinecone_llm/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | #Desktop Services Store 7 | .DS_Store 8 | 9 | #User cookie 10 | .user.yml -------------------------------------------------------------------------------- /data_to_pinecone_llm/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | - dbt run 7 | - dbt test 8 | 9 | 10 | ### Resources: 11 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 12 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 13 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 14 | - Find [dbt events](https://events.getdbt.com) near you 15 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 16 | -------------------------------------------------------------------------------- /data_to_pinecone_llm/dbt_project/analyses/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/data_to_pinecone_llm/dbt_project/analyses/.gitkeep -------------------------------------------------------------------------------- /data_to_pinecone_llm/dbt_project/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/data_to_pinecone_llm/dbt_project/macros/.gitkeep -------------------------------------------------------------------------------- /data_to_pinecone_llm/dbt_project/models/notion.source.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: notion_raw 5 | database: "{{ env_var('BIGQUERY_PROJECT_ID') }}" 6 | schema: "{{ env_var('BIGQUERY_DATASET_ID') }}" 7 | tables: 8 | - name: notion_blocks 9 | - name: notion_iterator 10 | - name: notion_pages 11 | -------------------------------------------------------------------------------- /data_to_pinecone_llm/dbt_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_project: 2 | outputs: 3 | dev: 4 | type: bigquery 5 | project: "{{ env_var('BIGQUERY_PROJECT_ID') }}" 6 | dataset: "{{ env_var('BIGQUERY_DATASET_ID') }}" 7 | location: "{{ env_var('BIGQUERY_DATASET_LOCATION') }}" 8 | keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH') }}" 9 | method: service-account 10 | priority: interactive 11 | job_execution_timeout_seconds: 300 12 | job_retries: 1 13 | threads: 1 14 | target: dev 15 | -------------------------------------------------------------------------------- /data_to_pinecone_llm/dbt_project/seeds/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/data_to_pinecone_llm/dbt_project/seeds/.gitkeep -------------------------------------------------------------------------------- /data_to_pinecone_llm/dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/data_to_pinecone_llm/dbt_project/snapshots/.gitkeep -------------------------------------------------------------------------------- /data_to_pinecone_llm/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/data_to_pinecone_llm/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /data_to_pinecone_llm/infra/airbyte/output.tf: -------------------------------------------------------------------------------- 1 | // These variables will be printed at the end of a successful `terraform apply`, 2 | // or by running `terraform output` at any time. 3 | 4 | output "airbyte_cloud_url" { 5 | value = local.airbyte_cloud_url 6 | } 7 | -------------------------------------------------------------------------------- /data_to_pinecone_llm/secrets/.gitignore: -------------------------------------------------------------------------------- 1 | # Git should ignore everything in this directory except this gitignore file and the readme. 2 | * 3 | !README.md 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /data_to_pinecone_llm/secrets/README.md: -------------------------------------------------------------------------------- 1 | # Secrets directory 2 | 3 | You can store secrets here and they will be ignored by git. 4 | 5 | For instance, you can save your Google Credentials JSON file here in this directory. 6 | -------------------------------------------------------------------------------- /data_to_pinecone_llm/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="data_to_pinecone_llm", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-bigquery", 8 | "pinecone-client", 9 | "langchain", 10 | "openai==0.28.1", 11 | "tiktoken", 12 | "rich" 13 | ], 14 | extras_require={"dev": ["pytest"]}, 15 | ) 16 | -------------------------------------------------------------------------------- /database_snapshot/images/layout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/database_snapshot/images/layout.png -------------------------------------------------------------------------------- /database_snapshot/infra/airbyte/connections/variables.tf: -------------------------------------------------------------------------------- 1 | variable "s3_source_id" { 2 | type = string 3 | } 4 | 5 | variable "postgres_id" { 6 | type = string 7 | } 8 | 9 | variable "s3_dest_id" { 10 | type = string 11 | } 12 | 13 | variable "bigquery_id" { 14 | type = string 15 | } 16 | 17 | variable "airbyte_password" { 18 | type = string 19 | default = "password" 20 | } 21 | -------------------------------------------------------------------------------- /database_snapshot/infra/airbyte/destinations/outputs.tf: -------------------------------------------------------------------------------- 1 | output "bigquery_id" { 2 | value = airbyte_destination_bigquery.bigquery.destination_id 3 | } 4 | 5 | output "s3_id" { 6 | value = airbyte_destination_s3.s3.destination_id 7 | } 8 | -------------------------------------------------------------------------------- /database_snapshot/infra/airbyte/destinations/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | variable "airbyte_password" { 6 | type = string 7 | default = "password" 8 | } 9 | -------------------------------------------------------------------------------- /database_snapshot/infra/airbyte/main.tf: -------------------------------------------------------------------------------- 1 | // Airbyte Terraform provider documentation: https://registry.terraform.io/providers/airbytehq/airbyte/latest/docs 2 | 3 | module "sources" { 4 | source = "./sources" 5 | workspace_id = "...my_workspace_id..." 6 | } 7 | 8 | module "destination" { 9 | source = "./destinations" 10 | workspace_id = "...my_workspace_id..." 11 | } 12 | 13 | module "connections" { 14 | source = "./connections" 15 | s3_dest_id = module.destination.s3_id 16 | s3_source_id = module.sources.s3_id 17 | postgres_id = module.sources.postgres_id 18 | bigquery_id = module.destination.bigquery_id 19 | } 20 | -------------------------------------------------------------------------------- /database_snapshot/infra/airbyte/sources/outputs.tf: -------------------------------------------------------------------------------- 1 | 2 | output "postgres_id" { 3 | value = airbyte_source_postgres.postgres.source_id 4 | } 5 | 6 | output "s3_id" { 7 | value = airbyte_source_s3.s3.source_id 8 | } -------------------------------------------------------------------------------- /database_snapshot/infra/airbyte/sources/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | variable "airbyte_password" { 6 | type = string 7 | default = "password" 8 | } 9 | -------------------------------------------------------------------------------- /database_snapshot/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | variable "airbyte_password" { 6 | type = string 7 | default = "password" 8 | } 9 | -------------------------------------------------------------------------------- /database_snapshot/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="database-snapshot", 5 | packages=find_packages(), 6 | install_requires=[ 7 | ], 8 | extras_require={"dev": ["pytest"]}, 9 | ) -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | #Desktop Services Store 7 | .DS_Store 8 | 9 | #User cookie 10 | .user.yml -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/marts/avarage_time_to_merge_pr_analysis.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | AVG(TIMESTAMP_DIFF(merged_at, created_at, SECOND)) AS avg_merge_time_seconds 3 | FROM 4 | transformed_data.stg_pull_requests 5 | WHERE 6 | merged_at IS NOT NULL 7 | -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/marts/commits_over_time_per_dev_analysis.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | DATE(created_at) AS commit_date, 3 | JSON_EXTRACT_SCALAR(author, '$.login') AS developer_username, 4 | COUNT(*) AS num_commits 5 | FROM 6 | transformed_data.stg_commits 7 | WHERE 8 | JSON_EXTRACT_SCALAR(author, '$.login') = 'developer_username' 9 | GROUP BY 10 | commit_date, developer_username 11 | ORDER BY 12 | commit_date 13 | -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/marts/dev_activity_by_day_of_week_analysis.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | EXTRACT(DAYOFWEEK FROM created_at) AS day_of_week, 3 | EXTRACT(HOUR FROM created_at) AS hour_of_day, 4 | JSON_EXTRACT_SCALAR(author, '$.login') AS developer_username, 5 | COUNT(*) AS num_commits 6 | FROM 7 | transformed_data.stg_commits 8 | GROUP BY 9 | day_of_week, hour_of_day, developer_username 10 | -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/marts/dev_collaboration_network_analysis.sql: -------------------------------------------------------------------------------- 1 | WITH Collaboration AS ( 2 | SELECT 3 | JSON_EXTRACT_SCALAR(a.author, '$.login') AS developer1, 4 | JSON_EXTRACT_SCALAR(b.author, '$.login') AS developer2, 5 | COUNT(*) AS num_collaborations 6 | FROM 7 | transformed_data.stg_commits AS a 8 | JOIN 9 | transformed_data.stg_commits AS b 10 | ON 11 | a.repository = b.repository 12 | AND a.sha <> b.sha 13 | GROUP BY 14 | developer1, developer2 15 | ) 16 | SELECT 17 | developer1, 18 | developer2, 19 | num_collaborations 20 | FROM 21 | Collaboration 22 | WHERE 23 | num_collaborations > 5 -- Adjust threshold as needed 24 | -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/marts/freq_of_code_contribution_analysis.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | JSON_EXTRACT_SCALAR(author, '$.login') AS developer_username, 3 | COUNT(*) AS num_contributions 4 | FROM 5 | transformed_data.stg_commits 6 | GROUP BY 7 | developer_username 8 | -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/marts/no_of_code_reviews_per_dev_analysis.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | JSON_EXTRACT_SCALAR(requested_reviewers, '$.users[0].login') AS developer_username, 3 | COUNT(*) AS num_reviews 4 | FROM 5 | transformed_data.stg_pull_requests 6 | WHERE 7 | JSON_EXTRACT_SCALAR(requested_reviewers, '$.users[0].login') IS NOT NULL 8 | GROUP BY 9 | developer_username 10 | -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/marts/no_of_commits_per_dev_per_repo_analysis.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | JSON_EXTRACT_SCALAR(author, '$.login') AS developer_username, 3 | repository, 4 | COUNT(*) AS num_commits 5 | FROM 6 | transformed_data.stg_commits 7 | GROUP BY 8 | developer_username, repository 9 | -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/marts/no_of_pr_per_dev_analysis.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | JSON_EXTRACT_SCALAR(user, '$.login') AS developer_username, 3 | COUNT(*) AS num_pull_requests_opened 4 | FROM 5 | transformed_data.stg_pull_requests 6 | GROUP BY 7 | developer_username -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/marts/number_of_pr_open_or_closed.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | JSON_EXTRACT_SCALAR(user, '$.login') AS username, 3 | SUM(CASE WHEN state = 'opened' THEN 1 ELSE 0 END) AS opened_prs, 4 | SUM(CASE WHEN state = 'closed' THEN 1 ELSE 0 END) AS closed_prs 5 | FROM transformed_data.stg_pull_requests 6 | GROUP BY username 7 | -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/marts/track_issues_assigned_by_dev_analysis.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | JSON_EXTRACT_SCALAR(assignee, '$.login') AS developer_username, 3 | COUNT(*) AS num_issues_assigned 4 | FROM 5 | transformed_data.stg_issues 6 | GROUP BY 7 | developer_username 8 | -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/staging/stg_branches.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('github', 'branches') }} 4 | -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/staging/stg_collaborators.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('github', 'collaborators') }} -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/staging/stg_comments.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('github', 'comments') }} -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/staging/stg_commits.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('github', 'commits') }} -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/staging/stg_issues.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('github', 'issues') }} -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/staging/stg_organizations.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('github', 'organizations') }} -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/staging/stg_pull_requests.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('github', 'pull_requests') }} -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/staging/stg_repositories.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('github', 'repositories') }} -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/staging/stg_review_comments.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('github', 'review_comments') }} -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/staging/stg_reviews.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('github', 'reviews') }} -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/staging/stg_stargazers.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('github', 'stargazers') }} -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/staging/stg_tags.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('github', 'tags') }} -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/staging/stg_teams.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('github', 'teams') }} -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/models/staging/stg_users.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('github', 'users') }} 4 | -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_project: 2 | outputs: 3 | dev: 4 | dataset: transformed_data 5 | job_execution_timeout_seconds: 300 6 | job_retries: 1 7 | # Use an env variable to indicate your JSON key file path 8 | keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH', '') }}" 9 | location: US 10 | method: service-account 11 | priority: interactive 12 | # Indicate your BigQuery project ID 13 | project: "{{ env_var('BIGQUERY_PROJECT_ID', '') }}" 14 | threads: 1 15 | type: bigquery 16 | target: dev -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/developer_productivity_analytics_github/dbt_project/snapshots/.gitkeep -------------------------------------------------------------------------------- /developer_productivity_analytics_github/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/developer_productivity_analytics_github/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /developer_productivity_analytics_github/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "personal_access_token" { 2 | type = string 3 | } 4 | 5 | variable "project_id" { 6 | type = string 7 | } 8 | 9 | variable "workspace_id" { 10 | type = string 11 | } 12 | 13 | variable "dataset_id" { 14 | type = string 15 | } 16 | 17 | variable "credentials_json" { 18 | type = string 19 | } 20 | 21 | variable "repository" { 22 | type = string 23 | } 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /developer_productivity_analytics_github/orchestration/orchestration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/developer_productivity_analytics_github/orchestration/orchestration/__init__.py -------------------------------------------------------------------------------- /developer_productivity_analytics_github/orchestration/orchestration/definitions.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dagster import Definitions 4 | from dagster_dbt import DbtCliResource 5 | 6 | from .assets import dbt_project_dbt_assets, airbyte_assets 7 | from .constants import dbt_project_dir 8 | from .schedules import schedules 9 | 10 | defs = Definitions( 11 | assets=[dbt_project_dbt_assets, airbyte_assets], 12 | schedules=schedules, 13 | resources={ 14 | "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)), 15 | }, 16 | ) 17 | -------------------------------------------------------------------------------- /developer_productivity_analytics_github/orchestration/orchestration/schedules.py: -------------------------------------------------------------------------------- 1 | """ 2 | To add a daily schedule that materializes your dbt assets, uncomment the following lines. 3 | """ 4 | from dagster_dbt import build_schedule_from_dbt_selection 5 | 6 | from .assets import dbt_project_dbt_assets 7 | 8 | schedules = [ 9 | # build_schedule_from_dbt_selection( 10 | # [dbt_project_dbt_assets], 11 | # job_name="materialize_dbt_models", 12 | # cron_schedule="0 0 * * *", 13 | # dbt_select="fqn:*", 14 | # ), 15 | ] 16 | -------------------------------------------------------------------------------- /developer_productivity_analytics_github/orchestration/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.dagster] 6 | module_name = "orchestration.definitions" 7 | code_location_name = "orchestration" -------------------------------------------------------------------------------- /developer_productivity_analytics_github/orchestration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="orchestration", 5 | version="0.0.1", 6 | packages=find_packages(), 7 | install_requires=[ 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dbt-core>=1.4.0", 12 | "dbt-bigquery", 13 | ], 14 | extras_require={ 15 | "dev": [ 16 | "dagster-webserver", 17 | ] 18 | }, 19 | ) -------------------------------------------------------------------------------- /developer_productivity_analytics_github/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="airbyte-dbt-dagster", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-bigquery", 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dagster-airbyte", 12 | ], 13 | extras_require={"dev": ["dagit", "pytest"]}, 14 | ) -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | .user.yml 7 | -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | - dbt run 7 | - dbt test 8 | 9 | 10 | ### Resources: 11 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 12 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 13 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 14 | - Find [dbt events](https://events.getdbt.com) near you 15 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 16 | -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/dbt_project/analyses/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/ecommerce_analytics_bigquery/dbt_project/analyses/.gitkeep -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/dbt_project/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/ecommerce_analytics_bigquery/dbt_project/macros/.gitkeep -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/dbt_project/models/marts/product_popularity.sql: -------------------------------------------------------------------------------- 1 | WITH base AS ( 2 | SELECT 3 | product_id, 4 | COUNT(id) AS purchase_count 5 | FROM {{ ref('stg_purchases') }} 6 | GROUP BY 1 7 | ) 8 | 9 | SELECT 10 | p.id, 11 | p.make, 12 | p.model, 13 | b.purchase_count 14 | FROM {{ ref('stg_products') }} p 15 | LEFT JOIN base b ON p.id = b.product_id 16 | ORDER BY b.purchase_count DESC 17 | -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/dbt_project/models/marts/purchase_patterns.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | user_id, 3 | product_id, 4 | purchased_at, 5 | added_to_cart_at, 6 | TIMESTAMP_DIFF(purchased_at, added_to_cart_at, SECOND) AS time_to_purchase_seconds, 7 | returned_at 8 | FROM {{ ref('stg_purchases') }} 9 | -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/dbt_project/models/marts/user_demographics.sql: -------------------------------------------------------------------------------- 1 | WITH base AS ( 2 | SELECT 3 | id AS user_id, 4 | gender, 5 | academic_degree, 6 | nationality, 7 | age 8 | FROM {{ ref('stg_users') }} 9 | ) 10 | 11 | SELECT 12 | gender, 13 | academic_degree, 14 | nationality, 15 | AVG(age) AS average_age, 16 | COUNT(user_id) AS user_count 17 | FROM base 18 | GROUP BY 1, 2, 3 19 | -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/dbt_project/models/staging/stg_products.sql: -------------------------------------------------------------------------------- 1 | select 2 | id, 3 | year, 4 | price, 5 | model, 6 | make, 7 | created_at, 8 | updated_at, 9 | _airbyte_extracted_at, 10 | from {{ source('faker', 'products') }} 11 | -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/dbt_project/models/staging/stg_purchases.sql: -------------------------------------------------------------------------------- 1 | select 2 | id, 3 | user_id, 4 | product_id, 5 | updated_at, 6 | purchased_at, 7 | returned_at, 8 | created_at, 9 | added_to_cart_at, 10 | _airbyte_extracted_at, 11 | from {{ source('faker', 'purchases') }} 12 | -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/dbt_project/models/staging/stg_users.sql: -------------------------------------------------------------------------------- 1 | select 2 | id, 3 | gender, 4 | academic_degree, 5 | title, 6 | nationality, 7 | age, 8 | name, 9 | email, 10 | created_at, 11 | updated_at, 12 | _airbyte_extracted_at, 13 | from {{ source('faker', 'users') }} 14 | -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/dbt_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_project: 2 | outputs: 3 | dev: 4 | dataset: transformed_data 5 | job_execution_timeout_seconds: 300 6 | job_retries: 1 7 | # Use an env variable to indicate your JSON key file path 8 | keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH', '') }}" 9 | location: US 10 | method: service-account 11 | priority: interactive 12 | # Indicate your BigQuery project ID 13 | project: your_project_id 14 | threads: 1 15 | type: bigquery 16 | target: dev -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/dbt_project/seeds/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/ecommerce_analytics_bigquery/dbt_project/seeds/.gitkeep -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/ecommerce_analytics_bigquery/dbt_project/snapshots/.gitkeep -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/ecommerce_analytics_bigquery/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | variable "dataset_id" { 6 | type = string 7 | } 8 | 9 | variable "project_id" { 10 | type = string 11 | } 12 | 13 | variable "credentials_json" { 14 | type = string 15 | } 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/orchestration/orchestration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/ecommerce_analytics_bigquery/orchestration/orchestration/__init__.py -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/orchestration/orchestration/definitions.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dagster import Definitions 4 | from dagster_dbt import DbtCliResource 5 | 6 | from .assets import dbt_project_dbt_assets, airbyte_assets 7 | from .constants import dbt_project_dir 8 | from .schedules import schedules 9 | 10 | defs = Definitions( 11 | assets=[dbt_project_dbt_assets, airbyte_assets], 12 | schedules=schedules, 13 | resources={ 14 | "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)), 15 | }, 16 | ) -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/orchestration/orchestration/schedules.py: -------------------------------------------------------------------------------- 1 | """ 2 | To add a daily schedule that materializes your dbt assets, uncomment the following lines. 3 | """ 4 | from dagster_dbt import build_schedule_from_dbt_selection 5 | 6 | from .assets import dbt_project_dbt_assets 7 | 8 | schedules = [ 9 | # build_schedule_from_dbt_selection( 10 | # [dbt_project_dbt_assets], 11 | # job_name="materialize_dbt_models", 12 | # cron_schedule="0 0 * * *", 13 | # dbt_select="fqn:*", 14 | # ), 15 | ] -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/orchestration/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.dagster] 6 | module_name = "orchestration.definitions" 7 | code_location_name = "orchestration" -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/orchestration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="orchestration", 5 | version="0.0.1", 6 | packages=find_packages(), 7 | install_requires=[ 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dbt-core>=1.4.0", 12 | "dbt-bigquery", 13 | ], 14 | extras_require={ 15 | "dev": [ 16 | "dagster-webserver", 17 | ] 18 | }, 19 | ) -------------------------------------------------------------------------------- /ecommerce_analytics_bigquery/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="airbyte-dbt-dagster", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-bigquery", 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dagster-airbyte", 12 | ], 13 | extras_require={"dev": ["dagit", "pytest"]}, 14 | ) -------------------------------------------------------------------------------- /elt_simplified_stack/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | #Desktop Services Store 7 | .DS_Store 8 | 9 | #User cookie 10 | .user.yml 11 | -------------------------------------------------------------------------------- /elt_simplified_stack/dbt_project/models/marts/commits-per-repo.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | JSON_EXTRACT_SCALAR(author, '$.login') AS developer_username, 3 | repository, 4 | COUNT(*) AS num_commits 5 | FROM 6 | transformed_data.stg_commits 7 | GROUP BY 8 | developer_username, repository 9 | -------------------------------------------------------------------------------- /elt_simplified_stack/dbt_project/models/marts/pr-per-dev.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | JSON_EXTRACT_SCALAR(user, '$.login') AS developer_username, 3 | COUNT(*) AS num_pull_requests_opened 4 | FROM 5 | transformed_data.stg_pull_requests 6 | GROUP BY 7 | developer_username 8 | 9 | -------------------------------------------------------------------------------- /elt_simplified_stack/dbt_project/models/marts/pr-per-status.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | JSON_EXTRACT_SCALAR(user, '$.login') AS username, 3 | SUM(CASE WHEN state = 'opened' THEN 1 ELSE 0 END) AS opened_prs, 4 | SUM(CASE WHEN state = 'closed' THEN 1 ELSE 0 END) AS closed_prs 5 | FROM transformed_data.stg_pull_requests 6 | GROUP BY username 7 | -------------------------------------------------------------------------------- /elt_simplified_stack/dbt_project/models/staging/stg_commits.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('github', 'commits') }} 4 | -------------------------------------------------------------------------------- /elt_simplified_stack/dbt_project/models/staging/stg_pull_requests.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('github', 'pull_requests') }} 4 | -------------------------------------------------------------------------------- /elt_simplified_stack/dbt_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_project: 2 | outputs: 3 | dev: 4 | dataset: transformed_data 5 | job_execution_timeout_seconds: 300 6 | job_retries: 1 7 | # Use an env variable to indicate your JSON key file path 8 | keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH', '') }}" 9 | location: US 10 | method: service-account 11 | priority: interactive 12 | # Indicate your BigQuery project ID 13 | project: "{{ env_var('BIGQUERY_PROJECT_ID', '') }}" 14 | threads: 1 15 | type: bigquery 16 | target: dev 17 | -------------------------------------------------------------------------------- /elt_simplified_stack/dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/elt_simplified_stack/dbt_project/snapshots/.gitkeep -------------------------------------------------------------------------------- /elt_simplified_stack/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/elt_simplified_stack/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /elt_simplified_stack/images/infrastrcuture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/elt_simplified_stack/images/infrastrcuture.png -------------------------------------------------------------------------------- /elt_simplified_stack/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "personal_access_token" { 2 | type = string 3 | } 4 | 5 | variable "project_id" { 6 | type = string 7 | } 8 | 9 | variable "workspace_id" { 10 | type = string 11 | } 12 | 13 | variable "dataset_id" { 14 | type = string 15 | } 16 | 17 | variable "credentials_json" { 18 | type = string 19 | } 20 | 21 | variable "repository" { 22 | type = string 23 | } 24 | -------------------------------------------------------------------------------- /elt_simplified_stack/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="elt-simplified" 5 | packages=find_packages(), 6 | install_requires=[ 7 | "prefect", 8 | "prefect-airbyte", 9 | "prefect-dbt", 10 | "dbt-core>=1.4.0", 11 | "dbt-bigquery", 12 | ], 13 | extras_require={"dev": ["pytest"]}, 14 | ) 15 | -------------------------------------------------------------------------------- /error_analysis_stack_sentry/assets/Infrastructure_layout.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/error_analysis_stack_sentry/assets/Infrastructure_layout.jpg -------------------------------------------------------------------------------- /error_analysis_stack_sentry/assets/orchestration_lineage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/error_analysis_stack_sentry/assets/orchestration_lineage.png -------------------------------------------------------------------------------- /error_analysis_stack_sentry/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | .user.yml -------------------------------------------------------------------------------- /error_analysis_stack_sentry/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | - dbt run 7 | - dbt test 8 | 9 | 10 | ### Resources: 11 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 12 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 13 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 14 | - Find [dbt events](https://events.getdbt.com) near you 15 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 16 | -------------------------------------------------------------------------------- /error_analysis_stack_sentry/dbt_project/analyses/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/error_analysis_stack_sentry/dbt_project/analyses/.gitkeep -------------------------------------------------------------------------------- /error_analysis_stack_sentry/dbt_project/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/error_analysis_stack_sentry/dbt_project/macros/.gitkeep -------------------------------------------------------------------------------- /error_analysis_stack_sentry/dbt_project/models/example/schema.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2 3 | 4 | models: 5 | - name: Insight_Table 6 | description: "A starter dbt model" 7 | columns: 8 | - name: id 9 | description: "The primary key for this table" 10 | tests: 11 | - unique 12 | - not_null 13 | 14 | -------------------------------------------------------------------------------- /error_analysis_stack_sentry/dbt_project/seeds/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/error_analysis_stack_sentry/dbt_project/seeds/.gitkeep -------------------------------------------------------------------------------- /error_analysis_stack_sentry/dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/error_analysis_stack_sentry/dbt_project/snapshots/.gitkeep -------------------------------------------------------------------------------- /error_analysis_stack_sentry/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/error_analysis_stack_sentry/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /error_analysis_stack_sentry/error_analysis_stack.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 2.1 2 | Name: error-analysis-stack 3 | Version: 0.0.1 4 | Provides-Extra: dev 5 | -------------------------------------------------------------------------------- /error_analysis_stack_sentry/error_analysis_stack.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | setup.py 2 | error_analysis_stack.egg-info/PKG-INFO 3 | error_analysis_stack.egg-info/SOURCES.txt 4 | error_analysis_stack.egg-info/dependency_links.txt 5 | error_analysis_stack.egg-info/requires.txt 6 | error_analysis_stack.egg-info/top_level.txt -------------------------------------------------------------------------------- /error_analysis_stack_sentry/error_analysis_stack.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /error_analysis_stack_sentry/error_analysis_stack.egg-info/requires.txt: -------------------------------------------------------------------------------- 1 | dagster 2 | dagster-cloud 3 | dagster-airbyte 4 | dagster-dbt 5 | dbt-core>=1.4.0 6 | dbt-snowflake 7 | 8 | [dev] 9 | dagster-webserver 10 | pytest 11 | -------------------------------------------------------------------------------- /error_analysis_stack_sentry/error_analysis_stack.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /error_analysis_stack_sentry/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "workspace_id" { 3 | type = string 4 | } 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /error_analysis_stack_sentry/orchestration/orchestration.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 2.1 2 | Name: orchestration 3 | Version: 0.0.1 4 | Requires-Dist: dagster 5 | Requires-Dist: dagster-cloud 6 | Requires-Dist: dagster-dbt 7 | Requires-Dist: dbt-core>=1.4.0 8 | Requires-Dist: dbt-snowflake 9 | Provides-Extra: dev 10 | Requires-Dist: dagster-webserver; extra == "dev" 11 | -------------------------------------------------------------------------------- /error_analysis_stack_sentry/orchestration/orchestration.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | pyproject.toml 2 | setup.py 3 | orchestration/__init__.py 4 | orchestration/assets.py 5 | orchestration/constants.py 6 | orchestration/definitions.py 7 | orchestration/schedules.py 8 | orchestration.egg-info/PKG-INFO 9 | orchestration.egg-info/SOURCES.txt 10 | orchestration.egg-info/dependency_links.txt 11 | orchestration.egg-info/requires.txt 12 | orchestration.egg-info/top_level.txt -------------------------------------------------------------------------------- /error_analysis_stack_sentry/orchestration/orchestration.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /error_analysis_stack_sentry/orchestration/orchestration.egg-info/requires.txt: -------------------------------------------------------------------------------- 1 | dagster 2 | dagster-cloud 3 | dagster-dbt 4 | dbt-core>=1.4.0 5 | dbt-snowflake 6 | 7 | [dev] 8 | dagster-webserver 9 | -------------------------------------------------------------------------------- /error_analysis_stack_sentry/orchestration/orchestration.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | orchestration 2 | -------------------------------------------------------------------------------- /error_analysis_stack_sentry/orchestration/orchestration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/error_analysis_stack_sentry/orchestration/orchestration/__init__.py -------------------------------------------------------------------------------- /error_analysis_stack_sentry/orchestration/orchestration/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/error_analysis_stack_sentry/orchestration/orchestration/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /error_analysis_stack_sentry/orchestration/orchestration/__pycache__/assets.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/error_analysis_stack_sentry/orchestration/orchestration/__pycache__/assets.cpython-310.pyc -------------------------------------------------------------------------------- /error_analysis_stack_sentry/orchestration/orchestration/__pycache__/constants.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/error_analysis_stack_sentry/orchestration/orchestration/__pycache__/constants.cpython-310.pyc -------------------------------------------------------------------------------- /error_analysis_stack_sentry/orchestration/orchestration/__pycache__/definitions.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/error_analysis_stack_sentry/orchestration/orchestration/__pycache__/definitions.cpython-310.pyc -------------------------------------------------------------------------------- /error_analysis_stack_sentry/orchestration/orchestration/__pycache__/schedules.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/error_analysis_stack_sentry/orchestration/orchestration/__pycache__/schedules.cpython-310.pyc -------------------------------------------------------------------------------- /error_analysis_stack_sentry/orchestration/orchestration/definitions.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dagster import Definitions 4 | from dagster_dbt import DbtCliResource 5 | 6 | from .assets import dbt_project_dbt_assets, airbyte_assets 7 | from .constants import dbt_project_dir 8 | from .schedules import schedules 9 | 10 | defs = Definitions( 11 | assets=[dbt_project_dbt_assets, airbyte_assets], 12 | schedules=schedules, 13 | resources={ 14 | "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)), 15 | }, 16 | ) -------------------------------------------------------------------------------- /error_analysis_stack_sentry/orchestration/orchestration/schedules.py: -------------------------------------------------------------------------------- 1 | """ 2 | To add a daily schedule that materializes your dbt assets, uncomment the following lines. 3 | """ 4 | from dagster_dbt import build_schedule_from_dbt_selection 5 | 6 | from .assets import dbt_project_dbt_assets 7 | 8 | schedules = [ 9 | # build_schedule_from_dbt_selection( 10 | # [dbt_project_dbt_assets], 11 | # job_name="materialize_dbt_models", 12 | # cron_schedule="0 0 * * *", 13 | # dbt_select="fqn:*", 14 | # ), 15 | ] -------------------------------------------------------------------------------- /error_analysis_stack_sentry/orchestration/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.dagster] 6 | module_name = "orchestration.definitions" 7 | code_location_name = "orchestration" -------------------------------------------------------------------------------- /error_analysis_stack_sentry/orchestration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="orchestration", 5 | version="0.0.1", 6 | packages=find_packages(), 7 | install_requires=[ 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dbt-core>=1.4.0", 12 | "dbt-snowflake", 13 | ], 14 | extras_require={ 15 | "dev": [ 16 | "dagster-webserver", 17 | ] 18 | }, 19 | ) -------------------------------------------------------------------------------- /error_analysis_stack_sentry/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="error-analysis-stack", 5 | version="0.0.1", 6 | packages=find_packages(), 7 | install_requires=[ 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-airbyte", 11 | "dagster-dbt", 12 | "dbt-core>=1.4.0", 13 | "dbt-snowflake", 14 | ], 15 | extras_require={ 16 | "dev": [ 17 | "dagster-webserver", 18 | "pytest", 19 | ] 20 | }, 21 | ) -------------------------------------------------------------------------------- /github_insight_stack/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/github_insight_stack/architecture.png -------------------------------------------------------------------------------- /github_insight_stack/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | .user.yml 7 | -------------------------------------------------------------------------------- /github_insight_stack/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | - dbt run 7 | - dbt test 8 | 9 | 10 | ### Resources: 11 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 12 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 13 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 14 | - Find [dbt events](https://events.getdbt.com) near you 15 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 16 | -------------------------------------------------------------------------------- /github_insight_stack/dbt_project/analyses/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/github_insight_stack/dbt_project/analyses/.gitkeep -------------------------------------------------------------------------------- /github_insight_stack/dbt_project/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/github_insight_stack/dbt_project/macros/.gitkeep -------------------------------------------------------------------------------- /github_insight_stack/dbt_project/models/Readme.md: -------------------------------------------------------------------------------- 1 | 2 | # Notes 3 | 4 | - These models are basic starting points. You can further refine and expand on them based on the insights you 5 | want and the intricacies of your GitHub data. 6 | 7 | - Make sure to adjust field names if they don't match the schema of the data you've extracted from GitHub into BigQuery. 8 | 9 | - It might also be beneficial to integrate other dbt packages or custom macros to handle more advanced transformations or aggregations as needed. 10 | 11 | - Always remember to test your models with dbt to ensure they run correctly and produce the expected results. -------------------------------------------------------------------------------- /github_insight_stack/dbt_project/models/sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: github_raw 5 | database: your_bigquery_project_id 6 | schema: github_raw_data 7 | 8 | tables: 9 | - name: repositories 10 | - name: commits 11 | - name: pull_requests 12 | -------------------------------------------------------------------------------- /github_insight_stack/dbt_project/models/test-models/code_quality.sql: -------------------------------------------------------------------------------- 1 | WITH commit_analysis AS ( 2 | SELECT 3 | committer_name, 4 | COUNT(*) AS number_of_commits, 5 | AVG(LENGTH(commit_message)) AS avg_commit_message_length 6 | FROM {{ source('github_raw', 'commits') }} 7 | GROUP BY 1 8 | ) 9 | 10 | SELECT * FROM commit_analysis -------------------------------------------------------------------------------- /github_insight_stack/dbt_project/models/test-models/collaboration_patterns.sql: -------------------------------------------------------------------------------- 1 | WITH pr_analysis AS ( 2 | SELECT 3 | author_name, 4 | COUNT(*) AS number_of_prs, 5 | AVG(LENGTH(pr_title)) AS avg_pr_title_length 6 | FROM {{ source('github_raw', 'pull_requests') }} 7 | GROUP BY 1 8 | ) 9 | 10 | SELECT * FROM pr_analysis 11 | -------------------------------------------------------------------------------- /github_insight_stack/dbt_project/models/test-models/project_health.sql: -------------------------------------------------------------------------------- 1 | WITH contributor_analysis AS ( 2 | SELECT 3 | committer_name, 4 | COUNT(DISTINCT repo_name) AS repos_contributed_to, 5 | COUNT(*) AS number_of_commits 6 | FROM {{ source('github_raw', 'commits') }} 7 | GROUP BY 1 8 | ) 9 | 10 | SELECT * FROM contributor_analysis 11 | -------------------------------------------------------------------------------- /github_insight_stack/dbt_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_project: 2 | outputs: 3 | dev: 4 | dataset: github_transformed_data 5 | job_execution_timeout_seconds: 300 6 | job_retries: 1 7 | # Use an env variable to indicate your JSON key file path 8 | keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH', '') }}" 9 | location: US 10 | method: service-account 11 | priority: interactive 12 | # Indicate your BigQuery project ID 13 | project: your_project_id 14 | threads: 1 15 | type: bigquery 16 | target: dev -------------------------------------------------------------------------------- /github_insight_stack/dbt_project/seeds/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/github_insight_stack/dbt_project/seeds/.gitkeep -------------------------------------------------------------------------------- /github_insight_stack/dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/github_insight_stack/dbt_project/snapshots/.gitkeep -------------------------------------------------------------------------------- /github_insight_stack/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/github_insight_stack/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /github_insight_stack/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | variable "dataset_id" { 6 | type = string 7 | } 8 | 9 | variable "project_id" { 10 | type = string 11 | } 12 | 13 | variable "credentials_json" { 14 | type = string 15 | } 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /github_insight_stack/orchestration/orchestration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/github_insight_stack/orchestration/orchestration/__init__.py -------------------------------------------------------------------------------- /github_insight_stack/orchestration/orchestration/definitions.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dagster import Definitions 4 | from dagster_dbt import DbtCliResource 5 | 6 | from .assets import dbt_project_dbt_assets, airbyte_assets 7 | from .constants import dbt_project_dir 8 | from .schedules import schedules 9 | 10 | defs = Definitions( 11 | assets=[dbt_project_dbt_assets, airbyte_assets], 12 | schedules=schedules, 13 | resources={ 14 | "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)), 15 | }, 16 | ) -------------------------------------------------------------------------------- /github_insight_stack/orchestration/orchestration/schedules.py: -------------------------------------------------------------------------------- 1 | """ 2 | To add a daily schedule that materializes your dbt assets, uncomment the following lines. 3 | """ 4 | from dagster_dbt import build_schedule_from_dbt_selection 5 | 6 | from .assets import dbt_project_dbt_assets 7 | 8 | schedules = [ 9 | # build_schedule_from_dbt_selection( 10 | # [dbt_project_dbt_assets], 11 | # job_name="materialize_dbt_models", 12 | # cron_schedule="0 0 * * *", 13 | # dbt_select="fqn:*", 14 | # ), 15 | ] -------------------------------------------------------------------------------- /github_insight_stack/orchestration/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.dagster] 6 | module_name = "orchestration.definitions" 7 | code_location_name = "orchestration" -------------------------------------------------------------------------------- /github_insight_stack/orchestration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="orchestration", 5 | version="0.0.1", 6 | packages=find_packages(), 7 | install_requires=[ 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dbt-core>=1.4.0", 12 | "dbt-bigquery", 13 | ], 14 | extras_require={ 15 | "dev": [ 16 | "dagster-webserver", 17 | ] 18 | }, 19 | ) -------------------------------------------------------------------------------- /github_insight_stack/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="airbyte-dbt-dagster", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-bigquery", 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dagster-airbyte", 12 | ], 13 | extras_require={"dev": ["dagit", "pytest"]}, 14 | ) -------------------------------------------------------------------------------- /low_latency_data_availability/images/infrastructure_layout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/low_latency_data_availability/images/infrastructure_layout.png -------------------------------------------------------------------------------- /low_latency_data_availability/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /low_latency_data_availability/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="low-latency-data-availability", 5 | packages=find_packages(), 6 | install_requires=[ 7 | ], 8 | extras_require={"dev": ["pytest"]}, 9 | ) -------------------------------------------------------------------------------- /mongodb_mysql_integration/images/Infrastructure_layout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/mongodb_mysql_integration/images/Infrastructure_layout.png -------------------------------------------------------------------------------- /mongodb_mysql_integration/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /mongodb_mysql_integration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="postgres-data-replication", 5 | packages=find_packages(), 6 | install_requires=[ 7 | ], 8 | extras_require={"dev": ["pytest"]}, 9 | ) 10 | -------------------------------------------------------------------------------- /multisource_aggregation/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | #Desktop Services Store 7 | .DS_Store 8 | 9 | #User cookie 10 | .user.yml -------------------------------------------------------------------------------- /multisource_aggregation/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | - dbt run 7 | - dbt test 8 | 9 | 10 | ### Resources: 11 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 12 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 13 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 14 | - Find [dbt events](https://events.getdbt.com) near you 15 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 16 | -------------------------------------------------------------------------------- /multisource_aggregation/dbt_project/analyses/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/multisource_aggregation/dbt_project/analyses/.gitkeep -------------------------------------------------------------------------------- /multisource_aggregation/dbt_project/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/multisource_aggregation/dbt_project/macros/.gitkeep -------------------------------------------------------------------------------- /multisource_aggregation/dbt_project/models/example/my_second_dbt_model.sql: -------------------------------------------------------------------------------- 1 | 2 | -- Use the `ref` function to select from other models 3 | 4 | select * 5 | from {{ ref('my_first_dbt_model') }} 6 | where id = 1 7 | -------------------------------------------------------------------------------- /multisource_aggregation/dbt_project/models/example/schema.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2 3 | 4 | models: 5 | - name: my_first_dbt_model 6 | description: "A starter dbt model" 7 | columns: 8 | - name: id 9 | description: "The primary key for this table" 10 | tests: 11 | - unique 12 | - not_null 13 | 14 | - name: my_second_dbt_model 15 | description: "A starter dbt model" 16 | columns: 17 | - name: id 18 | description: "The primary key for this table" 19 | tests: 20 | - unique 21 | - not_null 22 | -------------------------------------------------------------------------------- /multisource_aggregation/dbt_project/models/sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: bigquery 5 | tables: 6 | - name: sample_table 7 | meta: 8 | dagster: 9 | asset_key: ["pg_sample_table"] # This metadata specifies the corresponding Dagster asset for this dbt source. 10 | 11 | - name: test_table 12 | meta: 13 | dagster: 14 | asset_key: ["mysql_test_table"] # This metadata specifies the corresponding Dagster asset for this dbt source. -------------------------------------------------------------------------------- /multisource_aggregation/dbt_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_project: 2 | outputs: 3 | dev: 4 | dataset: my_dataset 5 | job_execution_timeout_seconds: 300 6 | job_retries: 1 7 | keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH', '') }}" 8 | location: my_dataset_location 9 | method: service-account 10 | priority: interactive 11 | project: my_project_id 12 | threads: 1 13 | type: bigquery 14 | target: dev -------------------------------------------------------------------------------- /multisource_aggregation/dbt_project/seeds/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/multisource_aggregation/dbt_project/seeds/.gitkeep -------------------------------------------------------------------------------- /multisource_aggregation/dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/multisource_aggregation/dbt_project/snapshots/.gitkeep -------------------------------------------------------------------------------- /multisource_aggregation/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/multisource_aggregation/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /multisource_aggregation/images/layout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/multisource_aggregation/images/layout.png -------------------------------------------------------------------------------- /multisource_aggregation/infra/airbyte/connections/variables.tf: -------------------------------------------------------------------------------- 1 | variable "mysql_id" { 2 | type = string 3 | } 4 | 5 | variable "postgres_id" { 6 | type = string 7 | } 8 | 9 | variable "bigquery_id" { 10 | type = string 11 | } 12 | 13 | variable "airbyte_password" { 14 | type = string 15 | default = "password" 16 | } 17 | -------------------------------------------------------------------------------- /multisource_aggregation/infra/airbyte/destination_warehouse/outputs.tf: -------------------------------------------------------------------------------- 1 | output "bigquery_id" { 2 | value = airbyte_destination_bigquery.bigquery.destination_id 3 | } -------------------------------------------------------------------------------- /multisource_aggregation/infra/airbyte/destination_warehouse/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | variable "airbyte_password" { 6 | type = string 7 | default = "password" 8 | } 9 | -------------------------------------------------------------------------------- /multisource_aggregation/infra/airbyte/main.tf: -------------------------------------------------------------------------------- 1 | // Airbyte Terraform provider documentation: https://registry.terraform.io/providers/airbytehq/airbyte/latest/docs 2 | 3 | module "sources" { 4 | source = "./source_databases" 5 | workspace_id = "...my_workspace_id..." 6 | } 7 | 8 | module "destination" { 9 | source = "./destination_warehouse" 10 | workspace_id = "...my_workspace_id..." 11 | } 12 | 13 | module "connections" { 14 | source = "./connections" 15 | mysql_id = module.sources.mysql_id 16 | postgres_id = module.sources.postgres_id 17 | bigquery_id = module.destination.bigquery_id 18 | } 19 | -------------------------------------------------------------------------------- /multisource_aggregation/infra/airbyte/source_databases/outputs.tf: -------------------------------------------------------------------------------- 1 | 2 | output "postgres_id" { 3 | value = airbyte_source_postgres.postgres.source_id 4 | } 5 | 6 | output "mysql_id" { 7 | value = airbyte_source_mysql.mysql.source_id 8 | } -------------------------------------------------------------------------------- /multisource_aggregation/infra/airbyte/source_databases/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | variable "airbyte_password" { 6 | type = string 7 | default = "password" 8 | } 9 | -------------------------------------------------------------------------------- /multisource_aggregation/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | variable "airbyte_password" { 6 | type = string 7 | default = "password" 8 | } 9 | -------------------------------------------------------------------------------- /multisource_aggregation/orchestration/orchestration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/multisource_aggregation/orchestration/orchestration/__init__.py -------------------------------------------------------------------------------- /multisource_aggregation/orchestration/orchestration/definitions.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dagster import Definitions 4 | from dagster_dbt import DbtCliResource 5 | 6 | from .assets import dbt_project_dbt_assets, airbyte_assets 7 | from .constants import dbt_project_dir 8 | from .schedules import schedules 9 | 10 | defs = Definitions( 11 | assets=[dbt_project_dbt_assets, airbyte_assets], 12 | schedules=schedules, 13 | resources={ 14 | "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)), 15 | }, 16 | ) -------------------------------------------------------------------------------- /multisource_aggregation/orchestration/orchestration/schedules.py: -------------------------------------------------------------------------------- 1 | """ 2 | To add a daily schedule that materializes your dbt assets, uncomment the following lines. 3 | """ 4 | from dagster_dbt import build_schedule_from_dbt_selection 5 | 6 | from .assets import dbt_project_dbt_assets 7 | 8 | schedules = [ 9 | # build_schedule_from_dbt_selection( 10 | # [dbt_project_dbt_assets], 11 | # job_name="materialize_dbt_models", 12 | # cron_schedule="0 0 * * *", 13 | # dbt_select="fqn:*", 14 | # ), 15 | ] -------------------------------------------------------------------------------- /multisource_aggregation/orchestration/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.dagster] 6 | module_name = "orchestration.definitions" 7 | code_location_name = "orchestration" -------------------------------------------------------------------------------- /multisource_aggregation/orchestration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="orchestration", 5 | version="0.0.1", 6 | packages=find_packages(), 7 | install_requires=[ 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dbt-core>=1.4.0", 12 | "dbt-bigquery", 13 | ], 14 | extras_require={ 15 | "dev": [ 16 | "dagster-webserver", 17 | ] 18 | }, 19 | ) -------------------------------------------------------------------------------- /multisource_aggregation/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="multisource-database-aggregation", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-bigquery", 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dagster-airbyte", 12 | ], 13 | extras_require={"dev": ["dagit", "pytest"]}, 14 | ) -------------------------------------------------------------------------------- /mysql_to_postgres_incremental_stack/image/sample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/mysql_to_postgres_incremental_stack/image/sample.png -------------------------------------------------------------------------------- /mysql_to_postgres_incremental_stack/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /mysql_to_postgres_incremental_stack/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="postgres-to-mysql-migration", 5 | packages=find_packages(), 6 | install_requires=[ 7 | ], 8 | extras_require={"dev": ["pytest"]}, 9 | ) -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | .user.yml 7 | -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | 7 | - dbt run 8 | - dbt test 9 | 10 | ### Resources: 11 | 12 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 13 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 14 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 15 | - Find [dbt events](https://events.getdbt.com) near you 16 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 17 | -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/dbt_project/models/marts/campsite_availability_over_time.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | DATE(CreatedDate) AS date, 3 | COUNT(*) AS campsite_count 4 | FROM 5 | transformed_data.stg_campsites 6 | GROUP BY 7 | date 8 | ORDER BY 9 | date -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/dbt_project/models/marts/campsite_type_counts.sql: -------------------------------------------------------------------------------- 1 | WITH campsite_type_counts AS ( 2 | SELECT 3 | CampsiteType, 4 | COUNT(*) AS campsite_count 5 | FROM 6 | {{ ref('stg_campsites') }} 7 | GROUP BY 8 | CampsiteType 9 | ) 10 | 11 | SELECT 12 | * 13 | FROM 14 | campsite_type_counts -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/dbt_project/models/marts/count_recarea_by_activity_analysis.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | a.ActivityName, 3 | COUNT(r.RecAreaID) AS rec_area_count 4 | FROM 5 | transformed_data.stg_activities AS a 6 | LEFT JOIN 7 | transformed_data.stg_recreationareas AS r 8 | ON 9 | a.ActivityID = CAST(JSON_EXTRACT_SCALAR(r.ACTIVITY, '$.ActivityID') AS INT64) 10 | GROUP BY 11 | a.ActivityName 12 | -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/dbt_project/models/staging/stg_activities.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('recreation', 'activities') }} -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/dbt_project/models/staging/stg_campsites.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('recreation', 'campsites') }} -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/dbt_project/models/staging/stg_facilities.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('recreation', 'facilities') }} -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/dbt_project/models/staging/stg_recreationareas.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('recreation', 'recreationareas') }} -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/dbt_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_project: 2 | outputs: 3 | dev: 4 | dataset: transformed_data 5 | job_execution_timeout_seconds: 300 6 | job_retries: 1 7 | # Use an env variable to indicate your JSON key file path 8 | keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH', '') }}" 9 | location: US 10 | method: service-account 11 | priority: interactive 12 | # Indicate your BigQuery project ID 13 | project: "{{ env_var('BIGQUERY_PROJECT_ID', '') }}" 14 | threads: 1 15 | type: bigquery 16 | target: dev -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "api_key" { 2 | type = string 3 | } 4 | 5 | variable "workspace_id" { 6 | type = string 7 | } 8 | 9 | variable "dataset_id" { 10 | type = string 11 | } 12 | 13 | variable "project_id" { 14 | type = string 15 | } 16 | 17 | variable "credentials_json" { 18 | type = string 19 | } 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/orchestration/orchestration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/outdoor_activity_analytics_recreation/orchestration/orchestration/__init__.py -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/orchestration/orchestration/definitions.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dagster import Definitions 4 | from dagster_dbt import DbtCliResource 5 | 6 | from .assets import dbt_project_dbt_assets, airbyte_assets 7 | from .constants import dbt_project_dir 8 | from .schedules import schedules 9 | 10 | defs = Definitions( 11 | assets=[dbt_project_dbt_assets, airbyte_assets], 12 | schedules=schedules, 13 | resources={ 14 | "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)), 15 | }, 16 | ) 17 | -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/orchestration/orchestration/schedules.py: -------------------------------------------------------------------------------- 1 | """ 2 | To add a daily schedule that materializes your dbt assets, uncomment the following lines. 3 | """ 4 | from dagster_dbt import build_schedule_from_dbt_selection 5 | 6 | from .assets import dbt_project_dbt_assets 7 | 8 | schedules = [ 9 | # build_schedule_from_dbt_selection( 10 | # [dbt_project_dbt_assets], 11 | # job_name="materialize_dbt_models", 12 | # cron_schedule="0 0 * * *", 13 | # dbt_select="fqn:*", 14 | # ), 15 | ] 16 | -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/orchestration/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.dagster] 6 | module_name = "orchestration.definitions" 7 | code_location_name = "orchestration" -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/orchestration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="orchestration", 5 | version="0.0.1", 6 | packages=find_packages(), 7 | install_requires=[ 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dbt-core>=1.4.0", 12 | "dbt-bigquery", 13 | ], 14 | extras_require={ 15 | "dev": [ 16 | "dagster-webserver", 17 | ] 18 | }, 19 | ) -------------------------------------------------------------------------------- /outdoor_activity_analytics_recreation/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="airbyte-dbt-dagster", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-bigquery", 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dagster-airbyte", 12 | ], 13 | extras_require={"dev": ["dagit", "pytest"]}, 14 | ) -------------------------------------------------------------------------------- /postgres_data_replication/images/infrastructure_layout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/postgres_data_replication/images/infrastructure_layout.png -------------------------------------------------------------------------------- /postgres_data_replication/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /postgres_data_replication/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="postgres-data-replication", 5 | packages=find_packages(), 6 | install_requires=[ 7 | ], 8 | extras_require={"dev": ["pytest"]}, 9 | ) -------------------------------------------------------------------------------- /postgres_snowflake_integration/images/Infrastructure_layout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/postgres_snowflake_integration/images/Infrastructure_layout.png -------------------------------------------------------------------------------- /postgres_snowflake_integration/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /postgres_snowflake_integration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="postgres-snowflake-integration", 5 | packages=find_packages(), 6 | install_requires=[ 7 | ], 8 | extras_require={"dev": ["pytest"]}, 9 | ) 10 | -------------------------------------------------------------------------------- /postgres_to_mysql_migration/images/infrastructure_layout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/postgres_to_mysql_migration/images/infrastructure_layout.png -------------------------------------------------------------------------------- /postgres_to_mysql_migration/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "workspace_id" { 2 | type = string 3 | } 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /postgres_to_mysql_migration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="postgres-to-mysql-migration", 5 | packages=find_packages(), 6 | install_requires=[ 7 | ], 8 | extras_require={"dev": ["pytest"]}, 9 | ) -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | .user.yml 7 | -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | 7 | - dbt run 8 | - dbt test 9 | 10 | ### Resources: 11 | 12 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 13 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 14 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 15 | - Find [dbt events](https://events.getdbt.com) near you 16 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 17 | -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/analysis/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/dbt_project/analysis/.gitkeep -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/marts/analyze_satisfaction_score_over_time.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | DATE_TRUNC(created_at, DAY) AS date, 3 | AVG(CAST(score AS FLOAT64)) AS avg_satisfaction_score 4 | FROM 5 | transformed_data.stg_satisfaction_ratings 6 | WHERE 7 | score IS NOT NULL 8 | GROUP BY 9 | date 10 | ORDER BY 11 | date 12 | -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/marts/avarage_satisfaction_rating.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | AVG(CAST(score AS FLOAT64)) AS avg_satisfaction_score 3 | FROM 4 | transformed_data.stg_satisfaction_ratings 5 | WHERE 6 | score IS NOT NULL 7 | 8 | -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/marts/feedback_analysis_for_low_score.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | reason, 3 | COUNT(*) AS count 4 | FROM 5 | transformed_data.stg_satisfaction_ratings 6 | WHERE 7 | CAST(score AS INT64) <= 2 8 | AND reason IS NOT NULL 9 | GROUP BY 10 | reason 11 | ORDER BY 12 | count DESC 13 | -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/marts/trend_analysis_by_score.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | score, 3 | COUNT(*) AS count 4 | FROM 5 | transformed_data.stg_satisfaction_ratings 6 | WHERE 7 | score IS NOT NULL 8 | GROUP BY 9 | score 10 | ORDER BY 11 | score 12 | -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/staging/stg_brands.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'brands') }} -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/staging/stg_groups.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'groups') }} -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/staging/stg_organizations.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'organizations') }} -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/staging/stg_satisfaction_ratings.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'satisfaction_ratings') }} 4 | -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/staging/stg_tags.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'tags') }} -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/staging/stg_ticket_audits.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'ticket_audits') }} -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/staging/stg_ticket_comments.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'ticket_comments') }} -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/staging/stg_ticket_fields.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'ticket_fields') }} -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/staging/stg_ticket_forms.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'ticket_forms') }} -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/staging/stg_ticket_metric_events.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'ticket_metric_events') }} -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/staging/stg_ticket_metrics.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'ticket_metrics') }} -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/staging/stg_tickets.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'tickets') }} 4 | -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/models/staging/stg_users.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'users') }} 4 | -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_project: 2 | outputs: 3 | dev: 4 | dataset: transformed_data 5 | job_execution_timeout_seconds: 300 6 | job_retries: 1 7 | # Use an env variable to indicate your JSON key file path 8 | keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH', '') }}" 9 | location: US 10 | method: service-account 11 | priority: interactive 12 | # Indicate your BigQuery project ID 13 | project: "{{ env_var('BIGQUERY_PROJECT_ID', '') }}" 14 | threads: 1 15 | type: bigquery 16 | target: dev -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "api_token" { 2 | type = string 3 | } 4 | 5 | variable "email" { 6 | type = string 7 | } 8 | 9 | variable "workspace_id" { 10 | type = string 11 | } 12 | 13 | variable "dataset_id" { 14 | type = string 15 | } 16 | 17 | variable "project_id" { 18 | type = string 19 | } 20 | 21 | variable "credentials_json" { 22 | type = string 23 | } 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/orchestration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/orchestration/__init__.py -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/orchestration/definitions.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dagster import Definitions 4 | from dagster_dbt import DbtCliResource 5 | 6 | from .assets import dbt_project_dbt_assets, airbyte_assets 7 | from .constants import dbt_project_dir 8 | from .schedules import schedules 9 | 10 | defs = Definitions( 11 | assets=[dbt_project_dbt_assets, airbyte_assets], 12 | schedules=schedules, 13 | resources={ 14 | "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)), 15 | }, 16 | ) 17 | -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/orchestration/schedules.py: -------------------------------------------------------------------------------- 1 | """ 2 | To add a daily schedule that materializes your dbt assets, uncomment the following lines. 3 | """ 4 | from dagster_dbt import build_schedule_from_dbt_selection 5 | 6 | from .assets import dbt_project_dbt_assets 7 | 8 | schedules = [ 9 | # build_schedule_from_dbt_selection( 10 | # [dbt_project_dbt_assets], 11 | # job_name="materialize_dbt_models", 12 | # cron_schedule="0 0 * * *", 13 | # dbt_select="fqn:*", 14 | # ), 15 | ] 16 | -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.dagster] 6 | module_name = "orchestration.definitions" 7 | code_location_name = "orchestration" -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="orchestration", 5 | version="0.0.1", 6 | packages=find_packages(), 7 | install_requires=[ 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dbt-core>=1.4.0", 12 | "dbt-bigquery", 13 | ], 14 | extras_require={ 15 | "dev": [ 16 | "dagster-webserver", 17 | ] 18 | }, 19 | ) -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/history/runs.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/history/runs.db -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/history/runs/ad21fadd-c131-4a7c-98a7-fa5ad3a929de.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/history/runs/ad21fadd-c131-4a7c-98a7-fa5ad3a929de.db -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/history/runs/f7507115-918d-443f-ab91-a065e84fa403.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/history/runs/f7507115-918d-443f-ab91-a065e84fa403.db -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/history/runs/index.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/history/runs/index.db -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/schedules/schedules.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/schedules/schedules.db -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/ad21fadd-c131-4a7c-98a7-fa5ad3a929de/compute_logs/mdvhnoik.complete: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/ad21fadd-c131-4a7c-98a7-fa5ad3a929de/compute_logs/mdvhnoik.complete -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/ad21fadd-c131-4a7c-98a7-fa5ad3a929de/compute_logs/mdvhnoik.err: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/ad21fadd-c131-4a7c-98a7-fa5ad3a929de/compute_logs/mdvhnoik.err -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/ad21fadd-c131-4a7c-98a7-fa5ad3a929de/compute_logs/mdvhnoik.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/ad21fadd-c131-4a7c-98a7-fa5ad3a929de/compute_logs/mdvhnoik.out -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/ad21fadd-c131-4a7c-98a7-fa5ad3a929de/compute_logs/uzgmeijp.complete: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/ad21fadd-c131-4a7c-98a7-fa5ad3a929de/compute_logs/uzgmeijp.complete -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/ad21fadd-c131-4a7c-98a7-fa5ad3a929de/compute_logs/uzgmeijp.err: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/ad21fadd-c131-4a7c-98a7-fa5ad3a929de/compute_logs/uzgmeijp.err -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/ad21fadd-c131-4a7c-98a7-fa5ad3a929de/compute_logs/uzgmeijp.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/ad21fadd-c131-4a7c-98a7-fa5ad3a929de/compute_logs/uzgmeijp.out -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/f7507115-918d-443f-ab91-a065e84fa403/compute_logs/aeebjmfa.complete: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/f7507115-918d-443f-ab91-a065e84fa403/compute_logs/aeebjmfa.complete -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/f7507115-918d-443f-ab91-a065e84fa403/compute_logs/aeebjmfa.err: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/f7507115-918d-443f-ab91-a065e84fa403/compute_logs/aeebjmfa.err -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/f7507115-918d-443f-ab91-a065e84fa403/compute_logs/aeebjmfa.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/f7507115-918d-443f-ab91-a065e84fa403/compute_logs/aeebjmfa.out -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/f7507115-918d-443f-ab91-a065e84fa403/compute_logs/zqbkkiww.complete: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/f7507115-918d-443f-ab91-a065e84fa403/compute_logs/zqbkkiww.complete -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/f7507115-918d-443f-ab91-a065e84fa403/compute_logs/zqbkkiww.err: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/f7507115-918d-443f-ab91-a065e84fa403/compute_logs/zqbkkiww.err -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/f7507115-918d-443f-ab91-a065e84fa403/compute_logs/zqbkkiww.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmp3ks7pwhz/storage/f7507115-918d-443f-ab91-a065e84fa403/compute_logs/zqbkkiww.out -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/history/runs.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/history/runs.db -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/history/runs/0bc4e544-546d-44df-b79c-e75413c56ecb.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/history/runs/0bc4e544-546d-44df-b79c-e75413c56ecb.db -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/history/runs/1eac78ed-12d1-4147-9c48-79b27dd586ed.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/history/runs/1eac78ed-12d1-4147-9c48-79b27dd586ed.db -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/history/runs/index.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/history/runs/index.db -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/schedules/schedules.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/schedules/schedules.db -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/0bc4e544-546d-44df-b79c-e75413c56ecb/compute_logs/xozgecli.complete: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/0bc4e544-546d-44df-b79c-e75413c56ecb/compute_logs/xozgecli.complete -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/0bc4e544-546d-44df-b79c-e75413c56ecb/compute_logs/xozgecli.err: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/0bc4e544-546d-44df-b79c-e75413c56ecb/compute_logs/xozgecli.err -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/0bc4e544-546d-44df-b79c-e75413c56ecb/compute_logs/xozgecli.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/0bc4e544-546d-44df-b79c-e75413c56ecb/compute_logs/xozgecli.out -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/0bc4e544-546d-44df-b79c-e75413c56ecb/compute_logs/yyxjctam.complete: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/0bc4e544-546d-44df-b79c-e75413c56ecb/compute_logs/yyxjctam.complete -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/0bc4e544-546d-44df-b79c-e75413c56ecb/compute_logs/yyxjctam.err: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/0bc4e544-546d-44df-b79c-e75413c56ecb/compute_logs/yyxjctam.err -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/0bc4e544-546d-44df-b79c-e75413c56ecb/compute_logs/yyxjctam.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/0bc4e544-546d-44df-b79c-e75413c56ecb/compute_logs/yyxjctam.out -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/1eac78ed-12d1-4147-9c48-79b27dd586ed/compute_logs/iqvvuhde.complete: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/1eac78ed-12d1-4147-9c48-79b27dd586ed/compute_logs/iqvvuhde.complete -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/1eac78ed-12d1-4147-9c48-79b27dd586ed/compute_logs/iqvvuhde.err: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/1eac78ed-12d1-4147-9c48-79b27dd586ed/compute_logs/iqvvuhde.err -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/1eac78ed-12d1-4147-9c48-79b27dd586ed/compute_logs/iqvvuhde.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/1eac78ed-12d1-4147-9c48-79b27dd586ed/compute_logs/iqvvuhde.out -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/1eac78ed-12d1-4147-9c48-79b27dd586ed/compute_logs/izklbfmq.complete: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/1eac78ed-12d1-4147-9c48-79b27dd586ed/compute_logs/izklbfmq.complete -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/1eac78ed-12d1-4147-9c48-79b27dd586ed/compute_logs/izklbfmq.err: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/1eac78ed-12d1-4147-9c48-79b27dd586ed/compute_logs/izklbfmq.err -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/1eac78ed-12d1-4147-9c48-79b27dd586ed/compute_logs/izklbfmq.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/satisfaction_analytics_zendesk_support/orchestration/tmpb3ctnsbk/storage/1eac78ed-12d1-4147-9c48-79b27dd586ed/compute_logs/izklbfmq.out -------------------------------------------------------------------------------- /satisfaction_analytics_zendesk_support/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="airbyte-dbt-dagster", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-bigquery", 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dagster-airbyte", 12 | ], 13 | extras_require={"dev": ["dagit", "pytest"]}, 14 | ) -------------------------------------------------------------------------------- /shopping_cart_analytics_shopify/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | .user.yml -------------------------------------------------------------------------------- /shopping_cart_analytics_shopify/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | 7 | - dbt run 8 | - dbt test 9 | 10 | ### Resources: 11 | 12 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 13 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 14 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 15 | - Find [dbt events](https://events.getdbt.com) near you 16 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 17 | -------------------------------------------------------------------------------- /shopping_cart_analytics_shopify/dbt_project/models/marts/abandoned_checkout_ratio.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | COUNTIF(completed_at IS NULL) AS abandoned_checkouts, 3 | COUNT(*) AS total_checkouts, 4 | COUNTIF(completed_at IS NULL) / COUNT(*) AS abandonment_rate 5 | FROM 6 | transformed_data.stg_abandoned_checkouts 7 | -------------------------------------------------------------------------------- /shopping_cart_analytics_shopify/dbt_project/models/marts/location_based_abandoned_checkouts.sql: -------------------------------------------------------------------------------- 1 | WITH abandoned_checkouts AS ( 2 | SELECT 3 | *, 4 | IF(billing_address IS NOT NULL, JSON_EXTRACT_SCALAR(billing_address, '$.country'), JSON_EXTRACT_SCALAR(shipping_address, '$.country')) AS checkout_country 5 | FROM 6 | transformed_data.stg_abandoned_checkouts 7 | ) 8 | SELECT 9 | checkout_country, 10 | COUNTIF(completed_at IS NULL) AS abandoned_checkouts 11 | FROM 12 | abandoned_checkouts 13 | GROUP BY 14 | checkout_country 15 | ORDER BY 16 | abandoned_checkouts DESC 17 | -------------------------------------------------------------------------------- /shopping_cart_analytics_shopify/dbt_project/models/marts/most_abandoned_products.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | JSON_EXTRACT_SCALAR(line_items, '$.title') AS product_title, 3 | COUNTIF(completed_at IS NULL) AS abandoned_checkouts 4 | FROM 5 | transformed_data.stg_abandoned_checkouts 6 | CROSS JOIN UNNEST(JSON_EXTRACT_ARRAY(line_items)) AS items 7 | GROUP BY 8 | product_title 9 | ORDER BY 10 | abandoned_checkouts DESC 11 | -------------------------------------------------------------------------------- /shopping_cart_analytics_shopify/dbt_project/models/marts/time_based.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | EXTRACT(DAYOFWEEK FROM created_at) AS day_of_week, 3 | EXTRACT(HOUR FROM created_at) AS hour_of_day, 4 | COUNTIF(completed_at IS NULL) AS abandoned_checkouts 5 | FROM 6 | transformed_data.stg_abandoned_checkouts 7 | GROUP BY 8 | day_of_week, 9 | hour_of_day 10 | ORDER BY 11 | day_of_week, hour_of_day 12 | -------------------------------------------------------------------------------- /shopping_cart_analytics_shopify/dbt_project/models/staging/stg_abandoned_checkouts.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('shopify', 'abandoned_checkouts') }} -------------------------------------------------------------------------------- /shopping_cart_analytics_shopify/dbt_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_project: 2 | outputs: 3 | dev: 4 | dataset: transformed_data 5 | job_execution_timeout_seconds: 300 6 | job_retries: 1 7 | # Use an env variable to indicate your JSON key file path 8 | keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH', '') }}" 9 | location: US 10 | method: service-account 11 | priority: interactive 12 | # Indicate your BigQuery project ID 13 | project: "{{ env_var('BIGQUERY_PROJECT_ID', '') }}" 14 | threads: 1 15 | type: bigquery 16 | target: dev -------------------------------------------------------------------------------- /shopping_cart_analytics_shopify/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "api_password" { 2 | type = string 3 | } 4 | 5 | variable "workspace_id" { 6 | type = string 7 | } 8 | 9 | variable "dataset_id" { 10 | type = string 11 | } 12 | 13 | variable "project_id" { 14 | type = string 15 | } 16 | 17 | variable "credentials_json" { 18 | type = string 19 | } 20 | 21 | variable "shop" { 22 | type = string 23 | } -------------------------------------------------------------------------------- /shopping_cart_analytics_shopify/orchestration/orchestration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/shopping_cart_analytics_shopify/orchestration/orchestration/__init__.py -------------------------------------------------------------------------------- /shopping_cart_analytics_shopify/orchestration/orchestration/definitions.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dagster import Definitions 4 | from dagster_dbt import DbtCliResource 5 | 6 | from .assets import dbt_project_dbt_assets, airbyte_assets 7 | from .constants import dbt_project_dir 8 | from .schedules import schedules 9 | 10 | defs = Definitions( 11 | assets=[dbt_project_dbt_assets, airbyte_assets], 12 | schedules=schedules, 13 | resources={ 14 | "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)), 15 | }, 16 | ) 17 | -------------------------------------------------------------------------------- /shopping_cart_analytics_shopify/orchestration/orchestration/schedules.py: -------------------------------------------------------------------------------- 1 | """ 2 | To add a daily schedule that materializes your dbt assets, uncomment the following lines. 3 | """ 4 | from dagster_dbt import build_schedule_from_dbt_selection 5 | 6 | from .assets import dbt_project_dbt_assets 7 | 8 | schedules = [ 9 | # build_schedule_from_dbt_selection( 10 | # [dbt_project_dbt_assets], 11 | # job_name="materialize_dbt_models", 12 | # cron_schedule="0 0 * * *", 13 | # dbt_select="fqn:*", 14 | # ), 15 | ] 16 | -------------------------------------------------------------------------------- /shopping_cart_analytics_shopify/orchestration/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.dagster] 6 | module_name = "orchestration.definitions" 7 | code_location_name = "orchestration" -------------------------------------------------------------------------------- /shopping_cart_analytics_shopify/orchestration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="orchestration", 5 | version="0.0.1", 6 | packages=find_packages(), 7 | install_requires=[ 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dbt-core>=1.4.0", 12 | "dbt-bigquery", 13 | ], 14 | extras_require={ 15 | "dev": [ 16 | "dagster-webserver", 17 | ] 18 | }, 19 | ) -------------------------------------------------------------------------------- /shopping_cart_analytics_shopify/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="airbyte-dbt-dagster", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-bigquery", 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dagster-airbyte", 12 | ], 13 | extras_require={"dev": ["dagit", "pytest"]}, 14 | ) -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | .user.yml 7 | -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | 7 | - dbt run 8 | - dbt test 9 | 10 | ### Resources: 11 | 12 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 13 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 14 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 15 | - Find [dbt events](https://events.getdbt.com) near you 16 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 17 | -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/dbt_project/models/marts/busier_day_of_week_analysis.sql: -------------------------------------------------------------------------------- 1 | WITH ticket_day_of_week_counts AS ( 2 | SELECT 3 | EXTRACT(DAYOFWEEK FROM created_at) AS day_of_week, 4 | COUNT(*) AS ticket_count 5 | FROM {{ ref('stg_tickets') }} 6 | GROUP BY 1 7 | ) 8 | SELECT * FROM ticket_day_of_week_counts -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/dbt_project/models/marts/pattern_and_trend_analysis.sql: -------------------------------------------------------------------------------- 1 | with ticket_hourly_counts as ( 2 | select 3 | extract(hour from created_at) as hour_of_day, 4 | count(*) as ticket_count 5 | from {{ ref('stg_tickets') }} 6 | group by 1 7 | ) 8 | select * from ticket_hourly_counts -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/dbt_project/models/marts/seasonal_analysis.sql: -------------------------------------------------------------------------------- 1 | with ticket_seasonal_counts as ( 2 | select 3 | extract(month from created_at) as month, 4 | count(*) as ticket_count 5 | from {{ ref('stg_tickets') }} 6 | group by 1 7 | ) 8 | select * from ticket_seasonal_counts -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/dbt_project/models/marts/ticket_priority_analysis.sql: -------------------------------------------------------------------------------- 1 | with ticket_priority_counts as ( 2 | select 3 | priority, 4 | count(*) as ticket_count 5 | from {{ ref('stg_tickets') }} 6 | group by 1 7 | ) 8 | select * from ticket_priority_counts -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/dbt_project/models/marts/ticket_resolution_time_analysis.sql: -------------------------------------------------------------------------------- 1 | with ticket_resolution_times as ( 2 | select 3 | TIMESTAMP_DIFF(solved_at, created_at, HOUR) as resolution_time, 4 | count(*) as ticket_count 5 | from transformed_data.stg_ticket_metrics 6 | where solved_at is not null 7 | group by 1 8 | ) 9 | select * from ticket_resolution_times -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/dbt_project/models/marts/ticket_source_analysis.sql: -------------------------------------------------------------------------------- 1 | with source_data as ( 2 | select 3 | JSON_EXTRACT_SCALAR(json_data, '$.channel') as ticket_source, 4 | count(*) as ticket_count 5 | from ( 6 | select 7 | JSON_EXTRACT_ARRAY(via)[OFFSET(0)] as json_data 8 | from transformed_data.stg_tickets 9 | ) t 10 | group by 1 11 | ) 12 | select * from source_data -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/dbt_project/models/marts/ticket_volume_analysis.sql: -------------------------------------------------------------------------------- 1 | with ticket_counts as ( 2 | select 3 | date(created_at) as ticket_date, 4 | count(*) as ticket_count 5 | from {{ ref('stg_tickets') }} 6 | group by 1 7 | ) 8 | select * from ticket_counts -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/dbt_project/models/staging/stg_schedules.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'schedules') }} -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/dbt_project/models/staging/stg_ticket_metrics.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'ticket_metrics') }} -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/dbt_project/models/staging/stg_tickets.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'tickets') }} 4 | -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/dbt_project/models/staging/stg_users.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('zendesk_support', 'users') }} 4 | -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/dbt_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_project: 2 | outputs: 3 | dev: 4 | dataset: transformed_data 5 | job_execution_timeout_seconds: 300 6 | job_retries: 1 7 | # Use an env variable to indicate your JSON key file path 8 | keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH', '') }}" 9 | location: US 10 | method: service-account 11 | priority: interactive 12 | # Indicate your BigQuery project ID 13 | project: "{{ env_var('BIGQUERY_PROJECT_ID', '') }}" 14 | threads: 1 15 | type: bigquery 16 | target: dev -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/ticket_volume_analytics_zendesk_support/dbt_project/snapshots/.gitkeep -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/ticket_volume_analytics_zendesk_support/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/infra/airbyte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "api_token" { 2 | type = string 3 | } 4 | 5 | variable "email" { 6 | type = string 7 | } 8 | 9 | variable "workspace_id" { 10 | type = string 11 | } 12 | 13 | variable "dataset_id" { 14 | type = string 15 | } 16 | 17 | variable "project_id" { 18 | type = string 19 | } 20 | 21 | variable "credentials_json" { 22 | type = string 23 | } 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/orchestration/orchestration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/ticket_volume_analytics_zendesk_support/orchestration/orchestration/__init__.py -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/orchestration/orchestration/definitions.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dagster import Definitions 4 | from dagster_dbt import DbtCliResource 5 | 6 | from .assets import dbt_project_dbt_assets, airbyte_assets 7 | from .constants import dbt_project_dir 8 | from .schedules import schedules 9 | 10 | defs = Definitions( 11 | assets=[dbt_project_dbt_assets, airbyte_assets], 12 | schedules=schedules, 13 | resources={ 14 | "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)), 15 | }, 16 | ) 17 | -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/orchestration/orchestration/schedules.py: -------------------------------------------------------------------------------- 1 | """ 2 | To add a daily schedule that materializes your dbt assets, uncomment the following lines. 3 | """ 4 | from dagster_dbt import build_schedule_from_dbt_selection 5 | 6 | from .assets import dbt_project_dbt_assets 7 | 8 | schedules = [ 9 | # build_schedule_from_dbt_selection( 10 | # [dbt_project_dbt_assets], 11 | # job_name="materialize_dbt_models", 12 | # cron_schedule="0 0 * * *", 13 | # dbt_select="fqn:*", 14 | # ), 15 | ] 16 | -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/orchestration/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.dagster] 6 | module_name = "orchestration.definitions" 7 | code_location_name = "orchestration" -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/orchestration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="orchestration", 5 | version="0.0.1", 6 | packages=find_packages(), 7 | install_requires=[ 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dbt-core>=1.4.0", 12 | "dbt-bigquery", 13 | ], 14 | extras_require={ 15 | "dev": [ 16 | "dagster-webserver", 17 | ] 18 | }, 19 | ) -------------------------------------------------------------------------------- /ticket_volume_analytics_zendesk_support/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="airbyte-dbt-dagster", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-bigquery", 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dagster-airbyte", 12 | ], 13 | extras_require={"dev": ["dagit", "pytest"]}, 14 | ) -------------------------------------------------------------------------------- /vector_store_integration/AI_assistant_streamlit_app/.gitignore: -------------------------------------------------------------------------------- 1 | venv/ 2 | 3 | .secrets 4 | .streamlit 5 | dist/ 6 | build/ 7 | *.egg-info/ 8 | 9 | .DS_STORE 10 | 11 | -------------------------------------------------------------------------------- /vector_store_integration/AI_assistant_streamlit_app/requirements.txt: -------------------------------------------------------------------------------- 1 | streamlit 2 | openai 3 | snowflake-connector-python 4 | langchain 5 | langchain-community 6 | tiktoken -------------------------------------------------------------------------------- /weather_data_stack/assets/Weather.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/weather_data_stack/assets/Weather.png -------------------------------------------------------------------------------- /weather_data_stack/dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | 6 | .user.yml 7 | -------------------------------------------------------------------------------- /weather_data_stack/dbt_project/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | - dbt run 7 | - dbt test 8 | 9 | 10 | ### Resources: 11 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 12 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 13 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 14 | - Find [dbt events](https://events.getdbt.com) near you 15 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 16 | -------------------------------------------------------------------------------- /weather_data_stack/dbt_project/analyses/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/weather_data_stack/dbt_project/analyses/.gitkeep -------------------------------------------------------------------------------- /weather_data_stack/dbt_project/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/weather_data_stack/dbt_project/macros/.gitkeep -------------------------------------------------------------------------------- /weather_data_stack/dbt_project/models/marts/historial_weather_trends.sql: -------------------------------------------------------------------------------- 1 | {% set start_date = '2023-01-01' %} 2 | {% set end_date = '2023-12-31' %} 3 | 4 | SELECT 5 | JSON_EXTRACT_SCALAR(`current`, '$.location.name') AS location_name, 6 | JSON_EXTRACT_SCALAR(`current`, '$.current.temperature') AS temperature, 7 | DATE(JSON_EXTRACT_SCALAR(`current`, '$.current.observation_time')) AS date 8 | FROM 9 | transformed_data.stg_current_weather 10 | WHERE 11 | DATE(JSON_EXTRACT_SCALAR(`current`, '$.current.observation_time')) BETWEEN '{{ start_date }}' AND '{{ end_date }}' 12 | ORDER BY 13 | date, location_name -------------------------------------------------------------------------------- /weather_data_stack/dbt_project/models/sources/weatherstack_source.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: weatherstack 5 | # Use your BigQuery project ID 6 | database: "{{ env_var('BIGQUERY_PROJECT_ID', '') }}" 7 | # Use your BigQuery dataset name 8 | schema: weatherstack_airbyte 9 | 10 | tables: 11 | 12 | - name: current_weather 13 | description: "Simulated current_weather data from the weatherstack connector." 14 | columns: 15 | - name: current 16 | - name: location 17 | - name: request -------------------------------------------------------------------------------- /weather_data_stack/dbt_project/models/staging/stg_current_weather.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from {{ source('weatherstack', 'current_weather') }} -------------------------------------------------------------------------------- /weather_data_stack/dbt_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_project: 2 | outputs: 3 | dev: 4 | dataset: transformed_data 5 | job_execution_timeout_seconds: 300 6 | job_retries: 1 7 | # Use an env variable to indicate your JSON key file path 8 | keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH', '') }}" 9 | location: US 10 | method: service-account 11 | priority: interactive 12 | # Indicate your BigQuery project ID 13 | project: your_project_id 14 | threads: 1 15 | type: bigquery 16 | target: dev -------------------------------------------------------------------------------- /weather_data_stack/dbt_project/seeds/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/weather_data_stack/dbt_project/seeds/.gitkeep -------------------------------------------------------------------------------- /weather_data_stack/dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/weather_data_stack/dbt_project/snapshots/.gitkeep -------------------------------------------------------------------------------- /weather_data_stack/dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/weather_data_stack/dbt_project/tests/.gitkeep -------------------------------------------------------------------------------- /weather_data_stack/orchestration/orchestration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbytehq/quickstarts/d00a6307442510d5b49e29a9317aca48dbabace2/weather_data_stack/orchestration/orchestration/__init__.py -------------------------------------------------------------------------------- /weather_data_stack/orchestration/orchestration/definitions.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dagster import Definitions 4 | from dagster_dbt import DbtCliResource 5 | 6 | from .assets import dbt_project_dbt_assets, airbyte_assets 7 | from .constants import dbt_project_dir 8 | from .schedules import schedules 9 | 10 | defs = Definitions( 11 | assets=[dbt_project_dbt_assets, airbyte_assets], 12 | schedules=schedules, 13 | resources={ 14 | "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)), 15 | }, 16 | ) -------------------------------------------------------------------------------- /weather_data_stack/orchestration/orchestration/schedules.py: -------------------------------------------------------------------------------- 1 | """ 2 | To add a daily schedule that materializes your dbt assets, uncomment the following lines. 3 | """ 4 | from dagster_dbt import build_schedule_from_dbt_selection 5 | 6 | from .assets import dbt_project_dbt_assets 7 | 8 | schedules = [ 9 | # build_schedule_from_dbt_selection( 10 | # [dbt_project_dbt_assets], 11 | # job_name="materialize_dbt_models", 12 | # cron_schedule="0 0 * * *", 13 | # dbt_select="fqn:*", 14 | # ), 15 | ] -------------------------------------------------------------------------------- /weather_data_stack/orchestration/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.dagster] 6 | module_name = "orchestration.definitions" 7 | code_location_name = "orchestration" -------------------------------------------------------------------------------- /weather_data_stack/orchestration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="orchestration", 5 | version="0.0.1", 6 | packages=find_packages(), 7 | install_requires=[ 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dbt-core>=1.4.0", 12 | "dbt-bigquery", 13 | ], 14 | extras_require={ 15 | "dev": [ 16 | "dagster-webserver", 17 | ] 18 | }, 19 | ) -------------------------------------------------------------------------------- /weather_data_stack/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="airbyte-dbt-dagster", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-bigquery", 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dagster-airbyte", 12 | ], 13 | extras_require={"dev": ["dagit", "pytest"]}, 14 | ) --------------------------------------------------------------------------------