├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ └── main.yml ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── README.md ├── dagster_orchestration ├── README.md ├── dagster_orchestration │ ├── __init__.py │ └── assets │ │ └── __init__.py ├── dagster_orchestration_tests │ ├── __init__.py │ └── test_assets.py ├── pyproject.toml ├── setup.cfg └── setup.py ├── dbt_transformation ├── .gitignore ├── README.md ├── analyses │ └── .gitkeep ├── config │ └── profiles.yml ├── dbt_project.yml ├── macros │ └── .gitkeep ├── models │ └── .gitkeep ├── seeds │ └── .gitkeep ├── snapshots │ └── .gitkeep └── tests │ └── .gitkeep └── setup.py /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Codeowners for these exercise files: 2 | # * (asterisk) denotes "all files and folders" 3 | # Example: * @producer @instructor 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | ## Issue Overview 9 | 10 | 11 | ## Describe your environment 12 | 13 | 14 | ## Steps to Reproduce 15 | 16 | 1. 17 | 2. 18 | 3. 19 | 4. 20 | 21 | ## Expected Behavior 22 | 23 | 24 | ## Current Behavior 25 | 26 | 27 | ## Possible Solution 28 | 29 | 30 | ## Screenshots / Video 31 | 32 | 33 | ## Related Issues 34 | 35 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Copy To Branches 2 | on: 3 | workflow_dispatch: 4 | jobs: 5 | copy-to-branches: 6 | runs-on: ubuntu-latest 7 | steps: 8 | - uses: actions/checkout@v2 9 | with: 10 | fetch-depth: 0 11 | - name: Copy To Branches Action 12 | uses: planetoftheweb/copy-to-branches@v1.2 13 | env: 14 | key: main 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 2 | Contribution Agreement 3 | ====================== 4 | 5 | This repository does not accept pull requests (PRs). All pull requests will be closed. 6 | 7 | However, if any contributions (through pull requests, issues, feedback or otherwise) are provided, as a contributor, you represent that the code you submit is your original work or that of your employer (in which case you represent you have the right to bind your employer). By submitting code (or otherwise providing feedback), you (and, if applicable, your employer) are licensing the submitted code (and/or feedback) to LinkedIn and the open source community subject to the BSD 2-Clause license. 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | LinkedIn Learning Exercise Files License Agreement 2 | ================================================== 3 | 4 | This License Agreement (the "Agreement") is a binding legal agreement 5 | between you (as an individual or entity, as applicable) and LinkedIn 6 | Corporation (“LinkedIn”). By downloading or using the LinkedIn Learning 7 | exercise files in this repository (“Licensed Materials”), you agree to 8 | be bound by the terms of this Agreement. If you do not agree to these 9 | terms, do not download or use the Licensed Materials. 10 | 11 | 1. License. 12 | - a. Subject to the terms of this Agreement, LinkedIn hereby grants LinkedIn 13 | members during their LinkedIn Learning subscription a non-exclusive, 14 | non-transferable copyright license, for internal use only, to 1) make a 15 | reasonable number of copies of the Licensed Materials, and 2) make 16 | derivative works of the Licensed Materials for the sole purpose of 17 | practicing skills taught in LinkedIn Learning courses. 18 | - b. Distribution. Unless otherwise noted in the Licensed Materials, subject 19 | to the terms of this Agreement, LinkedIn hereby grants LinkedIn members 20 | with a LinkedIn Learning subscription a non-exclusive, non-transferable 21 | copyright license to distribute the Licensed Materials, except the 22 | Licensed Materials may not be included in any product or service (or 23 | otherwise used) to instruct or educate others. 24 | 25 | 2. Restrictions and Intellectual Property. 26 | - a. You may not to use, modify, copy, make derivative works of, publish, 27 | distribute, rent, lease, sell, sublicense, assign or otherwise transfer the 28 | Licensed Materials, except as expressly set forth above in Section 1. 29 | - b. Linkedin (and its licensors) retains its intellectual property rights 30 | in the Licensed Materials. Except as expressly set forth in Section 1, 31 | LinkedIn grants no licenses. 32 | - c. You indemnify LinkedIn and its licensors and affiliates for i) any 33 | alleged infringement or misappropriation of any intellectual property rights 34 | of any third party based on modifications you make to the Licensed Materials, 35 | ii) any claims arising from your use or distribution of all or part of the 36 | Licensed Materials and iii) a breach of this Agreement. You will defend, hold 37 | harmless, and indemnify LinkedIn and its affiliates (and our and their 38 | respective employees, shareholders, and directors) from any claim or action 39 | brought by a third party, including all damages, liabilities, costs and 40 | expenses, including reasonable attorneys’ fees, to the extent resulting from, 41 | alleged to have resulted from, or in connection with: (a) your breach of your 42 | obligations herein; or (b) your use or distribution of any Licensed Materials. 43 | 44 | 3. Open source. This code may include open source software, which may be 45 | subject to other license terms as provided in the files. 46 | 47 | 4. Warranty Disclaimer. LINKEDIN PROVIDES THE LICENSED MATERIALS ON AN “AS IS” 48 | AND “AS AVAILABLE” BASIS. LINKEDIN MAKES NO REPRESENTATION OR WARRANTY, 49 | WHETHER EXPRESS OR IMPLIED, ABOUT THE LICENSED MATERIALS, INCLUDING ANY 50 | REPRESENTATION THAT THE LICENSED MATERIALS WILL BE FREE OF ERRORS, BUGS OR 51 | INTERRUPTIONS, OR THAT THE LICENSED MATERIALS ARE ACCURATE, COMPLETE OR 52 | OTHERWISE VALID. TO THE FULLEST EXTENT PERMITTED BY LAW, LINKEDIN AND ITS 53 | AFFILIATES DISCLAIM ANY IMPLIED OR STATUTORY WARRANTY OR CONDITION, INCLUDING 54 | ANY IMPLIED WARRANTY OR CONDITION OF MERCHANTABILITY OR FITNESS FOR A 55 | PARTICULAR PURPOSE, AVAILABILITY, SECURITY, TITLE AND/OR NON-INFRINGEMENT. 56 | YOUR USE OF THE LICENSED MATERIALS IS AT YOUR OWN DISCRETION AND RISK, AND 57 | YOU WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE THAT RESULTS FROM USE OF THE 58 | LICENSED MATERIALS TO YOUR COMPUTER SYSTEM OR LOSS OF DATA. NO ADVICE OR 59 | INFORMATION, WHETHER ORAL OR WRITTEN, OBTAINED BY YOU FROM US OR THROUGH OR 60 | FROM THE LICENSED MATERIALS WILL CREATE ANY WARRANTY OR CONDITION NOT 61 | EXPRESSLY STATED IN THESE TERMS. 62 | 63 | 5. Limitation of Liability. LINKEDIN SHALL NOT BE LIABLE FOR ANY INDIRECT, 64 | INCIDENTAL, SPECIAL, PUNITIVE, CONSEQUENTIAL OR EXEMPLARY DAMAGES, INCLUDING 65 | BUT NOT LIMITED TO, DAMAGES FOR LOSS OF PROFITS, GOODWILL, USE, DATA OR OTHER 66 | INTANGIBLE LOSSES . IN NO EVENT WILL LINKEDIN'S AGGREGATE LIABILITY TO YOU 67 | EXCEED $100. THIS LIMITATION OF LIABILITY SHALL: 68 | - i. APPLY REGARDLESS OF WHETHER (A) YOU BASE YOUR CLAIM ON CONTRACT, TORT, 69 | STATUTE, OR ANY OTHER LEGAL THEORY, (B) WE KNEW OR SHOULD HAVE KNOWN ABOUT 70 | THE POSSIBILITY OF SUCH DAMAGES, OR (C) THE LIMITED REMEDIES PROVIDED IN THIS 71 | SECTION FAIL OF THEIR ESSENTIAL PURPOSE; AND 72 | - ii. NOT APPLY TO ANY DAMAGE THAT LINKEDIN MAY CAUSE YOU INTENTIONALLY OR 73 | KNOWINGLY IN VIOLATION OF THESE TERMS OR APPLICABLE LAW, OR AS OTHERWISE 74 | MANDATED BY APPLICABLE LAW THAT CANNOT BE DISCLAIMED IN THESE TERMS. 75 | 76 | 6. Termination. This Agreement automatically terminates upon your breach of 77 | this Agreement or termination of your LinkedIn Learning subscription. On 78 | termination, all licenses granted under this Agreement will terminate 79 | immediately and you will delete the Licensed Materials. Sections 2-7 of this 80 | Agreement survive any termination of this Agreement. LinkedIn may discontinue 81 | the availability of some or all of the Licensed Materials at any time for any 82 | reason. 83 | 84 | 7. Miscellaneous. This Agreement will be governed by and construed in 85 | accordance with the laws of the State of California without regard to conflict 86 | of laws principles. The exclusive forum for any disputes arising out of or 87 | relating to this Agreement shall be an appropriate federal or state court 88 | sitting in the County of Santa Clara, State of California. If LinkedIn does 89 | not act to enforce a breach of this Agreement, that does not mean that 90 | LinkedIn has waived its right to enforce this Agreement. The Agreement does 91 | not create a partnership, agency relationship, or joint venture between the 92 | parties. Neither party has the power or authority to bind the other or to 93 | create any obligation or responsibility on behalf of the other. You may not, 94 | without LinkedIn’s prior written consent, assign or delegate any rights or 95 | obligations under these terms, including in connection with a change of 96 | control. Any purported assignment and delegation shall be ineffective. The 97 | Agreement shall bind and inure to the benefit of the parties, their respective 98 | successors and permitted assigns. If any provision of the Agreement is 99 | unenforceable, that provision will be modified to render it enforceable to the 100 | extent possible to give effect to the parties’ intentions and the remaining 101 | provisions will not be affected. This Agreement is the only agreement between 102 | you and LinkedIn regarding the Licensed Materials, and supersedes all prior 103 | agreements relating to the Licensed Materials. 104 | 105 | Last Updated: March 2019 106 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2023 LinkedIn Corporation 2 | All Rights Reserved. 3 | 4 | Licensed under the LinkedIn Learning Exercise File License (the "License"). 5 | See LICENSE in the project root for license information. 6 | 7 | Please note, this project may automatically load third party code from external 8 | repositories (for example, NPM modules, Composer packages, or other dependencies). 9 | If so, such third party code may be subject to other license terms than as set 10 | forth above. In addition, such third party code may also depend on and load 11 | multiple tiers of dependencies. Please review the applicable licenses of the 12 | additional dependencies. 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # End-to-End Data Engineering Project 2 | This is the repository for the LinkedIn Learning course End-to-End Data Engineering Project. The full course is available from [LinkedIn Learning][lil-course-url]. 3 | 4 | ![End-to-End Data Engineering Project][lil-thumbnail-url] 5 | 6 | The world of data engineering is ever-changing, with new tools and technologies emerging on a regular basis. Building an effective analytics platform can be a daunting task, especially if you’re not familiar with all the tools available. How do you turn scattered, complex data into a model that drives insights and decision-making? 7 | In this course, Thalia Barrera teaches data professionals how to implement an end-to-end data engineering project using open tools from the modern data stack. She touches on best practices such as data modeling, testing, documentation and version control and shows you how to efficiently extract, load, and transform data into a unified, analytics-ready format. Thalia shows you how to confidently select and use tools through practical examples—taking you through the construction of a robust data pipeline for a fictional ecommerce company—and how to implement best practices in data engineering. 8 | 9 | ## Instructions 10 | This repository has two branches: `main` holds the initial state of the project, and `finished` holds the final state. You can use the branch pop up menu in github to switch to a specific branch and take a look at the course at that stage, or you can add `/tree/BRANCH_NAME` to the URL to go to the branch you want to access. 11 | 12 | ## Branches 13 | You will be working in the `main` branch throughout the course. At any time, you can checkout the `finished` branch to consult how the finished project looks like. 14 | 15 | ## Prerequisites 16 | Ensure you have Python 3 installed. If not, you can download and install it from Python's official website. 17 | 18 | ## Installing 19 | 1. Fork the Repository: 20 | - Click the "Fork" button on the top right corner of this repository. 21 | 2. Clone the repository: 22 | - `git clone https://github.com/YOUR_USERNAME/end-to-end-data-engineering-project-4413618.git` 23 | - Note: Replace YOUR_USERNAME with your GitHub username 24 | 3. Navigate to the directory: 25 | - `cd end-to-end-data-engineering-project-4413618` 26 | 4. Set Up a Virtual Environment: 27 | - For Mac: 28 | - `python3 -m venv venv` 29 | - `source venv/bin/activate` 30 | - For Windows: 31 | - `python -m venv venv` 32 | - `.\venv\Scripts\activate` 33 | 5. Install Dependencies: 34 | - `pip install -e ".[dev]"` 35 | 36 | 37 | ### Instructor 38 | 39 | Thalia Barrera 40 | 41 | 42 | 43 | 44 | 45 | Check out my other courses on [LinkedIn Learning](https://www.linkedin.com/learning/instructors/thalia-barrera). 46 | 47 | [lil-course-url]: https://www.linkedin.com/learning/end-to-end-data-engineering-project?dApp=59033956&leis=LAA 48 | [lil-thumbnail-url]: https://media.licdn.com/dms/image/D4D0DAQFQihfehsNCiQ/learning-public-crop_288_512/0/1698869440746?e=2147483647&v=beta&t=3G9Icq-7JuCKrWsa5lQMv3mLiqyy5NkXwj8urZEXCWw 49 | 50 | -------------------------------------------------------------------------------- /dagster_orchestration/README.md: -------------------------------------------------------------------------------- 1 | # dagster_orchestration 2 | 3 | This is a [Dagster](https://dagster.io/) project scaffolded with [`dagster project scaffold`](https://docs.dagster.io/getting-started/create-new-project). 4 | 5 | ## Getting started 6 | 7 | First, install your Dagster code location as a Python package. By using the --editable flag, pip will install your Python package in ["editable mode"](https://pip.pypa.io/en/latest/topics/local-project-installs/#editable-installs) so that as you develop, local code changes will automatically apply. 8 | 9 | ```bash 10 | pip install -e ".[dev]" 11 | ``` 12 | 13 | Then, start the Dagster UI web server: 14 | 15 | ```bash 16 | dagster dev 17 | ``` 18 | 19 | Open http://localhost:3000 with your browser to see the project. 20 | 21 | You can start writing assets in `dagster_orchestration/assets.py`. The assets are automatically loaded into the Dagster code location as you define them. 22 | 23 | ## Development 24 | 25 | 26 | ### Adding new Python dependencies 27 | 28 | You can specify new Python dependencies in `setup.py`. 29 | 30 | ### Unit testing 31 | 32 | Tests are in the `dagster_orchestration_tests` directory and you can run tests using `pytest`: 33 | 34 | ```bash 35 | pytest dagster_orchestration_tests 36 | ``` 37 | 38 | ### Schedules and sensors 39 | 40 | If you want to enable Dagster [Schedules](https://docs.dagster.io/concepts/partitions-schedules-sensors/schedules) or [Sensors](https://docs.dagster.io/concepts/partitions-schedules-sensors/sensors) for your jobs, the [Dagster Daemon](https://docs.dagster.io/deployment/dagster-daemon) process must be running. This is done automatically when you run `dagster dev`. 41 | 42 | Once your Dagster Daemon is running, you can start turning on schedules and sensors for your jobs. 43 | 44 | ## Deploy on Dagster Cloud 45 | 46 | The easiest way to deploy your Dagster project is to use Dagster Cloud. 47 | 48 | Check out the [Dagster Cloud Documentation](https://docs.dagster.cloud) to learn more. 49 | -------------------------------------------------------------------------------- /dagster_orchestration/dagster_orchestration/__init__.py: -------------------------------------------------------------------------------- 1 | from dagster import Definitions, load_assets_from_modules 2 | 3 | from . import assets 4 | 5 | all_assets = load_assets_from_modules([assets]) 6 | 7 | defs = Definitions( 8 | assets=all_assets, 9 | ) 10 | -------------------------------------------------------------------------------- /dagster_orchestration/dagster_orchestration/assets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LinkedInLearning/end-to-end-data-engineering-project-4413618/f6bc98302ee58aea70b87b02aadcfc8b01b435f0/dagster_orchestration/dagster_orchestration/assets/__init__.py -------------------------------------------------------------------------------- /dagster_orchestration/dagster_orchestration_tests/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /dagster_orchestration/dagster_orchestration_tests/test_assets.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /dagster_orchestration/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.dagster] 6 | module_name = "dagster_orchestration" 7 | -------------------------------------------------------------------------------- /dagster_orchestration/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = dagster_orchestration 3 | -------------------------------------------------------------------------------- /dagster_orchestration/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="dagster_orchestration", 5 | packages=find_packages(exclude=["dagster_orchestration_tests"]), 6 | install_requires=[ 7 | "dagster", 8 | "dagster-cloud" 9 | ], 10 | extras_require={"dev": ["dagster-webserver", "pytest"]}, 11 | ) 12 | -------------------------------------------------------------------------------- /dbt_transformation/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | config/.user.yml 6 | -------------------------------------------------------------------------------- /dbt_transformation/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | - dbt run 7 | - dbt test 8 | 9 | 10 | ### Resources: 11 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 12 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 13 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 14 | - Find [dbt events](https://events.getdbt.com) near you 15 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 16 | -------------------------------------------------------------------------------- /dbt_transformation/analyses/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LinkedInLearning/end-to-end-data-engineering-project-4413618/f6bc98302ee58aea70b87b02aadcfc8b01b435f0/dbt_transformation/analyses/.gitkeep -------------------------------------------------------------------------------- /dbt_transformation/config/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_transformation: 2 | outputs: 3 | dev: 4 | dataset: "{{ env_var('DBT_BIGQUERY_DATASET', '') }}" 5 | job_execution_timeout_seconds: 300 6 | job_retries: 1 7 | keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH', '') }}" 8 | location: "{{ env_var('DBT_BIGQUERY_LOCATION', '') }}" 9 | method: service-account 10 | priority: interactive 11 | project: "{{ env_var('DBT_BIGQUERY_PROJECT', '') }}" 12 | threads: 1 13 | type: bigquery 14 | target: dev -------------------------------------------------------------------------------- /dbt_transformation/dbt_project.yml: -------------------------------------------------------------------------------- 1 | # Name your project! Project names should contain only lowercase characters 2 | # and underscores. A good package name should reflect your organization's 3 | # name or the intended use of these models 4 | name: "dbt_transformation" 5 | version: "1.0.0" 6 | config-version: 2 7 | 8 | # This setting configures which "profile" dbt uses for this project. 9 | profile: "dbt_transformation" 10 | 11 | # These configurations specify where dbt should look for different types of files. 12 | # The `model-paths` config, for example, states that models in this project can be 13 | # found in the "models/" directory. You probably won't need to change these! 14 | model-paths: ["models"] 15 | analysis-paths: ["analyses"] 16 | test-paths: ["tests"] 17 | seed-paths: ["seeds"] 18 | macro-paths: ["macros"] 19 | snapshot-paths: ["snapshots"] 20 | 21 | clean-targets: # directories to be removed by `dbt clean` 22 | - "target" 23 | - "dbt_packages" 24 | # Configuring models 25 | # Full documentation: https://docs.getdbt.com/docs/configuring-models 26 | 27 | # In this example config, we tell dbt to build all models in the example/ 28 | # directory as views. These settings can be overridden in the individual model 29 | # files using the `{{ config(...) }}` macro. 30 | -------------------------------------------------------------------------------- /dbt_transformation/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LinkedInLearning/end-to-end-data-engineering-project-4413618/f6bc98302ee58aea70b87b02aadcfc8b01b435f0/dbt_transformation/macros/.gitkeep -------------------------------------------------------------------------------- /dbt_transformation/models/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LinkedInLearning/end-to-end-data-engineering-project-4413618/f6bc98302ee58aea70b87b02aadcfc8b01b435f0/dbt_transformation/models/.gitkeep -------------------------------------------------------------------------------- /dbt_transformation/seeds/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LinkedInLearning/end-to-end-data-engineering-project-4413618/f6bc98302ee58aea70b87b02aadcfc8b01b435f0/dbt_transformation/seeds/.gitkeep -------------------------------------------------------------------------------- /dbt_transformation/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LinkedInLearning/end-to-end-data-engineering-project-4413618/f6bc98302ee58aea70b87b02aadcfc8b01b435f0/dbt_transformation/snapshots/.gitkeep -------------------------------------------------------------------------------- /dbt_transformation/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LinkedInLearning/end-to-end-data-engineering-project-4413618/f6bc98302ee58aea70b87b02aadcfc8b01b435f0/dbt_transformation/tests/.gitkeep -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="modern-data-stack-project", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "dbt-bigquery", 8 | "dagster", 9 | "dagster-cloud", 10 | "dagster-dbt", 11 | "dagster-airbyte", 12 | ], 13 | extras_require={"dev": ["dagit", "pytest"]}, 14 | ) --------------------------------------------------------------------------------