├── .github ├── CODEOWNERS └── workflows │ ├── CODEOWNERS │ ├── quality-check-dagster-and-dbt.yml │ ├── quality-check-dagster-essentials.yml │ ├── quality-check-dagster-and-etl.yml │ └── quality-check-dagster-testing.yml ├── dagster_university ├── dagster_and_dbt │ ├── data │ │ ├── raw │ │ │ └── .gitkeep │ │ ├── outputs │ │ │ └── .gitkeep │ │ └── staging │ │ │ └── .gitkeep │ ├── tests │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── run_config.yaml │ │ ├── test_lesson_starter.py │ │ ├── fixtures.py │ │ ├── test_lesson_3.py │ │ ├── test_lesson_4.py │ │ └── test_lesson_6.py │ ├── src │ │ └── dagster_and_dbt │ │ │ ├── __init__.py │ │ │ ├── defs │ │ │ ├── __init__.py │ │ │ ├── project.py │ │ │ ├── assets │ │ │ │ └── __init__.py │ │ │ ├── schedules.py │ │ │ ├── partitions.py │ │ │ ├── jobs.py │ │ │ └── resources.py │ │ │ ├── completed │ │ │ ├── __init__.py │ │ │ ├── lesson_2 │ │ │ │ ├── __init__.py │ │ │ │ ├── defs │ │ │ │ │ ├── assets │ │ │ │ │ │ └── __init__.py │ │ │ │ │ ├── schedules.py │ │ │ │ │ ├── partitions.py │ │ │ │ │ ├── jobs.py │ │ │ │ │ └── resources.py │ │ │ │ └── definitions.py │ │ │ ├── lesson_3 │ │ │ │ ├── __init__.py │ │ │ │ ├── defs │ │ │ │ │ ├── assets │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── dbt.py │ │ │ │ │ ├── project.py │ │ │ │ │ ├── schedules.py │ │ │ │ │ ├── partitions.py │ │ │ │ │ ├── jobs.py │ │ │ │ │ └── resources.py │ │ │ │ ├── analytics │ │ │ │ │ ├── .gitignore │ │ │ │ │ ├── packages.yml │ │ │ │ │ ├── package-lock.yml │ │ │ │ │ ├── models │ │ │ │ │ │ ├── sources │ │ │ │ │ │ │ └── raw_taxis.yml │ │ │ │ │ │ └── staging │ │ │ │ │ │ │ └── stg_zones.sql │ │ │ │ │ ├── profiles.yml │ │ │ │ │ └── README.md │ │ │ │ └── definitions.py │ │ │ ├── lesson_4 │ │ │ │ ├── __init__.py │ │ │ │ ├── defs │ │ │ │ │ ├── assets │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── dbt.py │ │ │ │ │ ├── project.py │ │ │ │ │ ├── schedules.py │ │ │ │ │ ├── partitions.py │ │ │ │ │ ├── jobs.py │ │ │ │ │ └── resources.py │ │ │ │ ├── analytics │ │ │ │ │ ├── .gitignore │ │ │ │ │ ├── packages.yml │ │ │ │ │ ├── package-lock.yml │ │ │ │ │ ├── models │ │ │ │ │ │ ├── sources │ │ │ │ │ │ │ └── raw_taxis.yml │ │ │ │ │ │ └── staging │ │ │ │ │ │ │ └── stg_zones.sql │ │ │ │ │ ├── profiles.yml │ │ │ │ │ └── README.md │ │ │ │ └── definitions.py │ │ │ ├── lesson_5 │ │ │ │ ├── __init__.py │ │ │ │ ├── defs │ │ │ │ │ ├── assets │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── dbt.py │ │ │ │ │ ├── project.py │ │ │ │ │ ├── schedules.py │ │ │ │ │ ├── partitions.py │ │ │ │ │ ├── jobs.py │ │ │ │ │ └── resources.py │ │ │ │ ├── analytics │ │ │ │ │ ├── .gitignore │ │ │ │ │ ├── packages.yml │ │ │ │ │ ├── package-lock.yml │ │ │ │ │ ├── models │ │ │ │ │ │ ├── sources │ │ │ │ │ │ │ └── raw_taxis.yml │ │ │ │ │ │ ├── staging │ │ │ │ │ │ │ └── stg_zones.sql │ │ │ │ │ │ └── marts │ │ │ │ │ │ │ └── location_metrics.sql │ │ │ │ │ ├── profiles.yml │ │ │ │ │ └── README.md │ │ │ │ └── definitions.py │ │ │ ├── lesson_6 │ │ │ │ ├── __init__.py │ │ │ │ ├── defs │ │ │ │ │ ├── assets │ │ │ │ │ │ └── __init__.py │ │ │ │ │ ├── project.py │ │ │ │ │ ├── partitions.py │ │ │ │ │ ├── schedules.py │ │ │ │ │ ├── resources.py │ │ │ │ │ └── jobs.py │ │ │ │ ├── analytics │ │ │ │ │ ├── .gitignore │ │ │ │ │ ├── packages.yml │ │ │ │ │ ├── package-lock.yml │ │ │ │ │ ├── models │ │ │ │ │ │ ├── sources │ │ │ │ │ │ │ └── raw_taxis.yml │ │ │ │ │ │ ├── staging │ │ │ │ │ │ │ └── stg_zones.sql │ │ │ │ │ │ └── marts │ │ │ │ │ │ │ ├── daily_metrics.sql │ │ │ │ │ │ │ └── location_metrics.sql │ │ │ │ │ ├── profiles.yml │ │ │ │ │ └── README.md │ │ │ │ └── definitions.py │ │ │ └── lesson_7 │ │ │ │ ├── __init__.py │ │ │ │ ├── defs │ │ │ │ ├── assets │ │ │ │ │ └── __init__.py │ │ │ │ ├── project.py │ │ │ │ ├── partitions.py │ │ │ │ ├── schedules.py │ │ │ │ ├── resources.py │ │ │ │ └── jobs.py │ │ │ │ ├── analytics │ │ │ │ ├── .gitignore │ │ │ │ ├── packages.yml │ │ │ │ ├── package-lock.yml │ │ │ │ ├── models │ │ │ │ │ ├── sources │ │ │ │ │ │ └── raw_taxis.yml │ │ │ │ │ ├── staging │ │ │ │ │ │ └── stg_zones.sql │ │ │ │ │ └── marts │ │ │ │ │ │ ├── daily_metrics.sql │ │ │ │ │ │ └── location_metrics.sql │ │ │ │ ├── profiles.yml │ │ │ │ └── README.md │ │ │ │ └── definitions.py │ │ │ ├── analytics │ │ │ ├── .gitignore │ │ │ ├── packages.yml │ │ │ ├── models │ │ │ │ ├── sources │ │ │ │ │ └── raw_taxis.yml │ │ │ │ └── staging │ │ │ │ │ └── stg_zones.sql │ │ │ └── README.md │ │ │ └── definitions.py │ ├── .env │ ├── pytest.ini │ ├── Makefile │ ├── dagster_cloud.yaml │ └── README.md ├── dagster_and_etl │ ├── tests │ │ ├── __init__.py │ │ ├── postgres │ │ │ └── Dockerfile │ │ ├── pytest.ini │ │ ├── test_lesson_starter.py │ │ ├── conftest.py │ │ └── docker-compose.yaml │ ├── data │ │ ├── staging │ │ │ └── .gitkeep │ │ └── source │ │ │ ├── 2018-01-22.csv │ │ │ ├── 2018-01-23.csv │ │ │ └── 2018-01-24.csv │ ├── src │ │ └── dagster_and_etl │ │ │ ├── __init__.py │ │ │ ├── defs │ │ │ ├── jobs.py │ │ │ ├── __init__.py │ │ │ ├── assets.py │ │ │ ├── resources.py │ │ │ ├── schedules.py │ │ │ └── sensors.py │ │ │ ├── completed │ │ │ ├── lesson_3 │ │ │ │ ├── __init__.py │ │ │ │ └── defs │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── schedules.py │ │ │ │ │ ├── resources.py │ │ │ │ │ └── jobs.py │ │ │ ├── lesson_4 │ │ │ │ ├── __init__.py │ │ │ │ └── defs │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── jobs.py │ │ │ │ │ └── schedules.py │ │ │ ├── lesson_5 │ │ │ │ ├── __init__.py │ │ │ │ ├── defs │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── resources.py │ │ │ │ └── dlt_quick_start.py │ │ │ ├── lesson_6 │ │ │ │ ├── __init__.py │ │ │ │ └── defs │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── sling_replication.yaml │ │ │ │ │ ├── schedules.py │ │ │ │ │ ├── jobs.py │ │ │ │ │ ├── resources.py │ │ │ │ │ └── assets.py │ │ │ └── lesson_7 │ │ │ │ ├── __init__.py │ │ │ │ └── defs │ │ │ │ ├── __init__.py │ │ │ │ ├── ingest_files │ │ │ │ ├── replication.yaml │ │ │ │ └── defs.yaml │ │ │ │ └── assets.py │ │ │ └── definitions.py │ ├── Makefile │ ├── .gitignore │ ├── README.md │ └── pyproject.toml ├── dagster_testing │ ├── tests │ │ ├── __init__.py │ │ ├── completed │ │ │ ├── __init__.py │ │ │ └── test_lesson_starter.py │ │ ├── data │ │ │ └── test.csv │ │ ├── postgres │ │ │ ├── Dockerfile │ │ │ └── postgres_bootstrap.sql │ │ ├── configs │ │ │ └── lesson_3.yaml │ │ ├── pytest.ini │ │ └── docker-compose.yaml │ ├── src │ │ └── dagster_testing │ │ │ ├── __init__.py │ │ │ ├── defs │ │ │ ├── __init__.py │ │ │ ├── assets │ │ │ │ ├── __init__.py │ │ │ │ └── lesson_5.py │ │ │ ├── data │ │ │ │ ├── ny.csv │ │ │ │ ├── ca.csv │ │ │ │ └── mn.csv │ │ │ ├── configs │ │ │ │ └── run_config.yaml │ │ │ ├── schedules.py │ │ │ ├── resources.py │ │ │ ├── sensors.py │ │ │ └── jobs.py │ │ │ └── definitions.py │ ├── Makefile │ └── pyproject.toml └── dagster_essentials │ ├── data │ ├── raw │ │ └── .gitkeep │ ├── outputs │ │ └── .gitkeep │ └── staging │ │ └── .gitkeep │ ├── tests │ ├── __init__.py │ ├── conftest.py │ ├── run_config.yaml │ ├── test_lesson_starter.py │ ├── utils.py │ ├── fixtures.py │ ├── test_lesson_3.py │ ├── test_lesson_4.py │ └── test_lesson_5.py │ ├── src │ └── dagster_essentials │ │ ├── __init__.py │ │ ├── defs │ │ ├── __init__.py │ │ └── assets │ │ │ ├── __init__.py │ │ │ └── constants.py │ │ ├── completed │ │ ├── __init__.py │ │ ├── lesson_3 │ │ │ ├── __init__.py │ │ │ └── defs │ │ │ │ └── assets │ │ │ │ ├── __init__.py │ │ │ │ └── constants.py │ │ ├── lesson_4 │ │ │ ├── __init__.py │ │ │ └── defs │ │ │ │ └── assets │ │ │ │ ├── __init__.py │ │ │ │ └── constants.py │ │ ├── lesson_5 │ │ │ ├── __init__.py │ │ │ └── defs │ │ │ │ └── assets │ │ │ │ ├── __init__.py │ │ │ │ └── constants.py │ │ ├── lesson_6 │ │ │ ├── __init__.py │ │ │ └── defs │ │ │ │ ├── assets │ │ │ │ ├── __init__.py │ │ │ │ └── constants.py │ │ │ │ └── resources.py │ │ ├── lesson_7 │ │ │ ├── __init__.py │ │ │ └── defs │ │ │ │ ├── assets │ │ │ │ ├── __init__.py │ │ │ │ └── constants.py │ │ │ │ ├── resources.py │ │ │ │ ├── jobs.py │ │ │ │ └── schedules.py │ │ ├── lesson_8 │ │ │ ├── __init__.py │ │ │ └── defs │ │ │ │ ├── assets │ │ │ │ ├── __init__.py │ │ │ │ └── constants.py │ │ │ │ ├── resources.py │ │ │ │ ├── partitions.py │ │ │ │ ├── schedules.py │ │ │ │ └── jobs.py │ │ └── lesson_9 │ │ │ ├── __init__.py │ │ │ └── defs │ │ │ ├── assets │ │ │ ├── __init__.py │ │ │ └── constants.py │ │ │ ├── resources.py │ │ │ ├── partitions.py │ │ │ ├── schedules.py │ │ │ └── jobs.py │ │ └── definitions.py │ ├── .env │ ├── pytest.ini │ ├── Makefile │ ├── dagster_cloud.yaml │ └── README.md ├── course ├── .prettierignore ├── components │ ├── index.js │ ├── Heading.tsx │ └── Callout.tsx ├── postcss.config.js ├── .prettierrc.js ├── .yarnrc.yml ├── public │ ├── fonts │ │ ├── PPNeueMontreal-Bold.woff │ │ ├── ABCDiatypeMono-Medium.woff │ │ ├── ABCDiatypeMono-Regular.woff │ │ ├── PPNeueMontreal-Italic.woff │ │ ├── PPNeueMontreal-Medium.woff │ │ ├── PPNeueMontreal-Regular.woff │ │ └── PPNeueMontreal-SemiBolditalic.woff │ └── images │ │ ├── dagster-etl │ │ ├── lesson-5 │ │ │ ├── dlt.png │ │ │ └── dlt-etl.png │ │ ├── lesson-3 │ │ │ ├── csv-etl.png │ │ │ ├── asset-check.png │ │ │ ├── asset-check-page.png │ │ │ ├── asset-check-pass.png │ │ │ ├── file-import-dag.png │ │ │ ├── partition-execution.png │ │ │ └── partition-execution-success.png │ │ ├── lesson-4 │ │ │ ├── api-etl.png │ │ │ ├── rolling-windows.png │ │ │ ├── rolling-windows-overlap.png │ │ │ └── rolling-windows-overlap-failure.png │ │ ├── lesson-1 │ │ │ ├── what-is-etl.png │ │ │ ├── etl-assets-1.png │ │ │ ├── etl-assets-2.png │ │ │ ├── etl-as-assets-1.png │ │ │ └── etl-as-assets-2.png │ │ └── extra-credit │ │ │ └── components-vs-asset.png │ │ ├── dagster-dbt │ │ ├── lesson-2 │ │ │ └── asset-graph.png │ │ ├── lesson-7 │ │ │ └── asset-graph.png │ │ ├── lesson-3 │ │ │ ├── dbt-asset-code.png │ │ │ ├── dbt-logs-one-asset.png │ │ │ ├── dbt-logs-dbt-command.png │ │ │ ├── dbt-run-details-page.png │ │ │ ├── dbt-asset-table-schema.png │ │ │ ├── asset-description-metadata.png │ │ │ └── dbt-logs-materialization-events.png │ │ ├── lesson-1 │ │ │ └── example-asset-graph.png │ │ ├── lesson-4 │ │ │ ├── std-zones-success.png │ │ │ ├── stg-zones-code-version.png │ │ │ └── stg-zones-stdout-failure.png │ │ ├── lesson-5 │ │ │ ├── airport-trips-asset.png │ │ │ ├── job-with-dbt-models.png │ │ │ ├── job-without-dbt-models.png │ │ │ ├── new-location-metrics-asset.png │ │ │ ├── trip-update-job-dbt-assets.png │ │ │ └── unconnected-sources-assets.png │ │ └── lesson-6 │ │ │ └── daily-metrics-asset.png │ │ ├── dagster-testing │ │ ├── lesson-6 │ │ │ ├── monorepo.png │ │ │ ├── multi-repo.png │ │ │ ├── asset-check.png │ │ │ └── asset-check-success.png │ │ └── lesson-5 │ │ │ └── docker-test.png │ │ ├── shared │ │ └── codespaces │ │ │ ├── codespaces-create.png │ │ │ ├── codespaces-launch.png │ │ │ └── codespaces-stop.png │ │ └── dagster-essentials │ │ ├── lesson-3 │ │ ├── run-logs.png │ │ ├── run-stats.png │ │ ├── failed-asset.png │ │ ├── run-timeline.png │ │ ├── overview-page.png │ │ ├── assets-overview.png │ │ ├── data-raw-parquet.png │ │ ├── global-asset-view.png │ │ ├── raw-compute-logs.png │ │ ├── run-failure-step.png │ │ ├── stacktrace-error.png │ │ ├── materialize-button.png │ │ ├── materialized-asset.png │ │ ├── run-details-diagram.png │ │ ├── highlighted-re-execute.png │ │ ├── re-execute-successful.png │ │ ├── failed-run-details-page.png │ │ └── highlighted-materialization-date.png │ │ ├── lesson-1 │ │ ├── cookie-etl.png │ │ ├── email-as-dag.png │ │ ├── cookie-assets-one-cookie.png │ │ └── cookie-assets-two-cookies.png │ │ ├── lesson-2 │ │ └── dagster-ui.png │ │ ├── lesson-4 │ │ ├── asset-graph.png │ │ ├── materialized-map.png │ │ ├── cookie-dependencies.png │ │ ├── materialization-graph.png │ │ ├── parent-child-ancestor.png │ │ └── taxi-trips-in-asset-graph.png │ │ ├── lesson-7 │ │ ├── ui-jobs-tab.png │ │ ├── cookie-schedule.png │ │ ├── ui-backfills-tab.png │ │ ├── ui-schedules-tab.png │ │ ├── ui-test-schedule.png │ │ ├── ui-enabled-schedule.png │ │ ├── ui-schedule-details.png │ │ ├── crontogo-cron-syntax.png │ │ ├── ui-mock-evaluation-time.png │ │ ├── ui-schedule-test-results.png │ │ └── ui-trip-update-job-asset-graph.png │ │ ├── lesson-9 │ │ ├── trips-graph.png │ │ ├── ui-launchpad.png │ │ ├── ui-sensors-tab.png │ │ ├── ui-sensor-details.png │ │ ├── ui-test-sensor-dialog.png │ │ └── ui-asset-lineage-with-sensors.png │ │ ├── extra-credit │ │ ├── ui-plots-tab.png │ │ ├── ui-asset-groups.png │ │ ├── definition-cookies.png │ │ ├── ui-markdown-chart.png │ │ ├── materialization-cookies.png │ │ ├── ui-global-asset-lineage.png │ │ ├── ui-asset-groups-practice-answer.png │ │ └── ui-selected-adhoc-request-asset.png │ │ ├── lesson-5 │ │ ├── bakery-locations.png │ │ ├── code-locations-ui.png │ │ ├── cookie-definitions.png │ │ ├── code-location-error-ui.png │ │ ├── code-locations-reload.png │ │ ├── dagster-architecture.png │ │ └── kitchen-code-location.png │ │ ├── lesson-6 │ │ ├── resource-details.png │ │ ├── resource-uses-tab.png │ │ ├── resources-not-dry.png │ │ ├── resources-tab-2.png │ │ └── cookie-resources-dry.png │ │ └── lesson-8 │ │ ├── ui-backfills-tab.png │ │ ├── ui-partition-details.png │ │ ├── ui-asset-partition-info.png │ │ ├── ui-taxi-trips-file-asset.png │ │ ├── ui-asset-partition-details.png │ │ └── ui-assets-with-partitions.png ├── markdoc │ ├── nodes │ │ ├── index.ts │ │ ├── fence.markdoc.ts │ │ ├── link.markdoc.ts │ │ └── heading.markdoc.ts │ └── functions.ts ├── .babelrc ├── next.config.js ├── .gitignore ├── next-env.d.ts ├── pages │ ├── index.md │ ├── dagster-essentials │ │ ├── lesson-8 │ │ │ ├── 9-recap.md │ │ │ ├── 0-overview.md │ │ │ └── 3-coding-practice-weekly-partition.md │ │ ├── lesson-7 │ │ │ ├── 0-overview.md │ │ │ ├── 5-updating-the-definitions-object.md │ │ │ ├── 2-coding-practice-weekly-update-job.md │ │ │ └── 4-coding-practice-weekly-update-schedule.md │ │ ├── lesson-9 │ │ │ ├── 6-updating-the-definitions-object.md │ │ │ └── 0-overview.md │ │ ├── lesson-5 │ │ │ └── 0-overview.md │ │ ├── lesson-3 │ │ │ └── 7-recap.md │ │ ├── extra-credit │ │ │ └── 0-overview.md │ │ ├── lesson-6 │ │ │ └── 6-recap.md │ │ ├── lesson-1 │ │ │ └── 4-project-preview.md │ │ └── lesson-4 │ │ │ └── 0-overview.md │ ├── dagster-dbt │ │ ├── lesson-3 │ │ │ ├── 6-updating-the-definitions-object.md │ │ │ ├── 1-overview.md │ │ │ └── knowledge-check.md │ │ ├── lesson-4 │ │ │ └── 1-overview.md │ │ ├── lesson-5 │ │ │ └── 1-overview.md │ │ ├── lesson-1 │ │ │ ├── deprecation.md │ │ │ └── 4-project-preview.md │ │ ├── lesson-6 │ │ │ └── 4-lesson-recap.md │ │ ├── lesson-7 │ │ │ └── 1-overview.md │ │ └── lesson-2 │ │ │ ├── 4-set-up-the-dbt-project.md │ │ │ └── 3-set-up-dagster-project.md │ ├── dagster-testing │ │ ├── lesson-1 │ │ │ └── 1-project-preview.md │ │ ├── lesson-4 │ │ │ └── 0-overview.md │ │ ├── lesson-5 │ │ │ └── 0-overview.md │ │ ├── lesson-6 │ │ │ └── 0-overview.md │ │ └── lesson-3 │ │ │ └── 0-overview.md │ ├── installation │ │ └── requirements.md │ └── dagster-etl │ │ ├── lesson-4 │ │ └── 0-overview.md │ │ ├── lesson-3 │ │ └── 0-overview.md │ │ ├── lesson-6 │ │ ├── 0-overview.md │ │ └── 2-sling.md │ │ ├── lesson-1 │ │ └── 3-project-preview.md │ │ └── lesson-5 │ │ ├── 1-dlt.md │ │ └── 0-overview.md ├── .remarkrc.js ├── .eslintrc.js └── tsconfig.json ├── Makefile └── .devcontainer ├── dagster-and-dbt ├── Dockerfile └── devcontainer.json ├── dagster-and-etl ├── Dockerfile └── devcontainer.json ├── dagster-testing ├── Dockerfile └── devcontainer.json └── dagster-essentials ├── Dockerfile └── devcontainer.json /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @dehume -------------------------------------------------------------------------------- /.github/workflows/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @dehume -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/data/raw/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_testing/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /course/.prettierignore: -------------------------------------------------------------------------------- 1 | .yarn 2 | .yarnrc.yml 3 | yarn.lock -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/data/outputs/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/data/staging/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/data/staging/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/data/raw/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/data/outputs/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/data/staging/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/tests/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /dagster_university/dagster_testing/tests/completed/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/defs/jobs.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_testing/src/dagster_testing/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/defs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/defs/project.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/defs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/defs/assets.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/defs/resources.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/defs/schedules.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/defs/sensors.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_testing/src/dagster_testing/defs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/defs/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/defs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_testing/src/dagster_testing/defs/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/.env: -------------------------------------------------------------------------------- 1 | DUCKDB_DATABASE=data/staging/data.duckdb -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_2/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_3/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_4/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_3/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_4/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_5/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_6/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_7/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/defs/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_3/defs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_4/defs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_5/defs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_6/defs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_7/defs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_3/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_4/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_5/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_6/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_7/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_8/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_9/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_2/defs/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_3/defs/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_4/defs/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/defs/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/defs/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/defs/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_3/defs/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_4/defs/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_5/defs/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_6/defs/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_7/defs/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_8/defs/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_9/defs/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/.env: -------------------------------------------------------------------------------- 1 | DUCKDB_DATABASE=data/staging/data.duckdb 2 | DAGSTER_ENVIRONMENT=dev -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | filterwarnings = 3 | ignore::DeprecationWarning -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | filterwarnings = 3 | ignore::DeprecationWarning -------------------------------------------------------------------------------- /course/components/index.js: -------------------------------------------------------------------------------- 1 | export * from './Callout'; 2 | export * from './CodeBlock'; 3 | export * from './Heading'; 4 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/analytics/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | -------------------------------------------------------------------------------- /course/postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | }; 7 | -------------------------------------------------------------------------------- /dagster_university/dagster_testing/tests/data/test.csv: -------------------------------------------------------------------------------- 1 | City,Population 2 | Example 1,4500000 3 | Example 2,3000000 4 | Example 3,1000000 -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_3/analytics/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_4/analytics/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/analytics/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/analytics/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/analytics/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/Makefile: -------------------------------------------------------------------------------- 1 | ruff: 2 | ruff check . --select I --fix 3 | ruff check . --fix 4 | ruff format . 5 | ruff check . -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/analytics/packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: 1.1.1 -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/tests/postgres/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM postgres:alpine 2 | 3 | COPY postgres_bootstrap.sql /docker-entrypoint-initdb.d/ -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/Makefile: -------------------------------------------------------------------------------- 1 | ruff: 2 | ruff check . --select I --fix 3 | ruff check . --fix 4 | ruff format . 5 | ruff check . -------------------------------------------------------------------------------- /dagster_university/dagster_testing/src/dagster_testing/defs/data/ny.csv: -------------------------------------------------------------------------------- 1 | City,Population 2 | New York,8804190 3 | Buffalo,278349 4 | Yonkers,211569 -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/Makefile: -------------------------------------------------------------------------------- 1 | ruff: 2 | ruff check . --select I --fix 3 | ruff check . --fix 4 | ruff format . 5 | ruff check . 6 | -------------------------------------------------------------------------------- /dagster_university/dagster_testing/Makefile: -------------------------------------------------------------------------------- 1 | ruff: 2 | ruff check . --select I --fix 3 | ruff check . --fix 4 | ruff format . 5 | ruff check . 6 | -------------------------------------------------------------------------------- /dagster_university/dagster_testing/src/dagster_testing/defs/data/ca.csv: -------------------------------------------------------------------------------- 1 | City,Population 2 | Los Angeles,3898747 3 | San Diego,1386932 4 | San Jose,1013240 -------------------------------------------------------------------------------- /dagster_university/dagster_testing/src/dagster_testing/defs/data/mn.csv: -------------------------------------------------------------------------------- 1 | City,Population 2 | Minneapolis,425115 3 | Saint Paul,303820 4 | Rochester,122413 -------------------------------------------------------------------------------- /dagster_university/dagster_testing/tests/postgres/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM postgres:17-alpine 2 | 3 | COPY postgres_bootstrap.sql /docker-entrypoint-initdb.d/ -------------------------------------------------------------------------------- /course/.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | bracketSpacing: false, 3 | printWidth: 100, 4 | singleQuote: true, 5 | trailingComma: 'all', 6 | }; 7 | -------------------------------------------------------------------------------- /dagster_university/dagster_testing/tests/configs/lesson_3.yaml: -------------------------------------------------------------------------------- 1 | ops: 2 | state_population_file_config: 3 | config: 4 | path: "tests/data/test.csv" -------------------------------------------------------------------------------- /course/.yarnrc.yml: -------------------------------------------------------------------------------- 1 | compressionLevel: mixed 2 | 3 | enableGlobalCache: false 4 | 5 | nodeLinker: node-modules 6 | 7 | yarnPath: .yarn/releases/yarn-4.5.0.cjs 8 | -------------------------------------------------------------------------------- /course/public/fonts/PPNeueMontreal-Bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/fonts/PPNeueMontreal-Bold.woff -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/dagster_cloud.yaml: -------------------------------------------------------------------------------- 1 | locations: 2 | - location_name: dagster_and_dbt 3 | code_source: 4 | package_name: dagster_and_dbt -------------------------------------------------------------------------------- /course/public/fonts/ABCDiatypeMono-Medium.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/fonts/ABCDiatypeMono-Medium.woff -------------------------------------------------------------------------------- /course/public/fonts/ABCDiatypeMono-Regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/fonts/ABCDiatypeMono-Regular.woff -------------------------------------------------------------------------------- /course/public/fonts/PPNeueMontreal-Italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/fonts/PPNeueMontreal-Italic.woff -------------------------------------------------------------------------------- /course/public/fonts/PPNeueMontreal-Medium.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/fonts/PPNeueMontreal-Medium.woff -------------------------------------------------------------------------------- /course/public/fonts/PPNeueMontreal-Regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/fonts/PPNeueMontreal-Regular.woff -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_3/analytics/packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: 1.1.1 -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_4/analytics/packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: 1.1.1 -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/analytics/packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: 1.1.1 -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/analytics/packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: 1.1.1 -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/analytics/packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: 1.1.1 -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-5/dlt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-5/dlt.png -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/dagster_cloud.yaml: -------------------------------------------------------------------------------- 1 | locations: 2 | - location_name: dagster_essentials 3 | code_source: 4 | package_name: dagster_essentials -------------------------------------------------------------------------------- /dagster_university/dagster_testing/src/dagster_testing/defs/configs/run_config.yaml: -------------------------------------------------------------------------------- 1 | ops: 2 | population_file_config: 3 | config: 4 | path: "tests/data/test.csv" -------------------------------------------------------------------------------- /course/public/fonts/PPNeueMontreal-SemiBolditalic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/fonts/PPNeueMontreal-SemiBolditalic.woff -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-3/csv-etl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-3/csv-etl.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-4/api-etl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-4/api-etl.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-5/dlt-etl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-5/dlt-etl.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-2/asset-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-2/asset-graph.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-7/asset-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-7/asset-graph.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-1/what-is-etl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-1/what-is-etl.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-3/asset-check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-3/asset-check.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-3/dbt-asset-code.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-3/dbt-asset-code.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-1/etl-assets-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-1/etl-assets-1.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-1/etl-assets-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-1/etl-assets-2.png -------------------------------------------------------------------------------- /course/public/images/dagster-testing/lesson-6/monorepo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-testing/lesson-6/monorepo.png -------------------------------------------------------------------------------- /course/public/images/dagster-testing/lesson-6/multi-repo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-testing/lesson-6/multi-repo.png -------------------------------------------------------------------------------- /course/public/images/shared/codespaces/codespaces-create.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/shared/codespaces/codespaces-create.png -------------------------------------------------------------------------------- /course/public/images/shared/codespaces/codespaces-launch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/shared/codespaces/codespaces-launch.png -------------------------------------------------------------------------------- /course/public/images/shared/codespaces/codespaces-stop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/shared/codespaces/codespaces-stop.png -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | test_all: 2 | @for dir in dagster_essentials dagster_and_dbt dagster_testing dagster_and_etl; do \ 3 | ( cd dagster_university/$$dir && uv run pytest tests -p no:warnings ); \ 4 | done -------------------------------------------------------------------------------- /course/markdoc/nodes/index.ts: -------------------------------------------------------------------------------- 1 | /* Use this file to export your markdoc nodes */ 2 | export * from './fence.markdoc'; 3 | export * from './heading.markdoc'; 4 | export * from './link.markdoc'; 5 | -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/run-logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/run-logs.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/run-stats.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/run-stats.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-1/etl-as-assets-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-1/etl-as-assets-1.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-1/etl-as-assets-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-1/etl-as-assets-2.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-3/asset-check-page.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-3/asset-check-page.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-3/asset-check-pass.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-3/asset-check-pass.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-3/file-import-dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-3/file-import-dag.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-4/rolling-windows.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-4/rolling-windows.png -------------------------------------------------------------------------------- /course/public/images/dagster-testing/lesson-5/docker-test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-testing/lesson-5/docker-test.png -------------------------------------------------------------------------------- /course/public/images/dagster-testing/lesson-6/asset-check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-testing/lesson-6/asset-check.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-1/example-asset-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-1/example-asset-graph.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-3/dbt-logs-one-asset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-3/dbt-logs-one-asset.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-4/std-zones-success.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-4/std-zones-success.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-5/airport-trips-asset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-5/airport-trips-asset.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-5/job-with-dbt-models.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-5/job-with-dbt-models.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-6/daily-metrics-asset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-6/daily-metrics-asset.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-1/cookie-etl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-1/cookie-etl.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-1/email-as-dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-1/email-as-dag.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-2/dagster-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-2/dagster-ui.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/failed-asset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/failed-asset.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/run-timeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/run-timeline.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-4/asset-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-4/asset-graph.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-7/ui-jobs-tab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-7/ui-jobs-tab.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-9/trips-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-9/trips-graph.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-9/ui-launchpad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-9/ui-launchpad.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-3/partition-execution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-3/partition-execution.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-3/dbt-logs-dbt-command.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-3/dbt-logs-dbt-command.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-3/dbt-run-details-page.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-3/dbt-run-details-page.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/overview-page.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/overview-page.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-9/ui-sensors-tab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-9/ui-sensors-tab.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-3/dbt-asset-table-schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-3/dbt-asset-table-schema.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-4/stg-zones-code-version.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-4/stg-zones-code-version.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-4/stg-zones-stdout-failure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-4/stg-zones-stdout-failure.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-5/job-without-dbt-models.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-5/job-without-dbt-models.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/extra-credit/ui-plots-tab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/extra-credit/ui-plots-tab.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/assets-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/assets-overview.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/data-raw-parquet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/data-raw-parquet.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/global-asset-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/global-asset-view.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/raw-compute-logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/raw-compute-logs.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/run-failure-step.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/run-failure-step.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/stacktrace-error.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/stacktrace-error.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-4/materialized-map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-4/materialized-map.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-5/bakery-locations.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-5/bakery-locations.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-5/code-locations-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-5/code-locations-ui.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-6/resource-details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-6/resource-details.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-6/resource-uses-tab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-6/resource-uses-tab.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-6/resources-not-dry.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-6/resources-not-dry.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-6/resources-tab-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-6/resources-tab-2.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-7/cookie-schedule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-7/cookie-schedule.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-7/ui-backfills-tab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-7/ui-backfills-tab.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-7/ui-schedules-tab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-7/ui-schedules-tab.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-7/ui-test-schedule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-7/ui-test-schedule.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-8/ui-backfills-tab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-8/ui-backfills-tab.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-9/ui-sensor-details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-9/ui-sensor-details.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/extra-credit/components-vs-asset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/extra-credit/components-vs-asset.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-4/rolling-windows-overlap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-4/rolling-windows-overlap.png -------------------------------------------------------------------------------- /course/public/images/dagster-testing/lesson-6/asset-check-success.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-testing/lesson-6/asset-check-success.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-3/asset-description-metadata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-3/asset-description-metadata.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-5/new-location-metrics-asset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-5/new-location-metrics-asset.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-5/trip-update-job-dbt-assets.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-5/trip-update-job-dbt-assets.png -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-5/unconnected-sources-assets.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-5/unconnected-sources-assets.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/extra-credit/ui-asset-groups.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/extra-credit/ui-asset-groups.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/materialize-button.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/materialize-button.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/materialized-asset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/materialized-asset.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/run-details-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/run-details-diagram.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-4/cookie-dependencies.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-4/cookie-dependencies.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-5/cookie-definitions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-5/cookie-definitions.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-7/ui-enabled-schedule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-7/ui-enabled-schedule.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-7/ui-schedule-details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-7/ui-schedule-details.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/extra-credit/definition-cookies.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/extra-credit/definition-cookies.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/extra-credit/ui-markdown-chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/extra-credit/ui-markdown-chart.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/highlighted-re-execute.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/highlighted-re-execute.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/re-execute-successful.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/re-execute-successful.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-4/materialization-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-4/materialization-graph.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-4/parent-child-ancestor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-4/parent-child-ancestor.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-5/code-location-error-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-5/code-location-error-ui.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-5/code-locations-reload.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-5/code-locations-reload.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-5/dagster-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-5/dagster-architecture.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-5/kitchen-code-location.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-5/kitchen-code-location.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-6/cookie-resources-dry.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-6/cookie-resources-dry.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-7/crontogo-cron-syntax.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-7/crontogo-cron-syntax.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-8/ui-partition-details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-8/ui-partition-details.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-9/ui-test-sensor-dialog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-9/ui-test-sensor-dialog.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-3/partition-execution-success.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-3/partition-execution-success.png -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/tests/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | markers = 3 | integration: these tests require Docker (deselect with '-m "not integration"') 4 | 5 | filterwarnings = 6 | ignore::DeprecationWarning -------------------------------------------------------------------------------- /dagster_university/dagster_testing/tests/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | markers = 3 | integration: these tests require Docker (deselect with '-m "not integration"') 4 | 5 | filterwarnings = 6 | ignore::DeprecationWarning -------------------------------------------------------------------------------- /course/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["next/babel"], 3 | "plugins": [ 4 | [ 5 | "prismjs", 6 | { 7 | "languages": ["python"], 8 | "css": true 9 | } 10 | ] 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /course/public/images/dagster-dbt/lesson-3/dbt-logs-materialization-events.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-dbt/lesson-3/dbt-logs-materialization-events.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-1/cookie-assets-one-cookie.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-1/cookie-assets-one-cookie.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/failed-run-details-page.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/failed-run-details-page.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-7/ui-mock-evaluation-time.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-7/ui-mock-evaluation-time.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-7/ui-schedule-test-results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-7/ui-schedule-test-results.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-8/ui-asset-partition-info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-8/ui-asset-partition-info.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-8/ui-taxi-trips-file-asset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-8/ui-taxi-trips-file-asset.png -------------------------------------------------------------------------------- /course/public/images/dagster-etl/lesson-4/rolling-windows-overlap-failure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-etl/lesson-4/rolling-windows-overlap-failure.png -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture(autouse=True) 5 | def duckdb_path_env(monkeypatch): 6 | monkeypatch.setenv("DUCKDB_DATABASE", "data/staging/data.duckdb") 7 | -------------------------------------------------------------------------------- /course/next.config.js: -------------------------------------------------------------------------------- 1 | const withMarkdoc = require('@markdoc/next.js'); 2 | 3 | module.exports = withMarkdoc({schemaPath: './markdoc', mode: 'static'})({ 4 | pageExtensions: ['js', 'jsx', 'ts', 'tsx', 'md', 'mdoc'], 5 | }); 6 | -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/extra-credit/materialization-cookies.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/extra-credit/materialization-cookies.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/extra-credit/ui-global-asset-lineage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/extra-credit/ui-global-asset-lineage.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-1/cookie-assets-two-cookies.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-1/cookie-assets-two-cookies.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-4/taxi-trips-in-asset-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-4/taxi-trips-in-asset-graph.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-8/ui-asset-partition-details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-8/ui-asset-partition-details.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-8/ui-assets-with-partitions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-8/ui-assets-with-partitions.png -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/analytics/models/sources/raw_taxis.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: raw_taxis 5 | schema: main 6 | tables: 7 | - name: zones 8 | - name: trips -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_3/analytics/package-lock.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: 1.1.1 4 | sha1_hash: a158c48c59c2bb7d729d2a4e215aabe5bb4f3353 5 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_4/analytics/package-lock.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: 1.1.1 4 | sha1_hash: a158c48c59c2bb7d729d2a4e215aabe5bb4f3353 5 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/analytics/package-lock.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: 1.1.1 4 | sha1_hash: a158c48c59c2bb7d729d2a4e215aabe5bb4f3353 5 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/analytics/package-lock.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: 1.1.1 4 | sha1_hash: a158c48c59c2bb7d729d2a4e215aabe5bb4f3353 5 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/analytics/package-lock.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: 1.1.1 4 | sha1_hash: a158c48c59c2bb7d729d2a4e215aabe5bb4f3353 5 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture(autouse=True) 5 | def duckdb_path_env(monkeypatch): 6 | monkeypatch.setenv("DUCKDB_DATABASE", "data/staging/data.duckdb") 7 | -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-9/ui-asset-lineage-with-sensors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-9/ui-asset-lineage-with-sensors.png -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/tests/run_config.yaml: -------------------------------------------------------------------------------- 1 | ops: 2 | adhoc_request: 3 | config: 4 | filename: "example.json" 5 | borough: "Staten Island" 6 | start_date: "2023-01-10" 7 | end_date: "2023-01-25" -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/tests/run_config.yaml: -------------------------------------------------------------------------------- 1 | ops: 2 | adhoc_request: 3 | config: 4 | filename: "example.json" 5 | borough: "Staten Island" 6 | start_date: "2023-01-10" 7 | end_date: "2023-01-25" -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-3/highlighted-materialization-date.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-3/highlighted-materialization-date.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/lesson-7/ui-trip-update-job-asset-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/lesson-7/ui-trip-update-job-asset-graph.png -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/tests/test_lesson_starter.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | import dagster_and_dbt.defs 4 | 5 | 6 | def test_defs(): 7 | assert dg.Definitions.merge(dg.components.load_defs(dagster_and_dbt.defs)) 8 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/tests/test_lesson_starter.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | import dagster_and_etl.defs 4 | 5 | 6 | def test_defs(): 7 | assert dg.Definitions.merge(dg.components.load_defs(dagster_and_etl.defs)) 8 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/tests/test_lesson_starter.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | import dagster_essentials.defs 4 | 5 | 6 | def test_defs(): 7 | assert dg.Definitions.merge(dg.components.load_defs(dagster_essentials.defs)) 8 | -------------------------------------------------------------------------------- /course/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | .DS_Store 3 | .next/ 4 | .vercel 5 | package-lock.json 6 | 7 | .pnp.* 8 | .yarn/* 9 | !.yarn/patches 10 | !.yarn/plugins 11 | !.yarn/releases 12 | !.yarn/sdks 13 | !.yarn/versions 14 | 15 | *.tsbuildinfo -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/extra-credit/ui-asset-groups-practice-answer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/extra-credit/ui-asset-groups-practice-answer.png -------------------------------------------------------------------------------- /course/public/images/dagster-essentials/extra-credit/ui-selected-adhoc-request-asset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagster-io/project-dagster-university/HEAD/course/public/images/dagster-essentials/extra-credit/ui-selected-adhoc-request-asset.png -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_3/analytics/models/sources/raw_taxis.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: raw_taxis 5 | schema: main 6 | tables: 7 | - name: zones 8 | - name: trips -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_4/analytics/models/sources/raw_taxis.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: raw_taxis 5 | schema: main 6 | tables: 7 | - name: zones 8 | - name: trips -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/analytics/models/sources/raw_taxis.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: raw_taxis 5 | schema: main 6 | tables: 7 | - name: zones 8 | - name: trips -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/analytics/models/sources/raw_taxis.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: raw_taxis 5 | schema: main 6 | tables: 7 | - name: zones 8 | - name: trips -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/analytics/models/sources/raw_taxis.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: raw_taxis 5 | schema: main 6 | tables: 7 | - name: zones 8 | - name: trips -------------------------------------------------------------------------------- /dagster_university/dagster_testing/tests/completed/test_lesson_starter.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | import dagster_testing.defs 4 | 5 | 6 | def test_defs(): 7 | assert dg.Definitions.merge(dg.components.load_defs(dagster_testing.defs)) 8 | -------------------------------------------------------------------------------- /course/next-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | /// 3 | 4 | // NOTE: This file should not be edited 5 | // see https://nextjs.org/docs/pages/building-your-application/configuring/typescript for more information. 6 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_3/defs/project.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from dagster_dbt import DbtProject 4 | 5 | dbt_project = DbtProject( 6 | project_dir=Path(__file__).joinpath("../..", "analytics").resolve(), 7 | ) 8 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_3/defs/assets/constants.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | TAXI_ZONES_FILE_PATH = os.path.join("data", "raw", "taxi_zones.csv") 4 | TAXI_TRIPS_TEMPLATE_FILE_PATH = os.path.join("data", "raw", "taxi_trips_{}.parquet") 5 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/definitions.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from dagster import definitions, load_from_defs_folder 4 | 5 | 6 | @definitions 7 | def defs(): 8 | return load_from_defs_folder(project_root=Path(__file__).parent.parent.parent) 9 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/definitions.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from dagster import definitions, load_from_defs_folder 4 | 5 | 6 | @definitions 7 | def defs(): 8 | return load_from_defs_folder(project_root=Path(__file__).parent.parent.parent) 9 | -------------------------------------------------------------------------------- /dagster_university/dagster_testing/src/dagster_testing/definitions.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from dagster import definitions, load_from_defs_folder 4 | 5 | 6 | @definitions 7 | def defs(): 8 | return load_from_defs_folder(project_root=Path(__file__).parent.parent.parent) 9 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_2/definitions.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | import dagster_and_dbt.completed.lesson_2.defs 4 | 5 | 6 | @dg.definitions 7 | def defs(): 8 | return dg.load_defs(defs_root=dagster_and_dbt.completed.lesson_2.defs) 9 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_3/definitions.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | import dagster_and_dbt.completed.lesson_3.defs 4 | 5 | 6 | @dg.definitions 7 | def defs(): 8 | return dg.load_defs(defs_root=dagster_and_dbt.completed.lesson_3.defs) 9 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_4/definitions.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | import dagster_and_dbt.completed.lesson_4.defs 4 | 5 | 6 | @dg.definitions 7 | def defs(): 8 | return dg.load_defs(defs_root=dagster_and_dbt.completed.lesson_4.defs) 9 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/definitions.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | import dagster_and_dbt.completed.lesson_5.defs 4 | 5 | 6 | @dg.definitions 7 | def defs(): 8 | return dg.load_defs(defs_root=dagster_and_dbt.completed.lesson_5.defs) 9 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/definitions.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | import dagster_and_dbt.completed.lesson_6.defs 4 | 5 | 6 | @dg.definitions 7 | def defs(): 8 | return dg.load_defs(defs_root=dagster_and_dbt.completed.lesson_6.defs) 9 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/definitions.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | import dagster_and_dbt.completed.lesson_7.defs 4 | 5 | 6 | @dg.definitions 7 | def defs(): 8 | return dg.load_defs(defs_root=dagster_and_dbt.completed.lesson_7.defs) 9 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/.gitignore: -------------------------------------------------------------------------------- 1 | # ignore secrets, virtual environments and typical python compilation artifacts 2 | secrets.toml 3 | # ignore basic python artifacts 4 | .env 5 | **/__pycache__/ 6 | **/*.py[cod] 7 | **/*$py.class 8 | # ignore duckdb 9 | *.duckdb 10 | *.wal -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/definitions.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from dagster import definitions, load_from_defs_folder 4 | 5 | 6 | @definitions 7 | def defs(): 8 | return load_from_defs_folder(project_root=Path(__file__).parent.parent.parent) 9 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_3/defs/schedules.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | import dagster_and_etl.completed.lesson_3.defs.jobs as jobs 4 | 5 | asset_partitioned_schedule = dg.build_schedule_from_partitioned_job( 6 | jobs.import_partition_job, 7 | ) 8 | -------------------------------------------------------------------------------- /course/pages/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Dagster University Course Content 3 | --- 4 | 5 | # Dagster University Course Content 6 | 7 | - [Dagster Essentials](/dagster-essentials) 8 | - [Dagster + dbt](/dagster-dbt) 9 | - [Dagster + ETL](/dagster-etl) 10 | - [Testing with Dagster](/dagster-testing) 11 | -------------------------------------------------------------------------------- /.devcontainer/dagster-and-dbt/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/devcontainers/python:1-3.12-bullseye 2 | ENV PYTHONUNBUFFERED 1 3 | 4 | COPY --from=ghcr.io/astral-sh/uv:0.6.10 /uv /bin/uv 5 | 6 | COPY dagster_university/dagster_and_dbt /workspace 7 | WORKDIR /workspace 8 | 9 | RUN uv pip install -r pyproject.toml --system -------------------------------------------------------------------------------- /.devcontainer/dagster-and-etl/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/devcontainers/python:1-3.12-bullseye 2 | ENV PYTHONUNBUFFERED 1 3 | 4 | COPY --from=ghcr.io/astral-sh/uv:0.6.10 /uv /bin/uv 5 | 6 | COPY dagster_university/dagster_and_etl /workspace 7 | WORKDIR /workspace 8 | 9 | RUN uv pip install -r pyproject.toml --system -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_3/analytics/profiles.yml: -------------------------------------------------------------------------------- 1 | dagster_dbt_university: 2 | target: dev 3 | outputs: 4 | dev: 5 | type: duckdb 6 | # Different path for nested project 7 | path: '../../../../../{{ env_var("DUCKDB_DATABASE", "data/staging/data.duckdb") }}' -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_4/analytics/profiles.yml: -------------------------------------------------------------------------------- 1 | dagster_dbt_university: 2 | target: dev 3 | outputs: 4 | dev: 5 | type: duckdb 6 | # Different path for nested project 7 | path: '../../../../../{{ env_var("DUCKDB_DATABASE", "data/staging/data.duckdb") }}' -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_4/defs/project.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from dagster_dbt import DbtProject 4 | 5 | dbt_project = DbtProject( 6 | project_dir=Path(__file__).joinpath("../..", "analytics").resolve(), 7 | ) 8 | 9 | dbt_project.prepare_if_dev() 10 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/analytics/profiles.yml: -------------------------------------------------------------------------------- 1 | dagster_dbt_university: 2 | target: dev 3 | outputs: 4 | dev: 5 | type: duckdb 6 | # Different path for nested project 7 | path: '../../../../../{{ env_var("DUCKDB_DATABASE", "data/staging/data.duckdb") }}' -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/defs/project.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from dagster_dbt import DbtProject 4 | 5 | dbt_project = DbtProject( 6 | project_dir=Path(__file__).joinpath("../..", "analytics").resolve(), 7 | ) 8 | 9 | dbt_project.prepare_if_dev() 10 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/analytics/profiles.yml: -------------------------------------------------------------------------------- 1 | dagster_dbt_university: 2 | target: dev 3 | outputs: 4 | dev: 5 | type: duckdb 6 | # Different path for nested project 7 | path: '../../../../../{{ env_var("DUCKDB_DATABASE", "data/staging/data.duckdb") }}' -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/defs/project.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from dagster_dbt import DbtProject 4 | 5 | dbt_project = DbtProject( 6 | project_dir=Path(__file__).joinpath("../..", "analytics").resolve(), 7 | ) 8 | 9 | dbt_project.prepare_if_dev() 10 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/analytics/profiles.yml: -------------------------------------------------------------------------------- 1 | dagster_dbt_university: 2 | target: dev 3 | outputs: 4 | dev: 5 | type: duckdb 6 | # Different path for nested project 7 | path: '../../../../../{{ env_var("DUCKDB_DATABASE", "data/staging/data.duckdb") }}' -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/defs/project.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from dagster_dbt import DbtProject 4 | 5 | dbt_project = DbtProject( 6 | project_dir=Path(__file__).joinpath("../..", "analytics").resolve(), 7 | ) 8 | 9 | dbt_project.prepare_if_dev() 10 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/analytics/models/staging/stg_zones.sql: -------------------------------------------------------------------------------- 1 | with raw_zones as ( 2 | select * 3 | from {{ source('raw_taxis', 'zones') }} 4 | ) 5 | select 6 | zone_id, 7 | zone as zone_name, 8 | borough, 9 | zone_name like '%Airport' as is_airport, 10 | from raw_zones -------------------------------------------------------------------------------- /.devcontainer/dagster-testing/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/devcontainers/python:1-3.12-bullseye 2 | ENV PYTHONUNBUFFERED 1 3 | 4 | COPY --from=ghcr.io/astral-sh/uv:0.6.10 /uv /bin/uv 5 | 6 | COPY dagster_university/dagster_testing /workspace 7 | WORKDIR /workspace 8 | 9 | RUN uv pip install -r pyproject.toml --system 10 | -------------------------------------------------------------------------------- /.devcontainer/dagster-essentials/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/devcontainers/python:1-3.12-bullseye 2 | ENV PYTHONUNBUFFERED 1 3 | 4 | COPY --from=ghcr.io/astral-sh/uv:0.6.10 /uv /bin/uv 5 | 6 | COPY dagster_university/dagster_essentials /workspace 7 | WORKDIR /workspace 8 | 9 | RUN uv pip install -r pyproject.toml --system 10 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_3/analytics/models/staging/stg_zones.sql: -------------------------------------------------------------------------------- 1 | with raw_zones as ( 2 | select * 3 | from {{ source('raw_taxis', 'zones') }} 4 | ) 5 | select 6 | zone_id, 7 | zone as zone_name, 8 | borough, 9 | zone_name like '%Airport' as is_airport, 10 | from raw_zones -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/analytics/models/staging/stg_zones.sql: -------------------------------------------------------------------------------- 1 | with raw_zones as ( 2 | select * 3 | from {{ source('raw_taxis', 'zones') }} 4 | ) 5 | select 6 | zone_id, 7 | zone as zone_name, 8 | borough, 9 | zone_name like '%Airport' as is_airport, 10 | from raw_zones -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/analytics/models/staging/stg_zones.sql: -------------------------------------------------------------------------------- 1 | with raw_zones as ( 2 | select * 3 | from {{ source('raw_taxis', 'zones') }} 4 | ) 5 | select 6 | zone_id, 7 | zone as zone_name, 8 | borough, 9 | zone_name like '%Airport' as is_airport, 10 | from raw_zones -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/analytics/models/staging/stg_zones.sql: -------------------------------------------------------------------------------- 1 | with raw_zones as ( 2 | select * 3 | from {{ source('raw_taxis', 'zones') }} 4 | ) 5 | select 6 | zone_id, 7 | zone as zone_name, 8 | borough, 9 | zone_name like '%Airport' as is_airport, 10 | from raw_zones -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_5/defs/resources.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | from dagster_dlt import DagsterDltResource 3 | 4 | 5 | @dg.definitions 6 | def resources(): 7 | return dg.Definitions( 8 | resources={ 9 | "dlt": DagsterDltResource(), 10 | }, 11 | ) 12 | -------------------------------------------------------------------------------- /.devcontainer/dagster-and-dbt/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Dagster & dbt", 3 | "build": { 4 | "dockerfile": "Dockerfile", 5 | "context": "../.." 6 | }, 7 | "forwardPorts": [ 8 | 3000 9 | ], 10 | "portsAttributes": { 11 | "3000": { 12 | "label": "Dagster" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /dagster_university/dagster_testing/tests/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | services: 4 | postgresql: 5 | build: 6 | context: ./postgres 7 | container_name: postgresql 8 | environment: 9 | POSTGRES_DB: "test_db" 10 | POSTGRES_USER: "test_user" 11 | POSTGRES_PASSWORD: "test_pass" 12 | ports: 13 | - "5432:5432" 14 | -------------------------------------------------------------------------------- /.devcontainer/dagster-essentials/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Dagster Essentials", 3 | "build": { 4 | "dockerfile": "Dockerfile", 5 | "context": "../.." 6 | }, 7 | "forwardPorts": [ 8 | 3000 9 | ], 10 | "portsAttributes": { 11 | "3000": { 12 | "label": "Dagster" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture(autouse=True) 5 | def duckdb_path_env(monkeypatch): 6 | monkeypatch.setenv("DUCKDB_DATABASE", "data/staging/data.duckdb") 7 | 8 | 9 | @pytest.fixture(autouse=True) 10 | def nasa_api_key_env(monkeypatch): 11 | monkeypatch.setenv("NASA_API_KEY", "DEMO_KEY") 12 | -------------------------------------------------------------------------------- /dagster_university/dagster_testing/tests/postgres/postgres_bootstrap.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA data; 2 | 3 | CREATE TABLE data.city_population ( 4 | state_name VARCHAR(100), 5 | city_name VARCHAR(100), 6 | population INT 7 | ); 8 | 9 | INSERT INTO data.city_population VALUES ('NY', 'New York', 8804190), ('NY', 'Buffalo', 278349), ('CA', 'Los Angeles', 3898747); 10 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_4/analytics/models/staging/stg_zones.sql: -------------------------------------------------------------------------------- 1 | with raw_zones as ( 2 | select * 3 | from {{ source('raw_taxis', 'zones') }} 4 | ) 5 | select 6 | zone_id, 7 | zone as zone_name, 8 | borough, 9 | zone_name like '%Airport' as is_airport, 10 | -- zone_population ## new column 11 | from raw_zones -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_6/defs/resources.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | from dagster_duckdb import DuckDBResource 3 | 4 | database_resource = DuckDBResource( 5 | database=dg.EnvVar("DUCKDB_DATABASE"), 6 | ) 7 | 8 | 9 | @dg.definitions 10 | def resources(): 11 | return dg.Definitions(resources={"database": database_resource}) 12 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_7/defs/resources.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | from dagster_duckdb import DuckDBResource 3 | 4 | database_resource = DuckDBResource( 5 | database=dg.EnvVar("DUCKDB_DATABASE"), 6 | ) 7 | 8 | 9 | @dg.definitions 10 | def resources(): 11 | return dg.Definitions(resources={"database": database_resource}) 12 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_8/defs/resources.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | from dagster_duckdb import DuckDBResource 3 | 4 | database_resource = DuckDBResource( 5 | database=dg.EnvVar("DUCKDB_DATABASE"), 6 | ) 7 | 8 | 9 | @dg.definitions 10 | def resources(): 11 | return dg.Definitions(resources={"database": database_resource}) 12 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_9/defs/resources.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | from dagster_duckdb import DuckDBResource 3 | 4 | database_resource = DuckDBResource( 5 | database=dg.EnvVar("DUCKDB_DATABASE"), 6 | ) 7 | 8 | 9 | @dg.definitions 10 | def resources(): 11 | return dg.Definitions(resources={"database": database_resource}) 12 | -------------------------------------------------------------------------------- /course/pages/dagster-essentials/lesson-8/9-recap.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 8: Recap' 3 | module: 'dagster_essentials' 4 | lesson: '8' 5 | --- 6 | 7 | # Recap 8 | 9 | In this lesson, you partitioned your data by time. However, Dagster enables you to partition by any dimensions you have. You can partition with a static list of dimensions or by a dynamic list that you define the logic for. 10 | -------------------------------------------------------------------------------- /course/pages/dagster-dbt/lesson-3/6-updating-the-definitions-object.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 3: Updating the Definitions object' 3 | module: 'dagster_dbt' 4 | lesson: '3' 5 | --- 6 | 7 | # Updating the Definitions object 8 | 9 | Since we established the mapping of the resources in the `defs/resources.py` there is no need to update the `definitions.py` which will automatically load all of our Dagster objects. 10 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_3/defs/resources.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | from dagster_duckdb import DuckDBResource 3 | 4 | 5 | @dg.definitions 6 | def resources(): 7 | return dg.Definitions( 8 | resources={ 9 | "database": DuckDBResource( 10 | database="data/staging/data.duckdb", 11 | ), 12 | } 13 | ) 14 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_6/defs/sling_replication.yaml: -------------------------------------------------------------------------------- 1 | source: MY_POSTGRES 2 | target: MY_DUCKDB 3 | 4 | defaults: 5 | mode: full-refresh 6 | 7 | object: "{stream_schema}_{stream_table}" 8 | 9 | streams: 10 | data.customers: 11 | data.products: 12 | data.orders: 13 | mode: incremental 14 | primary_key: order_id 15 | update_key: order_date 16 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_7/defs/ingest_files/replication.yaml: -------------------------------------------------------------------------------- 1 | source: MY_POSTGRES 2 | target: MY_DUCKDB 3 | 4 | defaults: 5 | mode: full-refresh 6 | 7 | object: "{stream_schema}_{stream_table}" 8 | 9 | streams: 10 | data.customers: 11 | data.products: 12 | data.orders: 13 | mode: incremental 14 | primary_key: order_id 15 | update_key: order_date 16 | -------------------------------------------------------------------------------- /course/.remarkrc.js: -------------------------------------------------------------------------------- 1 | exports.settings = { 2 | bullet: '-', 3 | emphasis: '_', 4 | strong: '*', 5 | listItemIndent: 'one', 6 | rule: '-', 7 | // https://github.com/remarkjs/remark/tree/main/packages/remark-stringify#options 8 | }; 9 | exports.plugins = [ 10 | require('remark-frontmatter'), 11 | // GitHub Flavored Markdown's table pipe alignment 12 | [require('remark-gfm'), {tablePipeAlign: true}], 13 | ]; 14 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_7/defs/jobs.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | trips_by_week = dg.AssetSelection.assets("trips_by_week") 4 | 5 | trip_update_job = dg.define_asset_job( 6 | name="trip_update_job", selection=dg.AssetSelection.all() - trips_by_week 7 | ) 8 | 9 | weekly_update_job = dg.define_asset_job( 10 | name="weekly_update_job", 11 | selection=trips_by_week, 12 | ) 13 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/tests/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | services: 4 | postgresql: 5 | build: 6 | context: ./postgres 7 | container_name: postgresql 8 | environment: 9 | POSTGRES_DB: "test_db" 10 | POSTGRES_USER: "test_user" 11 | POSTGRES_PASSWORD: "test_pass" 12 | ports: 13 | - "5432:5432" 14 | volumes: 15 | - pgdata:/var/lib/postgresql/data 16 | 17 | volumes: 18 | pgdata: 19 | -------------------------------------------------------------------------------- /.devcontainer/dagster-and-etl/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Dagster & ETL", 3 | "build": { 4 | "dockerfile": "Dockerfile", 5 | "context": "../.." 6 | }, 7 | "forwardPorts": [ 8 | 3000 9 | ], 10 | "portsAttributes": { 11 | "3000": { 12 | "label": "Dagster" 13 | } 14 | }, 15 | "features": { 16 | "ghcr.io/devcontainers/features/docker-in-docker:2": {} 17 | } 18 | } -------------------------------------------------------------------------------- /.devcontainer/dagster-testing/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Testing with Dagster", 3 | "build": { 4 | "dockerfile": "Dockerfile", 5 | "context": "../.." 6 | }, 7 | "forwardPorts": [ 8 | 3000 9 | ], 10 | "portsAttributes": { 11 | "3000": { 12 | "label": "Dagster" 13 | } 14 | }, 15 | "features": { 16 | "ghcr.io/devcontainers/features/docker-in-docker:2": {} 17 | } 18 | } -------------------------------------------------------------------------------- /course/pages/dagster-essentials/lesson-7/0-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 7: Overview' 3 | module: 'dagster_essentials' 4 | lesson: '7' 5 | --- 6 | 7 | # Overview 8 | 9 | So far, you’ve built a data pipeline and materialized it manually. However, the primary role of an orchestrator is to kick off runs without human intervention. 10 | 11 | In this lesson, you’ll learn the simplest way to automate the regular materialization of assets: using schedules. 12 | -------------------------------------------------------------------------------- /course/pages/dagster-essentials/lesson-7/5-updating-the-definitions-object.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 7: Updating the Definitions object' 3 | module: 'dagster_essentials' 4 | lesson: '7' 5 | --- 6 | 7 | ## Adding jobs and schedules to Definitions 8 | 9 | Similar to assets, schedules and jobs are loaded into the `Definitions` object with the rest of the module. And unlike resources which required mapping to specific resource keys, jobs and schedules are included automatically. -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_3/defs/assets/dbt.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | from dagster_dbt import DbtCliResource, dbt_assets 3 | 4 | from dagster_and_dbt.completed.lesson_3.defs.project import dbt_project 5 | 6 | 7 | @dbt_assets( 8 | manifest=dbt_project.manifest_path, 9 | ) 10 | def dbt_analytics(context: dg.AssetExecutionContext, dbt: DbtCliResource): 11 | yield from dbt.cli(["run"], context=context).stream() 12 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_4/defs/assets/dbt.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | from dagster_dbt import DbtCliResource, dbt_assets 3 | 4 | from dagster_and_dbt.completed.lesson_4.defs.project import dbt_project 5 | 6 | 7 | @dbt_assets( 8 | manifest=dbt_project.manifest_path, 9 | ) 10 | def dbt_analytics(context: dg.AssetExecutionContext, dbt: DbtCliResource): 11 | yield from dbt.cli(["run"], context=context).stream() 12 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/tests/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import duckdb 4 | from dagster._utils.backoff import backoff 5 | 6 | 7 | def duckdb_query(query: str): 8 | conn = backoff( 9 | fn=duckdb.connect, 10 | retry_on=(RuntimeError, duckdb.IOException), 11 | kwargs={ 12 | "database": os.getenv("DUCKDB_DATABASE"), 13 | }, 14 | max_retries=10, 15 | ) 16 | return conn.execute(query).fetchall() 17 | -------------------------------------------------------------------------------- /course/markdoc/functions.ts: -------------------------------------------------------------------------------- 1 | /* Use this file to export your Markdoc functions */ 2 | 3 | export const includes = { 4 | transform(parameters) { 5 | const [array, value] = Object.values(parameters); 6 | 7 | return Array.isArray(array) ? array.includes(value) : false; 8 | }, 9 | }; 10 | 11 | export const upper = { 12 | transform(parameters) { 13 | const string = parameters[0]; 14 | return typeof string === 'string' ? string.toUpperCase() : string; 15 | }, 16 | }; 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/defs/schedules.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_dbt.defs.jobs import trip_update_job, weekly_update_job 4 | 5 | trip_update_schedule = dg.ScheduleDefinition( 6 | job=trip_update_job, 7 | cron_schedule="0 0 5 * *", # every 5th of the month at midnight 8 | ) 9 | 10 | weekly_update_schedule = dg.ScheduleDefinition( 11 | job=weekly_update_job, 12 | cron_schedule="0 0 * * 1", # every Monday at midnight 13 | ) 14 | -------------------------------------------------------------------------------- /.github/workflows/quality-check-dagster-and-dbt.yml: -------------------------------------------------------------------------------- 1 | name: quality-check-dagster-and-dbt 2 | 3 | on: 4 | schedule: 5 | - cron: "0 0 * * 0" 6 | 7 | pull_request: 8 | types: [opened, synchronize, reopened] 9 | paths: 10 | - dagster_university/dagster_and_dbt/** 11 | 12 | jobs: 13 | check: 14 | if: github.event.pull_request.draft == false 15 | uses: ./.github/workflows/template-quality-check.yml 16 | with: 17 | working_directory: ./dagster_university/dagster_and_dbt -------------------------------------------------------------------------------- /course/pages/dagster-testing/lesson-1/1-project-preview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Lesson 1: Project preview" 3 | module: 'dbt_testing' 4 | lesson: '1' 5 | --- 6 | 7 | # Project preview 8 | 9 | In this course, we’ll focus on the craft of testing and how to maintain test coverage for a production Dagster platform. 10 | 11 | If you get stuck or want to jump ahead, check out the [finished project here on GitHub](https://github.com/dagster-io/project-dagster-university/tree/main/dagster_university/dagster_testing). -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_7/defs/assets.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | 4 | @dg.asset(deps=["orders"]) 5 | def downstream_orders(context: dg.AssetExecutionContext): 6 | pass 7 | 8 | 9 | @dg.asset(deps=["products"]) 10 | def downstream_products(context: dg.AssetExecutionContext): 11 | pass 12 | 13 | 14 | @dg.asset(deps=[downstream_orders, downstream_products]) 15 | def downstream_orders_and_products(context: dg.AssetExecutionContext): 16 | pass 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_8/defs/partitions.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_essentials.completed.lesson_8.defs.assets import constants 4 | 5 | start_date = constants.START_DATE 6 | end_date = constants.END_DATE 7 | 8 | monthly_partition = dg.MonthlyPartitionsDefinition( 9 | start_date=start_date, end_date=end_date 10 | ) 11 | 12 | weekly_partition = dg.WeeklyPartitionsDefinition( 13 | start_date=start_date, end_date=end_date 14 | ) 15 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_9/defs/partitions.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_essentials.completed.lesson_9.defs.assets import constants 4 | 5 | start_date = constants.START_DATE 6 | end_date = constants.END_DATE 7 | 8 | monthly_partition = dg.MonthlyPartitionsDefinition( 9 | start_date=start_date, end_date=end_date 10 | ) 11 | 12 | weekly_partition = dg.WeeklyPartitionsDefinition( 13 | start_date=start_date, end_date=end_date 14 | ) 15 | -------------------------------------------------------------------------------- /.github/workflows/quality-check-dagster-essentials.yml: -------------------------------------------------------------------------------- 1 | name: quality-check-dagster-essentials 2 | 3 | on: 4 | schedule: 5 | - cron: "0 0 * * 0" 6 | 7 | pull_request: 8 | types: [opened, synchronize, reopened] 9 | paths: 10 | - dagster_university/dagster_essentials/** 11 | 12 | jobs: 13 | check: 14 | if: github.event.pull_request.draft == false 15 | uses: ./.github/workflows/template-quality-check.yml 16 | with: 17 | working_directory: ./dagster_university/dagster_essentials -------------------------------------------------------------------------------- /course/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | // You might want to additionally set this in monorepos where Next.js app is in a subdir 3 | root: true, 4 | extends: ['next', 'prettier'], 5 | overrides: [ 6 | { 7 | // Adapt to your needs (e.g. some might want to only override "next.config.js") 8 | files: ['*.js'], 9 | // This is the default parser of ESLint 10 | parser: 'espree', 11 | parserOptions: { 12 | ecmaVersion: 2023, 13 | }, 14 | }, 15 | ], 16 | }; 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_6/defs/schedules.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_etl.completed.lesson_6.defs.jobs import ( 4 | orders_refresh_job, 5 | postgres_refresh_job, 6 | ) 7 | 8 | postgres_refresh_schedule = dg.ScheduleDefinition( 9 | job=postgres_refresh_job, 10 | cron_schedule="0 6 * * *", 11 | ) 12 | 13 | orders_refresh_schedule = dg.ScheduleDefinition( 14 | job=orders_refresh_job, 15 | cron_schedule="0 12,18 * * *", 16 | ) 17 | -------------------------------------------------------------------------------- /course/pages/dagster-essentials/lesson-9/6-updating-the-definitions-object.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 9: Updating the Definitions object' 3 | module: 'dagster_essentials' 4 | lesson: '9' 5 | --- 6 | 7 | # Updating the Definitions object 8 | 9 | Once again, sensors can be automatically loaded as part of the module in our `Definitions`. That means there is no changes that need to be made. 10 | 11 | Congratulations on creating a sensor! We’ll discuss how to enable the sensor in a bit, but let’s start by going through the Dagster UI. 12 | -------------------------------------------------------------------------------- /course/components/Heading.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | 3 | export function Heading({id = '', level = 1, children}) { 4 | switch (level) { 5 | case 1: 6 | return

{children}

; 7 | case 2: 8 | return

{children}

; 9 | case 3: 10 | return

{children}

; 11 | case 4: 12 | return

{children}

; 13 | case 5: 14 | return
{children}
; 15 | default: 16 | return
{children}
; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /course/pages/dagster-testing/lesson-4/0-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 4: Overview' 3 | module: 'dagster_testing' 4 | lesson: '4' 5 | --- 6 | 7 | # Overview 8 | 9 | As your data platform scales, it will interact with more and more systems. Writing tests for external systems can present a problem for testing as you now have to rely on components outside of your application. 10 | 11 | In this lesson we will discuss how to write tests when you need work with other services. Specifically how to emulate the function of those systems for the testing. -------------------------------------------------------------------------------- /course/pages/dagster-essentials/lesson-5/0-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 5: Overview' 3 | module: 'dagster_essentials' 4 | lesson: '5' 5 | --- 6 | 7 | # Overview 8 | 9 | Now that you’ve learned how to write data pipelines using Dagster’s asset-based model, let’s take a second to zoom out and discuss how Dagster works. 10 | 11 | In this lesson, you’ll learn: 12 | 13 | - What a `Definitions` object is 14 | - What a code location is 15 | - How code locations fit into Dagster’s architecture 16 | - How to view and manage code locations in the Dagster UI 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_2/defs/schedules.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_dbt.completed.lesson_2.defs.jobs import ( 4 | trip_update_job, 5 | weekly_update_job, 6 | ) 7 | 8 | trip_update_schedule = dg.ScheduleDefinition( 9 | job=trip_update_job, 10 | cron_schedule="0 0 5 * *", # every 5th of the month at midnight 11 | ) 12 | 13 | weekly_update_schedule = dg.ScheduleDefinition( 14 | job=weekly_update_job, 15 | cron_schedule="0 0 * * 1", # every Monday at midnight 16 | ) 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_3/defs/schedules.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_dbt.completed.lesson_3.defs.jobs import ( 4 | trip_update_job, 5 | weekly_update_job, 6 | ) 7 | 8 | trip_update_schedule = dg.ScheduleDefinition( 9 | job=trip_update_job, 10 | cron_schedule="0 0 5 * *", # every 5th of the month at midnight 11 | ) 12 | 13 | weekly_update_schedule = dg.ScheduleDefinition( 14 | job=weekly_update_job, 15 | cron_schedule="0 0 * * 1", # every Monday at midnight 16 | ) 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_4/defs/schedules.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_dbt.completed.lesson_4.defs.jobs import ( 4 | trip_update_job, 5 | weekly_update_job, 6 | ) 7 | 8 | trip_update_schedule = dg.ScheduleDefinition( 9 | job=trip_update_job, 10 | cron_schedule="0 0 5 * *", # every 5th of the month at midnight 11 | ) 12 | 13 | weekly_update_schedule = dg.ScheduleDefinition( 14 | job=weekly_update_job, 15 | cron_schedule="0 0 * * 1", # every Monday at midnight 16 | ) 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/defs/schedules.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_dbt.completed.lesson_5.defs.jobs import ( 4 | trip_update_job, 5 | weekly_update_job, 6 | ) 7 | 8 | trip_update_schedule = dg.ScheduleDefinition( 9 | job=trip_update_job, 10 | cron_schedule="0 0 5 * *", # every 5th of the month at midnight 11 | ) 12 | 13 | weekly_update_schedule = dg.ScheduleDefinition( 14 | job=weekly_update_job, 15 | cron_schedule="0 0 * * 1", # every Monday at midnight 16 | ) 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/defs/partitions.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_dbt.defs.assets import constants 4 | 5 | start_date = constants.START_DATE 6 | end_date = constants.END_DATE 7 | 8 | monthly_partition = dg.MonthlyPartitionsDefinition( 9 | start_date=start_date, end_date=end_date 10 | ) 11 | 12 | weekly_partition = dg.WeeklyPartitionsDefinition( 13 | start_date=start_date, end_date=end_date 14 | ) 15 | 16 | daily_partition = dg.DailyPartitionsDefinition(start_date=start_date, end_date=end_date) 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_4/defs/jobs.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | import dagster_and_etl.completed.lesson_4.defs.assets as assets 4 | 5 | asteroid_job = dg.define_asset_job( 6 | name="asteroid_job", 7 | selection=[ 8 | assets.asteroids, 9 | assets.asteroids_file, 10 | assets.duckdb_table, 11 | ], 12 | ) 13 | 14 | asteroid_partition_job = dg.define_asset_job( 15 | name="asteroid_partition_job", 16 | selection=[ 17 | assets.asteroids_partition, 18 | ], 19 | ) 20 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_6/defs/jobs.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | postgres_refresh_job = dg.define_asset_job( 4 | "postgres_refresh", 5 | selection=[ 6 | dg.AssetKey(["target", "data", "customers"]), 7 | dg.AssetKey(["target", "data", "products"]), 8 | dg.AssetKey(["target", "data", "orders"]), 9 | ], 10 | ) 11 | 12 | orders_refresh_job = dg.define_asset_job( 13 | "orders_refresh", 14 | selection=[ 15 | dg.AssetKey(["target", "data", "orders"]), 16 | ], 17 | ) 18 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_7/defs/schedules.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_essentials.completed.lesson_7.defs.jobs import ( 4 | trip_update_job, 5 | weekly_update_job, 6 | ) 7 | 8 | trip_update_schedule = dg.ScheduleDefinition( 9 | job=trip_update_job, 10 | cron_schedule="0 0 5 * *", # every 5th of the month at midnight 11 | ) 12 | 13 | weekly_update_schedule = dg.ScheduleDefinition( 14 | job=weekly_update_job, 15 | cron_schedule="0 0 * * 1", # every Monday at midnight 16 | ) 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_8/defs/schedules.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_essentials.completed.lesson_8.defs.jobs import ( 4 | trip_update_job, 5 | weekly_update_job, 6 | ) 7 | 8 | trip_update_schedule = dg.ScheduleDefinition( 9 | job=trip_update_job, 10 | cron_schedule="0 0 5 * *", # every 5th of the month at midnight 11 | ) 12 | 13 | weekly_update_schedule = dg.ScheduleDefinition( 14 | job=weekly_update_job, 15 | cron_schedule="0 0 * * 1", # every Monday at midnight 16 | ) 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_9/defs/schedules.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_essentials.completed.lesson_9.defs.jobs import ( 4 | trip_update_job, 5 | weekly_update_job, 6 | ) 7 | 8 | trip_update_schedule = dg.ScheduleDefinition( 9 | job=trip_update_job, 10 | cron_schedule="0 0 5 * *", # every 5th of the month at midnight 11 | ) 12 | 13 | weekly_update_schedule = dg.ScheduleDefinition( 14 | job=weekly_update_job, 15 | cron_schedule="0 0 * * 1", # every Monday at midnight 16 | ) 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/README.md: -------------------------------------------------------------------------------- 1 | # ETL with Dagster 2 | 3 | Learn how to ingest data to power your assets. You’ll build custom pipelines and see how to use Embedded ETL and Dagster Components to build out your data platform. 4 | 5 | ## Completed code 6 | 7 | If you are stuck you can reference the completed code for each lesson. 8 | 9 | ``` 10 | src 11 | └── dagster_and_etl 12 | └── completed 13 |       ├── lesson_2 14 |       ├── lesson_3 15 |       ├── lesson_4 16 |       ├── lesson_5 17 |       ├── lesson_6 18 |       └── lesson_7 19 | ``` 20 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_7/defs/ingest_files/defs.yaml: -------------------------------------------------------------------------------- 1 | type: dagster_sling.SlingReplicationCollectionComponent 2 | 3 | attributes: 4 | sling: 5 | connections: 6 | - name: MY_POSTGRES 7 | type: postgres 8 | host: localhost 9 | port: 5432 10 | database: test_db 11 | user: test_user 12 | password: test_pass 13 | - name: MY_DUCKDB 14 | type: duckdb 15 | instance: "{{ env('DUCKDB_DATABASE') }}" 16 | replications: 17 | - path: replication.yaml 18 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/tests/fixtures.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import duckdb 4 | import pytest 5 | from dagster._utils.backoff import backoff 6 | 7 | 8 | @pytest.fixture() 9 | def drop_tax_trips_table(): 10 | query = """ 11 | drop table if exists trips; 12 | """ 13 | 14 | conn = backoff( 15 | fn=duckdb.connect, 16 | retry_on=(RuntimeError, duckdb.IOException), 17 | kwargs={ 18 | "database": os.getenv("DUCKDB_DATABASE"), 19 | }, 20 | max_retries=10, 21 | ) 22 | conn.execute(query) 23 | -------------------------------------------------------------------------------- /course/pages/dagster-dbt/lesson-4/1-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 4: Overview' 3 | module: 'dagster_dbt' 4 | lesson: '4' 5 | --- 6 | 7 | # Overview 8 | 9 | In the last lesson, you connected the dbt project to Dagster, loaded some dbt models as assets, and even materialized them in the Dagster UI. Things were (hopefully) fairly straightforward in the last lesson, and development was smooth sailing. 10 | 11 | But what if we could improve development further? In this lesson, we’ll walk you through a better way to run `dbt parse` and how to debug failed runs in Dagster. 12 | -------------------------------------------------------------------------------- /course/pages/dagster-testing/lesson-5/0-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 5: Overview' 3 | module: 'dagster_testing' 4 | lesson: '5' 5 | --- 6 | 7 | # Overview 8 | 9 | By this point you should feel comfortable writing unit tests for your assets. Even tests that rely on external systems. 10 | 11 | However while it helpful to know how tests function with expected results, it can still be good to have tests that can connect to other systems. These tests require more set up and are as commonly invoked as unit tests but they add another critical layer to testing data applications. -------------------------------------------------------------------------------- /course/pages/installation/requirements.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Lesson 2: Requirements" 3 | lesson: '2' 4 | --- 5 | 6 | ## Set up requirements 7 | 8 | This is an interactive class where you will be coding. In order to follow along you can either run the project locally on your own machine or work with the code in Github Codespaces where all requirements will be set. 9 | 10 | ***You will only need to follow the set up for one of these options, please skip the other:*** 11 | 12 | - [Local Development](/installation/set-up-local) 13 | - [Github Codespaces](/installation/set-up-codespace) -------------------------------------------------------------------------------- /dagster_university/dagster_testing/src/dagster_testing/defs/schedules.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | import dagster_testing.defs.jobs as jobs 4 | from dagster_testing.defs.assets import lesson_6 5 | 6 | my_schedule = dg.ScheduleDefinition( 7 | name="my_schedule", 8 | job=jobs.my_job, 9 | cron_schedule="0 0 5 * *", # every 5th of the month at midnight 10 | run_config=dg.RunConfig( 11 | { 12 | "state_population_file_config": lesson_6.FilepathConfig( 13 | path="tests/data/test.csv" 14 | ) 15 | } 16 | ), 17 | ) 18 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_2/defs/partitions.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_dbt.completed.lesson_2.defs.assets import constants 4 | 5 | start_date = constants.START_DATE 6 | end_date = constants.END_DATE 7 | 8 | monthly_partition = dg.MonthlyPartitionsDefinition( 9 | start_date=start_date, end_date=end_date 10 | ) 11 | 12 | weekly_partition = dg.WeeklyPartitionsDefinition( 13 | start_date=start_date, end_date=end_date 14 | ) 15 | 16 | daily_partition = dg.DailyPartitionsDefinition(start_date=start_date, end_date=end_date) 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_3/defs/partitions.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_dbt.completed.lesson_3.defs.assets import constants 4 | 5 | start_date = constants.START_DATE 6 | end_date = constants.END_DATE 7 | 8 | monthly_partition = dg.MonthlyPartitionsDefinition( 9 | start_date=start_date, end_date=end_date 10 | ) 11 | 12 | weekly_partition = dg.WeeklyPartitionsDefinition( 13 | start_date=start_date, end_date=end_date 14 | ) 15 | 16 | daily_partition = dg.DailyPartitionsDefinition(start_date=start_date, end_date=end_date) 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_4/defs/partitions.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_dbt.completed.lesson_4.defs.assets import constants 4 | 5 | start_date = constants.START_DATE 6 | end_date = constants.END_DATE 7 | 8 | monthly_partition = dg.MonthlyPartitionsDefinition( 9 | start_date=start_date, end_date=end_date 10 | ) 11 | 12 | weekly_partition = dg.WeeklyPartitionsDefinition( 13 | start_date=start_date, end_date=end_date 14 | ) 15 | 16 | daily_partition = dg.DailyPartitionsDefinition(start_date=start_date, end_date=end_date) 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/defs/partitions.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_dbt.completed.lesson_5.defs.assets import constants 4 | 5 | start_date = constants.START_DATE 6 | end_date = constants.END_DATE 7 | 8 | monthly_partition = dg.MonthlyPartitionsDefinition( 9 | start_date=start_date, end_date=end_date 10 | ) 11 | 12 | weekly_partition = dg.WeeklyPartitionsDefinition( 13 | start_date=start_date, end_date=end_date 14 | ) 15 | 16 | daily_partition = dg.DailyPartitionsDefinition(start_date=start_date, end_date=end_date) 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/defs/partitions.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_dbt.completed.lesson_6.defs.assets import constants 4 | 5 | start_date = constants.START_DATE 6 | end_date = constants.END_DATE 7 | 8 | monthly_partition = dg.MonthlyPartitionsDefinition( 9 | start_date=start_date, end_date=end_date 10 | ) 11 | 12 | weekly_partition = dg.WeeklyPartitionsDefinition( 13 | start_date=start_date, end_date=end_date 14 | ) 15 | 16 | daily_partition = dg.DailyPartitionsDefinition(start_date=start_date, end_date=end_date) 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/defs/partitions.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_dbt.completed.lesson_7.defs.assets import constants 4 | 5 | start_date = constants.START_DATE 6 | end_date = constants.END_DATE 7 | 8 | monthly_partition = dg.MonthlyPartitionsDefinition( 9 | start_date=start_date, end_date=end_date 10 | ) 11 | 12 | weekly_partition = dg.WeeklyPartitionsDefinition( 13 | start_date=start_date, end_date=end_date 14 | ) 15 | 16 | daily_partition = dg.DailyPartitionsDefinition(start_date=start_date, end_date=end_date) 17 | -------------------------------------------------------------------------------- /course/markdoc/nodes/fence.markdoc.ts: -------------------------------------------------------------------------------- 1 | import {Tag, nodes} from '@markdoc/markdoc'; 2 | import {CodeBlock} from '../../components'; 3 | 4 | export const fence = { 5 | render: CodeBlock, 6 | attributes: { 7 | obfuscated: { 8 | type: Boolean, 9 | }, 10 | ...nodes.fence.attributes, 11 | }, 12 | transform(node, config) { 13 | const attributes = node.transformAttributes(config); 14 | const {language} = node.children[0].attributes; 15 | const content = node.attributes.content; 16 | return new Tag(this.render, {...attributes, language}, [content]); 17 | }, 18 | }; 19 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/README.md: -------------------------------------------------------------------------------- 1 | # Dagster and dbt 2 | 3 | ## Overview 4 | 5 | Learn how to integrate and orchestrate dbt projects with Dagster. You'll load dbt models into Dagster as assets, build dependencies, and ready your project for production deployment. 6 | 7 | ## Completed code 8 | 9 | If you are stuck you can reference the completed code for each lesson. 10 | 11 | ``` 12 | src 13 | └── dagster_and_dbt 14 | └── completed 15 |       ├── lesson_2 16 |       ├── lesson_3 17 |       ├── lesson_4 18 |       ├── lesson_5 19 |       ├── lesson_6 20 |       └── lesson_7 21 | ``` 22 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/defs/schedules.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | # from dagster_and_dbt.completed.lesson_6.defs.jobs import trip_update_job, weekly_update_job 4 | from dagster_and_dbt.completed.lesson_6.defs.jobs import weekly_update_job 5 | 6 | # trip_update_schedule = dg.ScheduleDefinition( 7 | # job=trip_update_job, 8 | # cron_schedule="0 0 5 * *", # every 5th of the month at midnight 9 | # ) 10 | 11 | weekly_update_schedule = dg.ScheduleDefinition( 12 | job=weekly_update_job, 13 | cron_schedule="0 0 * * 1", # every Monday at midnight 14 | ) 15 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/defs/schedules.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | # from dagster_and_dbt.completed.lesson_7.defs.jobs import trip_update_job, weekly_update_job 4 | from dagster_and_dbt.completed.lesson_7.defs.jobs import weekly_update_job 5 | 6 | # trip_update_schedule = dg.ScheduleDefinition( 7 | # job=trip_update_job, 8 | # cron_schedule="0 0 5 * *", # every 5th of the month at midnight 9 | # ) 10 | 11 | weekly_update_schedule = dg.ScheduleDefinition( 12 | job=weekly_update_job, 13 | cron_schedule="0 0 * * 1", # every Monday at midnight 14 | ) 15 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/tests/test_lesson_3.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | import pytest 3 | 4 | import dagster_essentials.completed.lesson_3.defs 5 | 6 | 7 | @pytest.fixture() 8 | def defs(): 9 | return dg.components.load_defs(dagster_essentials.completed.lesson_3.defs) 10 | 11 | 12 | def test_assets(defs): 13 | assets = [ 14 | defs.get_assets_def(dg.AssetKey(["taxi_trips_file"])), 15 | defs.get_assets_def(dg.AssetKey(["taxi_zones_file"])), 16 | ] 17 | result = dg.materialize(assets) 18 | assert result.success 19 | 20 | 21 | def test_defs(defs): 22 | assert defs 23 | -------------------------------------------------------------------------------- /course/pages/dagster-dbt/lesson-5/1-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 5: Overview' 3 | module: 'dagster_dbt' 4 | lesson: '5' 5 | --- 6 | 7 | # Overview 8 | 9 | In Lesson 3, you loaded your dbt project's models as assets into Dagster. You also materialized some of those models. 10 | 11 | In this lesson, we’ll integrate more dbt models with the rest of your Dagster project. You’ll use the existing Dagster assets as sources in your dbt project and learn how to further customize how Dagster maps your dbt project with the `DagsterDbtTranslator` class. To wrap things up, we’ll show you how to automate the running of the dbt models in Dagster. -------------------------------------------------------------------------------- /course/markdoc/nodes/link.markdoc.ts: -------------------------------------------------------------------------------- 1 | import {Tag, nodes} from '@markdoc/markdoc'; 2 | import Link from 'next/link'; 3 | 4 | export const link = { 5 | render: Link, 6 | attributes: nodes.link.attributes, 7 | transform(node, config) { 8 | const attributes = node.transformAttributes(config); 9 | const children = node.transformChildren(config); 10 | const {href} = attributes; 11 | 12 | // If the link is to another site, open in a new window. 13 | return new Tag( 14 | this.render, 15 | {...attributes, target: href.startsWith('http') ? '_blank' : undefined}, 16 | children, 17 | ); 18 | }, 19 | }; 20 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_3/defs/jobs.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | import dagster_and_etl.completed.lesson_3.defs.assets as assets 4 | 5 | import_partition_job = dg.define_asset_job( 6 | name="import_partition_job", 7 | selection=[ 8 | assets.import_partition_file, 9 | assets.duckdb_partition_table, 10 | ], 11 | ) 12 | 13 | import_dynamic_partition_job = dg.define_asset_job( 14 | name="import_dynamic_partition_job", 15 | selection=[ 16 | assets.import_dynamic_partition_file, 17 | assets.duckdb_dynamic_partition_table, 18 | ], 19 | ) 20 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/defs/assets/constants.py: -------------------------------------------------------------------------------- 1 | TAXI_ZONES_FILE_PATH = "data/raw/taxi_zones.csv" 2 | TAXI_TRIPS_TEMPLATE_FILE_PATH = "data/raw/taxi_trips_{}.parquet" 3 | 4 | TRIPS_BY_AIRPORT_FILE_PATH = "data/outputs/trips_by_airport.csv" 5 | TRIPS_BY_WEEK_FILE_PATH = "data/outputs/trips_by_week.csv" 6 | MANHATTAN_STATS_FILE_PATH = "data/staging/manhattan_stats.geojson" 7 | MANHATTAN_MAP_FILE_PATH = "data/outputs/manhattan_map.png" 8 | 9 | REQUEST_DESTINATION_TEMPLATE_FILE_PATH = "data/outputs/{}.png" 10 | 11 | DATE_FORMAT = "%Y-%m-%d" 12 | 13 | START_DATE = "2023-01-01" 14 | END_DATE = "2023-04-01" 15 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_8/defs/jobs.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_essentials.completed.lesson_8.defs.partitions import ( 4 | monthly_partition, 5 | weekly_partition, 6 | ) 7 | 8 | trips_by_week = dg.AssetSelection.assets("trips_by_week") 9 | 10 | trip_update_job = dg.define_asset_job( 11 | name="trip_update_job", 12 | partitions_def=monthly_partition, 13 | selection=dg.AssetSelection.all() - trips_by_week, 14 | ) 15 | 16 | weekly_update_job = dg.define_asset_job( 17 | name="weekly_update_job", partitions_def=weekly_partition, selection=trips_by_week 18 | ) 19 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_5/dlt_quick_start.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import dlt 4 | 5 | data = [ 6 | {"id": 1, "name": "Alice"}, 7 | {"id": 2, "name": "Bob"}, 8 | ] 9 | 10 | 11 | @dlt.source 12 | def simple_source(): 13 | @dlt.resource 14 | def load_dict(): 15 | yield data 16 | 17 | return load_dict 18 | 19 | 20 | pipeline = dlt.pipeline( 21 | pipeline_name="simple_pipeline", 22 | destination=dlt.destinations.duckdb(os.getenv("DUCKDB_DATABASE")), 23 | dataset_name="mydata", 24 | ) 25 | 26 | if __name__ == "__main__": 27 | load_info = pipeline.run(simple_source()) 28 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_4/defs/assets/constants.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | TAXI_ZONES_FILE_PATH = os.path.join("data", "raw", "taxi_zones.csv") 4 | TAXI_TRIPS_TEMPLATE_FILE_PATH = os.path.join("data", "raw", "taxi_trips_{}.parquet") 5 | 6 | TRIPS_BY_AIRPORT_FILE_PATH = os.path.join("data", "outputs", "trips_by_airport.csv") 7 | TRIPS_BY_WEEK_FILE_PATH = os.path.join("data", "outputs", "trips_by_week.csv") 8 | MANHATTAN_STATS_FILE_PATH = os.path.join("data", "staging", "manhattan_stats.geojson") 9 | MANHATTAN_MAP_FILE_PATH = os.path.join("data", "outputs", "manhattan_map.png") 10 | 11 | DATE_FORMAT = "%Y-%m-%d" 12 | -------------------------------------------------------------------------------- /course/pages/dagster-essentials/lesson-9/0-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 9: Overview' 3 | module: 'dagster_essentials' 4 | lesson: '9' 5 | --- 6 | 7 | # Overview 8 | 9 | In Lesson 7, you wrote a schedule to run your pipelines every month. Using schedules is only one of the various ways to trigger asset materializations in Dagster. 10 | 11 | Event-driven runs are common in real life. When running our bakery, we may get deliveries of ingredients and need to move them into the kitchen. Just like in baking cookies, data pipelines may also need to be responsive and reactive to events. In Dagster, you can use **sensors** to automate your pipelines for these situations. 12 | -------------------------------------------------------------------------------- /course/pages/dagster-etl/lesson-4/0-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Lesson 4: Overview" 3 | module: 'dagster_etl' 4 | lesson: '4' 5 | --- 6 | 7 | # Overview 8 | 9 | Now that we’ve covered CSV-based ingestion, let’s move on to another common pattern: loading data from an API. We’ll continue using the same destination (DuckDB) but introduce some new Dagster functionality to fetch data from an external API, apply light processing, and load the results into the database. This pattern is useful when working with third-party services, public datasets, or internal APIs that expose structured data for integration. 10 | 11 | ![API ETL](/images/dagster-etl/lesson-4/api-etl.png) 12 | -------------------------------------------------------------------------------- /.github/workflows/quality-check-dagster-and-etl.yml: -------------------------------------------------------------------------------- 1 | name: quality-check-dagster-and-etl 2 | 3 | on: 4 | schedule: 5 | - cron: "0 0 * * 0" 6 | 7 | pull_request: 8 | types: [opened, synchronize, reopened] 9 | paths: 10 | - dagster_university/dagster_and_etl/** 11 | 12 | jobs: 13 | check: 14 | if: github.event.pull_request.draft == false 15 | uses: ./.github/workflows/template-quality-check.yml 16 | with: 17 | working_directory: ./dagster_university/dagster_and_etl 18 | # TODO: Disable integration tests from GHA 19 | # postgres image has no windows/amd64 20 | windows_pytest_cmd: uv run pytest tests -v -m "not integration" -------------------------------------------------------------------------------- /course/pages/dagster-testing/lesson-6/0-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 6: Overview' 3 | module: 'dagster_testing' 4 | lesson: '6' 5 | --- 6 | 7 | # Overview 8 | 9 | So far everything we have learned applies to general best practices in Python testing. We have learned how to apply unit testing, mocks and integration testing in the context of Dagster but these apply universally to almost any code you could write. 10 | 11 | In this final lesson we will discuss some Dagster specific elements of testing. This will cover Dagster features to help you run your platform in production and some tips on tests you should add to your project to make sure you can always build with confidence. -------------------------------------------------------------------------------- /.github/workflows/quality-check-dagster-testing.yml: -------------------------------------------------------------------------------- 1 | name: quality-check-dagster-testing 2 | 3 | on: 4 | schedule: 5 | - cron: "0 0 * * 0" 6 | 7 | pull_request: 8 | types: [opened, synchronize, reopened] 9 | paths: 10 | - dagster_university/dagster_testing/** 11 | 12 | jobs: 13 | check: 14 | if: github.event.pull_request.draft == false 15 | uses: ./.github/workflows/template-quality-check.yml 16 | with: 17 | working_directory: ./dagster_university/dagster_testing 18 | # TODO: Disable integration tests from GHA 19 | # postgres image has no windows/amd64 20 | windows_pytest_cmd: uv run pytest tests/completed -v -m "not integration" -------------------------------------------------------------------------------- /course/pages/dagster-essentials/lesson-3/7-recap.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 3: Recap' 3 | module: 'dagster_essentials' 4 | lesson: '3' 5 | --- 6 | 7 | ## Recap 8 | 9 | In this lesson, you learned: 10 | 11 | - How to use `dg` to help build a Dagster project 12 | - What an asset is 13 | - What makes up an asset 14 | - How to define and materialize an asset 15 | - How to find run info and troubleshoot issues 16 | 17 | You also wrote your first assets - `taxi_trips_file` and `taxi_zones_file` - which you’ll use in the next lesson to create more assets and have them depend on each other. 18 | 19 | > ‼️ Make sure you update your code to match the answer before continuing to Lesson 4! 20 | -------------------------------------------------------------------------------- /course/pages/dagster-dbt/lesson-3/1-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 3: Overview' 3 | module: 'dagster_dbt' 4 | lesson: '3' 5 | --- 6 | 7 | # Overview 8 | 9 | As you learned in Lesson 1, Dagster and dbt have similar mental models because both frameworks excel at building data models, defining relationships between them, and materializing them. You should have also grasped that dbt models are conceptually data assets and that these data assets can be represented in Dagster. 10 | 11 | In this lesson, you’ll learn how to turn that conceptual understanding into real life by connecting a dbt project to Dagster, manually running your dbt models, and understanding what happens when Dagster runs dbt. -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/analytics/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | 7 | - dbt run 8 | - dbt test 9 | 10 | ### Resources: 11 | 12 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 13 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 14 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 15 | - Find [dbt events](https://events.getdbt.com) near you 16 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/data/source/2018-01-22.csv: -------------------------------------------------------------------------------- 1 | "date","share_price","amount","spend","shift","spread" 2 | "2018/09/28","264.7700","33597290.0000","270.2600","278.0000","260.5550" 3 | "2018/09/27","307.5200","7337760.0000","312.9000","314.9600","306.9100" 4 | "2018/09/26","309.5800","7835863.0000","301.9100","313.8900","301.1093" 5 | "2018/09/25","300.9900","4472287.0000","300.0000","304.6000","296.5000" 6 | "2018/09/24","299.6800","4834384.0000","298.4800","302.9993","293.5800" 7 | "2018/09/21","299.1000","5038497.0000","297.7000","300.5800","295.3700" 8 | "2018/09/20","298.3300","7332477.0000","303.5600","305.9800","293.3300" 9 | "2018/09/19","299.0200","8264353.0000","280.5100","300.0000","280.5000" -------------------------------------------------------------------------------- /course/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": ["dom", "dom.iterable", "esnext"], 5 | "allowJs": true, 6 | "skipLibCheck": true, 7 | "strict": false, 8 | "forceConsistentCasingInFileNames": true, 9 | "noEmit": true, 10 | "esModuleInterop": true, 11 | "module": "esnext", 12 | "moduleResolution": "node", 13 | "resolveJsonModule": true, 14 | "allowUnreachableCode": true, 15 | "isolatedModules": true, 16 | "jsx": "preserve", 17 | "baseUrl": ".", 18 | "downlevelIteration": true, 19 | "incremental": true 20 | }, 21 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], 22 | "exclude": ["node_modules"] 23 | } 24 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_3/analytics/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | 7 | - dbt run 8 | - dbt test 9 | 10 | ### Resources: 11 | 12 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 13 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 14 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 15 | - Find [dbt events](https://events.getdbt.com) near you 16 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_4/analytics/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | 7 | - dbt run 8 | - dbt test 9 | 10 | ### Resources: 11 | 12 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 13 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 14 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 15 | - Find [dbt events](https://events.getdbt.com) near you 16 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/analytics/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | 7 | - dbt run 8 | - dbt test 9 | 10 | ### Resources: 11 | 12 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 13 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 14 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 15 | - Find [dbt events](https://events.getdbt.com) near you 16 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/analytics/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | 7 | - dbt run 8 | - dbt test 9 | 10 | ### Resources: 11 | 12 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 13 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 14 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 15 | - Find [dbt events](https://events.getdbt.com) near you 16 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/analytics/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | 7 | - dbt run 8 | - dbt test 9 | 10 | ### Resources: 11 | 12 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 13 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 14 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 15 | - Find [dbt events](https://events.getdbt.com) near you 16 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_testing/src/dagster_testing/defs/resources.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | 4 | class StatePopulation(dg.ConfigurableResource): 5 | def get_cities(self, state: str) -> list[dict]: 6 | return [ 7 | { 8 | "City": "Milwaukee", 9 | "Population": 577222, 10 | }, 11 | { 12 | "City": "Madison", 13 | "Population": 269840, 14 | }, 15 | ] 16 | 17 | 18 | @dg.definitions 19 | def resources(): 20 | return dg.Definitions( 21 | resources={ 22 | "state_population_resource": StatePopulation(), 23 | "database": dg.ResourceDefinition.mock_resource(), 24 | }, 25 | ) 26 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/defs/jobs.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_dbt.defs.partitions import monthly_partition, weekly_partition 4 | 5 | trips_by_week = dg.AssetSelection.assets("trips_by_week") 6 | adhoc_request = dg.AssetSelection.assets("adhoc_request") 7 | 8 | trip_update_job = dg.define_asset_job( 9 | name="trip_update_job", 10 | partitions_def=monthly_partition, 11 | selection=dg.AssetSelection.all() - trips_by_week - adhoc_request, 12 | ) 13 | 14 | weekly_update_job = dg.define_asset_job( 15 | name="weekly_update_job", partitions_def=weekly_partition, selection=trips_by_week 16 | ) 17 | 18 | adhoc_request_job = dg.define_asset_job( 19 | name="adhoc_request_job", selection=adhoc_request 20 | ) 21 | -------------------------------------------------------------------------------- /dagster_university/dagster_testing/src/dagster_testing/defs/assets/lesson_5.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | 4 | @dg.asset 5 | def state_population_database(database: dg.ConfigurableResource) -> list[tuple]: 6 | query = """ 7 | SELECT 8 | city_name, 9 | population 10 | FROM data.city_population 11 | WHERE state_name = 'NY'; 12 | """ 13 | with database.get_connection() as conn: 14 | cur = conn.cursor() 15 | cur.execute(query) 16 | return cur.fetchall() 17 | 18 | 19 | class StateConfig(dg.Config): 20 | name: str 21 | 22 | 23 | @dg.asset 24 | def total_population_database(state_population_database: list[tuple]) -> int: 25 | return sum(value for _, value in state_population_database) 26 | -------------------------------------------------------------------------------- /dagster_university/dagster_testing/src/dagster_testing/defs/sensors.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | import dagster as dg 4 | 5 | import dagster_testing.defs.jobs as jobs 6 | 7 | 8 | def check_for_new_files() -> list[str]: 9 | if random.random() > 0.5: 10 | return ["file1", "file2"] 11 | return [] 12 | 13 | 14 | @dg.sensor( 15 | name="my_sensor", 16 | job=jobs.my_job_configured, 17 | minimum_interval_seconds=5, 18 | ) 19 | def my_sensor(): 20 | new_files = check_for_new_files() 21 | # New files, run `my_job` 22 | if new_files: 23 | for filename in new_files: 24 | yield dg.RunRequest(run_key=filename) 25 | # No new files, skip the run and log the reason 26 | else: 27 | yield dg.SkipReason("No new files found") 28 | -------------------------------------------------------------------------------- /dagster_university/dagster_testing/src/dagster_testing/defs/jobs.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import dagster as dg 4 | import yaml 5 | 6 | from dagster_testing.defs.assets import lesson_6 7 | 8 | my_job = dg.define_asset_job( 9 | name="jobs", 10 | selection=[ 11 | lesson_6.population_file_config, 12 | lesson_6.population_api_resource, 13 | lesson_6.population_combined, 14 | ], 15 | ) 16 | 17 | 18 | my_job_configured = dg.define_asset_job( 19 | name="jobs_config", 20 | selection=[ 21 | lesson_6.population_file_config, 22 | lesson_6.population_api_resource, 23 | lesson_6.population_combined, 24 | ], 25 | config=yaml.safe_load( 26 | (Path(__file__).absolute().parent / "configs/run_config.yaml").open() 27 | ), 28 | ) 29 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_5/defs/assets/constants.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | TAXI_ZONES_FILE_PATH = os.path.join("data", "raw", "taxi_zones.csv") 4 | TAXI_TRIPS_TEMPLATE_FILE_PATH = os.path.join("data", "raw", "taxi_trips_{}.parquet") 5 | 6 | TRIPS_BY_AIRPORT_FILE_PATH = os.path.join("data", "outputs", "trips_by_airport.csv") 7 | TRIPS_BY_WEEK_FILE_PATH = os.path.join("data", "outputs", "trips_by_week.csv") 8 | MANHATTAN_STATS_FILE_PATH = os.path.join("data", "staging", "manhattan_stats.geojson") 9 | MANHATTAN_MAP_FILE_PATH = os.path.join("data", "outputs", "manhattan_map.png") 10 | 11 | REQUEST_DESTINATION_TEMPLATE_FILE_PATH = os.path.join("data", "outputs", "{}.png") 12 | 13 | DATE_FORMAT = "%Y-%m-%d" 14 | 15 | START_DATE = "2023-01-01" 16 | END_DATE = "2023-04-01" 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_6/defs/assets/constants.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | TAXI_ZONES_FILE_PATH = os.path.join("data", "raw", "taxi_zones.csv") 4 | TAXI_TRIPS_TEMPLATE_FILE_PATH = os.path.join("data", "raw", "taxi_trips_{}.parquet") 5 | 6 | TRIPS_BY_AIRPORT_FILE_PATH = os.path.join("data", "outputs", "trips_by_airport.csv") 7 | TRIPS_BY_WEEK_FILE_PATH = os.path.join("data", "outputs", "trips_by_week.csv") 8 | MANHATTAN_STATS_FILE_PATH = os.path.join("data", "staging", "manhattan_stats.geojson") 9 | MANHATTAN_MAP_FILE_PATH = os.path.join("data", "outputs", "manhattan_map.png") 10 | 11 | REQUEST_DESTINATION_TEMPLATE_FILE_PATH = os.path.join("data", "outputs", "{}.png") 12 | 13 | DATE_FORMAT = "%Y-%m-%d" 14 | 15 | START_DATE = "2023-01-01" 16 | END_DATE = "2023-04-01" 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_7/defs/assets/constants.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | TAXI_ZONES_FILE_PATH = os.path.join("data", "raw", "taxi_zones.csv") 4 | TAXI_TRIPS_TEMPLATE_FILE_PATH = os.path.join("data", "raw", "taxi_trips_{}.parquet") 5 | 6 | TRIPS_BY_AIRPORT_FILE_PATH = os.path.join("data", "outputs", "trips_by_airport.csv") 7 | TRIPS_BY_WEEK_FILE_PATH = os.path.join("data", "outputs", "trips_by_week.csv") 8 | MANHATTAN_STATS_FILE_PATH = os.path.join("data", "staging", "manhattan_stats.geojson") 9 | MANHATTAN_MAP_FILE_PATH = os.path.join("data", "outputs", "manhattan_map.png") 10 | 11 | REQUEST_DESTINATION_TEMPLATE_FILE_PATH = os.path.join("data", "outputs", "{}.png") 12 | 13 | DATE_FORMAT = "%Y-%m-%d" 14 | 15 | START_DATE = "2023-01-01" 16 | END_DATE = "2023-04-01" 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_8/defs/assets/constants.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | TAXI_ZONES_FILE_PATH = os.path.join("data", "raw", "taxi_zones.csv") 4 | TAXI_TRIPS_TEMPLATE_FILE_PATH = os.path.join("data", "raw", "taxi_trips_{}.parquet") 5 | 6 | TRIPS_BY_AIRPORT_FILE_PATH = os.path.join("data", "outputs", "trips_by_airport.csv") 7 | TRIPS_BY_WEEK_FILE_PATH = os.path.join("data", "outputs", "trips_by_week.csv") 8 | MANHATTAN_STATS_FILE_PATH = os.path.join("data", "staging", "manhattan_stats.geojson") 9 | MANHATTAN_MAP_FILE_PATH = os.path.join("data", "outputs", "manhattan_map.png") 10 | 11 | REQUEST_DESTINATION_TEMPLATE_FILE_PATH = os.path.join("data", "outputs", "{}.png") 12 | 13 | DATE_FORMAT = "%Y-%m-%d" 14 | 15 | START_DATE = "2023-01-01" 16 | END_DATE = "2023-04-01" 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_9/defs/assets/constants.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | TAXI_ZONES_FILE_PATH = os.path.join("data", "raw", "taxi_zones.csv") 4 | TAXI_TRIPS_TEMPLATE_FILE_PATH = os.path.join("data", "raw", "taxi_trips_{}.parquet") 5 | 6 | TRIPS_BY_AIRPORT_FILE_PATH = os.path.join("data", "outputs", "trips_by_airport.csv") 7 | TRIPS_BY_WEEK_FILE_PATH = os.path.join("data", "outputs", "trips_by_week.csv") 8 | MANHATTAN_STATS_FILE_PATH = os.path.join("data", "staging", "manhattan_stats.geojson") 9 | MANHATTAN_MAP_FILE_PATH = os.path.join("data", "outputs", "manhattan_map.png") 10 | 11 | REQUEST_DESTINATION_TEMPLATE_FILE_PATH = os.path.join("data", "outputs", "{}.png") 12 | 13 | DATE_FORMAT = "%Y-%m-%d" 14 | 15 | START_DATE = "2023-01-01" 16 | END_DATE = "2023-04-01" 17 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_2/defs/jobs.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_dbt.completed.lesson_2.defs.partitions import ( 4 | monthly_partition, 5 | weekly_partition, 6 | ) 7 | 8 | trips_by_week = dg.AssetSelection.assets("trips_by_week") 9 | adhoc_request = dg.AssetSelection.assets("adhoc_request") 10 | 11 | trip_update_job = dg.define_asset_job( 12 | name="trip_update_job", 13 | partitions_def=monthly_partition, 14 | selection=dg.AssetSelection.all() - trips_by_week - adhoc_request, 15 | ) 16 | 17 | weekly_update_job = dg.define_asset_job( 18 | name="weekly_update_job", partitions_def=weekly_partition, selection=trips_by_week 19 | ) 20 | 21 | adhoc_request_job = dg.define_asset_job( 22 | name="adhoc_request_job", selection=adhoc_request 23 | ) 24 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_3/defs/jobs.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_dbt.completed.lesson_3.defs.partitions import ( 4 | monthly_partition, 5 | weekly_partition, 6 | ) 7 | 8 | trips_by_week = dg.AssetSelection.assets("trips_by_week") 9 | adhoc_request = dg.AssetSelection.assets("adhoc_request") 10 | 11 | trip_update_job = dg.define_asset_job( 12 | name="trip_update_job", 13 | partitions_def=monthly_partition, 14 | selection=dg.AssetSelection.all() - trips_by_week - adhoc_request, 15 | ) 16 | 17 | weekly_update_job = dg.define_asset_job( 18 | name="weekly_update_job", partitions_def=weekly_partition, selection=trips_by_week 19 | ) 20 | 21 | adhoc_request_job = dg.define_asset_job( 22 | name="adhoc_request_job", selection=adhoc_request 23 | ) 24 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_4/defs/jobs.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_dbt.completed.lesson_4.defs.partitions import ( 4 | monthly_partition, 5 | weekly_partition, 6 | ) 7 | 8 | trips_by_week = dg.AssetSelection.assets("trips_by_week") 9 | adhoc_request = dg.AssetSelection.assets("adhoc_request") 10 | 11 | trip_update_job = dg.define_asset_job( 12 | name="trip_update_job", 13 | partitions_def=monthly_partition, 14 | selection=dg.AssetSelection.all() - trips_by_week - adhoc_request, 15 | ) 16 | 17 | weekly_update_job = dg.define_asset_job( 18 | name="weekly_update_job", partitions_def=weekly_partition, selection=trips_by_week 19 | ) 20 | 21 | adhoc_request_job = dg.define_asset_job( 22 | name="adhoc_request_job", selection=adhoc_request 23 | ) 24 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/defs/resources.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import boto3 4 | import dagster as dg 5 | from dagster_duckdb import DuckDBResource 6 | 7 | database_resource = DuckDBResource( 8 | database=dg.EnvVar("DUCKDB_DATABASE"), 9 | ) 10 | 11 | if os.getenv("DAGSTER_ENVIRONMENT") == "prod": 12 | session = boto3.Session( 13 | aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), 14 | aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), 15 | region_name=os.getenv("AWS_REGION"), 16 | ) 17 | smart_open_config = {"client": session.client("s3")} 18 | else: 19 | smart_open_config = {} 20 | 21 | 22 | @dg.definitions 23 | def resources(): 24 | return dg.Definitions( 25 | resources={ 26 | "database": database_resource, 27 | }, 28 | ) 29 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/src/dagster_essentials/completed/lesson_9/defs/jobs.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_essentials.completed.lesson_9.defs.partitions import ( 4 | monthly_partition, 5 | weekly_partition, 6 | ) 7 | 8 | trips_by_week = dg.AssetSelection.assets("trips_by_week") 9 | adhoc_request = dg.AssetSelection.assets("adhoc_request") 10 | 11 | trip_update_job = dg.define_asset_job( 12 | name="trip_update_job", 13 | partitions_def=monthly_partition, 14 | selection=dg.AssetSelection.all() - trips_by_week - adhoc_request, 15 | ) 16 | 17 | weekly_update_job = dg.define_asset_job( 18 | name="weekly_update_job", partitions_def=weekly_partition, selection=trips_by_week 19 | ) 20 | 21 | adhoc_request_job = dg.define_asset_job( 22 | name="adhoc_request_job", selection=adhoc_request 23 | ) 24 | -------------------------------------------------------------------------------- /course/pages/dagster-dbt/lesson-3/knowledge-check.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Knowledge check' 3 | module: 'dagster_essentials' 4 | lesson: '3' 5 | --- 6 | 7 | # Knowledge check 8 | 9 | 1. Open the `defs/assets/dbt.py` file. 10 | 11 | 2. Modify `dbt_analytics` to run `dbt build` instead of `dbt run`. The function should look like this afterward: 12 | 13 | ```python 14 | # src/dagster_and_dbt/defs/assets/dbt.py 15 | @dbt_assets( 16 | manifest=dbt_project.manifest_path 17 | ) 18 | def dbt_analytics(context: dg.AssetExecutionContext, dbt: DbtCliResource): 19 | yield from dbt.cli(["build"], context=context).stream() 20 | ``` 21 | 22 | 3. In the Dagster UI, re-materialize both of the dbt models. 23 | 24 | 4. Navigate to the details page for the run you just started, then look at the logs. 25 | 26 | When finished, proceed to the next page. -------------------------------------------------------------------------------- /course/pages/dagster-essentials/extra-credit/0-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Extra credit: Overview' 3 | module: 'dagster_essentials' 4 | lesson: 'extra-credit' 5 | --- 6 | 7 | # Overview 8 | 9 | Welcome to extra credit, ambitious student! 10 | 11 | By this point, you have two working data pipelines: one you built progressively throughout this course’s lessons, and the other during the course capstone. Building a data pipeline from end-to-end is a useful skill, but there’s still another skill you can learn to level up. Using **metadata**, or data that provides information about other data, you can document your pipeline and make it easy for others to understand. 12 | 13 | In this lesson, we’ll cover: 14 | 15 | - What’s metadata and the benefits of using it 16 | - Definition metadata 17 | - Materialization metadata 18 | - How to locate metadata in the UI 19 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_2/defs/resources.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import boto3 4 | import dagster as dg 5 | from dagster_duckdb import DuckDBResource 6 | 7 | database_resource = DuckDBResource( 8 | database=dg.EnvVar("DUCKDB_DATABASE"), 9 | ) 10 | 11 | if os.getenv("DAGSTER_ENVIRONMENT") == "prod": 12 | session = boto3.Session( 13 | aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), 14 | aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), 15 | region_name=os.getenv("AWS_REGION"), 16 | ) 17 | smart_open_config = {"client": session.client("s3")} 18 | else: 19 | smart_open_config = {} 20 | 21 | 22 | @dg.definitions 23 | def resources(): 24 | return dg.Definitions( 25 | resources={ 26 | "database": database_resource, 27 | }, 28 | ) 29 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_4/defs/schedules.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | 3 | from dagster_and_etl.completed.lesson_4.defs.jobs import ( 4 | asteroid_job, 5 | asteroid_partition_job, 6 | ) 7 | 8 | 9 | @dg.schedule(job=asteroid_job, cron_schedule="0 6 * * *") 10 | def date_range_schedule(context): 11 | scheduled_date = context.scheduled_execution_time.strftime("%Y-%m-%d") 12 | 13 | return dg.RunRequest( 14 | run_config={ 15 | "ops": { 16 | "asteroids": { 17 | "config": { 18 | "date": scheduled_date, 19 | }, 20 | }, 21 | }, 22 | }, 23 | ) 24 | 25 | 26 | asset_partitioned_schedule = dg.build_schedule_from_partitioned_job( 27 | asteroid_partition_job, 28 | ) 29 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_6/defs/resources.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | from dagster_sling import SlingConnectionResource, SlingResource 3 | 4 | source = SlingConnectionResource( 5 | name="MY_POSTGRES", 6 | type="postgres", 7 | host="localhost", 8 | port=5432, 9 | database="test_db", 10 | user="test_user", 11 | password="test_pass", 12 | ) 13 | 14 | 15 | destination = SlingConnectionResource( 16 | name="MY_DUCKDB", 17 | type="duckdb", 18 | connection_string="duckdb:///var/tmp/duckdb.db", 19 | ) 20 | 21 | sling = SlingResource( 22 | connections=[ 23 | source, 24 | destination, 25 | ] 26 | ) 27 | 28 | 29 | @dg.definitions 30 | def resources(): 31 | return dg.Definitions( 32 | resources={ 33 | "sling": sling, 34 | }, 35 | ) 36 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/src/dagster_and_etl/completed/lesson_6/defs/assets.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | from dagster_sling import SlingResource, sling_assets 3 | 4 | replication_config = dg.file_relative_path(__file__, "sling_replication.yaml") 5 | 6 | 7 | @sling_assets(replication_config=replication_config) 8 | def postgres_sling_assets(context, sling: SlingResource): 9 | yield from sling.replicate(context=context).fetch_column_metadata() 10 | 11 | 12 | @dg.asset(deps=[dg.AssetKey("orders")]) 13 | def downstream_orders(context: dg.AssetExecutionContext): 14 | pass 15 | 16 | 17 | @dg.asset(deps=[dg.AssetKey("products")]) 18 | def downstream_products(context: dg.AssetExecutionContext): 19 | pass 20 | 21 | 22 | @dg.asset(deps=[downstream_orders, downstream_products]) 23 | def downstream_orders_and_products(context: dg.AssetExecutionContext): 24 | pass 25 | -------------------------------------------------------------------------------- /course/pages/dagster-etl/lesson-3/0-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Lesson 3: Overview" 3 | module: 'dagster_etl' 4 | lesson: '3' 5 | --- 6 | 7 | # Overview 8 | 9 | We'll begin by loading data from an external system into a data warehouse. In production, this source might be something like S3 or Azure Blob Storage, but for now, we’ll keep things simple by using the local filesystem. 10 | 11 | This is one of the most common ETL use cases, and many data warehouses support it natively. For example, loading data from S3 can often be accomplished by executing a SQL command directly within the warehouse. However, there are also situations where some level of custom processing is needed before the data can be ingested. 12 | 13 | In either case, this provides a solid starting point to explore the different ways ETL can be implemented in Dagster. 14 | 15 | ![API ETL](/images/dagster-etl/lesson-3/csv-etl.png) 16 | -------------------------------------------------------------------------------- /course/pages/dagster-essentials/lesson-6/6-recap.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 6: Lesson recap' 3 | module: 'dagster_essentials' 4 | lesson: '6' 5 | --- 6 | 7 | # Lesson recap 8 | 9 | Dagster comes with many resources and integrations out of the box. However, an integration library is not required to use your tools within Dagster. 10 | 11 | You can pass any object or connection into your code location’s `Definitions` and add it as a resource. The only requirement is that you will need to add `ResourceParam[]` to your asset argument’s type hints for your asset function to understand that you’re referencing a resource. 12 | 13 | Check out the [Dagster Docs](https://docs.dagster.io/integrations) for more info about all of Dagster’s integrations. 14 | 15 | {% callout %} 16 | 17 | > ‼️ Make sure you update your code to match the practice problem answer before continuing! 18 | 19 | {% /callout %} 20 | -------------------------------------------------------------------------------- /course/pages/dagster-dbt/lesson-1/deprecation.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Lesson 1: Deprecation" 3 | module: 'dbt_dagster' 4 | lesson: '1' 5 | --- 6 | 7 | **Deprecation Notice** 8 | 9 | This course was originally built around the Python-based integration of Dagster and dbt. With the introduction of [Dagster Components](https://docs.dagster.io/guides/build/components), connecting to a dbt project is now far more streamlined and beginner-friendly. 10 | 11 | If you’re new to Dagster and dbt, we recommend starting with the simpler, component-based guides instead: 12 | 13 | * [dbt integration](https://docs.dagster.io/integrations/libraries/dbt) 14 | * [Dagster and dbt example](https://docs.dagster.io/examples/dbt) 15 | 16 | If you’d like to explore Dagster internals and work directly with the Python APIs for the Dagster–dbt integration, you are still welcome to take this course. You’ll also receive a certificate upon completion. -------------------------------------------------------------------------------- /course/pages/dagster-essentials/lesson-1/4-project-preview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 1: Project preview' 3 | module: 'dagster_essentials' 4 | lesson: '1' 5 | --- 6 | 7 | # Project preview 8 | 9 | In this course, we will focus on asset-aware orchestrators and how they make data pipelines easier to manage. You’ll use Dagster, an open-source orchestrator, to build a sample data pipeline. 10 | 11 | Using data from [NYC OpenData,](https://opendata.cityofnewyork.us/) you’ll build a data pipeline that: 12 | 13 | - Extracts the data, stored in Parquet files, from NYC OpenData 14 | - Loads it into a DuckDB database 15 | - Transforms and prepares it for analysis 16 | - Creates a visualization using the transformed data 17 | 18 | If you get stuck or want to jump ahead, check out the [finished project here on GitHub](https://github.com/dagster-io/project-dagster-university/tree/main/dagster_university/dagster_essentials). 19 | -------------------------------------------------------------------------------- /course/components/Callout.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | 3 | interface Props { 4 | children: React.ReactNode; 5 | } 6 | 7 | export const Callout = ({children}) => { 8 | return ( 9 |
10 |
11 |
12 | 21 |
22 |
{children}
23 |
24 |
25 | ); 26 | }; 27 | -------------------------------------------------------------------------------- /course/pages/dagster-etl/lesson-6/0-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Lesson 6: Overview" 3 | module: 'dagster_etl' 4 | lesson: '6' 5 | --- 6 | 7 | # Overview 8 | 9 | We’ve explored several ways to ingest data from external sources, but one approach we haven’t covered yet is moving data between databases. This is a broad category, as databases vary widely in type and behavior. However, the most common scenario and one that many people associate with traditional ETL, is replicating data from an OLTP database (like Postgres or MySQL) into a data warehouse (such as Snowflake or Redshift). 10 | 11 | Although this flow is extremely common, and something nearly every data-driven organization performs to some extent, it's also a nuanced and potentially error-prone process. From schema drift and type mismatches to performance bottlenecks and data consistency challenges, moving data between systems requires careful handling and thoughtful architecture. 12 | -------------------------------------------------------------------------------- /course/pages/dagster-testing/lesson-3/0-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 3: Overview' 3 | module: 'dagster_testing' 4 | lesson: '3' 5 | --- 6 | 7 | # Overview 8 | 9 | In the previous lesson, you learned that there are many different types of testing. This lesson will focus on unit tests and creating the fundamental tests for your Dagster assets. We will discuss the various aspects of writing tests for your assets before we cover other Dagster concepts in future lessons. 10 | 11 | Tests in Dagster leverage conventional Python tooling and methodology. Learning about tests in Python generally is relevant to implementing tests in Dagster. The Dagster-specific learning curve is limited to efficiently using Dagster’s built-in tests features and a practical working knowledge of which tests to use when with reference examples. If this is your first time writing tests in Python, most of this lesson will carry over to your general Python knowledge. -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/tests/test_lesson_4.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | import pytest 3 | 4 | import dagster_essentials.completed.lesson_4.defs 5 | 6 | 7 | @pytest.fixture() 8 | def defs(): 9 | return dg.components.load_defs(dagster_essentials.completed.lesson_4.defs) 10 | 11 | 12 | def test_assets(defs): 13 | assets = [ 14 | defs.get_assets_def(dg.AssetKey(["taxi_trips_file"])), 15 | defs.get_assets_def(dg.AssetKey(["taxi_zones_file"])), 16 | defs.get_assets_def(dg.AssetKey(["taxi_trips"])), 17 | defs.get_assets_def(dg.AssetKey(["taxi_zones"])), 18 | defs.get_assets_def(dg.AssetKey(["trips_by_week"])), 19 | defs.get_assets_def(dg.AssetKey(["manhattan_stats"])), 20 | defs.get_assets_def(dg.AssetKey(["manhattan_map"])), 21 | ] 22 | result = dg.materialize(assets) 23 | assert result.success 24 | 25 | 26 | def test_defs(defs): 27 | assert defs 28 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/tests/test_lesson_5.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | import pytest 3 | 4 | import dagster_essentials.completed.lesson_5.defs 5 | 6 | 7 | @pytest.fixture() 8 | def defs(): 9 | return dg.components.load_defs(dagster_essentials.completed.lesson_5.defs) 10 | 11 | 12 | def test_assets(defs): 13 | assets = [ 14 | defs.get_assets_def(dg.AssetKey(["taxi_trips_file"])), 15 | defs.get_assets_def(dg.AssetKey(["taxi_zones_file"])), 16 | defs.get_assets_def(dg.AssetKey(["taxi_trips"])), 17 | defs.get_assets_def(dg.AssetKey(["taxi_zones"])), 18 | defs.get_assets_def(dg.AssetKey(["trips_by_week"])), 19 | defs.get_assets_def(dg.AssetKey(["manhattan_stats"])), 20 | defs.get_assets_def(dg.AssetKey(["manhattan_map"])), 21 | ] 22 | result = dg.materialize(assets) 23 | assert result.success 24 | 25 | 26 | def test_defs(defs): 27 | assert defs 28 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/data/source/2018-01-23.csv: -------------------------------------------------------------------------------- 1 | "date","share_price","amount","spend","shift","spread" 2 | "2018/10/15","259.5900","6189026.0000","259.0600","263.2800","254.5367" 3 | "2018/10/12","258.7800","7189257.0000","261.0000","261.9900","252.0100" 4 | "2018/10/11","252.2300","8128184.0000","257.5300","262.2500","249.0300" 5 | "2018/10/10","256.8800","12781560.0000","264.6100","265.5100","247.7700" 6 | "2018/10/09","262.8000","12037780.0000","255.2500","266.7700","253.3000" 7 | "2018/10/08","250.5600","13371180.0000","264.5200","267.7599","249.0000" 8 | "2018/10/05","261.9500","17900710.0000","274.6500","274.8800","260.0000" 9 | "2018/10/04","281.8300","9638885.0000","293.9500","294.0000","277.6700" 10 | "2018/10/03","294.8000","7982272.0000","303.3300","304.6000","291.5700" 11 | "2018/10/02","301.0200","11699690.0000","313.9500","316.8400","299.1500" 12 | "2018/10/01","310.7000","21714210.0000","305.7700","311.4400","301.0500" -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/data/source/2018-01-24.csv: -------------------------------------------------------------------------------- 1 | "date","share_price","amount","spend","shift","spread" 2 | "2018/10/15","259.5900","6189026.0000","259.0600","263.2800","254.5367" 3 | "2018/10/12","258.7800","7189257.0000","261.0000","261.9900","252.0100" 4 | "2018/10/11","252.2300","8128184.0000","257.5300","262.2500","249.0300" 5 | "2018/10/10","256.8800","12781560.0000","264.6100","265.5100","247.7700" 6 | "2018/10/09","262.8000","12037780.0000","255.2500","266.7700","253.3000" 7 | "2018/10/08","250.5600","13371180.0000","264.5200","267.7599","249.0000" 8 | "2018/10/05","261.9500","17900710.0000","274.6500","274.8800","260.0000" 9 | "2018/10/04","281.8300","9638885.0000","293.9500","294.0000","277.6700" 10 | "2018/10/03","294.8000","7982272.0000","303.3300","304.6000","291.5700" 11 | "2018/10/02","301.0200","11699690.0000","313.9500","316.8400","299.1500" 12 | "2018/10/01","310.7000","21714210.0000","305.7700","311.4400","301.0500" -------------------------------------------------------------------------------- /course/pages/dagster-dbt/lesson-6/4-lesson-recap.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 6: Lesson recap' 3 | module: 'dagster_dbt' 4 | lesson: '6' 5 | --- 6 | 7 | # Lesson recap 8 | 9 | In this lesson, you: 10 | 11 | - Learned the benefits of partitioning your incremental models 12 | - Added a time-based partition to an incremental model 13 | - Created a second `@dbt_assets` definition specifically for incremental dbt models 14 | 15 | The patterns you used are general enough that they can also be applied to any type of partition, allowing you to partition your incremental models by location, customer, or other dimensions. Tinker around with the `context.partition_key` property if you’re interested! 16 | 17 | {% callout %} 18 | > 💡 **Tip:** Did you know dbt models can resolve schema changes on their own? Using `on_schema_change: "sync_all_columns"`, you can avoid needing to fully refresh your dbt models and instead only orchestrate with Dagster backfills. 19 | {% /callout %} -------------------------------------------------------------------------------- /course/pages/dagster-dbt/lesson-7/1-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Lesson 7: Overview" 3 | module: 'dbt_dagster' 4 | lesson: '7' 5 | --- 6 | 7 | # Overview 8 | 9 | At this point, you have a fully integrated Dagster and dbt project! You’ve learned how to load dbt models as Dagster assets, create dependencies, add partitions, and execute and monitor the resulting pipeline in the Dagster UI. 10 | 11 | In this lesson, we’ll deploy your Dagster and dbt project to have it running in both local and production environments. We’ll walk through some considerations involved in bundling your dbt project up with Dagster. 12 | 13 | You’ll learn how to deploy your unified Dagster and dbt project to production, including pushing the project to GitHub and setting up CI/CD to factor in your dbt project. We’ll use Dagster+ because it’s a standardized and controlled experience that we can walk you through, but all of the general patterns can be applied to however you deploy Dagster. 14 | -------------------------------------------------------------------------------- /dagster_university/dagster_essentials/README.md: -------------------------------------------------------------------------------- 1 | # Dagster Essentials 2 | 3 | ## Overview 4 | 5 | Learn the basics of Dagster, a Python-based platform that enables you to build robust, production-ready data pipelines. In this course, you’ll learn how to represent a data pipeline as the data assets it produces and orchestrate a pipeline you’ll make with Dagster. 6 | 7 | ## Completed code 8 | 9 | If you are stuck you can reference the completed code for each lesson. 10 | 11 | ``` 12 | src 13 | └── dagster_essentials 14 | └── completed 15 |       ├── lesson_2 16 |       ├── lesson_3 17 |       ├── lesson_4 18 |       ├── lesson_5 19 |       ├── lesson_6 20 |       ├── lesson_7 21 |       ├── lesson_8 22 |       └── lesson_9 23 | ``` 24 | 25 | ## Deploy on Dagster Cloud 26 | 27 | The easiest way to deploy your Dagster project is to use Dagster Cloud. 28 | 29 | Check out the [Dagster Cloud Documentation](https://docs.dagster.cloud) to learn more. 30 | -------------------------------------------------------------------------------- /course/pages/dagster-dbt/lesson-1/4-project-preview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Lesson 1: Project preview" 3 | module: 'dbt_dagster' 4 | lesson: '1' 5 | --- 6 | 7 | # Project preview 8 | 9 | In this course, we’ll focus on integrating a dbt project with Dagster from end to end. We’ll build on the Dagster project used in the Dagster Essentials course, which uses data from [NYC OpenData](https://opendata.cityofnewyork.us/). If you haven’t completed Dagster Essentials, no worries - we'll show you what to do in the next lesson. 10 | 11 | By the end of the course, you will: 12 | 13 | - Create dbt models and load them into Dagster as assets 14 | - Run dbt and store the transformed data in a DuckDB database 15 | - Apply partitions to incremental dbt models 16 | - Deploy the Dagster project to Dagster+ 17 | 18 | If you get stuck or want to jump ahead, check out the [finished project here on GitHub](https://github.com/dagster-io/project-dagster-university/tree/main/dagster_university/dagster_and_dbt). -------------------------------------------------------------------------------- /course/pages/dagster-essentials/lesson-4/0-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 4: Overview' 3 | module: 'dagster_essentials' 4 | lesson: '4' 5 | --- 6 | 7 | # Overview 8 | 9 | In the previous lesson, you learned what an asset is and applied that knowledge to create your first assets in Dagster. Now, you’ll learn how to expand your Dagster project to create a data pipeline. 10 | 11 | Data pipelines are a chain of events that produce data assets. As introduced in [Lesson 1](https://www.notion.so/Lesson-1-Introduction-43c6dcd35f6b4a6bb0729d3fd185ce88?pvs=21), Dagster empowers you to efficiently create data pipelines that produce multiple assets. 12 | 13 | Now, you’ll build a complete data pipeline that: 14 | 15 | - Loads your downloaded files into a database 16 | - Combines and aggregates your data into metrics about taxi trips 17 | - Visualizes those metrics and saves the report 18 | 19 | While doing so, you’ll learn how to use Dagster to define dependencies between assets to produce pipelines. 20 | -------------------------------------------------------------------------------- /course/pages/dagster-essentials/lesson-7/2-coding-practice-weekly-update-job.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 7: Practice: Create a weekly_update_job' 3 | module: 'dagster_essentials' 4 | lesson: '7' 5 | --- 6 | 7 | # Practice: Create a weekly_update_job 8 | 9 | To practice what you’ve learned, add a job to `src/dagster_essentials/defs/jobs.py` that will materialize the `trips_by_week` asset. 10 | 11 | --- 12 | 13 | ## Check your work 14 | 15 | The job you built should look similar to the following code. Click **View answer** to view it. 16 | 17 | **If there are differences**, compare what you wrote to the job below and change them, as this job will be used as-is in future lessons. 18 | 19 | ```python {% obfuscated="true" %} 20 | # src/dagster_essentials/defs/jobs.py 21 | import dagster as dg 22 | 23 | trips_by_week = dg.AssetSelection.assets(["trips_by_week"]) 24 | 25 | weekly_update_job = dg.define_asset_job( 26 | name="weekly_update_job", 27 | selection=trips_by_week, 28 | ) 29 | ``` 30 | -------------------------------------------------------------------------------- /course/pages/dagster-etl/lesson-1/3-project-preview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Lesson 1: Project preview" 3 | module: 'dbt_etl' 4 | lesson: '1' 5 | --- 6 | 7 | # Project preview 8 | 9 | In this course, we’ll focus on ETL and how to manage data ingestion using Dagster. All of the examples will walk through real-world ETL workflows you're likely to encounter, covering a variety of data sources and the unique challenges they present. 10 | 11 | By the end of the course, you will: 12 | 13 | - Create scheduled and event-driven pipelines to ingest files 14 | - Build a custom resource to pull data from an external API 15 | - Use Embedded ETL (with dlt) to build more resilient applications 16 | - Replicate data across databases 17 | - Refactor your code using Dagster Components for better modularity and reuse 18 | 19 | If you get stuck or want to jump ahead, check out the [finished project here on GitHub](https://github.com/dagster-io/project-dagster-university/tree/main/dagster_university/dagster_and_etl). 20 | -------------------------------------------------------------------------------- /course/markdoc/nodes/heading.markdoc.ts: -------------------------------------------------------------------------------- 1 | import {Tag} from '@markdoc/markdoc'; 2 | 3 | import {Heading} from '../../components'; 4 | 5 | function generateID(children, attributes) { 6 | if (attributes.id && typeof attributes.id === 'string') { 7 | return attributes.id; 8 | } 9 | return children 10 | .filter((child) => typeof child === 'string') 11 | .join(' ') 12 | .replace(/[?]/g, '') 13 | .replace(/\s+/g, '-') 14 | .toLowerCase(); 15 | } 16 | 17 | export const heading = { 18 | render: Heading, 19 | children: ['inline'], 20 | attributes: { 21 | id: {type: String}, 22 | level: {type: Number, required: true, default: 1}, 23 | className: {type: String}, 24 | }, 25 | transform(node, config) { 26 | const attributes = node.transformAttributes(config); 27 | const children = node.transformChildren(config); 28 | const id = generateID(children, attributes); 29 | 30 | return new Tag(this.render, {...attributes, id}, children); 31 | }, 32 | }; 33 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/tests/fixtures.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | 4 | import pytest 5 | 6 | 7 | def _dbt_command(cmd: str, lesson: str): 8 | """Run a dbt command within a specific project lesson dbt project""" 9 | dir = os.path.join( 10 | os.path.dirname(__file__), 11 | f"../src/dagster_and_dbt/completed/{lesson}/analytics", 12 | ) 13 | 14 | cmd = ["dbt", cmd, "--project-dir", dir, "--profiles-dir", dir] 15 | print(cmd) 16 | result = subprocess.run(cmd, capture_output=True, text=True) 17 | print(result.stdout) 18 | 19 | if result.returncode != 0: 20 | pytest.fail(f"dbt command failed: {result.returncode}") 21 | 22 | 23 | @pytest.fixture(scope="session", autouse=True) 24 | def setup_dbt_env(request): 25 | """Run necessary dbt commands to generate the dbt manifest.json required for Dagster to parse""" 26 | lesson_num = request.param 27 | _dbt_command("deps", lesson_num) 28 | _dbt_command("parse", lesson_num) 29 | -------------------------------------------------------------------------------- /course/pages/dagster-dbt/lesson-2/4-set-up-the-dbt-project.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Lesson 2: Set up the dbt project" 3 | module: 'dagster_dbt' 4 | lesson: '2' 5 | --- 6 | 7 | # Set up the dbt project 8 | 9 | Next, you’ll notice that there is a dbt project called `analytics` in the repository you cloned. Throughout the duration of this module, you’ll add new dbt models and see them reflected in Dagster. 10 | 11 | 1. Navigate into the directory by running: 12 | 13 | ```bash 14 | cd dagster_university/dagster_and_dbt/src/dagster_and_dbt/analytics 15 | ``` 16 | 17 | 2. Next, install dbt package dependencies by running: 18 | 19 | ```bash 20 | dbt deps 21 | ``` 22 | 23 | 3. In a file explorer or IDE, open the `analytics` directory. You should see the following files, which are the models we’ll use to get started: 24 | 25 | - `models/sources/raw_taxis.yml` 26 | - `models/staging/staging.yml` 27 | - `models/staging/stg_trips.sql` 28 | - `models/staging/stg_zones.sql` 29 | -------------------------------------------------------------------------------- /course/pages/dagster-dbt/lesson-2/3-set-up-dagster-project.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Lesson 2: Set up the Dagster project" 3 | module: 'dagster_dbt' 4 | lesson: '2' 5 | --- 6 | 7 | # Set up the Dagster project 8 | 9 | To confirm everything works: 10 | 11 | 1. Run `dagster dev` from the directory `dagster_university/dagster_and_dbt`. 12 | 2. Navigate to the Dagster UI ([`http://localhost:3000`](http://localhost:3000/)) in your browser. 13 | 3. Open the asset graph by clicking **Assets > View global asset lineage** and confirm the asset graph you see matches the graph below. 14 | 15 | ![The Asset Graph in the Dagster UI](/images/dagster-dbt/lesson-2/asset-graph.png) 16 | 17 | 4. Let's confirm that you can materialize these assets by: 18 | 1. Navigating to **Overview > Jobs** 19 | 2. Clicking on the `trip_update_job` job and then **Materialize all...**. 20 | 3. When prompted to select a partition, materialize the most recent one (`2023-03-01`). It will start a run/backfill and your assets should materialize successfully. -------------------------------------------------------------------------------- /course/pages/dagster-etl/lesson-5/1-dlt.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Lesson 5: dlt" 3 | module: 'dagster_etl' 4 | lesson: '5' 5 | --- 6 | 7 | # dlt 8 | 9 | One ETL framework we’re particularly excited about is [data load tool (dlt)](https://dlthub.com/docs/intro). dlt is an open-source, lightweight Python library designed to simplify data loading. It takes care of many of the more tedious aspects of ETL, including schema management, data type handling, and normalization, so you can focus on what matters most. 10 | 11 | dlt supports a wide range of popular sources and destinations, which means you can move data between systems without having to build and maintain all the supporting infrastructure yourself. While it still gives you the flexibility to handle custom or complex data workflows, it eliminates much of the boilerplate code you'd otherwise need to write, making your pipelines cleaner, more maintainable, and faster to develop. 12 | 13 | ![dlt](/images/dagster-etl/lesson-5/dlt.png) 14 | 15 | Let's look at a simple dlt example. 16 | -------------------------------------------------------------------------------- /course/pages/dagster-essentials/lesson-8/0-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 8: Overview' 3 | module: 'dagster_essentials' 4 | lesson: '8' 5 | --- 6 | 7 | # Overview 8 | 9 | In the previous lesson, you learned about schedules and running your pipelines regularly. Going back to the cookie analogy, imagine that your cookie business is blossoming, and you’ve started taking orders in advance. Making every cookie right as each order came in would create problems. For example, an order for today is more urgent than an order for next week. Meanwhile, on some days, you would receive 100 orders, but you may receive zero orders on other days. 10 | 11 | Therefore, you batch your orders by the day they’re expected to be picked up. Every morning, you look at the orders to be fulfilled that day and only make the cookies for those orders. 12 | 13 | By looking at each day at a time, you are **partitioning** the orders. In this lesson, you’ll learn why to partition your data assets and how to do it in Dagster by partitioning the taxi trip data. 14 | -------------------------------------------------------------------------------- /course/pages/dagster-etl/lesson-5/0-overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Lesson 5: Overview" 3 | module: 'dagster_etl' 4 | lesson: '5' 5 | --- 6 | 7 | # Overview 8 | 9 | The pipelines we've built so far are great for learning the foundations of ETL pipeline construction and exploring how to apply various Dagster features. However, these aren't necessarily the kinds of pipelines you’d want to deploy directly to production. The main issue is that we're reinventing too many parts of the ETL process, developing logic and handling edge cases that could be handled more efficiently with existing tools. 10 | 11 | The good news is that ETL is a universal problem, one that nearly every data-driven organization faces. As a result, a wide range of tools and frameworks have emerged to simplify and standardize ETL workflows, from data extraction and transformation to orchestration and monitoring. Leveraging these tools not only reduces boilerplate and bugs, but also allows you to focus on the parts of the pipeline that are unique to your business logic. 12 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/analytics/models/marts/daily_metrics.sql: -------------------------------------------------------------------------------- 1 | {{ 2 | config( 3 | materialized='incremental', 4 | unique_key='date_of_business' 5 | ) 6 | }} 7 | 8 | with 9 | trips as ( 10 | select * 11 | from {{ ref('stg_trips') }} 12 | ), 13 | daily_summary as ( 14 | select 15 | date_trunc('day', pickup_datetime) as date_of_business, 16 | count(*) as trip_count, 17 | sum(duration) as total_duration, 18 | sum(duration) / count(*) as average_duration, 19 | sum(total_amount) as total_amount, 20 | sum(total_amount) / count(*) as average_amount, 21 | sum(case when duration > 30 then 1 else 0 end) / count(*) as pct_over_30_min 22 | from trips 23 | group by all 24 | ) 25 | select * 26 | from daily_summary 27 | {% if is_incremental() %} 28 | where date_of_business between '{{ var('min_date') }}' and '{{ var('max_date') }}' 29 | {% endif %} -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/analytics/models/marts/daily_metrics.sql: -------------------------------------------------------------------------------- 1 | {{ 2 | config( 3 | materialized='incremental', 4 | unique_key='date_of_business' 5 | ) 6 | }} 7 | 8 | with 9 | trips as ( 10 | select * 11 | from {{ ref('stg_trips') }} 12 | ), 13 | daily_summary as ( 14 | select 15 | date_trunc('day', pickup_datetime) as date_of_business, 16 | count(*) as trip_count, 17 | sum(duration) as total_duration, 18 | sum(duration) / count(*) as average_duration, 19 | sum(total_amount) as total_amount, 20 | sum(total_amount) / count(*) as average_amount, 21 | sum(case when duration > 30 then 1 else 0 end) / count(*) as pct_over_30_min 22 | from trips 23 | group by all 24 | ) 25 | select * 26 | from daily_summary 27 | {% if is_incremental() %} 28 | where date_of_business between '{{ var('min_date') }}' and '{{ var('max_date') }}' 29 | {% endif %} -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/tests/test_lesson_3.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | import pytest 3 | 4 | import dagster_and_dbt.completed.lesson_3.defs 5 | from tests.fixtures import setup_dbt_env # noqa: F401 6 | 7 | 8 | @pytest.mark.parametrize("setup_dbt_env", ["lesson_3"], indirect=True) 9 | def test_dbt_assets(setup_dbt_env): # noqa: F811 10 | from dagster_and_dbt.completed.lesson_3.defs.assets import dbt 11 | from dagster_and_dbt.completed.lesson_3.defs.resources import dbt_resource 12 | 13 | dbt_analytics_assets = dg.load_assets_from_modules(modules=[dbt]) 14 | 15 | result = dg.materialize( 16 | assets=[*dbt_analytics_assets], 17 | resources={ 18 | "dbt": dbt_resource, 19 | }, 20 | ) 21 | assert result.success 22 | 23 | 24 | @pytest.mark.parametrize("setup_dbt_env", ["lesson_3"], indirect=True) 25 | def test_defs(setup_dbt_env): # noqa: F811 26 | assert dg.Definitions.merge( 27 | dg.components.load_defs(dagster_and_dbt.completed.lesson_3.defs) 28 | ) 29 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/tests/test_lesson_4.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | import pytest 3 | 4 | import dagster_and_dbt.completed.lesson_4.defs 5 | from tests.fixtures import setup_dbt_env # noqa: F401 6 | 7 | 8 | @pytest.mark.parametrize("setup_dbt_env", ["lesson_4"], indirect=True) 9 | def test_dbt_assets(setup_dbt_env): # noqa: F811 10 | from dagster_and_dbt.completed.lesson_4.defs.assets import dbt 11 | from dagster_and_dbt.completed.lesson_4.defs.resources import dbt_resource 12 | 13 | dbt_analytics_assets = dg.load_assets_from_modules(modules=[dbt]) 14 | 15 | result = dg.materialize( 16 | assets=[*dbt_analytics_assets], 17 | resources={ 18 | "dbt": dbt_resource, 19 | }, 20 | ) 21 | assert result.success 22 | 23 | 24 | @pytest.mark.parametrize("setup_dbt_env", ["lesson_4"], indirect=True) 25 | def test_defs(setup_dbt_env): # noqa: F811 26 | assert dg.Definitions.merge( 27 | dg.components.load_defs(dagster_and_dbt.completed.lesson_4.defs) 28 | ) 29 | -------------------------------------------------------------------------------- /course/pages/dagster-etl/lesson-6/2-sling.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Lesson 6: Sling" 3 | module: 'dagster_etl' 4 | lesson: '6' 5 | --- 6 | 7 | # Sling 8 | 9 | In the previous lesson we used dlt. dlt is great for flexibility but since we know we are doing replication between two systems, we can try something else (though dlt can also do replication). 10 | 11 | For this lesson we’ll introduce another open-source ETL framework: [Sling](https://docs.slingdata.io/). Sling is a modern tool designed to simplify both real-time and batch data replication. It helps teams move data from databases like Postgres and MySQL into cloud data warehouses such as Snowflake or Redshift with minimal setup. Exposing all configuration through a simple YAML interface. 12 | 13 | Sling differs from dlt in that it is declarative by design. While dlt offers greater flexibility and can handle data ingestion from a wide variety of sources (such as our custom NASA API), Sling is purpose-built for database replication and excels at managing the complexities of moving data and schema evolution. 14 | -------------------------------------------------------------------------------- /dagster_university/dagster_testing/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "dagster-testing" 3 | version = "0.0.1" 4 | description = "Dagster Testing" 5 | readme = "README.md" 6 | requires-python = ">=3.10,<3.14" 7 | dependencies = [ 8 | "dagster==1.12.4", 9 | "dagster-dg-cli", 10 | "dagster-webserver", 11 | "dagster-snowflake", 12 | "psycopg2-binary", 13 | "pytest", 14 | ] 15 | 16 | [tool.uv] 17 | dev-dependencies = [ 18 | "ruff", 19 | ] 20 | 21 | [build-system] 22 | requires = ["hatchling"] 23 | build-backend = "hatchling.build" 24 | 25 | [tool.dg] 26 | directory_type = "project" 27 | 28 | [tool.dg.project] 29 | root_module = "dagster_testing" 30 | registry_modules = [ 31 | "dagster_testing.components.*", 32 | ] 33 | 34 | [tool.ruff.lint] 35 | ignore = [ 36 | "ERA001", # comments are fine for a learning environment 37 | "E501", # Line too long for docstrings 38 | "F401", # unused imports already caught by pylance 39 | ] 40 | 41 | fixable = ["ALL"] 42 | unfixable = [ 43 | "F401", # dont remove unused imports 44 | ] 45 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/analytics/models/marts/location_metrics.sql: -------------------------------------------------------------------------------- 1 | with 2 | trips as ( 3 | select * 4 | from {{ ref('stg_trips') }} 5 | ), 6 | zones as ( 7 | select * 8 | from {{ ref('stg_zones') }} 9 | ), 10 | trips_by_zone as ( 11 | select 12 | pickup_zones.zone_name as zone, 13 | dropoff_zones.borough as destination_borough, 14 | pickup_zones.is_airport as from_airport, 15 | count(*) as trips, 16 | sum(trips.trip_distance) as total_distance, 17 | sum(trips.duration) as total_duration, 18 | sum(trips.total_amount) as fare, 19 | sum(case when duration > 30 then 1 else 0 end) as trips_over_30_min 20 | from trips 21 | left join zones as pickup_zones on trips.pickup_zone_id = pickup_zones.zone_id 22 | left join zones as dropoff_zones on trips.dropoff_zone_id = dropoff_zones.zone_id 23 | group by all 24 | ) 25 | select * 26 | from trips_by_zone 27 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/analytics/models/marts/location_metrics.sql: -------------------------------------------------------------------------------- 1 | with 2 | trips as ( 3 | select * 4 | from {{ ref('stg_trips') }} 5 | ), 6 | zones as ( 7 | select * 8 | from {{ ref('stg_zones') }} 9 | ), 10 | trips_by_zone as ( 11 | select 12 | pickup_zones.zone_name as zone, 13 | dropoff_zones.borough as destination_borough, 14 | pickup_zones.is_airport as from_airport, 15 | count(*) as trips, 16 | sum(trips.trip_distance) as total_distance, 17 | sum(trips.duration) as total_duration, 18 | sum(trips.total_amount) as fare, 19 | sum(case when duration > 30 then 1 else 0 end) as trips_over_30_min 20 | from trips 21 | left join zones as pickup_zones on trips.pickup_zone_id = pickup_zones.zone_id 22 | left join zones as dropoff_zones on trips.dropoff_zone_id = dropoff_zones.zone_id 23 | group by all 24 | ) 25 | select * 26 | from trips_by_zone 27 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/analytics/models/marts/location_metrics.sql: -------------------------------------------------------------------------------- 1 | with 2 | trips as ( 3 | select * 4 | from {{ ref('stg_trips') }} 5 | ), 6 | zones as ( 7 | select * 8 | from {{ ref('stg_zones') }} 9 | ), 10 | trips_by_zone as ( 11 | select 12 | pickup_zones.zone_name as zone, 13 | dropoff_zones.borough as destination_borough, 14 | pickup_zones.is_airport as from_airport, 15 | count(*) as trips, 16 | sum(trips.trip_distance) as total_distance, 17 | sum(trips.duration) as total_duration, 18 | sum(trips.total_amount) as fare, 19 | sum(case when duration > 30 then 1 else 0 end) as trips_over_30_min 20 | from trips 21 | left join zones as pickup_zones on trips.pickup_zone_id = pickup_zones.zone_id 22 | left join zones as dropoff_zones on trips.dropoff_zone_id = dropoff_zones.zone_id 23 | group by all 24 | ) 25 | select * 26 | from trips_by_zone 27 | -------------------------------------------------------------------------------- /course/pages/dagster-essentials/lesson-8/3-coding-practice-weekly-partition.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 8: Practice: Create a weekly partition' 3 | module: 'dagster_essentials' 4 | lesson: '8' 5 | --- 6 | 7 | # Practice: Create a weekly partition 8 | 9 | To practice what you’ve learned, create a `weekly_partition` using Dagster’s `WeeklyPartitionsDefinition` with the same start and end dates. 10 | 11 | --- 12 | 13 | ## Check your work 14 | 15 | The partition you built should look similar to the following code. Click **View answer** to view it. 16 | 17 | **If there are differences**, compare what you wrote to the partition below and change them, as this partition will be used as-is in future lessons. 18 | 19 | ```python {% obfuscated="true" %} 20 | # src/dagster_essentials/defs/partitions.py 21 | import dagster as dg 22 | from dagster_essentials.defs.assets import constants 23 | 24 | start_date = constants.START_DATE 25 | end_date = constants.END_DATE 26 | 27 | weekly_partition = dg.WeeklyPartitionsDefinition( 28 | start_date=start_date, 29 | end_date=end_date 30 | ) 31 | ``` 32 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/defs/jobs.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | from dagster_dbt import build_dbt_asset_selection 3 | 4 | from dagster_and_dbt.completed.lesson_5.defs.assets.dbt import dbt_analytics 5 | from dagster_and_dbt.completed.lesson_5.defs.partitions import ( 6 | monthly_partition, 7 | weekly_partition, 8 | ) 9 | 10 | trips_by_week = dg.AssetSelection.assets("trips_by_week") 11 | adhoc_request = dg.AssetSelection.assets("adhoc_request") 12 | 13 | dbt_trips_selection = build_dbt_asset_selection([dbt_analytics], "stg_trips+") 14 | 15 | trip_update_job = dg.define_asset_job( 16 | name="trip_update_job", 17 | partitions_def=monthly_partition, 18 | selection=dg.AssetSelection.all() 19 | - trips_by_week 20 | - adhoc_request 21 | - dbt_trips_selection, 22 | ) 23 | 24 | weekly_update_job = dg.define_asset_job( 25 | name="weekly_update_job", partitions_def=weekly_partition, selection=trips_by_week 26 | ) 27 | 28 | adhoc_request_job = dg.define_asset_job( 29 | name="adhoc_request_job", selection=adhoc_request 30 | ) 31 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/tests/test_lesson_6.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | import pytest 3 | 4 | import dagster_and_dbt.completed.lesson_6.defs 5 | from tests.fixtures import setup_dbt_env # noqa: F401 6 | 7 | 8 | @pytest.mark.parametrize("setup_dbt_env", ["lesson_6"], indirect=True) 9 | def test_dbt_partitioned_incremental_assets(setup_dbt_env): # noqa: F811 10 | from dagster_and_dbt.completed.lesson_6.defs.assets import dbt 11 | from dagster_and_dbt.completed.lesson_6.defs.resources import dbt_resource 12 | 13 | dbt_analytics_assets = dg.load_assets_from_modules(modules=[dbt]) 14 | 15 | result = dg.materialize( 16 | assets=[*dbt_analytics_assets], 17 | resources={ 18 | "dbt": dbt_resource, 19 | }, 20 | partition_key="2023-01-01", 21 | ) 22 | assert result.success 23 | 24 | 25 | @pytest.mark.parametrize("setup_dbt_env", ["lesson_6"], indirect=True) 26 | def test_defs(setup_dbt_env): # noqa: F811 27 | assert dg.Definitions.merge( 28 | dg.components.load_defs(dagster_and_dbt.completed.lesson_6.defs) 29 | ) 30 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_3/defs/resources.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import boto3 4 | import dagster as dg 5 | from dagster_dbt import DbtCliResource 6 | from dagster_duckdb import DuckDBResource 7 | 8 | from dagster_and_dbt.completed.lesson_3.defs.project import dbt_project 9 | 10 | database_resource = DuckDBResource( 11 | database=dg.EnvVar("DUCKDB_DATABASE"), 12 | ) 13 | 14 | if os.getenv("DAGSTER_ENVIRONMENT") == "prod": 15 | session = boto3.Session( 16 | aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), 17 | aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), 18 | region_name=os.getenv("AWS_REGION"), 19 | ) 20 | smart_open_config = {"client": session.client("s3")} 21 | else: 22 | smart_open_config = {} 23 | 24 | 25 | dbt_resource = DbtCliResource( 26 | project_dir=dbt_project, 27 | ) 28 | 29 | 30 | @dg.definitions 31 | def resources(): 32 | return dg.Definitions( 33 | resources={ 34 | "database": database_resource, 35 | "dbt": dbt_resource, 36 | }, 37 | ) 38 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_4/defs/resources.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import boto3 4 | import dagster as dg 5 | from dagster_dbt import DbtCliResource 6 | from dagster_duckdb import DuckDBResource 7 | 8 | from dagster_and_dbt.completed.lesson_4.defs.project import dbt_project 9 | 10 | database_resource = DuckDBResource( 11 | database=dg.EnvVar("DUCKDB_DATABASE"), 12 | ) 13 | 14 | if os.getenv("DAGSTER_ENVIRONMENT") == "prod": 15 | session = boto3.Session( 16 | aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), 17 | aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), 18 | region_name=os.getenv("AWS_REGION"), 19 | ) 20 | smart_open_config = {"client": session.client("s3")} 21 | else: 22 | smart_open_config = {} 23 | 24 | 25 | dbt_resource = DbtCliResource( 26 | project_dir=dbt_project, 27 | ) 28 | 29 | 30 | @dg.definitions 31 | def resources(): 32 | return dg.Definitions( 33 | resources={ 34 | "database": database_resource, 35 | "dbt": dbt_resource, 36 | }, 37 | ) 38 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/defs/resources.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import boto3 4 | import dagster as dg 5 | from dagster_dbt import DbtCliResource 6 | from dagster_duckdb import DuckDBResource 7 | 8 | from dagster_and_dbt.completed.lesson_5.defs.project import dbt_project 9 | 10 | database_resource = DuckDBResource( 11 | database=dg.EnvVar("DUCKDB_DATABASE"), 12 | ) 13 | 14 | if os.getenv("DAGSTER_ENVIRONMENT") == "prod": 15 | session = boto3.Session( 16 | aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), 17 | aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), 18 | region_name=os.getenv("AWS_REGION"), 19 | ) 20 | smart_open_config = {"client": session.client("s3")} 21 | else: 22 | smart_open_config = {} 23 | 24 | 25 | dbt_resource = DbtCliResource( 26 | project_dir=dbt_project, 27 | ) 28 | 29 | 30 | @dg.definitions 31 | def resources(): 32 | return dg.Definitions( 33 | resources={ 34 | "database": database_resource, 35 | "dbt": dbt_resource, 36 | }, 37 | ) 38 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/defs/resources.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import boto3 4 | import dagster as dg 5 | from dagster_dbt import DbtCliResource 6 | from dagster_duckdb import DuckDBResource 7 | 8 | from dagster_and_dbt.completed.lesson_6.defs.project import dbt_project 9 | 10 | database_resource = DuckDBResource( 11 | database=dg.EnvVar("DUCKDB_DATABASE"), 12 | ) 13 | 14 | if os.getenv("DAGSTER_ENVIRONMENT") == "prod": 15 | session = boto3.Session( 16 | aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), 17 | aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), 18 | region_name=os.getenv("AWS_REGION"), 19 | ) 20 | smart_open_config = {"client": session.client("s3")} 21 | else: 22 | smart_open_config = {} 23 | 24 | 25 | dbt_resource = DbtCliResource( 26 | project_dir=dbt_project, 27 | ) 28 | 29 | 30 | @dg.definitions 31 | def resources(): 32 | return dg.Definitions( 33 | resources={ 34 | "database": database_resource, 35 | "dbt": dbt_resource, 36 | }, 37 | ) 38 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/defs/resources.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import boto3 4 | import dagster as dg 5 | from dagster_dbt import DbtCliResource 6 | from dagster_duckdb import DuckDBResource 7 | 8 | from dagster_and_dbt.completed.lesson_7.defs.project import dbt_project 9 | 10 | database_resource = DuckDBResource( 11 | database=dg.EnvVar("DUCKDB_DATABASE"), 12 | ) 13 | 14 | if os.getenv("DAGSTER_ENVIRONMENT") == "prod": 15 | session = boto3.Session( 16 | aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), 17 | aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), 18 | region_name=os.getenv("AWS_REGION"), 19 | ) 20 | smart_open_config = {"client": session.client("s3")} 21 | else: 22 | smart_open_config = {} 23 | 24 | 25 | dbt_resource = DbtCliResource( 26 | project_dir=dbt_project, 27 | ) 28 | 29 | 30 | @dg.definitions 31 | def resources(): 32 | return dg.Definitions( 33 | resources={ 34 | "database": database_resource, 35 | "dbt": dbt_resource, 36 | }, 37 | ) 38 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_6/defs/jobs.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | from dagster_dbt import build_dbt_asset_selection 3 | 4 | from dagster_and_dbt.completed.lesson_6.defs.assets.dbt import dbt_analytics 5 | from dagster_and_dbt.completed.lesson_6.defs.partitions import weekly_partition 6 | 7 | trips_by_week = dg.AssetSelection.assets("trips_by_week") 8 | adhoc_request = dg.AssetSelection.assets("adhoc_request") 9 | 10 | dbt_trips_selection = build_dbt_asset_selection([dbt_analytics], "stg_trips+") 11 | 12 | # TODO: Fix job for different partitions 13 | # trip_update_job = dg.define_asset_job( 14 | # name="trip_update_job", 15 | # partitions_def=monthly_partition, 16 | # selection=dg.AssetSelection.all() 17 | # - trips_by_week 18 | # - adhoc_request 19 | # - dbt_trips_selection, 20 | # ) 21 | 22 | weekly_update_job = dg.define_asset_job( 23 | name="weekly_update_job", partitions_def=weekly_partition, selection=trips_by_week 24 | ) 25 | 26 | adhoc_request_job = dg.define_asset_job( 27 | name="adhoc_request_job", selection=adhoc_request 28 | ) 29 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_7/defs/jobs.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | from dagster_dbt import build_dbt_asset_selection 3 | 4 | from dagster_and_dbt.completed.lesson_7.defs.assets.dbt import dbt_analytics 5 | from dagster_and_dbt.completed.lesson_7.defs.partitions import weekly_partition 6 | 7 | trips_by_week = dg.AssetSelection.assets("trips_by_week") 8 | adhoc_request = dg.AssetSelection.assets("adhoc_request") 9 | 10 | dbt_trips_selection = build_dbt_asset_selection([dbt_analytics], "stg_trips+") 11 | 12 | # TODO: Fix job for different partitions 13 | # trip_update_job = dg.define_asset_job( 14 | # name="trip_update_job", 15 | # partitions_def=monthly_partition, 16 | # selection=dg.AssetSelection.all() 17 | # - trips_by_week 18 | # - adhoc_request 19 | # - dbt_trips_selection, 20 | # ) 21 | 22 | weekly_update_job = dg.define_asset_job( 23 | name="weekly_update_job", partitions_def=weekly_partition, selection=trips_by_week 24 | ) 25 | 26 | adhoc_request_job = dg.define_asset_job( 27 | name="adhoc_request_job", selection=adhoc_request 28 | ) 29 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_dbt/src/dagster_and_dbt/completed/lesson_5/defs/assets/dbt.py: -------------------------------------------------------------------------------- 1 | import dagster as dg 2 | from dagster_dbt import DagsterDbtTranslator, DbtCliResource, dbt_assets 3 | 4 | from dagster_and_dbt.completed.lesson_5.defs.project import dbt_project 5 | 6 | 7 | class CustomizedDagsterDbtTranslator(DagsterDbtTranslator): 8 | # SCREENSHOT: Comment out screenshots before "Group dbt models" 9 | def get_group_name(self, dbt_resource_props): 10 | return dbt_resource_props["fqn"][1] 11 | 12 | def get_asset_key(self, dbt_resource_props): 13 | resource_type = dbt_resource_props["resource_type"] 14 | name = dbt_resource_props["name"] 15 | if resource_type == "source": 16 | return dg.AssetKey(f"taxi_{name}") 17 | else: 18 | return super().get_asset_key(dbt_resource_props) 19 | 20 | 21 | @dbt_assets( 22 | manifest=dbt_project.manifest_path, 23 | dagster_dbt_translator=CustomizedDagsterDbtTranslator(), 24 | ) 25 | def dbt_analytics(context: dg.AssetExecutionContext, dbt: DbtCliResource): 26 | yield from dbt.cli(["run"], context=context).stream() 27 | -------------------------------------------------------------------------------- /dagster_university/dagster_and_etl/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "dagster-and-etl" 3 | version = "0.0.1" 4 | description = "Dagster ETL" 5 | readme = "README.md" 6 | requires-python = ">=3.10,<3.14" 7 | dependencies = [ 8 | "dagster==1.12.4", 9 | "dagster-dg-cli", 10 | "dagster-webserver", 11 | "dagster-duckdb", 12 | "dagster-dlt", 13 | "dagster-sling", 14 | "psycopg2-binary", 15 | "dlt[duckdb]", 16 | "pandas", 17 | ] 18 | 19 | [tool.uv] 20 | dev-dependencies = [ 21 | "ruff", 22 | "pytest", 23 | ] 24 | 25 | [build-system] 26 | requires = ["hatchling"] 27 | build-backend = "hatchling.build" 28 | 29 | [tool.dg] 30 | directory_type = "project" 31 | 32 | [tool.dg.project] 33 | root_module = "dagster_and_etl" 34 | registry_modules = [ 35 | "dagster_and_etl.components.*", 36 | ] 37 | 38 | [tool.ruff.lint] 39 | ignore = [ 40 | "ERA001", # comments are fine for a learning environment 41 | "E501", # Line too long for docstrings 42 | "F401", # unused imports already caught by pylance 43 | ] 44 | 45 | fixable = ["ALL"] 46 | unfixable = [ 47 | "F401", # dont remove unused imports 48 | ] -------------------------------------------------------------------------------- /course/pages/dagster-essentials/lesson-7/4-coding-practice-weekly-update-schedule.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Lesson 7: Practice: Create a weekly_update_schedule' 3 | module: 'dagster_essentials' 4 | lesson: '7' 5 | --- 6 | 7 | # Practice: Create a weekly_update_schedule 8 | 9 | To practice what you’ve learned, add a schedule to `src/dagster_essentials/defs/schedules.py` that: 10 | 11 | - Is named `weekly_update_schedule` 12 | - Materializes the `trips_by_week` asset 13 | - Runs every Monday at midnight 14 | 15 | --- 16 | 17 | ## Check your work 18 | 19 | The schedule you built should look similar to the following code. Click **View answer** to view it. 20 | 21 | **If there are differences**, compare what you wrote to the schedule below and change them, as this schedule will be used as-is in future lessons. 22 | 23 | ```python {% obfuscated="true" %} 24 | # src/dagster_essentials/defs/schedules.py 25 | import dagster as dg 26 | 27 | from dagster_essentials.defs.jobs import weekly_update_job 28 | 29 | weekly_update_schedule = dg.ScheduleDefinition( 30 | job=weekly_update_job, 31 | cron_schedule="0 0 * * 1", # every Monday at midnight 32 | ) 33 | ``` 34 | --------------------------------------------------------------------------------