├── .gitignore ├── 01_course_overview ├── overview.ipynb └── single_pane_of_glass.png ├── 02_negative_engineering_and_workflow_orchestration └── workflow_orchestration.ipynb ├── 03_prefect_and_basic_orchestration_features ├── basic_orchestration.ipynb ├── flow_code │ └── nike_flow.py └── work_queue_and_agent.ipynb ├── 04_parallel_execution └── introduction_to_dask.ipynb ├── 05_docker_and_python_packaging ├── docker_architecture.png ├── docker_with_custom_module │ ├── Dockerfile │ ├── components │ │ ├── __init__.py │ │ ├── componentA.py │ │ └── componentB.py │ ├── mypackage.egg-info │ │ ├── PKG-INFO │ │ ├── SOURCES.txt │ │ ├── dependency_links.txt │ │ ├── requires.txt │ │ └── top_level.txt │ ├── requirements.txt │ ├── setup.py │ └── workflow │ │ └── custom_flow.py └── packaging_and_docker.ipynb ├── 06_advanced_patterns_and_subflows ├── advanced_patterns.ipynb └── shoes_in_my_budget.md ├── 07_demo_orchestrator ├── airbyte_bootscript.sh ├── demo_orchestrator.ipynb ├── imgs │ ├── airbyte-connectors.png │ ├── airbyte.png │ ├── airbyte.webp │ ├── dbt-logo.png │ ├── dbt-schematic.png │ ├── logos.png │ ├── marvin.png │ ├── prefect-orca.svg │ ├── snowflake-logo.webp │ └── snowflake-schematic.png └── my_dbt_project │ ├── .gitignore │ ├── README.md │ ├── analyses │ └── .gitkeep │ ├── dbt_project.yml │ ├── macros │ ├── .gitkeep │ └── airbyte_json_column_grab.sql │ ├── models │ └── demo │ │ ├── epidemic.yml │ │ └── stats.sql │ ├── seeds │ └── .gitkeep │ ├── snapshots │ └── .gitkeep │ └── tests │ └── .gitkeep ├── README.md └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/.gitignore -------------------------------------------------------------------------------- /01_course_overview/overview.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/01_course_overview/overview.ipynb -------------------------------------------------------------------------------- /01_course_overview/single_pane_of_glass.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/01_course_overview/single_pane_of_glass.png -------------------------------------------------------------------------------- /02_negative_engineering_and_workflow_orchestration/workflow_orchestration.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/02_negative_engineering_and_workflow_orchestration/workflow_orchestration.ipynb -------------------------------------------------------------------------------- /03_prefect_and_basic_orchestration_features/basic_orchestration.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/03_prefect_and_basic_orchestration_features/basic_orchestration.ipynb -------------------------------------------------------------------------------- /03_prefect_and_basic_orchestration_features/flow_code/nike_flow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/03_prefect_and_basic_orchestration_features/flow_code/nike_flow.py -------------------------------------------------------------------------------- /03_prefect_and_basic_orchestration_features/work_queue_and_agent.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/03_prefect_and_basic_orchestration_features/work_queue_and_agent.ipynb -------------------------------------------------------------------------------- /04_parallel_execution/introduction_to_dask.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/04_parallel_execution/introduction_to_dask.ipynb -------------------------------------------------------------------------------- /05_docker_and_python_packaging/docker_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/05_docker_and_python_packaging/docker_architecture.png -------------------------------------------------------------------------------- /05_docker_and_python_packaging/docker_with_custom_module/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/05_docker_and_python_packaging/docker_with_custom_module/Dockerfile -------------------------------------------------------------------------------- /05_docker_and_python_packaging/docker_with_custom_module/components/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /05_docker_and_python_packaging/docker_with_custom_module/components/componentA.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/05_docker_and_python_packaging/docker_with_custom_module/components/componentA.py -------------------------------------------------------------------------------- /05_docker_and_python_packaging/docker_with_custom_module/components/componentB.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/05_docker_and_python_packaging/docker_with_custom_module/components/componentB.py -------------------------------------------------------------------------------- /05_docker_and_python_packaging/docker_with_custom_module/mypackage.egg-info/PKG-INFO: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/05_docker_and_python_packaging/docker_with_custom_module/mypackage.egg-info/PKG-INFO -------------------------------------------------------------------------------- /05_docker_and_python_packaging/docker_with_custom_module/mypackage.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/05_docker_and_python_packaging/docker_with_custom_module/mypackage.egg-info/SOURCES.txt -------------------------------------------------------------------------------- /05_docker_and_python_packaging/docker_with_custom_module/mypackage.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /05_docker_and_python_packaging/docker_with_custom_module/mypackage.egg-info/requires.txt: -------------------------------------------------------------------------------- 1 | pandas 2 | numpy 3 | -------------------------------------------------------------------------------- /05_docker_and_python_packaging/docker_with_custom_module/mypackage.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | components 2 | -------------------------------------------------------------------------------- /05_docker_and_python_packaging/docker_with_custom_module/requirements.txt: -------------------------------------------------------------------------------- 1 | pandas 2 | numpy 3 | -------------------------------------------------------------------------------- /05_docker_and_python_packaging/docker_with_custom_module/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/05_docker_and_python_packaging/docker_with_custom_module/setup.py -------------------------------------------------------------------------------- /05_docker_and_python_packaging/docker_with_custom_module/workflow/custom_flow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/05_docker_and_python_packaging/docker_with_custom_module/workflow/custom_flow.py -------------------------------------------------------------------------------- /05_docker_and_python_packaging/packaging_and_docker.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/05_docker_and_python_packaging/packaging_and_docker.ipynb -------------------------------------------------------------------------------- /06_advanced_patterns_and_subflows/advanced_patterns.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/06_advanced_patterns_and_subflows/advanced_patterns.ipynb -------------------------------------------------------------------------------- /06_advanced_patterns_and_subflows/shoes_in_my_budget.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/06_advanced_patterns_and_subflows/shoes_in_my_budget.md -------------------------------------------------------------------------------- /07_demo_orchestrator/airbyte_bootscript.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/airbyte_bootscript.sh -------------------------------------------------------------------------------- /07_demo_orchestrator/demo_orchestrator.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/demo_orchestrator.ipynb -------------------------------------------------------------------------------- /07_demo_orchestrator/imgs/airbyte-connectors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/imgs/airbyte-connectors.png -------------------------------------------------------------------------------- /07_demo_orchestrator/imgs/airbyte.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/imgs/airbyte.png -------------------------------------------------------------------------------- /07_demo_orchestrator/imgs/airbyte.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/imgs/airbyte.webp -------------------------------------------------------------------------------- /07_demo_orchestrator/imgs/dbt-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/imgs/dbt-logo.png -------------------------------------------------------------------------------- /07_demo_orchestrator/imgs/dbt-schematic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/imgs/dbt-schematic.png -------------------------------------------------------------------------------- /07_demo_orchestrator/imgs/logos.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/imgs/logos.png -------------------------------------------------------------------------------- /07_demo_orchestrator/imgs/marvin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/imgs/marvin.png -------------------------------------------------------------------------------- /07_demo_orchestrator/imgs/prefect-orca.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/imgs/prefect-orca.svg -------------------------------------------------------------------------------- /07_demo_orchestrator/imgs/snowflake-logo.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/imgs/snowflake-logo.webp -------------------------------------------------------------------------------- /07_demo_orchestrator/imgs/snowflake-schematic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/imgs/snowflake-schematic.png -------------------------------------------------------------------------------- /07_demo_orchestrator/my_dbt_project/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | -------------------------------------------------------------------------------- /07_demo_orchestrator/my_dbt_project/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/my_dbt_project/README.md -------------------------------------------------------------------------------- /07_demo_orchestrator/my_dbt_project/analyses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /07_demo_orchestrator/my_dbt_project/dbt_project.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/my_dbt_project/dbt_project.yml -------------------------------------------------------------------------------- /07_demo_orchestrator/my_dbt_project/macros/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /07_demo_orchestrator/my_dbt_project/macros/airbyte_json_column_grab.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/my_dbt_project/macros/airbyte_json_column_grab.sql -------------------------------------------------------------------------------- /07_demo_orchestrator/my_dbt_project/models/demo/epidemic.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/my_dbt_project/models/demo/epidemic.yml -------------------------------------------------------------------------------- /07_demo_orchestrator/my_dbt_project/models/demo/stats.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/07_demo_orchestrator/my_dbt_project/models/demo/stats.sql -------------------------------------------------------------------------------- /07_demo_orchestrator/my_dbt_project/seeds/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /07_demo_orchestrator/my_dbt_project/snapshots/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /07_demo_orchestrator/my_dbt_project/tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/README.md -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzstoatzz/oreilly-workflow-orchestration/HEAD/requirements.txt --------------------------------------------------------------------------------