├── .github └── FUNDING.yml ├── .gitignore ├── README.md ├── images └── dagster-practical-data-engineering-pipeline.png └── src ├── .gitkeep └── pipelines ├── .pylintrc └── real-estate ├── .tool-versions ├── Makefile ├── debugging.py ├── dev-requirements.txt ├── docker-compose.yml ├── notes.md ├── pyproject.toml ├── realestate ├── .gitignore ├── __init__.py ├── common │ ├── __init__.py │ ├── helper_functions.py │ ├── resource_delta_lake.py │ ├── resources.py │ ├── solids.py │ ├── solids_druid.py │ ├── solids_filehandle.py │ ├── solids_jupyter.py │ ├── solids_notebook.py │ ├── solids_scraping.py │ ├── solids_spark_delta.py │ ├── types.py │ └── types_realestate.py ├── config_environments │ ├── __init__.py │ └── local_base.yaml ├── config_pipelines │ ├── __init__.py │ └── scrape_realestate.yaml ├── notebooks │ ├── comprehensive-real-estate-data-exploration.ipynb │ ├── create initial property table.ipynb │ ├── get_changed_or_new_properties.ipynb │ ├── playing with immo24.ipynb │ ├── read immo24 and plot maps.ipynb │ └── templates │ │ ├── Immo - Hedonische Berechnung.ipynb │ │ ├── Pedro Marcelino - Comprehensive data exploration with Python.ipynb │ │ └── Yusuke Nakaichi - House Prices.ipynb ├── pipelines.py ├── resources.py └── resources_test.py ├── realestate_tests ├── __init__.py ├── create_property_delta.py ├── pipeline_tests.py ├── property.json ├── property2.json └── read_property_json_test.py ├── setup.cfg ├── setup.py └── tox.ini /.github/FUNDING.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/.github/FUNDING.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/README.md -------------------------------------------------------------------------------- /images/dagster-practical-data-engineering-pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/images/dagster-practical-data-engineering-pipeline.png -------------------------------------------------------------------------------- /src/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/pipelines/.pylintrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/.pylintrc -------------------------------------------------------------------------------- /src/pipelines/real-estate/.tool-versions: -------------------------------------------------------------------------------- 1 | python 3.11.7 2 | -------------------------------------------------------------------------------- /src/pipelines/real-estate/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/Makefile -------------------------------------------------------------------------------- /src/pipelines/real-estate/debugging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/debugging.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/dev-requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/dev-requirements.txt -------------------------------------------------------------------------------- /src/pipelines/real-estate/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/docker-compose.yml -------------------------------------------------------------------------------- /src/pipelines/real-estate/notes.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/notes.md -------------------------------------------------------------------------------- /src/pipelines/real-estate/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/pyproject.toml -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/.gitignore -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/__init__.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/common/__init__.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/common/helper_functions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/common/helper_functions.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/common/resource_delta_lake.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/common/resource_delta_lake.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/common/resources.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/common/resources.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/common/solids.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/common/solids.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/common/solids_druid.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/common/solids_druid.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/common/solids_filehandle.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/common/solids_filehandle.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/common/solids_jupyter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/common/solids_jupyter.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/common/solids_notebook.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/common/solids_notebook.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/common/solids_scraping.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/common/solids_scraping.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/common/solids_spark_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/common/solids_spark_delta.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/common/types.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/common/types.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/common/types_realestate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/common/types_realestate.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/config_environments/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/config_environments/local_base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/config_environments/local_base.yaml -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/config_pipelines/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/config_pipelines/scrape_realestate.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/config_pipelines/scrape_realestate.yaml -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/notebooks/comprehensive-real-estate-data-exploration.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/notebooks/comprehensive-real-estate-data-exploration.ipynb -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/notebooks/create initial property table.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/notebooks/create initial property table.ipynb -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/notebooks/get_changed_or_new_properties.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/notebooks/get_changed_or_new_properties.ipynb -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/notebooks/playing with immo24.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/notebooks/playing with immo24.ipynb -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/notebooks/read immo24 and plot maps.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/notebooks/read immo24 and plot maps.ipynb -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/notebooks/templates/Immo - Hedonische Berechnung.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/notebooks/templates/Immo - Hedonische Berechnung.ipynb -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/notebooks/templates/Pedro Marcelino - Comprehensive data exploration with Python.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/notebooks/templates/Pedro Marcelino - Comprehensive data exploration with Python.ipynb -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/notebooks/templates/Yusuke Nakaichi - House Prices.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/notebooks/templates/Yusuke Nakaichi - House Prices.ipynb -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/pipelines.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/resources.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/resources.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate/resources_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate/resources_test.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate_tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate_tests/create_property_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate_tests/create_property_delta.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate_tests/pipeline_tests.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate_tests/pipeline_tests.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate_tests/property.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate_tests/property.json -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate_tests/property2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate_tests/property2.json -------------------------------------------------------------------------------- /src/pipelines/real-estate/realestate_tests/read_property_json_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/realestate_tests/read_property_json_test.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/setup.cfg: -------------------------------------------------------------------------------- 1 | 2 | [metadata] 3 | name = realestate 4 | -------------------------------------------------------------------------------- /src/pipelines/real-estate/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/setup.py -------------------------------------------------------------------------------- /src/pipelines/real-estate/tox.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssp-data/practical-data-engineering/HEAD/src/pipelines/real-estate/tox.ini --------------------------------------------------------------------------------