├── .appveyor.yml ├── .bumpversion.cfg ├── .cookiecutterrc ├── .coveragerc ├── .editorconfig ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── pull_request_template.md └── workflows │ └── build.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .prospector.yaml ├── .readthedocs.yml ├── AUTHORS.rst ├── CHANGELOG.rst ├── CONTRIBUTING.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── ci ├── appveyor-with-compiler.cmd ├── bootstrap.py ├── requirements.txt └── templates │ ├── .appveyor.yml │ ├── .travis.yml │ └── tox.ini ├── docs ├── _templates │ ├── module.rst_t │ ├── package.rst_t │ └── toc.rst_t ├── about.rst ├── authors.rst ├── changelog.rst ├── conf.py ├── contributing.rst ├── data.rst ├── data │ ├── DLR.rst │ ├── DSM.rst │ ├── batteries.rst │ ├── e-mobility.rst │ ├── electricity_demand.rst │ ├── electricity_grids.rst │ ├── electricity_supply.rst │ ├── gas_demand.rst │ ├── gas_grids.rst │ ├── gas_stores.rst │ ├── gas_supply.rst │ ├── heat_demand.rst │ ├── heat_stores.rst │ ├── heat_supply.rst │ ├── input_data.rst │ └── mobility_demand.rst ├── getting_started.rst ├── images │ ├── 100RE_gasgrid_enTitle_00.png │ ├── 2035_gasgrid_enTitle_00.png │ ├── DP_Workflow_15012021.svg │ ├── DSM_potential.png │ ├── Erzeugerpark.png │ ├── GasGen_eGon2035.png │ ├── H2_underground_stores_withTitle.png │ ├── Logos_Projektpartner_egon_data.png │ ├── PV_freiflaeche.png │ ├── S27-3.png │ ├── S27-4a.png │ ├── Toolchain_web_desktop.svg │ ├── combined_heat_and_power_plants.png │ ├── ding0_mv_lv_grid.png │ ├── district_heating_areas.png │ ├── eGon2035_gas_ind_load_repartition_DE_withTitle.png │ ├── eGon_emob_MIT_model.png │ ├── egon-modell-szenario-egon2035.png │ ├── flowchart_cts_disaggregation.jpg │ ├── grid_topology_ehv_hv.png │ ├── ind_gas_demand.png │ ├── offshore_power_plants_areas.png │ ├── regions_DLR.png │ ├── residential_heat_demand.png │ ├── residential_heat_demand_profile_summer.png │ ├── residential_heat_demand_profile_winter.png │ ├── shifted_dsm-example.png │ └── table_max_capacity_DLR.png ├── index.rst ├── literature.rst ├── reference │ ├── egon.data.airflow.dags.pipeline.rst │ ├── egon.data.airflow.dags.rst │ ├── egon.data.airflow.rst │ ├── egon.data.cli.rst │ ├── egon.data.config.rst │ ├── egon.data.datasets.DSM_cts_ind.rst │ ├── egon.data.datasets.calculate_dlr.rst │ ├── egon.data.datasets.ch4_prod.rst │ ├── egon.data.datasets.ch4_storages.rst │ ├── egon.data.datasets.chp.match_nep.rst │ ├── egon.data.datasets.chp.rst │ ├── egon.data.datasets.chp.small_chp.rst │ ├── egon.data.datasets.chp_etrago.rst │ ├── egon.data.datasets.data_bundle.rst │ ├── egon.data.datasets.database.rst │ ├── egon.data.datasets.demandregio.install_disaggregator.rst │ ├── egon.data.datasets.demandregio.rst │ ├── egon.data.datasets.district_heating_areas.plot.rst │ ├── egon.data.datasets.district_heating_areas.rst │ ├── egon.data.datasets.electrical_neighbours.rst │ ├── egon.data.datasets.electricity_demand.rst │ ├── egon.data.datasets.electricity_demand.temporal.rst │ ├── egon.data.datasets.electricity_demand_etrago.rst │ ├── egon.data.datasets.electricity_demand_timeseries.cts_buildings.rst │ ├── egon.data.datasets.electricity_demand_timeseries.hh_buildings.rst │ ├── egon.data.datasets.electricity_demand_timeseries.hh_profiles.rst │ ├── egon.data.datasets.electricity_demand_timeseries.mapping.rst │ ├── egon.data.datasets.electricity_demand_timeseries.rst │ ├── egon.data.datasets.electricity_demand_timeseries.tools.rst │ ├── egon.data.datasets.emobility.heavy_duty_transport.create_h2_buses.rst │ ├── egon.data.datasets.emobility.heavy_duty_transport.data_io.rst │ ├── egon.data.datasets.emobility.heavy_duty_transport.db_classes.rst │ ├── egon.data.datasets.emobility.heavy_duty_transport.h2_demand_distribution.rst │ ├── egon.data.datasets.emobility.heavy_duty_transport.rst │ ├── egon.data.datasets.emobility.motorized_individual_travel.db_classes.rst │ ├── egon.data.datasets.emobility.motorized_individual_travel.ev_allocation.rst │ ├── egon.data.datasets.emobility.motorized_individual_travel.helpers.rst │ ├── egon.data.datasets.emobility.motorized_individual_travel.model_timeseries.rst │ ├── egon.data.datasets.emobility.motorized_individual_travel.rst │ ├── egon.data.datasets.emobility.motorized_individual_travel.tests.rst │ ├── egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure.db_classes.rst │ ├── egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure.infrastructure_allocation.rst │ ├── egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure.rst │ ├── egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure.use_cases.rst │ ├── egon.data.datasets.emobility.rst │ ├── egon.data.datasets.era5.rst │ ├── egon.data.datasets.etrago_helpers.rst │ ├── egon.data.datasets.etrago_setup.rst │ ├── egon.data.datasets.fill_etrago_gen.rst │ ├── egon.data.datasets.fix_ehv_subnetworks.rst │ ├── egon.data.datasets.gas_areas.rst │ ├── egon.data.datasets.gas_grid.rst │ ├── egon.data.datasets.gas_neighbours.eGon100RE.rst │ ├── egon.data.datasets.gas_neighbours.eGon2035.rst │ ├── egon.data.datasets.gas_neighbours.gas_abroad.rst │ ├── egon.data.datasets.gas_neighbours.rst │ ├── egon.data.datasets.generate_voronoi.rst │ ├── egon.data.datasets.heat_demand.rst │ ├── egon.data.datasets.heat_demand_europe.rst │ ├── egon.data.datasets.heat_demand_timeseries.daily.rst │ ├── egon.data.datasets.heat_demand_timeseries.idp_pool.rst │ ├── egon.data.datasets.heat_demand_timeseries.rst │ ├── egon.data.datasets.heat_demand_timeseries.service_sector.rst │ ├── egon.data.datasets.heat_etrago.hts_etrago.rst │ ├── egon.data.datasets.heat_etrago.power_to_heat.rst │ ├── egon.data.datasets.heat_etrago.rst │ ├── egon.data.datasets.heat_supply.district_heating.rst │ ├── egon.data.datasets.heat_supply.geothermal.rst │ ├── egon.data.datasets.heat_supply.individual_heating.rst │ ├── egon.data.datasets.heat_supply.rst │ ├── egon.data.datasets.hydrogen_etrago.bus.rst │ ├── egon.data.datasets.hydrogen_etrago.h2_grid.rst │ ├── egon.data.datasets.hydrogen_etrago.h2_to_ch4.rst │ ├── egon.data.datasets.hydrogen_etrago.power_to_h2.rst │ ├── egon.data.datasets.hydrogen_etrago.rst │ ├── egon.data.datasets.hydrogen_etrago.storage.rst │ ├── egon.data.datasets.industrial_gas_demand.rst │ ├── egon.data.datasets.industrial_sites.rst │ ├── egon.data.datasets.industry.rst │ ├── egon.data.datasets.industry.temporal.rst │ ├── egon.data.datasets.loadarea.rst │ ├── egon.data.datasets.low_flex_scenario.rst │ ├── egon.data.datasets.mastr.rst │ ├── egon.data.datasets.mv_grid_districts.rst │ ├── egon.data.datasets.osm.rst │ ├── egon.data.datasets.osm_buildings_streets.rst │ ├── egon.data.datasets.osmtgmod.rst │ ├── egon.data.datasets.osmtgmod.substation.rst │ ├── egon.data.datasets.power_etrago.match_ocgt.rst │ ├── egon.data.datasets.power_etrago.rst │ ├── egon.data.datasets.power_plants.assign_weather_data.rst │ ├── egon.data.datasets.power_plants.conventional.rst │ ├── egon.data.datasets.power_plants.mastr.rst │ ├── egon.data.datasets.power_plants.pv_ground_mounted.rst │ ├── egon.data.datasets.power_plants.pv_rooftop.rst │ ├── egon.data.datasets.power_plants.pv_rooftop_buildings.rst │ ├── egon.data.datasets.power_plants.rst │ ├── egon.data.datasets.power_plants.wind_farms.rst │ ├── egon.data.datasets.power_plants.wind_offshore.rst │ ├── egon.data.datasets.pypsaeursec.rst │ ├── egon.data.datasets.re_potential_areas.rst │ ├── egon.data.datasets.renewable_feedin.rst │ ├── egon.data.datasets.rst │ ├── egon.data.datasets.saltcavern.rst │ ├── egon.data.datasets.sanity_checks.rst │ ├── egon.data.datasets.scenario_capacities.rst │ ├── egon.data.datasets.scenario_parameters.parameters.rst │ ├── egon.data.datasets.scenario_parameters.rst │ ├── egon.data.datasets.society_prognosis.rst │ ├── egon.data.datasets.storages.home_batteries.rst │ ├── egon.data.datasets.storages.pumped_hydro.rst │ ├── egon.data.datasets.storages.rst │ ├── egon.data.datasets.storages_etrago.rst │ ├── egon.data.datasets.substation.rst │ ├── egon.data.datasets.substation_voronoi.rst │ ├── egon.data.datasets.tyndp.rst │ ├── egon.data.datasets.vg250.rst │ ├── egon.data.datasets.vg250_mv_grid_districts.rst │ ├── egon.data.datasets.zensus.rst │ ├── egon.data.datasets.zensus_mv_grid_districts.rst │ ├── egon.data.datasets.zensus_vg250.rst │ ├── egon.data.db.rst │ ├── egon.data.metadata.rst │ ├── egon.data.rst │ └── egon.data.subprocess.rst ├── requirements.txt ├── spelling_wordlist.txt ├── troubleshooting.rst └── workflow.rst ├── pyproject.toml ├── setup.cfg ├── setup.py ├── src └── egon │ └── data │ ├── __init__.py │ ├── __main__.py │ ├── airflow │ ├── Dockerfile.postgis │ ├── __init__.py │ ├── airflow.cfg │ ├── dags │ │ ├── pipeline.py │ │ └── pipeline_status_quo.py │ ├── docker-compose.yml │ └── webserver_config.py │ ├── cli.py │ ├── config.py │ ├── datasets.yml │ ├── datasets │ ├── DSM_cts_ind.py │ ├── __init__.py │ ├── calculate_dlr.py │ ├── ch4_prod.py │ ├── ch4_storages.py │ ├── chp │ │ ├── __init__.py │ │ ├── match_nep.py │ │ └── small_chp.py │ ├── chp_etrago.py │ ├── data_bundle │ │ └── __init__.py │ ├── database.py │ ├── demandregio │ │ ├── __init__.py │ │ └── install_disaggregator.py │ ├── district_heating_areas │ │ ├── __init__.py │ │ └── plot.py │ ├── electrical_neighbours.py │ ├── electricity_demand │ │ ├── __init__.py │ │ └── temporal.py │ ├── electricity_demand_etrago.py │ ├── electricity_demand_timeseries │ │ ├── __init__.py │ │ ├── cts_buildings.py │ │ ├── hh_buildings.py │ │ ├── hh_profiles.py │ │ ├── mapping.py │ │ └── tools.py │ ├── emobility │ │ ├── __init__.py │ │ ├── heavy_duty_transport │ │ │ ├── __init__.py │ │ │ ├── create_h2_buses.py │ │ │ ├── data_io.py │ │ │ ├── db_classes.py │ │ │ └── h2_demand_distribution.py │ │ ├── motorized_individual_travel │ │ │ ├── __init__.py │ │ │ ├── db_classes.py │ │ │ ├── ev_allocation.py │ │ │ ├── helpers.py │ │ │ ├── model_timeseries.py │ │ │ └── tests.py │ │ └── motorized_individual_travel_charging_infrastructure │ │ │ ├── __init__.py │ │ │ ├── db_classes.py │ │ │ ├── infrastructure_allocation.py │ │ │ └── use_cases.py │ ├── era5.py │ ├── etrago_helpers.py │ ├── etrago_setup.py │ ├── fill_etrago_gen.py │ ├── fix_ehv_subnetworks.py │ ├── gas_areas.py │ ├── gas_grid.py │ ├── gas_neighbours │ │ ├── __init__.py │ │ ├── eGon100RE.py │ │ ├── eGon2035.py │ │ └── gas_abroad.py │ ├── generate_voronoi.py │ ├── heat_demand │ │ ├── __init__.py │ │ └── raster2cells-and-centroids.sql │ ├── heat_demand_europe.py │ ├── heat_demand_timeseries │ │ ├── __init__.py │ │ ├── daily.py │ │ ├── idp_pool.py │ │ └── service_sector.py │ ├── heat_etrago │ │ ├── __init__.py │ │ ├── hts_etrago.py │ │ └── power_to_heat.py │ ├── heat_supply │ │ ├── __init__.py │ │ ├── district_heating.py │ │ ├── geothermal.py │ │ └── individual_heating.py │ ├── hydrogen_etrago │ │ ├── __init__.py │ │ ├── bus.py │ │ ├── h2_grid.py │ │ ├── h2_to_ch4.py │ │ ├── power_to_h2.py │ │ └── storage.py │ ├── industrial_gas_demand.py │ ├── industrial_sites │ │ └── __init__.py │ ├── industry │ │ ├── __init__.py │ │ └── temporal.py │ ├── loadarea │ │ ├── __init__.py │ │ ├── census_cells_melt.sql │ │ ├── drop_temp_tables.sql │ │ ├── loadareas_add_demand_cts.sql │ │ ├── loadareas_add_demand_hh.sql │ │ ├── loadareas_add_demand_ind.sql │ │ ├── loadareas_create.sql │ │ ├── osm_landuse_census_cells_melt.sql │ │ ├── osm_landuse_extraction.sql │ │ └── osm_landuse_melt.sql │ ├── low_flex_scenario │ │ ├── __init__.py │ │ └── low_flex_eGon2035.sql │ ├── mastr.py │ ├── mv_grid_districts.py │ ├── osm │ │ ├── __init__.py │ │ └── oedb.style │ ├── osm_buildings_streets │ │ ├── __init__.py │ │ ├── drop_temp_tables.sql │ │ ├── osm_amenities_shops_preprocessing.sql │ │ ├── osm_buildings_extend_residential.sql │ │ ├── osm_buildings_extract.sql │ │ ├── osm_buildings_filter.sql │ │ ├── osm_buildings_filter_amenities.sql │ │ ├── osm_buildings_filter_residential.sql │ │ ├── osm_buildings_filtered_all_zensus_mapping.sql │ │ ├── osm_buildings_filtered_zensus_mapping.sql │ │ ├── osm_buildings_residential_zensus_mapping.sql │ │ ├── osm_buildings_temp_tables.sql │ │ ├── osm_results_amenities.sql │ │ ├── osm_results_buildings_w_amenities.sql │ │ ├── osm_results_buildings_wo_amenities.sql │ │ └── osm_ways_preprocessing.sql │ ├── osmtgmod │ │ ├── __init__.py │ │ └── substation.py │ ├── power_etrago │ │ ├── __init__.py │ │ └── match_ocgt.py │ ├── power_plants │ │ ├── __init__.py │ │ ├── assign_weather_data.py │ │ ├── conventional.py │ │ ├── mastr.py │ │ ├── mastr_db_classes.py │ │ ├── metadata.py │ │ ├── pv_ground_mounted.py │ │ ├── pv_rooftop.py │ │ ├── pv_rooftop_buildings.py │ │ ├── wind_farms.py │ │ └── wind_offshore.py │ ├── pypsaeur │ │ ├── Snakefile │ │ ├── __init__.py │ │ ├── config.yaml │ │ ├── config_eGon100RE_overnight.yaml │ │ ├── config_myopic.yaml │ │ ├── config_prepare.yaml │ │ ├── config_solve.yaml │ │ └── custom_extra_functionality.py │ ├── re_potential_areas │ │ └── __init__.py │ ├── renewable_feedin.py │ ├── saltcavern │ │ └── __init__.py │ ├── sanity_checks.py │ ├── scenario_capacities.py │ ├── scenario_parameters │ │ ├── __init__.py │ │ └── parameters.py │ ├── scenario_path │ │ ├── __init__.py │ │ └── import_status2019.py │ ├── society_prognosis.py │ ├── storages │ │ ├── __init__.py │ │ ├── home_batteries.py │ │ └── pumped_hydro.py │ ├── storages_etrago │ │ └── __init__.py │ ├── substation │ │ ├── __init__.py │ │ ├── ehv_substation.sql │ │ └── hvmv_substation.sql │ ├── substation_voronoi.py │ ├── tyndp.py │ ├── vg250 │ │ ├── __init__.py │ │ ├── cleaning_and_preparation.sql │ │ └── vg250_lan_nuts_id_mview.sql │ ├── vg250_mv_grid_districts.py │ ├── zensus │ │ ├── __init__.py │ │ └── create_combined_zensus_table.sql │ ├── zensus_mv_grid_districts.py │ └── zensus_vg250.py │ ├── datasets_original.yml │ ├── db.py │ ├── metadata │ ├── __init__.py │ ├── boundaries.egon_map_zensus_climate_zones.json │ ├── boundaries.egon_map_zensus_mvgd_buildings.json │ ├── boundaries.egon_map_zensus_weather_cell.json │ ├── demand.egon_building_electricity_peak_loads.json │ ├── demand.egon_building_heat_peak_loads.json │ ├── demand.egon_cts_electricity_demand_building_share.json │ ├── demand.egon_cts_heat_demand_building_share.json │ ├── demand.egon_daily_heat_demand_per_climate_zone.json │ ├── demand.egon_etrago_electricity_cts.json │ ├── demand.egon_etrago_electricity_households.json │ ├── demand.egon_etrago_heat_cts.json │ ├── demand.egon_heat_idp_pool.json │ ├── demand.egon_heat_timeseries_selected_profiles.json │ ├── demand.egon_household_electricity_profile_in_census_cell.json │ ├── demand.egon_household_electricity_profile_of_buildings.json │ ├── demand.iee_household_load_profiles.json │ ├── grid.egon_mv_grid_district.json │ ├── openstreetmap.osm_buildings_filtered.json │ ├── openstreetmap.osm_buildings_synthetic.json │ ├── society.destatis_zensus_population_per_ha.json │ └── society.egon_destatis_zensus_apartment_building_population_per_ha.json │ └── subprocess.py ├── tests ├── test_dataset_class.py └── test_egon-data.py └── tox.ini /.appveyor.yml: -------------------------------------------------------------------------------- 1 | version: '{branch}-{build}' 2 | build: off 3 | environment: 4 | global: 5 | COVERALLS_EXTRAS: '-v' 6 | COVERALLS_REPO_TOKEN: dnzlyTHfE7lWVrBEdhR1DBfXiYDCSISvl 7 | matrix: 8 | - TOXENV: check 9 | TOXPYTHON: C:\Python36\python.exe 10 | PYTHON_HOME: C:\Python36 11 | PYTHON_VERSION: '3.6' 12 | PYTHON_ARCH: '32' 13 | - TOXENV: py36-cover,codecov,coveralls 14 | TOXPYTHON: C:\Python36\python.exe 15 | PYTHON_HOME: C:\Python36 16 | PYTHON_VERSION: '3.6' 17 | PYTHON_ARCH: '32' 18 | - TOXENV: py36-cover,codecov,coveralls 19 | TOXPYTHON: C:\Python36-x64\python.exe 20 | PYTHON_HOME: C:\Python36-x64 21 | PYTHON_VERSION: '3.6' 22 | PYTHON_ARCH: '64' 23 | - TOXENV: py36-nocov 24 | TOXPYTHON: C:\Python36\python.exe 25 | PYTHON_HOME: C:\Python36 26 | PYTHON_VERSION: '3.6' 27 | PYTHON_ARCH: '32' 28 | WHEEL_PATH: .tox/dist 29 | - TOXENV: py36-nocov 30 | TOXPYTHON: C:\Python36-x64\python.exe 31 | PYTHON_HOME: C:\Python36-x64 32 | PYTHON_VERSION: '3.6' 33 | PYTHON_ARCH: '64' 34 | WHEEL_PATH: .tox/dist 35 | - TOXENV: py37-cover,codecov,coveralls 36 | TOXPYTHON: C:\Python37\python.exe 37 | PYTHON_HOME: C:\Python37 38 | PYTHON_VERSION: '3.7' 39 | PYTHON_ARCH: '32' 40 | - TOXENV: py37-cover,codecov,coveralls 41 | TOXPYTHON: C:\Python37-x64\python.exe 42 | PYTHON_HOME: C:\Python37-x64 43 | PYTHON_VERSION: '3.7' 44 | PYTHON_ARCH: '64' 45 | - TOXENV: py37-nocov 46 | TOXPYTHON: C:\Python37\python.exe 47 | PYTHON_HOME: C:\Python37 48 | PYTHON_VERSION: '3.7' 49 | PYTHON_ARCH: '32' 50 | WHEEL_PATH: .tox/dist 51 | - TOXENV: py37-nocov 52 | TOXPYTHON: C:\Python37-x64\python.exe 53 | PYTHON_HOME: C:\Python37-x64 54 | PYTHON_VERSION: '3.7' 55 | PYTHON_ARCH: '64' 56 | WHEEL_PATH: .tox/dist 57 | - TOXENV: py38-cover,codecov,coveralls 58 | TOXPYTHON: C:\Python38\python.exe 59 | PYTHON_HOME: C:\Python38 60 | PYTHON_VERSION: '3.8' 61 | PYTHON_ARCH: '32' 62 | - TOXENV: py38-cover,codecov,coveralls 63 | TOXPYTHON: C:\Python38-x64\python.exe 64 | PYTHON_HOME: C:\Python38-x64 65 | PYTHON_VERSION: '3.8' 66 | PYTHON_ARCH: '64' 67 | - TOXENV: py38-nocov 68 | TOXPYTHON: C:\Python38\python.exe 69 | PYTHON_HOME: C:\Python38 70 | PYTHON_VERSION: '3.8' 71 | PYTHON_ARCH: '32' 72 | WHEEL_PATH: .tox/dist 73 | - TOXENV: py38-nocov 74 | TOXPYTHON: C:\Python38-x64\python.exe 75 | PYTHON_HOME: C:\Python38-x64 76 | PYTHON_VERSION: '3.8' 77 | PYTHON_ARCH: '64' 78 | WHEEL_PATH: .tox/dist 79 | init: 80 | - ps: echo $env:TOXENV 81 | - ps: ls C:\Python* 82 | install: 83 | - '%PYTHON_HOME%\python -mpip install --progress-bar=off tox -rci/requirements.txt' 84 | - '%PYTHON_HOME%\Scripts\virtualenv --version' 85 | - '%PYTHON_HOME%\Scripts\easy_install --version' 86 | - '%PYTHON_HOME%\Scripts\pip --version' 87 | - '%PYTHON_HOME%\Scripts\tox --version' 88 | test_script: 89 | - cmd /E:ON /V:ON /C .\ci\appveyor-with-compiler.cmd %PYTHON_HOME%\Scripts\tox 90 | on_failure: 91 | - ps: dir "env:" 92 | - ps: get-content .tox\*\log\* 93 | 94 | ### To enable remote debugging uncomment this (also, see: http://www.appveyor.com/docs/how-to/rdp-to-build-worker): 95 | # on_finish: 96 | # - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) 97 | -------------------------------------------------------------------------------- /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.0.0 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.py] 7 | search = version='{current_version}' 8 | replace = version='{new_version}' 9 | 10 | [bumpversion:file:README.rst] 11 | search = v{current_version}. 12 | replace = v{new_version}. 13 | 14 | [bumpversion:file:docs/conf.py] 15 | search = version = release = '{current_version}' 16 | replace = version = release = '{new_version}' 17 | 18 | [bumpversion:file:src/egon.data/__init__.py] 19 | search = __version__ = '{current_version}' 20 | replace = __version__ = '{new_version}' 21 | -------------------------------------------------------------------------------- /.cookiecutterrc: -------------------------------------------------------------------------------- 1 | # This file exists so you can easily regenerate your project. 2 | # 3 | # `cookiepatcher` is a convenient shim around `cookiecutter` 4 | # for regenerating projects (it will generate a .cookiecutterrc 5 | # automatically for any template). To use it: 6 | # 7 | # pip install cookiepatcher 8 | # cookiepatcher gh:ionelmc/cookiecutter-pylibrary eGon-data 9 | # 10 | # See: 11 | # https://pypi.org/project/cookiepatcher 12 | # 13 | # Alternatively, you can run: 14 | # 15 | # cookiecutter --overwrite-if-exists --config-file=eGon-data/.cookiecutterrc gh:ionelmc/cookiecutter-pylibrary 16 | 17 | default_context: 18 | 19 | _extensions: ['jinja2_time.TimeExtension'] 20 | _template: 'gh:ionelmc/cookiecutter-pylibrary' 21 | allow_tests_inside_package: 'yes' 22 | appveyor: 'yes' 23 | c_extension_function: 'longest' 24 | c_extension_module: '_egon.data' 25 | c_extension_optional: 'yes' 26 | c_extension_support: 'no' 27 | c_extension_test_pypi: 'no' 28 | c_extension_test_pypi_username: 'openego' 29 | codacy: 'yes' 30 | codacy_projectid: 'd639ac4296a04edb8da5c882ea36e98b' 31 | codeclimate: 'yes' 32 | codecov: 'yes' 33 | command_line_interface: 'click' 34 | command_line_interface_bin_name: 'egon-data' 35 | coveralls: 'yes' 36 | coveralls_token: 'dnzlyTHfE7lWVrBEdhR1DBfXiYDCSISvl' 37 | distribution_name: 'egon.data' 38 | email: 'guido.plessmann@rl-institut.de, ilka.cussmann@hs-flensburg.de, stephan.guenther@ovgu.de' 39 | full_name: 'Guido Pleßmann, Ilka Cußman, Stephan Günther' 40 | landscape: 'no' 41 | license: 'GNU Lesser General Public License v3 or later (LGPLv3+)' 42 | linter: 'flake8' 43 | package_name: 'egon.data' 44 | pre_commit: 'yes' 45 | project_name: 'eGo^N Data' 46 | project_short_description: 'The data used in the eGo^N project along with the code importing, generating and processing it.' 47 | pypi_badge: 'yes' 48 | pypi_disable_upload: 'no' 49 | release_date: '2020-10-13' 50 | repo_hosting: 'github.com' 51 | repo_hosting_domain: 'github.com' 52 | repo_name: 'eGon-data' 53 | repo_username: 'openego' 54 | requiresio: 'yes' 55 | scrutinizer: 'yes' 56 | setup_py_uses_setuptools_scm: 'no' 57 | setup_py_uses_test_runner: 'no' 58 | sphinx_docs: 'yes' 59 | sphinx_docs_hosting: 'https://eGon-data.readthedocs.io/' 60 | sphinx_doctest: 'no' 61 | sphinx_theme: 'sphinx-rtd-theme' 62 | test_matrix_configurator: 'yes' 63 | test_matrix_separate_coverage: 'yes' 64 | test_runner: 'pytest' 65 | travis: 'yes' 66 | travis_osx: 'yes' 67 | version: '0.0.0' 68 | website: 'https://github.com/openego/eGon-data' 69 | year_from: '2020' 70 | year_to: '2022' 71 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [paths] 2 | source = src 3 | 4 | [run] 5 | branch = true 6 | source = 7 | src 8 | tests 9 | parallel = true 10 | 11 | [report] 12 | show_missing = true 13 | precision = 2 14 | omit = *migrations* 15 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # see https://editorconfig.org/ 2 | root = true 3 | 4 | [*] 5 | end_of_line = lf 6 | trim_trailing_whitespace = true 7 | insert_final_newline = true 8 | indent_style = space 9 | indent_size = 4 10 | charset = utf-8 11 | 12 | [*.{bat,cmd,ps1}] 13 | end_of_line = crlf 14 | 15 | [*.{yml,yaml}] 16 | indent_size = 2 17 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG] " 5 | labels: ":bug: bug" 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | --- 14 | 15 | **To Reproduce** 16 | Steps to reproduce the behavior: 17 | 1. Go to '...' 18 | 2. Click on '...' 19 | 3. Scroll down to '...' 20 | 4. See error 21 | 22 | --- 23 | 24 | **Expected behavior** 25 | A clear and concise description of what you expected to happen. 26 | 27 | --- 28 | 29 | **Screenshots** 30 | If applicable, add screenshots to help explain your problem. 31 | 32 | --- 33 | 34 | **Environment:** 35 | - OS: [e.g. Windows 11, Ubuntu 22.04] 36 | - Python version: [e.g. Python 3.10, Python 3.11] 37 | - Scenario: [e.g. eGon2035, eGon100RE] 38 | - Dataset Boundary: [Everything or Schleswig-Holstein] 39 | 40 | --- 41 | 42 | **Proposed solution (optional)** 43 | If you have an idea of how this issue could be fixed, describe it here. 44 | 45 | --- 46 | 47 | **Additional context** 48 | Add any other context about the problem here. 49 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "[FEATURE] " 5 | labels: ":rocket: feature" 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. 12 | E.g. *I'm always frustrated when [...]* 13 | 14 | --- 15 | 16 | **Describe the solution you'd like** 17 | A clear and concise description of what you want to happen. 18 | 19 | --- 20 | 21 | **Describe alternatives you've considered** 22 | A clear and concise description of any alternative solutions or features you’ve considered. 23 | 24 | --- 25 | 26 | **Additional context** 27 | Add any other context or screenshots about the feature request here. 28 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | Fixes # . 2 | 3 | ## Before merging into `dev`-branch, please make sure that 4 | 5 | - [ ] the `CHANGELOG.rst` was updated. 6 | - [ ] new and adjusted code is formated using `black` and `isort`. 7 | - [ ] the `Dataset`-version is updated when existing datasets are adjusted. 8 | - [ ] the workflow is running successful in `test mode`. 9 | - [ ] the workflow is running successful in `Everything` mode. 10 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | # NB: this name is used in the status badge 2 | name: "Tests, code style & coverage" 3 | 4 | on: 5 | push: 6 | branches: 7 | - '**' 8 | pull_request: 9 | branches: 10 | - dev 11 | - main 12 | 13 | # Copied from travis.yml, might need to be re-included 14 | #env: 15 | # global: 16 | # - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so 17 | # - SEGFAULT_SIGNALS=all 18 | # - LANG=en_US.UTF-8 19 | 20 | jobs: 21 | build: 22 | name: Python ${{ matrix.python-version }} (${{ matrix.platform }}) 23 | runs-on: ${{ matrix.platform }} 24 | 25 | strategy: 26 | matrix: 27 | python-version: 28 | - "3.8" 29 | platform: [ubuntu-latest, macos-latest] 30 | exclude: 31 | - platform: macos-latest 32 | python-version: "3.8" 33 | max-parallel: 4 34 | fail-fast: false 35 | 36 | steps: 37 | - uses: "actions/checkout@v2" 38 | - uses: "actions/setup-python@v2" 39 | with: 40 | python-version: "${{ matrix.python-version }}" 41 | - name: "Install dependencies" 42 | run: | 43 | python -mpip install --progress-bar=off tox tox-gh-actions -r'ci/requirements.txt' 44 | virtualenv --version 45 | pip --version 46 | tox --version 47 | - name: "Run tox targets for ${{ matrix.python-version }}" 48 | run: "python -m tox -vv" 49 | env: 50 | PLATFORM: ${{ matrix.platform }} 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | __pycache__ 3 | 4 | # C extensions 5 | *.so 6 | 7 | # Packages 8 | *.egg 9 | *.egg-info 10 | dist 11 | build 12 | eggs 13 | .eggs 14 | parts 15 | bin 16 | var 17 | sdist 18 | wheelhouse 19 | develop-eggs 20 | .installed.cfg 21 | lib 22 | lib64 23 | venv*/ 24 | pyvenv*/ 25 | pip-wheel-metadata/ 26 | 27 | # Installer logs 28 | pip-log.txt 29 | 30 | # Unit test / coverage reports 31 | .coverage 32 | .tox 33 | .coverage.* 34 | .pytest_cache/ 35 | nosetests.xml 36 | coverage.xml 37 | htmlcov 38 | 39 | # Translations 40 | *.mo 41 | 42 | # Mr Developer 43 | .mr.developer.cfg 44 | .project 45 | .pydevproject 46 | .idea 47 | *.iml 48 | *.komodoproject 49 | 50 | # Complexity 51 | output/*.html 52 | output/*/index.html 53 | 54 | # Sphinx 55 | docs/_build 56 | 57 | .DS_Store 58 | *~ 59 | .*.sw[po] 60 | .build 61 | .ve 62 | .env 63 | .cache 64 | .pytest 65 | .benchmarks 66 | .bootstrap 67 | .appveyor.token 68 | *.bak 69 | 70 | # Mypy Cache 71 | .mypy_cache/ 72 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # To install the git pre-commit hook run: 2 | # pre-commit install 3 | # To update the pre-commit hooks run: 4 | # pre-commit install-hooks 5 | exclude: '^(\.tox|ci/templates|\.bumpversion\.cfg)(/|$)' 6 | repos: 7 | - repo: https://github.com/pre-commit/pre-commit-hooks 8 | rev: v4.4.0 9 | hooks: 10 | - id: trailing-whitespace 11 | - id: end-of-file-fixer 12 | - id: debug-statements 13 | - repo: https://github.com/timothycrosley/isort 14 | rev: 5.11.4 15 | hooks: 16 | - id: isort 17 | - repo: https://github.com/psf/black 18 | rev: 22.12.0 19 | hooks: 20 | - id: black 21 | - repo: https://github.com/pycqa/flake8 22 | rev: 6.0.0 23 | hooks: 24 | - id: flake8 25 | language_version: python3.8 26 | -------------------------------------------------------------------------------- /.prospector.yaml: -------------------------------------------------------------------------------- 1 | ignore-paths: ["ci"] 2 | pep257: 3 | disable: ["210", "212", "213"] 4 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 2 | version: 2 3 | sphinx: 4 | configuration: docs/conf.py 5 | formats: all 6 | python: 7 | install: 8 | - requirements: docs/requirements.txt 9 | - method: pip 10 | path: . 11 | # Set the version of Python 12 | build: 13 | os: ubuntu-22.04 14 | tools: 15 | python: "3.8" 16 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | 2 | Authors 3 | ======= 4 | 5 | * Jonathan Amme, Clara Büttner, Ilka Cußmann, Julian Endres, Carlos Epia, Kilian Helfenbein, Stephan Günther, Ulf Müller, Amélia Nadal, Guido Pleßmann, Francesco Witte - https://github.com/openego/eGon-data 6 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | graft docs 2 | graft src 3 | graft ci 4 | graft tests 5 | 6 | include .bumpversion.cfg 7 | include .coveragerc 8 | include .cookiecutterrc 9 | include .editorconfig 10 | 11 | include AUTHORS.rst 12 | include CHANGELOG.rst 13 | include CONTRIBUTING.rst 14 | include LICENSE 15 | include README.rst 16 | 17 | include tox.ini .travis.yml .appveyor.yml .readthedocs.yml .pre-commit-config.yaml 18 | include .prospector.yaml 19 | 20 | global-exclude *.py[cod] __pycache__/* *.so *.dylib 21 | global-exclude airflow.db logs/**/* unittests.cfg 22 | global-exclude *.swp 23 | -------------------------------------------------------------------------------- /ci/appveyor-with-compiler.cmd: -------------------------------------------------------------------------------- 1 | :: Very simple setup: 2 | :: - if WINDOWS_SDK_VERSION is set then activate the SDK. 3 | :: - disable the WDK if it's around. 4 | 5 | SET COMMAND_TO_RUN=%* 6 | SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows 7 | SET WIN_WDK="c:\Program Files (x86)\Windows Kits\10\Include\wdf" 8 | ECHO SDK: %WINDOWS_SDK_VERSION% ARCH: %PYTHON_ARCH% 9 | 10 | IF EXIST %WIN_WDK% ( 11 | REM See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/ 12 | REN %WIN_WDK% 0wdf 13 | ) 14 | IF "%WINDOWS_SDK_VERSION%"=="" GOTO main 15 | 16 | SET DISTUTILS_USE_SDK=1 17 | SET MSSdk=1 18 | "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% 19 | CALL "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release 20 | 21 | :main 22 | ECHO Executing: %COMMAND_TO_RUN% 23 | CALL %COMMAND_TO_RUN% || EXIT 1 24 | -------------------------------------------------------------------------------- /ci/bootstrap.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from __future__ import absolute_import 4 | from __future__ import print_function 5 | from __future__ import unicode_literals 6 | 7 | import os 8 | import subprocess 9 | import sys 10 | from os.path import abspath 11 | from os.path import dirname 12 | from os.path import exists 13 | from os.path import join 14 | 15 | base_path = dirname(dirname(abspath(__file__))) 16 | 17 | 18 | def check_call(args): 19 | print("+", *args) 20 | subprocess.check_call(args) 21 | 22 | 23 | def exec_in_env(): 24 | env_path = join(base_path, ".tox", "bootstrap") 25 | if sys.platform == "win32": 26 | bin_path = join(env_path, "Scripts") 27 | else: 28 | bin_path = join(env_path, "bin") 29 | if not exists(env_path): 30 | import subprocess 31 | 32 | print("Making bootstrap env in: {0} ...".format(env_path)) 33 | try: 34 | check_call([sys.executable, "-m", "venv", env_path]) 35 | except subprocess.CalledProcessError: 36 | try: 37 | check_call([sys.executable, "-m", "virtualenv", env_path]) 38 | except subprocess.CalledProcessError: 39 | check_call(["virtualenv", env_path]) 40 | print("Installing `jinja2` into bootstrap environment...") 41 | check_call([join(bin_path, "pip"), "install", "jinja2", "tox", "matrix"]) 42 | python_executable = join(bin_path, "python") 43 | if not os.path.exists(python_executable): 44 | python_executable += '.exe' 45 | 46 | print("Re-executing with: {0}".format(python_executable)) 47 | print("+ exec", python_executable, __file__, "--no-env") 48 | os.execv(python_executable, [python_executable, __file__, "--no-env"]) 49 | 50 | 51 | def main(): 52 | import jinja2 53 | import matrix 54 | 55 | print("Project path: {0}".format(base_path)) 56 | 57 | jinja = jinja2.Environment( 58 | loader=jinja2.FileSystemLoader(join(base_path, "ci", "templates")), 59 | trim_blocks=True, 60 | lstrip_blocks=True, 61 | keep_trailing_newline=True 62 | ) 63 | 64 | tox_environments = {} 65 | for (alias, conf) in matrix.from_file(join(base_path, "setup.cfg")).items(): 66 | deps = conf["dependencies"] 67 | tox_environments[alias] = { 68 | "deps": deps.split(), 69 | } 70 | if "coverage_flags" in conf: 71 | cover = {"false": False, "true": True}[conf["coverage_flags"].lower()] 72 | tox_environments[alias].update(cover=cover) 73 | if "environment_variables" in conf: 74 | env_vars = conf["environment_variables"] 75 | tox_environments[alias].update(env_vars=env_vars.split()) 76 | 77 | for name in os.listdir(join("ci", "templates")): 78 | with open(join(base_path, name), "w") as fh: 79 | fh.write(jinja.get_template(name).render(tox_environments=tox_environments)) 80 | print("Wrote {}".format(name)) 81 | print("DONE.") 82 | 83 | 84 | if __name__ == "__main__": 85 | args = sys.argv[1:] 86 | if args == ["--no-env"]: 87 | main() 88 | elif not args: 89 | exec_in_env() 90 | else: 91 | print("Unexpected arguments {0}".format(args), file=sys.stderr) 92 | sys.exit(1) 93 | -------------------------------------------------------------------------------- /ci/requirements.txt: -------------------------------------------------------------------------------- 1 | virtualenv>=16.6.0 2 | pip>=19.1.1 3 | setuptools>=18.0.1 4 | six>=1.14.0 5 | -------------------------------------------------------------------------------- /ci/templates/.appveyor.yml: -------------------------------------------------------------------------------- 1 | version: '{branch}-{build}' 2 | build: off 3 | environment: 4 | global: 5 | COVERALLS_EXTRAS: '-v' 6 | COVERALLS_REPO_TOKEN: dnzlyTHfE7lWVrBEdhR1DBfXiYDCSISvl 7 | matrix: 8 | - TOXENV: check 9 | TOXPYTHON: C:\Python36\python.exe 10 | PYTHON_HOME: C:\Python36 11 | PYTHON_VERSION: '3.6' 12 | PYTHON_ARCH: '32' 13 | {% for env, config in tox_environments|dictsort %} 14 | {% if env.startswith(('py2', 'py3')) %} 15 | - TOXENV: {{ env }}{% if config.cover %},codecov,coveralls{% endif %}{{ "" }} 16 | TOXPYTHON: C:\Python{{ env[2:4] }}\python.exe 17 | PYTHON_HOME: C:\Python{{ env[2:4] }} 18 | PYTHON_VERSION: '{{ env[2] }}.{{ env[3] }}' 19 | PYTHON_ARCH: '32' 20 | {% if 'nocov' in env %} 21 | WHEEL_PATH: .tox/dist 22 | {% endif %} 23 | - TOXENV: {{ env }}{% if config.cover %},codecov,coveralls{% endif %}{{ "" }} 24 | TOXPYTHON: C:\Python{{ env[2:4] }}-x64\python.exe 25 | PYTHON_HOME: C:\Python{{ env[2:4] }}-x64 26 | PYTHON_VERSION: '{{ env[2] }}.{{ env[3] }}' 27 | PYTHON_ARCH: '64' 28 | {% if 'nocov' in env %} 29 | WHEEL_PATH: .tox/dist 30 | {% endif %} 31 | {% if env.startswith('py2') %} 32 | WINDOWS_SDK_VERSION: v7.0 33 | {% endif %} 34 | {% endif %}{% endfor %} 35 | init: 36 | - ps: echo $env:TOXENV 37 | - ps: ls C:\Python* 38 | install: 39 | - '%PYTHON_HOME%\python -mpip install --progress-bar=off tox -rci/requirements.txt' 40 | - '%PYTHON_HOME%\Scripts\virtualenv --version' 41 | - '%PYTHON_HOME%\Scripts\easy_install --version' 42 | - '%PYTHON_HOME%\Scripts\pip --version' 43 | - '%PYTHON_HOME%\Scripts\tox --version' 44 | test_script: 45 | - cmd /E:ON /V:ON /C .\ci\appveyor-with-compiler.cmd %PYTHON_HOME%\Scripts\tox 46 | on_failure: 47 | - ps: dir "env:" 48 | - ps: get-content .tox\*\log\* 49 | 50 | ### To enable remote debugging uncomment this (also, see: http://www.appveyor.com/docs/how-to/rdp-to-build-worker): 51 | # on_finish: 52 | # - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) 53 | -------------------------------------------------------------------------------- /ci/templates/.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | dist: xenial 3 | cache: false 4 | env: 5 | global: 6 | - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so 7 | - SEGFAULT_SIGNALS=all 8 | - LANG=en_US.UTF-8 9 | matrix: 10 | include: 11 | - python: '3.6' 12 | env: 13 | - TOXENV=check 14 | - python: '3.6' 15 | env: 16 | - TOXENV=docs 17 | {%- for env, config in tox_environments|dictsort %}{{ '' }} 18 | {%- if 'py38' in env or 'py27' in env %}{{ '' }} 19 | - os: osx 20 | osx_image: xcode11 21 | language: generic 22 | env: 23 | - TOXENV={{ env }} 24 | {%- endif %}{{ '' }} 25 | - env: 26 | - TOXENV={{ env }}{% if config.cover %},codecov,coveralls{% endif %} 27 | {%- if env.startswith('pypy3') %}{{ '' }} 28 | - TOXPYTHON=pypy3 29 | python: 'pypy3' 30 | {%- elif env.startswith('pypy') %}{{ '' }} 31 | python: 'pypy' 32 | {%- else %}{{ '' }} 33 | python: '{{ '{0[2]}.{0[3]}'.format(env) }}' 34 | {%- endif %}{{ '' }} 35 | {%- endfor %}{{ '' }} 36 | before_install: 37 | - python --version 38 | - uname -a 39 | - lsb_release -a || true 40 | - | 41 | if [[ $TRAVIS_OS_NAME == 'osx' ]]; then 42 | [[ $TOXENV =~ py3 ]] && brew upgrade python 43 | [[ $TOXENV =~ py2 ]] && brew install python@2 44 | export PATH="/usr/local/opt/python/libexec/bin:${PATH}" 45 | fi 46 | install: 47 | - python -mpip install --progress-bar=off tox -rci/requirements.txt 48 | - virtualenv --version 49 | - easy_install --version 50 | - pip --version 51 | - tox --version 52 | script: 53 | - tox -v 54 | after_failure: 55 | - cat .tox/log/* 56 | - cat .tox/*/log/* 57 | notifications: 58 | email: 59 | on_success: never 60 | on_failure: always 61 | -------------------------------------------------------------------------------- /ci/templates/tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | clean, 4 | check, 5 | docs, 6 | {% for env in tox_environments|sort %} 7 | {{ env }}, 8 | {% endfor %} 9 | report 10 | 11 | [testenv] 12 | basepython = 13 | {bootstrap,clean,check,report,docs,codecov,coveralls}: {env:TOXPYTHON:python3} 14 | setenv = 15 | PYTHONPATH={toxinidir}/tests 16 | PYTHONUNBUFFERED=yes 17 | passenv = 18 | * 19 | deps = 20 | pytest 21 | pytest-travis-fold 22 | commands = 23 | {posargs:pytest -vv --ignore=src} 24 | 25 | [testenv:bootstrap] 26 | deps = 27 | jinja2 28 | matrix 29 | skip_install = true 30 | commands = 31 | python ci/bootstrap.py --no-env 32 | 33 | [testenv:check] 34 | deps = 35 | docutils 36 | check-manifest 37 | flake8 38 | readme-renderer 39 | pygments 40 | isort 41 | skip_install = true 42 | commands = 43 | python setup.py check --strict --metadata --restructuredtext 44 | check-manifest {toxinidir} 45 | flake8 46 | isort --verbose --check-only --diff --filter-files . 47 | 48 | 49 | [testenv:docs] 50 | usedevelop = true 51 | deps = 52 | -r{toxinidir}/docs/requirements.txt 53 | commands = 54 | sphinx-build {posargs:-E} -b html docs dist/docs 55 | sphinx-build -b linkcheck docs dist/docs 56 | 57 | [testenv:coveralls] 58 | deps = 59 | coveralls 60 | skip_install = true 61 | commands = 62 | coveralls [] 63 | 64 | 65 | 66 | [testenv:codecov] 67 | deps = 68 | codecov 69 | skip_install = true 70 | commands = 71 | codecov [] 72 | 73 | [testenv:report] 74 | deps = coverage 75 | skip_install = true 76 | commands = 77 | coverage report 78 | coverage html 79 | 80 | [testenv:clean] 81 | commands = coverage erase 82 | skip_install = true 83 | deps = coverage 84 | {% for env, config in tox_environments|dictsort %} 85 | 86 | [testenv:{{ env }}] 87 | basepython = {env:TOXPYTHON:{{ env.split("-")[0] if env.startswith("pypy") else "python{0[2]}.{0[3]}".format(env) }}} 88 | {% if config.cover or config.env_vars %} 89 | setenv = 90 | {[testenv]setenv} 91 | {% endif %} 92 | {% for var in config.env_vars %} 93 | {{ var }} 94 | {% endfor %} 95 | {% if config.cover %} 96 | usedevelop = true 97 | commands = 98 | {posargs:pytest --cov --cov-report=term-missing -vv} 99 | {% endif %} 100 | {% if config.cover or config.deps %} 101 | deps = 102 | {[testenv]deps} 103 | {% endif %} 104 | {% if config.cover %} 105 | pytest-cov 106 | {% endif %} 107 | {% for dep in config.deps %} 108 | {{ dep }} 109 | {% endfor -%} 110 | {% endfor -%} 111 | -------------------------------------------------------------------------------- /docs/_templates/module.rst_t: -------------------------------------------------------------------------------- 1 | {%- if show_headings %} 2 | {{- basename.split(".")[-1] | e | heading }} 3 | 4 | {% endif -%} 5 | .. automodule:: {{ qualname }} 6 | {%- for option in automodule_options %} 7 | :{{ option }}: 8 | {%- endfor %} 9 | 10 | -------------------------------------------------------------------------------- /docs/_templates/package.rst_t: -------------------------------------------------------------------------------- 1 | {% macro automodule(modname, options) %} 2 | 3 | .. automodule:: {{ modname }} 4 | {%- for option in options %} 5 | :{{ option }}: 6 | {%- endfor %} 7 | {%- endmacro %} 8 | 9 | {% macro toctree(docnames) -%} 10 | 11 | .. toctree:: 12 | :maxdepth: {{ maxdepth }} 13 | {% for docname in docnames %} 14 | {{ docname }} 15 | {%- endfor %} 16 | {%- endmacro %} 17 | 18 | {{- pkgname.split(".")[-1] | e | heading }} 19 | 20 | {%- if is_namespace %} 21 | 22 | .. py:module:: {{ pkgname }} 23 | {% endif %} 24 | 25 | {%- if modulefirst and not is_namespace %} 26 | {{ automodule(pkgname, automodule_options) }} 27 | {% endif %} 28 | 29 | {%- if submodules %} 30 | {% if separatemodules %} 31 | {{- toctree(submodules + subpackages) }} 32 | {% else %} 33 | {%- if subpackages %} 34 | {{ toctree(subpackages) }} 35 | {% endif %} 36 | {%- for submodule in submodules %} 37 | {% if show_headings %} 38 | {{- [submodule, "module"] | join(" ") | e | heading(2) }} 39 | {% endif %} 40 | {{ automodule(submodule, automodule_options) }} 41 | {% endfor %} 42 | {%- endif %} 43 | {%- endif %} 44 | 45 | {%- if not modulefirst and not is_namespace %} 46 | {{ automodule(pkgname, automodule_options) }} 47 | {% endif %} 48 | -------------------------------------------------------------------------------- /docs/_templates/toc.rst_t: -------------------------------------------------------------------------------- 1 | {{ header | heading }} 2 | 3 | .. toctree:: 4 | :maxdepth: {{ maxdepth }} 5 | {% for docname in docnames %} 6 | {{ docname }} 7 | {%- endfor %} 8 | -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGELOG.rst 2 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from pathlib import Path 5 | import io 6 | import os 7 | 8 | 9 | def read_authors(**kwargs): 10 | with io.open( 11 | Path(__file__).parent.parent / "AUTHORS.rst", 12 | encoding=kwargs.get("encoding", "utf8"), 13 | ) as fh: 14 | return ( 15 | fh.read() 16 | .replace("\nAuthors\n=======\n\n* ", "") 17 | .replace(" - https://github.com/openego/eGon-data\n", "") 18 | ) 19 | 20 | 21 | extensions = [ 22 | "sphinx.ext.autodoc", 23 | "sphinx.ext.autosummary", 24 | "sphinx.ext.coverage", 25 | "sphinx.ext.doctest", 26 | "sphinx.ext.extlinks", 27 | "sphinx.ext.ifconfig", 28 | "sphinx.ext.napoleon", 29 | "sphinx.ext.todo", 30 | "sphinx.ext.viewcode", 31 | "sphinx.ext.autosectionlabel", 32 | ] 33 | source_suffix = ".rst" 34 | master_doc = "index" 35 | project = "eGo^N Data" 36 | year = "2020-2022" 37 | author = read_authors() 38 | copyright = "{0}, {1}".format(year, author) 39 | version = release = "1.0.0" 40 | 41 | pygments_style = "trac" 42 | templates_path = ["."] 43 | extlinks = { 44 | "issue": ("https://github.com/openego/eGon-data/issues/%s", "issue #"), 45 | "pr": ("https://github.com/openego/eGon-data/pull/%s", "PR #"), 46 | } 47 | # on_rtd is whether we are on readthedocs.org 48 | # on_rtd = os.environ.get("READTHEDOCS", None) == "True" 49 | 50 | #if not on_rtd: # only set the theme if we're building docs locally 51 | html_theme = "sphinx_rtd_theme" 52 | 53 | html_use_smartypants = True 54 | html_last_updated_fmt = "%b %d, %Y" 55 | html_split_index = False 56 | html_sidebars = {"**": ["searchbox.html", "globaltoc.html", "sourcelink.html"]} 57 | html_short_title = "%s-%s" % (project, version) 58 | 59 | napoleon_use_ivar = True 60 | napoleon_use_rtype = False 61 | napoleon_use_param = False 62 | 63 | add_module_names = False 64 | modindex_common_prefix = ["egon.data.", "egon.data.datasets."] 65 | 66 | autodoc_type_aliases = { 67 | "Dependencies": "egon.data.datasets.Dependencies", 68 | "Tasks": "egon.data.datasets.Tasks" 69 | } 70 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /docs/data.rst: -------------------------------------------------------------------------------- 1 | **** 2 | Data 3 | **** 4 | The description of the methods, input data and results of the eGon-data pipeline is given in the following section. 5 | References to datasets and functions are integrated if more detailed information is required. 6 | 7 | Main input data and their processing 8 | ==================================== 9 | 10 | All methods in the eGon-data workflow rely on public and freely available data from different external sources. The most important data sources 11 | and their processing within the eGon-data pipeline are described here. 12 | 13 | .. include:: data/input_data.rst 14 | 15 | Grid models 16 | =========== 17 | 18 | Power grid models of different voltage levels form a central part of the eGon-data model, which is required for cross-grid-level optimization. 19 | In addition, sector coupling necessitates the representation of the gas grid infrastructure, which is also described in this section. 20 | 21 | Electricity grid 22 | ---------------- 23 | 24 | .. include:: data/electricity_grids.rst 25 | 26 | Gas grid 27 | -------- 28 | 29 | .. include:: data/gas_grids.rst 30 | 31 | Demand 32 | ====== 33 | 34 | Electricity, heat and gas demands from different consumption sectors are taken into account in eGon-data. The related methods to distribute and 35 | process the demand data are described in the following chapters for the different consumption sectors separately. 36 | 37 | .. _electricity-demand-ref: 38 | 39 | Electricity 40 | ----------- 41 | 42 | .. include:: data/electricity_demand.rst 43 | 44 | .. _heat-demand-ref: 45 | 46 | Heat 47 | ---- 48 | 49 | .. include:: data/heat_demand.rst 50 | 51 | Gas 52 | --- 53 | 54 | .. include:: data/gas_demand.rst 55 | 56 | .. _mobility-demand-ref: 57 | 58 | Mobility 59 | -------- 60 | 61 | .. include:: data/mobility_demand.rst 62 | 63 | 64 | Supply 65 | ====== 66 | 67 | The distribution and assignment of supply capacities or potentials are carried out technology-specific. The different methods are described in the 68 | following chapters. 69 | 70 | Electricity 71 | ----------- 72 | 73 | .. include:: data/electricity_supply.rst 74 | 75 | Heat 76 | ---- 77 | 78 | .. include:: data/heat_supply.rst 79 | 80 | Gas 81 | --- 82 | .. include:: data/gas_supply.rst 83 | 84 | 85 | Flexibility options 86 | =================== 87 | 88 | Different flexibility options are part of the model and can be utilized in the optimization of the energy system. Therefore detailed information about 89 | flexibility potentials and their distribution are needed. The considered technologies described in the following chapters range from different storage units, 90 | through dynamic line rating to Demand-Side-Management measures. 91 | 92 | Demand-Side Management 93 | ---------------------- 94 | 95 | .. include:: data/DSM.rst 96 | 97 | Dynamic line rating 98 | ------------------- 99 | 100 | .. include:: data/DLR.rst 101 | 102 | .. _flexible-charging-ref: 103 | 104 | Flexible charging of EVs 105 | --------------------------- 106 | 107 | .. include:: data/e-mobility.rst 108 | 109 | Battery stores 110 | ---------------- 111 | 112 | .. include:: data/batteries.rst 113 | 114 | Gas stores 115 | ---------- 116 | 117 | .. include:: data/gas_stores.rst 118 | 119 | Heat stores 120 | ------------- 121 | 122 | .. include:: data/heat_stores.rst 123 | 124 | 125 | Published data 126 | ============== 127 | 128 | 129 | -------------------------------------------------------------------------------- /docs/data/DLR.rst: -------------------------------------------------------------------------------- 1 | To calculate the transmission capacity of each transmission line in the model, 2 | the procedure suggested in the **Principles for the Expansion Planning of the 3 | German Transmission Network** [NEP2021a]_ where used: 4 | 5 | 1. Import the temperature and wind temporal raster layers from ERA-5. Hourly 6 | resolution data from the year 2011 was used. Raster resolution 7 | latitude-longitude grids at 0.25° x 0.25°. 8 | 9 | 2. Import shape file for the 9 regions proposed by the Principles for 10 | the Expansion Planning. See Figure 1. 11 | 12 | .. image:: images/regions_DLR.png 13 | :width: 400 14 | :alt: regions DLR 15 | 16 | Figure 1: Representative regions in Germany for DLR analysis [NEP2021a]_ 17 | 18 | 3. Find the lowest wind speed in each region. To perform this, for each 19 | independent region, the wind speed of every cell in the raster layer should be 20 | extracted and compared. This procedure is repeated for each hour in the 21 | year 2011. The results are the 8760 lowest wind speed per region. 22 | 23 | 4. Find the highest temperature in each region. To perform this, for each 24 | independent region, the temperature of every cell in the raster layer should 25 | be extracted and compared. This procedure is repeated for each hour in the 26 | year 2011. The results are the 8760 maximum temperature per region. 27 | 28 | 5. Calculate the maximum capacity for each region using the parameters shown in 29 | Figure 2. 30 | 31 | .. image:: images/table_max_capacity_DLR.png 32 | :width: 400 33 | :alt: table_max_capacity_DLR 34 | 35 | Figure 2: transmission capacity based on max temperature and min wind speed [NEP2021a]_ 36 | 37 | 6. Assign the maximum capacity of the corresponding region to each transmission 38 | line inside each one of them. Crossborder lines and underground lines receive 39 | no values. It means that their capacities are static and equal to their nominal 40 | values. Lines that cross borders between regions receive the lowest 41 | capacity per hour of the regions containing the line. 42 | -------------------------------------------------------------------------------- /docs/data/DSM.rst: -------------------------------------------------------------------------------- 1 | Demand-side management (DSM) potentials are calculated in function :func:`dsm_cts_ind_processing`. 2 | Potentials relevant for the high and extra-high voltage grid are identified in the function :func:`dsm_cts_ind`, 3 | potentials within the medium- and low-voltage grids are determined within the function :func:`dsm_cts_ind_individual` 4 | in a higher spatial resolution. All this is part of the dataset :py:class:`DsmPotential `. 5 | The implementation is documented in detail within the following student work (in German): [EsterlDentzien]_. 6 | 7 | Loads eligible to be shifted are assumed within industrial loads and loads from Commercial, Trade and Service (CTS). 8 | Therefore, load time series from these sectors are used as input data (see section ref:`elec_demand-ref`). 9 | Shiftable shares of loads mainly derive from heating and cooling processes and selected energy-intensive 10 | industrial processes (cement production, wood pulp, paper production, recycling paper). Technical and sociotechnical 11 | constraints are considered using the parametrization elaborated in [Heitkoetter]_. An overview over the 12 | resulting potentials for scenario ``eGon2035`` can be seen in figure :ref:`dsm_potential`. The table below summarizes the 13 | aggregated potential for Germany per scenario. As the annual conventional electrical loads are assumed to be lower in the 14 | scenario ``eGon100RE``, also the DSM potential decreases compared to the scenario ``eGon2035``. 15 | 16 | .. figure:: /images/DSM_potential.png 17 | :name: dsm_potential 18 | :width: 600 19 | 20 | Aggregated DSM potential in Germany for scenario ``eGon2035`` 21 | 22 | .. list-table:: Aggregated DSM Potential for Germany 23 | :widths: 20 20 20 24 | :header-rows: 1 25 | 26 | * - 27 | - CTS 28 | - Industry 29 | 30 | * - eGon2035 31 | - 1.2 GW 32 | - 150 MW 33 | 34 | * - eGon100RE 35 | - 900 MW 36 | - 150 MW 37 | 38 | DSM is modelled following the approach of [Kleinhans]_. DSM components are created wherever 39 | respective loads are seen. Minimum and maximum shiftable power per time step depict time-dependent 40 | charging and discharging power of a storage-equivalent buffers. Time-dependent capacities 41 | of those buffers account for the time frame of management bounding the period within which 42 | the shifting can be conducted. Figure :ref:`dsm_shifted_p-example` shows the resulting potential at one exemplary bus. 43 | 44 | .. figure:: /images/shifted_dsm-example.png 45 | :name: dsm_shifted_p-example 46 | :width: 600 47 | 48 | Time-dependent DSM potential at one exemplary bus 49 | 50 | 51 | -------------------------------------------------------------------------------- /docs/data/batteries.rst: -------------------------------------------------------------------------------- 1 | Battery storage units comprise home batteries and larger, grid-supportive batteries. National capacities for home batteries arise from external sources, e.g. the Grid Development Plan for the scenario ``eGon2035``, whereas the capacities of large-scale batteries are a result of the grid optimization tool `eTraGo `_. 2 | 3 | Home battery capacities are first distributed to medium-voltage grid districts (MVGD) and based on that further disaggregated to single buildings. The distribution on MVGD level is done proportional to the installed capacities of solar rooftop power plants, assuming that they are used as solar home storage. 4 | 5 | Potential large-scale batteries are included in the data model at every substation. The data model includes technical and economic parameters, such as efficiencies and investment costs. The energy-to-power ratio is set to a fixed value of 6 hours. Other central parameters are given in the following table 6 | 7 | .. list-table:: Parameters of batteries for scenario eGon2035 8 | :widths: 40 30 30 9 | :header-rows: 1 10 | 11 | * - 12 | - Value 13 | - Sources 14 | 15 | * - Efficiency store 16 | - 98 % 17 | - [DAE_store]_ 18 | 19 | * - Efficiency dispatch 20 | - 98 % 21 | - [DAE_store]_ 22 | 23 | * - Standing loss 24 | - 0 % 25 | - [DAE_store]_ 26 | 27 | * - Investment costs 28 | - 838 €/kW 29 | - [DAE_store]_ 30 | 31 | * - Home storage units 32 | - 16.8 GW 33 | - [NEP2021]_ 34 | 35 | 36 | On transmission grid level, distinguishing between home batteries and large-scale batteries was not possible. Therefore, the capacities of home batteries were set as a lower boundary of the large-scale battery capacities. 37 | This is implemented in the dataset :py:class:`StorageEtrago `, the data for batteries in the transmission grid is stored in the database table :py:class:`grid.egon_etrago_storage `. 38 | -------------------------------------------------------------------------------- /docs/data/e-mobility.rst: -------------------------------------------------------------------------------- 1 | The flexibility potential of EVs is determined on the basis of the trip data 2 | created with SimBEV (see :ref:`mobility-demand-mit-ref`). 3 | It is assumed, that only charging at private charging points, comprising charging points at 4 | home and at the workplace, can be flexibilized. Public fast (e.g. gas stations) and slow charging (e.g. schools 5 | and shopping facilities) stations are assumed not to provide demand-side flexibility. 6 | Further, vehicle-to-grid is not considered and it is assumed that charging can only be shifted 7 | within a charging event. Shifting charging demand to a later charging event, for example 8 | from charging at work during working hours to charging at home in the evening, is therefore 9 | not possible. In the generation of the trip data itself it is already considered, that 10 | EVs are not charged everytime a charging point is available, but only if a certain 11 | lower state of charge (SoC) is reached or the energy level is not sufficient for the next ride. 12 | 13 | In `eTraGo `_, the flexibility of the EVs is modeled 14 | using a storage model based on [Brown2018]_ and [Wulff2020]_. 15 | The used model is visualised in the upper right in figure :ref:`mit-model`. 16 | Its parametrization is for both the eGon2035 and eGon100RE scenario conducted in the 17 | :py:class:`MotorizedIndividualTravel` 18 | dataset in the function 19 | :py:func:`generate_load_time_series`. 20 | The model consists of loads for static driving demands and stores for the fleet’s batteries. 21 | The stores are constrained by hourly lower and upper SoC limits. 22 | The lower SoC limit represents the inflexible charging demand while the 23 | SoC band between the lower and upper SoC limit represents the flexible charging demand. 24 | Further, the charging infrastructure is represented by unidirectional links from electricity 25 | buses to EV buses. Its maximum charging power per hour is set to the available charging power 26 | of grid-connected EVs. 27 | 28 | In `eDisGo `_, the flexibility potential for 29 | controlled charging is modeled using 30 | so-called flexibility bands. These bands comprise an upper and lower power band for 31 | the charging power and an upper and lower energy band for the energy to be recharged 32 | for each charging point in an hourly resolution. These flexibility bands are not 33 | set up in eGon-data but in eDisGo, using the trip data from eGon-data. 34 | For further information on the flexibility bands see 35 | `eDisGo documentation `_. 36 | -------------------------------------------------------------------------------- /docs/data/gas_demand.rst: -------------------------------------------------------------------------------- 1 | The industrial gas demand is modelled with PyPSA *loads*. 2 | 3 | In Germany 4 | ~~~~~~~~~~ 5 | 6 | In the scenario eGon2035, the industrial gas loads in Germany are taken 7 | directly from the eXtremos project data [eXtremos]_. They model the hourly-resolved 8 | industrial loads in Germany with a NUTS-3 spatial resolution, for methane 9 | (:math:`CH_4`), as well as for hydrogen (:math:`H_2`), for the year 2035. 10 | 11 | The spatial repartition of these loads is represented in the figure below 12 | (methane demand in grey and hydrogen in cyan). The size of the rounds on 13 | the figure corresponds to the total annual demand at the considered spot 14 | in the scenario eGon2035. 15 | 16 | .. image:: images/eGon2035_gas_ind_load_repartition_DE_withTitle.png 17 | :width: 400 18 | 19 | In eGon100RE, the global industrial demand for methane and hydrogen (for 20 | whole Germany and for one year) is calculated by the PyPSA-eur-sec run. 21 | The spatial and the temporal repartitions used to distribute these values 22 | are corresponding to the spatial and temporal repartition of the hydrogen 23 | industrial demand from the eXtremos project [eXtremos]_ for the year 2050. 24 | (The same repartition is used, due to the lack of data were available for 25 | methane, because the eXtremos project considers that there won't be any 26 | industrial methane load in 2050.) 27 | 28 | .. image:: images/ind_gas_demand.png 29 | :width: 400 30 | 31 | The figure above shows the temporal evolution of the methane (in grey) and 32 | hydrogen (in cyan) industrial demands in the year for both scenarios 33 | (eGon100RE in dashed). 34 | The total demands for whole Germany are to be found in the following table. 35 | 36 | .. list-table:: Total indutrial hydrogen and methane demands in Germany considered in eGon-data 37 | :widths: 25 25 25 38 | :header-rows: 1 39 | 40 | * - 41 | - eGon2035 42 | - eGon100RE 43 | * - :math:`CH_4` (in TWh) 44 | - 195 45 | - 105 46 | * - :math:`H_2` (in TWh) 47 | - 16 48 | - 42 49 | 50 | The hydrogen loads are attributed only to the *H2_grid* buses. 51 | 52 | The implementation of these data is detailed in the :py:mod:`industrial_gas_demand 53 | ` page of our documentation. 54 | 55 | In the neighboring countries 56 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 57 | 58 | In the scenario **eGon2035**, there are no hydrogen buses modelled in the 59 | neighboring countries. For this reason, the hydrogen industrial demand 60 | is there attributed to the electrical sector. 61 | The total demand is taken from the 'Distributed Energy' scenario of the 62 | Ten-Year Network Development Plan 2020 ([TYNDP]_). For the year 63 | 2035, it has been linearly interpolated between the values for the year 2030 64 | and 2040. These industrial hydrogen loads are considered as constant in 65 | time. In other words, at each hour of the year, the same load should by 66 | fulfilled by electricity to supply the hydrogen industrial demand. 67 | 68 | Contrary to all the other loads described in the section, the modelled 69 | methane load abroad includes not only the industrial demand, but also the 70 | heat demand, that is supplied by methane. The total demand is again taken 71 | from the 'Distributed Energy' scenario of the TYNDP 2020 (linear interpolation 72 | between 2030 and 2040). For the temporal disaggregation of the demand in 73 | the year, the time series 'rural heat' from PyPSA-eur-sec is used to approximated 74 | the temporal profile, because the heat sector represents the biggest load. 75 | 76 | The implementation of these data is detailed in the :py:mod:`gas_neighbours.eGon2035 77 | ` page of our documentation. 78 | 79 | In the scenario **eGon100RE**, the industrial gas loads (for methane as 80 | well as for hydrogen) in the neighboring countries are directly imported 81 | from the PyPSA-eur-sec run. 82 | -------------------------------------------------------------------------------- /docs/data/gas_grids.rst: -------------------------------------------------------------------------------- 1 | In the gas sector, only the transport grids are represented. They are modelled 2 | with bidirectional PyPSA *links* which allow to represent the energy exchange 3 | between distinct PyPSA *buses*. No losses are considered and the 4 | energy that can be transferred through a pipeline is limited by the e_nom 5 | parameter. 6 | 7 | In the scenario **eGon2035**, only the methane grid is modelled. 8 | 9 | In the scenario **eGon100RE**, there are tree type of gas grid: 10 | 11 | * the remaining methane grid, 12 | * the retrofitted hydrogen grid, 13 | * the potential hydrogen grid extension (only present in Germany). 14 | 15 | In Germany 16 | ~~~~~~~~~~ 17 | 18 | To determine the topology of the grid, the status quo methane grid is used, 19 | with the data from SciGRID_gas [SciGRID_gas]_. 20 | 21 | eGon2035 22 | """""""" 23 | 24 | In the scenario eGon2035, the capacities of the methane pipelines are determined 25 | using the status quo pipeline diameters [SciGRID_gas]_ and the correspondence 26 | table [Kunz]_. These capacities are fixed. 27 | 28 | There are two types of hydrogen buses and none of them are connected through 29 | a grid: 30 | 31 | * H2_grid buses: these buses are located at the places than the methane 32 | buses, 33 | * H2_saltcavern buses: these buses are located at the intersection of power 34 | substations and potential saltcavern areas, adapted for hydrogen storage. 35 | 36 | In this scenario, there are no hydrogen bus modelled in the neighboring 37 | countries. 38 | 39 | The modelling of the methane grid in the scenario eGon2035 is represented 40 | below. 41 | 42 | .. image:: images/2035_gasgrid_enTitle_00.png 43 | :width: 400 44 | 45 | The implementation of these data is detailed respectively in the 46 | :py:mod:`gas_grid ` and :py:mod:`hydrogen_etrago.bus 47 | ` pages of our documentation. 48 | 49 | eGon100RE 50 | """"""""" 51 | 52 | In the scenario eGon100RE, the methane grid has the same topology than in 53 | the scenario eGon2035. The capacities of the methane pipelines are still 54 | fixed, but they are reduced, because a part of the grid is retrofitted 55 | into an hydrogen grid. 56 | 57 | The retrofitted hydrogen grid has the same topology than the methane grid. 58 | The share of the methane grid which is retrofitted is calculated by the 59 | PyPSA-eur-sec run. 60 | 61 | The potential hydrogen grid extensions link each hydrogen saltcavern to 62 | its nearest (retrofitted) hydrogen grid bus. The extension has no upper 63 | boundary and its technical parameters are taken from the PyPSA technology 64 | data [technoData]_. This option is modelled exclusively in Germany. 65 | 66 | The modelling of the methane grid in the scenario eGon100RE is represented 67 | below. 68 | 69 | .. image:: images/100RE_gasgrid_enTitle_00.png 70 | :width: 400 71 | 72 | The implementation of these data is detailed respectively in the :py:mod:`gas_grid 73 | ` and :py:mod:`h2_grid ` 74 | pages of our documentation. 75 | 76 | 77 | Cross-bordering pipelines 78 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 79 | 80 | The cross-bordering pipelines capacities are taken respectively from the 81 | TYNDP [TYNDP]_ for eGon2035 and from PyPSA-eur-sec for eGon100RE. 82 | 83 | For the cross-bordering gas pipelines with Germany, the capacity is uniformly 84 | distributed between the all pipelines connecting one country to Germany. 85 | 86 | The implementation of these data is detailed respectively in the 87 | :py:func:`gas_neighbours.eGon2035.grid ` and 88 | :py:func:`insert_gas_neigbours_eGon100RE ` 89 | pages of our documentation. 90 | -------------------------------------------------------------------------------- /docs/data/gas_stores.rst: -------------------------------------------------------------------------------- 1 | Hydrogen stores 2 | ~~~~~~~~~~~~~~~ 3 | 4 | In both scenarios is there the option to store hydrogen. In the scenario 5 | eGon2035, these potentials are modelled only in Germany, because the hydrogen 6 | sector is not modelled in the neighboring countries. 7 | Hydrogen is stored either in steel tanks (*overground storage*) or in saltcavern 8 | (*underground storage*), which are modelled by PyPSA *stores*. The technical 9 | parameters (investment and marginal costs, efficiency, lifetime) comes 10 | from the PyPSA technology data [technoData]_. The steel tank storage potentials 11 | are not limited. On the contrary, the saltcavern storage potentials do have 12 | an upper boundary. 13 | 14 | In Germany 15 | """""""""" 16 | 17 | The saltcavern potentials are located at the intersection between the power 18 | substations and the adapted saltcavern surface areas, defined in the InSpEE-DS 19 | report from the Bundesanstalt für Geowissenschaften und Rohstoffe [BGR]_. 20 | These potentials, identical in the both scenarios, are represented in the 21 | figure below. 22 | 23 | .. image:: images/H2_underground_stores_withTitle.png 24 | :width: 400 25 | 26 | The implementation of these data is detailed in the :py:mod:`hydrogen_etrago.storage 27 | ` page of our documentation. 28 | 29 | In eGon100RE, the retrofitted hydrogen grid can also be used as storage, 30 | with unvariable capacity. This implementation is detailed in the 31 | :py:mod:`grid_storage ` page of our documentation. 32 | 33 | In the neighboring countries 34 | """""""""""""""""""""""""""" 35 | 36 | In the scenario eGon100RE the hydrogen storage potentials in the neighboring 37 | countries are taken directly from the PyPSA-eur-sec run. 38 | 39 | Methane stores 40 | ~~~~~~~~~~~~~~ 41 | 42 | The methane stores are modelled as PyPSA *stores* and they are invariable. 43 | In Germany, two types of methane stores are modelled: 44 | 45 | * the underground caverns: capacities from SciGRID_gas [SciGRID_gas]_, 46 | * the storage capacity of the German gas grid. 47 | 48 | The storage capacity of the gas grid (130 GWh according to the Bundesnetzagentur 49 | [Bna]_) is uniformly distributed between all the methane nodes in Germany 50 | in the scenario eGon2035. 51 | In the scenario eGon100RE, the retrofitted hydrogen grid take a share of 52 | this storage capacity over. 53 | 54 | The implementation of the methane storage in the data model is detailed in the 55 | :py:mod:`ch4_storages ` page of our documentation. 56 | 57 | In the neighboring countries, the methane store capacities from SciGRID_gas [SciGRID_gas]_ 58 | are used in the eGon2035 scenario and data from PyPSA-eur-sec are used in 59 | the eGon100RE scenario. 60 | -------------------------------------------------------------------------------- /docs/data/heat_stores.rst: -------------------------------------------------------------------------------- 1 | The heat sector can provide flexibility through stores that allow shifting energy in time. The data model includes hot water tanks as heat stores in individual buildings and pit thermal energy storage for district heating grids (further described in :ref:`district-heating`). 2 | 3 | Within the data model, potential locations as well as technical and economic parameters of these stores are defined. The installed store and (dis-)charging capacities are part of the grid optimization methods that can be performed by `eTraGo `_. The power-to-energy ratio is not predefined but a result of the optimization, which allows to build heat stores with various time horizons. 4 | 5 | Individual heat stores can be built in every building with an individual heat pump. Central heat stores can be built next to district heating grids. There are no maximum limits for the energy output as well as (dis-)charging capacities implemented yet. 6 | 7 | Central cost assumptions for central and decentral heat stores are listed in the table below. The parameters can differ for each scenario in order to include technology updates and learning curves. The table focuses on the scenario ``eGon2035``. 8 | 9 | .. list-table:: Parameters of heat stores 10 | :widths: 16 16 16 16 16 16 11 | :header-rows: 1 12 | 13 | * - 14 | - Technology 15 | - Costs for store capacity 16 | - Costs for (dis-)charging capacity 17 | - Round-trip efficiency 18 | - Sources 19 | 20 | * - District heating 21 | - Pit thermal energy storage 22 | - 0.51 EUR / kWh 23 | - 0 EUR / kW 24 | - 70 % 25 | - [DAE_store]_ 26 | 27 | * - Buildings with heat pump 28 | - Water tank 29 | - 1.84 EUR / kWh 30 | - 0 EUR / kW 31 | - 70 % 32 | - [DAE_store]_ 33 | 34 | The heat stores are implemented as a part of the dataset :py:class:`HeatEtrago `, the data is written into the tables :py:class:`grid.egon_etrago_bus `, :py:class:`grid.egon_etrago_link ` and :py:class:`grid.egon_etrago_store `. 35 | 36 | -------------------------------------------------------------------------------- /docs/getting_started.rst: -------------------------------------------------------------------------------- 1 | *************** 2 | Getting Started 3 | *************** 4 | 5 | .. include:: ../README.rst 6 | :start-after: .. begin-getting-started-information 7 | :end-before: .. end-getting-started-information 8 | -------------------------------------------------------------------------------- /docs/images/100RE_gasgrid_enTitle_00.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/100RE_gasgrid_enTitle_00.png -------------------------------------------------------------------------------- /docs/images/2035_gasgrid_enTitle_00.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/2035_gasgrid_enTitle_00.png -------------------------------------------------------------------------------- /docs/images/DSM_potential.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/DSM_potential.png -------------------------------------------------------------------------------- /docs/images/Erzeugerpark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/Erzeugerpark.png -------------------------------------------------------------------------------- /docs/images/GasGen_eGon2035.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/GasGen_eGon2035.png -------------------------------------------------------------------------------- /docs/images/H2_underground_stores_withTitle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/H2_underground_stores_withTitle.png -------------------------------------------------------------------------------- /docs/images/Logos_Projektpartner_egon_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/Logos_Projektpartner_egon_data.png -------------------------------------------------------------------------------- /docs/images/PV_freiflaeche.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/PV_freiflaeche.png -------------------------------------------------------------------------------- /docs/images/S27-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/S27-3.png -------------------------------------------------------------------------------- /docs/images/S27-4a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/S27-4a.png -------------------------------------------------------------------------------- /docs/images/combined_heat_and_power_plants.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/combined_heat_and_power_plants.png -------------------------------------------------------------------------------- /docs/images/ding0_mv_lv_grid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/ding0_mv_lv_grid.png -------------------------------------------------------------------------------- /docs/images/district_heating_areas.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/district_heating_areas.png -------------------------------------------------------------------------------- /docs/images/eGon2035_gas_ind_load_repartition_DE_withTitle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/eGon2035_gas_ind_load_repartition_DE_withTitle.png -------------------------------------------------------------------------------- /docs/images/eGon_emob_MIT_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/eGon_emob_MIT_model.png -------------------------------------------------------------------------------- /docs/images/egon-modell-szenario-egon2035.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/egon-modell-szenario-egon2035.png -------------------------------------------------------------------------------- /docs/images/flowchart_cts_disaggregation.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/flowchart_cts_disaggregation.jpg -------------------------------------------------------------------------------- /docs/images/grid_topology_ehv_hv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/grid_topology_ehv_hv.png -------------------------------------------------------------------------------- /docs/images/ind_gas_demand.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/ind_gas_demand.png -------------------------------------------------------------------------------- /docs/images/offshore_power_plants_areas.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/offshore_power_plants_areas.png -------------------------------------------------------------------------------- /docs/images/regions_DLR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/regions_DLR.png -------------------------------------------------------------------------------- /docs/images/residential_heat_demand.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/residential_heat_demand.png -------------------------------------------------------------------------------- /docs/images/residential_heat_demand_profile_summer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/residential_heat_demand_profile_summer.png -------------------------------------------------------------------------------- /docs/images/residential_heat_demand_profile_winter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/residential_heat_demand_profile_winter.png -------------------------------------------------------------------------------- /docs/images/shifted_dsm-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/shifted_dsm-example.png -------------------------------------------------------------------------------- /docs/images/table_max_capacity_DLR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/docs/images/table_max_capacity_DLR.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Contents 3 | ======== 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | 8 | about 9 | workflow 10 | getting_started 11 | troubleshooting 12 | data 13 | literature 14 | contributing 15 | authors 16 | changelog 17 | API Reference: egon.data 18 | 19 | Indices and tables 20 | ================== 21 | 22 | * :ref:`genindex` 23 | * :ref:`modindex` 24 | * :ref:`search` 25 | -------------------------------------------------------------------------------- /docs/reference/egon.data.airflow.dags.pipeline.rst: -------------------------------------------------------------------------------- 1 | pipeline 2 | ======== 3 | 4 | .. automodule:: egon.data.airflow.dags.pipeline 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.airflow.dags.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | dags 4 | ==== 5 | 6 | .. py:module:: egon.data.airflow.dags 7 | 8 | .. toctree:: 9 | :maxdepth: 1 10 | 11 | egon.data.airflow.dags.pipeline 12 | -------------------------------------------------------------------------------- /docs/reference/egon.data.airflow.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | airflow 4 | ======= 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | egon.data.airflow.dags 10 | -------------------------------------------------------------------------------- /docs/reference/egon.data.cli.rst: -------------------------------------------------------------------------------- 1 | cli 2 | === 3 | 4 | .. automodule:: egon.data.cli 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.config.rst: -------------------------------------------------------------------------------- 1 | config 2 | ====== 3 | 4 | .. automodule:: egon.data.config 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.DSM_cts_ind.rst: -------------------------------------------------------------------------------- 1 | DSM\_cts\_ind 2 | ============= 3 | 4 | .. automodule:: egon.data.datasets.DSM_cts_ind 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.calculate_dlr.rst: -------------------------------------------------------------------------------- 1 | calculate\_dlr 2 | ============== 3 | 4 | .. automodule:: egon.data.datasets.calculate_dlr 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.ch4_prod.rst: -------------------------------------------------------------------------------- 1 | ch4\_prod 2 | ========= 3 | 4 | .. automodule:: egon.data.datasets.ch4_prod 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.ch4_storages.rst: -------------------------------------------------------------------------------- 1 | ch4\_storages 2 | ============= 3 | 4 | .. automodule:: egon.data.datasets.ch4_storages 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.chp.match_nep.rst: -------------------------------------------------------------------------------- 1 | match\_nep 2 | ========== 3 | 4 | .. automodule:: egon.data.datasets.chp.match_nep 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.chp.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | chp 4 | === 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.chp.match_nep 9 | egon.data.datasets.chp.small_chp 10 | 11 | 12 | 13 | .. automodule:: egon.data.datasets.chp 14 | :members: 15 | :undoc-members: 16 | :show-inheritance: 17 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.chp.small_chp.rst: -------------------------------------------------------------------------------- 1 | small\_chp 2 | ========== 3 | 4 | .. automodule:: egon.data.datasets.chp.small_chp 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.chp_etrago.rst: -------------------------------------------------------------------------------- 1 | chp\_etrago 2 | =========== 3 | 4 | .. automodule:: egon.data.datasets.chp_etrago 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.data_bundle.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | data\_bundle 4 | ============ 5 | 6 | 7 | .. automodule:: egon.data.datasets.data_bundle 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.database.rst: -------------------------------------------------------------------------------- 1 | database 2 | ======== 3 | 4 | .. automodule:: egon.data.datasets.database 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.demandregio.install_disaggregator.rst: -------------------------------------------------------------------------------- 1 | install\_disaggregator 2 | ====================== 3 | 4 | .. automodule:: egon.data.datasets.demandregio.install_disaggregator 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.demandregio.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | demandregio 4 | =========== 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.demandregio.install_disaggregator 9 | 10 | 11 | 12 | .. automodule:: egon.data.datasets.demandregio 13 | :members: 14 | :undoc-members: 15 | :show-inheritance: 16 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.district_heating_areas.plot.rst: -------------------------------------------------------------------------------- 1 | plot 2 | ==== 3 | 4 | .. automodule:: egon.data.datasets.district_heating_areas.plot 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.district_heating_areas.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | district\_heating\_areas 4 | ======================== 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.district_heating_areas.plot 9 | 10 | 11 | 12 | .. automodule:: egon.data.datasets.district_heating_areas 13 | :members: 14 | :undoc-members: 15 | :show-inheritance: 16 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.electrical_neighbours.rst: -------------------------------------------------------------------------------- 1 | electrical\_neighbours 2 | ====================== 3 | 4 | .. automodule:: egon.data.datasets.electrical_neighbours 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.electricity_demand.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | electricity\_demand 4 | =================== 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.electricity_demand.temporal 9 | 10 | 11 | 12 | .. automodule:: egon.data.datasets.electricity_demand 13 | :members: 14 | :undoc-members: 15 | :show-inheritance: 16 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.electricity_demand.temporal.rst: -------------------------------------------------------------------------------- 1 | temporal 2 | ======== 3 | 4 | .. automodule:: egon.data.datasets.electricity_demand.temporal 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.electricity_demand_etrago.rst: -------------------------------------------------------------------------------- 1 | electricity\_demand\_etrago 2 | =========================== 3 | 4 | .. automodule:: egon.data.datasets.electricity_demand_etrago 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.electricity_demand_timeseries.cts_buildings.rst: -------------------------------------------------------------------------------- 1 | cts\_buildings 2 | ============== 3 | 4 | .. automodule:: egon.data.datasets.electricity_demand_timeseries.cts_buildings 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.electricity_demand_timeseries.hh_buildings.rst: -------------------------------------------------------------------------------- 1 | hh\_buildings 2 | ============= 3 | 4 | .. automodule:: egon.data.datasets.electricity_demand_timeseries.hh_buildings 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.electricity_demand_timeseries.hh_profiles.rst: -------------------------------------------------------------------------------- 1 | hh\_profiles 2 | ============ 3 | 4 | .. automodule:: egon.data.datasets.electricity_demand_timeseries.hh_profiles 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.electricity_demand_timeseries.mapping.rst: -------------------------------------------------------------------------------- 1 | mapping 2 | ======= 3 | 4 | .. automodule:: egon.data.datasets.electricity_demand_timeseries.mapping 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.electricity_demand_timeseries.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | electricity\_demand\_timeseries 4 | =============================== 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.electricity_demand_timeseries.cts_buildings 9 | egon.data.datasets.electricity_demand_timeseries.hh_buildings 10 | egon.data.datasets.electricity_demand_timeseries.hh_profiles 11 | egon.data.datasets.electricity_demand_timeseries.mapping 12 | egon.data.datasets.electricity_demand_timeseries.tools 13 | 14 | 15 | 16 | .. automodule:: egon.data.datasets.electricity_demand_timeseries 17 | :members: 18 | :undoc-members: 19 | :show-inheritance: 20 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.electricity_demand_timeseries.tools.rst: -------------------------------------------------------------------------------- 1 | tools 2 | ===== 3 | 4 | .. automodule:: egon.data.datasets.electricity_demand_timeseries.tools 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.emobility.heavy_duty_transport.create_h2_buses.rst: -------------------------------------------------------------------------------- 1 | create\_h2\_buses 2 | ================= 3 | 4 | .. automodule:: egon.data.datasets.emobility.heavy_duty_transport.create_h2_buses 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.emobility.heavy_duty_transport.data_io.rst: -------------------------------------------------------------------------------- 1 | data\_io 2 | ======== 3 | 4 | .. automodule:: egon.data.datasets.emobility.heavy_duty_transport.data_io 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.emobility.heavy_duty_transport.db_classes.rst: -------------------------------------------------------------------------------- 1 | db\_classes 2 | =========== 3 | 4 | .. automodule:: egon.data.datasets.emobility.heavy_duty_transport.db_classes 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.emobility.heavy_duty_transport.h2_demand_distribution.rst: -------------------------------------------------------------------------------- 1 | h2\_demand\_distribution 2 | ======================== 3 | 4 | .. automodule:: egon.data.datasets.emobility.heavy_duty_transport.h2_demand_distribution 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.emobility.heavy_duty_transport.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | heavy\_duty\_transport 4 | ====================== 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.emobility.heavy_duty_transport.create_h2_buses 9 | egon.data.datasets.emobility.heavy_duty_transport.data_io 10 | egon.data.datasets.emobility.heavy_duty_transport.db_classes 11 | egon.data.datasets.emobility.heavy_duty_transport.h2_demand_distribution 12 | 13 | 14 | 15 | .. automodule:: egon.data.datasets.emobility.heavy_duty_transport 16 | :members: 17 | :undoc-members: 18 | :show-inheritance: 19 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.emobility.motorized_individual_travel.db_classes.rst: -------------------------------------------------------------------------------- 1 | db\_classes 2 | =========== 3 | 4 | .. automodule:: egon.data.datasets.emobility.motorized_individual_travel.db_classes 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.emobility.motorized_individual_travel.ev_allocation.rst: -------------------------------------------------------------------------------- 1 | ev\_allocation 2 | ============== 3 | 4 | .. automodule:: egon.data.datasets.emobility.motorized_individual_travel.ev_allocation 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.emobility.motorized_individual_travel.helpers.rst: -------------------------------------------------------------------------------- 1 | helpers 2 | ======= 3 | 4 | .. automodule:: egon.data.datasets.emobility.motorized_individual_travel.helpers 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.emobility.motorized_individual_travel.model_timeseries.rst: -------------------------------------------------------------------------------- 1 | model\_timeseries 2 | ================= 3 | 4 | .. automodule:: egon.data.datasets.emobility.motorized_individual_travel.model_timeseries 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.emobility.motorized_individual_travel.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | motorized\_individual\_travel 4 | ============================= 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.emobility.motorized_individual_travel.db_classes 9 | egon.data.datasets.emobility.motorized_individual_travel.ev_allocation 10 | egon.data.datasets.emobility.motorized_individual_travel.helpers 11 | egon.data.datasets.emobility.motorized_individual_travel.model_timeseries 12 | egon.data.datasets.emobility.motorized_individual_travel.tests 13 | 14 | 15 | 16 | .. automodule:: egon.data.datasets.emobility.motorized_individual_travel 17 | :members: 18 | :undoc-members: 19 | :show-inheritance: 20 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.emobility.motorized_individual_travel.tests.rst: -------------------------------------------------------------------------------- 1 | tests 2 | ===== 3 | 4 | .. automodule:: egon.data.datasets.emobility.motorized_individual_travel.tests 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure.db_classes.rst: -------------------------------------------------------------------------------- 1 | db\_classes 2 | =========== 3 | 4 | .. automodule:: egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure.db_classes 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure.infrastructure_allocation.rst: -------------------------------------------------------------------------------- 1 | infrastructure\_allocation 2 | ========================== 3 | 4 | .. automodule:: egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure.infrastructure_allocation 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | motorized\_individual\_travel\_charging\_infrastructure 4 | ======================================================= 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure.db_classes 9 | egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure.infrastructure_allocation 10 | egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure.use_cases 11 | 12 | 13 | 14 | .. automodule:: egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure 15 | :members: 16 | :undoc-members: 17 | :show-inheritance: 18 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure.use_cases.rst: -------------------------------------------------------------------------------- 1 | use\_cases 2 | ========== 3 | 4 | .. automodule:: egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure.use_cases 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.emobility.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | emobility 4 | ========= 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.emobility.heavy_duty_transport 9 | egon.data.datasets.emobility.motorized_individual_travel 10 | egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure 11 | 12 | .. automodule:: egon.data.datasets.emobility 13 | :members: 14 | :undoc-members: 15 | :show-inheritance: 16 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.era5.rst: -------------------------------------------------------------------------------- 1 | era5 2 | ==== 3 | 4 | .. automodule:: egon.data.datasets.era5 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.etrago_helpers.rst: -------------------------------------------------------------------------------- 1 | etrago\_helpers 2 | =============== 3 | 4 | .. automodule:: egon.data.datasets.etrago_helpers 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.etrago_setup.rst: -------------------------------------------------------------------------------- 1 | etrago\_setup 2 | ============= 3 | 4 | .. automodule:: egon.data.datasets.etrago_setup 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.fill_etrago_gen.rst: -------------------------------------------------------------------------------- 1 | fill\_etrago\_gen 2 | ================= 3 | 4 | .. automodule:: egon.data.datasets.fill_etrago_gen 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.fix_ehv_subnetworks.rst: -------------------------------------------------------------------------------- 1 | fix\_ehv\_subnetworks 2 | ===================== 3 | 4 | .. automodule:: egon.data.datasets.fix_ehv_subnetworks 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.gas_areas.rst: -------------------------------------------------------------------------------- 1 | gas\_areas 2 | ========== 3 | 4 | .. automodule:: egon.data.datasets.gas_areas 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.gas_grid.rst: -------------------------------------------------------------------------------- 1 | gas\_grid 2 | ========= 3 | 4 | .. automodule:: egon.data.datasets.gas_grid 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.gas_neighbours.eGon100RE.rst: -------------------------------------------------------------------------------- 1 | eGon100RE 2 | ========= 3 | 4 | .. automodule:: egon.data.datasets.gas_neighbours.eGon100RE 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.gas_neighbours.eGon2035.rst: -------------------------------------------------------------------------------- 1 | eGon2035 2 | ======== 3 | 4 | .. automodule:: egon.data.datasets.gas_neighbours.eGon2035 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.gas_neighbours.gas_abroad.rst: -------------------------------------------------------------------------------- 1 | gas\_abroad 2 | =========== 3 | 4 | .. automodule:: egon.data.datasets.gas_neighbours.gas_abroad 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.gas_neighbours.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | gas\_neighbours 4 | =============== 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.gas_neighbours.eGon100RE 9 | egon.data.datasets.gas_neighbours.eGon2035 10 | egon.data.datasets.gas_neighbours.gas_abroad 11 | 12 | 13 | 14 | .. automodule:: egon.data.datasets.gas_neighbours 15 | :members: 16 | :undoc-members: 17 | :show-inheritance: 18 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.generate_voronoi.rst: -------------------------------------------------------------------------------- 1 | generate\_voronoi 2 | ================= 3 | 4 | .. automodule:: egon.data.datasets.generate_voronoi 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.heat_demand.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | heat\_demand 4 | ============ 5 | 6 | 7 | .. automodule:: egon.data.datasets.heat_demand 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.heat_demand_europe.rst: -------------------------------------------------------------------------------- 1 | heat\_demand\_europe 2 | ==================== 3 | 4 | .. automodule:: egon.data.datasets.heat_demand_europe 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.heat_demand_timeseries.daily.rst: -------------------------------------------------------------------------------- 1 | daily 2 | ===== 3 | 4 | .. automodule:: egon.data.datasets.heat_demand_timeseries.daily 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.heat_demand_timeseries.idp_pool.rst: -------------------------------------------------------------------------------- 1 | idp\_pool 2 | ========= 3 | 4 | .. automodule:: egon.data.datasets.heat_demand_timeseries.idp_pool 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.heat_demand_timeseries.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | heat\_demand\_timeseries 4 | ======================== 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.heat_demand_timeseries.daily 9 | egon.data.datasets.heat_demand_timeseries.idp_pool 10 | egon.data.datasets.heat_demand_timeseries.service_sector 11 | 12 | 13 | 14 | .. automodule:: egon.data.datasets.heat_demand_timeseries 15 | :members: 16 | :undoc-members: 17 | :show-inheritance: 18 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.heat_demand_timeseries.service_sector.rst: -------------------------------------------------------------------------------- 1 | service\_sector 2 | =============== 3 | 4 | .. automodule:: egon.data.datasets.heat_demand_timeseries.service_sector 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.heat_etrago.hts_etrago.rst: -------------------------------------------------------------------------------- 1 | hts\_etrago 2 | =========== 3 | 4 | .. automodule:: egon.data.datasets.heat_etrago.hts_etrago 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.heat_etrago.power_to_heat.rst: -------------------------------------------------------------------------------- 1 | power\_to\_heat 2 | =============== 3 | 4 | .. automodule:: egon.data.datasets.heat_etrago.power_to_heat 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.heat_etrago.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | heat\_etrago 4 | ============ 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.heat_etrago.hts_etrago 9 | egon.data.datasets.heat_etrago.power_to_heat 10 | 11 | 12 | 13 | .. automodule:: egon.data.datasets.heat_etrago 14 | :members: 15 | :undoc-members: 16 | :show-inheritance: 17 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.heat_supply.district_heating.rst: -------------------------------------------------------------------------------- 1 | district\_heating 2 | ================= 3 | 4 | .. automodule:: egon.data.datasets.heat_supply.district_heating 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.heat_supply.geothermal.rst: -------------------------------------------------------------------------------- 1 | geothermal 2 | ========== 3 | 4 | .. automodule:: egon.data.datasets.heat_supply.geothermal 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.heat_supply.individual_heating.rst: -------------------------------------------------------------------------------- 1 | individual\_heating 2 | =================== 3 | 4 | .. automodule:: egon.data.datasets.heat_supply.individual_heating 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.heat_supply.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | heat\_supply 4 | ============ 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.heat_supply.district_heating 9 | egon.data.datasets.heat_supply.geothermal 10 | egon.data.datasets.heat_supply.individual_heating 11 | 12 | 13 | 14 | .. automodule:: egon.data.datasets.heat_supply 15 | :members: 16 | :undoc-members: 17 | :show-inheritance: 18 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.hydrogen_etrago.bus.rst: -------------------------------------------------------------------------------- 1 | bus 2 | === 3 | 4 | .. automodule:: egon.data.datasets.hydrogen_etrago.bus 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.hydrogen_etrago.h2_grid.rst: -------------------------------------------------------------------------------- 1 | h2\_grid 2 | ======== 3 | 4 | .. automodule:: egon.data.datasets.hydrogen_etrago.h2_grid 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.hydrogen_etrago.h2_to_ch4.rst: -------------------------------------------------------------------------------- 1 | h2\_to\_ch4 2 | =========== 3 | 4 | .. automodule:: egon.data.datasets.hydrogen_etrago.h2_to_ch4 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.hydrogen_etrago.power_to_h2.rst: -------------------------------------------------------------------------------- 1 | power\_to\_h2 2 | ============= 3 | 4 | .. automodule:: egon.data.datasets.hydrogen_etrago.power_to_h2 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.hydrogen_etrago.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | hydrogen\_etrago 4 | ================ 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.hydrogen_etrago.bus 9 | egon.data.datasets.hydrogen_etrago.h2_grid 10 | egon.data.datasets.hydrogen_etrago.h2_to_ch4 11 | egon.data.datasets.hydrogen_etrago.power_to_h2 12 | egon.data.datasets.hydrogen_etrago.storage 13 | 14 | 15 | 16 | .. automodule:: egon.data.datasets.hydrogen_etrago 17 | :members: 18 | :undoc-members: 19 | :show-inheritance: 20 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.hydrogen_etrago.storage.rst: -------------------------------------------------------------------------------- 1 | storage 2 | ======= 3 | 4 | .. automodule:: egon.data.datasets.hydrogen_etrago.storage 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.industrial_gas_demand.rst: -------------------------------------------------------------------------------- 1 | industrial\_gas\_demand 2 | ======================= 3 | 4 | .. automodule:: egon.data.datasets.industrial_gas_demand 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.industrial_sites.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | industrial\_sites 4 | ================= 5 | 6 | 7 | .. automodule:: egon.data.datasets.industrial_sites 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.industry.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | industry 4 | ======== 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.industry.temporal 9 | 10 | 11 | 12 | .. automodule:: egon.data.datasets.industry 13 | :members: 14 | :undoc-members: 15 | :show-inheritance: 16 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.industry.temporal.rst: -------------------------------------------------------------------------------- 1 | temporal 2 | ======== 3 | 4 | .. automodule:: egon.data.datasets.industry.temporal 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.loadarea.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | loadarea 4 | ======== 5 | 6 | 7 | .. automodule:: egon.data.datasets.loadarea 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.low_flex_scenario.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | low\_flex\_scenario 4 | =================== 5 | 6 | 7 | .. automodule:: egon.data.datasets.low_flex_scenario 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.mastr.rst: -------------------------------------------------------------------------------- 1 | mastr 2 | ===== 3 | 4 | .. automodule:: egon.data.datasets.mastr 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.mv_grid_districts.rst: -------------------------------------------------------------------------------- 1 | mv\_grid\_districts 2 | =================== 3 | 4 | .. automodule:: egon.data.datasets.mv_grid_districts 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.osm.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | osm 4 | === 5 | 6 | 7 | .. automodule:: egon.data.datasets.osm 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.osm_buildings_streets.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | osm\_buildings\_streets 4 | ======================= 5 | 6 | 7 | .. automodule:: egon.data.datasets.osm_buildings_streets 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.osmtgmod.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | osmtgmod 4 | ======== 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.osmtgmod.substation 9 | 10 | 11 | 12 | .. automodule:: egon.data.datasets.osmtgmod 13 | :members: 14 | :undoc-members: 15 | :show-inheritance: 16 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.osmtgmod.substation.rst: -------------------------------------------------------------------------------- 1 | substation 2 | ========== 3 | 4 | .. automodule:: egon.data.datasets.osmtgmod.substation 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.power_etrago.match_ocgt.rst: -------------------------------------------------------------------------------- 1 | match\_ocgt 2 | =========== 3 | 4 | .. automodule:: egon.data.datasets.power_etrago.match_ocgt 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.power_etrago.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | power\_etrago 4 | ============= 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.power_etrago.match_ocgt 9 | 10 | 11 | 12 | .. automodule:: egon.data.datasets.power_etrago 13 | :members: 14 | :undoc-members: 15 | :show-inheritance: 16 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.power_plants.assign_weather_data.rst: -------------------------------------------------------------------------------- 1 | assign\_weather\_data 2 | ===================== 3 | 4 | .. automodule:: egon.data.datasets.power_plants.assign_weather_data 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.power_plants.conventional.rst: -------------------------------------------------------------------------------- 1 | conventional 2 | ============ 3 | 4 | .. automodule:: egon.data.datasets.power_plants.conventional 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.power_plants.mastr.rst: -------------------------------------------------------------------------------- 1 | mastr 2 | ===== 3 | 4 | .. automodule:: egon.data.datasets.power_plants.mastr 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.power_plants.pv_ground_mounted.rst: -------------------------------------------------------------------------------- 1 | pv\_ground\_mounted 2 | =================== 3 | 4 | .. automodule:: egon.data.datasets.power_plants.pv_ground_mounted 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.power_plants.pv_rooftop.rst: -------------------------------------------------------------------------------- 1 | pv\_rooftop 2 | =========== 3 | 4 | .. automodule:: egon.data.datasets.power_plants.pv_rooftop 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.power_plants.pv_rooftop_buildings.rst: -------------------------------------------------------------------------------- 1 | pv\_rooftop\_buildings 2 | ====================== 3 | 4 | .. automodule:: egon.data.datasets.power_plants.pv_rooftop_buildings 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.power_plants.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | power\_plants 4 | ============= 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.power_plants.assign_weather_data 9 | egon.data.datasets.power_plants.conventional 10 | egon.data.datasets.power_plants.mastr 11 | egon.data.datasets.power_plants.pv_ground_mounted 12 | egon.data.datasets.power_plants.pv_rooftop 13 | egon.data.datasets.power_plants.pv_rooftop_buildings 14 | egon.data.datasets.power_plants.wind_farms 15 | egon.data.datasets.power_plants.wind_offshore 16 | 17 | 18 | 19 | .. automodule:: egon.data.datasets.power_plants 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.power_plants.wind_farms.rst: -------------------------------------------------------------------------------- 1 | wind\_farms 2 | =========== 3 | 4 | .. automodule:: egon.data.datasets.power_plants.wind_farms 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.power_plants.wind_offshore.rst: -------------------------------------------------------------------------------- 1 | wind\_offshore 2 | ============== 3 | 4 | .. automodule:: egon.data.datasets.power_plants.wind_offshore 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.pypsaeursec.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | pypsaeursec 4 | =========== 5 | 6 | 7 | .. automodule:: egon.data.datasets.pypsaeursec 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.re_potential_areas.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | re\_potential\_areas 4 | ==================== 5 | 6 | 7 | .. automodule:: egon.data.datasets.re_potential_areas 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.renewable_feedin.rst: -------------------------------------------------------------------------------- 1 | renewable\_feedin 2 | ================= 3 | 4 | .. automodule:: egon.data.datasets.renewable_feedin 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | datasets 4 | ======== 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.DSM_cts_ind 9 | egon.data.datasets.calculate_dlr 10 | egon.data.datasets.ch4_prod 11 | egon.data.datasets.ch4_storages 12 | egon.data.datasets.chp_etrago 13 | egon.data.datasets.database 14 | egon.data.datasets.electrical_neighbours 15 | egon.data.datasets.electricity_demand_etrago 16 | egon.data.datasets.era5 17 | egon.data.datasets.etrago_helpers 18 | egon.data.datasets.etrago_setup 19 | egon.data.datasets.fill_etrago_gen 20 | egon.data.datasets.fix_ehv_subnetworks 21 | egon.data.datasets.gas_areas 22 | egon.data.datasets.gas_grid 23 | egon.data.datasets.generate_voronoi 24 | egon.data.datasets.heat_demand_europe 25 | egon.data.datasets.industrial_gas_demand 26 | egon.data.datasets.mastr 27 | egon.data.datasets.mv_grid_districts 28 | egon.data.datasets.renewable_feedin 29 | egon.data.datasets.sanity_checks 30 | egon.data.datasets.scenario_capacities 31 | egon.data.datasets.society_prognosis 32 | egon.data.datasets.substation_voronoi 33 | egon.data.datasets.tyndp 34 | egon.data.datasets.vg250_mv_grid_districts 35 | egon.data.datasets.zensus_mv_grid_districts 36 | egon.data.datasets.zensus_vg250 37 | egon.data.datasets.chp 38 | egon.data.datasets.data_bundle 39 | egon.data.datasets.demandregio 40 | egon.data.datasets.district_heating_areas 41 | egon.data.datasets.electricity_demand 42 | egon.data.datasets.electricity_demand_timeseries 43 | egon.data.datasets.emobility 44 | egon.data.datasets.gas_neighbours 45 | egon.data.datasets.heat_demand 46 | egon.data.datasets.heat_demand_timeseries 47 | egon.data.datasets.heat_etrago 48 | egon.data.datasets.heat_supply 49 | egon.data.datasets.hydrogen_etrago 50 | egon.data.datasets.industrial_sites 51 | egon.data.datasets.industry 52 | egon.data.datasets.loadarea 53 | egon.data.datasets.low_flex_scenario 54 | egon.data.datasets.osm 55 | egon.data.datasets.osm_buildings_streets 56 | egon.data.datasets.osmtgmod 57 | egon.data.datasets.power_etrago 58 | egon.data.datasets.power_plants 59 | egon.data.datasets.pypsaeursec 60 | egon.data.datasets.re_potential_areas 61 | egon.data.datasets.saltcavern 62 | egon.data.datasets.scenario_parameters 63 | egon.data.datasets.storages 64 | egon.data.datasets.storages_etrago 65 | egon.data.datasets.substation 66 | egon.data.datasets.vg250 67 | egon.data.datasets.zensus 68 | 69 | 70 | 71 | .. automodule:: egon.data.datasets 72 | :members: 73 | :undoc-members: 74 | :show-inheritance: 75 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.saltcavern.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | saltcavern 4 | ========== 5 | 6 | 7 | .. automodule:: egon.data.datasets.saltcavern 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.sanity_checks.rst: -------------------------------------------------------------------------------- 1 | sanity\_checks 2 | ============== 3 | 4 | .. automodule:: egon.data.datasets.sanity_checks 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.scenario_capacities.rst: -------------------------------------------------------------------------------- 1 | scenario\_capacities 2 | ==================== 3 | 4 | .. automodule:: egon.data.datasets.scenario_capacities 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.scenario_parameters.parameters.rst: -------------------------------------------------------------------------------- 1 | parameters 2 | ========== 3 | 4 | .. automodule:: egon.data.datasets.scenario_parameters.parameters 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.scenario_parameters.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | scenario\_parameters 4 | ==================== 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.scenario_parameters.parameters 9 | 10 | 11 | 12 | .. automodule:: egon.data.datasets.scenario_parameters 13 | :members: 14 | :undoc-members: 15 | :show-inheritance: 16 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.society_prognosis.rst: -------------------------------------------------------------------------------- 1 | society\_prognosis 2 | ================== 3 | 4 | .. automodule:: egon.data.datasets.society_prognosis 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.storages.home_batteries.rst: -------------------------------------------------------------------------------- 1 | home\_batteries 2 | =============== 3 | 4 | .. automodule:: egon.data.datasets.storages.home_batteries 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.storages.pumped_hydro.rst: -------------------------------------------------------------------------------- 1 | pumped\_hydro 2 | ============= 3 | 4 | .. automodule:: egon.data.datasets.storages.pumped_hydro 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.storages.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | storages 4 | ======== 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | egon.data.datasets.storages.home_batteries 9 | egon.data.datasets.storages.pumped_hydro 10 | 11 | 12 | 13 | .. automodule:: egon.data.datasets.storages 14 | :members: 15 | :undoc-members: 16 | :show-inheritance: 17 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.storages_etrago.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | storages\_etrago 4 | ================ 5 | 6 | 7 | .. automodule:: egon.data.datasets.storages_etrago 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.substation.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | substation 4 | ========== 5 | 6 | 7 | .. automodule:: egon.data.datasets.substation 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.substation_voronoi.rst: -------------------------------------------------------------------------------- 1 | substation\_voronoi 2 | =================== 3 | 4 | .. automodule:: egon.data.datasets.substation_voronoi 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.tyndp.rst: -------------------------------------------------------------------------------- 1 | tyndp 2 | ===== 3 | 4 | .. automodule:: egon.data.datasets.tyndp 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.vg250.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | vg250 4 | ===== 5 | 6 | 7 | .. automodule:: egon.data.datasets.vg250 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.vg250_mv_grid_districts.rst: -------------------------------------------------------------------------------- 1 | vg250\_mv\_grid\_districts 2 | ========================== 3 | 4 | .. automodule:: egon.data.datasets.vg250_mv_grid_districts 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.zensus.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | zensus 4 | ====== 5 | 6 | 7 | .. automodule:: egon.data.datasets.zensus 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.zensus_mv_grid_districts.rst: -------------------------------------------------------------------------------- 1 | zensus\_mv\_grid\_districts 2 | =========================== 3 | 4 | .. automodule:: egon.data.datasets.zensus_mv_grid_districts 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.datasets.zensus_vg250.rst: -------------------------------------------------------------------------------- 1 | zensus\_vg250 2 | ============= 3 | 4 | .. automodule:: egon.data.datasets.zensus_vg250 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.db.rst: -------------------------------------------------------------------------------- 1 | db 2 | == 3 | 4 | .. automodule:: egon.data.db 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.metadata.rst: -------------------------------------------------------------------------------- 1 | metadata 2 | ======== 3 | 4 | .. automodule:: egon.data.metadata 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/reference/egon.data.rst: -------------------------------------------------------------------------------- 1 | egon.data 2 | ========= 3 | 4 | .. automodule:: egon.data 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | .. toctree:: 10 | egon.data.airflow 11 | egon.data.cli 12 | egon.data.config 13 | egon.data.datasets 14 | egon.data.db 15 | egon.data.metadata 16 | egon.data.subprocess 17 | -------------------------------------------------------------------------------- /docs/reference/egon.data.subprocess.rst: -------------------------------------------------------------------------------- 1 | subprocess 2 | ========== 3 | 4 | .. automodule:: egon.data.subprocess 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # See the accompanying commit message for details on the upper bound 2 | # for Sphinx's version. In short, until Sphinx's issue [10479][0] or our 3 | # issues with Python >= 3.9 are fixed, the upper bound is needed. 4 | # 5 | # [0]: https://github.com/sphinx-doc/sphinx/issues/10479 6 | sphinx>=1.3,<4.4 7 | sphinx-rtd-theme 8 | -------------------------------------------------------------------------------- /docs/spelling_wordlist.txt: -------------------------------------------------------------------------------- 1 | builtin 2 | builtins 3 | classmethod 4 | staticmethod 5 | classmethods 6 | staticmethods 7 | args 8 | kwargs 9 | callstack 10 | Changelog 11 | Indices 12 | -------------------------------------------------------------------------------- /docs/workflow.rst: -------------------------------------------------------------------------------- 1 | ******** 2 | Workflow 3 | ******** 4 | 5 | Workflow management 6 | ------------------- 7 | 8 | Execution 9 | --------- 10 | 11 | In principle egon-data is not limited to the use of a specific programming language as the workflow integrates different scripts using Apache Airflow, but Python and SQL are widely used within the process. Apache Airflow organizes the order of execution of processing steps through so-called operators. In the default case the SQL processing is executed on a containerized local PostgreSQL database using Docker. For further information on Docker and its installation please refer to their `documentation `_. Connection information of our local Docker database are defined in the corresponding `docker-compose.yml `_ 12 | 13 | The egon-data workflow is composed of four different sections: database setup, data import, data processing and data export to the OpenEnergy Platform. Each section consists of different tasks, which are managed by Apache Airflow and correspond with the local database. 14 | Only final datasets which function as an input for the optimization tools or selected interim results are uploaded to the `Open Energy Platform `_. 15 | The data processing in egon-data needs to be performed locally as calculations on the Open Energy Platform are prohibited. 16 | More information on how to run the workflow can be found in the `getting started section `_ of our documentation. 17 | 18 | .. _DP_workflow_sketch: 19 | .. figure:: images/DP_Workflow_15012021.svg 20 | 21 | 22 | Versioning 23 | ---------- 24 | 25 | .. warning:: 26 | Please note, the following is not implemented yet, but we are working on it. 27 | 28 | Source code and data are versioned independendly from each other. Every data table uploaded to the Open Energy Platform contains a column 'version' which is used to identify different versions of the same data set. The version number is maintained for every table separately. This is a major difference to the versioning concept applied in the former data processing where all (interim) results were versioned under the same version number. 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 79 3 | target-version = ["py36", "py37", "py38"] 4 | include = '\.pyi?$' 5 | exclude = ''' 6 | /( 7 | \.git 8 | | \.tox 9 | | build 10 | | ci 11 | | dist 12 | )/ 13 | ''' 14 | 15 | [build-system] 16 | requires = ["setuptools>=41.0", "wheel"] 17 | build-backend = "setuptools.build_meta" 18 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | [flake8] 5 | max-line-length = 79 6 | extend-ignore = E203, W503 7 | exclude = .tox,.eggs,ci/bootstrap.py,ci/templates,build,dist 8 | 9 | [tool:pytest] 10 | # If a pytest section is found in one of the possible config files 11 | # (pytest.ini, tox.ini or setup.cfg), then pytest will not look for any others, 12 | # so if you add a pytest config section elsewhere, 13 | # you will need to delete this section from setup.cfg. 14 | norecursedirs = 15 | .git 16 | .tox 17 | .env 18 | dist 19 | build 20 | migrations 21 | 22 | python_files = 23 | test_*.py 24 | *_test.py 25 | tests.py 26 | addopts = 27 | -ra 28 | --strict-markers 29 | --ignore=docs/conf.py 30 | --ignore=setup.py 31 | --ignore=src/egon/data/datasets/pypsaeur/custom_extra_functionality.py 32 | --ignore=.tox/py38-nocov-linux/lib/python3.8/site-packages/egon/data/datasets/pypsaeur/custom_extra_functionality.py 33 | --ignore=ci 34 | --ignore=.eggs 35 | --doctest-modules 36 | --doctest-glob=\*.rst 37 | --tb=short 38 | --pyargs 39 | # The order of these options matters. testpaths comes after addopts so that 40 | # egon.data in testpaths is interpreted as 41 | # --pyargs egon.data. 42 | # Any tests in the src/ directory (that is, tests installed with the package) 43 | # can be run by any user with pytest --pyargs egon.data. 44 | # Packages that are sensitive to the host machine, most famously NumPy, 45 | # include tests with the installed package so that any user can check 46 | # at any time that everything is working properly. 47 | # If you do choose to make installable tests, this will run the installed 48 | # tests as they are actually installed (same principle as when we ensure that 49 | # we always test the installed version of the package). 50 | # If you have no need for this (and your src/ directory is very large), 51 | # you can save a few milliseconds on testing by telling pytest not to search 52 | # the src/ directory by removing 53 | # --pyargs and egon.data from the options here. 54 | testpaths = 55 | egon.data 56 | tests/ 57 | 58 | [tool:isort] 59 | profile = black 60 | from_first = True 61 | line_length = 79 62 | known_first_party = egon, egon.data 63 | known_third_party = airflow 64 | default_section = THIRDPARTY 65 | skip = .tox,.eggs,ci/bootstrap.py,ci/templates,build,dist 66 | 67 | [matrix] 68 | # This is the configuration for the `./bootstrap.py` script. 69 | # It generates `.travis.yml`, `tox.ini` and `.appveyor.yml`. 70 | # 71 | # Syntax: [alias:] value [!variable[glob]] [&variable[glob]] 72 | # 73 | # alias: 74 | # - is used to generate the tox environment 75 | # - it's optional 76 | # - if not present the alias will be computed from the `value` 77 | # value: 78 | # - a value of "-" means empty 79 | # !variable[glob]: 80 | # - exclude the combination of the current `value` with 81 | # any value matching the `glob` in `variable` 82 | # - can use as many you want 83 | # &variable[glob]: 84 | # - only include the combination of the current `value` 85 | # when there's a value matching `glob` in `variable` 86 | # - can use as many you want 87 | 88 | python_versions = 89 | py36 90 | py37 91 | py38 92 | 93 | dependencies = 94 | # 1.4: Django==1.4.16 !python_versions[py3*] 95 | # 1.5: Django==1.5.11 96 | # 1.6: Django==1.6.8 97 | # 1.7: Django==1.7.1 !python_versions[py26] 98 | # Deps commented above are provided as examples. That's what you would use in a Django project. 99 | 100 | coverage_flags = 101 | cover: true 102 | nocov: false 103 | environment_variables = 104 | - 105 | -------------------------------------------------------------------------------- /src/egon/data/__init__.py: -------------------------------------------------------------------------------- 1 | from textwrap import wrap 2 | 3 | from loguru import logger 4 | import click 5 | import shutil 6 | __version__ = "0.0.0" 7 | 8 | 9 | def echo(message): 10 | prefix, message = message.split(" - ") 11 | lines = message.split("\n") 12 | width = min(72, shutil.get_terminal_size()[0]) 13 | wraps = ["\n".join(wrap(line, width)) for line in lines] 14 | message = "\n".join([prefix] + wraps) 15 | click.echo(message, err=True) 16 | 17 | 18 | logger.remove() 19 | logger.add(echo, colorize=True) 20 | -------------------------------------------------------------------------------- /src/egon/data/__main__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Entrypoint module, in case you use `python -megon.data`. 3 | 4 | 5 | Why does this file exist, and why __main__? For more info, read: 6 | 7 | - https://www.python.org/dev/peps/pep-0338/ 8 | - https://docs.python.org/2/using/cmdline.html#cmdoption-m 9 | - https://docs.python.org/3/using/cmdline.html#cmdoption-m 10 | """ 11 | from egon.data.cli import main 12 | 13 | if __name__ == "__main__": 14 | main() 15 | -------------------------------------------------------------------------------- /src/egon/data/airflow/Dockerfile.postgis: -------------------------------------------------------------------------------- 1 | FROM postgres:16 2 | 3 | RUN apt-get update 4 | RUN apt-get install -y postgresql-16-postgis-3 5 | RUN apt-get update 6 | RUN apt-get install -y postgresql-16-pgrouting 7 | -------------------------------------------------------------------------------- /src/egon/data/airflow/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/src/egon/data/airflow/__init__.py -------------------------------------------------------------------------------- /src/egon/data/airflow/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | egon-data-local-database-service: 4 | user: "{uid}:{gid}" 5 | image: postgis/postgis:16-3 6 | container_name: "{--docker-container-name}" 7 | restart: unless-stopped 8 | build: 9 | context: . 10 | dockerfile: "{airflow}/Dockerfile.postgis" 11 | ports: 12 | - "{--database-host}:{--database-port}:5432" 13 | environment: 14 | POSTGRES_DB: "{--airflow-database-name}" 15 | POSTGRES_USER: "{--database-user}" 16 | POSTGRES_PASSWORD: "{--database-password}" 17 | # Use md5 authentication because of old JDBC driver in osmosis 18 | POSTGRES_INITDB_ARGS: "--auth=md5" 19 | volumes: 20 | - ./database-data/:/var/lib/postgresql/data 21 | - /dev/shm:/dev/shm 22 | -------------------------------------------------------------------------------- /src/egon/data/datasets/data_bundle/__init__.py: -------------------------------------------------------------------------------- 1 | """The central module containing all code dealing with small scale input-data 2 | """ 3 | 4 | 5 | from pathlib import Path 6 | from urllib.request import urlretrieve 7 | import shutil 8 | import zipfile 9 | 10 | from egon.data import config 11 | from egon.data.datasets import Dataset 12 | 13 | 14 | def download(): 15 | """ 16 | Download small scale input data from Zenodo 17 | Parameters 18 | ---------- 19 | 20 | """ 21 | data_bundle_path = Path(".") / "data_bundle_egon_data" 22 | # Delete folder if it already exists 23 | if data_bundle_path.exists() and data_bundle_path.is_dir(): 24 | shutil.rmtree(data_bundle_path) 25 | # Get parameters from config and set download URL 26 | sources = config.datasets()["data-bundle"]["sources"]["zenodo"] 27 | url = ( 28 | f"https://zenodo.org/record/{sources['deposit_id']}/files/" 29 | "data_bundle_egon_data.zip" 30 | ) 31 | target_file = config.datasets()["data-bundle"]["targets"]["file"] 32 | 33 | # check if file exists 34 | if not Path(target_file).exists(): 35 | # Retrieve files 36 | urlretrieve(url, target_file) 37 | 38 | with zipfile.ZipFile(target_file, "r") as zip_ref: 39 | zip_ref.extractall(".") 40 | 41 | powerd_data_bundle_path = Path(".") / "data_bundle_powerd_data" 42 | # Delete folder if it already exists 43 | if powerd_data_bundle_path.exists() and powerd_data_bundle_path.is_dir(): 44 | shutil.rmtree(powerd_data_bundle_path) 45 | 46 | url = f"""https://zenodo.org/record/{sources['deposit_id_powerd']}/files/data_bundle_powerd_data.zip""" 47 | target_file = config.datasets()["data-bundle"]["targets"]["file_powerd"] 48 | 49 | # check if file exists 50 | if not Path(target_file).exists(): 51 | # Retrieve files 52 | urlretrieve(url, target_file) 53 | 54 | with zipfile.ZipFile(target_file, "r") as zip_ref: 55 | zip_ref.extractall(".") 56 | 57 | 58 | class DataBundle(Dataset): 59 | def __init__(self, dependencies): 60 | deposit_id = config.datasets()["data-bundle"]["sources"][ 61 | "zenodo" 62 | ]["deposit_id"] 63 | deposit_id_powerd = config.datasets()["data-bundle"]["sources"][ 64 | "zenodo" 65 | ]["deposit_id"] 66 | super().__init__( 67 | name="DataBundle", 68 | version=f"{deposit_id}-{deposit_id_powerd}-0.0.3", 69 | dependencies=dependencies, 70 | tasks=(download,), 71 | ) 72 | -------------------------------------------------------------------------------- /src/egon/data/datasets/database.py: -------------------------------------------------------------------------------- 1 | import functools 2 | 3 | from egon.data import db 4 | from egon.data.datasets import Dataset 5 | 6 | 7 | def setup(): 8 | """ Initialize the local database used for data processing. """ 9 | engine = db.engine() 10 | with engine.connect().execution_options(autocommit=True) as connection: 11 | for extension in ["hstore", "postgis", "postgis_raster", "pgrouting"]: 12 | connection.execute(f"CREATE EXTENSION IF NOT EXISTS {extension}") 13 | 14 | 15 | Setup = functools.partial( 16 | Dataset, 17 | name="Database Setup", 18 | version="0.0.0", 19 | dependencies=[], 20 | tasks=setup, 21 | ) 22 | -------------------------------------------------------------------------------- /src/egon/data/datasets/demandregio/install_disaggregator.py: -------------------------------------------------------------------------------- 1 | """This module downloads and installs demandregio's disaggregator from GitHub 2 | 3 | """ 4 | from pathlib import Path 5 | from subprocess import check_output 6 | import importlib.util 7 | import os 8 | import shutil 9 | 10 | from egon.data import logger, subprocess 11 | import egon.data.config 12 | 13 | 14 | def clone_and_install(): 15 | """Clone and install repository of demandregio's disaggregator 16 | 17 | Returns 18 | ------- 19 | None. 20 | 21 | """ 22 | 23 | source = egon.data.config.datasets()["demandregio_installation"]["sources"] 24 | 25 | repo_path = Path(".") / ( 26 | egon.data.config.datasets()["demandregio_installation"]["targets"][ 27 | "path" 28 | ] 29 | ) 30 | 31 | try: 32 | status = check_output( 33 | ["git", "status"], cwd=(repo_path / "disaggregator").absolute() 34 | ) 35 | if status.startswith(b"Auf Branch features/pandas-update"): 36 | logger.info("Demandregio cloned and right branch checked out.") 37 | else: 38 | raise ImportError( 39 | "Demandregio cloned but wrong branch checked " 40 | "out. Please checkout features/pandas-update" 41 | ) 42 | spec = importlib.util.find_spec("disaggregator") 43 | if spec is not None: 44 | logger.info("Demandregio is not installed. Installing now.") 45 | # Install disaggregator from path 46 | subprocess.run( 47 | [ 48 | "pip", 49 | "install", 50 | "-e", 51 | (repo_path / "disaggregator").absolute(), 52 | ] 53 | ) 54 | 55 | except subprocess.CalledProcessError and FileNotFoundError: 56 | # Create subfolder 57 | os.makedirs(repo_path, exist_ok=True) 58 | 59 | # Clone from GitHub repository 60 | subprocess.run( 61 | [ 62 | "git", 63 | "clone", 64 | "--single-branch", 65 | "--branch", 66 | source["branch"], 67 | source["git-repository"], 68 | ], 69 | cwd=repo_path, 70 | ) 71 | 72 | # Install disaggregator from path 73 | subprocess.run( 74 | ["pip", "install", "-e", (repo_path / "disaggregator").absolute()] 75 | ) 76 | -------------------------------------------------------------------------------- /src/egon/data/datasets/electricity_demand_timeseries/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/src/egon/data/datasets/electricity_demand_timeseries/__init__.py -------------------------------------------------------------------------------- /src/egon/data/datasets/electricity_demand_timeseries/mapping.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, Integer, String 2 | from sqlalchemy.ext.declarative import declarative_base 3 | from sqlalchemy.types import Boolean 4 | 5 | from egon.data import config, db 6 | 7 | Base = declarative_base() 8 | 9 | 10 | class EgonMapZensusMvgdBuildings(Base): 11 | """ 12 | A final mapping table including all buildings used for residential and 13 | cts, heat and electricity timeseries. Including census cells, mvgd bus_id, 14 | building type (osm or synthetic) 15 | """ 16 | __tablename__ = "egon_map_zensus_mvgd_buildings" 17 | __table_args__ = {"schema": "boundaries"} 18 | 19 | building_id = Column(Integer, primary_key=True) 20 | sector = Column(String, primary_key=True) 21 | zensus_population_id = Column(Integer, index=True) 22 | bus_id = Column(Integer, index=True) 23 | osm = Column(Boolean, index=True) 24 | electricity = Column(Boolean, index=True) 25 | heat = Column(Boolean, index=True) 26 | 27 | scenarios = config.settings()["egon-data"]["--scenarios"] 28 | 29 | def map_all_used_buildings(): 30 | """This function maps all used buildings from OSM and synthetic ones.""" 31 | 32 | EgonMapZensusMvgdBuildings.__table__.drop( 33 | bind=db.engine(), checkfirst=True 34 | ) 35 | EgonMapZensusMvgdBuildings.__table__.create(bind=db.engine()) 36 | 37 | db.execute_sql( 38 | sql_string=f""" 39 | INSERT INTO {EgonMapZensusMvgdBuildings.__table_args__["schema"]}. 40 | {EgonMapZensusMvgdBuildings.__tablename__} 41 | SELECT 42 | bld.id as building_id, 43 | peak.sector, 44 | zensus.id as zensus_population_id, 45 | mvgd.bus_id::integer 46 | FROM ( 47 | SELECT "id"::integer, geom_point 48 | FROM openstreetmap.osm_buildings_synthetic 49 | UNION 50 | SELECT "id"::integer, geom_point 51 | FROM openstreetmap.osm_buildings_filtered 52 | ) AS bld, 53 | demand.egon_building_electricity_peak_loads AS peak, 54 | society.destatis_zensus_population_per_ha 55 | AS zensus, 56 | boundaries.egon_map_zensus_grid_districts AS mvgd 57 | WHERE bld.id = peak.building_id 58 | -- Buildings do not change in the scenarios 59 | AND peak.scenario = '{scenarios[0]}' 60 | AND ST_Within(bld.geom_point, zensus.geom) 61 | AND mvgd.zensus_population_id = zensus.id; 62 | 63 | UPDATE boundaries.egon_map_zensus_mvgd_buildings 64 | SET "osm" = TRUE; 65 | 66 | UPDATE boundaries.egon_map_zensus_mvgd_buildings as bld 67 | SET "osm" = FALSE 68 | FROM ( 69 | SELECT "id"::integer 70 | FROM openstreetmap.osm_buildings_synthetic 71 | ) as synth 72 | WHERE bld.building_id = synth.id; 73 | 74 | UPDATE boundaries.egon_map_zensus_mvgd_buildings 75 | SET "electricity" = TRUE; 76 | 77 | UPDATE boundaries.egon_map_zensus_mvgd_buildings 78 | SET "heat" = FALSE; 79 | 80 | -- Only residentials 81 | UPDATE boundaries.egon_map_zensus_mvgd_buildings as bld 82 | SET "heat" = TRUE 83 | FROM ( 84 | SELECT distinct(building_id) 85 | FROM demand.egon_heat_timeseries_selected_profiles 86 | ) as heat 87 | WHERE bld.building_id = heat.building_id 88 | AND bld.sector = 'residential'; 89 | 90 | -- All electricity cts also are heat cts also 91 | UPDATE boundaries.egon_map_zensus_mvgd_buildings as bld 92 | SET "heat" = TRUE 93 | WHERE bld.sector = 'cts' AND electricity = TRUE; 94 | 95 | 96 | 97 | """ 98 | ) 99 | -------------------------------------------------------------------------------- /src/egon/data/datasets/emobility/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openego/eGon-data/8582b464ab6b05187d759ff1000080c8448d8128/src/egon/data/datasets/emobility/__init__.py -------------------------------------------------------------------------------- /src/egon/data/datasets/emobility/heavy_duty_transport/data_io.py: -------------------------------------------------------------------------------- 1 | """ 2 | Read data from DB and download. 3 | """ 4 | from pathlib import Path 5 | 6 | from loguru import logger 7 | import geopandas as gpd 8 | import pandas as pd 9 | 10 | from egon.data import config 11 | from egon.data.db import select_geodataframe 12 | 13 | DATASET_CFG = config.datasets()["mobility_hgv"] 14 | WORKING_DIR = Path(".", "heavy_duty_transport").resolve() 15 | TESTMODE_OFF = ( 16 | config.settings()["egon-data"]["--dataset-boundary"] == "Everything" 17 | ) 18 | 19 | 20 | def get_data(): 21 | """ 22 | Load all necessary data. 23 | """ 24 | return boundary_gdf(), bast_gdf(), nuts3_gdf() 25 | 26 | 27 | def boundary_gdf(): 28 | """ 29 | Get outer boundary from database. 30 | """ 31 | srid = DATASET_CFG["tables"]["srid"] 32 | 33 | gdf = select_geodataframe( 34 | """ 35 | SELECT id, geometry FROM boundaries.vg250_lan 36 | ORDER BY id 37 | """, 38 | geom_col="geometry", 39 | index_col="id", 40 | ).to_crs(epsg=srid) 41 | 42 | return gdf.dissolve() 43 | 44 | 45 | def bast_gdf(): 46 | """ 47 | Reads BAST data. 48 | """ 49 | sources = DATASET_CFG["original_data"]["sources"] 50 | file = sources["BAST"]["file"] 51 | 52 | path = WORKING_DIR / file 53 | relevant_columns = sources["BAST"]["relevant_columns"] 54 | 55 | df = pd.read_csv( 56 | path, 57 | delimiter=r",", 58 | decimal=r",", 59 | thousands=r".", 60 | encoding="ISO-8859-1", 61 | usecols=relevant_columns, 62 | ) 63 | 64 | init_srid = sources["BAST"]["srid"] 65 | final_srid = DATASET_CFG["tables"]["srid"] 66 | 67 | gdf = gpd.GeoDataFrame( 68 | df[relevant_columns[0]], 69 | geometry=gpd.points_from_xy( 70 | df[relevant_columns[1]], 71 | df[relevant_columns[2]], 72 | crs=f"EPSG:{init_srid}", 73 | ), 74 | ).to_crs(epsg=final_srid) 75 | 76 | logger.debug("Read in BAST data.") 77 | 78 | return gdf 79 | 80 | 81 | def nuts3_gdf(): 82 | """Read in NUTS3 geo shapes.""" 83 | srid = DATASET_CFG["tables"]["srid"] 84 | sql = """ 85 | SELECT nuts as nuts3, geometry FROM boundaries.vg250_krs 86 | WHERE gf = 4 87 | ORDER BY nuts 88 | """ 89 | 90 | gdf = select_geodataframe( 91 | sql, geom_col="geometry", index_col="nuts3" 92 | ).to_crs(epsg=srid) 93 | 94 | gdf["area"] = gdf.geometry.area 95 | 96 | logger.debug("Read in NUTS 3 districts.") 97 | 98 | return gdf 99 | -------------------------------------------------------------------------------- /src/egon/data/datasets/emobility/heavy_duty_transport/db_classes.py: -------------------------------------------------------------------------------- 1 | """ 2 | DB tables / SQLAlchemy ORM classes for heavy duty transport. 3 | """ 4 | 5 | from geoalchemy2 import Geometry 6 | from sqlalchemy import Column, Float, ForeignKey, String 7 | from sqlalchemy.ext.declarative import declarative_base 8 | 9 | from egon.data import config 10 | from egon.data.datasets.scenario_parameters import EgonScenario 11 | 12 | Base = declarative_base() 13 | DATASET_CFG = config.datasets()["mobility_hgv"] 14 | 15 | 16 | class EgonHeavyDutyTransportVoronoi(Base): 17 | """ 18 | Class definition of table demand.egon_heavy_duty_transport_voronoi. 19 | """ 20 | __tablename__ = "egon_heavy_duty_transport_voronoi" 21 | __table_args__ = {"schema": "demand"} 22 | 23 | nuts3 = Column(String, primary_key=True) 24 | geometry = Column(Geometry(srid=DATASET_CFG["tables"]["srid"])) 25 | area = Column(Float) 26 | truck_traffic = Column(Float) 27 | normalized_truck_traffic = Column(Float) 28 | hydrogen_consumption = Column(Float) 29 | scenario = Column(String, ForeignKey(EgonScenario.name), primary_key=True) 30 | -------------------------------------------------------------------------------- /src/egon/data/datasets/emobility/motorized_individual_travel/tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | Sanity checks for motorized individual travel 3 | """ 4 | 5 | from numpy.testing import assert_allclose 6 | 7 | from egon.data.datasets.emobility.motorized_individual_travel.helpers import ( 8 | CONFIG_EV, 9 | ) 10 | 11 | 12 | def validate_electric_vehicles_numbers(dataset_name, ev_data, ev_target): 13 | """Validate cumulative numbers of electric vehicles' distribution. 14 | 15 | Tests 16 | * Check if all cells are not NaN 17 | * Check if total number matches produced results (tolerance: 0.01 %) 18 | 19 | Parameters 20 | ---------- 21 | dataset_name : str 22 | Name of data, used for error printing 23 | ev_data : pd.DataFrame 24 | EV data 25 | ev_target : int 26 | Desired number of EVs 27 | """ 28 | assert not ev_data.isna().any().any() 29 | 30 | assert_allclose( 31 | ev_data[[_ for _ in CONFIG_EV.keys()]].sum().sum(), 32 | ev_target, 33 | rtol=0.001, 34 | err_msg=f"Dataset on EV numbers [{dataset_name}] seems to be flawed.", 35 | ) 36 | -------------------------------------------------------------------------------- /src/egon/data/datasets/gas_neighbours/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | The central module containing definition of the datasets dealing with gas neighbours 3 | """ 4 | 5 | from egon.data.datasets import Dataset 6 | from egon.data import config 7 | from egon.data.datasets.gas_neighbours.eGon100RE import ( 8 | insert_gas_neigbours_eGon100RE, 9 | ) 10 | from egon.data.datasets.gas_neighbours.eGon2035 import ( 11 | grid, 12 | insert_ocgt_abroad, 13 | tyndp_gas_demand, 14 | tyndp_gas_generation, 15 | ) 16 | 17 | 18 | def no_gas_neighbours_required(): 19 | print( 20 | """ 21 | None of the required scenarios need the creation of 22 | foreign gas buses 23 | """ 24 | ) 25 | return None 26 | 27 | 28 | tasks = () 29 | 30 | if "eGon2035" in config.settings()["egon-data"]["--scenarios"]: 31 | tasks = tasks + ( 32 | tyndp_gas_generation, 33 | tyndp_gas_demand, 34 | grid, 35 | insert_ocgt_abroad, 36 | ) 37 | 38 | if "eGon100RE" in config.settings()["egon-data"]["--scenarios"]: 39 | tasks = tasks + (insert_gas_neigbours_eGon100RE,) 40 | 41 | if tasks == (): 42 | tasks = tasks + (no_gas_neighbours_required,) 43 | 44 | 45 | class GasNeighbours(Dataset): 46 | """ 47 | Insert the missing gas data abroad. 48 | 49 | Insert the missing gas data into the database for the scenario 50 | eGon2035 by executing successively the functions 51 | :py:func:`tyndp_gas_generation `, 52 | :py:func:`tyndp_gas_demand `, 53 | :py:func:`grid ` and 54 | :py:func:`insert_ocgt_abroad `, 55 | and for the scenario eGon100RE with the function :py:func:`insert_gas_neigbours_eGon100RE `. 56 | 57 | *Dependencies* 58 | * :py:class:`PypsaEurSec ` 59 | * :py:class:`GasNodesAndPipes ` 60 | * :py:class:`ElectricalNeighbours ` 61 | * :py:class:`HydrogenBusEtrago ` 62 | * :py:class:`GasAreaseGon100RE ` 63 | 64 | *Resulting tables* 65 | * :py:class:`grid.egon_etrago_link ` is extended 66 | * :py:class:`grid.egon_etrago_generator ` is extended 67 | * :py:class:`grid.egon_etrago_load ` is extended 68 | * :py:class:`grid.egon_etrago_store ` is extended 69 | 70 | """ 71 | 72 | #: 73 | name: str = "GasNeighbours" 74 | #: 75 | version: str = "0.0.5" 76 | 77 | def __init__(self, dependencies): 78 | super().__init__( 79 | name=self.name, 80 | version=self.version, 81 | dependencies=dependencies, 82 | tasks=tasks, 83 | ) 84 | -------------------------------------------------------------------------------- /src/egon/data/datasets/generate_voronoi.py: -------------------------------------------------------------------------------- 1 | """ 2 | The central module containing code to create voronoi polygons 3 | 4 | """ 5 | from geoalchemy2.types import Geometry 6 | from geovoronoi import voronoi_regions_from_coords 7 | import geopandas as gpd 8 | 9 | 10 | def get_voronoi_geodataframe(buses, boundary): 11 | """ 12 | Create voronoi polygons for the passed buses within the boundaries. 13 | 14 | Parameters 15 | ---------- 16 | buses : geopandas.GeoDataFrame 17 | Buses to create the voronois for. 18 | 19 | boundary : Multipolygon, Polygon 20 | Bounding box for the voronoi generation. 21 | 22 | Returns 23 | ------- 24 | gdf : geopandas.GeoDataFrame 25 | GeoDataFrame containting the bus_ids and the respective voronoi 26 | polygons. 27 | 28 | """ 29 | buses = buses[buses.geometry.intersects(boundary)] 30 | 31 | coords = buses[["x", "y"]].values # coordinates of the respective buses 32 | 33 | region_polys, region_pts = voronoi_regions_from_coords( 34 | coords, boundary, per_geom=False 35 | ) 36 | 37 | gpd_input_dict = { 38 | "bus_id": [], # original bus_id in the buses dataframe 39 | "geometry": [], # voronoi object 40 | } 41 | 42 | for pt, poly in region_pts.items(): 43 | gpd_input_dict["geometry"] += [region_polys[pt]] 44 | gpd_input_dict["bus_id"] += [buses.iloc[poly[0]]["bus_id"]] 45 | 46 | gdf = gpd.GeoDataFrame(gpd_input_dict) 47 | 48 | # the id column is a relict of older voronoi generation methods 49 | gdf["id"] = gdf.index.values 50 | 51 | return gdf 52 | -------------------------------------------------------------------------------- /src/egon/data/datasets/heat_demand/raster2cells-and-centroids.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO demand.egon_peta_heat ( 2 | demand, sector, scenario, zensus_population_id 3 | ) SELECT 4 | (demands.centroid).val AS demand, 5 | sector, 6 | (regexp_matches(filename, '(ser|res)_hd_([^.]*).*', 'i'))[2] AS scenario, 7 | population.id AS zensus_population_id 8 | FROM ( 9 | SELECT 10 | ST_PixelAsCentroids(rast, 1) AS centroid, 11 | ST_PixelAsPolygons(rast, 1) AS polygon, 12 | '{"res": "residential", "ser": "service"}'::json 13 | ->> substring(filename, 1, 3) AS sector, 14 | filename 15 | FROM {{ source }} 16 | ) AS demands 17 | LEFT JOIN ( 18 | SELECT 19 | id, 20 | geom_point AS point 21 | FROM society.destatis_zensus_population_per_ha 22 | ) AS population 23 | ON ST_Intersects((demands.polygon).geom, population.point) 24 | ; 25 | -------------------------------------------------------------------------------- /src/egon/data/datasets/heat_demand_europe.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # This script is part of eGon-data. 4 | 5 | # license text - to be added. 6 | 7 | """ 8 | Central module containing all code downloading hotmaps heat demand data. 9 | 10 | The 2050 national heat demand of the Hotmaps current policy scenario for 11 | buildings are used in the eGon100RE scenario for assumptions on national 12 | heating demands in European countries, but not for Germany. 13 | The data are downloaded to be used in the PyPSA-Eur-Sec scenario generator 14 | (forked into open_ego). 15 | 16 | """ 17 | 18 | import egon.data.config 19 | from egon.data import subprocess 20 | from egon.data.datasets import Dataset 21 | import os 22 | 23 | 24 | class HeatDemandEurope(Dataset): 25 | """ 26 | Downloads annual heat demands for European countries from hotmaps 27 | 28 | This dataset downloads annual heat demands for all European countries for the year 2050 from 29 | hotmaps and stores the results into files. These are later used by pypsa-eur-sec. 30 | 31 | 32 | *Dependencies* 33 | * :py:func:`Setup ` 34 | 35 | """ 36 | 37 | #: 38 | name: str = "heat-demands-europe" 39 | #: 40 | version: str = ( 41 | egon.data.config.datasets()[ 42 | "hotmaps_current_policy_scenario_heat_demands_buildings" 43 | ]["targets"]["path"] 44 | + "_hotmaps.0.1" 45 | ) 46 | 47 | def __init__(self, dependencies): 48 | super().__init__( 49 | name=self.name, 50 | version=self.version, 51 | dependencies=dependencies, 52 | tasks=(download), 53 | ) 54 | 55 | 56 | def download(): 57 | """ 58 | Download Hotmaps current policy scenario for building heat demands. 59 | 60 | The downloaded data contain residential and non-residential-sector 61 | national heat demands for different years. 62 | 63 | Parameters 64 | ---------- 65 | None 66 | 67 | Returns 68 | ------- 69 | None 70 | 71 | """ 72 | 73 | data_config = egon.data.config.datasets() 74 | 75 | # heat demands 76 | hotmapsheatdemands_config = data_config[ 77 | "hotmaps_current_policy_scenario_heat_demands_buildings" 78 | ] 79 | 80 | target_file = hotmapsheatdemands_config["targets"]["path"] 81 | 82 | if not os.path.isfile(target_file): 83 | subprocess.run( 84 | f"curl { hotmapsheatdemands_config['sources']['url']} > {target_file}", 85 | shell=True, 86 | ) 87 | return None 88 | -------------------------------------------------------------------------------- /src/egon/data/datasets/loadarea/drop_temp_tables.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Drop temp tables, views and sequences 3 | 4 | __copyright__ = "Reiner Lemoine Institut" 5 | __license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" 6 | __url__ = "https://github.com/openego/eGon-data/blob/main/LICENSE" 7 | __author__ = "nesnoj" 8 | */ 9 | 10 | -- From script: osm_landuse_melt.sql 11 | DROP SEQUENCE IF EXISTS openstreetmap.osm_landuse_buffer100_mview_id CASCADE; 12 | DROP MATERIALIZED VIEW IF EXISTS openstreetmap.osm_landuse_buffer100_mview CASCADE; 13 | DROP TABLE IF EXISTS openstreetmap.osm_landuse_melted CASCADE; 14 | 15 | -- From script: census_cells_melt.sql 16 | DROP TABLE IF EXISTS society.egon_destatis_zensus_cells_melted CASCADE; 17 | DROP TABLE IF EXISTS society.egon_destatis_zensus_cells_melted_cluster CASCADE; 18 | DROP MATERIALIZED VIEW IF EXISTS openstreetmap.egon_society_zensus_per_la_mview CASCADE; 19 | 20 | -- From script: osm_landuse_census_cells_melt.sql 21 | DROP TABLE IF EXISTS demand.egon_loadarea_load_collect CASCADE; 22 | DROP TABLE IF EXISTS demand.egon_loadarea_load_collect_buffer100 CASCADE; 23 | DROP TABLE IF EXISTS demand.egon_loadarea_load_melt CASCADE; 24 | DROP MATERIALIZED VIEW IF EXISTS demand.egon_loadarea_load_melt_error_geom_mview CASCADE; 25 | DROP MATERIALIZED VIEW IF EXISTS demand.egon_loadarea_load_melt_error_geom_fix_mview CASCADE; 26 | DROP MATERIALIZED VIEW IF EXISTS demand.egon_loadarea_load_melt_error_2_geom_mview CASCADE; 27 | 28 | -- From script: loadareas_create.sql 29 | DROP MATERIALIZED VIEW IF EXISTS demand.egon_loadarea_smaller100m2_mview CASCADE; 30 | DROP TABLE IF EXISTS openstreetmap.egon_osm_sector_per_griddistrict_1_residential CASCADE; 31 | DROP TABLE IF EXISTS openstreetmap.egon_osm_sector_per_griddistrict_2_cts CASCADE; 32 | DROP TABLE IF EXISTS openstreetmap.egon_osm_sector_per_griddistrict_3_industrial CASCADE; 33 | DROP TABLE IF EXISTS openstreetmap.egon_osm_sector_per_griddistrict_4_agricultural CASCADE; 34 | DROP MATERIALIZED VIEW IF EXISTS demand.egon_loadarea_error_noags_mview CASCADE; 35 | -------------------------------------------------------------------------------- /src/egon/data/datasets/loadarea/osm_landuse_melt.sql: -------------------------------------------------------------------------------- 1 | /* 2 | OSM Loads from landuse 3 | Excludes large scale consumer. 4 | Buffer OSM urban sectors with 100m 5 | Unbuffer buffer with -100m 6 | 7 | __copyright__ = "Reiner Lemoine Institut" 8 | __license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" 9 | __url__ = "https://github.com/openego/eGon-data/blob/main/LICENSE" 10 | __author__ = "Ludee, nesnoj" 11 | */ 12 | 13 | 14 | -- From open_eGo: exclude large scale consumers, not required in eGo^n 15 | -- NOTE: This invalidates the columns 16 | -- sector_area_*, sector_share_* and sector_count_* in table demand.egon_loadarea 17 | -- 18 | --DELETE FROM openstreetmap.osm_landuse 19 | -- WHERE gid IN (SELECT polygon_id FROM model_draft.egon_demand_hv_largescaleconsumer); 20 | 21 | 22 | -- sequence 23 | DROP SEQUENCE IF EXISTS openstreetmap.osm_landuse_buffer100_mview_id CASCADE; 24 | CREATE SEQUENCE openstreetmap.osm_landuse_buffer100_mview_id; 25 | 26 | -- buffer with 100m 27 | DROP MATERIALIZED VIEW IF EXISTS openstreetmap.osm_landuse_buffer100_mview CASCADE; 28 | CREATE MATERIALIZED VIEW openstreetmap.osm_landuse_buffer100_mview AS 29 | SELECT nextval('openstreetmap.osm_landuse_buffer100_mview_id') ::integer AS id, 30 | (ST_DUMP(ST_MULTI(ST_UNION( 31 | ST_BUFFER(geom, 100) 32 | )))).geom ::geometry(Polygon,3035) AS geom 33 | FROM openstreetmap.osm_landuse; 34 | 35 | -- index (id) 36 | CREATE UNIQUE INDEX osm_landuse_buffer100_mview_gid_idx 37 | ON openstreetmap.osm_landuse_buffer100_mview (id); 38 | 39 | -- index GIST (geom) 40 | CREATE INDEX osm_landuse_buffer100_mview_geom_idx 41 | ON openstreetmap.osm_landuse_buffer100_mview USING GIST (geom); 42 | 43 | 44 | -- unbuffer with 100m 45 | DROP TABLE IF EXISTS openstreetmap.osm_landuse_melted CASCADE; 46 | CREATE TABLE openstreetmap.osm_landuse_melted ( 47 | id SERIAL NOT NULL, 48 | area_ha double precision, 49 | geom geometry(Polygon,3035), 50 | CONSTRAINT osm_landuse_melted_pkey PRIMARY KEY (id)); 51 | 52 | -- insert buffer 53 | INSERT INTO openstreetmap.osm_landuse_melted(area_ha,geom) 54 | SELECT ST_AREA(buffer.geom)/10000 ::double precision AS area_ha, 55 | buffer.geom ::geometry(Polygon,3035) AS geom 56 | FROM (SELECT (ST_DUMP(ST_MULTI(ST_UNION( 57 | ST_BUFFER(osm.geom, -100) 58 | )))).geom ::geometry(Polygon,3035) AS geom 59 | FROM openstreetmap.osm_landuse_buffer100_mview AS osm 60 | -- ORDER BY id 61 | ) AS buffer; 62 | 63 | -- index GIST (geom) 64 | CREATE INDEX osm_landuse_melted_geom_idx 65 | ON openstreetmap.osm_landuse_melted USING GIST (geom); 66 | -------------------------------------------------------------------------------- /src/egon/data/datasets/low_flex_scenario/__init__.py: -------------------------------------------------------------------------------- 1 | """The central module to create low flex scenarios 2 | 3 | """ 4 | from airflow.operators.postgres_operator import PostgresOperator 5 | from sqlalchemy.ext.declarative import declarative_base 6 | from importlib_resources import files 7 | 8 | from egon.data.datasets import Dataset 9 | 10 | 11 | Base = declarative_base() 12 | 13 | 14 | class LowFlexScenario(Dataset): 15 | def __init__(self, dependencies): 16 | super().__init__( 17 | name="low_flex_scenario", 18 | version="0.0.1", 19 | dependencies=dependencies, 20 | tasks=( 21 | { 22 | PostgresOperator( 23 | task_id="low_flex_eGon2035", 24 | sql=files(__name__) 25 | .joinpath("low_flex_eGon2035.sql") 26 | .read_text(encoding="utf-8"), 27 | postgres_conn_id="egon_data", 28 | autocommit=True, 29 | ), 30 | }, 31 | ), 32 | ) 33 | -------------------------------------------------------------------------------- /src/egon/data/datasets/mastr.py: -------------------------------------------------------------------------------- 1 | """ 2 | Download Marktstammdatenregister (MaStR) from Zenodo. 3 | 4 | """ 5 | 6 | from pathlib import Path 7 | from urllib.request import urlretrieve 8 | import os 9 | 10 | from egon.data.datasets import Dataset 11 | import egon.data.config 12 | 13 | WORKING_DIR_MASTR_OLD = Path(".", "bnetza_mastr", "dump_2021-05-03") 14 | WORKING_DIR_MASTR_NEW = Path(".", "bnetza_mastr", "dump_2024-01-08") 15 | 16 | 17 | def download_mastr_data(): 18 | """Download MaStR data from Zenodo.""" 19 | 20 | def download(dataset_name, download_dir): 21 | print(f"Downloading dataset {dataset_name} to {download_dir} ...") 22 | # Get parameters from config and set download URL 23 | data_config = egon.data.config.datasets()[dataset_name] 24 | zenodo_files_url = ( 25 | f"https://zenodo.org/record/" 26 | f"{data_config['deposit_id']}/files/" 27 | ) 28 | 29 | files = [] 30 | for technology in data_config["technologies"]: 31 | files.append( 32 | f"{data_config['file_basename']}_{technology}_cleaned.csv" 33 | ) 34 | files.append("location_elec_generation_raw.csv") 35 | 36 | # Retrieve specified files 37 | for filename in files: 38 | if not os.path.isfile(filename): 39 | urlretrieve( 40 | zenodo_files_url + filename, download_dir / filename 41 | ) 42 | 43 | if not os.path.exists(WORKING_DIR_MASTR_OLD): 44 | WORKING_DIR_MASTR_OLD.mkdir(exist_ok=True, parents=True) 45 | if not os.path.exists(WORKING_DIR_MASTR_NEW): 46 | WORKING_DIR_MASTR_NEW.mkdir(exist_ok=True, parents=True) 47 | 48 | download(dataset_name="mastr", download_dir=WORKING_DIR_MASTR_OLD) 49 | download(dataset_name="mastr_new", download_dir=WORKING_DIR_MASTR_NEW) 50 | 51 | 52 | class mastr_data_setup(Dataset): 53 | """ 54 | Download Marktstammdatenregister (MaStR) from Zenodo. 55 | 56 | *Dependencies* 57 | * :py:func:`Setup ` 58 | 59 | The downloaded data incorporates two different datasets: 60 | 61 | Dump 2021-04-30 62 | * Source: https://zenodo.org/records/10480930 63 | * Used technologies: PV plants, wind turbines, biomass, hydro plants, 64 | combustion, nuclear, gsgk, storage 65 | * Data is further processed in the :py:class:`PowerPlants 66 | ` dataset 67 | 68 | Dump 2022-11-17 69 | * Source: https://zenodo.org/records/10480958 70 | * Used technologies: PV plants, wind turbines, biomass, hydro plants 71 | * Data is further processed in module :py:mod:`mastr 72 | ` and :py:class:`PowerPlants 73 | ` 74 | 75 | See documentation section :ref:`mastr-ref` for more information. 76 | 77 | """ 78 | 79 | #: 80 | name: str = "MastrData" 81 | #: 82 | version: str = "0.0.2" 83 | #: 84 | tasks = (download_mastr_data,) 85 | 86 | def __init__(self, dependencies): 87 | super().__init__( 88 | name=self.name, 89 | version=self.version, 90 | dependencies=dependencies, 91 | tasks=self.tasks, 92 | ) 93 | -------------------------------------------------------------------------------- /src/egon/data/datasets/osm_buildings_streets/drop_temp_tables.sql: -------------------------------------------------------------------------------- 1 | -- drop temp tables 2 | 3 | DROP TABLE IF EXISTS openstreetmap.osm_ways_preprocessed; 4 | DROP TABLE IF EXISTS openstreetmap.osm_buildings_with_res_tmp; 5 | DROP TABLE IF EXISTS openstreetmap.osm_buildings_with_res_tmp2; 6 | DROP TABLE IF EXISTS openstreetmap.osm_amenities_in_buildings_tmp; 7 | -------------------------------------------------------------------------------- /src/egon/data/datasets/osm_buildings_streets/osm_amenities_shops_preprocessing.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Original Autor: IsGut (johnrobert@t-online.de) 3 | * Adapted by: nesnoj (jonathan.amme@rl-institut.de) 4 | * 5 | * extract specific amenities and all shops 6 | * preserve relevant columns 7 | */ 8 | 9 | DROP TABLE if exists openstreetmap.osm_amenities_shops_filtered; 10 | CREATE TABLE openstreetmap.osm_amenities_shops_filtered AS 11 | select 12 | pnt.osm_id, 13 | pnt.amenity, 14 | pnt.name, 15 | ST_TRANSFORM(pnt.geom, 3035) AS geom_amenity, 16 | pnt.tags 17 | FROM openstreetmap.osm_point pnt 18 | where 19 | pnt.amenity like 'bar' 20 | or pnt.amenity like 'biergarten' 21 | or pnt.amenity like 'cafe' 22 | or pnt.amenity like 'fast_food' 23 | or pnt.amenity like 'food_court' 24 | or pnt.amenity like 'ice_cream' 25 | or pnt.amenity like 'pub' 26 | or pnt.amenity like 'restaurant' 27 | or pnt.amenity like 'college' 28 | or pnt.amenity like 'driving_school' 29 | or pnt.amenity like 'kindergarten' 30 | or pnt.amenity like 'language_school' 31 | or pnt.amenity like 'library' 32 | or pnt.amenity like 'toy_library' 33 | or pnt.amenity like 'music_school' 34 | or pnt.amenity like 'school' 35 | or pnt.amenity like 'university' 36 | or pnt.amenity like 'car_wash' 37 | or pnt.amenity like 'vehicle_inspection' 38 | or pnt.amenity like 'charging_station' 39 | or pnt.amenity like 'fuel' 40 | or pnt.amenity like 'bank' 41 | or pnt.amenity like 'clinic' 42 | or pnt.amenity like 'dentist' 43 | or pnt.amenity like 'doctors' 44 | or pnt.amenity like 'hospital' 45 | or pnt.amenity like 'nursing_home' 46 | or pnt.amenity like 'pharmacy' 47 | or pnt.amenity like 'social_facility' 48 | or pnt.amenity like 'veterinary' 49 | or pnt.amenity like 'arts_centre' 50 | or pnt.amenity like 'brothel' 51 | or pnt.amenity like 'casino' 52 | or pnt.amenity like 'cinema' 53 | or pnt.amenity like 'community_centre' 54 | or pnt.amenity like 'conference_centre' 55 | or pnt.amenity like 'events_venue' 56 | or pnt.amenity like 'gambling' 57 | or pnt.amenity like 'love_hotel' 58 | or pnt.amenity like 'nightclub' 59 | or pnt.amenity like 'planetarium' 60 | or pnt.amenity like 'social_centre' 61 | or pnt.amenity like 'stripclub' 62 | or pnt.amenity like 'studio' 63 | or pnt.amenity like 'swingerclub' 64 | or pnt.amenity like 'exhibition_centre' 65 | or pnt.amenity like 'theatre' 66 | or pnt.amenity like 'courthouse' 67 | or pnt.amenity like 'embassy' 68 | or pnt.amenity like 'fire_station' 69 | or pnt.amenity like 'police' 70 | or pnt.amenity like 'post_depot' 71 | or pnt.amenity like 'post_office' 72 | or pnt.amenity like 'prison' 73 | or pnt.amenity like 'ranger_station' 74 | or pnt.amenity like 'townhall' 75 | or pnt.amenity like 'animal_boarding' 76 | or pnt.amenity like 'childcare' 77 | or pnt.amenity like 'dive_centre' 78 | or pnt.amenity like 'funeral_hall' 79 | or pnt.amenity like 'gym' 80 | or pnt.amenity like 'internet_cafe' 81 | or pnt.amenity like 'kitchen' 82 | or pnt.amenity like 'monastery' 83 | or pnt.amenity like 'place_of_mourning' 84 | or pnt.amenity like 'place_of_worship' 85 | or pnt.amenity like 'shop' 86 | or pnt.shop IS NOT NULL; 87 | 88 | -- add PK as some osm ids are not unique 89 | ALTER TABLE openstreetmap.osm_amenities_shops_filtered 90 | ADD COLUMN egon_amenity_id SERIAL PRIMARY KEY; 91 | 92 | CREATE INDEX idx_osm_amenities_shops_filtered_geom 93 | ON openstreetmap.osm_amenities_shops_filtered USING gist (geom_amenity); 94 | -------------------------------------------------------------------------------- /src/egon/data/datasets/osm_buildings_streets/osm_buildings_extend_residential.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Original Autor: nesnoj (jonathan.amme@rl-institut.de) 3 | */ 4 | 5 | -------------------------------------------------------------------------------- 6 | -- Extend residential buildings by finding census cells with population but -- 7 | -- no residential buildings before in osm_buildings_filter_residential.sql . -- 8 | -- Mark commercial, retail, office, hotel buildings as residential in those -- 9 | -- cells. -- 10 | -------------------------------------------------------------------------------- 11 | 12 | INSERT INTO openstreetmap.osm_buildings_residential 13 | SELECT * 14 | FROM openstreetmap.osm_buildings_filtered 15 | WHERE id IN ( 16 | SELECT id FROM ( 17 | -- get buildings from filtered table in census cells (by centroid) 18 | SELECT 19 | bld.id, 20 | zensus.grid_id, 21 | zensus.zensus_population_id AS cell_id 22 | FROM openstreetmap.osm_buildings_filtered bld 23 | LEFT JOIN society.egon_destatis_zensus_apartment_building_population_per_ha zensus 24 | ON ST_Within(bld.geom_point, zensus.geom) 25 | WHERE building in ('commercial', 'retail', 'office', 'hotel') 26 | AND zensus.zensus_population_id in ( 27 | -- census cell ids which have population but no res. buildings 28 | SELECT zensus.zensus_population_id 29 | FROM society.egon_destatis_zensus_apartment_building_population_per_ha zensus 30 | LEFT OUTER JOIN openstreetmap.osm_buildings_residential bld 31 | ON ST_Intersects(bld.geom_building, zensus.geom) 32 | WHERE bld.id IS NULL 33 | ) 34 | ) bld2 35 | WHERE bld2.id IS NOT NULL AND bld2.grid_id IS NOT NULL 36 | ) 37 | ; -------------------------------------------------------------------------------- /src/egon/data/datasets/osm_buildings_streets/osm_buildings_extract.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Original Autor: IsGut (johnrobert@t-online.de) 3 | * Adapted by: nesnoj (jonathan.amme@rl-institut.de) 4 | */ 5 | 6 | ------------------------------------------------------------- 7 | -- extract all buildings, calculate area, create centroids -- 8 | ------------------------------------------------------------- 9 | DROP TABLE IF EXISTS openstreetmap.osm_buildings; 10 | CREATE TABLE openstreetmap.osm_buildings AS 11 | SELECT 12 | poly.osm_id, 13 | poly.amenity, 14 | poly.building, 15 | poly.name, 16 | ST_TRANSFORM(poly.geom, 3035) AS geom_building, 17 | ST_AREA(ST_TRANSFORM(poly.geom, 3035)) AS area, 18 | ST_TRANSFORM(ST_CENTROID(poly.geom), 3035) AS geom_point, 19 | poly.tags 20 | FROM openstreetmap.osm_polygon poly 21 | WHERE poly.building IS NOT NULL; 22 | 23 | -- add PK as some osm ids are not unique 24 | ALTER TABLE openstreetmap.osm_buildings ADD COLUMN id SERIAL PRIMARY KEY; 25 | 26 | CREATE INDEX ON openstreetmap.osm_buildings USING gist (geom_building); 27 | CREATE INDEX ON openstreetmap.osm_buildings USING gist (geom_point); 28 | -------------------------------------------------------------------------------- /src/egon/data/datasets/osm_buildings_streets/osm_buildings_filter_residential.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Original Autor: IsGut (johnrobert@t-online.de) 3 | * Adapted by: nesnoj (jonathan.amme@rl-institut.de) 4 | */ 5 | 6 | -------------------------------------------------------------------------- 7 | -- extract residential buildings only, calculate area, create centroids -- 8 | -------------------------------------------------------------------------- 9 | DROP TABLE if exists openstreetmap.osm_buildings_residential; 10 | CREATE TABLE openstreetmap.osm_buildings_residential as 11 | select * 12 | from openstreetmap.osm_buildings bld 13 | where 14 | bld.building like 'yes' 15 | or bld.building like 'apartments' 16 | or bld.building like 'detached' 17 | or bld.building like 'farm' 18 | or bld.building like 'house' 19 | or bld.building like 'residential' 20 | or bld.building like 'semidetached_house' 21 | or bld.building like 'terrace' 22 | or bld.building like 'dormitory' 23 | or bld.building like 'terraced_house' 24 | 25 | -- retirement and assisted homes 26 | or bld.amenity like 'retirement_home' 27 | or ( 28 | bld.amenity like 'social_facility' 29 | and tags::hstore -> 'social_facility' in ('nursing_home', 'assisted_living', 'group_home') 30 | ) 31 | or bld.amenity like 'nursing_home' 32 | or bld.amenity like 'assisted_living' 33 | or bld.amenity like 'group_home'; 34 | 35 | ALTER TABLE openstreetmap.osm_buildings_residential 36 | ADD CONSTRAINT osm_buildings_residential_id_pkey PRIMARY KEY (id); 37 | 38 | CREATE INDEX ON openstreetmap.osm_buildings_residential USING gist (geom_building); 39 | CREATE INDEX ON openstreetmap.osm_buildings_residential USING gist (geom_point); 40 | -------------------------------------------------------------------------------- /src/egon/data/datasets/osm_buildings_streets/osm_buildings_filtered_all_zensus_mapping.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Original Autor: nailend (julian.endres@rl-institut.de) 3 | */ 4 | 5 | ---------------------------------------------------------------- 6 | -- Create mapping table of all filtered buildings to zensus cells -- 7 | ---------------------------------------------------------------- 8 | -- Cells with unpopulated areas are NOT purged 9 | -- A few buildings get lost in SH as they are outside of SH boundaries 10 | -- Effect at DE not yet tested 11 | 12 | DROP TABLE IF EXISTS boundaries.egon_map_zensus_buildings_filtered_all; 13 | CREATE TABLE boundaries.egon_map_zensus_buildings_filtered_all AS 14 | SELECT * FROM ( 15 | SELECT 16 | filtered.id, 17 | zensus.grid_id, 18 | zensus.id AS zensus_population_id 19 | FROM openstreetmap.osm_buildings_filtered AS filtered 20 | LEFT JOIN society.destatis_zensus_population_per_ha AS zensus 21 | ON ST_Within(filtered.geom_point, zensus.geom) 22 | ) AS t_1 23 | WHERE t_1.zensus_population_id IS NOT null; 24 | 25 | ALTER TABLE ONLY boundaries.egon_map_zensus_buildings_filtered_all 26 | ADD CONSTRAINT pk_egon_map_zensus_buildings_filtered_all PRIMARY KEY (id); 27 | 28 | CREATE INDEX idx_egon_map_zensus_buildings_filtered_all_zensus_population_id 29 | ON boundaries.egon_map_zensus_buildings_filtered_all USING btree (zensus_population_id); 30 | -------------------------------------------------------------------------------- /src/egon/data/datasets/osm_buildings_streets/osm_buildings_filtered_zensus_mapping.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Original Autor: IsGut (johnrobert@t-online.de) 3 | * Adapted by: nesnoj (jonathan.amme@rl-institut.de) 4 | */ 5 | 6 | ---------------------------------------------------------------- 7 | -- Create mapping table of filtered buildings to zensus cells -- 8 | ---------------------------------------------------------------- 9 | -- Only selecting buildings wihtin the purged census cells 10 | -- purged of cells with unpopulated areas 11 | -- https://github.com/openego/eGon-data/blob/59195926e41c8bd6d1ca8426957b97f33ef27bcc/src/egon/data/importing/zensus/__init__.py#L418-L449 12 | drop table if exists boundaries.egon_map_zensus_buildings_filtered; 13 | CREATE TABLE boundaries.egon_map_zensus_buildings_filtered as 14 | select * from ( 15 | select 16 | bld.id, 17 | zensus.grid_id, 18 | zensus.zensus_population_id as cell_id 19 | from openstreetmap.osm_buildings_filtered bld 20 | left join society.egon_destatis_zensus_apartment_building_population_per_ha zensus 21 | on ST_Within(bld.geom_point, zensus.geom) 22 | ) bld2 23 | where bld2.id is not null and bld2.grid_id is not null; 24 | 25 | ALTER TABLE ONLY boundaries.egon_map_zensus_buildings_filtered 26 | ADD CONSTRAINT pk_egon_map_zensus_buildings_filtered PRIMARY KEY (id); 27 | 28 | CREATE INDEX idx_egon_map_zensus_buildings_filtered_cell_id 29 | ON boundaries.egon_map_zensus_buildings_filtered USING btree (cell_id); 30 | -------------------------------------------------------------------------------- /src/egon/data/datasets/osm_buildings_streets/osm_buildings_residential_zensus_mapping.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Original Autor: IsGut (johnrobert@t-online.de) 3 | * Adapted by: nesnoj (jonathan.amme@rl-institut.de) 4 | */ 5 | 6 | ------------------------------------------------------------------- 7 | -- Create mapping table of residential buildings to zensus cells -- 8 | ------------------------------------------------------------------- 9 | -- Only selecting buildings wihtin the purged census cells 10 | -- purged of cells with unpopulated areas 11 | -- https://github.com/openego/eGon-data/blob/59195926e41c8bd6d1ca8426957b97f33ef27bcc/src/egon/data/importing/zensus/__init__.py#L418-L449 12 | drop table if exists boundaries.egon_map_zensus_buildings_residential; 13 | CREATE TABLE boundaries.egon_map_zensus_buildings_residential as 14 | select * from ( 15 | select 16 | bld.id, 17 | zensus.grid_id, 18 | zensus.zensus_population_id as cell_id 19 | from openstreetmap.osm_buildings_residential bld 20 | left join society.egon_destatis_zensus_apartment_building_population_per_ha zensus 21 | on ST_Within(bld.geom_point, zensus.geom) 22 | ) bld2 23 | where bld2.id is not null and bld2.grid_id is not null; 24 | -------------------------------------------------------------------------------- /src/egon/data/datasets/osm_buildings_streets/osm_buildings_temp_tables.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Original Autor: IsGut (johnrobert@t-online.de) 3 | * Adapted by: nesnoj (jonathan.amme@rl-institut.de) 4 | */ 5 | 6 | -- merge n_apartments representing households to buildings_filtered 7 | -- first a temp table is needed due to wanna add a count to share n_apartments for n_buildings 8 | -- all buildings within a radius of ~77.7 meters 9 | 10 | -- create temporary table of buildings containing zensus data (residentials) 11 | drop table if exists openstreetmap.osm_buildings_with_res_tmp; 12 | CREATE TABLE openstreetmap.osm_buildings_with_res_tmp as 13 | select * from ( 14 | select 15 | bld.osm_id, 16 | bld.id, 17 | bld.amenity, 18 | bld.building, 19 | bld.name, 20 | bld.geom_building, 21 | bld.area, 22 | bld.tags, 23 | zensus_apartments.apartment_count, 24 | zensus_apartments.grid_id 25 | from openstreetmap.osm_buildings_filtered bld 26 | left join society.egon_destatis_zensus_apartment_building_population_per_ha zensus_apartments 27 | on ST_DWithin(zensus_apartments.geom, bld.geom_building, 0.0007) -- radius is around 77.7 meters 28 | ) buildings_with_apartments 29 | where buildings_with_apartments.osm_id is not null; 30 | 31 | -- ^- this table will be droppped at a later point due to the need is only for preprocessing steps 32 | 33 | drop table if exists openstreetmap.osm_buildings_with_res_tmp2; 34 | create table openstreetmap.osm_buildings_with_res_tmp2 as 35 | select 36 | bld.osm_id, 37 | bld.id, 38 | bld.amenity, 39 | bld.building, 40 | bld.name, 41 | bld.geom_building, 42 | bld.area, 43 | bld.tags, 44 | bld.apartment_count, 45 | bld.grid_id, 46 | bwa.n_apartments_in_n_buildings -- number of buildings the apartments are shared by 47 | from openstreetmap.osm_buildings_with_res_tmp as bld 48 | left join ( 49 | select * from ( 50 | select 51 | b_res.grid_id, 52 | count(b_res.*) as n_apartments_in_n_buildings 53 | from openstreetmap.osm_buildings_with_res_tmp as b_res 54 | group by b_res.grid_id 55 | ) bwa ) bwa 56 | on bld.grid_id = bwa.grid_id; 57 | CREATE INDEX ON openstreetmap.osm_buildings_with_res_tmp2 USING gist (geom_building); 58 | 59 | 60 | drop table if exists openstreetmap.osm_amenities_in_buildings_tmp; 61 | CREATE TABLE openstreetmap.osm_amenities_in_buildings_tmp as 62 | with amenity as (select * from openstreetmap.osm_amenities_shops_filtered af) 63 | select 64 | bf.osm_id as osm_id_building, 65 | bf.id, 66 | bf.building, 67 | bf.area, 68 | bf.geom_building, 69 | amenity.osm_id as osm_id_amenity, 70 | amenity.egon_amenity_id, 71 | amenity.amenity, 72 | amenity.name, 73 | amenity.geom_amenity, 74 | bf.tags as tags_building, 75 | amenity.tags as tags_amenity, 76 | bf.apartment_count, 77 | bf.grid_id, 78 | bf.n_apartments_in_n_buildings 79 | from amenity, openstreetmap.osm_buildings_with_res_tmp2 bf 80 | where st_intersects(bf.geom_building, amenity.geom_amenity); 81 | 82 | CREATE INDEX idx_osm_amenities_in_buildings_tmp_geom_building 83 | ON openstreetmap.osm_amenities_in_buildings_tmp USING gist (geom_building); 84 | -------------------------------------------------------------------------------- /src/egon/data/datasets/osm_buildings_streets/osm_results_amenities.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Original Autor: IsGut (johnrobert@t-online.de) 3 | * Adapted by: nesnoj (jonathan.amme@rl-institut.de) 4 | */ 5 | 6 | -- amenities not located in a building 7 | drop table if exists openstreetmap.osm_amenities_not_in_buildings; 8 | CREATE TABLE openstreetmap.osm_amenities_not_in_buildings as 9 | select * 10 | from openstreetmap.osm_amenities_shops_filtered af 11 | where af.osm_id not in ( 12 | select aib.osm_id_amenity 13 | from openstreetmap.osm_amenities_in_buildings_tmp aib 14 | ); 15 | 16 | ALTER TABLE openstreetmap.osm_amenities_not_in_buildings 17 | ADD CONSTRAINT pk_osm_amenities_not_in_buildings PRIMARY KEY (egon_amenity_id); 18 | 19 | CREATE INDEX idx_osm_amenities_not_in_buildings_osm_id 20 | ON openstreetmap.osm_amenities_not_in_buildings using btree (osm_id); 21 | 22 | CREATE INDEX idx_osm_amenities_not_in_buildings_geom 23 | ON openstreetmap.osm_amenities_not_in_buildings USING gist (geom_amenity); 24 | -------------------------------------------------------------------------------- /src/egon/data/datasets/osm_buildings_streets/osm_results_buildings_w_amenities.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Original Autor: IsGut (johnrobert@t-online.de) 3 | * Adapted by: nesnoj (jonathan.amme@rl-institut.de) 4 | */ 5 | 6 | -- buildings containing amenities 7 | drop table if exists openstreetmap.osm_buildings_with_amenities; 8 | CREATE TABLE openstreetmap.osm_buildings_with_amenities as 9 | select 10 | bwa.osm_id_amenity, 11 | bwa.osm_id_building, 12 | bwa.id, 13 | bwa.building, 14 | bwa.area, 15 | bwa.geom_building, 16 | bwa.geom_amenity, 17 | CASE 18 | WHEN (ST_Contains(bwa.geom_building, ST_Centroid(bwa.geom_building))) IS TRUE 19 | THEN ST_Centroid(bwa.geom_building) 20 | ELSE ST_PointOnSurface(bwa.geom_building) 21 | END AS geom_point, 22 | bwa."name", 23 | bwa.tags_building, 24 | bwa.tags_amenity, 25 | bwa.n_amenities_inside, 26 | case 27 | when apartment_count > 0 28 | then bwa.apartment_count / bwa.n_apartments_in_n_buildings 29 | else 0 30 | end as apartment_count 31 | from ( 32 | select 33 | bwa.osm_id_amenity, 34 | bwa.osm_id_building, 35 | bwa.id, 36 | bwa.building, 37 | bwa.area, 38 | bwa.geom_building, 39 | bwa.geom_amenity, 40 | bwa.name, 41 | bwa.tags_building, 42 | bwa.tags_amenity, 43 | bwa.n_amenities_inside, 44 | SUM(bwa.apartment_count) as apartment_count, 45 | SUM(bwa.n_apartments_in_n_buildings) as n_apartments_in_n_buildings 46 | from ( 47 | select 48 | b.osm_id_amenity, 49 | b.osm_id_building, 50 | b.id, 51 | coalesce(b.amenity, b.building) as building, 52 | b.area, 53 | b.geom_building, 54 | b.geom_amenity, 55 | b.name, 56 | b.tags_building, 57 | b.tags_amenity, 58 | coalesce(b.apartment_count, 0) as apartment_count, 59 | coalesce(b.n_apartments_in_n_buildings, 0) as n_apartments_in_n_buildings, 60 | ainb.n_amenities_inside 61 | from openstreetmap.osm_amenities_in_buildings_tmp b 62 | left join ( 63 | select 64 | ainb.osm_id_building, 65 | count(*) as n_amenities_inside 66 | from openstreetmap.osm_amenities_in_buildings_tmp ainb 67 | group by ainb.osm_id_building ) ainb 68 | on b.osm_id_building = ainb.osm_id_building 69 | ) bwa 70 | group by 71 | bwa.osm_id_amenity, 72 | bwa.osm_id_building, 73 | bwa.id, 74 | bwa.building, 75 | bwa.area, 76 | bwa.geom_building, 77 | bwa.geom_amenity, 78 | bwa.name, 79 | bwa.tags_building, 80 | bwa.tags_amenity, 81 | bwa.n_amenities_inside 82 | ) bwa; 83 | 84 | 85 | -- osm_id_amenity, osm_id_building and id are no unique values 86 | 87 | CREATE INDEX idx_osm_buildings_with_amenities_osm_id_amenity 88 | ON openstreetmap.osm_buildings_with_amenities USING btree (osm_id_amenity); 89 | 90 | CREATE INDEX idx_osm_buildings_with_amenities_id 91 | ON openstreetmap.osm_buildings_with_amenities USING btree (id); 92 | 93 | CREATE INDEX idx_osm_buildings_with_amenities_osm_id_building 94 | ON openstreetmap.osm_buildings_with_amenities USING btree (osm_id_building); 95 | 96 | CREATE INDEX idx_osm_buildings_with_amenities_geom_building 97 | ON openstreetmap.osm_buildings_with_amenities USING gist (geom_building); 98 | -------------------------------------------------------------------------------- /src/egon/data/datasets/osm_buildings_streets/osm_results_buildings_wo_amenities.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Original Autor: IsGut (johnrobert@t-online.de) 3 | * Adapted by: nesnoj (jonathan.amme@rl-institut.de) 4 | */ 5 | 6 | -- buildings containing no amenities 7 | drop table if exists openstreetmap.osm_buildings_without_amenities; 8 | CREATE TABLE openstreetmap.osm_buildings_without_amenities as 9 | select 10 | bwa.osm_id, 11 | bwa.id, 12 | bwa.building, 13 | bwa.area, 14 | bwa.geom_building, 15 | CASE 16 | WHEN (ST_Contains(bwa.geom_building, ST_Centroid(bwa.geom_building))) IS TRUE 17 | THEN ST_Centroid(bwa.geom_building) 18 | ELSE ST_PointOnSurface(bwa.geom_building) 19 | END AS geom_point, 20 | bwa.name, 21 | bwa.tags, 22 | CASE 23 | WHEN apartment_count > 0 24 | THEN bwa.apartment_count / bwa.n_apartments_in_n_buildings 25 | ELSE 0 26 | END AS apartment_count 27 | from ( 28 | select 29 | bwa.osm_id, 30 | bwa.id, 31 | bwa.building, 32 | bwa.area, 33 | bwa.geom_building, 34 | bwa.name, 35 | bwa.tags, 36 | SUM(bwa.apartment_count) as apartment_count, 37 | SUM(bwa.n_apartments_in_n_buildings) as n_apartments_in_n_buildings 38 | from ( 39 | select 40 | bf.osm_id, 41 | bf.id, 42 | coalesce(bf.amenity, bf.building) as building, 43 | bf.name, 44 | bf.area, 45 | bf.geom_building, 46 | bf.tags, 47 | coalesce(bf.apartment_count, 0) as apartment_count, 48 | coalesce(bf.n_apartments_in_n_buildings, 0) as n_apartments_in_n_buildings 49 | from openstreetmap.osm_buildings_with_res_tmp2 bf 50 | -- NOT IN replaced by JOIN DUE TO performance problems 51 | -- cf. https://github.com/openego/eGon-data/issues/693 52 | LEFT JOIN openstreetmap.osm_amenities_in_buildings_tmp aib 53 | ON bf.osm_id = aib.osm_id_building 54 | WHERE aib.osm_id_building IS NULL 55 | ) bwa 56 | group by 57 | bwa.osm_id, 58 | bwa.id, 59 | bwa.building, 60 | bwa.area, 61 | bwa.geom_building, 62 | bwa.name, 63 | bwa.tags 64 | ) bwa; 65 | 66 | CREATE INDEX ON openstreetmap.osm_buildings_without_amenities USING gist (geom_building); 67 | -------------------------------------------------------------------------------- /src/egon/data/datasets/power_etrago/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | The central module containing all code dealing with open cycle gas turbine 3 | """ 4 | from egon.data.datasets import Dataset 5 | from egon.data.datasets.power_etrago.match_ocgt import ( 6 | insert_open_cycle_gas_turbines, 7 | ) 8 | 9 | 10 | class OpenCycleGasTurbineEtrago(Dataset): 11 | """ 12 | Insert the open cycle gas turbine links into the database 13 | 14 | Insert the open cycle gas turbine links into the database by using 15 | the function :py:func:`insert_open_cycle_gas_turbines `. 16 | 17 | *Dependencies* 18 | * :py:class:`GasAreaseGon2035 ` 19 | * :py:class:`GasNodesAndPipes ` 20 | * :py:class:`PowerPlants ` 21 | * :py:class:`mastr_data_setup ` 22 | 23 | *Resulting tables* 24 | * :py:class:`grid.egon_etrago_link ` is extended 25 | 26 | """ 27 | 28 | #: 29 | name: str = "OpenCycleGasTurbineEtrago" 30 | #: 31 | version: str = "0.0.2" 32 | 33 | def __init__(self, dependencies): 34 | super().__init__( 35 | name=self.name, 36 | version=self.version, 37 | dependencies=dependencies, 38 | tasks=(insert_open_cycle_gas_turbines,), 39 | ) 40 | -------------------------------------------------------------------------------- /src/egon/data/datasets/pypsaeur/Snakefile: -------------------------------------------------------------------------------- 1 | rule Main: 2 | conda: 3 | "pypsa-eur/envs/linux-pinned.yaml" 4 | shell: 5 | ( "snakemake" 6 | " --directory=pypsa-eur/" 7 | " --snakefile=pypsa-eur-sec/Snakefile" 8 | " -j1" 9 | ) 10 | 11 | rule prepare: 12 | conda: 13 | "pypsa-eur/envs/linux-pinned.yaml" 14 | shell: 15 | ( "snakemake" 16 | " --directory=pypsa-eur/" 17 | " --snakefile=pypsa-eur/Snakefile" 18 | " -call" 19 | " prepare_sector_networks" 20 | ) 21 | 22 | rule solve: 23 | conda: 24 | "pypsa-eur/envs/linux-pinned.yaml" 25 | shell: 26 | ( "snakemake" 27 | " --directory=pypsa-eur/" 28 | " --snakefile=pypsa-eur/Snakefile" 29 | " --consider-ancient" 30 | " add_existing_baseyear=powerplants" 31 | " -call" 32 | " solve_sector_networks" 33 | ) 34 | 35 | rule summary: 36 | conda: 37 | "pypsa-eur/envs/linux-pinned.yaml" 38 | shell: 39 | ( "snakemake" 40 | " --directory=pypsa-eur/" 41 | " --snakefile=pypsa-eur/Snakefile" 42 | " -call" 43 | " make_summary" 44 | ) 45 | -------------------------------------------------------------------------------- /src/egon/data/datasets/pypsaeur/custom_extra_functionality.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # SPDX-FileCopyrightText: : 2023- The PyPSA-Eur Authors 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | from pypsa.optimization.compat import get_var, define_constraints, linexpr 7 | 8 | def custom_extra_functionality(n, snapshots, snakemake): 9 | """ 10 | Add custom extra functionality constraints. 11 | """ 12 | n.model.constraints.remove("Generator-e_sum_max") 13 | 14 | 15 | #if n.meta["wildcards"]['planning_horizons'] == "2045": 16 | # min_wind_onshore = 100e6 17 | # min_solar = 200e6 18 | # min_wind_offshore = 50e6 19 | if n.meta["wildcards"]['planning_horizons'] == "2035": 20 | min_wind_onshore = 111309 21 | min_solar = 232228 22 | min_wind_offshore = 39122 23 | elif n.meta["wildcards"]['planning_horizons'] == "2030": 24 | min_wind_onshore = 86583 25 | min_solar = 165173 26 | min_wind_offshore = 27239 27 | elif n.meta["wildcards"]['planning_horizons'] == "2025": 28 | min_wind_onshore = 61856 29 | min_solar = 98119 30 | min_wind_offshore = 15356 31 | 32 | wind_offshore_ext = list( 33 | n.generators.index[ 34 | (n.generators.index.str.contains("offwind") 35 | & (n.generators.bus.str.contains("DE")) 36 | & (n.generators.p_nom_extendable) 37 | ) 38 | ] 39 | ) 40 | 41 | wind_offshore_fixed = n.generators[ 42 | (n.generators.index.str.contains("offwind") 43 | & (n.generators.bus.str.contains("DE")) 44 | & (n.generators.p_nom_extendable==False) 45 | ) 46 | ].p_nom.sum() 47 | 48 | 49 | wind_onshore_ext = list( 50 | n.generators.index[ 51 | (n.generators.index.str.contains("onwind") 52 | & (n.generators.bus.str.contains("DE"))) 53 | & (n.generators.p_nom_extendable) 54 | ] 55 | ) 56 | 57 | wind_onshore_fixed = n.generators[ 58 | (n.generators.index.str.contains("onwind") 59 | & (n.generators.bus.str.contains("DE")) 60 | & (n.generators.p_nom_extendable==False) 61 | ) 62 | ].p_nom.sum() 63 | 64 | 65 | solar_ext = list( 66 | n.generators.index[ 67 | (n.generators.carrier.isin(["solar", "solar-hst", "solar rooftop"]) 68 | & (n.generators.bus.str.contains("DE")) 69 | & (n.generators.p_nom_extendable)) 70 | ] 71 | ) 72 | 73 | solar_fixed = n.generators[ 74 | (n.generators.carrier.isin(["solar", "solar-hst", "solar rooftop"]) 75 | & (n.generators.bus.str.contains("DE")) 76 | & (n.generators.p_nom_extendable==False) 77 | ) 78 | ].p_nom.sum() 79 | 80 | if n.meta["wildcards"]['planning_horizons'] == "2045": 81 | min_wind_onshore = wind_onshore_fixed 82 | min_solar = solar_fixed 83 | min_wind_offshore = wind_offshore_fixed 84 | 85 | 86 | define_constraints(n, 87 | get_var(n, "Generator", "p_nom").loc[wind_offshore_ext].sum() , 88 | ">=", 89 | min_wind_offshore - wind_offshore_fixed , 90 | "Global", 91 | "min_offwind_de" 92 | ) 93 | 94 | define_constraints(n, 95 | get_var(n, "Generator", "p_nom").loc[wind_onshore_ext].sum() , 96 | ">=", 97 | min_wind_onshore - wind_onshore_fixed, 98 | "Global", 99 | "min_onwind_de" 100 | ) 101 | 102 | define_constraints(n, 103 | get_var(n, "Generator", "p_nom").loc[solar_ext].sum() , 104 | ">=", 105 | min_solar - solar_fixed, 106 | "Global", 107 | "min_solar_de" 108 | ) 109 | 110 | -------------------------------------------------------------------------------- /src/egon/data/datasets/saltcavern/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | The central module containing all code dealing with BGR data. 3 | 4 | This module directly contains the code dealing with importing data of 5 | Bundesanstalt für Geowissenschaften und Rohstoffe (BGR) from the 6 | databundle into the database. 7 | 8 | """ 9 | 10 | from pathlib import Path 11 | 12 | from geoalchemy2 import Geometry 13 | import geopandas as gpd 14 | 15 | from egon.data import db 16 | from egon.data.datasets import Dataset 17 | import egon.data.config 18 | 19 | 20 | def to_postgres(): 21 | """ 22 | Write BGR saline structures to database. 23 | 24 | This function inserts data into the database and has no return. 25 | 26 | """ 27 | 28 | # Get information from data configuraiton file 29 | data_config = egon.data.config.datasets() 30 | bgr_processed = data_config["bgr"]["processed"] 31 | 32 | # Create target schema 33 | db.execute_sql(f"CREATE SCHEMA IF NOT EXISTS {bgr_processed['schema']};") 34 | 35 | engine_local_db = db.engine() 36 | 37 | # Extract shapefiles from zip archive and send it to postgres db 38 | for filename, table in bgr_processed["file_table_map"].items(): 39 | # Open files and read .shp (within .zip) with geopandas 40 | shp_file_path = ( 41 | Path(".") 42 | / "data_bundle_egon_data" 43 | / "hydrogen_storage_potential_saltstructures" 44 | / filename 45 | ) 46 | data = gpd.read_file(shp_file_path).to_crs(epsg=4326) 47 | data = ( 48 | data[ 49 | (data["Bewertung"] == "Eignung InSpEE-DS") 50 | | (data["Bewertung"] == "geeignet") 51 | ] 52 | .drop(columns=["Bewertung", "Typ", "Salzstrukt"]) 53 | .rename( 54 | columns={ 55 | "Shape_Area": "shape_star", 56 | "Shape_Leng": "shape_stle", 57 | } 58 | ) 59 | ) 60 | 61 | # Set index column and format column headings 62 | data.index.set_names("saltstructure_id", inplace=True) 63 | data.columns = [x.lower() for x in data.columns] 64 | 65 | # Drop table before inserting data 66 | db.execute_sql( 67 | f"DROP TABLE IF EXISTS " 68 | f"{bgr_processed['schema']}.{table} CASCADE;" 69 | ) 70 | 71 | # create database table from geopandas dataframe 72 | data.to_postgis( 73 | table, 74 | engine_local_db, 75 | schema=bgr_processed["schema"], 76 | index=True, 77 | if_exists="replace", 78 | dtype={"geometry": Geometry()}, 79 | ) 80 | 81 | # add primary key 82 | db.execute_sql( 83 | f"ALTER TABLE {bgr_processed['schema']}.{table} " 84 | f"ADD PRIMARY KEY (saltstructure_id);" 85 | ) 86 | 87 | # Add index on geometry column 88 | db.execute_sql( 89 | f"CREATE INDEX {table}_geometry_idx ON " 90 | f"{bgr_processed['schema']}.{table} USING gist (geometry);" 91 | ) 92 | 93 | 94 | class SaltcavernData(Dataset): 95 | """ 96 | Insert the BGR saline structures into the database. 97 | 98 | Insert the BGR saline structures into the database by executing the 99 | function :py:func:`to_postgres`. 100 | 101 | *Dependencies* 102 | * :py:class:`DataBundle ` 103 | * :py:class:`Vg250 ` 104 | 105 | *Resulting tables* 106 | * boundaries.inspee_saltstructures is created 107 | 108 | """ 109 | 110 | #: 111 | name: str = "SaltcavernData" 112 | #: 113 | version: str = "0.0.1" 114 | 115 | def __init__(self, dependencies): 116 | super().__init__( 117 | name=self.name, 118 | version=self.version, 119 | dependencies=dependencies, 120 | tasks=(to_postgres,), 121 | ) 122 | -------------------------------------------------------------------------------- /src/egon/data/datasets/scenario_path/__init__.py: -------------------------------------------------------------------------------- 1 | from egon.data.datasets import Dataset 2 | from egon.data.datasets.scenario_path.import_status2019 import ( 3 | download_status2019, 4 | import_scn_status2019, 5 | ) 6 | 7 | 8 | class CreateIntermediateScenarios(Dataset): 9 | def __init__(self, dependencies): 10 | super().__init__( 11 | name="scenario_path", 12 | version="0.0.1", 13 | dependencies=dependencies, 14 | tasks=( 15 | download_status2019, 16 | import_scn_status2019, 17 | ), 18 | ) 19 | -------------------------------------------------------------------------------- /src/egon/data/datasets/scenario_path/import_status2019.py: -------------------------------------------------------------------------------- 1 | """ 2 | Read eTraGo tables for the status2019 and import it to db 3 | """ 4 | 5 | from pathlib import Path 6 | from urllib.request import urlretrieve 7 | import os 8 | import subprocess 9 | 10 | import pandas as pd 11 | 12 | from egon.data import config, db 13 | import egon.data.config 14 | 15 | sources = egon.data.config.datasets()["scenario_path"]["sources"] 16 | 17 | 18 | def download_status2019(): 19 | """ 20 | Download the status2019 etrago tables from Zenodo 21 | 22 | Returns 23 | ------- 24 | None. 25 | 26 | """ 27 | 28 | if "status2019" in config.settings()["egon-data"]["--scenarios"]: 29 | print("Scenario status2019 was created within the pipeline. " 30 | "It is not imported from zenodo.") 31 | return 32 | 33 | # Get parameters from config and set download URL 34 | url = sources["url_status2019"] 35 | status2019_path = Path(".") / "PoWerD_status2019.backup" 36 | 37 | # Retrieve files 38 | urlretrieve(url, status2019_path) 39 | 40 | return 41 | 42 | 43 | def import_scn_status2019(): 44 | """ 45 | Read and import the scenario status2019 and import it into db 46 | 47 | Parameters 48 | ---------- 49 | *No parameters required 50 | 51 | """ 52 | 53 | if "status2019" in config.settings()["egon-data"]["--scenarios"]: 54 | print("Scenario status2019 was created within the pipeline. " 55 | "It is not imported from zenodo.") 56 | return 57 | 58 | # Connect to the data base 59 | con = db.engine() 60 | 61 | # Clean existing data for status2019 62 | tables = pd.read_sql( 63 | """ 64 | SELECT tablename FROM pg_catalog.pg_tables 65 | WHERE schemaname = 'grid' 66 | """, 67 | con, 68 | ) 69 | 70 | tables = tables[ 71 | ~tables["tablename"].isin( 72 | [ 73 | "egon_etrago_carrier", 74 | "egon_etrago_temp_resolution", 75 | ] 76 | ) 77 | ] 78 | 79 | for table in tables["tablename"]: 80 | db.execute_sql( 81 | f""" 82 | DELETE FROM grid.{table} WHERE scn_name = 'status2019'; 83 | """ 84 | ) 85 | 86 | config_data = config.settings()["egon-data"] 87 | database = config_data["--database-name"] 88 | host = config_data["--database-host"] 89 | port = config_data["--database-port"] 90 | user = config_data["--database-user"] 91 | 92 | my_env = os.environ.copy() 93 | my_env["PGPASSWORD"] = config_data["--database-password"] 94 | 95 | for table in tables["tablename"]: 96 | subprocess.Popen( 97 | [ 98 | "pg_restore", 99 | "-d", 100 | database, 101 | "--host", 102 | host, 103 | "--port", 104 | port, 105 | "-U", 106 | user, 107 | "-a", 108 | "--single-transaction", 109 | f"--table={table}", 110 | "PoWerD_status2019.backup", 111 | ], 112 | env=my_env, 113 | ) 114 | return 115 | -------------------------------------------------------------------------------- /src/egon/data/datasets/tyndp.py: -------------------------------------------------------------------------------- 1 | """The central module containing all code dealing with downloading tyndp data 2 | """ 3 | 4 | import os 5 | from egon.data import config 6 | from egon.data.datasets import Dataset 7 | from urllib.request import urlretrieve 8 | 9 | 10 | class Tyndp(Dataset): 11 | """ 12 | Downloads data for foreign countries from Ten-Year-Network-Developement Plan 13 | 14 | This dataset downloads installed generation capacities and load time series for 15 | foreign countries from the website of the Ten-Year-Network-Developement Plan 2020 from ENTSO-E. 16 | That data is stored into files and later on written into the database 17 | (see :py:class:`ElectricalNeighbours `). 18 | 19 | 20 | *Dependencies* 21 | * :py:class:`Setup ` 22 | 23 | *Resulting tables* 24 | 25 | """ 26 | 27 | #: 28 | name: str = "Tyndp" 29 | #: 30 | version: str = "0.0.1" 31 | 32 | def __init__(self, dependencies): 33 | super().__init__( 34 | name=self.name, 35 | version=self.version, 36 | dependencies=dependencies, 37 | tasks=(download), 38 | ) 39 | 40 | 41 | def download(): 42 | """Download input data from TYNDP 2020 43 | Returns 44 | ------- 45 | None. 46 | """ 47 | sources = config.datasets()["tyndp"]["sources"] 48 | targets = config.datasets()["tyndp"]["targets"] 49 | 50 | if not os.path.exists("tyndp"): 51 | os.mkdir("tyndp") 52 | 53 | for dataset in ["capacities", "demand_2030", "demand_2040"]: 54 | target_file = targets[dataset] 55 | 56 | urlretrieve(sources[dataset], f"tyndp/{target_file}") 57 | -------------------------------------------------------------------------------- /src/egon/data/datasets/vg250/vg250_lan_nuts_id_mview.sql: -------------------------------------------------------------------------------- 1 | -- Create MView with NUTS IDs 2 | 3 | DROP MATERIALIZED VIEW IF EXISTS boundaries.vg250_lan_nuts_id; 4 | 5 | CREATE MATERIALIZED VIEW boundaries.vg250_lan_nuts_id AS 6 | SELECT lan.ags_0, 7 | lan.gen, 8 | lan.nuts, 9 | st_union(st_transform(lan.geometry, 3035)) AS geometry 10 | FROM ( SELECT vg.ags_0, 11 | vg.nuts, 12 | replace(vg.gen::text, ' (Bodensee)'::text, ''::text) AS gen, 13 | vg.geometry 14 | FROM boundaries.vg250_lan vg) lan 15 | GROUP BY lan.ags_0, lan.gen, lan.nuts 16 | ORDER BY lan.ags_0 17 | WITH DATA; 18 | -------------------------------------------------------------------------------- /src/egon/data/datasets/vg250_mv_grid_districts.py: -------------------------------------------------------------------------------- 1 | """The module containing all code to map MV grid districts to federal states. 2 | """ 3 | from sqlalchemy import Column, Integer, String 4 | from sqlalchemy.ext.declarative import declarative_base 5 | import geopandas as gpd 6 | import pandas as pd 7 | 8 | from egon.data import config, db 9 | 10 | Base = declarative_base() 11 | from egon.data.datasets import Dataset 12 | 13 | 14 | class Vg250MvGridDistricts(Dataset): 15 | """ 16 | Maps MV grid districts to federal states and writes it to database. 17 | 18 | *Dependencies* 19 | * :py:class:`Vg250 ` 20 | * :py:class:`MvGridDistricts ` 21 | 22 | *Resulting tables* 23 | * :py:class:`boundaries.egon_map_mvgriddistrict_vg250 ` 24 | is created and filled 25 | 26 | """ 27 | #: 28 | name: str = "Vg250MvGridDistricts" 29 | #: 30 | version: str = "0.0.1" 31 | 32 | def __init__(self, dependencies): 33 | super().__init__( 34 | name=self.name, 35 | version=self.version, 36 | dependencies=dependencies, 37 | tasks=(mapping), 38 | ) 39 | 40 | 41 | class MapMvgriddistrictsVg250(Base): 42 | """ 43 | Class definition of table boundaries.egon_map_mvgriddistrict_vg250. 44 | """ 45 | __tablename__ = "egon_map_mvgriddistrict_vg250" 46 | __table_args__ = {"schema": "boundaries"} 47 | bus_id = Column(Integer, primary_key=True) 48 | vg250_lan = Column(String) 49 | 50 | 51 | def create_tables(): 52 | """ 53 | Create table for mapping grid districts to federal states. 54 | 55 | """ 56 | 57 | db.execute_sql("CREATE SCHEMA IF NOT EXISTS boundaries;") 58 | engine = db.engine() 59 | MapMvgriddistrictsVg250.__table__.drop(bind=engine, checkfirst=True) 60 | MapMvgriddistrictsVg250.__table__.create(bind=engine, checkfirst=True) 61 | 62 | 63 | def mapping(): 64 | """ 65 | Map MV grid districts to federal states and write to database. 66 | 67 | Newly creates and fills table boundaries.egon_map_mvgriddistrict_vg250. 68 | 69 | """ 70 | # Create table 71 | create_tables() 72 | 73 | # Select sources and targets from dataset configuration 74 | sources = config.datasets()["map_mvgrid_vg250"]["sources"] 75 | target = config.datasets()["map_mvgrid_vg250"]["targets"]["map"] 76 | 77 | # Delete existing data 78 | db.execute_sql(f"DELETE FROM {target['schema']}.{target['table']}") 79 | 80 | # Select sources from database 81 | mv_grid_districts = db.select_geodataframe( 82 | f""" 83 | SELECT bus_id as bus_id, ST_Centroid(geom) as geom 84 | FROM {sources['egon_mv_grid_district']['schema']}. 85 | {sources['egon_mv_grid_district']['table']} 86 | """, 87 | index_col="bus_id", 88 | ) 89 | 90 | federal_states = db.select_geodataframe( 91 | f""" 92 | SELECT gen,geometry 93 | FROM {sources['federal_states']['schema']}. 94 | {sources['federal_states']['table']} 95 | """, 96 | geom_col="geometry", 97 | index_col="gen", 98 | ) 99 | 100 | # Join mv grid districts and federal states 101 | df = pd.DataFrame( 102 | gpd.sjoin(mv_grid_districts, federal_states)["index_right"] 103 | ) 104 | 105 | # Rename columns 106 | df.rename({"index_right": "vg250_lan"}, axis=1, inplace=True) 107 | 108 | # Insert to database 109 | df.to_sql( 110 | target["table"], 111 | schema=target["schema"], 112 | if_exists="append", 113 | con=db.engine(), 114 | ) 115 | -------------------------------------------------------------------------------- /src/egon/data/datasets/zensus/create_combined_zensus_table.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Join building, apartment and population data from Zensus. 3 | If there's no data on buildings or apartments for a certain cell, 4 | the value for building_count resp. apartment_count contains NULL. 5 | */ 6 | 7 | DROP TABLE IF EXISTS society.egon_destatis_zensus_apartment_building_population_per_ha; 8 | 9 | -- Join tables and filter data 10 | CREATE TABLE society.egon_destatis_zensus_apartment_building_population_per_ha AS 11 | SELECT GREATEST(bld_apt.grid_id, ze.grid_id) AS grid_id, 12 | GREATEST(bld_apt.zensus_population_id, ze.id) AS zensus_population_id, 13 | bld_apt.building_count, 14 | bld_apt.apartment_count, 15 | ze.population, 16 | ze.geom, 17 | ze.geom_point 18 | FROM 19 | (SELECT GREATEST(bld.zensus_population_id, apt.zensus_population_id) AS zensus_population_id, 20 | GREATEST(bld.grid_id, apt.grid_id) AS grid_id, 21 | bld.quantity as building_count, 22 | apt.quantity as apartment_count 23 | FROM 24 | (SELECT * 25 | FROM society.egon_destatis_zensus_building_per_ha 26 | WHERE attribute = 'INSGESAMT' AND quantity_q < 2 27 | ) bld 28 | FULL JOIN 29 | (SELECT * 30 | FROM society.egon_destatis_zensus_apartment_per_ha 31 | WHERE attribute = 'INSGESAMT' AND quantity_q < 2 32 | ) apt 33 | USING (grid_id) 34 | ) bld_apt 35 | FULL JOIN 36 | (SELECT * 37 | FROM society.destatis_zensus_population_per_ha_inside_germany 38 | ) ze 39 | USING (grid_id) 40 | ORDER BY zensus_population_id; 41 | 42 | -- Set empty geoms from raw Zensus dataset 43 | -- This is needed as table `society.destatis_zensus_population_per_ha_inside_germany` 44 | -- holds geoms on cells with population > 0 45 | UPDATE society.egon_destatis_zensus_apartment_building_population_per_ha AS t 46 | SET 47 | geom = t2.geom, 48 | geom_point = t2.geom_point 49 | FROM society.destatis_zensus_population_per_ha AS t2 50 | WHERE t.geom IS NULL AND t.grid_id = t2.grid_id; 51 | 52 | -- Create index 53 | CREATE INDEX ON society.egon_destatis_zensus_apartment_building_population_per_ha USING gist (geom); 54 | -------------------------------------------------------------------------------- /src/egon/data/datasets/zensus_mv_grid_districts.py: -------------------------------------------------------------------------------- 1 | """ 2 | Implements mapping between mv grid districts and zensus cells 3 | """ 4 | 5 | from sqlalchemy import Column, ForeignKey, Integer 6 | from sqlalchemy.ext.declarative import declarative_base 7 | import geopandas as gpd 8 | 9 | from egon.data import db 10 | from egon.data.datasets import Dataset 11 | from egon.data.datasets.mv_grid_districts import MvGridDistricts 12 | from egon.data.datasets.zensus_vg250 import DestatisZensusPopulationPerHa 13 | import egon.data.config 14 | 15 | 16 | class ZensusMvGridDistricts(Dataset): 17 | """ 18 | Maps zensus cells to MV grid districts and writes it to database. 19 | 20 | *Dependencies* 21 | * :py:class:`ZensusPopulation ` 22 | * :py:class:`MvGridDistricts ` 23 | 24 | *Resulting tables* 25 | * :py:class:`boundaries.egon_map_zensus_grid_districts ` 26 | is created and filled 27 | 28 | """ 29 | #: 30 | name: str = "ZensusMvGridDistricts" 31 | #: 32 | version: str = "0.0.1" 33 | def __init__(self, dependencies): 34 | super().__init__( 35 | name=self.name, 36 | version=self.version, 37 | dependencies=dependencies, 38 | tasks=(mapping), 39 | ) 40 | 41 | 42 | # will be later imported from another file ### 43 | Base = declarative_base() 44 | 45 | 46 | class MapZensusGridDistricts(Base): 47 | """ 48 | Class definition of table boundaries.egon_map_zensus_grid_districts. 49 | """ 50 | __tablename__ = "egon_map_zensus_grid_districts" 51 | __table_args__ = {"schema": "boundaries"} 52 | 53 | zensus_population_id = Column( 54 | Integer, 55 | ForeignKey(DestatisZensusPopulationPerHa.id), 56 | primary_key=True, 57 | index=True, 58 | ) 59 | bus_id = Column(Integer, ForeignKey(MvGridDistricts.bus_id)) 60 | 61 | 62 | def mapping(): 63 | """ 64 | Map zensus cells and MV grid districts and write to database. 65 | 66 | Newly creates and fills table boundaries.egon_map_zensus_grid_districts. 67 | 68 | """ 69 | 70 | MapZensusGridDistricts.__table__.drop(bind=db.engine(), checkfirst=True) 71 | MapZensusGridDistricts.__table__.create(bind=db.engine(), checkfirst=True) 72 | 73 | # Get information from data configuration file 74 | cfg = egon.data.config.datasets()["map_zensus_grid_districts"] 75 | 76 | # Delete existsing data 77 | db.execute_sql( 78 | f"""DELETE FROM 79 | {cfg['targets']['map']['schema']}.{cfg['targets']['map']['table']}""" 80 | ) 81 | 82 | # Select zensus cells 83 | zensus = db.select_geodataframe( 84 | f"""SELECT id as zensus_population_id, geom_point FROM 85 | {cfg['sources']['zensus_population']['schema']}. 86 | {cfg['sources']['zensus_population']['table']}""", 87 | geom_col="geom_point", 88 | ) 89 | 90 | grid_districts = db.select_geodataframe( 91 | f"""SELECT bus_id, geom 92 | FROM {cfg['sources']['egon_mv_grid_district']['schema']}. 93 | {cfg['sources']['egon_mv_grid_district']['table']}""", 94 | geom_col="geom", 95 | epsg=3035, 96 | ) 97 | 98 | # Join mv grid districts with zensus cells 99 | join = gpd.sjoin(zensus, grid_districts, how="inner", op="intersects") 100 | 101 | # Insert results to database 102 | join[["zensus_population_id", "bus_id"]].to_sql( 103 | cfg["targets"]["map"]["table"], 104 | schema=cfg["targets"]["map"]["schema"], 105 | con=db.engine(), 106 | if_exists="replace", 107 | ) 108 | -------------------------------------------------------------------------------- /src/egon/data/subprocess.py: -------------------------------------------------------------------------------- 1 | """Exensions to Python's :py:mod:`subprocess` module. 2 | 3 | More specifically, this module provides a customized version of 4 | :py:func:`subprocess.run`, which always sets `check=True`, 5 | `capture_output=True`, enhances the raised exceptions string representation 6 | with additional output information and makes it slightly more readable when 7 | encountered in a stack trace. 8 | """ 9 | 10 | from textwrap import indent, wrap 11 | import itertools 12 | import subprocess 13 | 14 | 15 | class CalledProcessError(subprocess.CalledProcessError): 16 | """A more verbose version of :py:class:`subprocess.CalledProcessError`. 17 | 18 | Replaces the standard string representation of a 19 | :py:class:`subprocess.CalledProcessError` with one that has more output and 20 | error information and is formatted to be more readable in a stack trace. 21 | """ 22 | 23 | def __str__(self): 24 | errors = self.stderr.split("\n") 25 | outputs = self.stdout.split("\n") 26 | 27 | lines = itertools.chain( 28 | wrap(f"{super().__str__()}"), 29 | ["Output:"], 30 | *( 31 | wrap(output, initial_indent=" ", subsequent_indent=" ") 32 | for output in outputs 33 | ), 34 | ["Errors:"], 35 | *( 36 | wrap(error, initial_indent=" ", subsequent_indent=" ") 37 | for error in errors 38 | ), 39 | ) 40 | lines = indent("\n".join(lines), "| ") 41 | return f"\n{lines}" 42 | 43 | 44 | def run(*args, **kwargs): 45 | """A "safer" version of :py:func:`subprocess.run`. 46 | 47 | "Safer" in this context means that this version always raises 48 | :py:class:`CalledProcessError` if the process in question returns a 49 | non-zero exit status. This is done by setting `check=True` and 50 | `capture_output=True`, so you don't have to specify these yourself anymore. 51 | You can though, if you want to override these defaults. 52 | Other than that, the function accepts the same parameters as 53 | :py:func:`subprocess.run`. 54 | """ 55 | for default in ["capture_output", "check", "text"]: 56 | kwargs[default] = kwargs.get(default, True) 57 | try: 58 | result = subprocess.run(*args, **kwargs) 59 | except subprocess.CalledProcessError as cpe: 60 | raise CalledProcessError( 61 | cpe.returncode, cpe.cmd, output=cpe.output, stderr=cpe.stderr 62 | ) from None 63 | return result 64 | -------------------------------------------------------------------------------- /tests/test_dataset_class.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Union 3 | 4 | from airflow.models.dag import DAG 5 | 6 | from egon.data.datasets import Dataset, TaskGraph, Tasks 7 | 8 | 9 | def test_uniqueness_of_automatically_generated_final_dataset_task(): 10 | """Test that the generated final dataset task is named uniquely. 11 | 12 | This is a regression test for issue #985. Having multiple `Dataset`s ending 13 | in parallel tasks doesn't work if those `Dataset`s are in a module below 14 | the `egon.data.datasets` package. In that case the code removing the module 15 | name prefix from task ids and the code generating the final dataset task 16 | which updates the dataset version once all parallel tasks have finished 17 | interact in a way that generates non-distinct task ids so that tasks 18 | generated later clobber the ones generated earlier. This leads to spurious 19 | cycles and other inconsistencies and bugs in the graph. 20 | """ 21 | 22 | noops = [(lambda: None) for _ in range(4)] 23 | for i, noop in enumerate(noops): 24 | noop.__name__ = f"noop-{i}" 25 | 26 | @dataclass 27 | class Dataset_1(Dataset): 28 | name: str = "DS1" 29 | version: str = "0.0.0" 30 | tasks: Union[Tasks, TaskGraph] = ({noops[0], noops[1]},) 31 | 32 | @dataclass 33 | class Dataset_2(Dataset): 34 | name: str = "DS2" 35 | version: str = "0.0.0" 36 | tasks: Union[Tasks, TaskGraph] = ({noops[2], noops[3]},) 37 | 38 | Dataset_1.__module__ = "egon.data.datasets.test.datasets" 39 | Dataset_2.__module__ = "egon.data.datasets.test.datasets" 40 | with DAG(dag_id="Test-DAG", default_args={"start_date": "1111-11-11"}): 41 | datasets = [Dataset_1(), Dataset_2()] 42 | ids = [list(dataset.tasks)[-1] for dataset in datasets] 43 | assert ( 44 | ids[0] != ids[1] 45 | ), "Expected unique names for final tasks of distinct datasets." 46 | -------------------------------------------------------------------------------- /tests/test_egon-data.py: -------------------------------------------------------------------------------- 1 | from importlib import import_module 2 | 3 | from click.testing import CliRunner 4 | import pytest 5 | 6 | from egon.data import __version__ 7 | from egon.data.cli import egon_data 8 | 9 | 10 | def test_main(): 11 | runner = CliRunner() 12 | with runner.isolated_filesystem(): 13 | result = runner.invoke(egon_data, ["--version"]) 14 | 15 | assert result.output == "{name}, version {version}\n".format( 16 | name=egon_data.name, version=__version__ 17 | ) 18 | assert result.exit_code == 0 19 | 20 | 21 | @pytest.mark.skip( 22 | reason=( 23 | "Needs `docker` and/or PostgreSQL and we're currently not making sure" 24 | "\nthese are present on the continuous integration service(s) we use." 25 | ) 26 | ) 27 | def test_airflow(): 28 | """ Test that `egon-data airflow` correctly forwards to airflow. """ 29 | runner = CliRunner() 30 | with runner.isolated_filesystem(): 31 | result = runner.invoke(egon_data, ["airflow", "--help"]) 32 | assert result.output == "" 33 | 34 | 35 | def test_pipeline_importability(): 36 | error = None 37 | for m in ["egon.data.airflow.dags.pipeline"]: 38 | try: 39 | import_module(m) 40 | except Exception as e: 41 | error = e 42 | assert error is None, ( 43 | "\nDid not expect an error when importing:\n\n `{}`\n\nGot: {}" 44 | ).format(m, error) 45 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [gh-actions] 2 | # Map python versions used in build.yml to tox environments 3 | # This mapping is used by tox-gh-actions 4 | python = 5 | 3.7: py37, clean, docs, report 6 | 3.8: py38, clean, docs, report 7 | 8 | [gh-actions:env] 9 | PLATFORM = 10 | ubuntu-latest: linux 11 | 12 | [tox] 13 | envlist = 14 | clean, 15 | docs, 16 | py{37,38}-{cover,nocov}-{linux}, 17 | report 18 | 19 | [testenv] 20 | basepython = 21 | {bootstrap,clean,report,docs,codecov,coveralls}: {env:TOXPYTHON:python3} 22 | setenv = 23 | PYTHONPATH={toxinidir}/tests 24 | PYTHONUNBUFFERED=yes 25 | passenv = 26 | * 27 | deps = 28 | pytest 29 | commands = 30 | {posargs:pytest -vv --ignore=src} 31 | 32 | [testenv:bootstrap] 33 | deps = 34 | jinja2 35 | matrix 36 | skip_install = true 37 | commands = 38 | python ci/bootstrap.py --no-env 39 | 40 | [testenv:check] 41 | deps = 42 | docutils 43 | check-manifest 44 | flake8 45 | readme-renderer 46 | pygments 47 | isort >= 5 48 | skip_install = true 49 | commands = 50 | python setup.py check --strict --metadata --restructuredtext 51 | check-manifest {toxinidir} 52 | flake8 53 | isort --verbose --check-only --diff --filter-files . 54 | 55 | 56 | [testenv:docs] 57 | usedevelop = true 58 | deps = 59 | -r{toxinidir}/docs/requirements.txt 60 | commands = 61 | sphinx-build {posargs:-E} -b html docs dist/docs 62 | sphinx-build -b linkcheck docs dist/docs 63 | 64 | [testenv:coveralls] 65 | deps = 66 | coveralls 67 | skip_install = true 68 | commands = 69 | coveralls [] 70 | 71 | 72 | 73 | [testenv:codecov] 74 | deps = 75 | codecov 76 | skip_install = true 77 | commands = 78 | codecov [] 79 | 80 | [testenv:report] 81 | deps = coverage 82 | skip_install = true 83 | commands = 84 | coverage report 85 | coverage html 86 | 87 | [testenv:clean] 88 | commands = coverage erase 89 | skip_install = true 90 | deps = coverage 91 | 92 | [testenv:py37-cover-{linux}] 93 | basepython = {env:TOXPYTHON:python3.7} 94 | setenv = 95 | {[testenv]setenv} 96 | usedevelop = true 97 | commands = 98 | {posargs:pytest --cov --cov-report=term-missing -vv} 99 | deps = 100 | {[testenv]deps} 101 | pytest-cov 102 | 103 | [testenv:py37-nocov-{linux}] 104 | basepython = {env:TOXPYTHON:python3.7} 105 | 106 | [testenv:py38-cover-{linux}] 107 | basepython = {env:TOXPYTHON:python3.8} 108 | setenv = 109 | {[testenv]setenv} 110 | usedevelop = true 111 | commands = 112 | {posargs:pytest --cov --cov-report=term-missing -vv} 113 | deps = 114 | {[testenv]deps} 115 | pytest-cov 116 | 117 | [testenv:py38-nocov-{linux}] 118 | basepython = {env:TOXPYTHON:python3.8} 119 | --------------------------------------------------------------------------------