├── .gitignore ├── LICENSE.md ├── README.md ├── checksums.txt ├── comparison_plot.ipynb ├── functions ├── __init__.py ├── helper_functions.py ├── plots.py └── soaf.py ├── input ├── ENTSO-E │ ├── Power Statistics │ │ └── NGC.xlsx │ └── Transparency │ │ └── InstalledGenerationCapacityAggregated │ │ ├── 2015_1_InstalledGenerationCapacityAggregated.csv │ │ ├── 2016_1_InstalledGenerationCapacityAggregated.csv │ │ ├── 2017_1_InstalledGenerationCapacityAggregated.csv │ │ ├── 2018_1_InstalledGenerationCapacityAggregated.csv │ │ ├── 2019_1_InstalledGenerationCapacityAggregated.csv │ │ └── 2020_1_InstalledGenerationCapacityAggregated.csv ├── Eurostat │ └── Eurostat.tsv.gz ├── National_Generation_Capacities.xlsx ├── definition_EUROSTAT_indic.txt ├── energy_source_mapping.csv └── metadata.yml ├── main.ipynb ├── processing.ipynb └── requirements.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints/ 2 | output/ 3 | original_data/ 4 | download 5 | __pycache__ 6 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Open Power System Data 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Aggregated Generation Capacity by Technology and Country 2 | This is data package of aggregated generation capacities for European countries. It includes generation capacities for nuclear, fossil and renewable fuels on an aggregated and, if possible, disaggregated technology levels. 3 | 4 | See the [main Jupter notebook](main.ipynb) for further details. 5 | 6 | ## Preparation 7 | 8 | To work on the Notebooks locally see the installation instructions in the 9 | [wiki](https://github.com/Open-Power-System-Data/common/wiki/Tutorial-to-run-OPSD-scripts). 10 | 11 | ## License 12 | 13 | This notebook as well as all other documents in this repository is published under the [MIT License](LICENSE.md). -------------------------------------------------------------------------------- /checksums.txt: -------------------------------------------------------------------------------- 1 | national_generation_capacity.xlsx,c3deb84bded1cd25a0fa6b1d00b1b7cfd1c96d69de5db8a7a748f3457dd37191 2 | national_generation_capacity_stacked.csv,4890bf0e329499fbe22da29bd7a1c29418775dbf0f10cd38eff93e3035153d70 3 | national_generation_capacity_stacked.xlsx,6a4ed2864516f899bbe3de92af846e7c86fa0c36a8bde7da186662d4b22425c2 4 | national_generation_capacity.sqlite,f23b1e9e16a5775b4c4a7a7397ac7d0ff258119bf7de30a7988689667a6b01ce 5 | -------------------------------------------------------------------------------- /comparison_plot.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | " \n", 9 | " \n", 18 | " \n", 19 | "
\n", 10 | " National generation capacity: Check notebook\n", 11 | " \n", 16 | "
This Notebook is part of the National Generation Capacity Datapackage of Open Power System Data.\n", 17 | "
" 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "metadata": {}, 25 | "source": [ 26 | "# Table of Contents\n", 27 | "* [1. Introductory notes](#1.-Introductory-notes)\n", 28 | "* [2. Script setup](#2.-Script-setup)\n", 29 | "* [3. Import of processed data](#3.-Import-of-processed-data)\n", 30 | "* [4. Visualisation of results for different energy source levels](#4.-Visualisation-of-results-for-different-energy-source-levels)\n", 31 | "\t* [4.1 Energy source level 1](#4.1-Energy-source-level-1)\n", 32 | "\t\t* [4.1.1 Table](#4.1.1-Table)\n", 33 | "\t\t* [4.1.2 Bokeh chart](#4.1.2-Bokeh-chart)\n", 34 | "\t* [4.2 Energy source level 2](#4.2-Energy-source-level-2)\n", 35 | "\t\t* [4.2.1 Table](#4.2.1-Table)\n", 36 | "\t\t* [4.2.2 Bokeh chart](#4.2.2-Bokeh-chart)\n", 37 | "\t* [4.3 Energy source level 3](#4.3-Energy-source-level-3)\n", 38 | "\t\t* [4.3.1 Table](#4.3.1-Table)\n", 39 | "\t\t* [4.3.2 Bokeh chart](#4.3.2-Bokeh-chart)\n", 40 | "\t* [4.4 Technology level](#4.4-Technology-level)\n", 41 | "\t\t* [4.4.1 Table](#4.4.1-Table)\n", 42 | "\t\t* [4.4.2 Bokeh chart](#4.4.2-Bokeh-chart)\n", 43 | "* [5. Comparison of total capacity for energy source levels](#5.-Comparison-of-total-capacity-for-energy-source-levels)\n", 44 | "\t* [5.1 Calculation of total capacity for energy source levels](#5.1-Calculation-of-total-capacity-for-energy-source-levels)\n", 45 | "\t* [5.2 Identifcation of capacity differences for energy source levels](#5.2-Identifcation-of-capacity-differences-for-energy-source-levels)\n" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": { 51 | "collapsed": true 52 | }, 53 | "source": [ 54 | "# 1. Introductory notes" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "metadata": {}, 60 | "source": [ 61 | "The notebook extends the [processing notebook](processing.ipynb) to make visualisations." 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": { 67 | "collapsed": true 68 | }, 69 | "source": [ 70 | "# 2. Script setup" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": null, 76 | "metadata": {}, 77 | "outputs": [], 78 | "source": [ 79 | "import os.path\n", 80 | "import math\n", 81 | "import functions.plots as fp # predefined functions in extra file\n", 82 | "\n", 83 | "import bokeh.plotting as plo\n", 84 | "from bokeh.io import show, output_notebook\n", 85 | "from bokeh.layouts import row, column\n", 86 | "from bokeh.models import Panel, Tabs\n", 87 | "from bokeh.models.widgets import RangeSlider, MultiSelect, Select\n", 88 | "output_notebook()" 89 | ] 90 | }, 91 | { 92 | "cell_type": "markdown", 93 | "metadata": {}, 94 | "source": [ 95 | "# 3. Data import" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": null, 101 | "metadata": {}, 102 | "outputs": [], 103 | "source": [ 104 | "data_file = 'national_generation_capacity_stacked.csv'\n", 105 | "filepath = os.path.join('output', data_file)\n", 106 | "data = fp.load_opsd_data(filepath)\n", 107 | "data.head()" 108 | ] 109 | }, 110 | { 111 | "cell_type": "markdown", 112 | "metadata": {}, 113 | "source": [ 114 | "# 4. Create interactive plot" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": {}, 120 | "source": [ 121 | "Select individual width and height that fits your jupyter notebook settings." 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": null, 127 | "metadata": {}, 128 | "outputs": [], 129 | "source": [ 130 | "width = 1000\n", 131 | "height = 500" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": null, 137 | "metadata": {}, 138 | "outputs": [], 139 | "source": [ 140 | "def comparison_plot(doc):\n", 141 | " \n", 142 | " # init of 5 plots for each energy level\n", 143 | " sources = []\n", 144 | " plots = []\n", 145 | "\n", 146 | " for level in fp.energy_levels:\n", 147 | " # init plots with the predfined function\n", 148 | " s, p = fp.init_plot(data, level, size=(width, height))\n", 149 | " sources.append(s)\n", 150 | " plots.append(p)\n", 151 | "\n", 152 | " # associate each plot with a tab of the interactive plot\n", 153 | " panels= []\n", 154 | " for p, level in zip(plots, fp.energy_levels):\n", 155 | " panels.append(Panel(child=p, title=level))\n", 156 | "\n", 157 | " tabs = Tabs(tabs=panels, tabs_location='below', active=2)\n", 158 | "\n", 159 | " # Range slider for available years\n", 160 | " oldest_year = min(data[\"year\"])\n", 161 | " newest_year = max(data[\"year\"])\n", 162 | " y_slider = RangeSlider(title=\"Years\",\n", 163 | " value=(2015,2016),\n", 164 | " start=oldest_year,\n", 165 | " end=newest_year,\n", 166 | " step=1)\n", 167 | "\n", 168 | " # Select field for sources\n", 169 | " m_select = MultiSelect(title=\"Available Sources:\", \n", 170 | " value=fp.global_sources,\n", 171 | " options=[(s,s) for s in fp.global_sources])\n", 172 | "\n", 173 | " # Select button for countries\n", 174 | " countries = list(data[\"country\"].unique())\n", 175 | " c_select = Select(title=\"Country\", options=countries, value='FR')\n", 176 | "\n", 177 | " # catch all widgets \n", 178 | " wid = [c_select, y_slider, m_select]\n", 179 | " rows = row(wid)\n", 180 | "\n", 181 | " # update function for `on_change` trigger\n", 182 | " def update(attrname, old, new):\n", 183 | "\n", 184 | " y = y_slider.value\n", 185 | " y_range = [x for x in range(y[0],y[1]+1)]\n", 186 | " s_selected = m_select.value\n", 187 | " co = c_select.value\n", 188 | "\n", 189 | " # run update for each plot\n", 190 | " for p, s, l in zip(plots, sources, fp.energy_levels):\n", 191 | " df = fp.filter_data_set(data, co, y_range, s_selected, l)\n", 192 | " source_data, x_axis = fp.prepare_data(df)\n", 193 | " s.data = source_data\n", 194 | " p.x_range.factors = x_axis\n", 195 | "\n", 196 | " # associate `update` function with each widget to apply updates for each change\n", 197 | " for w in wid:\n", 198 | " w.on_change('value', update)\n", 199 | "\n", 200 | " \n", 201 | " layout = column(rows, tabs)\n", 202 | " doc.add_root(layout)\n" 203 | ] 204 | }, 205 | { 206 | "cell_type": "markdown", 207 | "metadata": {}, 208 | "source": [ 209 | "After the bokeh plot is set up a bokeh server is started to make the plot interactive.\n", 210 | "\n", 211 | "Possible options:\n", 212 | "\n", 213 | " - Select a country from the dropdown menu in the top left\n", 214 | " - Select a range of years from the range slider\n", 215 | " - Multiselect the sources you want to compare in the top right\n", 216 | " - Choose which \"energy level\" you want to investigate with the tabs below the plot" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": null, 222 | "metadata": { 223 | "scrolled": true 224 | }, 225 | "outputs": [], 226 | "source": [ 227 | "show(comparison_plot)" 228 | ] 229 | }, 230 | { 231 | "cell_type": "code", 232 | "execution_count": null, 233 | "metadata": {}, 234 | "outputs": [], 235 | "source": [] 236 | } 237 | ], 238 | "metadata": { 239 | "@webio": { 240 | "lastCommId": null, 241 | "lastKernelId": null 242 | }, 243 | "anaconda-cloud": {}, 244 | "kernelspec": { 245 | "display_name": "Python 3", 246 | "language": "python", 247 | "name": "python3" 248 | }, 249 | "language_info": { 250 | "codemirror_mode": { 251 | "name": "ipython", 252 | "version": 3 253 | }, 254 | "file_extension": ".py", 255 | "mimetype": "text/x-python", 256 | "name": "python", 257 | "nbconvert_exporter": "python", 258 | "pygments_lexer": "ipython3", 259 | "version": "3.6.9" 260 | } 261 | }, 262 | "nbformat": 4, 263 | "nbformat_minor": 1 264 | } 265 | -------------------------------------------------------------------------------- /functions/__init__.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import logging 3 | 4 | # sets up the logger config 5 | logging.basicConfig( 6 | level=logging.INFO, 7 | format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', 8 | datefmt='%d %b %Y %H:%M:%S' 9 | ) 10 | 11 | #init a logger 12 | logger = logging.getLogger() 13 | 14 | # create output directories if they do not exist 15 | os.makedirs('output', exist_ok=True) 16 | os.makedirs(os.path.join('output'), exist_ok=True) 17 | os.makedirs(os.path.join('output', 'original_data'), exist_ok=True) 18 | 19 | -------------------------------------------------------------------------------- /functions/helper_functions.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import os.path 3 | import urllib.parse 4 | import urllib.request 5 | import zipfile 6 | import shutil 7 | from . import logger 8 | 9 | 10 | def get_sha_hash(path, blocksize=65536): 11 | sha_hasher = hashlib.sha256() 12 | with open(path, 'rb') as f: 13 | buffer = f.read(blocksize) 14 | while len(buffer) > 0: 15 | sha_hasher.update(buffer) 16 | buffer = f.read(blocksize) 17 | return sha_hasher.hexdigest() 18 | 19 | 20 | def createDownloadFolderPath(source): 21 | folderpath = os.path.join('download', source) 22 | 23 | return folderpath 24 | 25 | def isZipAndNotExcel(filepath): 26 | """ 27 | Returns false if the file is an Excel file. Return true if file is a zipped. 28 | Parameters 29 | ---------- 30 | filepath : str 31 | Path to the file 32 | """ 33 | 34 | if (os.path.splitext(filepath)[1] in [".xlsx", ".xls"] or not 35 | zipfile.is_zipfile(filepath)): 36 | 37 | return False 38 | 39 | else: 40 | return True 41 | 42 | 43 | 44 | def downloadandcache(url, filename, source): 45 | """ 46 | Download a file into a folder called "downloads". 47 | Returns the local filepath. 48 | 49 | Parameters 50 | ---------- 51 | url : str 52 | Url of a file to be downloaded 53 | filename : str 54 | Name of the downloaded file 55 | source : str 56 | Source of the downloaded file 57 | """ 58 | 59 | folderpath = createDownloadFolderPath(source) 60 | filepath = os.path.join(folderpath, filename) 61 | 62 | # check if file exists, otherwise download it 63 | if not os.path.exists(filepath): 64 | os.makedirs(folderpath, exist_ok=True) 65 | logger.info('Downloading file %s', filename) 66 | urllib.request.urlretrieve(url, filepath) 67 | 68 | # extract files if they are zipped 69 | if isZipAndNotExcel(filepath): 70 | logger.info('Extracting %s into the directory %s', filename, folderpath) 71 | with zipfile.ZipFile(filepath, "r") as O: 72 | O.extractall(folderpath) 73 | 74 | return folderpath 75 | else: 76 | return filepath 77 | 78 | else: 79 | if isZipAndNotExcel(filepath): 80 | logger.info('Using extracted zip files from %s', folderpath) 81 | return folderpath 82 | else: 83 | logger.info('Using local file from %s', filepath) 84 | return filepath 85 | 86 | 87 | def unstackData(df): 88 | 89 | pt = df.pivot_table(values='capacity', 90 | index=['country','year'], 91 | columns='technology') 92 | 93 | return pt 94 | 95 | def restackData(df): 96 | return df.stack().reset_index().rename(columns={0: 'capacity'}) 97 | 98 | def checkIfEmptyAndSetDefault(df, technology, default=0): 99 | sub_df = df.loc[df['technology'] == technology, 'capacity'] 100 | 101 | if len(sub_df) == 0: 102 | return default 103 | else: 104 | return sub_df.values[0] 105 | 106 | 107 | def copydir(source, dest): 108 | """Copy a directory structure overwriting existing files""" 109 | for root, dirs, files in os.walk(source): 110 | if not os.path.isdir(root): 111 | os.makedirs(root) 112 | 113 | for file in files: 114 | rel_path = root.replace(source, '').lstrip(os.sep) 115 | dest_path = os.path.join(dest, rel_path) 116 | 117 | if not os.path.isdir(dest_path): 118 | os.makedirs(dest_path) 119 | 120 | shutil.copyfile(os.path.join(root, file), os.path.join(dest_path, file)) 121 | -------------------------------------------------------------------------------- /functions/plots.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import math 3 | 4 | import bokeh.plotting as plt 5 | from bokeh.models import ColumnDataSource, Legend, LegendItem 6 | 7 | 8 | colormap = { 9 | 'Fossil fuels': 'Black', 10 | 'Lignite': '#754937', 11 | 'Hard coal': '#4d3e35', 12 | 'Oil': '#272724', 13 | 'Natural gas': '#e54213', 14 | 'Combined cycle': '#ff814b', 15 | 'Gas turbine': '#880000', 16 | 'Other and unknown natural gas': '#e54213', 17 | 'Differently categorized natural gas': '#e54213', 18 | 'Non-renewable waste': '#b7c7cd', 19 | 'Mixed fossil fuels': '#e9f9fd', 20 | 'Other fossil fuels': '#6f7d84', 21 | 'Differently categorized fossil fuels': '#6f7d84', 22 | 'Nuclear': '#ae393f', 23 | 'Renewable energy sources': '#24693d', 24 | 'Hydro': '#0d47a1', 25 | 'Run-of-river': '#6782e4', 26 | 'Reservoir': '#2e56b4', 27 | 'Reservoir including pumped storage': '#5472d3', 28 | 'Pumped storage': '#002171', 29 | 'Pumped storage with natural inflow': '#00125e', 30 | 'Differently categorized hydro': '#0d47a1', 31 | 'Wind': '#326776', 32 | 'Onshore': '#518696', 33 | 'Offshore': '#215968', 34 | 'Differently categorized wind': '#00303d', 35 | 'Solar': '#ffeb3b', 36 | 'Photovoltaics': '#fffb4e', 37 | 'Concentrated solar power': '#d5c200', 38 | 'Differently categorized solar': '#ffeb3b', 39 | 'Geothermal': '#ef9347', 40 | 'Marine': '#002171', 41 | 'Bioenergy and renewable waste': '#7cb342', 42 | 'Biomass and biogas': '#c2f08e', 43 | 'Sewage and landfill gas': '#95cb59', 44 | 'Other bioenergy and renewable waste': '#316a00', 45 | 'Differently categorized renewable energy sources': '#dce775', 46 | 'Other or unspecified energy sources': '#9526b7', 47 | 'Total': 'Black', 48 | 'renewable': '#316a00' 49 | } 50 | 51 | 52 | energy_levels = ['energy_source_level_0','energy_source_level_1', 53 | 'energy_source_level_2', 'energy_source_level_3', 54 | 'technology_level'] 55 | 56 | global_sources = ['EUROSTAT','ENTSO-E SOAF','ENTSO-E Data Portal', 57 | 'ENTSO-E Transparency Platform','ENTSO-E Power Statistics', 58 | 'National source'] 59 | 60 | 61 | def which_source(x): 62 | if x in global_sources: 63 | return x 64 | else: 65 | return 'National source' 66 | 67 | def load_opsd_data(filepath): 68 | data = pd.read_csv(filepath, index_col=0) 69 | data["capacity"] = data["capacity"]/1000 70 | data["source_new"] = data["source"].apply(which_source) 71 | return data 72 | 73 | 74 | 75 | def filter_data_set(data, input_co, input_year, input_source, level, unique=False): 76 | 77 | co_subset = data['country'] == input_co 78 | y_subset = data['year'].isin(input_year) 79 | energy_source_level = data[level] == True 80 | 81 | if isinstance(input_source, str): 82 | input_source = [input_source] 83 | 84 | source_subset = data['source_new'].isin(input_source) 85 | drop_list = co_subset & y_subset & energy_source_level & source_subset 86 | ret_data = data.loc[drop_list, :].copy() 87 | y_col = ret_data["year"].astype(str, copy=True).to_numpy() 88 | ret_data.drop(labels='year', axis=1, inplace=True) 89 | ret_data.loc[:,"year"] = y_col 90 | 91 | if unique: 92 | technologies = list(ret_data.technology.unique()) 93 | years = list(ret_data.year.unique()) 94 | return ret_data, technologies, years 95 | else: 96 | return ret_data 97 | 98 | 99 | def prepare_data(data): 100 | source_data = pd.pivot_table(data, 101 | index=('year','source'), 102 | columns='technology', 103 | values='capacity', 104 | aggfunc=sum, 105 | margins=False) 106 | 107 | source_data.fillna(0, inplace=True) 108 | x_axis = list(source_data.index) 109 | source_data.reset_index(inplace=True) 110 | source_data['index'] = x_axis 111 | 112 | return source_data, x_axis 113 | 114 | 115 | def init_plot(data, level, size=(800,800)): 116 | 117 | ret_data, technologies, years = filter_data_set(data, 118 | 'DE', 119 | [2015,2016], 120 | ['EUROSTAT','ENTSO-E SOAF'], 121 | level, 122 | unique=True) 123 | 124 | source_data, x_axis = prepare_data(ret_data) 125 | 126 | source = ColumnDataSource(source_data) 127 | colors = [colormap[t] for t in technologies] 128 | 129 | p = plt.figure(x_range=[], title="Comparison "+level, plot_width=size[0], plot_height=size[1], 130 | y_axis_label='GW', toolbar_location="above") 131 | 132 | p.x_range.factors = x_axis 133 | 134 | r = p.vbar_stack(technologies, x='index', width=0.8, source=source, 135 | color=colors) 136 | 137 | p.xaxis.major_label_orientation = math.pi/2 138 | 139 | it = [LegendItem(label=t, renderers=[r[i]]) for i,t in enumerate(technologies)] 140 | legend = Legend(items=it) 141 | p.add_layout(legend, 'right') 142 | 143 | return source, p 144 | 145 | 146 | 147 | 148 | 149 | -------------------------------------------------------------------------------- /functions/soaf.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import functions.helper_functions as f 3 | import pandas as pd 4 | import numpy as np 5 | from . import logger 6 | 7 | 8 | dict_energy_source = {'Nuclear Power': 'Nuclear', 9 | 'Fossil Fuels': 'Fossil fuels', 10 | 'Hard Coal': 'Hard coal', 11 | 'Lignite': 'Lignite', 12 | 'Gas': 'Natural gas', 13 | 'Oil': 'Oil', 14 | 'Mixed Fuels': 'Mixed fossil fuels', 15 | 'Hydro power (total)': 'Hydro', 16 | 'of which renewable hydro generation': 'NaN', 17 | 'of which run-of-river (pre-dominantly)': 'Run-of-river', 18 | 'of which storage and pumped storage (total)': 'Reservoir including pumped storage', # auxiliary class definition 19 | 'Renewable Energy Sources (other than hydro)': 'renewable', 20 | 'Solar': 'Solar', 21 | 'Wind': 'Wind', 22 | 'of which offshore': 'Offshore', 23 | 'of which onshore': 'Onshore', 24 | 'Biomass': 'Biomass and biogas', 25 | 'Not Clearly Identifiable Energy Sources': 'Other or unspecified energy sources', 26 | 'Net generating Capacity': 'NaN', 27 | 'Import Capacity': 'NaN', 28 | 'Export Capacity': 'NaN', 29 | 'Load': 'NaN', 30 | 'Load Management': 'NaN', 31 | 'Maintenance and Overhauls': 'NaN', 32 | 'Margin Against Seasonal Peak Load': 'NaN', 33 | 'Adequacy Reference Margin': 'NaN', 34 | 'National Power Data': 'NaN', 35 | 'Non-Usable Capacity': 'NaN', 36 | 'Outages': 'NaN', 37 | 'Reliable Available Capacity': 'NaN', 38 | 'Remaining Capacity': 'NaN', 39 | 'Spare Capacity': 'NaN', 40 | 'System Service Reserve': 'NaN', 41 | 'Unavailable Capacity': 'NaN', 42 | 'Simultaneous Exportable Capacity for Adequacy': 'NaN', 43 | 'Simultaneous Importable Capacity for Adequacy': 'NaN', 44 | '“The values of Simultaneous Importable/Exportable Capacity for Adequacy do not include the border with Austria as there is a common market between Germany and Austria for which no NTC exists.”': 'NaN'} 45 | 46 | class SoafDataRaw: 47 | 48 | 49 | def __init__(self, url, zipfile, xls, year): 50 | 51 | self.source = 'ENTSO-E' 52 | self.year = year 53 | self.sourcefolder = os.path.join('SO&AF', str(year)) 54 | 55 | self.folderpath = f.downloadandcache(url, 56 | zipfile, 57 | os.path.join(self.source, 58 | self.sourcefolder)) 59 | 60 | self.filepath = os.path.join(self.folderpath, xls) 61 | 62 | 63 | self.transformIntoDataFrame() 64 | 65 | 66 | 67 | def transformIntoDataFrame(self): 68 | 69 | s = "Reading {file} and transforming into a DataFrame".format(file=self.filepath) 70 | logger.info(s) 71 | 72 | dict_with_dfs = pd.read_excel(self.filepath, sheet_name=None, 73 | skiprows=11, header=[0], index_col=0) 74 | 75 | collect_arr = list() 76 | for key,df in dict_with_dfs.items(): 77 | df.reset_index(inplace=True) 78 | df = df[['index', self.year]].rename(columns={'index': 'technology'}) 79 | subset = df['technology'].isin(dict_energy_source.keys()) 80 | df= df[subset] 81 | 82 | df['technology'].replace(dict_energy_source, inplace=True) 83 | df['country'] = key[0:2] 84 | df['year'] = self.year 85 | df.rename(columns={self.year: 'value'}, inplace=True) 86 | df['value'] = 1000 * df['value'] 87 | df['value'].replace({'NaN' : 0, np.nan : 0}, inplace=True) 88 | df.rename(columns={'value' : 'capacity'}, inplace=True) 89 | df['technology'].replace('NaN', np.nan, inplace=True) 90 | df = df[df['technology'].isnull() == False] 91 | df['capacity'] = df['capacity'].astype(float) 92 | 93 | collect_arr.append(df) 94 | 95 | df = pd.concat(collect_arr) 96 | 97 | self.transformed_df = df -------------------------------------------------------------------------------- /input/ENTSO-E/Power Statistics/NGC.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Open-Power-System-Data/national_generation_capacity/9a404fd9cda6bef7e6a826e4ebc73dbb0e5591b2/input/ENTSO-E/Power Statistics/NGC.xlsx -------------------------------------------------------------------------------- /input/ENTSO-E/Transparency/InstalledGenerationCapacityAggregated/2015_1_InstalledGenerationCapacityAggregated.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Open-Power-System-Data/national_generation_capacity/9a404fd9cda6bef7e6a826e4ebc73dbb0e5591b2/input/ENTSO-E/Transparency/InstalledGenerationCapacityAggregated/2015_1_InstalledGenerationCapacityAggregated.csv -------------------------------------------------------------------------------- /input/ENTSO-E/Transparency/InstalledGenerationCapacityAggregated/2016_1_InstalledGenerationCapacityAggregated.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Open-Power-System-Data/national_generation_capacity/9a404fd9cda6bef7e6a826e4ebc73dbb0e5591b2/input/ENTSO-E/Transparency/InstalledGenerationCapacityAggregated/2016_1_InstalledGenerationCapacityAggregated.csv -------------------------------------------------------------------------------- /input/ENTSO-E/Transparency/InstalledGenerationCapacityAggregated/2017_1_InstalledGenerationCapacityAggregated.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Open-Power-System-Data/national_generation_capacity/9a404fd9cda6bef7e6a826e4ebc73dbb0e5591b2/input/ENTSO-E/Transparency/InstalledGenerationCapacityAggregated/2017_1_InstalledGenerationCapacityAggregated.csv -------------------------------------------------------------------------------- /input/ENTSO-E/Transparency/InstalledGenerationCapacityAggregated/2018_1_InstalledGenerationCapacityAggregated.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Open-Power-System-Data/national_generation_capacity/9a404fd9cda6bef7e6a826e4ebc73dbb0e5591b2/input/ENTSO-E/Transparency/InstalledGenerationCapacityAggregated/2018_1_InstalledGenerationCapacityAggregated.csv -------------------------------------------------------------------------------- /input/ENTSO-E/Transparency/InstalledGenerationCapacityAggregated/2019_1_InstalledGenerationCapacityAggregated.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Open-Power-System-Data/national_generation_capacity/9a404fd9cda6bef7e6a826e4ebc73dbb0e5591b2/input/ENTSO-E/Transparency/InstalledGenerationCapacityAggregated/2019_1_InstalledGenerationCapacityAggregated.csv -------------------------------------------------------------------------------- /input/ENTSO-E/Transparency/InstalledGenerationCapacityAggregated/2020_1_InstalledGenerationCapacityAggregated.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Open-Power-System-Data/national_generation_capacity/9a404fd9cda6bef7e6a826e4ebc73dbb0e5591b2/input/ENTSO-E/Transparency/InstalledGenerationCapacityAggregated/2020_1_InstalledGenerationCapacityAggregated.csv -------------------------------------------------------------------------------- /input/Eurostat/Eurostat.tsv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Open-Power-System-Data/national_generation_capacity/9a404fd9cda6bef7e6a826e4ebc73dbb0e5591b2/input/Eurostat/Eurostat.tsv.gz -------------------------------------------------------------------------------- /input/National_Generation_Capacities.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Open-Power-System-Data/national_generation_capacity/9a404fd9cda6bef7e6a826e4ebc73dbb0e5591b2/input/National_Generation_Capacities.xlsx -------------------------------------------------------------------------------- /input/definition_EUROSTAT_indic.txt: -------------------------------------------------------------------------------- 1 | 12_1176011 Electrical capacity, main activity producers - Combustible Fuels Fossil fuels 2 | 12_1176012 Electrical capacity, autoproducers - Combustible Fuels Fossil fuels 3 | 12_1176031 Electrical capacity, main activity producers - Nuclear Nuclear 4 | 12_1176032 Electrical capacity, autoproducers - Nuclear Nuclear 5 | 12_1176051 Electrical capacity, main activity producers - Hydro Hydro 6 | 12_1176052 Electrical capacity, autoproducers - Hydro Hydro 7 | 12_1176061 Electrical capacity, main activity producers - Mixed plants 8 | 12_1176071 Net electrical capacity, main activity producers - Pure Pumped Hydro Pumped storage 9 | 12_1176072 Net electrical capacity, autoproducers - Pure Pumped Hydro Pumped storage 10 | 12_1176081 Electrical capacity, main activity producers - Geothermal Geothermal 11 | 12_1176082 Electrical capacity, autoproducers - Geothermal Geothermal 12 | 12_1176083 Net maximum capacity - Geothermal 13 | 12_1176091 Electrical capacity, main activity producers - Wind Wind 14 | 12_1176092 Electrical capacity, autoproducers - Wind Wind 15 | 12_1176101 Electrical capacity, main activity producers - Other Sources 16 | 12_1176102 Electrical capacity, autoproducers - Other Sources 17 | 12_1176111 Electrical capacity, main activity producers - Steam 18 | 12_1176112 Electrical capacity, autoproducers - Steam 19 | 12_1176121 Electrical capacity, main activity producers - Gas Turbine 20 | 12_1176122 Electrical capacity, autoproducers - Gas Turbine 21 | 12_1176131 Electrical capacity, main activity producers - Combined Cycle 22 | 12_1176132 Electrical capacity, autoproducers - Combined Cycle 23 | 12_1176141 Electrical capacity, main activity producers - Internal Combustion 24 | 12_1176142 Electrical capacity, autoproducers - Internal Combustion 25 | 12_117615 Net maximum capacity - Hydro <1 MW 26 | 12_117616 Net maximum capacity - Hydro >= 1 MW and <= 10 MW 27 | 12_117617 Net maximum capacity - Hydro 10 MW and over 28 | 12_1176233 Net maximum capacity - Solar Photovoltaic Photovoltaics 29 | 12_1176243 Net maximum capacity - Solar Thermal Electric Concentrated solar power 30 | 12_1176253 Net maximum capacity - Municipal Wastes Non-renewable waste 31 | 12_1176263 Net maximum capacity - Wood/Wood Wastes/Other Solid Wastes Other bioenergy and renewable waste 32 | 12_1176273 Net maximum capacity - Biogases Biomass and biogas 33 | 12_1176283 Net maximum capacity - Industrial Wastes (non-renewable) Non-renewable waste 34 | 12_1176301 Electrical capacity, main activity producers - Tide, wave and ocean Marine 35 | 12_1176302 Electrical capacity, autoproducers - Tide, wave and ocean Marine 36 | 12_1176303 Net maximum capacity - Tide, Wave, Ocean 37 | 12_1176343 Net maximum capacity - Liquid Biofuels Biomass and biogas 38 | 12_1176401 Electrical capacity, main activity producers - Other Type of Generation 39 | 12_1176402 Electrical capacity, autoproducers - Other Type of Generation 40 | -------------------------------------------------------------------------------- /input/energy_source_mapping.csv: -------------------------------------------------------------------------------- 1 | name,Level 0,Level 1,Level 2,Level 3,Technology level 2 | Fossil fuels,0,1,0,0,0 3 | Lignite,0,0,1,1,1 4 | Hard coal,0,0,1,1,1 5 | Oil,0,0,1,1,1 6 | Natural gas,0,0,1,1,0 7 | Combined cycle,0,0,0,0,1 8 | Gas turbine,0,0,0,0,1 9 | Other and unknown natural gas,0,0,0,0,1 10 | Differently categorized natural gas,0,0,0,0,1 11 | Non-renewable waste,0,0,1,1,1 12 | Mixed fossil fuels,0,0,1,1,1 13 | Other fossil fuels,0,0,1,1,1 14 | Differently categorized fossil fuels,0,0,1,1,1 15 | Nuclear,0,1,1,1,1 16 | Renewable energy sources,0,1,0,0,0 17 | Hydro,0,0,1,1,0 18 | Run-of-river,0,0,0,0,1 19 | Reservoir,0,0,0,0,1 20 | Reservoir including pumped storage,0,0,0,0,1 21 | Pumped storage,0,0,0,0,1 22 | Pumped storage with natural inflow,0,0,0,0,1 23 | Differently categorized hydro,0,0,0,0,1 24 | Wind,0,0,1,1,0 25 | Onshore,0,0,0,0,1 26 | Offshore,0,0,0,0,1 27 | Differently categorized wind,0,0,0,0,1 28 | Solar,0,0,1,1,0 29 | Photovoltaics,0,0,0,0,1 30 | Concentrated solar power,0,0,0,0,1 31 | Differently categorized solar,0,0,0,0,1 32 | Geothermal,0,0,1,1,1 33 | Marine,0,0,1,1,1 34 | Bioenergy and renewable waste,0,0,1,0,0 35 | Biomass and biogas,0,0,0,1,1 36 | Sewage and landfill gas,0,0,0,1,1 37 | Other bioenergy and renewable waste,0,0,0,1,1 38 | Differently categorized renewable energy sources,0,0,1,1,1 39 | Other or unspecified energy sources,0,1,1,1,1 40 | Total,1,0,0,0,0 41 | -------------------------------------------------------------------------------- /input/metadata.yml: -------------------------------------------------------------------------------- 1 | name: opsd-national_generation_capacity 2 | title: National generation capacity 3 | description: Aggregated generation capacity by technology and country 4 | longDescription: This data package comprises technology-specific aggregated generation capacities for European countries. The generation capacities are consistently categorized based on fuel and technology. For each European country, various references are used ranging from international (e.g. ENTSOE or EUROSTAT) to national sources from e.g. regulatory authorities. The input data is processed in the script linked below. 5 | profile: data-package 6 | _external: false 7 | _metadataVersion: 1.2 8 | attribution: "Attribution should be given as follows: Open Power System Data. 2020. Data Package National generation capacity. Version 2020-10-01. https://doi.org/10.25832/national_generation_capacity/2020-10-01. (Primary data from various sources, for a complete list see URL)." 9 | keywords: 10 | - generation capacity 11 | - conventional generation 12 | - renewable generation 13 | - Open Power System Data 14 | - Europe 15 | spatial: 16 | location: Austria, Belgium, Bulgaria, Switzerland, Czech Republic, Germany, Denmark, Estonia, Spain, Finland, France, Greece, Hungary, Irland, Italy, Lithuania, Luxemburg, Latvia, the Netherlands, Norway, Poland, Portugal, Romania, Sweden, Slovenia, Slovakia, United Kingdom 17 | resolution: national aggregates 18 | temporal: 19 | start: '1990' 20 | end: '2020' 21 | version: '2020-10-01' 22 | publicationDate: '2020-10-01' 23 | id: https://doi.org/10.25832/national_generation_capacity/2020-10-01 24 | homepage: https://data.open-power-system-data.org/national_generation_capacity/2020-10-01 25 | lastChanges: Updated data for 2020, bugfixes 26 | documentation: https://github.com/Open-Power-System-Data/national_generation_capacity/blob/2020-10-01/main.ipynb 27 | contributors: 28 | - title: Mario Kendziorski 29 | email: mak@wip.tu-berlin.de 30 | path: https://www.wip.tu-berlin.de/ 31 | organization: TU Berlin, Workgroup for Infrastructure Policy (WIP) 32 | role: maintainer 33 | - title: Elmar Zozmann 34 | email: ez@wip.tu-berlin.de 35 | path: https://www.wip.tu-berlin.de/ 36 | organization: TU Berlin, Workgroup for Infrastructure Policy (WIP) 37 | role: maintainer 38 | - title: Friedrich Kunz 39 | organization: German Institute for Economic Research (DIW Berlin) 40 | role: author 41 | resources: 42 | - name: national_generation_capacity 43 | profile: tabular-data-resource 44 | title: Statistics of national generation capacities 45 | bytes: 46 | hash: 47 | path: national_generation_capacity.xlsx 48 | format: xlsx 49 | mediatype: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet 50 | - name: national_generation_capacity_stacked 51 | profile: tabular-data-resource 52 | title: Statistics of national generation capacities 53 | bytes: 54 | hash: 55 | path: national_generation_capacity_stacked.csv 56 | format: csv 57 | mediatype: text/csv 58 | encoding: UTF8 59 | dialect: 60 | csvddfVersion: 1.2 61 | delimiter: "," 62 | lineTerminator: "\\n" 63 | header: true 64 | alternativeFormats: 65 | - path: national_generation_capacity_stacked.xlsx 66 | stacking: Stacked 67 | format: xlsx 68 | schema: # Schema according to: http://dataprotocols.org/json-table-schema/ 69 | missingValues: "" 70 | primaryKey: id 71 | fields: 72 | - name: id 73 | description: ID for data entries 74 | type: integer 75 | - name: technology 76 | description: Generation technology defined by fuel and conversion technology 77 | type: string 78 | opsdContentfilter: "true" 79 | - name: source 80 | description: Source of data entry 81 | type: string 82 | opsdContentfilter: "true" 83 | - name: source_type 84 | description: Type of data source 85 | type: string 86 | opsdContentfilter: "true" 87 | - name: year 88 | description: Year of data entry 89 | type: integer 90 | format: YYYY 91 | opsdContentfilter: "true" 92 | - name: type 93 | description: Type of capacity (e.g. installed capacity) 94 | type: string 95 | - name: country 96 | description: Country ISO code 97 | type: string 98 | opsdContentfilter: "true" 99 | - name: capacity_definition 100 | description: Capacity definition used in the relevant source (net, gross, or unknown) 101 | type: string 102 | - name: comment 103 | description: Comments on data entry 104 | type: string 105 | - name: capacity 106 | description: Installed capacity in MW 107 | type: float 108 | - name: energy_source_level_0 109 | description: Energy source level 0 (total aggregated capacity) 110 | type: boolean 111 | - name: energy_source_level_1 112 | description: Energy source level 1 (aggregation or classification by type of fuel) 113 | type: boolean 114 | - name: energy_source_level_2 115 | description: Energy source level 2 (aggregation or classification by fuel) 116 | type: boolean 117 | - name: energy_source_level_3 118 | description: Energy source level 3 (aggregation or classification by fuel refined for bioenergy) 119 | type: boolean 120 | - name: technology_level 121 | description: Technology (aggregation or classification by fuel and technology) 122 | type: boolean 123 | sources: 124 | - title: EUROSTAT 125 | path: http://ec.europa.eu/eurostat/product?code=nrg_113a&mode=view 126 | - title: UN Statistical Office 127 | path: http://data.un.org/Data.aspx?d=EDATA&f=cmID%3AEC 128 | - title: ENTSOE Statistics 129 | path: https://www.entsoe.eu/db-query/miscellaneous/net-generating-capacity 130 | - title: ENTSOE Power Statistics 131 | path: https://www.entsoe.eu/data/power-stats/net-gen-capacity/ 132 | - title: ENTSOE SO&AF 133 | path: https://www.entsoe.eu/outlooks/maf/Pages/default.aspx 134 | - title: ENTSOE Transparency Platform 135 | path: https://transparency.entsoe.eu/generation/r2/installedGenerationCapacityAggregation/show 136 | - title: e-control 137 | path: http://www.e-control.at/statistik/strom/bestandsstatistik 138 | - title: ELIA 139 | path: http://www.elia.be/en/grid-data/power-generation/generating-facilities 140 | - title: TSO Bulgaria 141 | path: http://www.eso.bg/?did=79#Reports 142 | - title: BFE 143 | path: https://www.bfe.admin.ch/bfe/de/home/versorgung/statistik-und-geodaten/energiestatistiken/elektrizitaetsstatistik.html 144 | - title: ERU 145 | path: https://www.eru.cz/en/zpravy-o-provozu-elektrizacni-soustavy 146 | - title: BMWi 147 | path: https://www.bmwi.de/Redaktion/DE/Artikel/Energie/energiedaten-gesamtausgabe.html 148 | - title: DEA 149 | path: https://ens.dk/en/our-services/statistics-data-key-figures-and-energy-maps/annual-and-monthly-statistics 150 | - title: Statistics Estonia 151 | path: http://pub.stat.ee/px-web.2001/Dialog/varval.asp?ma=FE032&ti=CAPACITY+AND+PRODUCTION+OF+POWER+PLANTS&path=../I_Databas/Economy/07Energy/02Energy_consumption_and_production/01Annual_statistics/&lang=1 152 | - title: REE 153 | path: http://www.ree.es/en/statistical-data-of-spanish-electrical-system/national-indicators/national-indicators 154 | - title: RTE 155 | path: https://www.services-rte.com/en/view-data-published-by-rte/production-installed-capacity.html 156 | - title: Statistics Finland 157 | path: http://pxnet2.stat.fi/PXWeb/pxweb/en/StatFin_Passiivi/StatFin_Passiivi__ene__ehk/statfinpas_ehk_pxt_904_201500_en.px/ 158 | - title: Department of Energy & Climate Change UK 159 | path: https://www.gov.uk/government/statistics/electricity-chapter-5-digest-of-united-kingdom-energy-statistics-dukes 160 | - title: REA 161 | path: http://www.rae.gr/site/en_US/categories_new/about_rae/actions/reports_national.csp 162 | - title: HOPS 163 | path: https://www.hops.hr/en/basic-data 164 | - title: Mavir 165 | path: https://www.mavir.hu/hu/web/mavir-en/installed-generation-capacity 166 | - title: Eirgrid 167 | path: http://www.eirgridgroup.com/library/ 168 | - title: Terna 169 | path: https://www.terna.it/en/electric-system/transparency-report/installed-capacity 170 | - title: Litgrid 171 | path: https://www.litgrid.eu/index.php/power-system/power-system-information/generation-capacity/546 172 | - title: Central Statistical Bureau of Latvia 173 | path: http://data1.csb.gov.lv/pxweb/en/vide/vide__energetika__ikgad/?tablelist=true 174 | - title: Tennet 175 | path: https://www.tennet.org/english/operational_management/export_data.aspx 176 | - title: Statistics Norway 177 | path: https://www.ssb.no/en/energi-og-industri/statistikker/elektrisitet/aar 178 | - title: CIRE 179 | path: http://www.rynek-energii-elektrycznej.cire.pl/st,33,207,tr,75,0,0,0,0,0,podstawowe-dane.html 180 | - title: REN 181 | path: http://www.ren.pt/en-GB/media/publications/ 182 | - title: ANRE 183 | path: https://www.anre.ro/en/about-anre/annual-reports-archive 184 | - title: Swedish Energy Agency 185 | path: https://www.energimyndigheten.se/en/facts-and-figures/publications/ 186 | - title: Agencija za energijo 187 | path: https://www.agen-rs.si/web/en/publications/-/asset_publisher/TCdo6cWN9Shk/content/te-1?_101_INSTANCE_TCdo6cWN9Shk_redirect=%2Fweb%2Fen%2Fpublications 188 | - title: Statistical Office of Slovakia 189 | path: https://slovak.statistics.sk/wps/portal/ext/themes/multi/energy/publications -------------------------------------------------------------------------------- /main.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | " \n", 9 | " \n", 18 | " \n", 19 | "
\n", 10 | " National generation capacity: Main notebook\n", 11 | " \n", 16 | "
This Notebook is part of the National Generation Capacity Datapackage of Open Power System Data.\n", 17 | "
" 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "metadata": {}, 25 | "source": [ 26 | "# 1. About Open Power System Data \n", 27 | "This notebook is part of the project [Open Power System Data](http://open-power-system-data.org). Open Power System Data develops a platform for free and open data for electricity system modeling. We collect, check, process, document, and provide data that are publicly available but currently inconvenient to use. \n", 28 | "More info on Open Power System Data:\n", 29 | "- [Information on the project on our website](http://open-power-system-data.org)\n", 30 | "- [Data and metadata on our data platform](http://data.open-power-system-data.org)\n", 31 | "- [Data processing scripts on our GitHub page](https://github.com/Open-Power-System-Data)\n", 32 | "\n", 33 | "# 2. About Jupyter Notebooks and GitHub\n", 34 | "This file is a [Jupyter Notebook](http://jupyter.org/). A Jupyter Notebook is a file that combines executable programming code with visualizations and comments in markdown format, allowing for an intuitive documentation of the code. We use Jupyter Notebooks for combined coding and documentation. We use Python 3 as programming language. All Notebooks are stored on [GitHub](https://github.com/), a platform for software development, and are publicly available. More information on our IT-concept can be found [here](http://open-power-system-data.org/it). See also our [step-by-step manual](http://open-power-system-data.org/step-by-step) how to use the dataplatform.\n", 35 | "\n", 36 | "# 3. About this Data Package\n", 37 | "We provide data in different chunks, or [datapackages](http://frictionlessdata.io/data-packages/). The one you are looking at is on [national generation capacity](http://data.open-power-system-data.org/national_generation_capacity/), \n", 38 | "\n", 39 | "This notebook processes data on aggregated generation capacity for European countries. The data includes generation capacities for nuclear, fossil and renewable fuels on an aggregated and, if possible, disaggregated technology levels. The majority of data reflects the years 2013 and 2014.\n", 40 | "The final output of the notebook is a list of national generation capacity differentiated by input fuels and technologies, as well as by different data sources." 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "# 4. Data sources\n", 48 | "The raw data is based on an extensive research of available national and international statistics. Due to the diverse structure of data sources, a compiled and structured dataset is used as input for further processing. Generally, five cross-national data sources are considered, complemented by national data sources.\n", 49 | "\n", 50 | "The cross-national data sources are:\n", 51 | "- [EUROSTAT \"Annual data on electricity infrastructure\"](http://ec.europa.eu/eurostat/product?code=nrg_113a&mode=view)\n", 52 | "- [ENTSOE \"Historical data\" (until 2015)](https://www.entsoe.eu/db-query/miscellaneous/net-generating-capacity)\n", 53 | "- [ENTSOE \"Power Statistics\" (from 2016)](https://www.entsoe.eu/data/power-stats/)\n", 54 | "- [ENTSOE \"Scenario Outlook & Adequacy Forecasts\"](https://www.entsoe.eu/outlooks/maf/Pages/default.aspx)\n", 55 | "- [ENTSOE \"Transparency Platform\"](https://transparency.entsoe.eu/)\n", 56 | "\n", 57 | "The cross-national data source [UN Statistical Office](http://data.un.org/Data.aspx?d=EDATA&f=cmID%3AEC) is only used for countries which are not listed in EUROSTAT data, i.e. Switzerland and Norway.\n", 58 | "\n", 59 | "Beside this, the data is complemented by national sources, which includes national statistical offices, ministries, regulatory authorities, transmission system operators, as well as other associations. All data sources are listed subsequently and in the `datapackage.json` including their link. If data are more granularly reported than on an annual basis, the end of the year values are used.\n", 60 | "\n", 61 | "| Country | Data type | Source | Type of Institution | Description | Type of document | Coverage |\n", 62 | "| :--- | :--- | :---:|:---|:---|:---:|---:|\n", 63 | "| Multiple | Aggregated capacities | [ENTSOE](https://www.entsoe.eu/outlooks/maf/Pages/default.aspx) | Association of European Transmission Network Operators | System Outlook and Adequacy Forecasts | Report | 2011-2016|\n", 64 | "| Multiple | Aggregated capacities | [ENTSOE](https://www.entsoe.eu/db-query/miscellaneous/net-generating-capacity) | Association of European Transmission Network Operators | Net generating capacity | Statistics | 1994-2015 |\n", 65 | "| Multiple | Aggregated capacities | [ENTSOE](https://www.entsoe.eu/data/power-stats/net-gen-capacity/) | Association of European Transmission Network Operators | Net generating capacity | Statistics | 2015-2018 |\n", 66 | "| Multiple | Aggregated capacities | [ENTSOE](https://transparency.entsoe.eu/) | Association of European Transmission Network Operators | Net generating capacity | Statistics | 2015-2019 |\n", 67 | "| Multiple | Aggregated capacities | [EUROSTAT](http://ec.europa.eu/eurostat/product?code=nrg_113a&mode=view) | European Statistical Office | Country Datasheets 2016 by the European Commission | Statistics | 1990-2016 |\n", 68 | "| Multiple | Aggregated capacities | [UN Statistical Office](http://data.un.org/Data.aspx?d=EDATA&f=cmID%3AEC) | Statistical Office of the United Nations | Electricity, net installed capacity of electric power plants | Statistics | 1990-2013 |\n", 69 | "| AT | Aggregated capacities | [e-control](http://www.e-control.at/statistik/strom/bestandsstatistik) | Regulatory Authority | Engpassleistung nach Kraftwerkstypen | Statistics | 2013-2019 |\n", 70 | "| BE | List of generation units | [ELIA](http://www.elia.be/en/grid-data/power-generation/generating-facilities) | Transmission System Operator | Overview of Generating Facilities | Statistics | actual |\n", 71 | "| BG | Aggregated capacities | [TSO Bulgaria ](http://www.eso.bg/?did=380) | Transmission System Operator | Generation Capacities | Report | 2012, 2019 |\n", 72 | "| CH | Aggregated capacities | [BFE](https://www.bfe.admin.ch/bfe/de/home/versorgung/statistik-und-geodaten/energiestatistiken/elektrizitaetsstatistik.html) | Ministry | Schweizerische Elektrizitätsstatistik (Electricity statistics) | Report | 2012-2019 |\n", 73 | "| CZ | Aggregated capacities | [ERU](https://www.eru.cz/en/zpravy-o-provozu-elektrizacni-soustavy) | Regulatory Authority | Yearly Report on Operations | Report | 2013-2019 |\n", 74 | "| DE | Aggregated capacities | [BMWi](https://www.bmwi.de/Redaktion/DE/Artikel/Energie/energiedaten-gesamtausgabe.html) | Ministry | Energiedaten (Energy Data) | Statistics | 2013-2018 |\n", 75 | "| DK | Aggregated capacities | [DEA](https://ens.dk/en/our-services/statistics-data-key-figures-and-energy-maps/annual-and-monthly-statistics) | Energy Agency | Annual Energy Statistics | Statistics | 2013-2018 |\n", 76 | "| EE | Aggregated capacities | [Statistics Estonia](http://pub.stat.ee/px-web.2001/Dialog/varval.asp?ma=FE032&ti=CAPACITY+AND+PRODUCTION+OF+POWER+PLANTS&path=../I_Databas/Economy/07Energy/02Energy_consumption_and_production/01Annual_statistics/&lang=1) | National Statistical Office | Capacity and Production of Power Plants | Statistics | 2013-2018 |\n", 77 | "| ES | Aggregated capacities | [REE](https://www.ree.es/en/datos/publications/national-statistical-series) | Transmission System Operator | National Indicators | Statistics | 2013-2020 |\n", 78 | "| FR | Aggregated capacities | [RTE](https://www.services-rte.com/en/view-data-published-by-rte/production-installed-capacity.html) | Transmission System Operator | Installed capacity of units above 1 MW | Statistics | 2014-2020 |\n", 79 | "| FI | Aggregated capacities | [Statistics Finland](http://pxnet2.stat.fi/PXWeb/pxweb/en/StatFin_Passiivi/StatFin_Passiivi__ene__ehk/statfinpas_ehk_pxt_904_201500_en.px/) | National Statistical Office | Capacity of Electricity Generation | Statistics | 2012-2015 |\n", 80 | "| GB | Aggregated capacities | [Department of Energy & Climate Change UK](https://www.gov.uk/government/statistics/electricity-chapter-5-digest-of-united-kingdom-energy-statistics-dukes) | Ministry | Plant Capacity | Statistics | 2015-2019 |\n", 81 | "| GR | Aggregated capacities | [REA](http://www.rae.gr/site/el_GR/categories_new/about_rae/actions/reports_national.csp) | Regulatory Authority | National Report | Report | 2012-2018 |\n", 82 | "| HR | Aggregated capacities | [HOPS](https://www.hops.hr/en/basic-data) | Transmission System Operator | Basic Data | Report | 2013-2018 |\n", 83 | "| HU | List of generation units | [Mavir](https://www.mavir.hu/hu/web/mavir-en/power-plants-installed-capacity-data) | Transmission System Operator | Planned Installed Generation Capacity | Report | 2013-2015 |\n", 84 | "| IE | List of generation units | [Eirgrid](http://www.eirgridgroup.com/library/index.xml) | Transmission System Operator | All-Island Generation Capacity Statement | Report | 2014-2015, 2019-2020 |\n", 85 | "| IT | Installed capacity | [Terna](https://www.terna.it/en/electric-system/transparency-report/installed-capacity) | Transmission System Operator | Installed Generation Capacity | Statistics | 2014-2017 |\n", 86 | "| IT | The statistical data on electricity in Italy | [Terna](https://www.terna.it/en/electric-system/statistical-data-forecast/statistical-publications) | Transmission System Operator | Installed Generation Capacity | Report | 2018-2019 |\n", 87 | "| LT | List of generation units | [Litgrid](http://www.litgrid.eu/index.php/power-system/power-system-information/generation-capacity/546) | Transmission System Operator | Generation Capacities | Statistics | actual |\n", 88 | "| LU | Aggregated capacities | [ILR](https://web.ilr.lu/FR/Professionnels/Electricite/Commun/Statistiques) | Regulatory Authority | Centrales de productions au Luxembourg | Statistics | 2013-2014 |\n", 89 | "| LV | Aggregated capacities | [Central Statistical Bureau of Latvia](http://data1.csb.gov.lv/pxweb/en/vide/vide__energetika__ikgad/?tablelist=true) | National Statistical Office | Electrical Capacity | Statistics | 2013-2019 |\n", 90 | "| NL | Aggregated capacities | [Tennet](https://www.tennet.org/english/operational_management/export_data.aspx) | Transmission System Operator | Installed Generation Capacity | Statistics | 2013-2018 |\n", 91 | "| NO | Aggregated capacities | [Statistics Norway](https://www.ssb.no/en/energi-og-industri/statistikker/elektrisitet/aar) | National Statistical Office | Electricity | Statistics | 2013-2018 |\n", 92 | "| PL | Aggregated capacities | [CIRE](http://www.rynek-energii-elektrycznej.cire.pl/st,33,207,tr,75,0,0,0,0,0,podstawowe-dane.html) | Energy Market Information Center | Moc zainstalowana (Installed Capacity) | Statistics | 2014, 2018 |\n", 93 | "| PT | Aggregated capacities | [REN](https://www.ren.pt/en-GB/media/publicacoes) | Transmission System Operator | Technical Data | Report | 2013-2018 |\n", 94 | "| RO | Aggregated capacities | [ANRE](https://www.anre.ro/en/about-anre/annual-reports-archive) | Regulatory Authority | Report | Report | 2013-2018 |\n", 95 | "| SE | Aggregated capacities | [Svensk Energi](https://www.energiforetagen.se/statistik/energiaret/energiaret-2018/) | Energy Agency | Installed Capacity | Report | 2013-2015 |\n", 96 | "| SE | Aggregated capacities | [Swedish Energy Agency](https://www.energimyndigheten.se/en/facts-and-figures/publications/) | Energy Agency | Energy in Sweden | Report | 2016-2018 |\n", 97 | "| SI | Aggregated capacities | [Agencija za energijo](http://www.agen-rs.si/web/en/publications/-/asset_publisher/TCdo6cWN9Shk/content/te-1?_101_INSTANCE_TCdo6cWN9Shk_redirect=%2Fweb%2Fen%2Fpublications) | Regulatory Authority | Report on the Energy Sector in Slovenia | Report | 2011-2017 |\n", 98 | "| SK | Aggregated capacities | [Statistical Office of Slovakia](https://slovak.statistics.sk/wps/portal/ext/themes/multi/energy/publications) | National Statistical Office | Energy Statistics | Report | 2013-2018 |" 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": {}, 104 | "source": [ 105 | "# 5. Notation\n", 106 | "As we compile different source for national generation capacity, we define a consolidated classifiation scheme based on energy sources and technologies. The classification scheme is related to four different energy source levels, to reflect the structural relations among the categories:\n", 107 | "\n", 108 | "- **Energy source level 0** - Total generation capacity\n", 109 | "- **Energy source level 1** - Generation capacity by energy type (fossil, nuclear, renewable, other)\n", 110 | "- **Energy source level 2** - Generation capacity by fuel or energy source (e.g. coal, lignite, hard coal, natural gas, wind)\n", 111 | "- **Energy source level 3** - Generation capacity by fuel or energy source and detailed fuel for bioenergy\n", 112 | "- **Technology level** - Generation capacity by fuel and technology (e.g. combined cycle natural gas, gas turbine, onshore wind, offshore wind)\n", 113 | "\n", 114 | "Additionally, the definition of the energy source levels ensures that the total capacity is identical for each energy source level.\n", 115 | "\n", 116 | "Moreover, some specific comments on particular categories:\n", 117 | "\n", 118 | "|\tCategory/Technology\t|\tNote\t|\n", 119 | "| :--- | :--- | \n", 120 | "|*Differently categorized …*|\tThese categories comprise either capacities which cannot be assigned to specific categories at the same level, or capacities with unspecified fuel or technology.| \n", 121 | "|*Reservoir including pumped storage*|\tAggregated capacity of reservoirs and pumped storage, or combinations of both. Relevant for AT.|\n", 122 | "|*Pumped storage with natural inflow*|\tPumped storage with natural inflow, or reservoirs with pumping capacity.|\n", 123 | "\n", 124 | "\n", 125 | "The information on the technology levels is rearranged the following table. It displays the four general technology levels and the corresponding categories:" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": null, 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [ 134 | "import os, pandas as pd;\n", 135 | "energylevels = pd.read_excel(os.path.join('input', 'National_Generation_Capacities.xlsx'),\n", 136 | " sheetname='Summary',\n", 137 | " header=0,\n", 138 | " na_values=['-'],\n", 139 | " parse_cols='A:F',\n", 140 | " index_col=0,\n", 141 | " skiprows=5)\n", 142 | "energylevels = energylevels[pd.notnull(energylevels.index)]\n", 143 | "energylevels = energylevels.replace({0: ' ', 1: True})\n", 144 | "energylevels.index.name = 'Technology'\n", 145 | "energylevels" 146 | ] 147 | }, 148 | { 149 | "cell_type": "markdown", 150 | "metadata": {}, 151 | "source": [ 152 | "# 6. License\n", 153 | "This notebook as well as all other documents in this repository is published under the [MIT License](LICENSE.md)." 154 | ] 155 | } 156 | ], 157 | "metadata": { 158 | "@webio": { 159 | "lastCommId": null, 160 | "lastKernelId": null 161 | }, 162 | "anaconda-cloud": {}, 163 | "kernelspec": { 164 | "display_name": "Python 3", 165 | "language": "python", 166 | "name": "python3" 167 | }, 168 | "language_info": { 169 | "codemirror_mode": { 170 | "name": "ipython", 171 | "version": 3 172 | }, 173 | "file_extension": ".py", 174 | "mimetype": "text/x-python", 175 | "name": "python", 176 | "nbconvert_exporter": "python", 177 | "pygments_lexer": "ipython3", 178 | "version": "3.8.3" 179 | } 180 | }, 181 | "nbformat": 4, 182 | "nbformat_minor": 1 183 | } 184 | -------------------------------------------------------------------------------- /requirements.yaml: -------------------------------------------------------------------------------- 1 | name: national_generation_capacity 2 | 3 | channels: 4 | - conda-forge 5 | - anaconda 6 | 7 | dependencies: 8 | - python=3.7 9 | - notebook 10 | - pandas 11 | - numpy 12 | - openpyxl 13 | - xlrd 14 | - pyyaml 15 | - nb_conda_kernels 16 | - bokeh=1.4 17 | 18 | 19 | 20 | --------------------------------------------------------------------------------