├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── hbvpy ├── ThirdParty │ ├── __init__.py │ └── core.py ├── __init__.py └── core │ ├── __init__.py │ ├── config.py │ ├── data.py │ ├── model.py │ └── process.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, Marc Girons Lopez 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HBVpy 2 | Python functions to interact with the command line version of HBV-light. 3 | 4 | This package provides: 5 | 6 | - Bindings to run HBV-light from python scripts. 7 | - Functions to pre-process the necessary input data for HBV-light for Swiss catchments. 8 | - An easy way to generate HBV-light configuration files. 9 | - Functions to load and process results files from HBV-light. 10 | 11 | --- 12 | 13 | ## HBV-light 14 | 15 | HBV-light is a version of the HBV semi-distributed rainfall-runoff model developed and maintained by the Hydrology and Climate group of the University of Zurich. You can find more details on the project's [website](http://www.geo.uzh.ch/en/units/h2k/Services/HBV-Model.html). 16 | 17 | --- 18 | 19 | ## How to... 20 | 21 | ### ... Install the package 22 | 23 | Download the master branch of the [package](https://github.com/GironsLopez/hbvpy/archive/master.zip) to a suitable location of your hard drive an unzip the file. Open a command prompt (if you are using Anaconda Python you should use the Anaconda prompt instead), navigate to the package main folder and run the following command: 24 | 25 | ``` 26 | pip install -e . 27 | ``` 28 | 29 | The package should be installed as well as it's dependencies. Currently, HBVpy depends on the following packages: 30 | 31 | ``` 32 | 'lxml' 33 | 'netCDF4' 34 | 'numpy' 35 | 'gdal' 36 | 'pandas' 37 | 'pyproj' 38 | 'scipy' 39 | ``` 40 | 41 | It appears that there is a bug when installing `pyproj` using this method so I would recommend to ensure that the packages are already installed in your Python distribution when attempting to install HBVpy. If you are using Anaconda Python you can run the following command: 42 | 43 | ``` 44 | conda install lxml netCDF4 numpy gdal pandas pyproj scipy 45 | ``` 46 | 47 | ### ... Run HBV-light with default settings 48 | 49 | If you want to run a simple HBV-light simulation for a single catchment, with a default folder structure, and all the necessary configuration and data files in place (see the help function of HBV-light), you can use the following code: 50 | 51 | ```python 52 | from hbvpy import HBVsimulation, HBVcatchment 53 | 54 | # Create an instance of the HBVsimulation class with the default arguments. 55 | # HBVsimulation provides the location of the HBV-light executable, as well as 56 | # the names of the input data and configuration files to use in the simulation 57 | # and the output folder. 58 | simulation = HBVsimulation() 59 | 60 | # Set the path of the catchment directory 61 | catchment_dir = 'path\\to\\catchment\\directory' 62 | 63 | # Create an instance of the HBVcatchment class by providing the path to the 64 | # catchment folder and the simulation instance. HBVcatchment locates all the 65 | # files indicated by HBVsimulation for the given catchment and allows to run 66 | # HBV-light. 67 | catchment = HBVcatchment(catchment_dir, simulation) 68 | 69 | # Perform a single run of the model. 70 | catchment.run('SingleRun') 71 | ``` 72 | 73 | If you want to inspect the screen-dump messages from HBV-light when running the model (e.g. to track possible errors), you can use the flag `debug_mode=True` in the `catchment.run()` method. 74 | 75 | Besides performing a single model run (i.e. `SingleRun`) it is also possible to perfom Monte Carlo, Batch and GAP runs by using `MonteCarloRun`, `BatchRun`, and `GAPRun` respectively. 76 | 77 | ### ... Performing parallel runs 78 | 79 | If you need to run HBV-light for e.g. multiple catchments using the same simulation setup you can use the following code: 80 | 81 | ```python 82 | import multiprocessing as mp 83 | from hbvpy import HBVsimulation, HBVcatchment 84 | 85 | 86 | def main(catchment_dir): 87 | """Function containing all the necessary code to run the model. 88 | """ 89 | # Create an HBVsimulation instance 90 | simulation = HBVsimulation() 91 | 92 | # Create an HBVcatchment instance providing the path to the catchment folder 93 | # and the simulation instance. 94 | catchment = HBVcatchment(catchment_dir, simulation) 95 | 96 | # Run the model 97 | catchment.run('SingleRun') 98 | 99 | 100 | # Create a list of all the catchment directories (just as a simple example; 101 | # it can be more elegant than this!) 102 | catchment_dir_list = [catchment_dir_1, catchment_dir_2, catchment_dir_3] 103 | 104 | # Define the number of cores (threads) to use for the model simulations 105 | cores_n = 3 106 | 107 | # Set up the parallel simulations 108 | p = mp.Pool(processes=cores_n) 109 | p.starmap(main, iterable=catchment_dir_list) 110 | ``` 111 | 112 | For more complex cases such as when calibrating multiple catchments using different parameter values (but also different objective functions, input data...), we can use the `itertools` package. This will allow us to easily produce all combinations of e.g. catchments and parameter values to feed into the `iterable` option. Building on the previous example: 113 | 114 | ```python 115 | import itertools 116 | import multiprocessing as mp 117 | from hbvpy import HBVsimulation, HBVcatchment 118 | 119 | 120 | def main(catchment_dir, parameter_file): 121 | """Function containing all the necessary code to run the model. 122 | """ 123 | simulation = HBVsimulation(p=parameter_file) 124 | 125 | catchment = HBVcatchment(catchment_dir, simulation) 126 | 127 | catchment.run('SingleRun') 128 | 129 | 130 | catchment_dir_list = [catchment_dir_1, catchment_dir_2, catchment_dir_3] 131 | 132 | parameter_file_list = [parameter_file_1, parameter_file_2] 133 | 134 | combinations = list(itertools.product(catchment_dir_list, parameter_file_list)) 135 | 136 | cores_n = 3 137 | 138 | p = mp.Pool(processes=cores_n) 139 | p.starmap(main, iterable=combinations) 140 | ``` 141 | 142 | ### ... Performing other tasks 143 | 144 | HBVpy also allows to: 145 | * Generate HBV-light input data files for Switzerland from a range of data products by **MeteoSwiss**, **FOEN**, **swisstopo**, **SLF**, and **MODIS**. Check out the classes and functions in the `hbvpy.data` module for more details on the supported products and further documentation. 146 | 147 | * Create and/or modify HBV-light configuration files from a Python environment. Check out the `HBVconfig` class and it's associated methods for further documentation on specific configuration files. 148 | 149 | * Parse HBV-light output files into Python data structures. Check out the classes in the `hbvpy.process` module for further documentation. 150 | -------------------------------------------------------------------------------- /hbvpy/ThirdParty/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | HBVpy.ThirdParty 5 | ================ 6 | 7 | Third party classes and functions needed for HBVpy. 8 | 9 | """ 10 | 11 | from . import core 12 | from .core import * 13 | 14 | __all__ = ['core'] 15 | __all__.extend(core.__all__) 16 | -------------------------------------------------------------------------------- /hbvpy/ThirdParty/core.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | HBVpy.ThirdParty : a package to include third party classes and functions. 5 | 6 | .. author:: Marc Girons Lopez 7 | 8 | """ 9 | 10 | import sys 11 | import numpy as np 12 | 13 | 14 | __all__ = ['AnimatedProgressBar', 'ncdump', 'ProgressBar', ] 15 | 16 | 17 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 18 | # Name ncdump 19 | # Author Chris Slocum (Colorado State University) 20 | # URL http://schubert.atmos.colostate.edu/~cslocum/netcdf_example.html 21 | # License CC BY-NC-ND 3.0 22 | # ----------------------------------------------------------------------------- 23 | 24 | 25 | def ncdump(nc_fid, verb=True): 26 | ''' 27 | ncdump outputs dimensions, variables and their attribute information. 28 | The information is similar to that of NCAR's ncdump utility. 29 | ncdump requires a valid instance of Dataset. 30 | 31 | Parameters 32 | ---------- 33 | nc_fid : netCDF4.Dataset 34 | A netCDF4 dateset object 35 | verb : Boolean 36 | whether or not nc_attrs, nc_dims, and nc_vars are printed 37 | 38 | Returns 39 | ------- 40 | nc_attrs : list 41 | A Python list of the NetCDF file global attributes 42 | nc_dims : list 43 | A Python list of the NetCDF file dimensions 44 | nc_vars : list 45 | A Python list of the NetCDF file variables 46 | ''' 47 | def print_ncattr(key): 48 | """ 49 | Prints the NetCDF file attributes for a given key 50 | 51 | Parameters 52 | ---------- 53 | key : unicode 54 | a valid netCDF4.Dataset.variables key 55 | """ 56 | try: 57 | print("\t\ttype:", repr(nc_fid.variables[key].dtype)) 58 | for ncattr in nc_fid.variables[key].ncattrs(): 59 | print('\t\t%s:' % ncattr, 60 | repr(nc_fid.variables[key].getncattr(ncattr))) 61 | except KeyError: 62 | print("\t\tWARNING: %s does not contain variable attributes" % key) 63 | 64 | # NetCDF global attributes 65 | nc_attrs = nc_fid.ncattrs() 66 | if verb: 67 | print("NetCDF Global Attributes:") 68 | for nc_attr in nc_attrs: 69 | print('\t%s:' % nc_attr, repr(nc_fid.getncattr(nc_attr))) 70 | nc_dims = [dim for dim in nc_fid.dimensions] # list of nc dimensions 71 | # Dimension shape information. 72 | if verb: 73 | print("NetCDF dimension information:") 74 | for dim in nc_dims: 75 | print("\tName:", dim) 76 | print("\t\tsize:", len(nc_fid.dimensions[dim])) 77 | print_ncattr(dim) 78 | # Variable information. 79 | nc_vars = [var for var in nc_fid.variables] # list of nc variables 80 | if verb: 81 | print("NetCDF variable information:") 82 | for var in nc_vars: 83 | if var not in nc_dims: 84 | print('\tName:', var) 85 | print("\t\tdimensions:", nc_fid.variables[var].dimensions) 86 | print("\t\tsize:", nc_fid.variables[var].size) 87 | print_ncattr(var) 88 | return nc_attrs, nc_dims, nc_vars 89 | 90 | 91 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 92 | # Name : ProgressBar 93 | # Author : Anler Hp, 2013 94 | # URL : https://github.com/anler/progressbar 95 | # License : MIT License 96 | # ----------------------------------------------------------------------------- 97 | 98 | 99 | class ProgressBar(object): 100 | """ProgressBar class holds the options of the progress bar. 101 | The options are: 102 | start State from which start the progress. For example, if start is 103 | 5 and the end is 10, the progress of this state is 50% 104 | end State in which the progress has terminated. 105 | width -- 106 | fill String to use for "filled" used to represent the progress 107 | blank String to use for "filled" used to represent remaining space. 108 | format Format 109 | incremental 110 | """ 111 | def __init__(self, start=0, end=10, width=12, fill='=', blank='.', 112 | format='[%(fill)s>%(blank)s] %(progress)s%%', 113 | incremental=True): 114 | super(ProgressBar, self).__init__() 115 | 116 | self.start = start 117 | self.end = end 118 | self.width = width 119 | self.fill = fill 120 | self.blank = blank 121 | self.format = format 122 | self.incremental = incremental 123 | self.step = 100 / float(width) # fix 124 | self.reset() 125 | 126 | def __add__(self, increment): 127 | increment = self._get_progress(increment) 128 | if 100 > self.progress + increment: 129 | self.progress += increment 130 | else: 131 | self.progress = 100 132 | return self 133 | 134 | def __str__(self): 135 | progressed = int(self.progress / self.step) # fix 136 | fill = progressed * self.fill 137 | blank = (self.width - progressed) * self.blank 138 | return self.format % {'fill': fill, 'blank': blank, 139 | 'progress': int(self.progress)} 140 | 141 | __repr__ = __str__ 142 | 143 | def _get_progress(self, increment): 144 | return float(increment * 100) / self.end 145 | 146 | def reset(self): 147 | """Resets the current progress to the start point""" 148 | self.progress = self._get_progress(self.start) 149 | return self 150 | 151 | 152 | class AnimatedProgressBar(ProgressBar): 153 | """Extends ProgressBar to allow you to use it straighforward on a script. 154 | 155 | Accepts an extra keyword argument named `stdout` 156 | (by default use sys.stdout) and may be any file-object to which send 157 | the progress status. 158 | """ 159 | def __init__(self, *args, **kwargs): 160 | super(AnimatedProgressBar, self).__init__(*args, **kwargs) 161 | self.stdout = kwargs.get('stdout', sys.stdout) 162 | 163 | def show_progress(self): 164 | if hasattr(self.stdout, 'isatty') and self.stdout.isatty(): 165 | self.stdout.write('\r') 166 | else: 167 | self.stdout.write('\n') 168 | self.stdout.write(str(self)) 169 | self.stdout.flush() 170 | -------------------------------------------------------------------------------- /hbvpy/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | HBVpy 5 | ===== 6 | 7 | Provides : 8 | 1. Bindings to run HBV-light from python scripts. 9 | 2. Functions to pre-process input data to HBV-light. 10 | 2. An easy way to generate HBV-light configuration files. 11 | 3. Functions to load and process HBV-light result files. 12 | 13 | """ 14 | from . import core 15 | from .core import * 16 | from . import ThirdParty 17 | 18 | __all__ = [] 19 | __all__.extend(core.__all__) 20 | -------------------------------------------------------------------------------- /hbvpy/core/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from . import config 5 | from .config import * 6 | from . import data 7 | from .data import * 8 | from . import model 9 | from .model import * 10 | from . import process 11 | from .process import * 12 | 13 | __all__ = ['config', 'data', 'model', 'process'] 14 | __all__.extend(config.__all__) 15 | __all__.extend(data.__all__) 16 | __all__.extend(model.__all__) 17 | __all__.extend(process.__all__) 18 | -------------------------------------------------------------------------------- /hbvpy/core/config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | hbvpy.config 5 | ============ 6 | 7 | **A package to generate xml configuration files needed to run HBV-light.** 8 | 9 | This package is intended to provide functions and methods to generate 10 | the necessary configuration files to run HBV-light. More specifically, it 11 | allows to generate the following files: 12 | 13 | * SnowRoutineSettings.xml 14 | * Simulation.xml 15 | * Batch_Simulation.xml 16 | * Parameter.xml 17 | * GAP_Simulation.xml 18 | * MC_Simulation.xml 19 | * Clarea.xml 20 | 21 | .. author:: Marc Girons Lopez 22 | 23 | """ 24 | 25 | import os 26 | import pandas as pd 27 | from lxml import etree as ET 28 | from datetime import datetime as dt 29 | 30 | 31 | __all__ = ['HBVconfig'] 32 | 33 | 34 | class HBVconfig(object): 35 | """ 36 | Generate an HBV-light Catchment folder structure and configuration files. 37 | 38 | Attributes 39 | ---------- 40 | bsn_dir : str 41 | Basin directory. 42 | model_type : {'Standard', 'OnlySnowDistributed', 'DistributedSUZ', 43 | 'ThreeGWBoxes', ThreeGWBoxesDistributedSTZ', 44 | 'ThreeGWBoxesDistributedSTZandSUZ', 'ResponseDelayed' 45 | 'OneGWBox'}, optional 46 | Model type to use, default is 'Standard'. 47 | model_variant : {'Basic', 'Aspect', 'Glacier'}, optional 48 | Model variant to use, default is 'Basic'. 49 | old_suz : bool, optional 50 | Use UZL and K0 in SUZ-box and therefore two runoff components from 51 | the storage in the soil upper zone, default is True. 52 | pcalt_data : bool, optional 53 | Use observed precipitation lapse rate input data, default if False. 54 | tcalt_data : bool, optional 55 | Use observed temperature lapse rate input data, default is False. 56 | et_corr : bool, optional 57 | Use long-term monthly or daily air temperature average values to 58 | correct the potential evapotranspiration, default is False. 59 | 60 | """ 61 | XSI = 'http://www.w3.org/2001/XMLSchema-instance' 62 | XSD = 'http://www.w3.org/2001/XMLSchema' 63 | 64 | XSI_TYPE = '{%s}type' % XSI 65 | 66 | def __init__( 67 | self, bsn_dir, model_type='Standard', model_variant='Basic', 68 | old_suz=True, pcalt_data=False, tcalt_data=False, et_corr=False): 69 | 70 | # The configuration files are stored in the 'Data' subfolder. 71 | self.data_dir = bsn_dir + '\\Data\\' 72 | 73 | # Create the folder structure if it doesn't exist. 74 | if not os.path.exists(self.data_dir): 75 | os.makedirs(self.data_dir) 76 | 77 | # Model setup 78 | self.model_type = model_type 79 | self.model_variant = model_variant 80 | self.old_suz = old_suz 81 | 82 | # Additional input data 83 | self.pcalt_data = pcalt_data 84 | self.tcalt_data = tcalt_data 85 | self.et_corr = et_corr 86 | 87 | @staticmethod 88 | def _add_subelement(root, name, value=None, attribute=None): 89 | """ 90 | Add an xml sub-element and give it a value and/or attribute. 91 | 92 | The value of the element can either be None or any of the following 93 | types: str, int, float, bool. In all the cases the value is 94 | transformed to a str. 95 | 96 | Parameters 97 | ---------- 98 | root : lxml.etree.Element or lxml.etree.SubElement 99 | Element or sub-element to add a sub-element to. 100 | name : str 101 | Name of the sub-element. 102 | value : str or int or float or bool, optional 103 | Value of the sub-element, default is None. 104 | attribute : dict, optional 105 | Attributes of the sub-element, default is None. 106 | 107 | Returns 108 | ------- 109 | element : lxml.etree.SubElement 110 | xml sub-element with its value and/or attribute. 111 | 112 | Raises 113 | ------ 114 | ValueError 115 | If the value type is not supported. 116 | 117 | """ 118 | if attribute is not None: 119 | element = ET.SubElement(root, name, attrib=attribute) 120 | 121 | else: 122 | element = ET.SubElement(root, name) 123 | 124 | if value is not None: 125 | if isinstance(value, bool): 126 | element.text = str(value).lower() 127 | 128 | elif isinstance(value, int) or isinstance(value, float): 129 | element.text = str(value) 130 | 131 | elif isinstance(value, str): 132 | element.text = value 133 | 134 | else: 135 | raise ValueError('Error: type not supported') 136 | 137 | return element 138 | 139 | @staticmethod 140 | def _write(root, filename): 141 | """ 142 | Write an root element tree to an xml file. 143 | 144 | Parameters 145 | ---------- 146 | root : lxml.ElementTree 147 | Element tree with the xml information. 148 | filename : str 149 | Path and name of the xml file to save the element tree. 150 | 151 | """ 152 | tree = ET.ElementTree(root) 153 | tree.write(filename, pretty_print=True, 154 | xml_declaration=True, encoding='UTF-8') 155 | 156 | @staticmethod 157 | def _date_format(date): 158 | """ 159 | Set a given date to the correct HBV-light format. 160 | 161 | Parameters 162 | ---------- 163 | date : '%Y-%m-%d' 164 | Date. 165 | 166 | Returns 167 | ------- 168 | '%Y-%m-%dT%H:%M:%S' 169 | Date in the correct HBV-light format. 170 | 171 | """ 172 | return dt.strptime(date, '%Y-%m-%d').strftime('%Y-%m-%dT%H:%M:%S') 173 | 174 | def model_settings( 175 | self, filename='Simulation.xml', start_warm_up='1900-01-01', 176 | start_simulation='1900-01-01', end_simulation='1900-01-01', 177 | save_results=True, save_dist=True, compute_interval=False, 178 | intervals=None, compute_peak=True, compute_season=False, 179 | compute_weight=False, start_day=1, end_day=31, 180 | start_month='January', end_month='December', weights=None): 181 | """ 182 | Define the HBV-light model settings. 183 | 184 | Parameters 185 | ---------- 186 | filename : str, optional 187 | Path and file name of the model settings file name, 188 | default is 'Simulation.xml'. 189 | start_warm_up : '%Y-%m-%d', optional 190 | Start of the warming-up period, default is '1900-01-01'. 191 | start_simulation : '%Y-%m-%d', optional 192 | Start of the simulation period, default is '1900-01-01'. 193 | end_simulation : '%Y-%m-%d', optional 194 | End of the simulation period, default is '1900-01-01'. 195 | save_results : bool, optional 196 | Choose whether to save simulation results, default is True. 197 | save_dist : bool, optional 198 | Choose whether to save distributed simulation results, 199 | default is True. 200 | compute_interval : bool, optional 201 | Choose whether to compute efficiency based on intervals of n 202 | time steps, default is False. 203 | intervals : list of int, optional 204 | Time step intervals to calculate the efficiency for, 205 | default is None. 206 | compute_peak : bool, optional 207 | Choose whether to compute efficiency for peak flows, default is 208 | True. 209 | compute_season : bool, optional 210 | Choose whether to compute efficiency for the season between 211 | 'start_day''start_month' and 'end_day''end_month', 212 | default is False. 213 | compute_weight : bool, optional 214 | Choose whether to compute efficiency based on 'weights', 215 | default is False. 216 | start_day : int, optional 217 | Start day of the season, default is 1. 218 | end_day : int, optional 219 | End day of the season, default is 31. 220 | start_month : str, optional 221 | Start month of the season, default is 'January'. 222 | end_month : str, optional 223 | End month of the season, default is 'December'. 224 | weights : dict, optional 225 | Dictionary of {Q: weight}, default is None. 226 | 227 | Raises 228 | ------ 229 | ValueError 230 | If compute_interval is True but no Reff Intervals are specified. 231 | ValueError 232 | If compute_weight is True but no weights are specified. 233 | 234 | """ 235 | # Generate the XML configuration file 236 | # --------------------------------------------------------------------- 237 | root = ET.Element( 238 | 'Simulation', nsmap={'xsi': self.XSI, 'xsd': self.XSD}) 239 | 240 | # Model type and additional input data 241 | # --------------------------------------------------------------------- 242 | self._add_subelement(root, 'ModelType', self.model_type) 243 | self._add_subelement(root, 'ModelVariant', self.model_variant) 244 | self._add_subelement(root, 'UseOldSUZ', self.old_suz) 245 | 246 | # Simulation settings 247 | # --------------------------------------------------------------------- 248 | self._add_subelement(root, 'StartOfWarmingUpPeriod', 249 | self._date_format(start_warm_up)) 250 | self._add_subelement(root, 'StartOfSimulationPeriod', 251 | self._date_format(start_simulation)) 252 | self._add_subelement(root, 'EndOfSimulationPeriod', 253 | self._date_format(end_simulation)) 254 | self._add_subelement(root, 'SaveResults', save_results) 255 | self._add_subelement(root, 'SaveDistributedSimulations', save_dist) 256 | self._add_subelement(root, 'ComputeReffInterval', compute_interval) 257 | self._add_subelement(root, 'ComputeReffPeak', compute_peak) 258 | self._add_subelement(root, 'ComputeReffSeason', compute_season) 259 | self._add_subelement(root, 'ComputeReffWeightedQ', compute_weight) 260 | self._add_subelement(root, 'SeasonalReffStartDay', start_day) 261 | self._add_subelement(root, 'SeasonalReffEndDay', end_day) 262 | self._add_subelement(root, 'SeasonalReffStartMonth', start_month) 263 | self._add_subelement(root, 'SeasonalReffEndMonth', end_month) 264 | ri = self._add_subelement(root, 'ReffInterval') 265 | if compute_interval is True: 266 | if intervals is None: 267 | raise ValueError('Error: No Reff Intervals specified.') 268 | for value in intervals: 269 | # reff_intervals is a list of integers 270 | i = self._add_subelement(ri, 'ReffInterval') 271 | self._add_subelement(i, 'TimeStep', value) 272 | self._add_subelement(root, 'PlotPeriod', 365) 273 | rw = self._add_subelement(root, 'ReffWeights') 274 | if compute_weight is True: 275 | if weights is None: 276 | raise ValueError('Error: No weights specified.') 277 | for q, weight in weights.items(): 278 | r = self._add_subelement(rw, 'ReffWeight') 279 | self._add_subelement(r, 'Q', q) 280 | self._add_subelement(r, 'Weight', weight) 281 | 282 | # Write out the xml file 283 | # --------------------------------------------------------------------- 284 | self._write(root, self.data_dir + filename) 285 | 286 | def batch_settings( 287 | self, filename='Batch_Simulation.xml', save_qsim_rows=False, 288 | save_qsim_cols=True, save_qsim_comp=True, save_qsim_comp_gw=False, 289 | header_line=True, save_peaks=False, save_freq_dist=True, 290 | save_gw_stats=False, create_param_files=False, dynamic_id=False, 291 | window_width=0, n_classes=0, use_precip_series=False, 292 | use_temp_series=False, use_evap_series=False, run_all=False, 293 | seasons=None, gap_file_in=True): 294 | """ 295 | Generate an XML comfiguration file for the BatchRun function 296 | of HBV-light. 297 | 298 | Parameters 299 | ---------- 300 | filename : str, optional 301 | Path and file name of the BatchRun configuration file, 302 | default is 'Batch_Simulation.xml'. 303 | save_qsim_rows : bool, optional 304 | Save simulated runoff results in rows, default is False. 305 | save_qsim_cols : bool, optional 306 | Save simulated runoff results in columns, default is True. 307 | It also saves a summary of the simulated runoff results. 308 | save_qsim_comp : bool, optional 309 | Save runoff components (rain, snow, glacier), default is True. 310 | save_qsim_comp_gw : bool, optional 311 | Save runoff groundwater components, default is False. 312 | header_line : bool, optional 313 | Set a header line with column names in all files, default is True. 314 | save_peaks : bool, optional 315 | Save simulated runoff value, snow amount, and relative soil 316 | moisture for the peak data points, in addition to weather 317 | statistics, default is False. 318 | save_freq_dist : bool, optional 319 | Save simulation runoff percentiles, maximum and minimum seasonal 320 | runoff values and their associated dates, in addition to average 321 | monthly simulated runoff values, default is True. 322 | save_gw_stats : bool, optional 323 | Save the 10th, 50th, and 90th quantiles of the water content in 324 | each groundwater box, default is False. 325 | create_param_files : bool, optional 326 | Create a Parameter.xml file for each parameter set, 327 | default is False. 328 | dynamic_id : bool, optional 329 | Compute statistics for dynamic identification, default is False. 330 | window_width : int, optional 331 | Window width for the dynamic identification, default is 0. 332 | n_classes : int, optional 333 | Number of classes for the dynamic identification, default is 0. 334 | use_precip_series : bool, optional 335 | Check each parameter set against multiple precipitation series, 336 | default is False. 337 | use_temp_series : bool, optional 338 | Check each parameter set against multiple temperature series, 339 | default is False. 340 | use_evap_series : bool, optional 341 | Check each parameter set against multiple evaporation series, 342 | default is False. 343 | run_all : bool, optional 344 | Check each parameter set against all possible combinations of 345 | precipitation, temperature, and evaporation series, default is 346 | False. 347 | seasons : list of tuples 348 | List of the start of each desired season, default is None. 349 | e.g. [(1, January), (1, March), ...] 350 | gap_file_in : bool, optional 351 | Use a GAP Calibration file format to run the BatchRun, 352 | default is True. 353 | 354 | """ 355 | # Generate the XML configuration file 356 | # --------------------------------------------------------------------- 357 | root = ET.Element( 358 | 'BatchSimulation', nsmap={'xsi': self.XSI, 'xsd': self.XSD}) 359 | 360 | # Batch settings 361 | # --------------------------------------------------------------------- 362 | self._add_subelement(root, 'SaveQsim', save_qsim_rows) 363 | self._add_subelement(root, 'SaveQsimInColumns', save_qsim_cols) 364 | self._add_subelement(root, 'SaveQsimSourceComponents', save_qsim_comp) 365 | self._add_subelement(root, 'SaveQsimFluxComponents', save_qsim_comp_gw) 366 | self._add_subelement(root, 'SaveHeaderLine', header_line) 367 | self._add_subelement(root, 'SavePeaks', save_peaks) 368 | self._add_subelement(root, 'SaveFreqDist', save_freq_dist) 369 | self._add_subelement(root, 'SaveGroundwaterStatistics', save_gw_stats) 370 | self._add_subelement(root, 'CreateParameterFiles', create_param_files) 371 | self._add_subelement(root, 'DynamicIdentification', dynamic_id) 372 | self._add_subelement(root, 'WindowWidth', window_width) 373 | self._add_subelement(root, 'NoOfClasses', n_classes) 374 | self._add_subelement(root, 'UsePrecipitationSeries', use_precip_series) 375 | self._add_subelement(root, 'UseTemperatureSeries', use_temp_series) 376 | self._add_subelement(root, 'RunAllClimateSeriesCombinations', run_all) 377 | self._add_subelement(root, 'UseEvaporationSeries', use_evap_series) 378 | season = self._add_subelement(root, 'Seasons') 379 | if seasons is not None: 380 | for element in seasons: 381 | # seasons is a list of tuples (e.g. [(1, January), ...]) 382 | self._add_subelement(season, 'Day', int(element[0])) 383 | self._add_subelement(season, 'Month', element[1]) 384 | self._add_subelement(root, 'GapInputFile', gap_file_in) 385 | 386 | # Write out the xml file 387 | # --------------------------------------------------------------------- 388 | self._write(root, self.data_dir + filename) 389 | 390 | def parameter_settings( 391 | self, filename='Parameter.xml', cps=None, vgps=None): 392 | """ 393 | Define the HBV-light Parameter.xml settings file. 394 | 395 | Parameters 396 | ---------- 397 | filename : str, optional 398 | Path and file name of the Parameter settings file, 399 | default is 'Parameter.xml'. 400 | cps : dict, optional 401 | Dictionary containing catchment parameter names and values, 402 | default is None. 403 | vgps : dict, optional 404 | Dictionary containing vegetation zone parameter names and values, 405 | default is None. 406 | 407 | """ 408 | # Define the catchment and vegetation zone parameters 409 | # --------------------------------------------------------------------- 410 | # Catchment parameters 411 | if cps is None: 412 | cps = {'KSI': 0, 'KGmin': 0, 'RangeKG': 0, 'AG': 0, 'PERC': 1, 413 | 'Alpha': 0, 'UZL': 30, 'K0': 0.25, 'K1': 0.1, 'K2': 0.01, 414 | 'MAXBAS': 1, 'Cet': 0.1, 'PCALT': 10, 'TCALT': 0.6, 415 | 'Pelev': 0, 'Telev': 0, 'PART': 0.5, 'DELAY': 1} 416 | 417 | # Vegetation zone parameters 418 | if vgps is None: 419 | vgps = {'TT': 0, 'CFMAX': 3, 'SFCF': 1, 'CFR': 0.05, 'CWH': 0.1, 420 | 'CFGlacier': 0, 'CFSlope': 1, 'FC': 120, 'LP': 1, 421 | 'BETA': 2} 422 | 423 | # Generate the XML configuration file 424 | # --------------------------------------------------------------------- 425 | root = ET.Element( 426 | 'Catchment', nsmap={'xsi': self.XSI, 'xsd': self.XSD}) 427 | 428 | # Parameter settings 429 | # --------------------------------------------------------------------- 430 | cp = self._add_subelement(root, 'CatchmentParameters') 431 | for parameter, value in cps.items(): 432 | self._add_subelement(cp, parameter, value=value) 433 | vg = self._add_subelement(root, 'VegetationZone') 434 | vgp = self._add_subelement(vg, 'VegetationZoneParameters') 435 | for parameter, value in vgps.items(): 436 | self._add_subelement(vgp, parameter, value=value) 437 | sc = self._add_subelement(root, 'SubCatchment') 438 | scp = self._add_subelement(sc, 'SubCatchmentParameters') 439 | for parameter, value in cps.items(): 440 | self._add_subelement(scp, parameter, value=value) 441 | scvg = self._add_subelement(sc, 'SubCatchmentVegetationZone') 442 | scvgp = self._add_subelement( 443 | scvg, 'SubCatchmentVegetationZoneParameters') 444 | for parameter, value in vgps.items(): 445 | self._add_subelement(scvgp, parameter, value=value) 446 | 447 | # Write out the xml file 448 | # --------------------------------------------------------------------- 449 | self._write(root, self.data_dir + filename) 450 | 451 | @staticmethod 452 | def _set_param_units(param_name, time_step='day'): 453 | """ 454 | Set the appropriate unit for a given HBV-light parameter. 455 | 456 | Parameters 457 | ---------- 458 | param_name : str 459 | Name of the HBV-light parameter. 460 | time_step : {'day', 'hour'}, optional 461 | Time step of the HBV-light input data, default is 'day'. 462 | 463 | Returns 464 | ------- 465 | str 466 | Units of the given HBV-light parameter. 467 | 468 | Raises 469 | ------ 470 | ValueError 471 | If the provided time step is not recognised. 472 | ValueError 473 | If the provided parameter name is not recognised. 474 | 475 | """ 476 | # Set the appropriate format for the given time step. 477 | # --------------------------------------------------------------------- 478 | if time_step == 'day': 479 | ts = 'd' 480 | 481 | elif time_step == 'hour': 482 | ts = 'h' 483 | 484 | else: 485 | raise ValueError('Time step not recognised.') 486 | 487 | # Set the appropriate unit for the given HBV-light parameter. 488 | # --------------------------------------------------------------------- 489 | 490 | # Set the unicode for the degree sign 491 | degree_sign = u'\N{DEGREE SIGN}' 492 | 493 | if param_name in ['KSI', 'KGmin', 'RangeKG', 'K0', 'K1', 'K2']: 494 | return '1/' + ts 495 | 496 | elif param_name in ['Alpha', 'PART', 'SP', 'SFCF', 'CFR', 'CWH', 497 | 'CFGlacier', 'CFSlope', 'LP', 'BETA']: 498 | return '-' 499 | 500 | elif param_name in ['UZL', 'FC']: 501 | return 'mm' 502 | 503 | elif param_name == 'TT': 504 | return degree_sign + 'C' 505 | 506 | elif param_name == 'CFMAX': 507 | return 'mm/' + ts + ' ' + degree_sign + 'C' 508 | 509 | elif param_name in ['Pelev', 'Telev']: 510 | return 'm' 511 | 512 | elif param_name == 'PCALT': 513 | return '%/100m' 514 | 515 | elif param_name == 'TCALT': 516 | return degree_sign + 'C/100m' 517 | 518 | elif param_name == 'Cet': 519 | return '1/' + degree_sign + 'C' 520 | 521 | elif param_name in ['MAXBAS', 'DELAY']: 522 | return ts 523 | 524 | elif param_name == 'AG': 525 | return '1/mm' 526 | 527 | elif param_name == 'PERC': 528 | return 'mm/' + ts 529 | 530 | else: 531 | raise ValueError('Parameter name not recognised.') 532 | 533 | @staticmethod 534 | def _set_param_disp_name(param_name): 535 | """ 536 | Return the display name of a given HBV-light parameter. 537 | 538 | Parameters 539 | ---------- 540 | param_name : str 541 | HBV-light parameter name. 542 | 543 | Returns 544 | ------- 545 | str 546 | HBV-light parameter display name. 547 | 548 | """ 549 | if param_name == 'Pelev': 550 | return 'Elev. of P' 551 | 552 | elif param_name == 'Telev': 553 | return 'Elev. of T' 554 | 555 | else: 556 | return param_name 557 | 558 | @staticmethod 559 | def objective_function_name(obj_function): 560 | """ 561 | Set the HBV-light name for a given objective function. 562 | 563 | Parameters 564 | ---------- 565 | obj_fun : str 566 | Name of the objective function used in the analysis. 567 | 568 | Returns 569 | ------- 570 | str 571 | Name of the objective function according to HBV-light. 572 | 573 | """ 574 | if obj_function not in [ 575 | 'Reff', 'ReffWeighted', 'LogReff', 'R2', 'MeanDiff', 576 | 'VolumeError', 'LindstromMeasure', 'MAREMeasure', 577 | 'FlowWeightedReff', 'ReffInterval(i)', 'ReffSeason', 578 | 'ReffPeak', 'SpearmanRank', 'ReffQObsSample', 579 | 'SnowCover_RMSE', 'GW_Rspear', 'Glacier_MAE', 580 | 'SWE_Reff', 'SWE_MANE', 'F_time', 'F_flow']: 581 | return 'PythonScript' 582 | 583 | else: 584 | return obj_function 585 | 586 | def gap_settings( 587 | self, catchment_params, veg_zone_params, 588 | filename='GAP_Simulation.xml', time_step='day', 589 | runs=5000, powell_runs=1000, parameter_sets=50, populations=2, 590 | exchange_freq=0, exchange_count=0, prob_optimisation=0.01, 591 | prob_mutation=0.02, prob_optimised=0, prob_random=0.16, 592 | prob_one=0.82, small_change=0, c_value=2, multiple_times=False, 593 | calibrations=100, obj_functions={'Reff': 1}): 594 | """ 595 | Generate an XML configuration file for the GAP Calibration function 596 | of HBV-light. 597 | 598 | # TODO: So far it only works for one sub-catchment and one vegetation 599 | zone. 600 | 601 | Parameters 602 | ---------- 603 | catchment_params : dict 604 | Dictionary containing the upper and lower limits for each 605 | catchment parameter, e.g. 606 | catchment_params={'K0': {'LowerLimit': 0.1, 'UpperLimit': 0.5}}. 607 | veg_zone_params : dict 608 | Dictionary containing the upper and lower limits for each 609 | vegetation zone parameter, e.g. 610 | veg_zone_params={'TT': {'LowerLimit': -0.5, 'UpperLimit': 0.5}}. 611 | filename : str, optional 612 | Path and file name of the GAP calibration configuration file, 613 | default is 'GAP_Simulation.xml'. 614 | time_step : {'day', 'hour'}, optional 615 | Time step of the simulation, default is 'day'. 616 | runs : int, optional 617 | Number of model runs, default is 5000. 618 | powell_runs : int, optional 619 | Number of runos for local optimisation, default is 1000. 620 | paramter_sets : int, optional 621 | Number of parameter sets per population, default is 50. 622 | populations : int, optional 623 | Number of populations, default is 2. 624 | exchange_freq : int, optional 625 | Frequency of parameter set exchange between different populations, 626 | default is 0. 627 | exchnage_count : int, optional 628 | Number of parameter sets that exchange between two populations, 629 | default is 0. 630 | prob_optimisation : float, optional 631 | Probability for optimisation between two sets, default is 0.01. 632 | prob_mutation : float, optional 633 | Probability for a mutation, default is 0.02. 634 | prob_optimised : float, optional 635 | Probability for optimising a value, default is 0. 636 | prob_random : float, optional 637 | Probability for a random value between the old values, default 638 | is 0.16. 639 | prob_one : float, optional 640 | Probability for taking one of the old values, default is 0.82. 641 | small_change : float, optional 642 | Portion of range for small change, default is 0. 643 | c_value : int, optional 644 | Value of C, default is 2. 645 | multiple_times : bool, optional 646 | Select whether the calibration should be repeated multiple times, 647 | default is False. 648 | calibrations : int, optional 649 | Number of times to repeat the calibration, default is 100. 650 | obj_functions : dict, optional 651 | Dictionary containing the objective functions to use and their 652 | associated wheights. Default is {'Reff': 1}. 653 | free_params : {'Snow', 'Soil_moisture', 'Response', 654 | 'Routing, 'All'}, optional 655 | HBV-light routine to get the free parameters for calibration from, 656 | default is 'Snow'. 657 | input_data_params : bool, optional 658 | Use routine-independent parameters as free_parameters, 659 | default is False. 660 | 661 | """ 662 | # Generate the XML configuration file 663 | # --------------------------------------------------------------------- 664 | root = ET.Element( 665 | 'GAPSimulation', nsmap={'xsi': self.XSI, 'xsd': self.XSD}) 666 | 667 | # Model type and additional input data 668 | # --------------------------------------------------------------------- 669 | self._add_subelement(root, 'ModelType', self.model_type) 670 | self._add_subelement(root, 'ModelVariant', self.model_variant) 671 | self._add_subelement(root, 'UseOldSUZ', self.old_suz) 672 | self._add_subelement(root, 'ETCorrection', self.et_corr) 673 | self._add_subelement(root, 'PCALTFile', self.pcalt_data) 674 | self._add_subelement(root, 'TCALTFile', self.tcalt_data) 675 | 676 | # Catchment parameters 677 | # TODO: Parameter values for multiple subcatchments. 678 | # --------------------------------------------------------------------- 679 | c_params = self._add_subelement(root, 'GAPCatchmentParameters') 680 | for param in catchment_params: 681 | # Get parameter information 682 | unit = self._set_param_units(param, time_step='day') 683 | display_name = self._set_param_disp_name(param) 684 | value_low = catchment_params[param]['LowerLimit'] 685 | value_high = catchment_params[param]['UpperLimit'] 686 | # Append information to the XML tree 687 | cp = self._add_subelement(c_params, 'GAPCatchmentParameter') 688 | self._add_subelement(cp, 'Name', param) 689 | self._add_subelement(cp, 'DisplayName', display_name) 690 | self._add_subelement(cp, 'Unit', unit) 691 | self._add_subelement(cp, 'LowerLimit', value_low) 692 | self._add_subelement(cp, 'UpperLimit', value_high) 693 | self._add_subelement(cp, 'ValuesPerSubCatchment', False) 694 | 695 | # Vegetation zone parameters 696 | # TODO: Parameter values for multiple vegetation zones. 697 | # --------------------------------------------------------------------- 698 | vz_params = self._add_subelement(root, 'GAPVegetationZoneParameters') 699 | for param in veg_zone_params: 700 | # Get parameter information 701 | unit = self._set_param_units(param, time_step='day') 702 | display_name = self._set_param_disp_name(param) 703 | value_low = veg_zone_params[param]['LowerLimit'] 704 | value_high = veg_zone_params[param]['UpperLimit'] 705 | # Append information to the XML tree 706 | vzp = self._add_subelement(vz_params, 'GAPVegetationZoneParameter') 707 | self._add_subelement(vzp, 'Name', param) 708 | self._add_subelement(vzp, 'DisplayName', display_name) 709 | self._add_subelement(vzp, 'Unit', unit) 710 | self._add_subelement(vzp, 'LowerLimit', value_low) 711 | self._add_subelement(vzp, 'UpperLimit', value_high) 712 | self._add_subelement(vzp, 'ValuesPerSubCatchment', False) 713 | self._add_subelement(vzp, 'ValuesPerVegetationZone', False) 714 | self._add_subelement(vzp, 'RandomValuePerVegetationZone', False) 715 | 716 | # GAP run parameters 717 | # --------------------------------------------------------------------- 718 | self._add_subelement(root, 'NumberOfRuns', runs) 719 | self._add_subelement(root, 'NumberOfPowellRuns', powell_runs) 720 | self._add_subelement(root, 'NumberOfParameterSets', parameter_sets) 721 | self._add_subelement(root, 'NumberOfPopulations', populations) 722 | self._add_subelement(root, 'ExchangeFrequency', exchange_freq) 723 | self._add_subelement(root, 'ExchangeCount', exchange_count) 724 | self._add_subelement(root, 'ProbOptimizeBetweeSets', prob_optimisation) 725 | self._add_subelement(root, 'ProbMutation', prob_mutation) 726 | self._add_subelement(root, 'ProbOptimized', prob_optimised) 727 | self._add_subelement(root, 'ProbRandom', prob_random) 728 | self._add_subelement(root, 'ProbOne', prob_one) 729 | self._add_subelement(root, 'SmallChange', small_change) 730 | self._add_subelement(root, 'CValue', c_value) 731 | self._add_subelement(root, 'CalibrateMultipleTimes', multiple_times) 732 | self._add_subelement(root, 'NumberOfCalibrations', calibrations) 733 | pps = self._add_subelement(root, 'Populations') 734 | for p in range(populations): 735 | pp = self._add_subelement(pps, 'GAPPopulation') 736 | self._add_subelement(pp, 'Name', 'Population_' + str(p+1)) 737 | ofws = self._add_subelement(pp, 'ObjFunctionWeights') 738 | for name, weight in obj_functions.items(): 739 | ofw = self._add_subelement(ofws, 'ObjFunctionWeight') 740 | of = self._add_subelement(ofw, 'ObjFunction') 741 | self._add_subelement(of, 'Name', name) 742 | self._add_subelement(of, 'Index', -1) 743 | self._add_subelement(ofw, 'Weight', weight) 744 | # TODO: Set weights for different subcatchments. 745 | self._add_subelement(root, 'ObjFunctionUsage', 'UseOutlet') 746 | self._add_subelement(root, 'SubCatchmentWeights') 747 | 748 | # Write out the xml file 749 | # --------------------------------------------------------------------- 750 | self._write(root, self.data_dir + filename) 751 | 752 | def mc_settings( 753 | self, catchment_params, veg_zone_params, gap_folder=None, 754 | eff_measure=None, filename='MC_Simulation.xml', time_step='day', 755 | multi_periods=False, periods=None, runs=1000, save_runs='SaveAll', 756 | min_reff_val=0.6, gaussian=False, obj_functions=None): 757 | """ 758 | Generate an XML configuration file for the Monte Carlo Simulation 759 | function of HBV-light. 760 | 761 | Parameters 762 | ---------- 763 | catchment_params : dict 764 | Dictionary containing the upper and lower limits for each 765 | catchment parameter, e.g. 766 | catchment_params={'K0': {'LowerLimit': 0.1, 'UpperLimit': 0.5}}. 767 | veg_zone_params : dict 768 | Dictionary containing the upper and lower limits for each 769 | vegetation zone parameter, e.g. 770 | veg_zone_params={'TT': {'LowerLimit': -0.5, 'UpperLimit': 0.5}}. 771 | filename : str, optional 772 | Path and file name of the MonteCarlo configuration file, 773 | default is 'GAP_Simulation.xml'. 774 | time_step : {'day', 'hour'}, optional 775 | Time step of the simulation, default is 'day'. 776 | gap_folder : str, optional 777 | Name of the folder in which the GAP results are stored, 778 | relavant only for sensitivity analysis. Default is None. 779 | eff_measure : str or list, optional 780 | List of the efficiency measures to use to sort the calibrations, 781 | relevant for sensitivity analysis. Default is None. 782 | filename : str, optional 783 | Path and file name of the GAP calibration configuration file, 784 | default is 'MC_Simulation.xml'. 785 | multi_periods : bool, optional 786 | Divide the simulation period into multiple parts. The efficiency 787 | of the model run will be computed for each of the periods, 788 | default is False. 789 | periods : list, optional 790 | List of dates representing the start of each period. The dates 791 | should be provided with the the format '%Y-%m-%d'. Default is None. 792 | runs : int, optional 793 | Number of model runs that will be carried out during the Monte 794 | Carlo simulation, default is 1000. 795 | save_runs : {'SaveAll', 'SaveOnlyIf', 'Save100Best'}, optional 796 | Specify which model runs should be save, default is 'SaveAll'. 797 | min_reff_val : float, optional 798 | Select the model efficiency value above which model runs are being 799 | saved if the the save_runs parameter is set to 'SaveOnlyIf', 800 | default is 0.6. 801 | gaussian : bool, optional 802 | Selected whether random numbers should be generated from a 803 | Gaussian continuous probability distribution, default is False. 804 | obj_functions : dict, optional 805 | Dictionary containing the objective functions to use and their 806 | associated wheights, default is None. Example: {'Reff': 1} 807 | 808 | """ 809 | # Generate the XML configuration file 810 | # --------------------------------------------------------------------- 811 | root = ET.Element('MonteCarloSimulation', 812 | nsmap={'xsi': self.XSI, 'xsd': self.XSD}) 813 | 814 | # Model type and additional input data 815 | # --------------------------------------------------------------------- 816 | self._add_subelement(root, 'ModelType', self.model_type) 817 | self._add_subelement(root, 'ModelVariant', self.model_variant) 818 | self._add_subelement(root, 'UseOldSUZ', self.old_suz) 819 | self._add_subelement(root, 'ETCorrection', self.et_corr) 820 | self._add_subelement(root, 'PCALTFile', self.pcalt_data) 821 | self._add_subelement(root, 'TCALTFile', self.tcalt_data) 822 | 823 | # Catchment parameters 824 | # TODO: Parameter values for multiple subcatchments. 825 | # --------------------------------------------------------------------- 826 | c_params = self._add_subelement(root, 'MonteCarloCatchmentParameters') 827 | for param in catchment_params: 828 | # Get parameter information 829 | unit = self._set_param_units(param, time_step='day') 830 | display_name = self._set_param_disp_name(param) 831 | value_low = catchment_params[param]['LowerLimit'] 832 | value_high = catchment_params[param]['UpperLimit'] 833 | if gaussian is True: 834 | mean = catchment_params[param]['Mean'] 835 | sd = catchment_params[param]['SD'] 836 | # Append information to the XML tree 837 | cp = self._add_subelement(c_params, 'MonteCarloCatchmentParameter') 838 | self._add_subelement(cp, 'Name', param) 839 | self._add_subelement(cp, 'DisplayName', display_name) 840 | self._add_subelement(cp, 'Unit', unit) 841 | self._add_subelement(cp, 'LowerLimit', value_low) 842 | self._add_subelement(cp, 'UpperLimit', value_high) 843 | if gaussian is True: 844 | self._add_subelement(cp, 'Mean', mean) 845 | self._add_subelement(cp, 'SD', sd) 846 | self._add_subelement(cp, 'ValuesPerSubCatchment', False) 847 | self._add_subelement(cp, 'CatchmentOption', 'Random') 848 | 849 | # Vegetation zone parameters 850 | # TODO: Parameter values for multiple vegetation zones. 851 | # --------------------------------------------------------------------- 852 | vz_params = self._add_subelement( 853 | root, 'MonteCarloVegetationZoneParameters') 854 | for param in veg_zone_params: 855 | # Get parameter information 856 | unit = self._set_param_units(param, time_step='day') 857 | display_name = self._set_param_disp_name(param) 858 | value_low = veg_zone_params[param]['LowerLimit'] 859 | value_high = veg_zone_params[param]['UpperLimit'] 860 | if gaussian is True: 861 | mean = veg_zone_params[param]['Mean'] 862 | sd = veg_zone_params[param]['SD'] 863 | # Append information to the XML tree 864 | vzp = self._add_subelement( 865 | vz_params, 'MonteCarloVegetationZoneParameter') 866 | self._add_subelement(vzp, 'Name', param) 867 | self._add_subelement(vzp, 'DisplayName', display_name) 868 | self._add_subelement(vzp, 'Unit', unit) 869 | self._add_subelement(vzp, 'LowerLimit', value_low) 870 | self._add_subelement(vzp, 'UpperLimit', value_high) 871 | if gaussian is True: 872 | self._add_subelement(vzp, 'Mean', mean) 873 | self._add_subelement(vzp, 'SD', sd) 874 | self._add_subelement(vzp, 'ValuesPerSubCatchment', False) 875 | self._add_subelement(vzp, 'ValuesPerVegetationZone', False) 876 | self._add_subelement(vzp, 'CatchmentOption', 'Random') 877 | self._add_subelement(vzp, 'VegetationZoneOption', 'Random') 878 | 879 | # Monte Carlo parameters 880 | # --------------------------------------------------------------------- 881 | self._add_subelement(root, 'MultiPeriods', multi_periods) 882 | if multi_periods is False: 883 | p = self._add_subelement(root, 'Periods') 884 | elif multi_periods is True and periods is None: 885 | raise ValueError('Periods need to be specified.') 886 | else: 887 | for element in periods: 888 | self._add_subelement(p, 'Period', element) 889 | self._add_subelement(root, 'NumberOfRuns', runs) 890 | self._add_subelement(root, 'SaveRunsOption', save_runs) 891 | self._add_subelement(root, 'ObjFunctionUsage', 'UseOutlet') 892 | self._add_subelement(root, 'MinReffValue', min_reff_val) 893 | self._add_subelement(root, 'Gaussian', gaussian) 894 | ofws = self._add_subelement(root, 'ObjFunctionWeights') 895 | if save_runs in ['SaveOnlyIf', 'Save100Best']: 896 | for name, weight in obj_functions.items(): 897 | ofw = self._add_subelement(ofws, 'ObjFunctionWeight') 898 | of = self._add_subelement(ofw, 'ObjFunction') 899 | self._add_subelement(of, 'Name', name) 900 | self._add_subelement(of, 'Index', -1) 901 | self._add_subelement(ofw, 'Weight', weight) 902 | # TODO: Set weights for different subcatchments. 903 | self._add_subelement(root, 'SubCatchmentWeights') 904 | 905 | # Write out the xml file 906 | # --------------------------------------------------------------------- 907 | self._write(root, self.data_dir + filename) 908 | 909 | @classmethod 910 | def _add_evu_element(cls, root, model_variant='Basic'): 911 | """ 912 | Parameters 913 | ---------- 914 | root : lxml.etree.Element or lxml.etree.SubElement 915 | Element or sub-element to add a sub-element to. 916 | model_variant : {'Basic', 'Aspect', 'Glacier'}, optional 917 | Name of the model variant, default is 'Basic'. 918 | 919 | Returns 920 | ------- 921 | XML root subelement containing an 'EVU' type. 922 | 923 | """ 924 | return cls._add_subelement( 925 | root, 'EVU', attribute={cls.XSI_TYPE: model_variant + '_EVU'}) 926 | 927 | @staticmethod 928 | def _get_area_value(df, idx, scc, vzc, apc, igc=None): 929 | """ 930 | Parameters 931 | ---------- 932 | df : Pandas.DataFrame 933 | DataFrame containing the area fractions for each elevation zone, 934 | sub-catchment, and vegetation zone 935 | idx : int 936 | Index of the elevation zone 937 | scc : int 938 | Sub-catchment count value 939 | vzc : int 940 | Vegetation zone count value 941 | apc : str 942 | Aspect count 943 | igc : str 944 | Glacier count 945 | 946 | Returns 947 | ------- 948 | Pandas.DataFrame cell value 949 | Value corresponding to the given sub-catchment, vegetation zone, 950 | elevation, zone, aspect, and glacier. 951 | 952 | """ 953 | if igc is None: 954 | return df['Area_' + scc + '_' + vzc + '_' + apc].iloc[idx] 955 | 956 | else: 957 | return df['Area_' + scc + '_' + vzc + 958 | '_' + igc + '_' + apc].iloc[idx] 959 | 960 | @classmethod 961 | def _set_aspect_value(cls, root, aspect, value): 962 | """ 963 | Parameters 964 | ---------- 965 | root : lxml.etree.Element or lxml.etree.SubElement 966 | xml element to add the aspect value to. 967 | aspect : {'n', 's', 'ew'} 968 | Aspect to set the area for. 969 | value : float 970 | Area value for the given aspect. 971 | 972 | Raises 973 | ------ 974 | ValueError 975 | If an unknown aspect is provided. 976 | 977 | """ 978 | if aspect == 'n': 979 | cls._add_subelement(root, 'NorthArea', value) 980 | 981 | elif aspect == 's': 982 | cls._add_subelement(root, 'SouthArea', value) 983 | 984 | elif aspect == 'ew': 985 | cls._add_subelement(root, 'EastWestArea', value) 986 | 987 | else: 988 | raise ValueError('Error: Unknown aspect') 989 | 990 | def catchment_settings( 991 | self, data, filename='Clarea.xml', lakes=None, areas=None): 992 | """ 993 | Generate a clarea.xml file based on data structured as a Pandas 994 | DataFrame. 995 | 996 | # NOTE: Even if this method belongs with the HBV-light configuration 997 | settings (xml), it is a part of the input data for the model. It is 998 | defined as a class method so it can be used within the data module 999 | without any restrictions. 1000 | 1001 | Parameters 1002 | ---------- 1003 | data : Pandas.DataFrame 1004 | Where the elevations are in the index and each column 1005 | is a different SubCatchment/VegetationZone. 1006 | | Index || Area_1_1 | Area_1_2 | ... 1007 | | 1800 || 0.211 | 0.39 | ... 1008 | | ... 1009 | Where the first index indicates the SubCatchment and 1010 | the second index indicates the VegetationZone. 1011 | filename : str, optional 1012 | Path and filename of the output file, default is 'Clarea.xml'. 1013 | lakes : Dict, optional 1014 | lakes = {'Lake_1': {'Area': 0, 1015 | 'Elevation': 0, 1016 | 'TT': 0, 1017 | 'SFCF': 0}, 1018 | 'Lake_2': ...}, default is None. 1019 | areas : Dict, optional 1020 | areas = {'Area_1': 0, 'Area_2': 0, ...}, default is None. 1021 | 1022 | Raises 1023 | ------ 1024 | ValueError 1025 | If an invalid model variant is provided. 1026 | 1027 | """ 1028 | # Get the number of vegetation zones 1029 | # --------------------------------------------------------------------- 1030 | vg = [] 1031 | for value in data.columns: 1032 | vg.append(int(value[7])) 1033 | vegetation_zone_count = max(vg) 1034 | 1035 | # Get the number of subcatchments 1036 | # --------------------------------------------------------------------- 1037 | sc = [] 1038 | for value in data.columns: 1039 | sc.append(int(value[5])) 1040 | sub_catchment_count = max(sc) 1041 | 1042 | # Generate the XML configuration file 1043 | # --------------------------------------------------------------------- 1044 | root = ET.Element( 1045 | 'Catchment', nsmap={'xsi': self.XSI, 'xsd': self.XSD}) 1046 | 1047 | # Elevation zones 1048 | # --------------------------------------------------------------------- 1049 | self._add_subelement( 1050 | root, 'VegetationZoneCount', vegetation_zone_count) 1051 | elevz = self._add_subelement(root, 'ElevationZoneHeight') 1052 | for value in data.index: 1053 | self._add_subelement(elevz, 'double', int(value)) 1054 | 1055 | # Fraction areas 1056 | # --------------------------------------------------------------------- 1057 | for scc in range(1, sub_catchment_count+1): 1058 | scc = str(scc) 1059 | subc = self._add_subelement(root, 'SubCatchment') 1060 | for vzc in range(1, vegetation_zone_count+1): 1061 | vzc = str(vzc) 1062 | vegz = self._add_subelement(subc, 'VegetationZone') 1063 | if self.model_variant == 'Basic': 1064 | for value in data['Area_' + scc + '_' + vzc]: 1065 | evu = self._add_evu_element( 1066 | vegz, model_variant=self.model_variant) 1067 | self._add_subelement(evu, 'Area', value) 1068 | 1069 | elif self.model_variant == 'Aspect': 1070 | for idx in range(len(data.index)): 1071 | idx = str(idx) 1072 | evu = self._add_evu_element( 1073 | vegz, model_variant=self.model_variant) 1074 | for apc in ['n', 's', 'ew']: 1075 | value = self._get_area_value( 1076 | data, idx, scc, vzc, apc) 1077 | self._set_aspect_value(evu, apc, value) 1078 | 1079 | elif self.model_variant == 'Glacier': 1080 | if data['Area_' + scc + '_' + vzc + '_g_n']: 1081 | is_glacier = True 1082 | igc = 'g' # glacier 1083 | elif data['Area_' + scc + '_' + vzc + '_c_n']: 1084 | is_glacier = False 1085 | igc = 'c' # clear (no glacier) 1086 | else: 1087 | raise ValueError('Glacier definition not valid') 1088 | for idx in range(len(data.index)): 1089 | idx = str(idx) 1090 | evu = self._add_evu_element( 1091 | vegz, model_variant=self.model_variant) 1092 | self._add_subelement(evu, 'IsGlacier', is_glacier) 1093 | for apc in ['n', 's', 'ew']: 1094 | value = self._get_area_value( 1095 | data, idx, scc, vzc, apc, igc) 1096 | self._set_aspect_value(evu, apc, value) 1097 | 1098 | else: 1099 | raise ValueError('Invalid model variant provided') 1100 | 1101 | # Lake parameters 1102 | # ----------------------------------------------------------------- 1103 | n = str(scc) 1104 | lake = self._add_subelement(subc, 'Lake') 1105 | if lakes is None: 1106 | self._add_subelement(lake, 'Area', 0) 1107 | self._add_subelement(lake, 'Elevation', 0) 1108 | self._add_subelement(lake, 'TT', 0) 1109 | self._add_subelement(lake, 'SFCF', 0) 1110 | else: 1111 | self._add_subelement(lake, 'Area', lakes['Lake_' + n]['Area']) 1112 | self._add_subelement( 1113 | lake, 'Elevation', lakes['Lake_' + n]['Elevation']) 1114 | self._add_subelement(lake, 'TT', lakes['Lake_' + n]['TT']) 1115 | self._add_subelement(lake, 'SFCF', lakes['Lake_' + n]['SFCF']) 1116 | 1117 | # Absolute area 1118 | # ----------------------------------------------------------------- 1119 | if areas is None: 1120 | self._add_subelement(subc, 'AbsoluteArea', 0) 1121 | else: 1122 | self._add_subelement(subc, 'AbsoluteArea', areas['Area_' + n]) 1123 | 1124 | # Write out the xml file 1125 | # --------------------------------------------------------------------- 1126 | self._write(root, self.data_dir + filename) 1127 | 1128 | def load_catchment_settings(self, filename='Clarea.xml'): 1129 | """ 1130 | Load the elevation distribution data for the catchment. 1131 | 1132 | Parameters 1133 | ---------- 1134 | filename : str, optional 1135 | Name for the catchment settings file.'Clarea.xml' file, 1136 | default is 'Clarea.xml'. 1137 | 1138 | Returns 1139 | ------- 1140 | Pandas.Series 1141 | Data structure containing the mean elevation and area fraction 1142 | of each elevation band in which the catchment is partitioned. 1143 | 1144 | """ 1145 | # Parse the file contents 1146 | tree = ET.parse(self.data_dir + filename) 1147 | root = tree.getroot() 1148 | 1149 | # Initialise lists to store the average elevation and 1150 | # fraction area of each elevation band. 1151 | elevs = [] 1152 | areas = [] 1153 | 1154 | for child in root: 1155 | 1156 | if child.tag == 'ElevationZoneHeight': 1157 | for item in child: 1158 | elevs.append(int(item.text)) 1159 | 1160 | if child.tag == 'SubCatchment': 1161 | for item in child: 1162 | if item.tag == 'VegetationZone': 1163 | for area in item: 1164 | areas.append(float(area[0].text)) 1165 | 1166 | return pd.Series(data=areas, index=elevs, name='Area') 1167 | -------------------------------------------------------------------------------- /hbvpy/core/data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | hbvpy.data 5 | ========== 6 | 7 | **A package to pre-process the necessary data for HBV-light** 8 | 9 | This package is intended to provide functions and methods to pre-process 10 | all the types of input data necessary for running HBV-light in Swiss 11 | catchments. More specifically, it allows to process the following 12 | data types (products): 13 | 14 | * NetCDF : 15 | TabsD, TmaxD, TminD, RhiresD, RdisaggH (all from MeteoSwiss). 16 | * Raster : 17 | SWE (SLF), MOD10A1 (MODIS), DEM (swisstopo). 18 | * Shape : 19 | Basin shape (FOEN). 20 | * Other : 21 | runoff (FOEN) 22 | 23 | .. author:: Marc Girons Lopez 24 | 25 | """ 26 | 27 | import os 28 | import ssl 29 | import glob 30 | import shutil 31 | import numpy as np 32 | import pandas as pd 33 | import datetime as dt 34 | from io import BytesIO 35 | from pyproj import Proj 36 | from zipfile import ZipFile 37 | from netCDF4 import Dataset 38 | from osgeo import gdal, ogr, osr 39 | from urllib.request import urlopen 40 | from scipy.stats import linregress 41 | 42 | from . import HBVconfig 43 | from hbvpy.ThirdParty import ncdump 44 | 45 | 46 | gdal.UseExceptions() 47 | 48 | 49 | __all__ = [ 50 | 'BasinShape', 'Evap', 'DEM', 'HBVdata', 'MOD10A1', 'RdisaggH', 51 | 'RhiresD', 'Runoff', 'SWE', 'TabsD', 'TmaxD', 'TminD'] 52 | 53 | 54 | def remove_file(filename): 55 | """ 56 | Iteratively remove a file if a PermissionError is encountered. 57 | 58 | Parameters 59 | ---------- 60 | filename : str 61 | Path and file name of the file to be removed. 62 | 63 | """ 64 | for retry in range(100): 65 | try: 66 | os.remove(filename) 67 | break 68 | except PermissionError: 69 | continue 70 | 71 | 72 | class NetCDF(object): 73 | """ 74 | Methods to work with MeteoSwiss NetCDF data files. 75 | 76 | Attributes 77 | ---------- 78 | filename : str 79 | Path and file name of the NetCDF file object. 80 | 81 | """ 82 | # Variables 83 | LON = None # Name of the longitude variable 84 | LAT = None # Name of the latitude variable 85 | DATA = None # Name of the data variable 86 | 87 | # Projection 88 | PROJ4 = None # Proj4 format 89 | 90 | # Data resolution 91 | RES = None 92 | 93 | def __init__(self, filename): 94 | 95 | self.fn = filename 96 | 97 | # HACK: Correct MeteoSwiss datasets prior to 2014. 98 | self._check_time_var() 99 | 100 | def _check_time_var(self): 101 | """ 102 | Check the time variable name of a NetCDF dataset. 103 | 104 | In 2014 MeteoSwiss changed the time variable name from 'time' to 105 | 'REFERENCE_TS'. This function detects which variable name is used in 106 | the NetCDF object and sets it as default for the NetCDF instance. 107 | 108 | """ 109 | ds = Dataset(self.fn, 'r') 110 | 111 | if 'time' in ds.variables: 112 | self.TIME = 'time' 113 | 114 | elif 'REFERENCE_TS' in ds.variables: 115 | self.TIME = 'REFERENCE_TS' 116 | 117 | else: 118 | raise ValueError('The NetCDF file does not have a ' 119 | 'recognised time variable.') 120 | 121 | ds.close() 122 | 123 | def ncdump(self, verb=True): 124 | """ 125 | Retrieve the dimensions, variables, and attributes of a NetCDf dataset. 126 | 127 | This method calls the 'ncdump' function developed by Crhis Slocum 128 | from Colorado State University. 129 | 130 | Parameters 131 | ---------- 132 | verb : bool 133 | Select whether to print the NetCDF dimensions, variables, and 134 | attributes, default is True. 135 | 136 | Returns 137 | ------- 138 | nc_attrs : list 139 | List of the global attributes of the NetCDF file. 140 | nc_dims : list 141 | List of the dimensions of the NetCDF file. 142 | nc_vars : list 143 | List of the variables of the NetCDF file. 144 | 145 | """ 146 | ds = Dataset(self.fn, 'r') 147 | 148 | nc_attrs, nc_dims, nc_vars = ncdump(ds, verb=verb) 149 | 150 | ds.close() 151 | 152 | return nc_attrs, nc_dims, nc_vars 153 | 154 | def load(self): 155 | """ 156 | Load a NetCDF dataset. 157 | 158 | Returns 159 | ------- 160 | data : Numpy array 161 | Array containing the gridded NetCDF data. 162 | 163 | """ 164 | ds = Dataset(self.fn, 'r') 165 | 166 | data = ds.variables[self.DATA][:] 167 | 168 | # Set "_FillValue" (no data value) to NaN 169 | if '_FillValue' in ds.variables[self.DATA].ncattrs(): 170 | no_data = ds.variables[self.DATA].getncattr('_FillValue') 171 | data[data == no_data] = np.nan 172 | 173 | elif 'missing_value' in ds.variables[self.DATA].ncattrs(): 174 | no_data = ds.variables[self.DATA].getncattr('missing_value') 175 | data[data == no_data] = np.nan 176 | 177 | else: 178 | pass 179 | 180 | # Load the latitude variable to check for inverted array. 181 | lat = ds.variables[self.LAT][:] 182 | 183 | if lat[-1] > lat[0]: 184 | # HACK: Correct MeteoSwiss datasets prior to 2014. 185 | # Flip the latitude coordinate if the latitude variable is 186 | # inverted. This is the case for MeteoSwiss datasets before 2014. 187 | 188 | if isinstance(data, np.ma.MaskedArray): 189 | # HACK: Prevent FutureWarning 190 | data.unshare_mask() 191 | 192 | for i in range(len(ds.variables[self.TIME][:])): 193 | # Loop over the time variable and flip the latitude coordinate. 194 | data[i] = np.flipud(data[i]) 195 | 196 | ds.close() 197 | 198 | return data 199 | 200 | def geotransform(self): 201 | """ 202 | Get the geotransform information of the NetCDF dataset using the 203 | default format of the GDAL library. 204 | 205 | Format: (min(lon), res(lon), ??, max(lat), ??, -res(lat)) 206 | 207 | Returns 208 | ------- 209 | tuple 210 | Tuple containing the geotransform information. 211 | 212 | """ 213 | ds = Dataset(self.fn, 'r') 214 | 215 | # Get the relevant limits of the netCDF dataset 216 | # (smallest longitude and larges latitude). 217 | xmin = min(ds.variables[self.LON]) 218 | ymax = max(ds.variables[self.LAT]) 219 | 220 | ds.close() 221 | 222 | return (xmin, self.RES, 0.0, ymax, 0.0, np.negative(self.RES)) 223 | 224 | def meshgrid(self): 225 | """ 226 | Get the coordinate meshgrid of the NetCDF dataset. 227 | 228 | Returns 229 | ------- 230 | xx : Numpy array 231 | Array of longitude coordinates with the shape of the dataset. 232 | yy : Numpy array 233 | Array of latitude coordinates with the shape of the dataset. 234 | 235 | """ 236 | ds = Dataset(self.fn, 'r') 237 | 238 | lon = ds.variables[self.LON][:] 239 | lat = ds.variables[self.LAT][:] 240 | 241 | # HACK: Correct MeteoSwiss datasets prior to 2014. 242 | if lat[-1] > lat[0]: 243 | lat = lat[::-1] 244 | 245 | xx, yy = np.meshgrid(lon, lat) 246 | 247 | ds.close() 248 | 249 | return xx, yy 250 | 251 | def lonlat_meshgrid(self): 252 | """ 253 | Get the coordinate meshgrid of the NetCDF 254 | dataset in lon/lat (deg) units. 255 | 256 | Returns 257 | ------- 258 | lon : Numpy array 259 | Array of longitude coordinates with the shape of the dataset. 260 | lat : Numpy array 261 | Array of latitude coordinates with the shape of the dataset. 262 | 263 | """ 264 | xx, yy = self.meshgrid() 265 | 266 | ds = Dataset(self.nc_fn, 'r') 267 | 268 | if 'degrees' in ds.variables[self.LON].units: 269 | # Do nothing if the coordinates of the dataset 270 | # are already in degree units 271 | lon = xx 272 | lat = yy 273 | 274 | else: 275 | p = Proj(self.PROJ4) 276 | lon, lat = p(xx, yy, inverse=True) 277 | 278 | ds.close() 279 | 280 | return lon, lat 281 | 282 | def datenum_range(self): 283 | """ 284 | Get the range of date numbers of the NetCDF dataset. 285 | 286 | Returns 287 | ------- 288 | dates : Numpy array 289 | Array of datenums of the NetCDF file. 290 | 291 | """ 292 | ds = Dataset(self.fn, 'r') 293 | 294 | dates = ds.variables[self.TIME][:] 295 | 296 | ds.close() 297 | 298 | return dates 299 | 300 | def date(self, datenum): 301 | """ 302 | Obtain the date of a given date number of the NetCDF dataset. 303 | 304 | Parameters 305 | ---------- 306 | datenum : int 307 | Date number to convert to datetime. 308 | 309 | Returns 310 | ------- 311 | Datetime object 312 | Datetime object containing the corresponding date 313 | to the given datenum. 314 | 315 | Raises 316 | ------ 317 | ValueError 318 | If the time step is not recognised. 319 | 320 | """ 321 | ds = Dataset(self.fn, 'r') 322 | 323 | time_units = ds.variables[self.TIME].units 324 | 325 | ds.close() 326 | 327 | # HACK: Correct MeteoSwiss datasets prior to 2014. 328 | try: 329 | # After 2014 MeteoSwiss changed the format of the reference date 330 | # for daily gridded datasets, excluding sub-daily time units. 331 | orig = dt.datetime.strptime(time_units[-10:], '%Y-%m-%d') 332 | 333 | except ValueError: 334 | orig = dt.datetime.strptime(time_units[-19:], '%Y-%m-%d %H:%M:%S') 335 | 336 | if 'hours' in time_units: 337 | d = dt.timedelta(hours=datenum) 338 | 339 | elif 'days' in time_units: 340 | d = dt.timedelta(days=datenum) 341 | 342 | else: 343 | raise ValueError('Error: Time step not recognised!') 344 | 345 | return orig + d 346 | 347 | def mask(self, shp_fn, touch_all=False): 348 | """ 349 | Mask the NetCDF dataset using a shapefile. 350 | 351 | Parameters 352 | ---------- 353 | shp_fn : str 354 | Path and file name of the shapefile. 355 | touch_all : bool, optional 356 | May be set to True to set all pixels touched by the line or 357 | polygons, not just those whose center is within the polygon or 358 | that are selected by brezenhams line algorithm, default is False. 359 | 360 | Returns 361 | ------- 362 | masked_data : Numpy array 363 | Array containing the masked NetCDF data. 364 | 365 | """ 366 | # Load the NetCDF file and extract the necessary information 367 | data = self.load() 368 | ntimes, nrows, ncols = np.shape(data) 369 | src_gt = self.geotransform() 370 | proj = self.PROJ4 371 | 372 | # Mask eventual invalid values (e.g. NaN) in the data array. 373 | data = np.ma.masked_invalid(data) 374 | 375 | # Update the geotransform information so xmin and ymax correspond 376 | # to the edge of the cells (Gdal) instead of the mid-point (NetCDF). 377 | dst_gt = (src_gt[0] - src_gt[1] * 0.5, src_gt[1], src_gt[2], 378 | src_gt[3] - src_gt[5] * 0.5, src_gt[4], src_gt[5]) 379 | 380 | # Rasterise the shapefile 381 | shp = Shape(shp_fn) 382 | basin = shp.rasterise(nrows, ncols, dst_gt, proj, touch_all=touch_all) 383 | 384 | # Invert the values of the basin array so it can be used as a mask. 385 | basin_mask = np.logical_not(basin) 386 | 387 | # Mask the NetCDF dataset using the rasterised shapefile 388 | masked_data = np.empty_like(data) 389 | for d in range(ntimes): 390 | masked_data[d] = np.ma.array(data[d], mask=basin_mask) 391 | 392 | return masked_data 393 | 394 | def average( 395 | self, shp_fn=None, date_list=None, value_list=None, 396 | start=None, end=None, touch_all=False): 397 | """ 398 | Calculate the average value for each time step. 399 | 400 | Parameters 401 | ---------- 402 | shp_fn : str, optional 403 | Path and file name of the masking shapefile, default is None. 404 | date_list : list, optional 405 | List containing datetime objects, default is None. 406 | value_list : list, optional 407 | List containing average values, default is None. 408 | start : '%Y-%m-%d', optional 409 | Start date of the output dataset, default is None. 410 | end : '%Y-%m-%d', optional 411 | End date of the outpout dataset, default is None. 412 | touch_all : bool, optional 413 | May be set to True to set all pixels touched by the line or 414 | polygons, not just those whose center is within the polygon or 415 | that are selected by brezenhams line algorithm, default is False. 416 | 417 | Returns 418 | ------- 419 | date_list : list 420 | List containing datetime objects. 421 | value_list : list 422 | List containing average values. 423 | 424 | """ 425 | if date_list is None: 426 | date_list = [] 427 | 428 | if value_list is None: 429 | value_list = [] 430 | 431 | # Get the date number array of the file 432 | datenums = self.datenum_range() 433 | 434 | if shp_fn is None: 435 | # Load the data. 436 | data = self.load() 437 | 438 | else: 439 | # Load the data and mask it with the shapefile. 440 | data = self.mask(shp_fn, touch_all=touch_all) 441 | 442 | # Get the period in datetime format 443 | if start is not None: 444 | start_date = dt.datetime.strptime(start, '%Y-%m-%d') 445 | 446 | if end is not None: 447 | end_date = dt.datetime.strptime(end, '%Y-%m-%d') 448 | 449 | # Iterate over the dates and average the data 450 | for d, datenum in enumerate(datenums): 451 | # Get the date from the date number 452 | date = self.date(datenum) 453 | 454 | if start is not None: 455 | # Continue to the next date if the date is 456 | # before the start of the period. 457 | if date < start_date: 458 | continue 459 | 460 | if end is not None: 461 | # Break the loop if the date is after the end of the period. 462 | if date > end_date: 463 | break 464 | 465 | # Calculate the average value for the catchment 466 | if data[d].mask.all(): 467 | # No valid values in the array 468 | data_avg = np.nan 469 | else: 470 | data_avg = np.nanmean(data[d]) 471 | 472 | # Append values to the lists 473 | date_list.append(date) 474 | value_list.append(data_avg) 475 | 476 | return date_list, value_list 477 | 478 | def reproject(self, src_fn, src_proj, dst_fn): 479 | """ 480 | Reproject and resample a raster dataset to match the values of 481 | the NetCDF instance. 482 | 483 | Parameters 484 | ---------- 485 | src_fn : str 486 | Path and file name of the raster dataset to reproject. 487 | src_proj : str 488 | Projection string (Proj4) of the raster dataset. 489 | dst_fn : str 490 | Path and filename of the reprojected raster dataset. 491 | 492 | """ 493 | # GeoTransform and projecction information of the NetCDF dataset 494 | # used as a reference for the reprojection and resampling process. 495 | ndates, nrows, ncols = np.shape(self.load()) 496 | ref_gt = self.geotransform() 497 | ref_proj = self.PROJ4 498 | 499 | # Update the information of the dataset edges so xmin and ymax 500 | # correspond to the edge of the cells (Gdal) instead of 501 | # the mid-point (NetCDF). 502 | xmin = ref_gt[0] - ref_gt[1] * 0.5 503 | xmax = ref_gt[0] + ref_gt[1] * ncols - ref_gt[1] * 0.5 504 | ymin = ref_gt[3] + ref_gt[5] * nrows - ref_gt[5] * 0.5 505 | ymax = ref_gt[3] - ref_gt[5] * 0.5 506 | 507 | # Reproject and resample the source raster 508 | gdal.Warp(dst_fn, src_fn, srcSRS=src_proj, dstSRS=ref_proj, 509 | xRes=ref_gt[1], yRes=ref_gt[5], outputBoundsSRS=ref_proj, 510 | outputBounds=(xmin, ymin, xmax, ymax)) 511 | 512 | def mean_elev(self, dem_fn, shp_fn=None, touch_all=False): 513 | """ 514 | Calculate the mean elevation of (a fraction of) the dataset. 515 | 516 | Parameters 517 | ---------- 518 | dem_fn : str 519 | Path and file name of the DEM file to get the elevation data from. 520 | shp_fn : str, optional 521 | Path and file name of the masking shapefile, default is None. 522 | touch_all : bool, optional 523 | May be set to True to set all pixels touched by the line or 524 | polygons, not just those whose center is within the polygon or 525 | that are selected by brezenhams line algorithm, default is False. 526 | 527 | Returns 528 | ------- 529 | float 530 | Mean elevation of the dataset (m a.s.l.). 531 | 532 | """ 533 | # Resample and reproject the dem_fn to match the NetCDF data. 534 | dem_fn_tmp = os.getcwd() + '\\dem_tmp.tif' 535 | self.reproject(dem_fn, DEM.PROJ4, dem_fn_tmp) 536 | 537 | # Load the DEM file and set the correct projection 538 | dem_ds = DEM(dem_fn_tmp) 539 | dem_ds.PROJ4 = self.PROJ4 540 | 541 | if shp_fn is None: 542 | dem = dem_ds.load() 543 | 544 | else: 545 | dem = dem_ds.mask(shp_fn, touch_all=touch_all) 546 | 547 | # Remove the temporary DEM file 548 | remove_file(dem_fn_tmp) 549 | 550 | return np.nanmean(dem.compressed()) 551 | 552 | @staticmethod 553 | def data_gradient(data, d, dem): 554 | """ 555 | Get the gradient (slope) of data values with elevation. 556 | 557 | Parameters 558 | ---------- 559 | data : NetCDF 560 | 3D NETCDF array of data values with time. 561 | d : int 562 | NetCDF date index. 563 | dem : Numpy.array 564 | Numpy array of elevation values with the same size as "data". 565 | 566 | Returns 567 | ------- 568 | slp : float 569 | Data gradient with elevation. 570 | 571 | """ 572 | # Preallocate space 573 | elevs = [] 574 | vals = [] 575 | 576 | # Pair the cells of the data and dem arrays 577 | for i, elev in enumerate(dem.compressed()): 578 | elevs.append(elev) 579 | vals.append(data[d].compressed()[i]) 580 | 581 | # Get the linear regression of the data-elevation pairs 582 | slp, intr, r, p, err = linregress(elevs, vals) 583 | 584 | return slp 585 | 586 | @classmethod 587 | def lapse_rate_value(cls, data, d, dem, step=100, method='absolute'): 588 | """ 589 | Calculate the lapse rate of a given data array. 590 | 591 | Parameters 592 | ---------- 593 | data : NetCDF 594 | 3D NETCDF array of data values with time. 595 | d : int 596 | NetCDF date index. 597 | dem : Numpy.array 598 | Numpy array of elevation values with the same size as "data". 599 | step : float or int, optional 600 | Step to calculate the lapse rate for, default is 100. 601 | In the case of temperature lapse rate this would give deg C / 100m. 602 | method : {'absolute', 'relative'}, optional 603 | Choose whether to calculate the absolute or relative lapse rate, 604 | default is 'absolute'. 605 | 606 | Returns 607 | ------- 608 | lapse_rate : float 609 | Lapse rate value. 610 | 611 | Raises 612 | ------ 613 | ValueError 614 | If the method provided is not recognised. 615 | 616 | """ 617 | # Get the data gradient 618 | slp = cls.data_gradient(data, d, dem) 619 | 620 | # Calculate the lapse rate 621 | if method == 'absolute': 622 | return slp * step 623 | 624 | elif method == 'relative': 625 | # Calculate the data mean over the area 626 | avg = np.nanmean(data[d, :, :]) 627 | 628 | if avg == 0: 629 | return 0 630 | 631 | else: 632 | return (((avg + slp) / avg) - 1) * step 633 | 634 | else: 635 | raise ValueError('The provided method is not recognised.') 636 | 637 | def lapse_rate( 638 | self, dem_fn, shp_fn=None, date_list=None, value_list=None, 639 | step=100, method='absolute', start=None, end=None, 640 | touch_all=False): 641 | """ 642 | Calculate the lapse rate of a NetCDF variable with elevation. 643 | 644 | Parameters 645 | ---------- 646 | dem_fn : str 647 | Path and file name of the DEM file to get the elevation data from. 648 | shp_fn : str, optional 649 | Path and file name of the masking shapefile, default is None. 650 | date_list : list, optional 651 | List containing datetime objects, default is None. 652 | value_list : list, optional 653 | List containing lapse rate values, default is None. 654 | step : int or float, optional 655 | Elevation unit to calculate the lapse rate for, default is 100. 656 | In the case of temperature lapse rate this would give deg C / 100m. 657 | method : {'absolute', 'relative'}, optional 658 | Choose whether to calculate the absolute or relative lapse rate, 659 | default is 'absolute'. 660 | start : '%Y-%m-%d', optional 661 | Start date of the output dataset, default is None. 662 | end : '%Y-%m-%d', optional 663 | End date of the outpout dataset, default is None. 664 | touch_all : bool, optional 665 | May be set to True to set all pixels touched by the line or 666 | polygons, not just those whose center is within the polygon or 667 | that are selected by brezenhams line algorithm, default is False. 668 | 669 | Returns 670 | ------- 671 | date_list : list 672 | List containing datetime objects. 673 | value_list : list 674 | List containing lapse rate values. 675 | 676 | Raises 677 | ------ 678 | ValueError 679 | If the method provided is not recognised. 680 | 681 | """ 682 | if date_list is None: 683 | date_list = [] 684 | 685 | if value_list is None: 686 | value_list = [] 687 | 688 | # Reproject and resample the DEM file 689 | dem_fn_tmp = os.getcwd() + '\\dem_tmp.tif' 690 | self.reproject(dem_fn, DEM.PROJ4, dem_fn_tmp) 691 | 692 | # Load the DEM file and set the correct projection 693 | dem_ds = DEM(dem_fn_tmp) 694 | dem_ds.PROJ4 = self.PROJ4 695 | 696 | # Get the datenum range of the NetCDF instance 697 | datenums = self.datenum_range() 698 | 699 | if shp_fn is None: 700 | # Load the necessary data 701 | data = self.load() 702 | dem = dem_ds.load() 703 | 704 | else: 705 | # Load and mask the necessary data 706 | data = self.mask(shp_fn, touch_all=touch_all) 707 | dem = dem_ds.mask(shp_fn, touch_all=touch_all) 708 | 709 | # Get the period in datetime format 710 | if start is not None: 711 | start_date = dt.datetime.strptime(start, '%Y-%m-%d') 712 | 713 | if end is not None: 714 | end_date = dt.datetime.strptime(end, '%Y-%m-%d') 715 | 716 | # Extract the relevant cells to calculate the lapse rate 717 | for d, datenum in enumerate(datenums): 718 | # Get the date from the date number 719 | date = self.date(datenum) 720 | 721 | if start is not None: 722 | # Continue to the next date if the date is 723 | # before the start of the period. 724 | if date < start_date: 725 | continue 726 | 727 | if end is not None: 728 | # Break the loop if the date is after the end of the period. 729 | if date > end_date: 730 | break 731 | 732 | date_list.append(date) 733 | value_list.append(self.lapse_rate_value( 734 | data, d, dem, step=step, method=method)) 735 | 736 | # Remove the temporary DEM file 737 | remove_file(dem_fn_tmp) 738 | 739 | return date_list, value_list 740 | 741 | 742 | class Raster(object): 743 | """ 744 | Methods to work with raster datasets. 745 | 746 | Attributes 747 | ---------- 748 | filename : str 749 | Path and file name of the raster dataset. 750 | 751 | """ 752 | PROJ4 = None 753 | 754 | NO_DATA = [] 755 | 756 | def __init__(self, filename): 757 | 758 | self.fn = filename 759 | 760 | def load(self): 761 | """ 762 | Load a raster dataset. 763 | 764 | Returns 765 | ------- 766 | data : Numpy Array 767 | Array containing the raster data. 768 | 769 | """ 770 | ds = gdal.Open(self.fn, gdal.GA_ReadOnly) 771 | data = ds.ReadAsArray().astype(float) 772 | 773 | ds = None 774 | 775 | # Set the values specified as no-data as np.NaN 776 | for value in self.NO_DATA: 777 | data[data == value] = np.nan 778 | 779 | return data 780 | 781 | def geotransform(self): 782 | """ 783 | Get the geotransform information of the raster dataset. 784 | 785 | Returns 786 | ------- 787 | gt : tuple 788 | Tuple containing the geotransform information. 789 | 790 | """ 791 | ds = gdal.Open(self.fn, gdal.GA_ReadOnly) 792 | gt = ds.GetGeoTransform() 793 | 794 | ds = None 795 | 796 | return gt 797 | 798 | def meshgrid(self): 799 | """ 800 | Obtain the coordinate meshgrid of the raster dataset. 801 | 802 | Returns 803 | ------- 804 | xx : Numpy array 805 | Array of longitude coordinates with the shape of the dataset. 806 | yy : Numpy array 807 | Array of latitude coordinates with the shape of the dataset. 808 | 809 | """ 810 | ds = gdal.Open(self.fn, gdal.GA_ReadOnly) 811 | gt = ds.GetGeoTransform() 812 | 813 | # get the edge coordinates and add half the resolution 814 | # to go to center coordinates 815 | xmin = gt[0] 816 | xmax = gt[0] + (gt[1] * ds.RasterXSize) 817 | ymin = gt[3] + (gt[5] * ds.RasterYSize) 818 | ymax = gt[3] 819 | 820 | lons = np.linspace(xmin, xmax, ds.RasterXSize) 821 | lats = np.linspace(ymax, ymin, ds.RasterYSize) 822 | 823 | ds = None 824 | 825 | xx, yy = np.meshgrid(lons, lats) 826 | 827 | return xx, yy 828 | 829 | def lonlat_meshgrid(self): 830 | """ 831 | Get the coordinate meshgrid of the 832 | raster dataset in lon/lat (deg) units. 833 | 834 | Returns 835 | ------- 836 | lon : Numpy array 837 | Array of longitude coordinates with the shape of the dataset. 838 | lat : Numpy array 839 | Array of latitude coordinates with the shape of the dataset. 840 | 841 | """ 842 | p = Proj(self.PROJ4) 843 | 844 | xx, yy = self.meshgrid() 845 | 846 | lon, lat = p(xx, yy, inverse=True) 847 | 848 | return lon, lat 849 | 850 | def mask(self, shp_fn, touch_all=False): 851 | """ 852 | Mask the raster dataset using a shapefile. 853 | 854 | Parameters 855 | ---------- 856 | shp_fn : str 857 | Path and file name of the shapefile. 858 | touch_all : bool, optional 859 | May be set to True to set all pixels touched by the line or 860 | polygons, not just those whose center is within the polygon or 861 | that are selected by brezenhams line algorithm, default is False. 862 | 863 | Returns 864 | ------- 865 | masked_data : Numpy array 866 | Array containing the masked NetCDF data. 867 | 868 | """ 869 | # Load the NetCDF file and extract the necessary information 870 | data = self.load() 871 | nrows, ncols = np.shape(data) 872 | gt = self.geotransform() 873 | proj = self.PROJ4 874 | 875 | # Mask eventual invalid values (e.g. NaN) in the data array. 876 | data = np.ma.masked_invalid(data) 877 | 878 | # Rasterise the shapefile 879 | shp = Shape(shp_fn) 880 | array = shp.rasterise(nrows, ncols, gt, proj, touch_all=touch_all) 881 | 882 | # Invert the values of the basin array to use it as a mask. 883 | data_mask = np.logical_not(array) 884 | 885 | # Mask the raster dataset using the rasterised shapefile 886 | masked_data = np.empty_like(data) 887 | masked_data = np.ma.array(data, mask=data_mask) 888 | 889 | return masked_data 890 | 891 | def average(self, shp_fn=None, touch_all=False): 892 | """ 893 | Calculate the average value of the raster dataset. 894 | 895 | Parameters 896 | ---------- 897 | shp_fn : str, optional 898 | Pand and file name of the masking shapefile, default: None. 899 | touch_all : bool, optional 900 | May be set to True to set all pixels touched by the line or 901 | polygons, not just those whose center is within the polygon or 902 | that are selected by brezenhams line algorithm, default is False. 903 | 904 | Returns 905 | ------- 906 | float 907 | Average value of the raster dataset. 908 | 909 | """ 910 | if shp_fn is None: 911 | data = self.load() 912 | else: 913 | data = self.mask(shp_fn, touch_all=touch_all) 914 | 915 | if data.mask.all(): 916 | return np.nan 917 | else: 918 | return np.nanmean(data) 919 | 920 | def reproject(self, src_fn, src_proj, dst_fn): 921 | """ 922 | Reproject and resample a raster dataset to match the values of 923 | the raster instance. 924 | 925 | Parameters 926 | ---------- 927 | src_fn : str 928 | Path and file name of the raster dataset to reproject. 929 | src_proj : str 930 | Projection string (Proj4) of the raster dataset. 931 | dst_fn : str 932 | Path and filename of the reprojected raster dataset. 933 | 934 | """ 935 | # Load the reference raster GeoTransform and projection 936 | ref_ds = gdal.Open(self.fn, gdal.GA_ReadOnly) 937 | gt = ref_ds.GetGeoTransform() 938 | ref_proj = self.PROJ4 939 | 940 | # Get the edges of the reference dataset 941 | xmin = gt[0] 942 | xmax = gt[0] + gt[1] * ref_ds.RasterXSize 943 | ymin = gt[3] + gt[5] * ref_ds.RasterYSize 944 | ymax = gt[3] 945 | 946 | # Reproject and resample the source raster 947 | gdal.Warp(dst_fn, src_fn, srcSRS=src_proj, dstSRS=ref_proj, 948 | xRes=gt[1], yRes=gt[5], outputBoundsSRS=ref_proj, 949 | outputBounds=(xmin, ymin, xmax, ymax)) 950 | 951 | def lapse_rate( 952 | self, dem_fn, shp_fn=None, step=100, 953 | method='absolute', touch_all=False): 954 | """ 955 | Calculate the lapse rate of the raster dataset with elevation. 956 | 957 | Parameters 958 | ---------- 959 | dem_fn : str 960 | Path and file name of the DEM file to get the elevation data from. 961 | shape_fn : str, optional 962 | Path and file name of the masking shapefile, default: None. 963 | step : int or float, optional 964 | Elevation unit to calculate the lapse rate for, default: 100 965 | In the case of temperature lapse rate this would give deg C / 100m 966 | method : {'absolute', 'relative'}, optional 967 | Choose whether to calculate the absolute or relative lapse rate, 968 | default is 'absolute'. 969 | touch_all : bool, optional 970 | May be set to True to set all pixels touched by the line or 971 | polygons, not just those whose center is within the polygon or 972 | that are selected by brezenhams line algorithm, default is False. 973 | 974 | Returns 975 | ------- 976 | float 977 | Lapse rate value. 978 | 979 | Raises 980 | ------ 981 | ValueError 982 | If the method provided is not recognised. 983 | 984 | """ 985 | # Reproject and resample the DEM file 986 | dem_fn_tmp = os.getcwd() + '\\dem_tmp.tif' 987 | self.reproject(dem_fn, DEM.PROJ4, dem_fn_tmp) 988 | 989 | # Load the DEM file and set the correct projection 990 | dem_ds = DEM(dem_fn_tmp) 991 | dem_ds.PROJ4 = self.PROJ4 992 | 993 | if shp_fn is None: 994 | # Load the necessary data 995 | data = self.load() 996 | dem = dem_ds.load() 997 | 998 | else: 999 | # Load and mask the necessary data 1000 | data = self.mask(shp_fn, touch_all=touch_all) 1001 | dem = dem_ds.mask(shp_fn, touch_all=touch_all) 1002 | 1003 | # Extract the relevant cells to calculate the lapse rate 1004 | elevs = [] 1005 | vals = [] 1006 | for i, elev in enumerate(dem.compressed()): 1007 | elevs.append(elev) 1008 | vals.append(data.compressed()[i]) 1009 | 1010 | # The lapse rate is the slope of the linear regression 1011 | slp, intr, r, p, err = linregress(elevs, vals) 1012 | 1013 | # Remove the temporary DEM file 1014 | os.remove(dem_fn_tmp) 1015 | 1016 | if method == 'absolute': 1017 | return slp * step 1018 | 1019 | elif method == 'relative': 1020 | avg = np.nanmean(data) 1021 | if avg == 0: 1022 | return 0 1023 | else: 1024 | return (((avg + slp) / avg) - 1) * step 1025 | 1026 | else: 1027 | raise ValueError('The provided method is not recognised.') 1028 | 1029 | 1030 | class Shape(object): 1031 | """ 1032 | Methods to work with shapefiles. 1033 | 1034 | Attributes 1035 | ---------- 1036 | filename : str 1037 | Path and file name of the shapefile. 1038 | 1039 | """ 1040 | def __init__(self, filename): 1041 | 1042 | self.fn = filename 1043 | 1044 | def rasterise( 1045 | self, dst_nrows, dst_ncols, dst_gt, 1046 | dst_proj, dst_fn=None, touch_all=False): 1047 | """ 1048 | Convert a shapefile into a raster. 1049 | 1050 | If no output shapefile is specified the output is stored 1051 | in memory. 1052 | 1053 | Parameters 1054 | ---------- 1055 | dst_nrows : int 1056 | Number of rows of the output raster. 1057 | dst_ncols : int 1058 | Number of columns of the output raster. 1059 | dst_gt : tuple 1060 | Geotransformation informaiton of the output raster. 1061 | dst_proj : Proj4 1062 | Projection of the output raster (Proj4) 1063 | dst_fn : str, optional 1064 | Path and filename of the output raster file (e.g. *.tif), 1065 | default is None. 1066 | touch_all : bool, optional 1067 | May be set to True to set all pixels touched by the line or 1068 | polygons, not just those whose center is within the polygon or 1069 | that are selected by brezenhams line algorithm, default is False. 1070 | 1071 | Returns 1072 | ------- 1073 | array : Numpy array 1074 | Array containing the data of the output raster. 1075 | 1076 | """ 1077 | # Load the shapefile 1078 | driver = ogr.GetDriverByName('ESRI Shapefile') 1079 | src_ds = driver.Open(self.fn, gdal.GA_ReadOnly) 1080 | src_lyr = src_ds.GetLayer() 1081 | 1082 | # Initialise the rasterfile 1083 | if dst_fn is None: 1084 | driver = gdal.GetDriverByName('MEM') 1085 | dst_ds = driver.Create('', dst_ncols, dst_nrows, 1, gdal.GDT_Byte) 1086 | 1087 | else: 1088 | driver = gdal.GetDriverByName('GTiff') 1089 | dst_ds = driver.Create( 1090 | dst_fn, dst_ncols, dst_nrows, 1, gdal.GDT_Byte) 1091 | 1092 | # Set the attributes of the rasterfile 1093 | dst_rb = dst_ds.GetRasterBand(1) 1094 | dst_rb.Fill(0) 1095 | dst_rb.SetNoDataValue(0) 1096 | dst_ds.SetGeoTransform(dst_gt) 1097 | 1098 | # Set the reference system of the rasterfile 1099 | srs = osr.SpatialReference() 1100 | srs.ImportFromProj4(dst_proj) 1101 | dst_ds.SetProjection(srs.ExportToWkt()) 1102 | 1103 | # Rasterise the shapefile 1104 | if touch_all is True: 1105 | gdal.RasterizeLayer(dst_ds, [1], src_lyr, burn_values=[1], 1106 | options=['ALL_TOUCHED=TRUE']) 1107 | else: 1108 | gdal.RasterizeLayer(dst_ds, [1], src_lyr, burn_values=[1]) 1109 | 1110 | # Read the output as a Numpy array. 1111 | array = dst_rb.ReadAsArray() 1112 | 1113 | # Close the data source and target 1114 | del src_ds, dst_ds 1115 | 1116 | return array 1117 | 1118 | def lonlat(self): 1119 | """ 1120 | Get the latitude and longitude of the centre of mass 1121 | of the polgon shapefile. 1122 | 1123 | TODO: So far it only works for shapefiles with a single feature. 1124 | 1125 | Returns 1126 | ------- 1127 | lon : float 1128 | Longitude of the centre of mass of the polygon shapefile (deg). 1129 | lat : float 1130 | Latitude of the centre of mass of the polygon shapefile (deg). 1131 | 1132 | """ 1133 | # Load the shapefile 1134 | driver = ogr.GetDriverByName('ESRI Shapefile') 1135 | ds = driver.Open(self.fn, gdal.GA_ReadOnly) 1136 | lyr = ds.GetLayer() 1137 | 1138 | # Get the geometry 1139 | feature = lyr.GetNextFeature() 1140 | geometry = feature.GetGeometryRef() 1141 | 1142 | # Get the projection of the shapefile 1143 | src_ref = geometry.GetSpatialReference() 1144 | 1145 | # Get the output projection (WGS84) 1146 | # TODO: Provide an option to choose the output coordinate system. 1147 | # HACK: ImportFromEPSG is currently not working... 1148 | dst_ref = osr.SpatialReference() 1149 | epsg_4326 = '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs' 1150 | dst_ref.ImportFromProj4(epsg_4326) 1151 | 1152 | # Get the centre of mass of the polygon. 1153 | centroid = geometry.Centroid() 1154 | 1155 | # Transform the coordinate system of the centroid geometry 1156 | transform = osr.CoordinateTransformation(src_ref, dst_ref) 1157 | centroid.Transform(transform) 1158 | 1159 | # Get the latitude of the centroid 1160 | lon = centroid.GetX() 1161 | lat = centroid.GetY() 1162 | 1163 | # Close the dataset 1164 | del ds 1165 | 1166 | return lon, lat 1167 | 1168 | def area(self): 1169 | """ 1170 | Calculate the area of a polygon shapefile. 1171 | 1172 | # NOTE: So far it only works for shapefiles with a single feature. 1173 | 1174 | Returns 1175 | ------- 1176 | area : float 1177 | Area of the polygon shapefile (in the projection units). 1178 | 1179 | """ 1180 | # Load the shapefile 1181 | driver = ogr.GetDriverByName('ESRI Shapefile') 1182 | ds = driver.Open(self.fn, gdal.GA_ReadOnly) 1183 | lyr = ds.GetLayer() 1184 | 1185 | # Get the geometry 1186 | feature = lyr.GetNextFeature() 1187 | geometry = feature.GetGeometryRef() 1188 | 1189 | # Calculate the area of the basin 1190 | area = geometry.GetArea() 1191 | 1192 | del ds 1193 | 1194 | return area 1195 | 1196 | 1197 | class TabsD(NetCDF): 1198 | """ 1199 | Daily mean air temperature gridded dataset. 1200 | Author: MeteoSwiss 1201 | 1202 | Attributes 1203 | ---------- 1204 | filename : str 1205 | Path and filename of the TabsD NetCDF file. 1206 | 1207 | """ 1208 | LON = 'lon' 1209 | LAT = 'lat' 1210 | DATA = 'TabsD' 1211 | 1212 | PROJ4 = '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs' 1213 | 1214 | RES = 1.25 / 60 # Minutes to degrees 1215 | 1216 | def __init__(self, filename): 1217 | 1218 | super().__init__(filename) 1219 | 1220 | 1221 | class TmaxD(NetCDF): 1222 | """ 1223 | Daily maximum air temperature gridded dataset. 1224 | Author: MeteoSwiss 1225 | 1226 | Attributes 1227 | ---------- 1228 | filename : str 1229 | Path and filename of the TmaxD NetCDF file. 1230 | 1231 | """ 1232 | LON = 'lon' 1233 | LAT = 'lat' 1234 | DATA = 'TmaxD' 1235 | 1236 | PROJ4 = '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs' 1237 | 1238 | RES = 1.25 / 60 # Minutes to degrees 1239 | 1240 | def __init__(self, filename): 1241 | 1242 | super().__init__(filename) 1243 | 1244 | 1245 | class TminD(NetCDF): 1246 | """ 1247 | Daily minimum air temperature gridded dataset. 1248 | Author: MeteoSwiss 1249 | 1250 | Attributes 1251 | ---------- 1252 | filename : str 1253 | Path and filename of the TminD NetCDF file. 1254 | 1255 | """ 1256 | LON = 'lon' 1257 | LAT = 'lat' 1258 | DATA = 'TminD' 1259 | 1260 | PROJ4 = '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs' 1261 | 1262 | RES = 1.25 / 60 # Minutes to degrees 1263 | 1264 | def __init__(self, filename): 1265 | 1266 | super().__init__(filename) 1267 | 1268 | 1269 | class RhiresD(NetCDF): 1270 | """ 1271 | Daily precipitation (final analysis) gridded dataset. 1272 | Author: MeteoSwiss 1273 | 1274 | Attributes 1275 | ---------- 1276 | filename : str 1277 | Path and filename of the RhiresD NetCDF file. 1278 | 1279 | """ 1280 | LON = 'lon' 1281 | LAT = 'lat' 1282 | DATA = 'RhiresD' 1283 | 1284 | PROJ4 = '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs' 1285 | 1286 | RES = 1.25 / 60 # Minutes to degrees 1287 | 1288 | def __init__(self, filename): 1289 | 1290 | super().__init__(filename) 1291 | 1292 | 1293 | class RdisaggH(NetCDF): 1294 | """ 1295 | Hourly precipitation (experimental) gridded dataset. 1296 | Author: MeteoSwiss 1297 | 1298 | Attributes 1299 | ---------- 1300 | filename : str 1301 | Path and filename of the RdisaggH NetCDF file. 1302 | 1303 | """ 1304 | LON = 'chx' 1305 | LAT = 'chy' 1306 | DATA = 'RdisaggH' 1307 | 1308 | PROJ4 = '+proj=somerc +lat_0=46.95240555555556 '\ 1309 | '+lon_0=7.439583333333333 +k_0=1 +x_0=600000 '\ 1310 | '+y_0=200000 +ellps=bessel +towgs84=674.374,'\ 1311 | '15.056,405.346,0,0,0,0 +units=m +no_defs' 1312 | 1313 | RES = 1000 # metres 1314 | 1315 | def __init__(self, filename): 1316 | 1317 | super().__init__(filename) 1318 | 1319 | 1320 | class SWE(Raster): 1321 | """ 1322 | Daily Snow Water Equivalent (SWE) gridded dataset for Switzerland. 1323 | Author: SLF 1324 | 1325 | Attributes 1326 | ---------- 1327 | filename : str 1328 | Path and filename of the SWE raster file. 1329 | 1330 | """ 1331 | PROJ4 = '+proj=somerc +lat_0=46.95240555555556 '\ 1332 | '+lon_0=7.439583333333333 +k_0=1 +x_0=600000 '\ 1333 | '+y_0=200000 +ellps=bessel +towgs84=674.374,'\ 1334 | '15.056,405.346,0,0,0,0 +units=m +no_defs' 1335 | 1336 | NO_DATA = [-9999.] 1337 | 1338 | def __init__(self, filename): 1339 | 1340 | super().__init__(filename) 1341 | 1342 | def date(self): 1343 | """ 1344 | Obtain the date of the SWE dataset. 1345 | 1346 | Returns 1347 | ------- 1348 | Datetime object 1349 | Datetime object containing the date of the SWE dataset. 1350 | 1351 | """ 1352 | date = dt.datetime.strptime(self.fn[-23:-13], '%Y-%m-%d') 1353 | 1354 | return date - dt.timedelta(days=1) 1355 | 1356 | def elev_dist(self, dem_fn, shp_fn=None, step=100, touch_all=False): 1357 | """ 1358 | Calculate the elevation distribution of snow water equivalent. 1359 | 1360 | Parameters 1361 | ---------- 1362 | dem_fn : str 1363 | Path and filename of the DEM file used to get the elevation data. 1364 | shp_fn : str, optional 1365 | Path and filename of the shapefile to use for masking the data, 1366 | default is None. 1367 | step : int or float, optional 1368 | Elevation band width to perform the calculations, default is 100. 1369 | touch_all : bool, optional 1370 | May be set to True to set all pixels touched by the line or 1371 | polygons, not just those whose center is within the polygon or 1372 | that are selected by brezenhams line algorithm, default is False. 1373 | 1374 | Returns 1375 | ------- 1376 | vals : list 1377 | List of average SWE values for each elevation band. 1378 | 1379 | """ 1380 | # Instatiate a DEM dataset object 1381 | dem_ds = DEM(dem_fn) 1382 | 1383 | if shp_fn is None: 1384 | swe = self.load() 1385 | dem = dem_ds.load() 1386 | 1387 | else: 1388 | swe = self.mask(shp_fn, touch_all=touch_all) 1389 | dem = dem_ds.mask(shp_fn, touch_all=touch_all) 1390 | 1391 | # Get the elevation distribution of the catchment. 1392 | hist, bin_edges = DEM.histogram(dem, width=step) 1393 | names = DEM.bin_names(bin_edges, header_type='range') 1394 | 1395 | # Preallocate space to store the average snow cover fraction values. 1396 | obs_swe = pd.DataFrame(columns=names, index=[self.date()]) 1397 | 1398 | # HACK: Get rid of NaN values in the DEM dataset in order to be able 1399 | # to extract the elevation bands later on. 1400 | dem.unshare_mask() 1401 | dem[np.isnan(dem)] = -9999 1402 | 1403 | # Loop over the elevation bands. 1404 | for i, n in enumerate(hist): 1405 | # return NaN if no cells in the elevation band 1406 | if n == 0: 1407 | obs_swe.loc[self.date(), names[i]] = np.nan 1408 | 1409 | else: 1410 | # Create and elevation mask to filter the cells outside 1411 | # of the elevation band of interest 1412 | elev_m = np.ones_like(dem) 1413 | elev_m[(dem >= bin_edges[i]) & (dem < bin_edges[i+1])] = 0 1414 | 1415 | # Mask the data and extract the mean value 1416 | swe_m = np.ma.masked_array(data=swe, mask=elev_m) 1417 | obs_swe.loc[self.date(), names[i]] = np.mean(swe_m) 1418 | 1419 | # Delete empty columns 1420 | obs_swe = obs_swe.dropna(axis=1, how='all') 1421 | 1422 | # return vals 1423 | return obs_swe 1424 | 1425 | 1426 | class MOD10A1(Raster): 1427 | """ 1428 | MODIS snow cover gridded dataset. 1429 | 1430 | Attributes 1431 | ---------- 1432 | filename : str 1433 | Path and filename of the MOD10A1 file. 1434 | 1435 | """ 1436 | PROJ4 = '+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 '\ 1437 | '+b=6371007.181 +units=m +no_defs' # SR-ORG:6842 1438 | 1439 | NO_DATA = [200., 201., 211., 237., 239., 250., 254., 255.] 1440 | # NO_DATA = [200., 201.] 1441 | 1442 | def __init__(self, filename): 1443 | 1444 | self.file = filename 1445 | 1446 | # NOTE: Only designed to return the 'NDSI_Snow_Cover' band. 1447 | self.fn = 'HDF4_EOS:EOS_GRID:"' + filename + \ 1448 | '":MOD_Grid_Snow_500m:NDSI_Snow_Cover' 1449 | 1450 | super().__init__(self.fn) 1451 | 1452 | def date(self): 1453 | """ 1454 | Parse the date of the snow cover dataset. 1455 | 1456 | Returns 1457 | ------- 1458 | Datetime object 1459 | Date of the snow cover dataset. 1460 | 1461 | """ 1462 | return dt.datetime.strptime(self.file[-36:-29], '%Y%j') 1463 | 1464 | 1465 | class DEM(Raster): 1466 | """ 1467 | Methods to work with Digital Elevation Models (DEMs) from swisstopo. 1468 | 1469 | Attributes 1470 | ---------- 1471 | filename : str 1472 | Path and filename of the DEM file. 1473 | 1474 | """ 1475 | PROJ4 = '+proj=somerc +lat_0=46.95240555555556 '\ 1476 | '+lon_0=7.439583333333333 +k_0=1 +x_0=600000 '\ 1477 | '+y_0=200000 +ellps=bessel +towgs84=674.374,'\ 1478 | '15.056,405.346,0,0,0,0 +units=m +no_defs' 1479 | 1480 | NO_DATA = [-9999.] 1481 | 1482 | def __init__(self, filename): 1483 | 1484 | super().__init__(filename) 1485 | 1486 | @staticmethod 1487 | def histogram(dem_array, width=100): 1488 | """ 1489 | Get the elevation histogram of the DEM dataset. 1490 | 1491 | Parameters 1492 | ---------- 1493 | dem_array : Numpy array 1494 | Array containing the DEM data. 1495 | width : int or float, optional 1496 | Width of the elevation bins to calculate the histogram for, 1497 | default is 100. 1498 | 1499 | Returns 1500 | ------- 1501 | hist : Numpy array 1502 | The values of the histogram. 1503 | bin_edges : Numpy array of type float 1504 | Return the bin edges (length(hist)+1). 1505 | 1506 | """ 1507 | # Remove NaN cells 1508 | dem = dem_array.compressed() 1509 | 1510 | # Set the bins with the given width and min/max array values 1511 | bin_vals = np.arange(np.floor(min(dem) / width) * width, 1512 | np.ceil(max(dem) / width) * width, width) 1513 | 1514 | # Calculate the histogram 1515 | hist, bin_edges = np.histogram(a=dem, density=False, bins=bin_vals) 1516 | 1517 | return hist, bin_edges 1518 | 1519 | @staticmethod 1520 | def bin_names(bin_edges, header_type='range'): 1521 | """ 1522 | Generate a list of bin names for a DEM histogram. 1523 | 1524 | Parameters 1525 | ---------- 1526 | bin_edges : array of dtype float 1527 | Array containing the bin edges (ouput from numpy.histogram). 1528 | header_type : {'range', 'mid-point'}, optional 1529 | Type of information to display in the bin names, default is 1530 | 'range'. 1531 | 1532 | Returns 1533 | ------- 1534 | names : list 1535 | List of bin names. 1536 | 1537 | """ 1538 | names = [] 1539 | 1540 | for i in range(len(bin_edges)-1): 1541 | if header_type == 'range': 1542 | names.append(str(int(bin_edges[i])) + '-' + 1543 | str(int(bin_edges[i+1]))) 1544 | 1545 | elif header_type == 'mid-point': 1546 | names.append(str(int((bin_edges[i] + bin_edges[i+1]) / 2))) 1547 | 1548 | else: 1549 | raise ValueError('Header type not recognised') 1550 | 1551 | return names 1552 | 1553 | def elev_area_dist(self, shp_fn=None, step=100, touch_all=False): 1554 | """ 1555 | Calculate the area distribution with elevation. 1556 | 1557 | # TODO: This method only supports a 'Basic' model type with 1558 | one vegetation zone and one sub-catchment. 1559 | 1560 | Parameters 1561 | ---------- 1562 | shp_fn : str, optional 1563 | Path and filename of the shapefile to use for masking the data, 1564 | default is None. 1565 | step : int or float, optional 1566 | Elevation band width to perform the calculations. default is 100. 1567 | touch_all : bool, optional 1568 | May be set to True to set all pixels touched by the line or 1569 | polygons, not just those whose center is within the polygon or 1570 | that are selected by brezenhams line algorithm, default is False. 1571 | 1572 | Returns 1573 | ------- 1574 | df : Pandas.DataFrame 1575 | DataFrame containing the area percentage data for each elevation 1576 | band. 1577 | 1578 | """ 1579 | if shp_fn is None: 1580 | # Load the DEM file 1581 | dem = self.load() 1582 | 1583 | else: 1584 | # Load and mask the DEM file 1585 | dem = self.mask(shp_fn, touch_all=touch_all) 1586 | 1587 | # Calculate the elevation histogram of the catchment 1588 | hist, bin_edges = self.histogram(dem, width=step) 1589 | 1590 | elevs = self.bin_names(bin_edges, header_type='mid-point') 1591 | areas = [] 1592 | 1593 | for band_cells in hist: 1594 | # Calculate the percentage of cells within the elevation band 1595 | areas.append(band_cells / sum(hist)) 1596 | 1597 | # Create a Pandas.DataFrame with the area percentages 1598 | df = pd.DataFrame(data=areas, index=elevs, columns=['Area_1_1']) 1599 | 1600 | # HACK: Save only elevation zones with non-zero area percentages 1601 | return df[df['Area_1_1'] != 0] 1602 | 1603 | 1604 | class Evap(object): 1605 | """ 1606 | Methods to generate potential evaporation and evapotranspiration data. 1607 | 1608 | """ 1609 | # Constants 1610 | SC = 0.0820 # Solar constant (MJ m-2 min-1), 1611 | L = 2.26 # Latent heat flux (MJ Kg-1) 1612 | RHO = 1000 # Density of water (Kg m-3) 1613 | 1614 | @classmethod 1615 | def extraterrestrial_radiation(cls, lat, j): 1616 | """ 1617 | Calculate the potential extraterrestrial solar radiation. 1618 | 1619 | Based on Eq. 21, 23, 24, 25 in: 1620 | Allen et al. (1998) Crop evapotranspiration - Guidelines 1621 | for computing crop water requirements - FAO Irrigation and 1622 | drainage paper 56. 1623 | 1624 | Parameters 1625 | ---------- 1626 | lat : float 1627 | Latitude of the catchment (degrees). 1628 | j : int 1629 | Day of the year. 1630 | 1631 | Returns 1632 | ------- 1633 | float 1634 | Potential extraterrestrial solar radiation (kJ m-2 day-1). 1635 | 1636 | """ 1637 | 1638 | # Transform latitude from degrees to radians 1639 | lat_rad = lat * (np.pi / 180) 1640 | 1641 | # Calculate the solar declination 1642 | dec = 0.409 * np.sin(((2 * np.pi) / 365) * j - 1.39) 1643 | 1644 | # Calculate the inverse relative distance Earth-Sun 1645 | dr = 1 + 0.033 * np.cos(((2 * np.pi) / 365) * j) 1646 | 1647 | # Calculate the sunset hour angle 1648 | sha = np.arccos(-np.tan(lat_rad) * np.tan(dec)) 1649 | 1650 | # Calculate the extraterrestrial solar radiation (MJ m-2 day-1) 1651 | return ((24 * 60) / np.pi) * cls.SC * dr * ( 1652 | sha * np.sin(lat_rad) * np.sin(dec) + 1653 | np.cos(lat_rad) * np.cos(dec) * np.sin(sha)) 1654 | 1655 | @classmethod 1656 | def mean_monthly_radiation(cls, lat): 1657 | """ 1658 | Calculate the mean monthly potential extraterrestrial radiation for 1659 | an arbitrary year. 1660 | 1661 | Parameters 1662 | ---------- 1663 | lat : float 1664 | Latitude of the catchment (degrees). 1665 | 1666 | Returns 1667 | ------- 1668 | Pandas.Series 1669 | Pandas Series containing the mean monthly radiation values. 1670 | 1671 | """ 1672 | # Calculate the extraterrestrial radiation for each day of the year. 1673 | js = np.arange(1, 366, 1) 1674 | re = cls.extraterrestrial_radiation(lat, js) 1675 | 1676 | # Create an array of dates for a random year 1677 | idx = np.arange(dt.datetime(2017, 1, 1), dt.datetime(2018, 1, 1), 1678 | dt.timedelta(days=1)).astype(dt.datetime) 1679 | 1680 | # Calculate the monthly mean 1681 | ds = pd.Series(data=re, index=idx) 1682 | 1683 | return ds.groupby(ds.index.month).mean() 1684 | 1685 | @classmethod 1686 | def potential_evaporation(cls, re_m, temp_m): 1687 | """ 1688 | Calculate the mean monthly potential evaporation for a catchment. 1689 | 1690 | Based on Eq. 3 in: 1691 | Oudin et al. 2005, Which potential evapotranspiration input for 1692 | a lumped rainfall-runoff model? Part 2 --- Towards a simple and 1693 | efficient potential evapotranspiration model for rainfall-runoff 1694 | modelling. Journal of Hydrology, 303, p. 290-306. 1695 | 1696 | Parameters 1697 | ---------- 1698 | re_m : float 1699 | Mean monthly extraterrestrial solar radiation for a given month. 1700 | temp_m : float 1701 | Mean air temperature of a given month. 1702 | 1703 | Returns 1704 | ------- 1705 | float 1706 | Potential evapotranspiration (mm). 1707 | 1708 | """ 1709 | # Calculate the potential evaporation (m/day) 1710 | pet = (re_m / (cls.L * cls.RHO)) * ((temp_m + 5) / 100) 1711 | 1712 | # If temperature is below -5 deg C pe is considered to be 0 1713 | pet[temp_m <= -5] = 0 1714 | 1715 | # Transform the PET into mm 1716 | return pet * 1000 1717 | 1718 | 1719 | class Runoff(object): 1720 | """ 1721 | Methods to work with daily streamflow datasets from BAFU. 1722 | 1723 | Attributes 1724 | ---------- 1725 | stn_code : int 1726 | BAFU code of the streamflow station. 1727 | 1728 | """ 1729 | 1730 | def __init__(self, filename): 1731 | 1732 | self.fn = filename 1733 | 1734 | def load(self): 1735 | """ 1736 | Load streamflow data for a given BAFU hydrometric station. 1737 | 1738 | Returns 1739 | ------- 1740 | Pandas.DataFrame 1741 | DataFrame including streamflow data for each time step. 1742 | 1743 | """ 1744 | return pd.read_csv( 1745 | self.fn, sep='-|;', skiprows=7, skipinitialspace=True, 1746 | usecols=[1, 3], index_col=0, names=['Date', 'Q'], header=None, 1747 | parse_dates=True, squeeze=True, infer_datetime_format=True, 1748 | engine='python') 1749 | 1750 | def units(self): 1751 | """ 1752 | Get the units of the stream runoff data. 1753 | 1754 | BAFU stream runoff files usually have a line specifying the runoff 1755 | units. This method reads the runoff file line by line until it finds 1756 | the relevant line. It then extracts and returns the runoff units. 1757 | 1758 | Returns 1759 | ------- 1760 | str 1761 | Units of the stream runoff data. 1762 | 1763 | """ 1764 | with open(self.fn, 'r') as f: 1765 | lines = f.readlines() 1766 | 1767 | # Loop through the lines until the line containing 'Abfluss' is 1768 | # reached (this is the line containing information about the units). 1769 | # u_line = '' 1770 | for line in lines: 1771 | if 'Abfluss' in line: 1772 | units_line = line 1773 | break 1774 | 1775 | # Split the line into header ('Abfluss') and units 1776 | header, units = units_line.split(sep=' ') 1777 | 1778 | # HACK: Return the units removing the new string command ("\n") 1779 | return units[:-1] 1780 | 1781 | 1782 | class BasinShape(object): 1783 | """ 1784 | Define the contributing BAFU partial subcatchments for a given stream 1785 | gauge station and generate the corresponding polygon shapefile. 1786 | 1787 | Given a BAFU station code number the code downloads the shapefile 1788 | all the stream gauging stations from BAFU and extracts the coordinates 1789 | of the given station. It then downloads the shapefile with all the 1790 | sub-basins in which Switzerland is divided and computes which catchments 1791 | are upstream of the given station. The code finally returns a shapefile 1792 | with a single polygon representing the basin area corresponding to the 1793 | given station. 1794 | 1795 | Attributes 1796 | ---------- 1797 | stn_code : int 1798 | BAFU code of the desired hydrometric station. 1799 | 1800 | """ 1801 | # URL of the zip file containing the BAFU hydrometrical stations data 1802 | STATIONS_URL = ( 1803 | 'https://data.geo.admin.ch/' 1804 | 'ch.bafu.hydrologie-hydromessstationen/' 1805 | 'data.zip' 1806 | ) 1807 | 1808 | # URL of the zip file containing the BAFU subdivision of Switzerland in 1809 | # partial hydrological catchments 1810 | BASINS_URL = ( 1811 | 'https://www.bafu.admin.ch/dam/bafu/de/' 1812 | 'dokumente/wasser/geodaten/' 1813 | 'einzugsgebietsgliederungschweizausgabe2015.zip.download.zip/' 1814 | 'einzugsgebietsgliederungschweizausgabe2015.zip' 1815 | ) 1816 | 1817 | def __init__(self, stn_code): 1818 | 1819 | self.code = stn_code 1820 | 1821 | self.path_tmp = os.getcwd() + '\\BAFU_data\\' 1822 | 1823 | if not os.path.exists(self.path_tmp): 1824 | os.makedirs(self.path_tmp) 1825 | 1826 | def _download_data(self, zipurl): 1827 | """ 1828 | Download a zipfile and extract its contents. 1829 | 1830 | Parameters 1831 | ---------- 1832 | zipurl : str 1833 | URL of the zip file to download. 1834 | 1835 | """ 1836 | # Download the zipped data and extract its contents to the temp folder 1837 | context = ssl._create_unverified_context() 1838 | with urlopen(zipurl, context=context) as zipresp: 1839 | with ZipFile(BytesIO(zipresp.read())) as zfile: 1840 | zfile.extractall(self.path_tmp) 1841 | 1842 | if os.path.exists(self.path_tmp + 'lhg_UBST.zip'): 1843 | with ZipFile(self.path_tmp + 'lhg_UBST.zip') as zfile: 1844 | zfile.extractall(self.path_tmp) 1845 | os.remove(self.path_tmp + 'lhg_UBST.zip') 1846 | 1847 | def _get_station_coordinates(self): 1848 | """ 1849 | Get the coordinates of a given station. 1850 | 1851 | Download the BAFU shapefile containing the location of all the BAFU 1852 | hydrometric stations, and extract the coordinates of the desired 1853 | station. 1854 | 1855 | Returns 1856 | ------- 1857 | x : float 1858 | Longitude coordinate of the selected BAFU station. 1859 | y : float 1860 | Latitude coordinate of the selected BAFU station. 1861 | 1862 | """ 1863 | # Download and extract the data on the FOEN hydrometric stations 1864 | if not os.path.exists(self.path_tmp + 'lhg_UBST.shp'): 1865 | self._download_data(self.STATIONS_URL) 1866 | 1867 | # Load the shapefile containing the data 1868 | driver = ogr.GetDriverByName('ESRI Shapefile') 1869 | src_ds = driver.Open(self.path_tmp + 'lhg_UBST.shp', 0) 1870 | src_lyr = src_ds.GetLayer() 1871 | 1872 | # Filter the point feature representing the selected station 1873 | query = 'EDV_NR4 = {}'.format(self.code) 1874 | src_lyr.SetAttributeFilter(query) 1875 | 1876 | # Get the geometry of the feature 1877 | feature = src_lyr.GetNextFeature() 1878 | geometry = feature.GetGeometryRef() 1879 | 1880 | # Get the coordinates of the station from the feature's geometry 1881 | x = geometry.GetX() 1882 | y = geometry.GetY() 1883 | 1884 | # Close the data source 1885 | del src_ds 1886 | 1887 | return x, y 1888 | 1889 | def _get_auxiliary_codes(self, x, y): 1890 | """ 1891 | Get the auxiliary codes necessary to generate the upstream basin. 1892 | 1893 | Download the BAFU geodatabase containing all the partial sub-basins of 1894 | Switzerland and load it. Loop over the partial sub-basins and find 1895 | the one including the coordinates of the desired station. Finally, 1896 | get the auxiliary codes (H1 and H2) for the selected partial sub-basin. 1897 | 1898 | Parameters 1899 | ---------- 1900 | x : float 1901 | Longitude coordinate of the selected BAFU station. 1902 | y : float 1903 | Latitude coordinate of the selected BAFU station. 1904 | 1905 | Returns 1906 | ------- 1907 | h1 : float 1908 | Auxiliary code to generate the basin polygon. 1909 | h2 : float 1910 | Auxiliary code to generate the basin polygon. 1911 | 1912 | """ 1913 | # Download and extract the data on the FOEN partial catchments 1914 | if not os.path.exists(self.path_tmp + 'EZGG2015.gdb'): 1915 | self._download_data(self.BASINS_URL) 1916 | 1917 | # Load the database in which the data is stored and extract the 1918 | # shapefile where partial catchments are stored 1919 | driver = ogr.GetDriverByName('OpenFileGDB') 1920 | src_ds = driver.Open(self.path_tmp + 'EZGG2015.gdb', 0) 1921 | src_lyr = src_ds.GetLayer('basisgeometrie') 1922 | 1923 | # Create a point feature using the coordinates of the station 1924 | point = ogr.Geometry(ogr.wkbPoint) 1925 | point.SetPoint(0, x, y) 1926 | 1927 | # Loop over the different partial catchments to locate the partial 1928 | # subcatchment in which the hydrometric station is located and 1929 | # extract the "H1" and "H2" auxiliary field codes. 1930 | for feature in src_lyr: 1931 | polygon = feature.GetGeometryRef() 1932 | if point.Within(polygon): 1933 | h1 = feature.GetField('H1') 1934 | h2 = feature.GetField('H2') 1935 | 1936 | # Close the data source 1937 | del src_ds 1938 | 1939 | return h1, h2 1940 | 1941 | def _generate_shapefile(self, h1, h2, shp_fn): 1942 | """ 1943 | Generate a basin polygon shapefile for a given FOEN hydrometric 1944 | station code. 1945 | 1946 | Generate a polygon feature based on the auxiliary codes H1 and H2 1947 | and save it as a new shapefile. 1948 | 1949 | Parameters 1950 | ---------- 1951 | h1 : float 1952 | Auxiliary code to generate the basin polygon. 1953 | h2 : float 1954 | Auxiliary code to generate the basin polygon. 1955 | shp_fn : str 1956 | Path and file name of the output shapefile. 1957 | 1958 | """ 1959 | # Load the database in which the data is stored and extract the 1960 | # shapefile where partial catchments are stored 1961 | driver = ogr.GetDriverByName('OpenFileGDB') 1962 | src_ds = driver.Open(self.path_tmp + 'EZGG2015.gdb', 0) 1963 | src_lyr = src_ds.GetLayer('basisgeometrie') 1964 | src_proj = src_lyr.GetSpatialRef() 1965 | 1966 | # Create a new polygon geometry feature and merge all the partial 1967 | # catchments the are upstream of the given hydrometric station 1968 | # (as given by H1 and H2) 1969 | union_polygon = ogr.Geometry(ogr.wkbPolygon) 1970 | for feature in src_lyr: 1971 | if feature.GetField('H1') >= h1 and feature.GetField('H1') < h2: 1972 | geometry = feature.GetGeometryRef() 1973 | union_polygon = union_polygon.Union(geometry) 1974 | 1975 | # Create a new ogr driver to write the resulting shapefile 1976 | driver = ogr.GetDriverByName("ESRI Shapefile") 1977 | 1978 | # Check if a file with the same name exists, and if so delete it 1979 | if os.path.exists(shp_fn): 1980 | driver.DeleteDataSource(shp_fn) 1981 | 1982 | # Create a new layer to store the output data 1983 | dst_ds = driver.CreateDataSource(shp_fn) 1984 | dst_lyr = dst_ds.CreateLayer( 1985 | 'basin_' + str(self.code), src_proj, geom_type=ogr.wkbPolygon) 1986 | 1987 | # Assign an id field to the new layer 1988 | field_id = ogr.FieldDefn("id", ogr.OFTInteger) 1989 | dst_lyr.CreateField(field_id) 1990 | 1991 | # Write the output data to the newly created layer 1992 | feature_defn = dst_lyr.GetLayerDefn() 1993 | feature = ogr.Feature(feature_defn) 1994 | feature.SetGeometry(union_polygon) 1995 | feature.SetField("id", self.code) 1996 | dst_lyr.CreateFeature(feature) 1997 | 1998 | # Close the data source and target 1999 | del src_ds, dst_ds 2000 | 2001 | def generate(self, shp_fn, keep_files=False): 2002 | """ 2003 | """ 2004 | # Get the coordinates of the selected station 2005 | x, y = self._get_station_coordinates() 2006 | 2007 | # Get the corresponding auxiliary codes 2008 | h1, h2 = self._get_auxiliary_codes(x, y) 2009 | 2010 | # Generate the basin shapefile 2011 | self._generate_shapefile(h1, h2, shp_fn) 2012 | 2013 | if keep_files is False: 2014 | # Clean the temporary files 2015 | shutil.rmtree(self.path_tmp) 2016 | 2017 | def station_name(self, keep_files=False): 2018 | """ 2019 | """ 2020 | # Download and extract the data on the FOEN hydrometric stations 2021 | if not os.path.exists(self.path_tmp + 'lhg_UBST.shp'): 2022 | self._download_data(self.STATIONS_URL) 2023 | 2024 | # Load the shapefile containing the data 2025 | driver = ogr.GetDriverByName('ESRI Shapefile') 2026 | src_ds = driver.Open(self.path_tmp + 'lhg_UBST.shp', 0) 2027 | src_lyr = src_ds.GetLayer() 2028 | 2029 | # Filter the point feature representing the selected station 2030 | query = 'EDV_NR4 = {}'.format(self.code) 2031 | src_lyr.SetAttributeFilter(query) 2032 | 2033 | # Get the feature and retrieve the name of the station 2034 | feature = src_lyr.GetNextFeature() 2035 | name = feature['lhg_name'] 2036 | 2037 | # Close the data source and clean the temporary files 2038 | del src_ds 2039 | 2040 | if keep_files is False: 2041 | # Clean the temporary files 2042 | shutil.rmtree(self.path_tmp) 2043 | 2044 | return name 2045 | 2046 | 2047 | class HBVdata(object): 2048 | """ 2049 | Generate an HBV-light Catchment folder structure and data files. 2050 | 2051 | # TODO: Provide the possibility to choose between daily and hourly steps 2052 | (so far only daily time steps are used). 2053 | 2054 | # TODO: Provide the possibility to choose the input precipitation and 2055 | temperature data products (so far only RhiresD and TabsD are used). 2056 | 2057 | Attributes 2058 | ---------- 2059 | bsn_dir : str 2060 | Basin directory. 2061 | 2062 | """ 2063 | def __init__(self, bsn_dir): 2064 | 2065 | self.bsn_dir = bsn_dir 2066 | 2067 | # The data files are stored in the 'Data' subfolder. 2068 | self.data_dir = bsn_dir + '\\Data\\' 2069 | 2070 | # Create the folder structure if it doesn't exist. 2071 | if not os.path.exists(self.data_dir): 2072 | os.makedirs(self.data_dir) 2073 | 2074 | def _write_txt(self, df, filename, idx=False, header=None): 2075 | """ 2076 | Write a Pandas.DataFrame as a text file. 2077 | 2078 | Parameters 2079 | ---------- 2080 | df : Pandas.DataFrame 2081 | Pandas DataFrame containing the data to write as a text file. 2082 | filename : str 2083 | Name of the text file. 2084 | idx : bool, optional 2085 | Select whether the file should have an index column, default is 2086 | False. 2087 | header : str, optional 2088 | File header, default is None. 2089 | 2090 | """ 2091 | with open(self.data_dir + filename, 'w') as txt_file: 2092 | if header is not None: 2093 | # Write an additional header line 2094 | txt_file.write(str(header) + '\n') 2095 | 2096 | if idx is False: 2097 | # Save the file without an index. 2098 | df.to_csv(txt_file, sep='\t', header=True, 2099 | index=False, na_rep=-9999) 2100 | 2101 | else: 2102 | # Save the file with an index. 2103 | df.to_csv(txt_file, sep='\t', index_label='Date', 2104 | date_format='%Y%m%d', na_rep=-9999, header=True) 2105 | 2106 | @staticmethod 2107 | def _parse_precip(precip_dir, shp_fn=None, output='average', 2108 | dem_fn=None, start=None, end=None, touch_all=False): 2109 | """ 2110 | Parse the average precipitation data for the catchment. 2111 | 2112 | Parameters 2113 | ---------- 2114 | precip_dir : str 2115 | Directory where the precipitation data is stored. 2116 | shp_fn : str, optional 2117 | Path and filename of the shapefile defining the catchment boundary, 2118 | default is None. 2119 | output : {'average', 'lapse_rate'}, optional 2120 | Operation to perform to the precipitation data, 2121 | default is 'average'. 2122 | dem_fn : str, optional 2123 | Path and filename of the DEM file to get the elevation data from. 2124 | Needed if op == 'TCALT', default is None. 2125 | start : '%Y-%m-%d', optional 2126 | Start date of the output dataset, default is None. 2127 | end : '%Y-%m-%d', optional 2128 | End date of the outpout dataset, default is None. 2129 | touch_all : bool, optional 2130 | May be set to True to set all pixels touched by the line or 2131 | polygons, not just those whose center is within the polygon or 2132 | that are selected by brezenhams line algorithm, default is False. 2133 | 2134 | Returns 2135 | ------- 2136 | Pandas.Series 2137 | Pandas Series structure containing the precipitation data. 2138 | 2139 | Raises 2140 | ------ 2141 | ValueError 2142 | If the output format is not provided. 2143 | 2144 | """ 2145 | # Preallocate space to store dates and precipitation data. 2146 | dates = [] 2147 | vals = [] 2148 | 2149 | for file in glob.glob(precip_dir + '*.nc'): 2150 | # Loop over the precipitation NetCDF files in the directory. 2151 | 2152 | # Create an instance of the RhiresD class. 2153 | precip = RhiresD(file) 2154 | 2155 | if output == 'average': 2156 | # Get the average precipitation over the catchment. 2157 | dates, vals = precip.average( 2158 | shp_fn=shp_fn, date_list=dates, value_list=vals, 2159 | start=start, end=end, touch_all=touch_all) 2160 | series_name = 'P' 2161 | 2162 | elif output == 'lapse_rate': 2163 | # Calculate the temperature lapse rate over the catchment. 2164 | if dem_fn is None: 2165 | raise ValueError('DEM file for lapse rate ' 2166 | 'calculation not found.') 2167 | dates, vals = precip.lapse_rate( 2168 | dem_fn, shp_fn=shp_fn, date_list=dates, 2169 | value_list=vals, method='relative', start=start, 2170 | end=end, touch_all=touch_all) 2171 | series_name = 'PCALT' 2172 | 2173 | else: 2174 | raise ValueError('Output format is not recognised.') 2175 | 2176 | # Return a Pandas.Series object containing the precipitation data. 2177 | return pd.Series(data=vals, index=dates, name=series_name) 2178 | 2179 | @staticmethod 2180 | def _parse_temp(temp_dir, shp_fn=None, output='average', 2181 | dem_fn=None, start=None, end=None, touch_all=False): 2182 | """ 2183 | Parse the temperature data for the catchment. 2184 | 2185 | Parameters 2186 | ---------- 2187 | temp_dir : str 2188 | Directory where the temperature data is stored. 2189 | shp_fn : str, optional 2190 | Path and filename of the shapefile defining the catchment boundary, 2191 | default is None. 2192 | output : {'average', 'lapse_rate'}, optional 2193 | Operation to perform to the temperature data, default is 'average'. 2194 | dem_fn : str, optional 2195 | Path and filename of the DEM file to get the elevation data from. 2196 | Needed if op == 'TCALT', default is None. 2197 | start : '%Y-%m-%d', optional 2198 | Start date of the output dataset, default is None. 2199 | end : '%Y-%m-%d', optional 2200 | End date of the outpout dataset, default is None. 2201 | touch_all : bool, optional 2202 | May be set to True to set all pixels touched by the line or 2203 | polygons, not just those whose center is within the polygon or 2204 | that are selected by brezenhams line algorithm, default is False. 2205 | 2206 | Returns 2207 | ------- 2208 | Pandas.Series 2209 | Pandas Series structure containing the temperature data. 2210 | 2211 | Raises 2212 | ------ 2213 | ValueError 2214 | If the output format is not provided. 2215 | 2216 | """ 2217 | # Preallocate space to store dates and tempeature data. 2218 | dates = [] 2219 | vals = [] 2220 | 2221 | for file in glob.glob(temp_dir + '*.nc'): 2222 | # Loop over the temperature NetCDF files in the directory. 2223 | 2224 | # Create an instance of the TabsD class. 2225 | temp = TabsD(file) 2226 | 2227 | if output == 'average': 2228 | # Average temperature over the catchment. 2229 | dates, vals = temp.average( 2230 | shp_fn=shp_fn, date_list=dates, value_list=vals, 2231 | start=start, end=end, touch_all=touch_all) 2232 | series_name = 'T' 2233 | 2234 | elif output == 'lapse_rate': 2235 | # Calculate the temperature lapse rate over the catchment. 2236 | if dem_fn is None: 2237 | raise ValueError('DEM file for lapse rate ' 2238 | 'calculation not found.') 2239 | dates, vals = temp.lapse_rate( 2240 | dem_fn, shp_fn=shp_fn, date_list=dates, 2241 | value_list=vals, method='absolute', start=start, 2242 | end=end, touch_all=touch_all) 2243 | series_name = 'TCALT' 2244 | 2245 | else: 2246 | raise ValueError('Output format is not recognised.') 2247 | 2248 | # Return a Pandas.Series object containing the temperature data 2249 | return pd.Series(data=vals, index=dates, name=series_name) 2250 | 2251 | @staticmethod 2252 | def _parse_q(q_dir, shp_fn, stn_code, start=None, end=None): 2253 | """ 2254 | Parse the stream runoff data for the catchment outlet. 2255 | 2256 | Parameters 2257 | ---------- 2258 | q_dir : str 2259 | Directory where the stream runoff data is stored. 2260 | shp_fn : str 2261 | Path and filename of the shapefile defining the catchment boundary. 2262 | stn_code : int 2263 | Identification code of the BAFU hydrometric station defining 2264 | the catchment outlet. 2265 | start : '%Y-%m-%d', optional 2266 | Start date of the output dataset, default is None. 2267 | end : '%Y-%m-%d', optional 2268 | End date of the outpout dataset, default is None. 2269 | 2270 | Returns 2271 | ------- 2272 | Pandas.Series 2273 | Pandas Series structure containing the stream runoff data 2274 | in mm day-1. 2275 | 2276 | """ 2277 | # Get the appropriate data file given the BAFU station code 2278 | fn = q_dir + 'Q_' + str(stn_code) + '_Tagesmittel.asc' 2279 | 2280 | if not os.path.exists(fn): 2281 | raise ValueError('No streamflow data is available for the ' 2282 | 'given BAFU station.') 2283 | 2284 | # Load the runoff data 2285 | q = Runoff(fn).load() 2286 | # Get the runoff units to perform the calculations (m3 s-1 or l s-1) 2287 | u = Runoff(fn).units() 2288 | # Calculate the catchment area (m2) 2289 | area = Shape(shp_fn).area() 2290 | 2291 | # Re-index the data 2292 | if start is not None or end is not None: 2293 | date_index = pd.date_range(start=start, end=end, freq='D') 2294 | q = q.reindex(date_index) 2295 | 2296 | # Transform the runoff units to mm and return the resulting data. 2297 | if u == 'l/s': 2298 | return (q / area) * 3600 * 24 2299 | 2300 | else: # m3 s-1 2301 | return (q / area) * 1000 * 3600 * 24 2302 | 2303 | def generate_ptq( 2304 | self, precip_dir, temp_dir, q_dir, shp_fn, stn_code, step=100, 2305 | start=None, end=None, touch_all=False, filename='PTQ.txt'): 2306 | """ 2307 | Generate an HBV-light PTQ.txt file. 2308 | 2309 | Parameters 2310 | ---------- 2311 | precip_dir : str 2312 | Directory where the precipitation data is stored. 2313 | temp_dir : str 2314 | Directory where the temperature data is stored. 2315 | q_dir : str 2316 | Directory where the stream runoff data is stored. 2317 | shp_fn : str 2318 | Path and filename of the basin shapefile delimiting the catchment. 2319 | stn_code : int 2320 | Identification code of the BAFU hydrometric station defining 2321 | the catchment. 2322 | start : '%Y-%m-%d', optional 2323 | Start date of the output dataset, default is None. 2324 | end : '%Y-%m-%d', optional 2325 | End date of the outpout dataset, default is None. 2326 | touch_all : bool, optional 2327 | May be set to True to set all pixels touched by the line or 2328 | polygons, not just those whose center is within the polygon or 2329 | that are selected by brezenhams line algorithm, default is False. 2330 | filename : str 2331 | Name of the PTQ file, default is 'PTQ.txt'. 2332 | 2333 | Returns 2334 | ------- 2335 | ptq : Pandas.DataFrame 2336 | Pandas Dataframe containing the ptq data for the catchment. 2337 | 2338 | """ 2339 | print('...' + filename) 2340 | 2341 | # Parse the precipitation, temperature, and stream runoff data 2342 | p = self._parse_precip( 2343 | precip_dir, shp_fn=shp_fn, output='average', 2344 | start=start, end=end, touch_all=touch_all) 2345 | # Interpolate eventual missing data 2346 | p.interpolate(method='cubic', inplace=True) 2347 | # HACK: Filter potential negative precip values 2348 | p[p < 0] = 0 2349 | 2350 | t = self._parse_temp( 2351 | temp_dir, shp_fn=shp_fn, output='average', 2352 | start=start, end=end, touch_all=touch_all) 2353 | # Interpolate eventual missing data 2354 | t.interpolate(method='cubic', inplace=True) 2355 | 2356 | q = self._parse_q(q_dir, shp_fn, stn_code, start=start, end=end) 2357 | 2358 | # Merge the data into a Pandas.DataFrame and save it as a text file. 2359 | ptq = pd.concat([p, t, q], axis=1) 2360 | 2361 | # Re-index the data 2362 | if start is not None or end is not None: 2363 | date_index = pd.date_range(start=start, end=end, freq='D') 2364 | ptq = ptq.reindex(date_index) 2365 | 2366 | # Round the values. 2367 | ptq = ptq.round(decimals=3) 2368 | 2369 | # Set the header of the file 2370 | name = BasinShape(stn_code).station_name() 2371 | header = str(stn_code) + ' - ' + name 2372 | 2373 | self._write_txt(ptq, filename, idx=True, header=header) 2374 | 2375 | return ptq 2376 | 2377 | def generate_ptcalt( 2378 | self, precip_dir, temp_dir, dem_fn, shp_fn, stn_code, 2379 | start=None, end=None, pcalt=True, tcalt=True, touch_all=False, 2380 | filename='PTCALT.txt'): 2381 | """ 2382 | Generate an HBV-light PTCALT.txt file. 2383 | 2384 | Parameters 2385 | ---------- 2386 | precip_dir : str 2387 | Directory where the precipitation data is stored. 2388 | temp_dir : str 2389 | Directory where the temperature data is stored. 2390 | dem_fn : str 2391 | Path and filename of the DEM file to get the elevation data from. 2392 | shp_fn : str 2393 | Path and filename of the basin shapefile delimiting the catchment. 2394 | stn_code : int 2395 | Identification code of the BAFU hydrometric station defining 2396 | the catchment. 2397 | start : '%Y-%m-%d', optional 2398 | Start date of the output dataset, default is None. 2399 | end : '%Y-%m-%d', optional 2400 | End date of the outpout dataset, default is None. 2401 | pcalt : bool, optional 2402 | Choose whether to include precipitation lapse rate time series, 2403 | default is True. 2404 | tcalt : bool, optional 2405 | Choose whether to include temperature lapse rate time series, 2406 | default is True. 2407 | touch_all : bool, optional 2408 | May be set to True to set all pixels touched by the line or 2409 | polygons, not just those whose center is within the polygon or 2410 | that are selected by brezenhams line algorithm, default is False. 2411 | filename : str 2412 | Name of the PTCALT file, default is 'PTCALT.txt'. 2413 | 2414 | Returns 2415 | ------- 2416 | ptcalt : Pandas.Series 2417 | Pandas Series containing the temperature lapse rate. 2418 | 2419 | """ 2420 | print('...' + filename) 2421 | 2422 | if pcalt is False and tcalt is False: 2423 | # Return nothing if no variable is selected 2424 | return None 2425 | 2426 | else: 2427 | if pcalt is True and tcalt is False: 2428 | # Parse the precipitation data 2429 | ptcalt = self._parse_precip( 2430 | precip_dir, shp_fn=shp_fn, output='lapse_rate', 2431 | dem_fn=dem_fn, start=start, end=end, 2432 | touch_all=touch_all) 2433 | # Convert fraction to percentage 2434 | ptcalt = ptcalt * 100 2435 | 2436 | elif pcalt is False and tcalt is True: 2437 | # Parse the temperature data 2438 | ptcalt = self._parse_temp( 2439 | temp_dir, shp_fn=shp_fn, output='lapse_rate', 2440 | dem_fn=dem_fn, start=start, end=end, 2441 | touch_all=touch_all) 2442 | # Reverse the temperature lapse rate (HBV-light convention) 2443 | ptcalt = -ptcalt 2444 | 2445 | else: 2446 | # Parse the precipitation and temperature data 2447 | p_calt = self._parse_precip( 2448 | precip_dir, shp_fn=shp_fn, output='lapse_rate', 2449 | dem_fn=dem_fn, start=start, end=end, 2450 | touch_all=touch_all) 2451 | # Convert fraction to percentage 2452 | p_calt = p_calt * 100 2453 | t_calt = self._parse_temp( 2454 | temp_dir, shp_fn=shp_fn, output='lapse_rate', 2455 | dem_fn=dem_fn, start=start, end=end, 2456 | touch_all=touch_all) 2457 | # Reverse the temperature lapse rate (HBV-light convention) 2458 | t_calt = -t_calt 2459 | # Concatenate the precipitation and temperature series 2460 | ptcalt = pd.concat([p_calt, t_calt], axis=1) 2461 | 2462 | # Re-index the data 2463 | if start is not None or end is not None: 2464 | date_index = pd.date_range(start=start, end=end, freq='D') 2465 | ptcalt = ptcalt.reindex(date_index) 2466 | 2467 | # Round the number of decimals and save it as a text file. 2468 | ptcalt = ptcalt.round(decimals=3) 2469 | 2470 | # Set the header of the file 2471 | name = BasinShape(stn_code).station_name() 2472 | header = str(stn_code) + ' - ' + name 2473 | 2474 | self._write_txt(ptcalt, filename, idx=True, header=header) 2475 | 2476 | return ptcalt 2477 | 2478 | def generate_snow_cover( 2479 | self, sc_dir, shp_fn, start=None, end=None, 2480 | touch_all=False, filename='SnowCover.txt'): 2481 | """ 2482 | Generate an HBV-light SnowCover.txt file. 2483 | 2484 | Parameters 2485 | ---------- 2486 | sc_dir : str 2487 | Directory where the snow cover fraction data is stored. 2488 | shp_fn : str 2489 | Path and filename of the shapefile to use for masking the data. 2490 | start : '%Y-%m-%d', optional 2491 | Start date of the output dataset, default is None. 2492 | end : '%Y-%m-%d', optional 2493 | End date of the outpout dataset, default is None. 2494 | touch_all : bool, optional 2495 | May be set to True to set all pixels touched by the line or 2496 | polygons, not just those whose center is within the polygon or 2497 | that are selected by brezenhams line algorithm, default is False. 2498 | filename : str 2499 | Name of the SnowCover file, default is 'SnowCover.txt'. 2500 | 2501 | Returns 2502 | ------- 2503 | obs_sc : Pandas.DataFrame 2504 | Pandas DataFrame containing the average snow cover fraction values 2505 | for each elevation band and time step. 2506 | 2507 | """ 2508 | print('...' + filename) 2509 | 2510 | # Preallocate space to store dates and snow cover data. 2511 | dates = [] 2512 | scs = [] 2513 | 2514 | # Get the period in datetime format 2515 | if start is not None: 2516 | start_date = dt.datetime.strptime(start, '%Y-%m-%d') 2517 | 2518 | if end is not None: 2519 | end_date = dt.datetime.strptime(end, '%Y-%m-%d') 2520 | 2521 | for file in glob.glob(sc_dir + '*.hdf'): 2522 | # Loop over the snow cover hdf files in the directory. 2523 | 2524 | # Create an instance of the MOD10A1 class. 2525 | mod = MOD10A1(file) 2526 | 2527 | # Get the date of the current file. 2528 | date = mod.date() 2529 | 2530 | if start is not None: 2531 | # Continue to the next file if the end date of the file is 2532 | # before the start of the period. 2533 | if date < start_date: 2534 | continue 2535 | 2536 | if end is not None: 2537 | # Break the loop if the end date of the file is after 2538 | # the end of the period. 2539 | if date > end_date: 2540 | break 2541 | 2542 | # Calculate the average snow fraction over the catchment and 2543 | # append the date and value to the preallocated lists. 2544 | dates.append(date) 2545 | scs.append(mod.average(shp_fn=shp_fn, touch_all=touch_all)) 2546 | 2547 | # Store the data in a Pandas.Series object. 2548 | snow_cover = pd.Series(data=scs, index=dates, name='SnowCover') 2549 | 2550 | # Re-index the data 2551 | if start is not None or end is not None: 2552 | date_index = pd.date_range(start=start, end=end, freq='D') 2553 | snow_cover = snow_cover.reindex(date_index) 2554 | 2555 | # Round the values to 3 decimals. 2556 | snow_cover = snow_cover.round(decimals=3) 2557 | 2558 | self._write_txt(snow_cover, filename, idx=True, header=None) 2559 | 2560 | return snow_cover 2561 | 2562 | def generate_swe( 2563 | self, swe_dir, dem_fn, shp_fn, output='elev_dist', step=100, 2564 | start=None, end=None, touch_all=False, filename='ObsSWE.txt'): 2565 | """ 2566 | Generate an HBV-light ObsSWE.txt file. 2567 | 2568 | Parameters 2569 | ---------- 2570 | swe_dir : str 2571 | Directory where the snow water equivalent data is stored. 2572 | dem_fn : str 2573 | Path and filename of the DEM file used to get the elevation data. 2574 | shp_fn : str 2575 | Path and filename of the shapefile to use for masking the data. 2576 | output : {'elev_dist', 'average'} 2577 | Choose between calculating the average SWE value for each 2578 | elevation zone ('elev_dist') or for the entire catchment 2579 | ('average'), default is 'elev_dist'. 2580 | step : int or float, optional 2581 | Elevation band width to perform the calculations, default is 100. 2582 | start : '%Y-%m-%d', optional 2583 | Start date of the output dataset, default is None. 2584 | end : '%Y-%m-%d', optional 2585 | End date of the outpout dataset, default is None. 2586 | touch_all : bool, optional 2587 | May be set to True to set all pixels touched by the line or 2588 | polygons, not just those whose center is within the polygon or 2589 | that are selected by brezenhams line algorithm, default is False. 2590 | filename : str 2591 | Name of the SWE file, default is 'ObsSWE.txt'. 2592 | 2593 | Returns 2594 | ------- 2595 | obs_swe : Pandas.DataFrame 2596 | Pandas DataFrame containing the average SWE values for each 2597 | elevation band and time step. 2598 | 2599 | """ 2600 | print('...' + filename) 2601 | 2602 | # Initialise a Pandas.DataFrame object to store the ObsSWE data. 2603 | obs_swe = pd.DataFrame() 2604 | 2605 | # Get the period in datetime format 2606 | if start is not None: 2607 | start_date = dt.datetime.strptime(start, '%Y-%m-%d') 2608 | 2609 | if end is not None: 2610 | end_date = dt.datetime.strptime(end, '%Y-%m-%d') 2611 | 2612 | for file in glob.glob(swe_dir + '*.asc'): 2613 | # Loop over the SWE asc files in the directory. 2614 | 2615 | # Create an instance of the SWE class. 2616 | swe = SWE(file) 2617 | 2618 | # Get the date of the current file. 2619 | date = swe.date() 2620 | 2621 | if start is not None: 2622 | # Continue to the next file if the end date of the file is 2623 | # before the start of the period. 2624 | if date < start_date: 2625 | continue 2626 | 2627 | if end is not None: 2628 | # Break the loop if the end date of the file is after 2629 | # the end of the period. 2630 | if date > end_date: 2631 | break 2632 | 2633 | # Calculate the elevation distribution of SWE and append the date 2634 | # and value to the preallocated lists. 2635 | if output == 'elev_dist': 2636 | data = swe.elev_dist( 2637 | dem_fn, shp_fn=shp_fn, step=step, touch_all=touch_all) 2638 | 2639 | elif output == 'average': 2640 | data = swe.average(shp_fn=shp_fn, touch_all=touch_all) 2641 | # HACK: data needs to be formatted as a Pandas.DataFrame! 2642 | data = pd.DataFrame( 2643 | data=data, index=[swe.date()], columns=['SWE']) 2644 | 2645 | else: 2646 | raise ValueError('Selected output file not recognised.') 2647 | 2648 | obs_swe = obs_swe.append(data) 2649 | 2650 | # Re-index the data 2651 | if start is not None or end is not None: 2652 | date_index = pd.date_range(start=start, end=end, freq='D') 2653 | obs_swe = obs_swe.reindex(date_index) 2654 | 2655 | # Round the values. 2656 | obs_swe = obs_swe.round(decimals=3) 2657 | 2658 | self._write_txt(obs_swe, filename, idx=True, header=None) 2659 | 2660 | return obs_swe 2661 | 2662 | def generate_tmean( 2663 | self, temp_dir, shp_fn, freq='month', start=None, 2664 | end=None, touch_all=False, save_file=True, filename='T_mean.txt'): 2665 | """ 2666 | Generate an HBV-light T_mean.txt file. 2667 | 2668 | Parameters 2669 | ---------- 2670 | temp_dir : str 2671 | Directory where the temperature data is stored. 2672 | shp_fn : str 2673 | Path and filename of the basin shapefile delimiting the catchment. 2674 | freq : {'month', 'day'}, optional 2675 | Frequency of the long-term temperature averages, 2676 | default is 'month'. 2677 | start : '%Y-%m-%d', optional 2678 | Start date of the output dataset, default is None. 2679 | end : '%Y-%m-%d', optional 2680 | End date of the outpout dataset, default is None. 2681 | touch_all : bool, optional 2682 | May be set to True to set all pixels touched by the line or 2683 | polygons, not just those whose center is within the polygon or 2684 | that are selected by brezenhams line algorithm, default is False. 2685 | save_file : boolean, optional 2686 | Choose whether to save a file with the resulting data, 2687 | default is True. 2688 | filename : str, optional 2689 | Name of the T_mean file, default is 'T_mean.txt'. 2690 | 2691 | Returns 2692 | ------- 2693 | t_mean : Pandas.Series 2694 | Pandas series containing the monthly or daily average temperature 2695 | values. 2696 | 2697 | Raises 2698 | ------ 2699 | ValueError 2700 | If the provided averaging frequency is not recognised. 2701 | 2702 | """ 2703 | if save_file is True: 2704 | print('...' + filename) 2705 | 2706 | # Calculate the average temperature over the catchment. 2707 | temp = self._parse_temp( 2708 | temp_dir, shp_fn=shp_fn, output='average', 2709 | start=start, end=end, touch_all=touch_all) 2710 | 2711 | if freq == 'month': 2712 | # Calculate the monthly mean values. 2713 | temp_avg = temp.groupby(temp.index.month).mean() 2714 | 2715 | elif freq == 'day': 2716 | # Calculate the mean values for each day of the year (j). 2717 | temp_avg = temp.groupby([temp.index.month, temp.index.day]).mean() 2718 | # Remove February 29th as the list should be of 365 values. 2719 | temp_avg.drop((2, 29), inplace=True) 2720 | 2721 | else: 2722 | raise ValueError('Averaging frequency not recognised.') 2723 | 2724 | # Round the data values and rename the Pandas.Series object. 2725 | t_mean = temp_avg.round(decimals=3) 2726 | t_mean.name = 'T_mean' 2727 | 2728 | if save_file is True: 2729 | self._write_txt(t_mean, filename, idx=False, header=None) 2730 | 2731 | return t_mean 2732 | 2733 | def generate_evap(self, temp_dir, shp_fn, start=None, 2734 | end=None, touch_all=False, filename='EVAP.txt'): 2735 | """ 2736 | Generate an HBV-light EVAP.txt file. 2737 | 2738 | Parameters 2739 | ---------- 2740 | temp_dir : str 2741 | Directory where the temperature data is stored. 2742 | shp_fn : str 2743 | Path and filename of the basin shapefile delimiting the catchment. 2744 | start : '%Y-%m-%d', optional 2745 | Start date of the output dataset, default is None. 2746 | end : '%Y-%m-%d', optional 2747 | End date of the outpout dataset, default is None. 2748 | touch_all : bool, optional 2749 | May be set to True to set all pixels touched by the line or 2750 | polygons, not just those whose center is within the polygon or 2751 | that are selected by brezenhams line algorithm, default is False. 2752 | filename : bool, optional 2753 | Name of the Evap file, default is 'EVAP.txt'. 2754 | 2755 | Returns 2756 | ------- 2757 | pet : Pandas.Series 2758 | Pandas Series containing the mean monthly 2759 | evapotranspiration values. 2760 | 2761 | """ 2762 | print('...' + filename) 2763 | 2764 | # Get the monthly average temperature over the catchment 2765 | temp_m = self.generate_tmean( 2766 | temp_dir, shp_fn, freq='month', start=start, 2767 | end=end, touch_all=touch_all, save_file=False) 2768 | 2769 | # The the latitude of the centroid of the basin (deg) 2770 | lon, lat = Shape(shp_fn).lonlat() 2771 | 2772 | # Get the monthly average extraterrestrial radiation 2773 | re_m = Evap.mean_monthly_radiation(lat) 2774 | 2775 | # Calculate the mean monthly potential evaporation 2776 | pet = Evap.potential_evaporation(re_m, temp_m) 2777 | 2778 | # Round the data values and rename the Pandas.Series object. 2779 | pet = pet.round(decimals=3) 2780 | pet.name = 'EVAP' 2781 | 2782 | self._write_txt(pet, filename, idx=False, header=None) 2783 | 2784 | return pet 2785 | 2786 | def generate_clarea( 2787 | self, dem_fn, shp_fn, step=100, 2788 | touch_all=False, filename='Clarea.xml'): 2789 | """ 2790 | Generate an HBV-light Clarea.xml file. 2791 | 2792 | Parameters 2793 | ---------- 2794 | dem_fn : str, optional 2795 | Path and filename of the DEM file to get the elevation data from. 2796 | shp_fn : str 2797 | Path and filename of the shapefile defining the catchment boundary. 2798 | step : int or float, optional 2799 | Elevation band width to perform the calculations, default is 100. 2800 | touch_all : bool, optional 2801 | May be set to True to set all pixels touched by the line or 2802 | polygons, not just those whose center is within the polygon or 2803 | that are selected by brezenhams line algorithm, default is False. 2804 | filename : str 2805 | Name of the elevation distribution file, default is 'Clarea.xml'. 2806 | 2807 | Returns 2808 | ------- 2809 | clarea : Pandas.DataFrame 2810 | Pandas DataFrame containing the elevation distribution area 2811 | percentages at a given time step for the given catchment. 2812 | 2813 | """ 2814 | print('...' + 'Clarea.xml') 2815 | 2816 | clarea = DEM(dem_fn).elev_area_dist( 2817 | shp_fn=shp_fn, step=step, touch_all=touch_all) 2818 | 2819 | HBVconfig(self.bsn_dir).catchment_settings(clarea, filename=filename) 2820 | 2821 | return clarea 2822 | 2823 | def generate_metadata( 2824 | self, precip_dir, temp_dir, dem_fn, shp_fn, stn_code, 2825 | area_calc='shape', touch_all=False, filename='metadata.txt'): 2826 | """ 2827 | Generate a metadata file for the given catchment. 2828 | 2829 | The metadata currently being generated are: 2830 | - FOEN station code 2831 | - FOEN station name 2832 | - Latitude and longitude of the catchment centroid 2833 | - Average elevation of the precipitation and temperature data 2834 | - Average, maximum, and minimum catchment elevation 2835 | - Catchment area 2836 | 2837 | Parameters 2838 | ---------- 2839 | precip_dir : str 2840 | Directory where the precipitation data is stored. 2841 | temp_dir : str 2842 | Directory where the temperature data is stored. 2843 | dem_fn : str 2844 | Path and filename of the DEM file to get the elevation data from. 2845 | shp_fn : str 2846 | Path and filename of the basin shapefile delimiting the catchment. 2847 | stn_code : int 2848 | Identification code of the BAFU hydrometric station defining 2849 | the catchment. 2850 | area_calc : {'shape', 'dem'}, optional 2851 | Select if the catchment area should be calculated from the 2852 | shapefile or from the DEM, default is 'shape'. 2853 | touch_all : bool, optional 2854 | May be set to True to set all pixels touched by the line or 2855 | polygons, not just those whose center is within the polygon or 2856 | that are selected by brezenhams line algorithm, default is False. 2857 | filename : bool, optional 2858 | Name of the metadata file, default is 'metadata.txt'. 2859 | 2860 | Returns 2861 | ------- 2862 | meta : Pandas.DataFrame 2863 | Data structure containing the catchment metadata. 2864 | 2865 | Raises 2866 | ------ 2867 | ValueError 2868 | If the provided area_calc parameter is not 'shape' or 'dem'. 2869 | 2870 | """ 2871 | print('...' + filename) 2872 | 2873 | # Initialise a Pandas.DataFrame 2874 | meta = pd.DataFrame(index=[str(stn_code)]) 2875 | 2876 | # Get the name of the catchment 2877 | meta['Catchment'] = BasinShape(stn_code).station_name() 2878 | 2879 | # The the latitude of the centroid of the basin (deg) 2880 | meta['lon'], meta['lat'] = Shape(shp_fn).lonlat() 2881 | 2882 | # Load the DEM dataset 2883 | dem_ds = DEM(dem_fn) 2884 | dem = dem_ds.mask(shp_fn, touch_all=touch_all) 2885 | 2886 | # Get the average elevation of the precipitation data 2887 | p_file = glob.glob(precip_dir + '*.nc')[0] 2888 | meta['Pelev'] = RhiresD(p_file).mean_elev( 2889 | dem_fn, shp_fn=shp_fn, touch_all=touch_all) 2890 | 2891 | # Get the average elevation of the temperature data 2892 | t_file = glob.glob(temp_dir + '*.nc')[0] 2893 | meta['Telev'] = TabsD(t_file).mean_elev( 2894 | dem_fn, shp_fn=shp_fn, touch_all=touch_all) 2895 | 2896 | # Get the minimum, mean, and maximum elevation of the catchment 2897 | meta['Zmin'] = np.nanmin(dem) 2898 | meta['Zavg'] = np.nanmean(dem) 2899 | meta['Zmax'] = np.nanmax(dem) 2900 | # TODO: Calculate catchment slope. 2901 | 2902 | # Get the area of the catchment (in km2) 2903 | if area_calc == 'shape': 2904 | meta['Area'] = Shape(shp_fn).area() / 1e6 2905 | 2906 | elif area_calc == 'dem': 2907 | meta['Area_data'] = dem.count() 2908 | 2909 | else: 2910 | raise ValueError('Area calculation method not recognised.') 2911 | 2912 | meta = meta.round(decimals=3) 2913 | 2914 | with open(self.data_dir + filename, 'w') as f: 2915 | meta.to_csv(f, sep='\t', index_label='Code', header=True) 2916 | 2917 | return meta 2918 | 2919 | def generate_input_data( 2920 | self, precip_dir, temp_dir, q_dir, swe_dir, sc_dir, dem_fn, 2921 | stn_code, start=None, end=None, elev_step=100, 2922 | t_mean_freq='month', touch_all=False): 2923 | """ 2924 | Generate the necessary input data to run HBV-light. 2925 | 2926 | # TODO: Provide the possibility to decide which files to generate. 2927 | 2928 | Parameters 2929 | ---------- 2930 | precip_dir : str 2931 | Directory where the precipitation data is stored. 2932 | temp_dir : str 2933 | Directory where the temperature data is stored. 2934 | q_dir : str 2935 | Directory where the stream runoff data is stored. 2936 | swe_dir : str 2937 | Directory where the snow water equivalent data is stored. 2938 | sc_dir : str 2939 | Directory where the snow cover fraction data is stored. 2940 | dem_fn : str 2941 | Path and filename of the DEM file to get the elevation data from. 2942 | stn_code : int 2943 | Identification code of the BAFU hydrometric station defining 2944 | the catchment. 2945 | start : '%Y-%m-%d', optional 2946 | Start date of the output dataset, default is None. 2947 | end : '%Y-%m-%d', optional 2948 | End date of the outpout dataset, default is None. 2949 | elev_step : int or float, optional 2950 | Width of the elevation bands to do the calculations for, default 2951 | is 100. 2952 | t_mean_freq : {'month', 'day'}, optional 2953 | Frequency of average temperature values, default is 'month'. 2954 | touch_all : bool, optional 2955 | May be set to True to set all pixels touched by the line or 2956 | polygons, not just those whose center is within the polygon or 2957 | that are selected by brezenhams line algorithm, default is False. 2958 | 2959 | """ 2960 | print('Processing the input data for ' + str(stn_code) + '...') 2961 | 2962 | # Generate the basin shapefile (shp_fn) 2963 | shp_fn = self.data_dir + 'basin.shp' 2964 | BasinShape(stn_code).generate(shp_fn) 2965 | 2966 | # Generate the PTQ.txt file 2967 | self.generate_ptq( 2968 | precip_dir, temp_dir, q_dir, shp_fn, stn_code, 2969 | start=start, end=end, touch_all=touch_all) 2970 | 2971 | # Generate the PTCALT.txt file (considering the three alternatives) 2972 | self.generate_ptcalt( 2973 | precip_dir, temp_dir, dem_fn, shp_fn, stn_code, 2974 | start=start, end=end, pcalt=True, tcalt=True, 2975 | touch_all=touch_all) 2976 | 2977 | # Generate the ObsSWE.txt file 2978 | self.generate_swe( 2979 | swe_dir, dem_fn, shp_fn, output='elev_dist', step=elev_step, 2980 | start=start, end=end, touch_all=touch_all) 2981 | 2982 | # Generate the SnowCover.txt file 2983 | self.generate_snow_cover( 2984 | sc_dir, shp_fn, start=start, end=end, touch_all=touch_all) 2985 | 2986 | # Generate the EVAP.txt file 2987 | self.generate_evap( 2988 | temp_dir, shp_fn, start=start, end=end, touch_all=touch_all) 2989 | 2990 | # Generate the T_mean.txt file 2991 | self.generate_tmean( 2992 | temp_dir, shp_fn, freq=t_mean_freq, start=start, 2993 | end=end, touch_all=touch_all, save_file=True) 2994 | 2995 | # Generate the Clarea.xml file 2996 | self.generate_clarea( 2997 | dem_fn, shp_fn=shp_fn, step=elev_step, touch_all=touch_all) 2998 | 2999 | # Generate a metadata.txt file 3000 | self.generate_metadata( 3001 | precip_dir, temp_dir, dem_fn, shp_fn, 3002 | stn_code, touch_all=touch_all) 3003 | 3004 | # Remove temporary files 3005 | for file in glob.glob(self.data_dir + 'basin*'): 3006 | os.remove(file) 3007 | 3008 | def load_input_data(self, filename, no_data=-9999): 3009 | """ 3010 | Load the data from a predefined HBV-light input data file. 3011 | 3012 | NOTE: Only default input data names are currently accepted. See 3013 | the documentation of HBV-light for a description of the input data 3014 | and the default file names. 3015 | 3016 | Parameters 3017 | ---------- 3018 | filename : {'EVAP.txt', 'PTCALT.txt', 'ObsSWE.txt', 'PTQ.txt', 3019 | 'SnowCover.txt', 'T_mean.txt'} 3020 | Name of the input data file. 3021 | no_data : int or float, optional 3022 | Invalid data value, default is -9999. 3023 | 3024 | Returns 3025 | ------- 3026 | Pandas.DataFrame or Pandas.Serie 3027 | Data structure containing the selected input data type. 3028 | 3029 | Raises 3030 | ------ 3031 | ValueError 3032 | If the specified file does not exist. 3033 | 3034 | """ 3035 | filepath = self.data_dir + filename 3036 | 3037 | if not os.path.exists(filepath): 3038 | raise ValueError('The file does not exist.') 3039 | 3040 | if filename.lower() == 'ptq.txt': 3041 | board = pd.read_csv( 3042 | filepath, sep='\t', na_values=no_data, index_col=0, 3043 | parse_dates=True, skiprows=1, infer_datetime_format=True) 3044 | board.index.rename('Date', inplace=True) 3045 | 3046 | elif filename.lower() in ['evap.txt', 't_mean.txt']: 3047 | board = pd.read_csv(filepath) 3048 | if len(board.index) == 12: 3049 | board['Month'] = np.arange(1, 13) 3050 | board.set_index('Month', inplace=True) 3051 | elif len(board.index) == 365: 3052 | board['Day'] = np.arange(1, 366) 3053 | board.set_index('Day', inplace=True) 3054 | 3055 | elif filename.lower() == ' ptcalt.txt': 3056 | board = pd.read_csv( 3057 | filepath, sep='\t', index_col=0, parse_dates=True, 3058 | skiprows=1, infer_datetime_format=True, squeeze=True) 3059 | board.index.rename('Date', inplace=True) 3060 | 3061 | elif filename.lower() in ['snowcover.txt', 'obsswe.txt']: 3062 | board = pd.read_csv( 3063 | filepath, sep='\t', index_col=0, parse_dates=True, 3064 | na_values=no_data, infer_datetime_format=True, 3065 | squeeze=True) 3066 | board.index.rename('Date', inplace=True) 3067 | 3068 | else: 3069 | raise ValueError('The specified filename is not recognised.') 3070 | 3071 | return board 3072 | 3073 | 3074 | def load_metadata(catchments_dir): 3075 | """ 3076 | Load the metadata of all catchments in a given directory. 3077 | 3078 | Parameters 3079 | ---------- 3080 | catchments_dir : str 3081 | Path of the directory where the catchment folders are stored. 3082 | 3083 | Returns 3084 | ------- 3085 | metadata : Pandas.DataFrame 3086 | Data structure containing the metadata for all catchments in the 3087 | given directory. 3088 | 3089 | """ 3090 | metadata = pd.DataFrame() 3091 | 3092 | for root, dirs, files in os.walk(catchments_dir): 3093 | for file in files: 3094 | filename = os.path.join(root, file) 3095 | if file == 'metadata.txt': 3096 | meta = pd.read_csv( 3097 | filename, sep='\t', engine='python', index_col=0) 3098 | metadata = pd.concat([metadata, meta], axis=0) 3099 | 3100 | return metadata 3101 | -------------------------------------------------------------------------------- /hbvpy/core/model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | hbvpy.model 5 | =========== 6 | 7 | **A package to run the command line version of HBV-light.** 8 | 9 | This package is intended to provide bindngs to the command line version of 10 | HBV-light so the model can be run from a python script. 11 | 12 | .. author:: Marc Girons Lopez 13 | 14 | """ 15 | 16 | import os 17 | import subprocess 18 | 19 | from hbvpy.ThirdParty import AnimatedProgressBar 20 | 21 | 22 | __all__ = ['HBVcatchment', 'HBVsimulation'] 23 | 24 | 25 | class HBV(object): 26 | """ 27 | Set the command line version of HBV-light (HBV-light-CLI.exe). 28 | 29 | Attributes 30 | ---------- 31 | hbv_path : str, optional 32 | Non-default HBV-light-CLI.exe path, default is None. 33 | 34 | Raises 35 | ------ 36 | ValueError 37 | If the specified path to the HBV-light-CLI.exe file does not exist. 38 | 39 | """ 40 | def __init__(self, hbv_path=None): 41 | 42 | if hbv_path is None: 43 | self.hbv_path = ( 44 | 'C:\\Program Files (x86)\\HBV-light\\HBV-light-CLI.exe') 45 | 46 | else: 47 | self.hbv_path = hbv_path 48 | 49 | if not os.path.exists(self.hbv_path): 50 | raise ValueError( 51 | 'The specified HBV-ligh-CLI.exe file does not exist.') 52 | 53 | 54 | class HBVsimulation(HBV): 55 | """ 56 | HBV-light simulation setup. 57 | 58 | This class defines the input data and configuration files for setting up 59 | a particular simulation setup. 60 | 61 | HBV-light will search for the default files in the data directory and use 62 | them if they are present. If the user decides not to use a specific file 63 | (that is located in the data directory with a default name), the str 64 | 'dummy.txt' should be passed to the corresponding attribute. 65 | 66 | Attributes 67 | ---------- 68 | hbv_path : str, optional 69 | Non-default HBV-light-CLI.exe path, default is None. 70 | c : str, optional 71 | File with catchment settings, default is 'Clarea.xml'. 72 | p : str, optional 73 | File with parameter settings, default is 'Parameter.xml'. 74 | s : str, optional 75 | File with simulation settings, default is 'Simulation.xml'. 76 | ptq : str, optional 77 | File with daily precipitation, temperature and discharge values, 78 | default is 'ptq.txt'. 79 | evap : str, optional 80 | File with potential evaporation values, default is 'EVAP.txt'. 81 | tmean : str, optional 82 | File with long-term mean temperature values, default is 'T_mean.txt'. 83 | ptcalt : str, optional 84 | File with daily temperature and/or precipitation gradients, 85 | default is 'PTCALT'txt'. 86 | sc : str, optional 87 | File describing the spatial relation between different subcatchments, 88 | default is 'SubCatchment.txt'. 89 | b : str, optional 90 | File with parameter sets for batch simulation, default is 'Batch.txt'. 91 | ps : str, optional 92 | File with precipitation series, default is 'P_series.txt'. 93 | ts : str, optional 94 | File with temperature series, default is 'T_series.txt'. 95 | es : str, optional 96 | File with evaporation series, default is 'EVAP_series.txt'. 97 | bs : str, optional 98 | File with batch simulation settings, default is 'Batch_Simulation.txt'. 99 | mcs : str, optional 100 | File with Monte Carlo simulation settings, 101 | default is 'MC_Simulation.txt'. 102 | gaps : str, optional 103 | File with GAP simulation settings, default is 'GAP_Simulation.txt'. 104 | results : str, optional 105 | Results output folder, default is 'Results'. 106 | summary : str, optional 107 | Summary output file, default is 'Summary.txt'. 108 | g : str, optional 109 | Glacier profile file, default is 'GlacierProfile.txt'. 110 | swe : str, optional 111 | File with snow water equivalent data, default is 'ObsSWE.txt'. 112 | snowcover . str, optional 113 | File with snow cover data, default is 'SnowCover.txt'. 114 | python : str, optional 115 | Python objective function file, default is 'ObjFunc.py'. 116 | 117 | """ 118 | 119 | def __init__( 120 | self, hbv_path=None, c='Clarea.xml', p='Parameter.xml', 121 | s='Simulation.xml', ptq='ptq.txt', evap='EVAP.txt', 122 | tmean='T_mean.txt', ptcalt='PTCALT.txt', sc='SubCatchment.txt', 123 | b='Batch.txt', ps='P_series.txt', ts='T_series.txt', 124 | es='EVAP_series.txt', bs='Batch_Simulation.txt', 125 | mcs='MC_Simulation.txt', gaps='GAP_Simulation.txt', 126 | results='Results', summary='Summary.txt', g='GlacierProfile.txt', 127 | swe='ObsSWE.txt', snowcover='SnowCover.txt', python='ObjFunc.py'): 128 | 129 | super().__init__(hbv_path) 130 | 131 | self.results_folder = results 132 | 133 | self.files = { 134 | 'c': c, 'p': p, 's': s, 'ptq': ptq, 'evap': evap, 135 | 'tmean': tmean, 'ptcalt': ptcalt, 'sc': sc, 'b': b, 'ps': ps, 136 | 'ts': ts, 'es': es, 'bs': bs, 'mcs': mcs, 'gaps': gaps, 137 | 'summary': summary, 'g': g, 'swe': swe, 'snowcover': snowcover, 138 | 'python': python} 139 | 140 | 141 | class HBVcatchment(HBVsimulation): 142 | """ 143 | HBV-light catchment. 144 | 145 | This class defines the catchment folder for HBV-light and provides 146 | methods to run the model and show the progress. 147 | 148 | Attributes 149 | ---------- 150 | bsn_dir : str 151 | Path to the basin folder (containing a 'Data' sub-folder). 152 | simulation : hbvpy.model.Scenario instance 153 | Predefined HBV-light simulation setup to run for the chosen catchment. 154 | 155 | """ 156 | def __init__(self, bsn_dir, simulation): 157 | """ 158 | """ 159 | self.__simulation = simulation 160 | 161 | self.bsn_dir = bsn_dir 162 | 163 | self.basin_name = os.path.relpath(bsn_dir, bsn_dir + '..') 164 | 165 | if not os.path.exists(self.bsn_dir + self.results_folder): 166 | os.makedirs(self.bsn_dir + self.results_folder) 167 | 168 | def __getattr__(self, attr): 169 | """ 170 | """ 171 | return getattr(self.__simulation, attr) 172 | 173 | def __setattr__(self, attr, val): 174 | """ 175 | """ 176 | if attr == '_HBVcatchment__simulation': 177 | object.__setattr__(self, attr, val) 178 | 179 | return setattr(self.__simulation, attr, val) 180 | 181 | def _parse_files(self, command): 182 | """ 183 | Parse the necessary files to run HBV-light. 184 | 185 | Parameters 186 | ---------- 187 | command : list 188 | List of arguments needed to run HBV-light. 189 | 190 | Returns 191 | ------- 192 | command : list 193 | List of arguments and files needed to run HBV-light. 194 | 195 | """ 196 | for name, file in self.files.items(): 197 | if file is None: 198 | continue 199 | 200 | else: 201 | command.append('/' + name + ':' + file) 202 | 203 | return command 204 | 205 | def _print_progress(self, sim_type, process, debug_mode=False): 206 | """ 207 | Print the run progress of HBV-light. 208 | 209 | Parameters 210 | ---------- 211 | sim_type : str 212 | Simulation type. 213 | process : Subprocess.process 214 | Process to run the HBV model. 215 | debug_mode : bool, optional 216 | Choose whether to show the full HBV-light messages on the 217 | command line, default is False. 218 | 219 | """ 220 | print('\nProcessing: ' + str(sim_type) + 221 | ' | Catchment: ' + str(self.basin_name)) 222 | 223 | if debug_mode is True: 224 | while True: 225 | line = process.stdout.readline() 226 | if not line: 227 | break 228 | print(line) 229 | 230 | else: 231 | p = AnimatedProgressBar(end=100, width=50) 232 | while True: 233 | line = process.stdout.readline() 234 | if not line: 235 | break 236 | p + 1 237 | p.show_progress() 238 | print 239 | 240 | def run(self, sim_type, debug_mode=False): 241 | """ 242 | Run HBV-light. 243 | 244 | NOTE: Each simulation type (sim_type) requires specific configuration 245 | files. Please refer to the documentation of HBV-light for information 246 | on the different simulation types and required files. 247 | 248 | Parameters 249 | ---------- 250 | sim_type : {'SingleRun', 'MonteCarloRun', 'BatchRun', 'GAPRun'} 251 | Simulation type. 252 | debug_mode : bool, optional 253 | If False a progress bar is shown, otherwise the standard 254 | HBV-light output is shown, default is False. 255 | 256 | """ 257 | command = [ 258 | self.hbv_path, 'Run', self.bsn_dir, 259 | sim_type, self.results_folder] 260 | command = self._parse_files(command) 261 | 262 | process = subprocess.Popen( 263 | command, stdout=subprocess.PIPE, universal_newlines=True) 264 | 265 | self._print_progress(sim_type, process, debug_mode=debug_mode) 266 | -------------------------------------------------------------------------------- /hbvpy/core/process.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | hbvpy.process 5 | ============= 6 | 7 | **A package to process HBV-light simulation results.** 8 | 9 | This package is intended to provide functions and methods to parse and process 10 | the output of the different types HBV-light simulations (i.e. SingleRun, 11 | BatchRun, GAPRun). 12 | 13 | .. author:: Marc Girons Lopez 14 | 15 | """ 16 | 17 | import os 18 | import pandas as pd 19 | 20 | from . import HBVdata 21 | 22 | 23 | __all__ = ['BatchRun', 'GAPRun', 'MonteCarloRun', 'SingleRun'] 24 | 25 | 26 | class SingleRun(object): 27 | """ 28 | Process results from HBV-light single run simulations. 29 | 30 | Attributes 31 | ---------- 32 | bsn_dir : str 33 | Basin directory. 34 | 35 | """ 36 | def __init__(self, bsn_dir): 37 | 38 | self.bsn_dir = bsn_dir 39 | 40 | def load_results(self, results_folder='Results', sc=None): 41 | """ 42 | Load the results from a single HBV-light run. 43 | 44 | Parameters 45 | ---------- 46 | results_folder : str, optional 47 | Name of the results folder, default is 'Results'. 48 | sc : int, optional 49 | Sub-catchment number, in case there are more than one 50 | sub-catchments, default is None. 51 | 52 | Returns 53 | ------- 54 | Pandas.DataFrame 55 | Data structure containing the model results. 56 | 57 | Raises 58 | ------ 59 | ValueError 60 | If the specified file does not exist. 61 | 62 | """ 63 | # Set the results folder path. 64 | path = self.bsn_dir + '\\' + results_folder + '\\' 65 | 66 | # Set the results filename. 67 | if sc is not None: 68 | filepath = path + 'Results_SubCatchment_' + str(sc) + '.txt' 69 | else: 70 | filepath = path + 'Results.txt' 71 | 72 | # Check if the results file exists 73 | if not os.path.exists(filepath): 74 | raise ValueError('The file does not exist.') 75 | 76 | # Load the results file. 77 | return pd.read_csv( 78 | filepath, sep='\t', index_col=0, 79 | parse_dates=True, infer_datetime_format=True) 80 | 81 | def load_dist_results(self, results_folder='Results', sc=None): 82 | """ 83 | Load the distributed results from a single HBV-light run. 84 | 85 | Parameters 86 | ---------- 87 | results_folder : str, optional 88 | Name of the results folder, default is 'Results'. 89 | sc : int, optional 90 | Sub-catchment number, in case there are more than one 91 | sub-catchments, default is None. 92 | 93 | Returns 94 | ------- 95 | Pandas.DataFrame 96 | Data structure containing the distributed model results. 97 | 98 | Raises 99 | ------ 100 | ValueError 101 | If the specified file does not exist. 102 | 103 | """ 104 | # Set the results folder path. 105 | path = self.bsn_dir + '\\' + results_folder + '\\' 106 | 107 | # Set the results filename. 108 | if sc is not None: 109 | filepath = path + 'Dis_SubCatchment_' + str(sc) + '.txt' 110 | else: 111 | filepath = path + 'Dis.txt' 112 | 113 | # Check if the results file exists. 114 | if not os.path.exists(filepath): 115 | raise ValueError('The file does not exist.') 116 | 117 | # Load the results file. 118 | return pd.read_csv( 119 | filepath, sep='\t', index_col=0, 120 | parse_dates=True, infer_datetime_format=True) 121 | 122 | def load_summary(self, results_folder='Results'): 123 | """ 124 | Load the summary of the results from a single HBV-light run. 125 | 126 | Parameters 127 | ---------- 128 | results_folder : str, optional 129 | Name of the results folder, default is 'Results'. 130 | 131 | Returns 132 | ------- 133 | Pandas.DataFrame 134 | Data structure containing the distributed model results. 135 | 136 | Raises 137 | ------ 138 | ValueError 139 | If the specified file does not exist. 140 | 141 | """ 142 | # Set the results folder path. 143 | path = self.bsn_dir + '\\' + results_folder + '\\' 144 | 145 | # Set the summary filename. 146 | filepath = path + 'Summary.txt' 147 | 148 | # Check if the summary file exists. 149 | if not os.path.exists(filepath): 150 | raise ValueError('The file does not exist.') 151 | 152 | # Load the summary file. 153 | return pd.read_csv(filepath, sep='\t', index_col=0) 154 | 155 | def load_peaks(self, results_folder='Results'): 156 | """ 157 | Load the list of peak flows from a single HBV-light run. 158 | 159 | Following the documentation of HBV-light, a peak is defined as a data 160 | point with a Qobs value that is at least three times the average Qobs. 161 | Only a single peak is allowed in a window of 15 days. 162 | 163 | Parameters 164 | ---------- 165 | results_folder : str, optional 166 | Name of the results folder, default is 'Results'. 167 | 168 | Returns 169 | ------- 170 | Pandas.DataFrame 171 | Data structure containing the peak flow dates and values. 172 | 173 | Raises 174 | ------ 175 | ValueError 176 | If the specified file does not exist. 177 | 178 | """ 179 | # Set the results folder path. 180 | path = self.bsn_dir + '\\' + results_folder + '\\' 181 | 182 | # Set the peaks filename. 183 | filepath = path + 'Peaks.txt' 184 | 185 | # Check if the peaks file exists. 186 | if not os.path.exists(filepath): 187 | raise ValueError('The file does not exist.') 188 | 189 | # Load the peaks file. 190 | return pd.read_csv( 191 | filepath, sep='\t', index_col=0, parse_dates=True, 192 | infer_datetime_format=True, squeeze=True) 193 | 194 | def load_q_peaks(self, results_folder='Results'): 195 | """ 196 | Load the list of observed runoff and peak flows from a single 197 | HBV-light run. 198 | 199 | Following the documentation of HBV-light, a peak is defined as a data 200 | point with a Qobs value that is at least three times the average Qobs. 201 | Only a single peak is allowed in a window of 15 days. 202 | 203 | Parameters 204 | ---------- 205 | results_folder : str, optional 206 | Name of the results folder, default is 'Results'. 207 | 208 | Returns 209 | ------- 210 | Pandas.DataFrame 211 | Data structure containing the observed discharge values as well 212 | as the peak flow values. 213 | 214 | Raises 215 | ------ 216 | ValueError 217 | If the specified file does not exist. 218 | 219 | """ 220 | # Set the results folder path. 221 | path = self.bsn_dir + '\\' + results_folder + '\\' 222 | 223 | # Set the runoff peaks filename. 224 | filepath = path + 'Q_Peaks.txt' 225 | 226 | # Check if the runoff peaks file exists. 227 | if not os.path.exists(filepath): 228 | raise ValueError('The file does not exist.') 229 | 230 | # Load the runoff peaks file. 231 | return pd.read_csv( 232 | filepath, sep='\t', index_col=0, parse_dates=True, 233 | infer_datetime_format=True, squeeze=True) 234 | 235 | 236 | class GAPRun(object): 237 | """ 238 | Process results from HBV-light GAP run simulations. 239 | 240 | Attributes 241 | ---------- 242 | bsn_dir : str 243 | Basin directory. 244 | 245 | """ 246 | def __init__(self, bsn_dir): 247 | 248 | self.bsn_dir = bsn_dir 249 | 250 | def load_results(self, results_folder='Results'): 251 | """ 252 | Load the results from an HBV-light GAP calibration run. 253 | 254 | Parameters 255 | ---------- 256 | results_folder : str, optional 257 | Name of the GAP results folder, default is 'Results'. 258 | 259 | Returns 260 | ------- 261 | Pandas.DataFrame 262 | Data structure containing the GAP results. 263 | 264 | Raises 265 | ------ 266 | ValueError 267 | If the specified file does not exist. 268 | 269 | """ 270 | # Set the results folder path. 271 | path = self.bsn_dir + '\\' + results_folder + '\\' 272 | 273 | # Set the results filename. 274 | filepath = path + 'GA_best1.txt' 275 | 276 | # Check if the results file exists. 277 | if not os.path.exists(filepath): 278 | raise ValueError('The file does not exist.') 279 | 280 | # Load the results file. 281 | return pd.read_csv(filepath, sep='\t') 282 | 283 | 284 | class BatchRun(object): 285 | """ 286 | Process results from HBV-light batch run simulations. 287 | 288 | Attributes 289 | ---------- 290 | bsn_dir : str 291 | Basin directory. 292 | 293 | """ 294 | def __init__(self, bsn_dir): 295 | 296 | self.bsn_dir = bsn_dir 297 | 298 | def load_results(self, results_folder='Results', sc=None): 299 | """ 300 | Load the results from a batch HBV-light run. 301 | 302 | Parameters 303 | ---------- 304 | results_folder : str, optional 305 | Name of the Batch Run results folder, 306 | default is 'Results'. 307 | sc : int, optional 308 | Sub-catchment number, in case there are more than one 309 | sub-catchments, default is None. 310 | 311 | Returns 312 | ------- 313 | Pandas.DataFrame 314 | Data structure containing the Batch Run results. 315 | 316 | Raises 317 | ------ 318 | ValueError 319 | If the specified file does not exist. 320 | 321 | """ 322 | # Set the results folder path. 323 | path = self.bsn_dir + '\\' + results_folder + '\\' 324 | 325 | # Set the results filename. 326 | if sc is not None: 327 | filepath = path + 'BatchRun_SubCatchment_' + str(sc) + '.txt' 328 | else: 329 | filepath = path + 'BatchRun.txt' 330 | 331 | # Check if the results file exists. 332 | if not os.path.exists(filepath): 333 | raise ValueError('The file does not exist.') 334 | 335 | # Load the results file. 336 | return pd.read_csv(filepath, sep='\t') 337 | 338 | def load_runoff(self, results_folder='Results', data='columns', sc=None): 339 | """ 340 | Load the time series of observed and simulated runoff from 341 | a batch HBV-light Run. 342 | 343 | Parameters 344 | ---------- 345 | results_folder : str, optional 346 | Name of the Batch Run results folder, default is 'Results'. 347 | data : {'rows', 'columns'}, optional 348 | Organisation of the data in the results file. 349 | sc : int, optional 350 | Sub-catchment number, in case there are more than one 351 | sub-catchments, default is None. 352 | 353 | Returns 354 | ------- 355 | Pandas.DataFrame 356 | Data structure containing the Batch Run runoff time series. 357 | 358 | Raises 359 | ------ 360 | ValueError 361 | If the corresponding file does not exist. 362 | ValueError 363 | If the data structure is not recognised. 364 | 365 | """ 366 | # Set the results folder path. 367 | path = self.bsn_dir + '\\' + results_folder + '\\' 368 | 369 | # Load the data according to the predefined format of the results file. 370 | if data == 'columns': 371 | # Set the runoff results filename. 372 | if sc is not None: 373 | filepath = path + 'BatchQsim_(InColumns)_' + str(sc) + '.txt' 374 | else: 375 | filepath = path + 'BatchQsim_(InColumns).txt' 376 | 377 | # Check if the runoff results file exists. 378 | if not os.path.exists(filepath): 379 | raise ValueError('The file does not exist.') 380 | 381 | # Load the runoff results file. 382 | return pd.read_csv( 383 | filepath, sep='\t', parse_dates=True, 384 | index_col=0, infer_datetime_format=True) 385 | 386 | elif data == 'rows': 387 | # Set the runoff results filename. 388 | if sc is not None: 389 | filepath = path + 'BatchQsim_' + str(sc) + '.txt' 390 | else: 391 | filepath = path + 'BatchQsim.txt' 392 | 393 | # Check if the runoff results file exists. 394 | if not os.path.exists(filepath): 395 | raise ValueError('The file does not exist.') 396 | 397 | # Parse the index. 398 | dates = pd.read_csv( 399 | filepath, sep='\t', header=None, nrows=1, 400 | index_col=False, squeeze=True).transpose() 401 | 402 | # Parse the data. 403 | data = pd.read_csv( 404 | filepath, sep='\t', header=None, index_col=False, 405 | skiprows=1).transpose() 406 | 407 | # Rename the index and convert it to datetime format. 408 | dates.columns = ['Date'] 409 | dates = pd.to_datetime(dates['Date'], format='%Y%m%d') 410 | 411 | # Merge the index and data into a Pandas.DataFrame structure. 412 | df = pd.concat([data, dates], axis=1) 413 | 414 | # Set the index and return the DataFrame 415 | return df.set_index('Date') 416 | 417 | else: 418 | raise ValueError('Data organisation not recognised.') 419 | 420 | def load_runoff_stats(self, results_folder='Results', sc=None): 421 | """ 422 | Load the time series of observed and simulated runoff statistics 423 | from a batch HBV-light Run. 424 | 425 | The statistics contain: Qobs, Qmedian, Qmean, Qp10, Qp90. 426 | 427 | Parameters 428 | ---------- 429 | results_folder : str, optional 430 | Name of the Batch Run results folder, default is 'Results'. 431 | sc : int, optional 432 | Sub-catchment number, in case there are more than one 433 | sub-catchments, default is None. 434 | 435 | Returns 436 | ------- 437 | Pandas.DataFrame 438 | Data structure containing the Batch Run runoff statistics 439 | time series. 440 | 441 | Raises 442 | ------ 443 | ValueError 444 | If the specified file does not exist. 445 | 446 | """ 447 | # Set the results folder path. 448 | path = self.bsn_dir + '\\' + results_folder + '\\' 449 | 450 | # Set the runoff statistics filename. 451 | if sc is not None: 452 | filepath = path + 'BatchQsimSummary_' + str(sc) + '.txt' 453 | else: 454 | filepath = path + 'BatchQsimSummary.txt' 455 | 456 | # Check if the runoff statistics file exists. 457 | if not os.path.exists(filepath): 458 | raise ValueError('The file does not exist.') 459 | 460 | # Load the runoff statistics file. 461 | return pd.read_csv( 462 | filepath, sep='\t', parse_dates=True, 463 | index_col=0, infer_datetime_format=True) 464 | 465 | def load_runoff_component( 466 | self, results_folder='Results', component='Snow', sc=None): 467 | """ 468 | Load the time series of a given runoff component from a batch 469 | HBV-light run. 470 | 471 | Parameters 472 | ---------- 473 | results_folder : str, optional 474 | Name of the Batch Run results folder, default is 'Results'. 475 | component : {'Rain', 'Snow', 'Glacier', 'Q0', 'Q1', 'Q2'} 476 | Name of the runoff component to load, default 'Snow'. 477 | sc : int, optional 478 | Sub-catchment number, in case there are more than one 479 | sub-catchments, default is None. 480 | 481 | Returns 482 | ------- 483 | Pandas.DataFrame 484 | Data structure containing the Batch Run runoff component 485 | time series. 486 | 487 | Raises 488 | ------ 489 | ValueError 490 | If the provided runoff component is not recognised. 491 | ValueError 492 | If the specified file does not exist. 493 | 494 | """ 495 | # Check if the provided component is valid. 496 | if component not in ['Rain', 'Snow', 'Glacier', 'Q0', 'Q1', 'Q2']: 497 | raise ValueError('Provided runoff compoent not recognised.') 498 | 499 | # Set the results folder path. 500 | path = self.bsn_dir + '\\' + results_folder + '\\' 501 | 502 | # Set the runoff component filename. 503 | if sc is not None: 504 | filepath = path + 'BatchQsim_' + component + '_' + str(sc) + '.txt' 505 | else: 506 | filepath = path + 'BatchQsim_' + component + '.txt.' 507 | 508 | # Check if the runoff component file exists. 509 | if not os.path.exists(filepath): 510 | raise ValueError('The file does not exist.') 511 | 512 | # Parse the index. 513 | dates = pd.read_csv( 514 | filepath, sep='\t', header=None, nrows=1, 515 | index_col=False, squeeze=True).transpose() 516 | 517 | # Parse the data. 518 | data = pd.read_csv( 519 | filepath, sep='\t', header=None, index_col=False, 520 | skiprows=1).transpose() 521 | 522 | # Rename the index and convert it to datetime format. 523 | dates.columns = ['Date'] 524 | dates = pd.to_datetime(dates['Date'], format='%Y%m%d') 525 | 526 | # Merge the index and data into a single Pandas.DataFrame structure. 527 | df = pd.concat([data, dates], axis=1) 528 | 529 | # Set the index. 530 | return df.set_index('Date') 531 | 532 | def load_monthly_runoff(self, results_folder='Results', sc=None): 533 | """ 534 | Load the monthly average simulated runoff from each parameter set 535 | used for a batch HBV-light run. 536 | 537 | Parameters 538 | ---------- 539 | results_folder : str, optional 540 | Name of the Batch Run results folder, default is 'Results'. 541 | sc : int, optional 542 | Sub-catchment number, in case there are more than one 543 | sub-catchments, default is None. 544 | 545 | Returns 546 | ------- 547 | Pandas.DataFrame 548 | Data structure containing the monthly average runoff values 549 | from each parameter set. 550 | 551 | Raises 552 | ------ 553 | ValueError 554 | If the specifed file does not exist. 555 | 556 | """ 557 | # Set the results folder path. 558 | path = self.bsn_dir + '\\' + results_folder + '\\' 559 | 560 | # Set the monthly runoff filename. 561 | if sc is not None: 562 | filepath = path + 'Qseasonal_' + str(sc) + '.txt' 563 | else: 564 | filepath = path + 'Qseasonal.txt.' 565 | 566 | # Check if the monthly runoff file exists. 567 | if not os.path.exists(filepath): 568 | raise ValueError('The file does not exist.') 569 | 570 | # Load the monthly runoff file. 571 | return pd.read_csv(filepath, sep='\t') 572 | 573 | def load_swe(self, results_folder='Results'): 574 | """ 575 | Load the time series of simulated snow water equivalent for each 576 | elevation band and parameter set used for a batch HBV-light run. 577 | 578 | NOTE: This method ONLY works if no additional precipitaiton, 579 | temperature, or evaporation series are provided. 580 | 581 | Parameters 582 | ---------- 583 | results_folder : str, optional 584 | Name of the Batch Run results folder, default is 'Results'. 585 | 586 | Returns 587 | ------- 588 | swe : Pandas.DataFrame 589 | Data structure containing the simulated snow water equivalent 590 | data for each elevation band and parameter set. 591 | 592 | Raises 593 | ------ 594 | ValueError 595 | If the specifed file does not exist. 596 | 597 | """ 598 | # Set the results folder path. 599 | path = self.bsn_dir + '\\' + results_folder + '\\' 600 | 601 | # Set the snow water equivalent filename. 602 | filepath = path + 'Simulated_SWE.txt.' 603 | 604 | # Check if the snow water equivalent file exists. 605 | if not os.path.exists(filepath): 606 | raise ValueError('The file does not exist.') 607 | 608 | # Parse the raw data 609 | raw_data = pd.read_csv(filepath, sep='\t') 610 | 611 | # Get the number of parameter sets 612 | param_sets = raw_data['Parameterset'].unique() 613 | 614 | # Initialise a dict to write the SWE data 615 | swe = {ps: None for ps in param_sets} 616 | 617 | # Parse the data 618 | for ps in param_sets: 619 | data_tmp = raw_data[raw_data['Parameterset'] == ps] 620 | data_tmp.set_index('SWE serie', drop=True, inplace=True) 621 | data = data_tmp.drop( 622 | ['Parameterset', 'Prec. serie', 623 | 'Temp. serie', 'Evap. serie'], axis=1).transpose() 624 | data.index = pd.to_datetime(data.index, format='%Y%m%d') 625 | 626 | swe[ps] = data 627 | 628 | return swe 629 | 630 | def calculate_swe_stats(self, results_folder, clarea=None): 631 | """ 632 | Calculate the catchment-wide statistics of snow water equivalent 633 | taking into account all elevation bands and parameter sets used in 634 | batch HBV-light run. 635 | 636 | This method provides median and mean catchment snow water equivalent, 637 | in addition to the 10th and 90th percentiles. 638 | 639 | Parameters 640 | ---------- 641 | results_folder : str 642 | Name of the Batch Run results folder. 643 | clarea : str, optional 644 | Custom name for the 'Clarea.xml' file. If no value is provided 645 | the default file name is used, default is None. 646 | 647 | Returns 648 | ------- 649 | sim_swe : Pandas.Series 650 | Data structure containing the simulated catchment avverage 651 | snow water equivalent for each time step. 652 | 653 | """ 654 | # Instatiate the HBVdata class 655 | hbv_data = HBVdata(self.bsn_dir) 656 | 657 | # Use the default elevation distribution file name if no name is 658 | # specified. 659 | if clarea is None: 660 | clarea = 'Clarea.xml' 661 | 662 | # Check if the elevation distribution file exists. 663 | if not os.path.exists(hbv_data.data_dir + clarea): 664 | raise ValueError('The Clarea.xml file does not exist.') 665 | 666 | # Load the area fraction of each elevation band in the catchment. 667 | elev_dist = hbv_data.load_elev_dist(filename=clarea) 668 | 669 | # Load the distributed simulated snow water equivalent. 670 | dist_swe = self.load_swe(results_folder) 671 | 672 | # Calculate the catchment average snow water equivalent for 673 | # each of the parameter sets. 674 | avg_swe = pd.DataFrame() 675 | for p_set in dist_swe.keys(): 676 | for i, col in enumerate(dist_swe[p_set].columns): 677 | area = elev_dist.iloc[i] 678 | dist_swe[p_set][col] = dist_swe[p_set][col] * area 679 | avg_swe[p_set] = dist_swe[p_set].sum(axis=1) 680 | 681 | # Calculate the snow water equivalent statistics: 682 | # median, mean, 10p, and 90p 683 | swe_stats = pd.DataFrame() 684 | swe_stats['SWEmedian'] = avg_swe.median(axis=1) 685 | swe_stats['SWEmean'] = avg_swe.mean(axis=1) 686 | swe_stats['SWEp10'] = avg_swe.quantile(q=0.1, axis=1) 687 | swe_stats['SWEp90'] = avg_swe.quantile(q=0.9, axis=1) 688 | 689 | return swe_stats 690 | 691 | def calculate_runoff_quantile( 692 | self, results_folder='Results', data='Columns', quantile=0.5): 693 | """ 694 | Calculate the time series of runoff magnitudes corresponding to a given 695 | quantile. 696 | 697 | NOTE: This method only works if the catchment contains a single 698 | subcatchment. 699 | 700 | Parameters 701 | ---------- 702 | results_folder : str, optional 703 | Name of the Batch Run results folder, default is 'Results'. 704 | data : {'rows', 'columns'}, optional 705 | Organisation of the data in the results file. 706 | quantile : float, optional 707 | Quantile to get the runoff for, default 0.5. 708 | 709 | Returns 710 | ------- 711 | Pandas.Series 712 | Time series of runoff magnitudes corresponding to the given 713 | quantile. 714 | 715 | """ 716 | # Load the runoff results data 717 | runoff = self.load_runoff(results_folder, data=data, sc=None) 718 | 719 | # Drop the observed runoff column 720 | runoff = runoff.drop('Qobs', axis=1) 721 | 722 | # Return the time series of the given runoff quantile. 723 | return runoff.quantile(quantile, axis=1) 724 | 725 | 726 | class MonteCarloRun(object): 727 | """ 728 | Process results from HBV-light Monte Carlo simulations. 729 | 730 | Attributes 731 | ---------- 732 | bsn_dir : str 733 | Basin directory. 734 | 735 | """ 736 | def __init__(self, bsn_dir): 737 | 738 | self.bsn_dir = bsn_dir 739 | 740 | def load_results(self, results_folder='Results', sc=None): 741 | """ 742 | Load the results of a HBV-light Monte Carlo Run. 743 | 744 | Parameters 745 | ---------- 746 | results_folder : str, optional 747 | Name of the MC Run results folder, default is 'Results'. 748 | sc : int, optional 749 | Sub-catchment number, in case there are more than one 750 | sub-catchments, default is None. 751 | 752 | Returns 753 | ------- 754 | Pandas.DataFrame 755 | Data structure containing the MC Run results. 756 | 757 | Raises 758 | ------ 759 | ValueError 760 | If the specified file does not exist. 761 | 762 | """ 763 | # Set the results folder path. 764 | path = self.bsn_dir + '\\' + results_folder + '\\' 765 | 766 | # Set the results filename. 767 | if sc is not None: 768 | filepath = path + 'Multi_SubCatchment_' + str(sc) + '.txt' 769 | else: 770 | filepath = path + 'Multi.txt' 771 | 772 | # Check if the results file exists. 773 | if not os.path.exists(filepath): 774 | raise ValueError('The file does not exist.') 775 | 776 | # Load the results file. 777 | return pd.read_csv(filepath, sep='\t', index_col=0) 778 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | HBVpy: A module for interacting with the HBV-light hydrological model. 5 | 6 | HBVpy is a package designed to interact with the command line version of the 7 | HBV-light hydrological model. It allows generate the necessary configuration 8 | files adapted for the different modifications, to run the model, 9 | and to process the results. 10 | 11 | """ 12 | from setuptools import setup, find_packages 13 | 14 | 15 | def readme(): 16 | with open('README.md') as rm: 17 | return rm.read() 18 | 19 | 20 | setup( 21 | name='hbvpy', 22 | version='0.1', 23 | description='Python functions to interact with HBV-light', 24 | long_description=readme(), 25 | classifiers=[ 26 | 'Development Status :: 3 - Alpha', 27 | 'License :: OSI Approved :: BSD License', 28 | 'Programming Language :: Python :: 3.6', 29 | 'Topic :: Scientific/Engineering :: Hydrology', 30 | 'Operating System :: Microsoft :: Windows' 31 | ], 32 | keywords='HBV rainfall-runoff hydrology model', 33 | url='https://github.com/GironsLopez/hbvpy', 34 | author='Marc Girons Lopez', 35 | author_email='m.girons@gmail.com', 36 | license='BSD', 37 | packages=find_packages(), 38 | install_requires=[ 39 | 'lxml', 'netCDF4', 'numpy', 'gdal', 40 | 'pandas', 'pyproj', 'scipy' 41 | ], 42 | zip_safe=False, 43 | ) 44 | --------------------------------------------------------------------------------