├── docs ├── _static │ └── .gitignore ├── changelog.rst ├── fortran_api │ ├── gwgen.src.main.rst │ ├── gwgen.src.csv_file.rst │ ├── gwgen.src.geohashmod.rst │ ├── gwgen.src.parametersmod.rst │ ├── gwgen.src.randomdistmod.rst │ ├── gwgen.src.weathergenmod.rst │ └── index.rst ├── api │ ├── gwgen.main.rst │ ├── gwgen.utils.rst │ ├── gwgen.preproc.rst │ ├── gwgen.evaluation.rst │ ├── gwgen.parse_eecra.rst │ ├── gwgen.parseghcnrow.rst │ ├── gwgen.parameterization.rst │ ├── gwgen.sensitivity_analysis.rst │ └── gwgen.rst ├── command_line │ ├── gwgen.run.rst │ ├── gwgen.info.rst │ ├── gwgen.init.rst │ ├── gwgen.setup.rst │ ├── gwgen.remove.rst │ ├── gwgen.archive.rst │ ├── gwgen.compile.rst │ ├── gwgen.sens.run.rst │ ├── gwgen.bias.tmin.rst │ ├── gwgen.bias.wind.rst │ ├── gwgen.configure.rst │ ├── gwgen.del-value.rst │ ├── gwgen.get-value.rst │ ├── gwgen.param.day.rst │ ├── gwgen.sens.init.rst │ ├── gwgen.set-value.rst │ ├── gwgen.unarchive.rst │ ├── gwgen.param.cday.rst │ ├── gwgen.param.corr.rst │ ├── gwgen.param.prcp.rst │ ├── gwgen.param.temp.rst │ ├── gwgen.param.wind.rst │ ├── gwgen.sens.setup.rst │ ├── gwgen.evaluate.ks.rst │ ├── gwgen.param.cloud.rst │ ├── gwgen.param.month.rst │ ├── gwgen.sens.remove.rst │ ├── gwgen.param.cmonth.rst │ ├── gwgen.param.markov.rst │ ├── gwgen.preproc.test.rst │ ├── gwgen.sens.compile.rst │ ├── gwgen.preproc.select.rst │ ├── gwgen.evaluate.output.rst │ ├── gwgen.evaluate.quants.rst │ ├── gwgen.evaluate.prepare.rst │ ├── gwgen.evaluate.quality.rst │ ├── gwgen.sens.evaluate.ks.rst │ ├── gwgen.sens.plot.plot1d.rst │ ├── gwgen.sens.plot.plot2d.rst │ ├── gwgen.param.daily_cloud.rst │ ├── gwgen.param.yearly_cday.rst │ ├── gwgen.param.cdaily_cloud.rst │ ├── gwgen.param.hourly_cloud.rst │ ├── gwgen.param.cmonthly_wind.rst │ ├── gwgen.param.monthly_cloud.rst │ ├── gwgen.param.yearly_cmonth.rst │ ├── gwgen.param.cmonthly_cloud.rst │ ├── gwgen.sens.evaluate.quants.rst │ ├── gwgen.sens.evaluate.quality.rst │ ├── gwgen.bias.rst │ ├── gwgen.param.yearly_cdaily_cloud.rst │ ├── gwgen.param.yearly_cmonthly_wind.rst │ ├── gwgen.param.yearly_cmonthly_cloud.rst │ ├── gwgen.preproc.cloud.eecra_ghcn_map.rst │ ├── gwgen.preproc.cloud.eecra_inventory.rst │ ├── gwgen.sens.plot.rst │ ├── gwgen.preproc.rst │ ├── gwgen.preproc.cloud.rst │ ├── gwgen.sens.evaluate.rst │ ├── gwgen.evaluate.rst │ ├── gwgen.sens.rst │ ├── gwgen.rst │ └── gwgen.param.rst ├── apigen.bash ├── gwgen_environment.yml ├── environment.yml ├── command_line_api.py ├── install.rst ├── index.rst ├── getting_started.rst ├── Makefile └── conf.py ├── setup.cfg ├── tests ├── eecra_test_stations_short.dat ├── test_stations_short.dat ├── eecra_test_stations.dat ├── test_stations.dat ├── logging.yaml ├── conftest.py ├── test_main.py ├── test_evaluation.py ├── test_sensitivity_analysis.py ├── _base_testing.py ├── test_stations_long.dat └── test_parameterization.py ├── MANIFEST.in ├── ci ├── deploy_key.enc ├── environment_py2.7.yml ├── environment_py3.7.yml └── deploy.sh ├── gwgen ├── version.py ├── __init__.py ├── logging.yaml ├── parseghcnrow.py ├── parse_eecra.py ├── mo_parseghcnrow.f90 ├── preproc.py └── mo_parseeecra.f90 ├── readthedocs.yml ├── .gitmodules ├── CHANGELOG.rst ├── .gitignore ├── README.rst ├── .travis.yml ├── setup.py └── LICENSE /docs/_static/.gitignore: -------------------------------------------------------------------------------- 1 | prcp1.png 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [coverage:run] 2 | source = gwgen 3 | -------------------------------------------------------------------------------- /tests/eecra_test_stations_short.dat: -------------------------------------------------------------------------------- 1 | 1010 2 | 3502 3 | -------------------------------------------------------------------------------- /tests/test_stations_short.dat: -------------------------------------------------------------------------------- 1 | NOE00112080 2 | UKM00003502 3 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include gwgen/src/* 3 | include gwgen/data/* 4 | -------------------------------------------------------------------------------- /ci/deploy_key.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ARVE-Research/gwgen/HEAD/ci/deploy_key.enc -------------------------------------------------------------------------------- /gwgen/version.py: -------------------------------------------------------------------------------- 1 | """The version of the gwgen package""" 2 | 3 | __version__ = '1.0.3' 4 | -------------------------------------------------------------------------------- /tests/eecra_test_stations.dat: -------------------------------------------------------------------------------- 1 | 3980 2 | 11659 3 | 1028 4 | 1010 5 | 15480 6 | 3005 7 | 3162 8 | 3502 9 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. _changelog: 2 | 3 | Changelog 4 | ********* 5 | 6 | .. include:: ../CHANGELOG.rst 7 | -------------------------------------------------------------------------------- /readthedocs.yml: -------------------------------------------------------------------------------- 1 | conda: 2 | file: docs/environment.yml 3 | python: 4 | version: 2 5 | setup_py_install: true 6 | -------------------------------------------------------------------------------- /docs/fortran_api/gwgen.src.main.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.src.main: 2 | 3 | main program 4 | ============ 5 | 6 | .. f:autoprogram:: gwgen 7 | -------------------------------------------------------------------------------- /docs/fortran_api/gwgen.src.csv_file.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.src.csv_file: 2 | 3 | csv_file module 4 | =============== 5 | 6 | .. f:automodule:: csv_file 7 | -------------------------------------------------------------------------------- /tests/test_stations.dat: -------------------------------------------------------------------------------- 1 | EI000003980 2 | EZM00011659 3 | NO000099710 4 | NOE00112080 5 | ROE00108892 6 | UK000003005 7 | UK000003162 8 | UKM00003502 9 | -------------------------------------------------------------------------------- /docs/fortran_api/gwgen.src.geohashmod.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.src.geohashmod: 2 | 3 | geohashmod module 4 | ================= 5 | 6 | .. f:automodule:: geohashmod 7 | -------------------------------------------------------------------------------- /docs/api/gwgen.main.rst: -------------------------------------------------------------------------------- 1 | gwgen.main module 2 | ================= 3 | 4 | .. automodule:: gwgen.main 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/fortran_api/gwgen.src.parametersmod.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.src.parametersmod: 2 | 3 | parametersmod module 4 | ==================== 5 | 6 | .. f:automodule:: parametersmod 7 | -------------------------------------------------------------------------------- /docs/fortran_api/gwgen.src.randomdistmod.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.src.randomdistmod: 2 | 3 | randomdistmod module 4 | ==================== 5 | 6 | .. f:automodule:: randomdistmod 7 | -------------------------------------------------------------------------------- /docs/fortran_api/gwgen.src.weathergenmod.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.src.weathergenmod: 2 | 3 | weathergenmod module 4 | ==================== 5 | 6 | .. f:automodule:: weathergenmod 7 | -------------------------------------------------------------------------------- /docs/api/gwgen.utils.rst: -------------------------------------------------------------------------------- 1 | gwgen.utils module 2 | ================== 3 | 4 | .. automodule:: gwgen.utils 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/api/gwgen.preproc.rst: -------------------------------------------------------------------------------- 1 | gwgen.preproc module 2 | ==================== 3 | 4 | .. automodule:: gwgen.preproc 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/api/gwgen.evaluation.rst: -------------------------------------------------------------------------------- 1 | gwgen.evaluation module 2 | ======================= 3 | 4 | .. automodule:: gwgen.evaluation 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/api/gwgen.parse_eecra.rst: -------------------------------------------------------------------------------- 1 | gwgen.parse_eecra module 2 | ======================== 3 | 4 | .. automodule:: gwgen.parse_eecra 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/api/gwgen.parseghcnrow.rst: -------------------------------------------------------------------------------- 1 | gwgen.parseghcnrow module 2 | ========================= 3 | 4 | .. automodule:: gwgen.parseghcnrow 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.run.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.run: 2 | 3 | gwgen run 4 | ========= 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: run 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.info.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.info: 2 | 3 | gwgen info 4 | ========== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: info 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.init.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.init: 2 | 3 | gwgen init 4 | ========== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: init 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.setup.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.setup: 2 | 3 | gwgen setup 4 | =========== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: setup 11 | -------------------------------------------------------------------------------- /docs/api/gwgen.parameterization.rst: -------------------------------------------------------------------------------- 1 | gwgen.parameterization module 2 | ============================= 3 | 4 | .. automodule:: gwgen.parameterization 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.remove.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.remove: 2 | 3 | gwgen remove 4 | ============ 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: remove 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.archive.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.archive: 2 | 3 | gwgen archive 4 | ============= 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: archive 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.compile.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.compile: 2 | 3 | gwgen compile 4 | ============= 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: compile 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.sens.run.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.sens.run: 2 | 3 | gwgen sens run 4 | ============== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: sens run 11 | -------------------------------------------------------------------------------- /docs/api/gwgen.sensitivity_analysis.rst: -------------------------------------------------------------------------------- 1 | gwgen.sensitivity_analysis module 2 | ================================= 3 | 4 | .. automodule:: gwgen.sensitivity_analysis 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.bias.tmin.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.bias.tmin: 2 | 3 | gwgen bias tmin 4 | =============== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: bias tmin 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.bias.wind.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.bias.wind: 2 | 3 | gwgen bias wind 4 | =============== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: bias wind 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.configure.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.configure: 2 | 3 | gwgen configure 4 | =============== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: configure 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.del-value.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.del-value: 2 | 3 | gwgen del-value 4 | =============== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: del-value 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.get-value.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.get-value: 2 | 3 | gwgen get-value 4 | =============== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: get-value 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.day.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.day: 2 | 3 | gwgen param day 4 | =============== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param day 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.sens.init.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.sens.init: 2 | 3 | gwgen sens init 4 | =============== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: sens init 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.set-value.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.set-value: 2 | 3 | gwgen set-value 4 | =============== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: set-value 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.unarchive.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.unarchive: 2 | 3 | gwgen unarchive 4 | =============== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: unarchive 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.cday.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.cday: 2 | 3 | gwgen param cday 4 | ================ 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param cday 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.corr.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.corr: 2 | 3 | gwgen param corr 4 | ================ 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param corr 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.prcp.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.prcp: 2 | 3 | gwgen param prcp 4 | ================ 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param prcp 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.temp.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.temp: 2 | 3 | gwgen param temp 4 | ================ 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param temp 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.wind.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.wind: 2 | 3 | gwgen param wind 4 | ================ 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param wind 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.sens.setup.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.sens.setup: 2 | 3 | gwgen sens setup 4 | ================ 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: sens setup 11 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "tests/test_data"] 2 | path = tests/test_data 3 | url = https://github.com/ARVE-Research/gwgen_test_data.git 4 | [submodule "gwgen/src"] 5 | path = gwgen/src 6 | url = https://github.com/ARVE-Research/gwgen_f90.git 7 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.evaluate.ks.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.evaluate.ks: 2 | 3 | gwgen evaluate ks 4 | ================= 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: evaluate ks 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.cloud.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.cloud: 2 | 3 | gwgen param cloud 4 | ================= 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param cloud 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.month.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.month: 2 | 3 | gwgen param month 4 | ================= 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param month 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.sens.remove.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.sens.remove: 2 | 3 | gwgen sens remove 4 | ================= 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: sens remove 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.cmonth.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.cmonth: 2 | 3 | gwgen param cmonth 4 | ================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param cmonth 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.markov.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.markov: 2 | 3 | gwgen param markov 4 | ================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param markov 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.preproc.test.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.preproc.test: 2 | 3 | gwgen preproc test 4 | ================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: preproc test 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.sens.compile.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.sens.compile: 2 | 3 | gwgen sens compile 4 | ================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: sens compile 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.preproc.select.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.preproc.select: 2 | 3 | gwgen preproc select 4 | ==================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: preproc select 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.evaluate.output.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.evaluate.output: 2 | 3 | gwgen evaluate output 4 | ===================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: evaluate output 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.evaluate.quants.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.evaluate.quants: 2 | 3 | gwgen evaluate quants 4 | ===================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: evaluate quants 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.evaluate.prepare.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.evaluate.prepare: 2 | 3 | gwgen evaluate prepare 4 | ====================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: evaluate prepare 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.evaluate.quality.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.evaluate.quality: 2 | 3 | gwgen evaluate quality 4 | ====================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: evaluate quality 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.sens.evaluate.ks.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.sens.evaluate.ks: 2 | 3 | gwgen sens evaluate ks 4 | ====================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: sens evaluate ks 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.sens.plot.plot1d.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.sens.plot.plot1d: 2 | 3 | gwgen sens plot plot1d 4 | ====================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: sens plot plot1d 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.sens.plot.plot2d.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.sens.plot.plot2d: 2 | 3 | gwgen sens plot plot2d 4 | ====================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: sens plot plot2d 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.daily_cloud.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.daily_cloud: 2 | 3 | gwgen param daily_cloud 4 | ======================= 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param daily_cloud 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.yearly_cday.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.yearly_cday: 2 | 3 | gwgen param yearly_cday 4 | ======================= 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param yearly_cday 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.cdaily_cloud.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.cdaily_cloud: 2 | 3 | gwgen param cdaily_cloud 4 | ======================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param cdaily_cloud 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.hourly_cloud.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.hourly_cloud: 2 | 3 | gwgen param hourly_cloud 4 | ======================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param hourly_cloud 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.cmonthly_wind.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.cmonthly_wind: 2 | 3 | gwgen param cmonthly_wind 4 | ========================= 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param cmonthly_wind 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.monthly_cloud.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.monthly_cloud: 2 | 3 | gwgen param monthly_cloud 4 | ========================= 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param monthly_cloud 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.yearly_cmonth.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.yearly_cmonth: 2 | 3 | gwgen param yearly_cmonth 4 | ========================= 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param yearly_cmonth 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.cmonthly_cloud.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.cmonthly_cloud: 2 | 3 | gwgen param cmonthly_cloud 4 | ========================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param cmonthly_cloud 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.sens.evaluate.quants.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.sens.evaluate.quants: 2 | 3 | gwgen sens evaluate quants 4 | ========================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: sens evaluate quants 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.sens.evaluate.quality.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.sens.evaluate.quality: 2 | 3 | gwgen sens evaluate quality 4 | =========================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: sens evaluate quality 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.bias.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.bias: 2 | 3 | gwgen bias 4 | ========== 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | gwgen.bias.wind 10 | gwgen.bias.tmin 11 | 12 | .. argparse:: 13 | :module: gwgen.main 14 | :func: _get_parser 15 | :prog: gwgen 16 | :path: bias 17 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.yearly_cdaily_cloud.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.yearly_cdaily_cloud: 2 | 3 | gwgen param yearly_cdaily_cloud 4 | =============================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param yearly_cdaily_cloud 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.yearly_cmonthly_wind.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.yearly_cmonthly_wind: 2 | 3 | gwgen param yearly_cmonthly_wind 4 | ================================ 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param yearly_cmonthly_wind 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.yearly_cmonthly_cloud.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param.yearly_cmonthly_cloud: 2 | 3 | gwgen param yearly_cmonthly_cloud 4 | ================================= 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: param yearly_cmonthly_cloud 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.preproc.cloud.eecra_ghcn_map.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.preproc.cloud.eecra_ghcn_map: 2 | 3 | gwgen preproc cloud eecra_ghcn_map 4 | ================================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: preproc cloud eecra_ghcn_map 11 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.preproc.cloud.eecra_inventory.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.preproc.cloud.eecra_inventory: 2 | 3 | gwgen preproc cloud eecra_inventory 4 | =================================== 5 | 6 | .. argparse:: 7 | :module: gwgen.main 8 | :func: _get_parser 9 | :prog: gwgen 10 | :path: preproc cloud eecra_inventory 11 | -------------------------------------------------------------------------------- /docs/apigen.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # script to automatically generate the psyplot api documentation using 3 | # sphinx-apidoc and sed 4 | sphinx-apidoc -f -M -e -T -o api ../gwgen/ 5 | # replace chapter title in psyplot.rst 6 | sed -i '' -e 1,1s/.*/'Python API Reference'/ api/gwgen.rst 7 | sed -i '' -e 2,2s/.*/'===================='/ api/gwgen.rst 8 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.sens.plot.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.sens.plot: 2 | 3 | gwgen sens plot 4 | =============== 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | gwgen.sens.plot.plot2d 10 | gwgen.sens.plot.plot1d 11 | 12 | .. argparse:: 13 | :module: gwgen.main 14 | :func: _get_parser 15 | :prog: gwgen 16 | :path: sens plot 17 | -------------------------------------------------------------------------------- /gwgen/__init__.py: -------------------------------------------------------------------------------- 1 | """GWGEN: A WGEN-like weather generator for global daily weather 2 | """ 3 | from gwgen.version import __version__ 4 | 5 | __author__ = "Philipp Sommer and Jed Kaplan" 6 | __maintainer__ = "Philipp Sommer" 7 | __licence__ = "GPLv2" 8 | __email__ = "philipp.sommer@unil.ch" 9 | __copyright__ = "Copyright 2016, ARVE-Research" 10 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.preproc.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.preproc: 2 | 3 | gwgen preproc 4 | ============= 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | gwgen.preproc.select 10 | gwgen.preproc.cloud 11 | gwgen.preproc.test 12 | 13 | .. argparse:: 14 | :module: gwgen.main 15 | :func: _get_parser 16 | :prog: gwgen 17 | :path: preproc 18 | -------------------------------------------------------------------------------- /docs/gwgen_environment.yml: -------------------------------------------------------------------------------- 1 | name: gwgen 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - psyplot 6 | - dask 7 | - xarray 8 | - netCDF4 9 | - cartopy 10 | - seaborn 11 | - bottleneck 12 | - scipy<1.3 # for statsmodels compatibility 13 | - pip: 14 | - sqlalchemy 15 | - model-organization 16 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.preproc.cloud.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.preproc.cloud: 2 | 3 | gwgen preproc cloud 4 | =================== 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | gwgen.preproc.cloud.eecra_inventory 10 | gwgen.preproc.cloud.eecra_ghcn_map 11 | 12 | .. argparse:: 13 | :module: gwgen.main 14 | :func: _get_parser 15 | :prog: gwgen 16 | :path: preproc cloud 17 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.sens.evaluate.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.sens.evaluate: 2 | 3 | gwgen sens evaluate 4 | =================== 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | gwgen.sens.evaluate.quality 10 | gwgen.sens.evaluate.ks 11 | gwgen.sens.evaluate.quants 12 | 13 | .. argparse:: 14 | :module: gwgen.main 15 | :func: _get_parser 16 | :prog: gwgen 17 | :path: sens evaluate 18 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.evaluate.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.evaluate: 2 | 3 | gwgen evaluate 4 | ============== 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | gwgen.evaluate.quality 10 | gwgen.evaluate.output 11 | gwgen.evaluate.prepare 12 | gwgen.evaluate.ks 13 | gwgen.evaluate.quants 14 | 15 | .. argparse:: 16 | :module: gwgen.main 17 | :func: _get_parser 18 | :prog: gwgen 19 | :path: evaluate 20 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.sens.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.sens: 2 | 3 | gwgen sens 4 | ========== 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | gwgen.sens.setup 10 | gwgen.sens.compile 11 | gwgen.sens.init 12 | gwgen.sens.run 13 | gwgen.sens.evaluate 14 | gwgen.sens.plot 15 | gwgen.sens.remove 16 | 17 | .. argparse:: 18 | :module: gwgen.main 19 | :func: _get_parser 20 | :prog: gwgen 21 | :path: sens 22 | -------------------------------------------------------------------------------- /docs/api/gwgen.rst: -------------------------------------------------------------------------------- 1 | Python API Reference 2 | ==================== 3 | 4 | .. automodule:: gwgen 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | Submodules 10 | ---------- 11 | 12 | .. toctree:: 13 | 14 | gwgen.evaluation 15 | gwgen.main 16 | gwgen.parameterization 17 | gwgen.parse_eecra 18 | gwgen.parseghcnrow 19 | gwgen.preproc 20 | gwgen.sensitivity_analysis 21 | gwgen.utils 22 | 23 | -------------------------------------------------------------------------------- /docs/environment.yml: -------------------------------------------------------------------------------- 1 | name: gwgen-docs 2 | channels: 3 | - chilipp 4 | - conda-forge 5 | dependencies: 6 | - gcc 7 | - matplotlib 8 | - numpy 9 | - sphinx 10 | - seaborn 11 | - ipython 12 | - scipy<1.3 # for statsmodels compatibility 13 | - jupyter_client 14 | - psyplot 15 | - pip: 16 | - autodocsumm 17 | - sphinx-argparse 18 | - sphinxfortran 19 | - model-organization 20 | -------------------------------------------------------------------------------- /ci/environment_py2.7.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | channels: 3 | - conda-forge 4 | - chilipp 5 | dependencies: 6 | - python=2.7 7 | - scipy<1.3 # for statsmodels compatibility 8 | - matplotlib 9 | - dask 10 | - pip 11 | - xarray 12 | - numpy 13 | - netcdf4 14 | - seaborn 15 | - bottleneck 16 | - statsmodels 17 | - docrep 18 | - model-organization 19 | - f90nml 20 | - cartopy 21 | - sqlalchemy 22 | - psycopg2 23 | - psyplot 24 | - psy-simple 25 | - psy-reg 26 | -------------------------------------------------------------------------------- /ci/environment_py3.7.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | channels: 3 | - conda-forge 4 | - chilipp 5 | dependencies: 6 | - python=3.7 7 | - scipy<1.3 # for statsmodels compatibility 8 | - matplotlib 9 | - pip 10 | - dask 11 | - xarray 12 | - numpy 13 | - netcdf4 14 | - seaborn 15 | - bottleneck 16 | - statsmodels 17 | - docrep 18 | - model-organization 19 | - f90nml 20 | - cartopy 21 | - sqlalchemy 22 | - psycopg2 23 | - psyplot 24 | - psy-simple 25 | - psy-reg 26 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.rst: -------------------------------------------------------------------------------- 1 | .. _Command.Line.API.Reference: 2 | 3 | Command Line API Reference 4 | ========================== 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | gwgen.setup 10 | gwgen.compile 11 | gwgen.init 12 | gwgen.set-value 13 | gwgen.get-value 14 | gwgen.del-value 15 | gwgen.info 16 | gwgen.unarchive 17 | gwgen.configure 18 | gwgen.preproc 19 | gwgen.param 20 | gwgen.run 21 | gwgen.evaluate 22 | gwgen.bias 23 | gwgen.sens 24 | gwgen.archive 25 | gwgen.remove 26 | 27 | .. argparse:: 28 | :module: gwgen.main 29 | :func: _get_parser 30 | :prog: gwgen 31 | -------------------------------------------------------------------------------- /gwgen/logging.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # debug logging settings (sets the level of the nc2map logger to DEBUG) 3 | 4 | version: 1 5 | 6 | disable_existing_loggers: False 7 | 8 | formatters: 9 | 10 | simple: 11 | 12 | format: "[%(name)s] - %(levelname)s - %(message)s" 13 | 14 | level_message: 15 | 16 | format: "%(levelname)s: %(message)s" 17 | 18 | full: 19 | format: "%(asctime)s - [%(name)s.%(funcName)s] - %(levelname)s - %(message)s" 20 | 21 | 22 | handlers: 23 | 24 | console: 25 | 26 | class: logging.StreamHandler 27 | 28 | level: DEBUG 29 | 30 | formatter: full 31 | 32 | stream: ext://sys.stdout 33 | 34 | loggers: 35 | 36 | gwgen: 37 | 38 | handlers: [console] 39 | 40 | propagate: False 41 | 42 | level: INFO 43 | 44 | ... -------------------------------------------------------------------------------- /tests/logging.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # debug logging settings (sets the level of the nc2map logger to DEBUG) 3 | 4 | version: 1 5 | 6 | disable_existing_loggers: False 7 | 8 | formatters: 9 | 10 | simple: 11 | 12 | format: "[%(name)s] - %(levelname)s - %(message)s" 13 | 14 | level_message: 15 | 16 | format: "%(levelname)s: %(message)s" 17 | 18 | full: 19 | format: "%(asctime)s - [%(name)s.%(funcName)s] - %(levelname)s - %(message)s" 20 | 21 | 22 | handlers: 23 | 24 | console: 25 | 26 | class: logging.StreamHandler 27 | 28 | level: DEBUG 29 | 30 | formatter: full 31 | 32 | stream: ext://sys.stdout 33 | 34 | loggers: 35 | 36 | gwgen: 37 | 38 | handlers: [console] 39 | 40 | propagate: False 41 | 42 | level: DEBUG 43 | 44 | ... 45 | -------------------------------------------------------------------------------- /docs/command_line/gwgen.param.rst: -------------------------------------------------------------------------------- 1 | .. _gwgen.param: 2 | 3 | gwgen param 4 | =========== 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | gwgen.param.hourly_cloud 10 | gwgen.param.daily_cloud 11 | gwgen.param.monthly_cloud 12 | gwgen.param.cdaily_cloud 13 | gwgen.param.cmonthly_cloud 14 | gwgen.param.yearly_cdaily_cloud 15 | gwgen.param.corr 16 | gwgen.param.cmonthly_wind 17 | gwgen.param.wind 18 | gwgen.param.yearly_cmonthly_wind 19 | gwgen.param.cloud 20 | gwgen.param.yearly_cmonthly_cloud 21 | gwgen.param.day 22 | gwgen.param.month 23 | gwgen.param.cday 24 | gwgen.param.temp 25 | gwgen.param.markov 26 | gwgen.param.prcp 27 | gwgen.param.yearly_cday 28 | gwgen.param.yearly_cmonth 29 | gwgen.param.cmonth 30 | 31 | .. argparse:: 32 | :module: gwgen.main 33 | :func: _get_parser 34 | :prog: gwgen 35 | :path: param 36 | -------------------------------------------------------------------------------- /docs/fortran_api/index.rst: -------------------------------------------------------------------------------- 1 | .. _fortran_api: 2 | 3 | Fortran API Reference 4 | ===================== 5 | This section documents the real weather generator which is written as a 6 | FORTRAN program. All the FORTRAN files, including the Makefile are in the 7 | ``src`` directory of the ``gwgen`` package. If you initialize a new project, 8 | those files are copied to the ``'/src'`` directory. 9 | 10 | To compile the source code manually, you may go to the source directory and 11 | simply run:: 12 | 13 | make all 14 | 15 | This will create an executable called ``'weathergen'`` in the same directory. 16 | However, make sure that the right compiler is chosen in the ``Makefile`` ( 17 | by default: ``gfortran``). 18 | 19 | This API reference is created automatically from the FORTRAN source code using 20 | sphinx-fortran_ and is subject to further improvements. 21 | 22 | .. _sphinx-fortran: https://github.com/VACUMM/sphinx-fortran 23 | 24 | 25 | Submodules 26 | ---------- 27 | 28 | .. toctree:: 29 | 30 | gwgen.src.main 31 | gwgen.src.weathergenmod 32 | gwgen.src.randomdistmod 33 | gwgen.src.csv_file 34 | gwgen.src.geohashmod 35 | gwgen.src.parametersmod 36 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | v1.0.3 2 | ====== 3 | Compatibility fixes for pandas and dask, and bug fix for random number generator based on https://github.com/ARVE-Research/gwgen_f90/pull/1 4 | 5 | v1.0.2 6 | ====== 7 | Final version for Geoscientific Model Development 8 | 9 | Changed 10 | ------- 11 | * Minor changes in LICENSE file 12 | 13 | v1.0.1 14 | ====== 15 | Added 16 | ----- 17 | * Added changelog 18 | 19 | Changed 20 | ------- 21 | * Changed parameterization of temperatures and winds standard deviation 22 | * Implemented default parameterization based on `Sommer and Kaplan, 2017`_ 23 | * Several bug fixes in ``'mo_parseghcnrow.f90'`` 24 | * To decrease the size of the github repository, the old repository of 25 | GWGEN v1.0.0 has been deleted and recreated. These files can be accessed 26 | through the gwgen_old_ repository 27 | * The FORTRAN src files have been moved to an own repository, the gwgen_f90_ 28 | repository, and are not implemented as a submodule. Hence, when cloning the 29 | gwgen repository from github, one has to execute:: 30 | 31 | git submodule update --init gwgen/src 32 | 33 | .. _Sommer and Kaplan, 2017: https://doi.org/10.5194/gmd-2017-42 34 | .. _gwgen_old: https://github.com/ARVE-Research/gwgen_old 35 | .. _gwgen_f90: https://github.com/ARVE-Research/gwgen_f90 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | *.so.dSYM/ 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .pytest_cache 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | docs/index.doctree 58 | 59 | # PyBuilder 60 | target/ 61 | 62 | # Spyder project 63 | .spyderproject 64 | 65 | # Example ipython notebook checkpoints 66 | *.ipynb_checkpoints/ 67 | 68 | # Fortran 69 | *.mod 70 | *.o 71 | 72 | # test files 73 | tests/ghcn/ghcnd-stations.txt 74 | ci/recipe/gwgen/meta.yaml 75 | gwgen-conda 76 | gwgen-conda-*.sh 77 | -------------------------------------------------------------------------------- /ci/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e # Exit with nonzero exit code if anything fails 3 | set -x 4 | 5 | TARGET_BRANCH="gh-pages" 6 | 7 | SSH_REPO=${REPO/https:\/\/github.com\//git@github.com:} 8 | SHA=`git rev-parse --verify HEAD` 9 | 10 | # Get the deploy key by using Travis's stored variables to decrypt deploy_key.enc 11 | set +x 12 | ENCRYPTED_KEY_VAR="encrypted_${ENCRYPTION_LABEL}_key" 13 | ENCRYPTED_IV_VAR="encrypted_${ENCRYPTION_LABEL}_iv" 14 | ENCRYPTED_KEY=${!ENCRYPTED_KEY_VAR} 15 | ENCRYPTED_IV=${!ENCRYPTED_IV_VAR} 16 | openssl aes-256-cbc -K $ENCRYPTED_KEY -iv $ENCRYPTED_IV -in ci/deploy_key.enc -out deploy_key -d 17 | set -x 18 | chmod 600 deploy_key 19 | eval `ssh-agent -s` 20 | ssh-add deploy_key 21 | 22 | # Now let's go have some fun with the cloned repo 23 | cd deploy 24 | git config user.name "Travis" 25 | git config user.email "$COMMIT_AUTHOR_EMAIL" 26 | 27 | # If there are no changes to the compiled out (e.g. this is a README update) then just bail. 28 | if [ -z `git diff --exit-code` ]; then 29 | echo "No changes to the output on this push; exiting." 30 | exit 0 31 | fi 32 | 33 | # Commit the "changes", i.e. the new version. 34 | # The delta will show diffs between new and old versions. 35 | git add . 36 | git commit -m "Deploy to GitHub Pages: ${SHA}" 37 | 38 | # Now that we're all set up, we can push. 39 | git push $SSH_REPO $TARGET_BRANCH -------------------------------------------------------------------------------- /docs/command_line_api.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """Script to generate the command line api""" 4 | import os 5 | import os.path as osp 6 | from gwgen.main import _get_parser 7 | from psyplot.docstring import dedents 8 | 9 | 10 | def document_parsers(parser, base_key='gwgen'): 11 | f = open(osp.join(api_dir, base_key + '.rst'), 'w') 12 | if len(base_key.split('.')) == 1: 13 | title = 'Command Line API Reference' 14 | else: 15 | title = base_key.replace('.', ' ') 16 | 17 | f.write('.. _' + title.replace(' ', '.') + ':\n\n') 18 | f.write(title + '\n') 19 | f.write('=' * len(title) + '\n\n') 20 | 21 | if parser._subparsers: 22 | f.write(dedents(""" 23 | .. toctree:: 24 | :maxdepth: 1""") + '\n\n') 25 | for key, p in parser._subparsers_action.choices.items(): 26 | sp_key = base_key + '.' + key 27 | f.write(' ' + sp_key + '\n') 28 | document_parsers(p, sp_key) 29 | f.write('\n') 30 | path = base_key.split('.') 31 | if len(path) == 1: 32 | path = '' 33 | else: 34 | path = ':path: ' + ' '.join(path[1:]) 35 | f.write(dedents(""" 36 | .. argparse:: 37 | :module: gwgen.main 38 | :func: _get_parser 39 | :prog: gwgen 40 | """ + path) + '\n') 41 | f.close() 42 | 43 | 44 | parser = _get_parser() 45 | 46 | api_dir = 'command_line' 47 | 48 | if not osp.exists(api_dir): 49 | os.makedirs(api_dir) 50 | 51 | document_parsers(parser) 52 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | 2 | def pytest_addoption(parser): 3 | group = parser.getgroup("gwgen", "GWGEN specific options") 4 | group.addoption('--offline', help="Block outgoing internet connections", 5 | action='store_true') 6 | group.addoption('--no-db', help="Don't test postgres databases", 7 | action='store_true') 8 | group.addoption('--database', 9 | help=("The name of the postgres database to use. Default: " 10 | "%(default)s"), default='travis_ci_test') 11 | group.addoption('--user', 12 | help="The username to access the postgres database") 13 | group.addoption('--host', 14 | help=("The hostname for the postgres database. Default: " 15 | "%(default)s"), default='127.0.0.1') 16 | group.addoption('--port', help="The port of the postgres database.") 17 | group.addoption('--no-remove', action='store_true', 18 | help=("Do not remove the test directory at the end of the " 19 | "tests.")) 20 | group.addoption('--nprocs', type=int, 21 | help=("The number of processors to start.")) 22 | group.addoption('--serial', action='store_true', 23 | help="Run tests in serial") 24 | 25 | 26 | def pytest_configure(config): 27 | import _base_testing as bt 28 | if config.getoption('offline'): 29 | bt.online = False 30 | if config.getoption('no_db'): 31 | bt.BaseTest.use_db = False 32 | for option in ['database', 'user', 'host', 'port']: 33 | bt.db_config[option] = config.getoption(option) 34 | if config.getoption('no_remove'): 35 | bt.BaseTest.remove_at_cleanup = False 36 | bt.BaseTest.serial = config.getoption('serial') 37 | bt.BaseTest.nprocs = config.getoption('nprocs') 38 | -------------------------------------------------------------------------------- /gwgen/parseghcnrow.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from gwgen._parseghcnrow import parseghcnrow 3 | import six 4 | import pandas as pd 5 | import numpy as np 6 | from itertools import chain 7 | import datetime as dt 8 | import re 9 | 10 | if six.PY2: 11 | from itertools import imap as map, izip as zip 12 | 13 | daymon_patt = re.compile(r'(?:\w|-){11}(\d{6})(?:TMAX|TMIN|PRCP)') 14 | 15 | 16 | def read_ghcn_file(ifile): 17 | """Read in a GHCN station data file and convert it to a dataframe 18 | 19 | Parameters 20 | ---------- 21 | ifile: str 22 | The path to a ghcn datafile 23 | 24 | Returns 25 | ------- 26 | pandas.DataFrame 27 | The `ifile` converted to a dataframe""" 28 | # get number of days in the file 29 | with open(ifile) as f: 30 | ndays = np.sum(list(map(ndaymon, np.unique(list(map( 31 | lambda m: m.group(1), filter( 32 | None, map(daymon_patt.match, f.readlines())))))))) 33 | stationid, dates, variables, flags, j = parseghcnrow.parse_station( 34 | ifile, ndays or 100) 35 | dates = dates[:j] 36 | variables = variables[:j].astype(np.float64) 37 | flags = flags[:j].astype(np.str_) 38 | flags = np.core.defchararray.replace(flags, ' ', '') 39 | variables[np.isclose(variables, -9999.) | 40 | np.isclose(variables, -999.9)] = np.nan 41 | vlst = ['tmin', 'tmax', 'prcp'] 42 | df = pd.DataFrame.from_dict(dict(chain( 43 | [('id', np.repeat(np.array([stationid]).astype(np.str_), j))], 44 | zip(('year', 'month', 'day'), dates.T), 45 | zip(vlst, variables.T), 46 | chain(*[zip((var + '_m', var + '_q', var + '_s'), arr) 47 | for var, arr in zip(vlst, np.rollaxis(flags, 2, 1).T)])))) 48 | return df 49 | 50 | 51 | def ndaymon(yearmon): 52 | """Calculate the number of days for one month in a year 53 | 54 | Parameters 55 | ---------- 56 | yearmon: str 57 | The first 4 numbers stand for the year, the others for the month 58 | (in datetime writing: ``'%Y%m'``)""" 59 | year = int(yearmon[:4]) 60 | month = int(yearmon[4:]) 61 | d = dt.date(year, month, 1) 62 | d2 = d.replace( 63 | year=year + 1 if month == 12 else year, 64 | month=1 if month == 12 else month + 1) 65 | return (d2 - d).days 66 | -------------------------------------------------------------------------------- /tests/test_main.py: -------------------------------------------------------------------------------- 1 | """Test module for the :mod:`gwgen.main` module""" 2 | import os.path as osp 3 | import unittest 4 | from gwgen.utils import file_len 5 | import _base_testing as bt 6 | 7 | 8 | class OrganizerTest(bt.BaseTest): 9 | """Test the :class:`gwgen.main.ModuleOrganizer` class""" 10 | 11 | def test_compile_model(self): 12 | """Test the compilation of a model""" 13 | self.organizer.setup(self.test_dir) 14 | projectname = self.organizer.projectname 15 | self.organizer.compile_model() 16 | binpath = osp.join(self.test_dir, projectname, 'bin', 'weathergen') 17 | self.assertTrue(osp.exists(binpath), 18 | msg='binary %s does not exist!' % binpath) 19 | 20 | def test_run(self): 21 | self._test_init() 22 | organizer = self.organizer 23 | exp = organizer.experiment 24 | self.organizer.run(ifile=osp.join( 25 | bt.test_root, 'test_data', 'input.csv')) 26 | fname = osp.join(self.test_dir, organizer.projectname, 'experiments', 27 | exp, 'outdata', exp + '.csv') 28 | self.assertTrue(osp.exists(fname), msg='Output file %s not found!' % ( 29 | fname)) 30 | nlines = file_len(fname) 31 | self.assertGreater(nlines, 2, msg='No output generated!') 32 | 33 | def test_wind_bias_correction(self): 34 | """Test gwgen bias wind""" 35 | self._test_init() 36 | self.organizer.exp_config['reference'] = osp.join( 37 | bt.test_root, 'test_data', 'long_reference.csv') 38 | fname, ext = osp.splitext(self.stations_file) 39 | self.stations_file = osp.join(bt.test_root, 'test_stations_long.dat') 40 | self.organizer.exp_config['eval_stations'] = self.stations_file 41 | ifile = osp.join(bt.test_root, 'test_data', 'long_input.csv') 42 | self.organizer.exp_config['namelist'] = { 43 | 'weathergen_ctl': {'wind_bias_coeffs': [1.0] + [0.0] * 5, 44 | 'wind_intercept_bias_a': -9999., 45 | 'wind_intercept_bias_b': -9999.}, 46 | 'main_ctl': {}} 47 | self.organizer.parse_args(['run', '-i', ifile]) 48 | self.organizer.parse_args('bias -q 1-100-5,99 wind'.split()) 49 | self.organizer.fix_paths(self.organizer.exp_config) 50 | ofile = self.organizer.exp_config['postproc']['bias']['wind'][ 51 | 'plot_file'] 52 | self.assertTrue(osp.exists(ofile), msg=ofile + ' is missing') 53 | 54 | 55 | if __name__ == '__main__': 56 | unittest.main() 57 | -------------------------------------------------------------------------------- /tests/test_evaluation.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Test module for the parameterization. Note that the entire task framework is 3 | very sensible to internal errors , so if anything goes wrong, there should be 4 | an error within the setup""" 5 | import unittest 6 | import os.path as osp 7 | import pandas as pd 8 | import pytest 9 | # we use assert_frame_equal instead of pd.equal, because it is less strict 10 | from pandas.util.testing import assert_frame_equal 11 | import _base_testing as bt 12 | import test_parameterization as tp 13 | 14 | 15 | #: Message to provide a bit more information on the two data frames 16 | df_diff_msg = ( 17 | 'Task data and reference differ!\n{0}\n' 18 | 'Task\n%s\n{0}\n' 19 | 'Reference\n%s{0}').format('-' * 80) 20 | 21 | 22 | def df_equals(df, df_ref, *args, **kwargs): 23 | """Simple wrapper around assert_frame_equal to use unittests assertion 24 | 25 | Parameters 26 | ---------- 27 | df: pd.DataFrame 28 | The simulation data frame 29 | df_ref: pd.DataFrame 30 | The reference data frame 31 | 32 | Returns 33 | ------- 34 | None or Exception 35 | Either None if everything went fine, otherwise the raised Exception""" 36 | try: 37 | assert_frame_equal(df, df_ref, *args, **kwargs) 38 | except Exception as e: 39 | return e 40 | 41 | 42 | class EvaluationPreparationTest(bt.BaseTest): 43 | """Test case for the :class:`gwgen.evaluation.EvaluationPreparation` class 44 | """ 45 | 46 | @pytest.mark.long 47 | def test_setup(self): 48 | """Test the setup from scratch""" 49 | self._test_init() 50 | organizer = self.organizer 51 | organizer.evaluate(self.stations_file, prepare={}, to_csv=True) 52 | ifile1 = osp.join(self.test_dir, organizer.projectname, 'experiments', 53 | organizer.experiment, 'input', 'input.csv') 54 | self.assertTrue(osp.exists(ifile1), msg=ifile1 + ' is missing!') 55 | ref1 = osp.join(self.test_dir, organizer.projectname, 'experiments', 56 | organizer.experiment, 'evaluation', 'reference.csv') 57 | self.assertTrue(osp.exists(ref1), msg=ref1 + ' is missing!') 58 | index_cols = ['id', 'year', 'month', 'day'] 59 | 60 | # test for equality 61 | df1 = pd.read_csv(ifile1, index_col=index_cols[:-1]) 62 | ifile2 = osp.join(bt.test_root, 'test_data', 'input.csv') 63 | df2 = pd.read_csv(ifile2, index_col=index_cols[:-1]) 64 | merged = df2[[]].merge(df1, left_index=True, right_index=True, 65 | how='left') 66 | result = tp.df_equals(merged.sort_index(), df2.sort_index()) 67 | self.assertIsNone(result, msg=result) 68 | 69 | df1 = pd.read_csv(ref1, index_col=index_cols) 70 | ref2 = osp.join(bt.test_root, 'test_data', 'reference.csv') 71 | df2 = pd.read_csv(ref2, index_col=index_cols) 72 | merged = df2[[]].merge(df1, left_index=True, right_index=True, 73 | how='left') 74 | result = tp.df_equals(merged.sort_index(), df2.sort_index()) 75 | self.assertIsNone(result, msg=result) 76 | 77 | 78 | if __name__ == '__main__': 79 | unittest.main() 80 | -------------------------------------------------------------------------------- /gwgen/parse_eecra.py: -------------------------------------------------------------------------------- 1 | # -*- codng: utf-8 -*- 2 | import re 3 | import os.path as osp 4 | import numpy as np 5 | import pandas as pd 6 | from collections import OrderedDict 7 | from gwgen.utils import file_len 8 | from gwgen._parseeecra import parseeecra 9 | 10 | names = [ 11 | 'year', 'month', 'day', 'hour', 12 | 'IB', 13 | 'lat', 14 | 'lon', 15 | 'station_id', 16 | 'LO', 17 | 'ww', 18 | 'N', 19 | 'Nh', 20 | 'h', 21 | 'CL', 22 | 'CM', 23 | 'CH', 24 | 'AM', 25 | 'AH', 26 | 'UM', 27 | 'UH', 28 | 'IC', 29 | 'SA', 30 | 'RI', 31 | 'SLP', 32 | 'WS', 33 | 'WD', 34 | 'AT', 35 | 'DD', 36 | 'EL', 37 | 'IW', 38 | 'IP'] 39 | 40 | 41 | def parse_file(ifile, year=None): 42 | """Parse a raw data file from EECRA and as a pandas DataFrame 43 | 44 | Parameters 45 | ---------- 46 | ifile: str 47 | The raw (uncompressed) data file 48 | year: int 49 | The first year in the data file 50 | 51 | Returns 52 | ------- 53 | pandas.DataFrame 54 | `ifile` parsed into a dataframe""" 55 | if year is None: 56 | m = re.match(r'\w{3}(\d{2})L', osp.basename(ifile)) 57 | if not m: 58 | raise TypeError( 59 | "Could not infer year of file %s! Use the 'year' " 60 | "parameter!" % (ifile, )) 61 | year = int(m.group(1)) 62 | year += 1900 if year > 60 else 2000 63 | df = pd.DataFrame.from_dict(OrderedDict( 64 | zip(names, parseeecra.parse_file(ifile, year, file_len(ifile))))) 65 | return df 66 | 67 | 68 | def extract_data(ids, src_dir, target_dir, years=range(1971, 2010), 69 | imonths=range(1, 13)): 70 | """Extract the data for the given EECRA stations 71 | 72 | This function extracts the data for the given `ids` from the EECRA data 73 | base stored in `src_dir` into one file for each *id* in `ids`. The 74 | resulting filename will be like *id.csv*. 75 | 76 | Parameters 77 | ---------- 78 | ids: np.ndarray of dtype int 79 | The numpy integer array with the station ids to extract 80 | src_dir: str 81 | The path to the source directory containing the raw (uncompressed) 82 | EECRA database 83 | target_dir: str 84 | The path to the output directory 85 | years: np.ndarray of dtype int 86 | The numpy integer array with the years to extract (by default, all 87 | years between 1971 and 2010) 88 | imonths: np.ndarray of dtype int 89 | The numpy integer array with the months to extract (by default, all 90 | from january to december) 91 | 92 | Returns 93 | ------- 94 | numpy.ndarray 95 | The paths of the filenames corresponding to ids""" 96 | ids = np.asarray(ids).astype(int) 97 | years = np.asarray(years).astype(int) 98 | imonths = np.asarray(imonths).astype(int) 99 | for arr in [ids, years, imonths]: 100 | if arr.ndim == 0: 101 | arr.reshape((1,)) 102 | parseeecra.extract_data( 103 | ids, osp.join(src_dir, ''), osp.join(target_dir, ''), years, imonths) 104 | return np.array([osp.join(src_dir, str(station_id) + '.csv') 105 | for station_id in ids]) 106 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | GWGEN: A global weather generator for daily data 2 | ================================================ 3 | 4 | 5 | .. start-badges 6 | 7 | .. list-table:: 8 | :stub-columns: 1 9 | :widths: 10 90 10 | 11 | * - tests 12 | - |travis| |coveralls| 13 | * - package 14 | - |version| |supported-versions| |supported-implementations| 15 | 16 | .. |travis| image:: https://travis-ci.org/ARVE-Research/gwgen.svg?branch=master 17 | :alt: Travis 18 | :target: https://travis-ci.org/ARVE-Research/gwgen 19 | 20 | .. |coveralls| image:: https://coveralls.io/repos/github/ARVE-Research/gwgen/badge.svg?branch=master 21 | :alt: Coverage 22 | :target: https://coveralls.io/github/ARVE-Research/gwgen?branch=master 23 | 24 | .. |version| image:: https://img.shields.io/pypi/v/gwgen.svg?style=flat 25 | :alt: PyPI Package latest release 26 | :target: https://pypi.python.org/pypi/gwgen 27 | 28 | .. |supported-versions| image:: https://img.shields.io/pypi/pyversions/gwgen.svg?style=flat 29 | :alt: Supported versions 30 | :target: https://pypi.python.org/pypi/gwgen 31 | 32 | .. |supported-implementations| image:: https://img.shields.io/pypi/implementation/gwgen.svg?style=flat 33 | :alt: Supported implementations 34 | :target: https://pypi.python.org/pypi/gwgen 35 | 36 | .. end-badges 37 | 38 | 39 | Welcome! This synthesis of FORTRAN and Python is a globally applicable 40 | weather generator inspired by the original WGEN weather generator of 41 | [Richardson1981]_ and parameterized through a global dataset of [GHCN]_ and 42 | [EECRA]_ data. A paper with the scientific documentation is in progress, the 43 | `technical documentation`_ with more information is hosted on github.io. 44 | 45 | .. _technical documentation: https://arve-research.github.io/gwgen/ 46 | 47 | 48 | Authors 49 | ------- 50 | This package has been developped by `Philipp S. Sommer`_ and `Jed O. Kaplan`_. 51 | 52 | Citing GWGEN 53 | ------------ 54 | When using GWGEN, we kindly ask you to provide a reference to the corresponding 55 | paper published in Geoscientific Model Development: 56 | 57 | Sommer, P. S. and Kaplan, J. O.: *A globally calibrated scheme for 58 | generating daily meteorology from monthly statistics: Global-WGEN (GWGEN) 59 | v1.0*, Geosci. Model Dev., 10, 3771-3791, DOI: `10.5194/gmd-10-3771-2017`_, 60 | 2017. 61 | 62 | .. _10.5194/gmd-10-3771-2017: https://doi.org/10.5194/gmd-10-3771-2017 63 | 64 | 65 | Acknowledgements 66 | ---------------- 67 | This work was supported by the European Research Council (COEVOLVE, 313797) and 68 | the Swiss National Science Foundation (ACACIA, CR10I2\_146314). We thank 69 | `Shawn Koppenhoefer`_ for assistance compiling and querying the weather databases and 70 | Alexis Berne and Grégoire Mariéthoz for helpful suggestions on the analyses. We 71 | are grateful to NOAA NCDC and the University of Washingtion for providing free 72 | of charge the GHCN-Daily and EECRA databases, respectively. 73 | 74 | .. _Philipp S. Sommer: https://github.com/Chilipp 75 | .. _Jed O. Kaplan: https://github.com/jedokaplan 76 | .. _Shawn Koppenhoefer: http://arve.unil.ch/people/shawn-koppenhoefer/ 77 | 78 | 79 | References 80 | ---------- 81 | .. [Richardson1981] Richardson, C. W.: *Stochastic simulation of daily 82 | precipitation, temperature, and solar radiation*, Water Resources Research, 83 | 17, 182–190, doi:10.1029/WR017i001p00182, 1981. 84 | .. [GHCN] T. G.: Global Historical Climatology Network - Daily (GHCN-Daily), 85 | Version 3.22, doi:10.7289/V5D21VHZ, doi:10.7289/V5D21VHZ, 2012 86 | .. [EECRA] Hahn, C. and Warren, S.: *Extended Edited Synoptic Cloud Reports from 87 | Ships and Land Stations Over the Globe*, 1952-1996 (with Ship data 88 | updated through 2008), doi:10.3334/CDIAC/cli.ndp026c, 1999 89 | -------------------------------------------------------------------------------- /docs/install.rst: -------------------------------------------------------------------------------- 1 | .. _install: 2 | 3 | 4 | Installation 5 | ============ 6 | 7 | 8 | .. _install_fortran: 9 | 10 | Installing only the FORTRAN program 11 | ----------------------------------- 12 | If you are only interested in the FORTRAN source files for the weather 13 | generator, checkout the gwgen_f90_ repository. 14 | 15 | It is implemented as a submodule in this repository in the src_ directory 16 | 17 | .. _src: https://github.com/ARVE-Research/gwgen/blob/master/gwgen/src 18 | .. _gwgen_f90: https://github.com/ARVE-Research/gwgen_f90 19 | .. _Github: https://github.com/ARVE-Research/gwgen 20 | 21 | .. _install_full: 22 | 23 | Installing the full GWGEN 24 | ------------------------- 25 | If you not only want the source code, but also the 26 | :ref:`parameterization ` or the experiment organization 27 | features of the model, you need the full python package which includes the 28 | FORTRAN source files. 29 | 30 | The code is hosted open-source on Github_ and can be downloaded via 31 | 32 | .. code-block:: bash 33 | 34 | git clone https://github.com/ARVE-Research/gwgen.git 35 | 36 | or as a zipped archive directly from Github_. 37 | 38 | Installing the requirements 39 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 40 | You need a FORTRAN 95 compiler (see :ref:`install_fortran`). Furthermore, you 41 | need python and the following python packages: 42 | 43 | - model-organization_: For the command line utility and the experiments 44 | organization 45 | - matplotlib_, seaborn_ and psyplot_: For the visualization 46 | - xarray_, statsmodels_, `numpy and scipy`_: For the calculations 47 | - dask_ (only necessary for the 48 | :class:`cross correlation ` 49 | parameterization task) 50 | - cartopy_: For the Kolmogorov-Smirnoff 51 | (:class:`ks `) evaluation task. 52 | 53 | The recommended way to install these requirements is using conda_. We provide 54 | two methodologies here 55 | 56 | 1. Installing into an existing conda distribution 57 | ************************************************* 58 | If you already have conda_ installed, we recommend you just use 59 | :download:`this conda environment file ` and create a 60 | new virtual environment via 61 | 62 | .. code-block:: bash 63 | 64 | conda env create -f gwgen_environment.yml 65 | source activate gwgen 66 | 67 | 2. Installation including conda 68 | ******************************* 69 | With every new ``gwgen`` release, we also provide executables to install 70 | conda and gwgen on our releases_ page. This is probably the easiest way to 71 | install it. 72 | 73 | After selecting the right file for your operating system (MacOS or Linux), you 74 | can simply install it via 75 | 76 | .. code-block:: bash 77 | 78 | bash 79 | 80 | and follow the instructions. 81 | 82 | .. _model-organization: http://model-organization.readthedocs.io/en/latest/ 83 | .. _psyplot: http://psyplot.readthedocs.io/en/latest/ 84 | .. _numpy and scipy: https://docs.scipy.org/doc/ 85 | .. _statsmodels: http://statsmodels.sourceforge.net/ 86 | .. _matplotlib: http://matplotlib.org/ 87 | .. _xarray: http://xarray.pydata.org/en/stable/ 88 | .. _seaborn: http://seaborn.pydata.org/ 89 | .. _dask: http://dask.pydata.org/en/latest/ 90 | .. _cartopy: http://scitools.org.uk/cartopy/ 91 | .. _conda: https://www.continuum.io/downloads 92 | .. _releases: https://github.com/ARVE-Research/gwgen/releases 93 | 94 | 95 | Installing GWGEN 96 | ~~~~~~~~~~~~~~~~ 97 | After having successfully installed python, just install the gwgen package via 98 | 99 | .. code-block:: bash 100 | 101 | python setup.py install 102 | 103 | You can test whether it was successfully installed by typing:: 104 | 105 | gwgen -h 106 | 107 | .. note:: 108 | 109 | If you download the repository from Github_ via ``git clone``, you have to 110 | initialize the ``src`` submodule via:: 111 | 112 | git submodule update --init gwgen/src 113 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. gwgen documentation master file, created by 2 | sphinx-quickstart on Mon Jul 20 18:01:33 2015. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | GWGEN: A global weather generator for daily data 7 | ================================================ 8 | 9 | .. start-badges 10 | 11 | .. list-table:: 12 | :stub-columns: 1 13 | :widths: 10 90 14 | 15 | * - tests 16 | - |travis| |coveralls| 17 | * - package 18 | - |version| |supported-versions| |supported-implementations| 19 | 20 | .. |travis| image:: https://travis-ci.org/ARVE-Research/gwgen.svg?branch=master 21 | :alt: Travis 22 | :target: https://travis-ci.org/ARVE-Research/gwgen 23 | 24 | .. |coveralls| image:: https://coveralls.io/repos/github/ARVE-Research/gwgen/badge.svg?branch=master 25 | :alt: Coverage 26 | :target: https://coveralls.io/github/ARVE-Research/gwgen?branch=master 27 | 28 | .. |version| image:: https://img.shields.io/pypi/v/gwgen.svg?style=flat 29 | :alt: PyPI Package latest release 30 | :target: https://pypi.python.org/pypi/gwgen 31 | 32 | .. |supported-versions| image:: https://img.shields.io/pypi/pyversions/gwgen.svg?style=flat 33 | :alt: Supported versions 34 | :target: https://pypi.python.org/pypi/gwgen 35 | 36 | .. |supported-implementations| image:: https://img.shields.io/pypi/implementation/gwgen.svg?style=flat 37 | :alt: Supported implementations 38 | :target: https://pypi.python.org/pypi/gwgen 39 | 40 | .. end-badges 41 | 42 | Welcome! This synthesis of FORTRAN and Python is a globally applicable 43 | weather generator inspired by the original WGEN weather generator of 44 | [Richardson1981]_ and parameterized through a global dataset of [GHCN]_ and 45 | [EECRA]_ data. A paper with the scientific documentation is in progress, this 46 | document shall cover the technical documentation. 47 | 48 | 49 | Documentation 50 | ------------- 51 | 52 | .. toctree:: 53 | :maxdepth: 1 54 | 55 | install 56 | getting_started 57 | command_line/gwgen.rst 58 | api/gwgen.rst 59 | fortran_api/index.rst 60 | changelog 61 | 62 | 63 | Authors 64 | ------- 65 | This package has been developped by `Philipp S. Sommer`_ and `Jed O. Kaplan`_. 66 | 67 | Citing GWGEN 68 | ------------ 69 | When using GWGEN, we kindly ask you to provide a reference to the corresponding 70 | paper published in Geoscientific Model Development: 71 | 72 | Sommer, P. S. and Kaplan, J. O.: *A globally calibrated scheme for 73 | generating daily meteorology from monthly statistics: Global-WGEN (GWGEN) 74 | v1.0*, Geosci. Model Dev., 10, 3771-3791, :doi:`10.5194/gmd-10-3771-2017`, 75 | 2017. 76 | 77 | 78 | Acknowledgements 79 | ---------------- 80 | This work was supported by the European Research Council (COEVOLVE, 313797) and 81 | the Swiss National Science Foundation (ACACIA, CR10I2\_146314). We thank 82 | `Shawn Koppenhoefer`_ for assistance compiling and querying the weather databases and 83 | Alexis Berne and Grégoire Mariéthoz for helpful suggestions on the analyses. We 84 | are grateful to NOAA NCDC and the University of Washingtion for providing free 85 | of charge the GHCN-Daily and EECRA databases, respectively. 86 | 87 | .. _Philipp S. Sommer: https://github.com/Chilipp 88 | .. _Jed O. Kaplan: https://github.com/jedokaplan 89 | .. _Shawn Koppenhoefer: http://arve.unil.ch/people/shawn-koppenhoefer/ 90 | 91 | 92 | References 93 | ---------- 94 | .. [Richardson1981] Richardson, C. W.: *Stochastic simulation of daily 95 | precipitation, temperature, and solar radiation*, Water Resources Research, 96 | 17, 182–190, :doi:`10.1029/WR017i001p00182`, 1981. 97 | .. [GHCN] T. G.: Global Historical Climatology Network - Daily (GHCN-Daily), 98 | Version 3.22, doi:10.7289/V5D21VHZ, :doi:`10.7289/V5D21VHZ`, 2012 99 | .. [EECRA] Hahn, C. and Warren, S.: *Extended Edited Synoptic Cloud Reports from 100 | Ships and Land Stations Over the Globe*, 1952-1996 (with Ship data 101 | updated through 2008), :doi:`10.3334/CDIAC/cli.ndp026c`, 1999 102 | 103 | 104 | Indices and tables 105 | ================== 106 | 107 | * :ref:`genindex` 108 | * :ref:`modindex` 109 | * :ref:`search` 110 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: generic 2 | branches: 3 | except: 4 | - arve2 5 | env: 6 | global: 7 | - ENCRYPTION_LABEL: "d0a57c2d09c2" 8 | - COMMIT_AUTHOR_EMAIL: "philipp.sommer@unil.ch" 9 | - BUILD_DOCS: false 10 | - ONLINE_TESTS: false 11 | matrix: 12 | include: 13 | - env: BUILD_DOCS=true 14 | PYTHON_VERSION=2.7 15 | os: linux 16 | - env: PYTHON_VERSION=3.7 17 | ONLINE_TESTS=true 18 | os: linux 19 | - env: PYTHON_VERSION=2.7 20 | os: osx 21 | - env: PYTHON_VERSION=3.7 22 | os: osx 23 | services: 24 | - postgresql 25 | addons: # install gfortran 26 | apt: 27 | packages: 28 | - gfortran 29 | before_install: 30 | - "brew install gcc || brew link --overwrite gcc || :" 31 | install: 32 | # select the os name for the conda installer 33 | - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then 34 | OS_NAME=MacOSX; 35 | else 36 | OS_NAME=Linux; 37 | fi 38 | # We do this conditionally because it saves us some downloading if the 39 | # version is the same. 40 | - if [[ "$PYTHON_VERSION" == "2.7" ]]; then 41 | wget https://repo.continuum.io/miniconda/Miniconda2-latest-"${OS_NAME}"-x86_64.sh -O miniconda.sh; 42 | else 43 | wget https://repo.continuum.io/miniconda/Miniconda3-latest-"${OS_NAME}"-x86_64.sh -O miniconda.sh; 44 | fi 45 | # set build_docs to the real value 46 | - if [[ "$BUILD_DOCS" == "true" && "$TRAVIS_PULL_REQUEST" == "false" && ("$TRAVIS_BRANCH" == "master" || "$TRAVIS_TAG" != "") ]]; then 47 | export BUILD_DOCS="true"; 48 | else 49 | export BUILD_DOCS="false"; 50 | fi 51 | - echo "$BUILD_DOCS" 52 | # make sure we use an 'innocent' default matplotlib environment 53 | - touch matplotlibrc 54 | - bash miniconda.sh -b -p $HOME/miniconda 55 | - . $HOME/miniconda/etc/profile.d/conda.sh && conda activate base 56 | - hash -r 57 | - conda config --set always_yes yes --set changeps1 no 58 | - conda update -q conda 59 | # Useful for debugging any issues with conda 60 | - conda info -a 61 | - conda install -c conda-forge numpy 62 | - conda env create -f ci/environment_py${PYTHON_VERSION}.yml 63 | - conda activate test 64 | - if [[ "$PYTHON_VERSION" == "2.7" ]]; then 65 | pip install pathlib; 66 | fi 67 | - pip install . coveralls pytest 68 | # install necessary module for the documentation and clone the existing 69 | # gh-pages for this repo into the *deploy* folder and clear everything 70 | - export REPO="$(git config remote.origin.url)" 71 | - if [[ "$BUILD_DOCS" == "true" ]]; then 72 | pip install sphinx-fortran sphinx==1.3.5 sphinx_rtd_theme ipython sphinx-argparse==0.1.17 autodocsumm; 73 | git clone "$REPO" deploy; 74 | cd deploy; 75 | git checkout "gh-pages" || git checkout --orphan "gh-pages"; 76 | ls -ad * .* | grep -v ".git$" | grep -v "^\.$" | grep -v "^\.\.$" | xargs rm -r; 77 | touch .nojekyll; 78 | cd ..; 79 | fi 80 | # export the environment for debugging 81 | - conda env export -n test 82 | 83 | script: 84 | - gwgen -h 85 | # run test suite 86 | - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then 87 | coverage run --parallel-mode --concurrency=multiprocessing setup.py test -a "-v --serial -m 'not fullrun and not long' -k 'not test_parameterization' --user postgres --offline"; 88 | elif [[ "$ONLINE_TESTS" == "true" ]]; then 89 | coverage run --parallel-mode --concurrency=multiprocessing setup.py test -a "-v --nprocs 2 -m 'not fullrun' --user postgres"; 90 | else 91 | coverage run --parallel-mode --concurrency=multiprocessing setup.py test -a "-v --nprocs 2 -m 'not fullrun' --user postgres --offline"; 92 | fi 93 | # build docs 94 | - if [[ "$BUILD_DOCS" == "true" ]]; then 95 | cd docs; 96 | sphinx-build . ../deploy; 97 | cd ..; 98 | fi 99 | 100 | after_success: 101 | - coverage combine && coveralls 102 | 103 | deploy: 104 | - provider: script 105 | script: ci/deploy.sh 106 | skip_cleanup: true 107 | on: 108 | branch: master 109 | condition: "$BUILD_DOCS == true" 110 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages 2 | from setuptools.command.test import test as TestCommand 3 | from numpy.distutils.core import Extension 4 | import sys 5 | import os 6 | import os.path as osp 7 | 8 | 9 | # conda skeleton applies a patch that is not realised by 10 | # numpy.distutils.core.setup. Therefore we give the user the possibility to 11 | # choose to use the setuptools. Note that conda skeleton not actually installs 12 | # the package but just saves the information from the setup call 13 | if os.getenv('RUNNING_SKELETON', None): 14 | from setuptools import setup 15 | else: 16 | from numpy.distutils.core import setup 17 | 18 | parseghcnrow = Extension( 19 | name='gwgen._parseghcnrow', sources=[ 20 | osp.join('gwgen', 'mo_parseghcnrow.f90')]) 21 | parseeecra = Extension( 22 | name='gwgen._parseeecra', sources=[osp.join('gwgen', 'mo_parseeecra.f90')], 23 | f2py_options=['only:', 'parse_file', 'extract_data', ':'], 24 | extra_f90_compile_args=["-fopenmp"], extra_link_args=["-lgomp"]) 25 | 26 | 27 | class PyTest(TestCommand): 28 | user_options = [('pytest-args=', 'a', "Arguments to pass to pytest")] 29 | 30 | def initialize_options(self): 31 | TestCommand.initialize_options(self) 32 | self.pytest_args = [] 33 | 34 | def run_tests(self): 35 | import shlex 36 | # import here, cause outside the eggs aren't loaded 37 | import pytest 38 | errno = pytest.main(shlex.split(self.pytest_args)) 39 | sys.exit(errno) 40 | 41 | 42 | def readme(): 43 | with open('README.rst') as f: 44 | return f.read() 45 | 46 | 47 | # read the version from version.py 48 | with open(osp.join('gwgen', 'version.py')) as f: 49 | exec(f.read()) 50 | 51 | 52 | def configuration(parent_package='', top_path=None): 53 | from numpy.distutils.misc_util import Configuration 54 | 55 | config = Configuration(None, parent_package, top_path) 56 | config.set_options(ignore_setup_xxx_py=True, 57 | assume_default_configuration=True, 58 | delegate_options_to_subpackages=True, 59 | quiet=True) 60 | 61 | config.add_subpackage('gwgen') 62 | config.add_data_dir(osp.join('gwgen', 'src')) 63 | config.add_data_dir(osp.join('gwgen', 'data')) 64 | 65 | return config 66 | 67 | 68 | install_requires = ['f90nml', 'psyplot', 'scipy', 'sqlalchemy', 'psycopg2', 69 | 'statsmodels', 'docrep', 'model-organization', 'xarray', 70 | 'six'] 71 | 72 | 73 | setup(name='gwgen', 74 | version=__version__, 75 | description='A global weather generator for daily data', 76 | long_description=readme(), 77 | classifiers=[ 78 | 'Development Status :: 5 - Production/Stable', 79 | 'Intended Audience :: Developers', 80 | 'Intended Audience :: Education', 81 | 'Intended Audience :: Science/Research', 82 | 'Topic :: Scientific/Engineering :: Physics', 83 | 'Topic :: Scientific/Engineering', 84 | 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', 85 | 'Programming Language :: Python :: 2', 86 | 'Programming Language :: Python :: 2.7', 87 | 'Programming Language :: Python :: 3', 88 | 'Programming Language :: Python :: 3.2', 89 | 'Programming Language :: Python :: 3.3', 90 | 'Programming Language :: Python :: 3.4', 91 | 'Programming Language :: Python :: 3.5', 92 | 'Operating System :: Unix', 93 | 'Operating System :: MacOS', 94 | ], 95 | keywords='wgen weathergen ghcn eecra richardson geng', 96 | url='https://github.com/ARVE-Research/gwgen', 97 | author='Philipp Sommer', 98 | author_email='philipp.sommer@unil.ch', 99 | license="GPLv2", 100 | packages=find_packages(exclude=['docs', 'tests*', 'examples']), 101 | install_requires=install_requires, 102 | package_data={'gwgen': [ 103 | 'gwgen/src/*', 104 | 'gwgen/data/*', 105 | ]}, 106 | include_package_data=True, 107 | tests_require=['pytest'], 108 | cmdclass={'test': PyTest}, 109 | entry_points={'console_scripts': ['gwgen=gwgen.main:main']}, 110 | zip_safe=False, 111 | ext_modules=[parseghcnrow, parseeecra], 112 | configuration=configuration) 113 | -------------------------------------------------------------------------------- /gwgen/mo_parseghcnrow.f90: -------------------------------------------------------------------------------- 1 | module parseghcnrow 2 | 3 | contains 4 | 5 | subroutine parse_station(infile, ndays, stationid, dates, variables, flags, j) 6 | 7 | implicit none 8 | 9 | character(11), intent(out) :: stationid 10 | 11 | integer :: year 12 | integer :: month 13 | 14 | character(4) :: variable 15 | 16 | character(300), intent(in) :: infile 17 | integer, intent(in) :: ndays 18 | 19 | ! returns 20 | integer, intent(out) :: dates(ndays, 3), j 21 | real, intent(out) :: variables(ndays, 3) 22 | character(1), intent(out) :: flags(ndays, 3, 3) 23 | 24 | integer, parameter, dimension(12) :: ndm_normal = [ 31,28,31, 30, 31, 30, 31, 31, 30, 31, 30, 31 ] !number of days in each month 25 | integer, parameter, dimension(12) :: ndm_leapyr = [ 31,29,31, 30, 31, 30, 31, 31, 30, 31, 30, 31 ] !number of days in each month 26 | 27 | integer, dimension(12) :: ndaymon 28 | 29 | character(269) :: values 30 | 31 | real,dimension(31) :: tmax 32 | real,dimension(31) :: tmin 33 | real,dimension(31) :: prcp 34 | 35 | character(1), dimension(31,3) :: tmin_flags 36 | character(1), dimension(31,3) :: tmax_flags 37 | character(1), dimension(31,3) :: prcp_flags 38 | 39 | integer :: d 40 | integer :: start 41 | 42 | integer :: lastmonth 43 | integer :: lastyear 44 | 45 | real, parameter :: missing = -9999. 46 | 47 | 48 | !---- 49 | 50 | open(10,file=infile,status='old') 51 | 52 | 20 format(a11,i4,i2,a4,a247) 53 | 54 | lastyear = 0 55 | lastmonth = 0 56 | 57 | do 58 | 59 | read(10,20,end=99)stationid,year,month,variable,values 60 | if (variable == 'PRCP' .or. variable == 'TMAX' .or. variable == 'TMIN') then 61 | 62 | !initialize variables 63 | 64 | lastmonth = month 65 | lastyear = year 66 | exit 67 | end if 68 | end do 69 | 70 | if (lastmonth == 0) then 71 | STOP 'Could not find any data for tmin, tmax or prcp' 72 | end if 73 | 74 | tmin = missing 75 | tmax = missing 76 | prcp = missing 77 | 78 | tmin_flags = '' 79 | tmax_flags = '' 80 | prcp_flags = '' 81 | 82 | rewind(10) 83 | 84 | !---- 85 | j = 1 86 | do !read one month of one variable per row 87 | 88 | read(10,20,end=99)stationid,year,month,variable,values 89 | 90 | if (variable /= 'PRCP' .and. variable /= 'TMAX' .and. variable /= 'TMIN') cycle 91 | 92 | if (year /= lastyear .or. month /= lastmonth) then 93 | 94 | !write out the data 95 | 96 | do d = 1,ndaymon(lastmonth) 97 | dates(j, :) = (/lastyear, lastmonth, d/) 98 | variables(j, :) = (/tmin(d), tmax(d), prcp(d)/) 99 | flags(j, 1, :) = tmin_flags(d,:) 100 | flags(j, 2, :) = tmax_flags(d,:) 101 | flags(j, 3, :) = prcp_flags(d,:) 102 | j = j + 1 103 | end do 104 | 105 | lastmonth = month 106 | lastyear = year 107 | 108 | tmin = missing 109 | tmax = missing 110 | prcp = missing 111 | 112 | tmin_flags = '' 113 | tmax_flags = '' 114 | prcp_flags = '' 115 | 116 | end if 117 | 118 | if (leapyear(year)) then 119 | ndaymon = ndm_leapyr 120 | else 121 | ndaymon = ndm_normal 122 | end if 123 | 124 | do d = 1,ndaymon(month) 125 | 126 | start = 1+8*(d-1) 127 | 128 | !convert the value into a real number 129 | 130 | if (variable == 'TMAX') then 131 | 132 | read(values(start:start+4),*)tmax(d) 133 | !apply the scale factor 134 | tmax(d) = tmax(d) * 0.1 135 | ! Set values smaller than -100 degC to NaN 136 | if (tmax(d) < -100.0) tmax(d) = missing 137 | 138 | tmax_flags(d,1) = values(start+5:start+5) 139 | tmax_flags(d,2) = values(start+6:start+6) 140 | tmax_flags(d,3) = values(start+7:start+7) 141 | 142 | else if (variable == 'TMIN') then 143 | 144 | read(values(start:start+4),*)tmin(d) 145 | !apply the scale factor 146 | tmin(d) = tmin(d) * 0.1 147 | ! Set values smaller than -100 degC to NaN 148 | if (tmin(d) < -100.0) tmin(d) = missing 149 | 150 | tmin_flags(d,1) = values(start+5:start+5) 151 | tmin_flags(d,2) = values(start+6:start+6) 152 | tmin_flags(d,3) = values(start+7:start+7) 153 | 154 | else if (variable == 'PRCP') then 155 | 156 | read(values(start:start+4),*)prcp(d) 157 | !apply the scale factor 158 | prcp(d) = prcp(d) * 0.1 159 | if (prcp(d) < 0.0) prcp(d) = missing 160 | 161 | prcp_flags(d,1) = values(start+5:start+5) 162 | prcp_flags(d,2) = values(start+6:start+6) 163 | prcp_flags(d,3) = values(start+7:start+7) 164 | 165 | end if 166 | 167 | end do 168 | end do 169 | 170 | 99 continue 171 | 172 | if (lastmonth > 0) then 173 | do d = 1,ndaymon(lastmonth) 174 | dates(j, :) = (/lastyear, lastmonth, d/) 175 | variables(j, :) = (/tmin(d), tmax(d), prcp(d)/) 176 | flags(j, 1, :) = tmin_flags(d,:) 177 | flags(j, 2, :) = tmax_flags(d,:) 178 | flags(j, 3, :) = prcp_flags(d,:) 179 | j = j + 1 180 | end do 181 | end if 182 | j = j - 1 183 | 184 | end subroutine parse_station 185 | 186 | !---- 187 | 188 | logical function leapyear(year) 189 | 190 | integer, intent(in) :: year 191 | 192 | if ((mod(year,4) == 0 .and. mod(year,100) /= 0) .or. mod(year,400) == 0) then 193 | 194 | leapyear = .true. 195 | 196 | else 197 | 198 | leapyear = .false. 199 | 200 | end if 201 | 202 | end function leapyear 203 | 204 | !---- 205 | 206 | end module parseghcnrow 207 | -------------------------------------------------------------------------------- /docs/getting_started.rst: -------------------------------------------------------------------------------- 1 | .. _getting_started: 2 | 3 | Getting started 4 | =============== 5 | 6 | .. _running_raw: 7 | 8 | Running the raw ``weathergen`` 9 | ------------------------------ 10 | After successfully :ref:`installing ` the FORTRAN files, 11 | you can run the weather generator by typing:: 12 | 13 | ./weathergen 14 | 15 | ```` thereby is a csv-file (with header) containing the columns 16 | (the order is important!) 17 | 18 | station id (character string of 11 characters) 19 | a unique identifier of the weather station 20 | lon (float) 21 | the longitude of the weather station (only necessary if the 22 | ``use_geohash`` namelist parameter in the main namelist of 23 | ``weathergen.nml`` is True (the default)) 24 | lat (float) 25 | the latitude of the weather station (see ``lon``) 26 | year (int) 27 | the year of the month 28 | month (int) 29 | the month 30 | min. temperature (float) 31 | the minimum temperature degrees Celsius 32 | max. temperature (float) 33 | the maximum temperature in degrees Celsius 34 | cloud fraction (float) 35 | the mean cloud fraction during the month between 0 and 1 36 | wind speed (float) 37 | the mean wind speed during the month in m/s 38 | precipitation (float) 39 | the total precipitation in the month in mm/day 40 | wet (int) 41 | the number of wet days in the month 42 | 43 | The output file will contain the same columns except lon and lat but with an 44 | additional day column 45 | 46 | 47 | Using the python package 48 | ------------------------ 49 | The GWGEN package uses the model-organization_ framework and thus can be used 50 | from the command line. The corresponding subclass of the 51 | :class:`model_organization.ModelOrganizer` is the 52 | :class:`gwgen.main.GWGENOrganizer` class. 53 | 54 | After :ref:`having installed the full python package ` you can 55 | setup a new project with the :ref:`gwgen.setup` command via 56 | 57 | .. ipython:: 58 | 59 | @suppress 60 | In [1]: import os 61 | ...: os.environ['PYTHONWARNINGS'] = "ignore" 62 | 63 | In [1]: !gwgen setup . -p my_first_project 64 | 65 | This will copy the fortran source files into `my_first_project/src` where you 66 | can then modify them according to your needs. To compile the model, use the 67 | :ref:`gwgen.compile` command: 68 | 69 | .. ipython:: 70 | 71 | In [2]: !gwgen compile -p my_first_project 72 | 73 | Note that you can also omit the ``-p`` option. If you do so, it uses the last 74 | created project. 75 | 76 | To create a new experiment inside the project, use the :ref:`gwgen.init` 77 | command: 78 | 79 | .. ipython:: 80 | 81 | In [3]: !gwgen -id my_first_experiment init -p my_first_project 82 | 83 | To run the weather generator, use the :ref:`gwgen.run` command 84 | 85 | .. ipython:: 86 | :verbatim: 87 | 88 | In [4]: !gwgen -id my_first_experiment run -i 89 | 90 | (see :ref:`running_raw` for the format of ````). Note that you 91 | can as well omit the ``-id`` option. By doing so, it uses the last created 92 | experiment. 93 | 94 | .. _parameterization: 95 | 96 | Parameterizing an experiment 97 | ---------------------------- 98 | The default parameterization of the weather generator uses about 8500 stations 99 | world wide from the [GHCN]_ database and 8500 stations from the [EECRA]_ 100 | database. I you however want to have your own parameterization, you can use the 101 | :ref:`gwgen.param` command. 102 | 103 | The parameterization is split up into :class:`tasks `, 104 | where each :class:`task ` processed a given set of GHCN 105 | or EECRA stations. Each task that is used for the parameterization, requires 106 | some intermediate tasks. For example, the :ref:`prcp ` task 107 | that determines the relationship between mean precipitation, number of wet days 108 | and the precipitation distribution parameters, requires the 109 | :ref:`reading and downloading of the daily GHCN data `, the 110 | :ref:`calculation of the monthly averages ` and the 111 | :ref:`extraction of the complete months `. However, these 112 | dependencies are specified in the corresponding 113 | :class:`~gwgen.parametization.Parameterizer` subclass 114 | (e.g. :class:`gwgen.parameterization.PrcpDistParams`) and the only question you 115 | have to take care about is: What stations do you want to use for the 116 | parameterization? You can use the climap_ to select the stations you need for 117 | your region but note that you should have as many weather stations as you can. 118 | 119 | For our demonstration, we only use two weather stations from Hamburg: 120 | 121 | GM000010147 122 | Hamburg-Fuhlsbüttel 123 | GM000003865 124 | Hamburg-Bergedorf 125 | 126 | and save the corresponding IDs in a file 127 | 128 | .. ipython:: 129 | 130 | In [5]: with open('hamburg_stations.dat', 'w') as f: 131 | ...: f.write('GM000010147\n') 132 | ...: f.write('GM000003865') 133 | 134 | then, we use the :ref:`day ` task to download the necessary 135 | data files and run our :ref:`prcp ` parameterization: 136 | 137 | .. ipython:: 138 | 139 | In [6]: !gwgen param -s hamburg_stations.dat day --download single prcp 140 | 141 | @suppress 142 | In [6]: !echo 'bins: 100\ndensity: "kde"\nxrange: [3, 6]' > fmt.yml 143 | ...: !psyplot -p my_first_project/experiments/my_first_experiment/parameterization/prcp.pkl -o _static/prcp%i.png -fmt fmt.yml 144 | 145 | This then also creates a plot that shows the relation ship between the mean 146 | precipitation on wet days (as it is calculated in the weather generator) and 147 | the gamma shape parameter 148 | 149 | .. image:: ./_static/prcp1.png 150 | :alt: Relation ship of mean precipitation and gamma shape parameter 151 | 152 | This procedure now modified the namelist of our experiment and added two 153 | namelist parameters 154 | 155 | .. ipython:: 156 | 157 | In [7]: !gwgen get-value namelist 158 | 159 | Otherwise you can of course always modify the namelist of your experiment using 160 | the :ref:`set-value ` and :ref:`del-value ` 161 | commands or by modifying the configuration file (``gwgen info -ep``) by hand 162 | 163 | .. note:: 164 | Since we use the psyplot_ package, you can also easily change the 165 | above created plot after the parameterization. For example, to change the 166 | number of bins in the density plot of the above plot, just load the created 167 | psyplot project and update the plot: 168 | 169 | .. ipython:: 170 | :verbatim: 171 | 172 | In [8]: import psyplot.project as psy 173 | 174 | In [9]: p = psy.Project.load_project( 175 | ...: 'my_first_project/experiments/my_first_experiment/' 176 | ...: 'parameterization/prcp.pkl') 177 | 178 | In [10]: p.update(bins=20) 179 | 180 | .. ipython:: 181 | :suppress: 182 | 183 | In [10]: !gwgen remove -p my_first_project -ay 184 | 185 | In [11]: !rm hamburg_stations.dat fmt.yml 186 | 187 | 188 | .. _model-organization: http://model-organization.readthedocs.io/en/latest/ 189 | .. _psyplot: http://psyplot.readthedocs.io/en/latest/ 190 | .. _climap: http://arve.unil.ch/climap/ 191 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " applehelp to make an Apple Help Book" 34 | @echo " devhelp to make HTML files and a Devhelp project" 35 | @echo " epub to make an epub" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | 51 | clean: 52 | rm -rf $(BUILDDIR)/* 53 | 54 | html: 55 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 56 | @echo 57 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 58 | 59 | dirhtml: 60 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 61 | @echo 62 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 63 | 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | pickle: 70 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 71 | @echo 72 | @echo "Build finished; now you can process the pickle files." 73 | 74 | json: 75 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 76 | @echo 77 | @echo "Build finished; now you can process the JSON files." 78 | 79 | htmlhelp: 80 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 81 | @echo 82 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 83 | ".hhp project file in $(BUILDDIR)/htmlhelp." 84 | 85 | qthelp: 86 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 87 | @echo 88 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 89 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 90 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/syplot.qhcp" 91 | @echo "To view the help file:" 92 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/syplot.qhc" 93 | 94 | applehelp: 95 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 96 | @echo 97 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 98 | @echo "N.B. You won't be able to view it unless you put it in" \ 99 | "~/Library/Documentation/Help or install it in your application" \ 100 | "bundle." 101 | 102 | devhelp: 103 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 104 | @echo 105 | @echo "Build finished." 106 | @echo "To view the help file:" 107 | @echo "# mkdir -p $$HOME/.local/share/devhelp/syplot" 108 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/syplot" 109 | @echo "# devhelp" 110 | 111 | epub: 112 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 113 | @echo 114 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 115 | 116 | latex: 117 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 118 | @echo 119 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 120 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 121 | "(use \`make latexpdf' here to do that automatically)." 122 | 123 | latexpdf: 124 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 125 | @echo "Running LaTeX files through pdflatex..." 126 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 127 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 128 | 129 | latexpdfja: 130 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 131 | @echo "Running LaTeX files through platex and dvipdfmx..." 132 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 133 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 134 | 135 | text: 136 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 137 | @echo 138 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 139 | 140 | man: 141 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 142 | @echo 143 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 144 | 145 | texinfo: 146 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 147 | @echo 148 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 149 | @echo "Run \`make' in that directory to run these through makeinfo" \ 150 | "(use \`make info' here to do that automatically)." 151 | 152 | info: 153 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 154 | @echo "Running Texinfo files through makeinfo..." 155 | make -C $(BUILDDIR)/texinfo info 156 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 157 | 158 | gettext: 159 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 160 | @echo 161 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 162 | 163 | changes: 164 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 165 | @echo 166 | @echo "The overview file is in $(BUILDDIR)/changes." 167 | 168 | linkcheck: 169 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 170 | @echo 171 | @echo "Link check complete; look for any errors in the above output " \ 172 | "or in $(BUILDDIR)/linkcheck/output.txt." 173 | 174 | doctest: 175 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 176 | @echo "Testing of doctests in the sources finished, look at the " \ 177 | "results in $(BUILDDIR)/doctest/output.txt." 178 | 179 | coverage: 180 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 181 | @echo "Testing of coverage in the sources finished, look at the " \ 182 | "results in $(BUILDDIR)/coverage/python.txt." 183 | 184 | xml: 185 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 186 | @echo 187 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 188 | 189 | pseudoxml: 190 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 191 | @echo 192 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 193 | -------------------------------------------------------------------------------- /tests/test_sensitivity_analysis.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import unittest 3 | import os.path as osp 4 | import numpy as np 5 | import pandas as pd 6 | import _base_testing as bt 7 | import pytest 8 | 9 | 10 | class SensitivityAnalysisTest(bt.BaseTest): 11 | """Test case to test the sensitivity analysis""" 12 | 13 | @property 14 | def projectname(self): 15 | return self.organizer.exp_config['sensitivity_analysis']['project'] 16 | 17 | @property 18 | def sa_experiments(self): 19 | all_exps = self.organizer.config.experiments 20 | projectname = self.projectname 21 | return [exp_id for exp_id in all_exps 22 | if all_exps[exp_id]['project'] == projectname] 23 | 24 | @pytest.mark.fullrun 25 | def test_setup(self): 26 | self._test_init() 27 | self.organizer.exp_config['input'] = osp.join( 28 | bt.test_root, 'test_data', 'input.csv') 29 | self.organizer.exp_config['reference'] = osp.join( 30 | bt.test_root, 'test_data', 'reference.csv') 31 | self.organizer.sensitivity_analysis( 32 | setup={}, experiment=self.organizer.experiment) 33 | projectname = self.projectname 34 | self.assertTrue(osp.exists( 35 | self.organizer.config.projects[projectname]['root'])) 36 | 37 | def test_init(self, sparse=False): 38 | self.test_setup() 39 | experiment = self.organizer.experiment 40 | self.organizer.exp_config['eval_stations'] = self.stations_file 41 | if not sparse: 42 | self.organizer.parse_args( 43 | ('-id {} sens init -nml thresh=13,14-16 ' 44 | 'gp_shape=0.1,0.4-1.1-0.3').format(experiment).split()) 45 | threshs = np.arange(13, 16) 46 | shapes = np.arange(0.1, 1.1, 0.3) 47 | else: 48 | self.organizer.parse_args( 49 | ('-id {} sens init -nml thresh=13-15 ' 50 | 'gp_shape=0.1-0.5-0.3').format(experiment).split()) 51 | threshs = np.arange(13, 15) 52 | shapes = np.arange(0.1, 0.5, 0.3) 53 | shapes, threshs = np.meshgrid(shapes, threshs) 54 | n = threshs.size 55 | all_exps = self.organizer.config.experiments 56 | exp_names = self.sa_experiments 57 | self.assertEqual(len(exp_names), n) 58 | df = pd.DataFrame.from_dict([ 59 | all_exps[exp_id]['namelist']['weathergen_ctl'] 60 | for exp_id in exp_names]) 61 | self.assertAlmostArrayEqual(df.thresh.values, 62 | threshs.ravel()) 63 | self.assertAlmostArrayEqual( 64 | df.gp_shape.values, shapes.ravel()) 65 | 66 | @pytest.mark.fullrun 67 | def test_init_and_param(self): 68 | self.test_setup() 69 | experiment = self.organizer.experiment 70 | self.organizer.exp_config['eval_stations'] = self.stations_file 71 | self.organizer.exp_config['param_stations'] = self.stations_file 72 | n_shape = 2 73 | self.organizer.parse_args( 74 | ('-id {} sens init -nml thresh=5-8-2 ' 75 | 'gp_shape=-1err-1err-{}').format(experiment, n_shape).split()) 76 | threshs = np.arange(5, 8, 2) 77 | _, threshs = np.meshgrid(range(n_shape + 1), threshs) 78 | n = threshs.size 79 | all_exps = self.organizer.config.experiments 80 | exp_names = self.sa_experiments 81 | self.assertEqual(len(exp_names), n) 82 | df = pd.DataFrame([ 83 | all_exps[exp_id]['namelist']['weathergen_ctl'] 84 | for exp_id in exp_names], index=exp_names) 85 | self.assertAlmostArrayEqual(df.thresh.values, 86 | threshs.ravel()) 87 | # get the parameterization values 88 | df2 = pd.DataFrame([ 89 | all_exps[exp_id].get('parameterization', {}).get('prcp', {}) 90 | for exp_id in exp_names], index=exp_names)[ 91 | ['gpshape_std', 'gpshape_mean']] 92 | ranges = iter([]) 93 | for i, (exp, err, mean) in enumerate(df2.to_records()): 94 | if not np.isnan(err): 95 | ranges = iter(np.linspace(mean - err, mean + err, n_shape)) 96 | else: 97 | self.assertAlmostEqual( 98 | df.gp_shape[i], 99 | next(ranges), 100 | msg=("Wrong shape parameter for experiment %s.\n" 101 | "Namelists: \n%s \n" 102 | "Parameterization: \n%s") % (exp, df, df2)) 103 | 104 | def test_compile_model(self): 105 | self.test_setup() 106 | self.organizer.sensitivity_analysis( 107 | compile={}, experiment=self.organizer.experiment) 108 | binpath = osp.join(self.test_dir, self.projectname, 'bin', 109 | 'weathergen') 110 | self.assertTrue(osp.exists(binpath), 111 | msg='binary %s does not exist!' % binpath) 112 | 113 | @pytest.mark.fullrun 114 | def test_run(self, param=False): 115 | if param: 116 | self.test_init_and_param() 117 | else: 118 | self.test_init(sparse=True) 119 | self.organizer.sensitivity_analysis( 120 | compile={}, run={}, experiment=self.organizer.experiment) 121 | all_exps = self.organizer.config.experiments 122 | exp_names = self.sa_experiments 123 | for exp in exp_names: 124 | self.assertIn( 125 | 'outdata', all_exps[exp], 126 | msg=('Run failed for experiment %s. No outdata in ' 127 | 'configuration!') % (exp, )) 128 | self.assertTrue(osp.exists(all_exps[exp]['outdata']), 129 | msg='Missing output file %s for experiment %s' % ( 130 | all_exps[exp]['outdata'], exp)) 131 | 132 | @pytest.mark.fullrun 133 | def test_evaluate(self, param=False, full=False): 134 | self.test_run(param=param) 135 | orig_serial = self.organizer.global_config.get('serial') 136 | # parallel does not work due to matplotlib 137 | self.organizer.global_config['serial'] = True 138 | if not full: 139 | to_evaluate = ['testexp1_sens1', 'testexp1_sens2'] 140 | else: 141 | to_evaluate = None 142 | self.organizer.sensitivity_analysis( 143 | evaluate={'quants': {'names': ['prcp']}, 'ks': {}, 144 | 'experiments': to_evaluate}, 145 | experiment=self.organizer.experiment) 146 | self.organizer.global_config['serial'] = orig_serial 147 | all_exps = self.organizer.config.experiments 148 | if full: 149 | projectname = self.projectname 150 | to_evaluate = [exp_id for exp_id in all_exps 151 | if all_exps[exp_id]['project'] == projectname] 152 | for exp in to_evaluate: 153 | self.assertIn('quants', all_exps[exp]['evaluation'], 154 | msg='No quantile evaluation made for %s' % exp) 155 | self.assertIn('ks', all_exps[exp]['evaluation'], 156 | msg='No ks evaluation made for %s' % exp) 157 | 158 | @pytest.mark.long 159 | def test_plot(self): 160 | """Test whether the plot of a simple sensitivity analysis works""" 161 | self._test_plot() 162 | 163 | @pytest.mark.long 164 | def test_plot_complex(self): 165 | """Test whether the plot of a complex sensitivity analysis works""" 166 | self._test_plot(True) 167 | 168 | def _test_plot(self, param=False): 169 | self.test_evaluate(param=param, full=True) 170 | exp_names = self.sa_experiments 171 | plot1d_output = osp.join(self.test_dir, 'plot1d.pdf') 172 | plot2d_output = osp.join(self.test_dir, 'plot2d.pdf') 173 | self.organizer.sensitivity_analysis( 174 | experiment=self.organizer.experiment, 175 | evaluate={'quality': {}, 176 | # delete one experiment to test, if it still works 177 | 'experiments': exp_names[:1] + exp_names[2:]}, 178 | plot={'plot1d': {'plot_output': plot1d_output}, 179 | 'plot2d': {'plot_output': plot2d_output}, 180 | 'names': ['prcp']}) 181 | self.assertTrue(osp.exists(plot1d_output), 182 | msg='File %s is missing!' % plot1d_output) 183 | self.assertTrue(osp.exists(plot2d_output), 184 | msg='File %s is missing!' % plot2d_output) 185 | 186 | 187 | if __name__ == '__main__': 188 | unittest.main() 189 | -------------------------------------------------------------------------------- /tests/_base_testing.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import six 4 | import shutil 5 | import unittest 6 | import glob 7 | import gwgen 8 | import numpy as np 9 | import tempfile 10 | from gwgen.main import GWGENOrganizer 11 | import gwgen.utils as utils 12 | from model_organization.config import setup_logging 13 | 14 | test_root = osp.abspath(osp.dirname(__file__)) 15 | 16 | _test_stations = osp.join(test_root, 'test_stations.dat') 17 | 18 | _short_test_stations = osp.join(test_root, 'test_stations_short.dat') 19 | 20 | _long_test_stations = osp.join(test_root, 'test_stations_long.dat') 21 | 22 | _eecra_test_stations = osp.join(test_root, 'eecra_test_stations.dat') 23 | 24 | _short_eecra_test_stations = osp.join(test_root, 25 | 'eecra_test_stations_short.dat') 26 | 27 | 28 | setup_logging(osp.join(test_root, 'logging.yaml')) 29 | 30 | on_travis = os.getenv('TRAVIS') 31 | 32 | 33 | db_config = dict( 34 | database='travis_ci_test') 35 | 36 | 37 | class BaseTest(unittest.TestCase): 38 | """Test the :class:`gwgen.main.ModuleOrganizer` class""" 39 | 40 | test_dir = None 41 | 42 | remove_at_cleanup = True 43 | 44 | use_db = True 45 | 46 | stations_type = 'normal' 47 | 48 | #: number of cores to use 49 | nprocs = None 50 | 51 | #: Boolean that is True, if the tests should be run in serial 52 | serial = False 53 | 54 | _station_files = { 55 | 'normal': _test_stations, 56 | 'short': _short_test_stations, 57 | 'long': _long_test_stations, 58 | } 59 | 60 | _eecra_station_files = { 61 | 'normal': _eecra_test_stations, 62 | 'short': _short_eecra_test_stations, 63 | } 64 | 65 | @classmethod 66 | def setUpClass(cls): 67 | # try to connect to a postgres database 68 | if cls.use_db: 69 | try: 70 | utils.get_postgres_engine(create=True, test=True, **db_config) 71 | cls.use_db = True 72 | except: 73 | cls.use_db = False 74 | 75 | def setUp(self): 76 | from psyplot import rcParams 77 | self.test_dir = tempfile.mkdtemp(prefix='tmp_gwgentest') 78 | os.environ['GWGENCONFIGDIR'] = self.config_dir = osp.join( 79 | self.test_dir, 'config') 80 | if not osp.exists(self.test_dir): 81 | os.makedirs(self.test_dir) 82 | if not osp.exists(self.config_dir): 83 | os.makedirs(self.config_dir) 84 | self.test_db = osp.basename(self.test_dir) 85 | src_station_file = self._station_files[self.stations_type] 86 | self.stations_file = osp.join(self.test_dir, 87 | osp.basename(src_station_file)) 88 | shutil.copyfile(src_station_file, self.stations_file) 89 | 90 | src_eecra_file = self._eecra_station_files[self.stations_type] 91 | self.eecra_stations_file = osp.join(self.test_dir, 92 | osp.basename(src_eecra_file)) 93 | shutil.copyfile(src_eecra_file, self.eecra_stations_file) 94 | self.organizer = GWGENOrganizer() 95 | global_conf = self.organizer.config.global_config 96 | global_conf['data'] = osp.join(test_root, 'test_data') 97 | global_conf['use_relative_links'] = False 98 | global_conf['nprocs'] = self.nprocs or 'all' 99 | global_conf['serial'] = self.serial 100 | if self.use_db: 101 | self._clear_db() 102 | global_conf.update(db_config) 103 | rcParams['plotter.linreg.nboot'] = 1 104 | 105 | def tearDown(self): 106 | from psyplot import rcParams 107 | if self.remove_at_cleanup: 108 | if osp.exists(self.test_dir): 109 | shutil.rmtree(self.test_dir) 110 | if osp.exists(self.config_dir): 111 | shutil.rmtree(self.config_dir) 112 | if self.use_db: 113 | self._clear_db() 114 | rcParams.update_from_defaultParams(plotters=False) 115 | 116 | del self.organizer 117 | del self.test_dir 118 | del self.config_dir 119 | 120 | def _clear_db(self): 121 | engine = utils.get_postgres_engine(db_config['database'])[0] 122 | conn = engine.connect() 123 | for table in engine.table_names(): 124 | conn.execute("DROP TABLE %s;" % table) 125 | conn.close() 126 | 127 | @property 128 | def stations(self): 129 | """A numpy array of the stations in :attr:`test_stations`""" 130 | return np.loadtxt( 131 | self.stations_file, dtype='S11', usecols=[0]).astype( 132 | np.str_) 133 | 134 | def _test_setup(self): 135 | """Test the setup of a project. We make this method private such that 136 | it is not called everytime""" 137 | self.organizer.setup(self.test_dir, 'test_project0', link=False) 138 | mpath = osp.join(self.test_dir, 'test_project0') 139 | self.assertTrue(osp.isdir(mpath)) 140 | original_files = sorted(map(osp.basename, glob.glob(osp.join( 141 | osp.dirname(gwgen.__file__), 'src', '*.f90')))) 142 | copied_files = sorted(map(osp.basename, glob.glob(osp.join( 143 | mpath, 'src', '*.f90')))) 144 | self.assertEqual(original_files, copied_files) 145 | self.assertIn('test_project0', self.organizer.config.projects) 146 | 147 | # createa new project and let it automatically assign the name 148 | self.organizer.setup(self.test_dir) 149 | mpath = osp.join(self.test_dir, 'test_project1') 150 | self.assertTrue(osp.isdir(mpath)) 151 | original_files = sorted(map(osp.basename, glob.glob(osp.join( 152 | osp.dirname(gwgen.__file__), 'src', '*.f90')))) 153 | copied_files = sorted(map(osp.basename, glob.glob(osp.join( 154 | mpath, 'src', '*.f90')))) 155 | self.assertEqual(original_files, copied_files) 156 | self.assertIn('test_project1', self.organizer.config.projects) 157 | 158 | def _test_init(self): 159 | """Test the intialization of a new experiment. We make this method 160 | private such that it is not called everytime""" 161 | self.organizer.setup(self.test_dir) 162 | projectname = self.organizer.projectname 163 | self.organizer.init(experiment='testexp0') 164 | expdir = osp.join( 165 | self.test_dir, projectname, 'experiments', 'testexp0') 166 | self.assertTrue(osp.exists(expdir), 167 | msg='Experiment directory %s does not exist!' % expdir) 168 | self.assertIn('testexp0', self.organizer.config.experiments) 169 | 170 | # test without argument 171 | self.organizer.setup(self.test_dir) 172 | projectname = self.organizer.projectname 173 | self.organizer.init(experiment=None) 174 | expdir = osp.join( 175 | self.test_dir, projectname, 'experiments', 'testexp1') 176 | self.assertTrue(osp.exists(expdir), 177 | msg='Experiment directory %s does not exist!' % expdir) 178 | self.assertIn('testexp1', self.organizer.config.experiments) 179 | 180 | @staticmethod 181 | def _test_url(url, *args, **kwargs): 182 | if six.PY3: 183 | from urllib import request 184 | request.urlopen(url, *args, **kwargs) 185 | else: 186 | import urllib 187 | urllib.urlopen(url, *args, **kwargs) 188 | 189 | def assertAlmostArrayEqual(self, actual, desired, rtol=1e-07, atol=0, 190 | msg=None, **kwargs): 191 | """Asserts that the two given arrays are almost the same 192 | 193 | This method uses the :func:`numpy.testing.assert_allclose` function 194 | to compare the two given arrays. 195 | 196 | Parameters 197 | ---------- 198 | actual : array_like 199 | Array obtained. 200 | desired : array_like 201 | Array desired. 202 | rtol : float, optional 203 | Relative tolerance. 204 | atol : float, optional 205 | Absolute tolerance. 206 | equal_nan : bool, optional. 207 | If True, NaNs will compare equal. 208 | err_msg : str, optional 209 | The error message to be printed in case of failure. 210 | verbose : bool, optional 211 | If True, the conflicting values are appended to the error message. 212 | """ 213 | try: 214 | np.testing.assert_allclose(actual, desired, rtol=rtol, atol=atol, 215 | err_msg=msg or '', **kwargs) 216 | except AssertionError as e: 217 | if six.PY2: 218 | self.fail(e.message) 219 | else: 220 | self.fail(str(e)) 221 | 222 | 223 | # check if we are online by trying to connect to google 224 | try: 225 | BaseTest._test_url('https://www.google.de') 226 | online = True 227 | except: 228 | online = False 229 | -------------------------------------------------------------------------------- /tests/test_stations_long.dat: -------------------------------------------------------------------------------- 1 | EN000026242 2 | EN000026249 3 | KZ000028952 4 | RSM00026188 5 | RSM00026359 6 | RSM00026781 7 | RSM00027051 8 | RSM00027199 9 | RSM00027296 10 | RSM00027459 11 | RSM00027509 12 | RSM00027648 13 | RSM00027823 14 | RSM00027857 15 | RSM00027962 16 | RSM00028009 17 | RSM00028224 18 | RSM00028367 19 | RSM00028506 20 | RSM00028537 21 | RSM00028552 22 | RSM00028573 23 | RSM00028661 24 | RSM00028666 25 | RSM00028704 26 | RSM00029263 27 | JNM00001001 28 | NO000097250 29 | NO000099710 30 | NOE00105472 31 | NOE00105476 32 | NOE00109459 33 | NOE00109476 34 | NOE00109485 35 | NOE00109514 36 | NOE00109613 37 | NOE00109671 38 | NOE00110653 39 | NOE00110725 40 | NOE00111327 41 | NOE00112080 42 | NOE00133122 43 | NOE00134094 44 | NOE00134886 45 | NOM00001238 46 | SW000002196 47 | SWE00100026 48 | SWE00138166 49 | SWE00138674 50 | SWE00139070 51 | SWE00139276 52 | SWE00139324 53 | SWE00139588 54 | SWE00139900 55 | SWE00139968 56 | SWE00140016 57 | SWE00140240 58 | SWE00140898 59 | SWE00140960 60 | SWM00002584 61 | FR000007130 62 | FR000007190 63 | FR000007255 64 | FR000007560 65 | FR000007630 66 | GM000010147 67 | GM000010393 68 | GME00102177 69 | GME00102212 70 | GME00102236 71 | GME00102252 72 | GME00102340 73 | GME00102348 74 | GME00111430 75 | GME00114621 76 | GME00121018 77 | GME00121090 78 | GME00121102 79 | GME00121150 80 | GME00121258 81 | PO000008549 82 | SP000008215 83 | SP000008280 84 | SP000009981 85 | SPE00119711 86 | SPE00119963 87 | SPE00120134 88 | SPE00120188 89 | SPE00120629 90 | SZ000009480 91 | AU000005010 92 | AU000006306 93 | AU000015410 94 | BKM00014654 95 | GM000010962 96 | GME00102316 97 | GME00102332 98 | GME00102364 99 | GME00102372 100 | GME00102404 101 | GME00120946 102 | GME00121114 103 | GME00121126 104 | GME00121174 105 | GME00121186 106 | GR000016641 107 | GR000016734 108 | HRE00105182 109 | HRE00105189 110 | HRE00105210 111 | HRE00105217 112 | HRE00105227 113 | PL000012120 114 | RIE00100814 115 | RIE00100818 116 | ROE00100903 117 | ROE00108892 118 | ROE00108895 119 | ROE00108897 120 | ROE00108898 121 | SIE00105938 122 | SIM00014015 123 | BE000006447 124 | EI000003965 125 | EI000003980 126 | EIE00101859 127 | FI000002401 128 | FI000002963 129 | FI000007501 130 | FIE00142085 131 | FIE00142511 132 | FIE00142861 133 | FIE00143586 134 | FIE00144132 135 | FIE00144212 136 | FIE00144702 137 | FIE00145377 138 | FIE00146423 139 | FIE00146688 140 | GLE00100981 141 | IC000004048 142 | IC000004097 143 | NLM00006260 144 | SWE00100003 145 | SWE00137494 146 | SWE00137910 147 | SWE00138058 148 | SZ000003700 149 | UK000003162 150 | RSM00020069 151 | RSM00020087 152 | RSM00020292 153 | RSM00020891 154 | RSM00021802 155 | RSM00021921 156 | RSM00021946 157 | RSM00022028 158 | RSM00022165 159 | RSM00022235 160 | RSM00022324 161 | RSM00022355 162 | RSM00022408 163 | RSM00022438 164 | RSM00022471 165 | RSM00022583 166 | RSM00022602 167 | RSM00022619 168 | RSM00022641 169 | RSM00022768 170 | RSM00022802 171 | RSM00022820 172 | RSM00022837 173 | RSM00022845 174 | RSM00023032 175 | KZ000029807 176 | RSM00029348 177 | RSM00029539 178 | RSM00029570 179 | RSM00029594 180 | RSM00029605 181 | RSM00029698 182 | RSM00029789 183 | RSM00029862 184 | RSM00029923 185 | RSM00029939 186 | RSM00030054 187 | RSM00030328 188 | RSM00030469 189 | RSM00030493 190 | RSM00030504 191 | RSM00030636 192 | RSM00030692 193 | RSM00030703 194 | RSM00030710 195 | RSM00030815 196 | RSM00030879 197 | RSM00030925 198 | RSM00030949 199 | RSM00031168 200 | RSM00031371 201 | RSM00031416 202 | RSM00031443 203 | RSM00023274 204 | RSM00023365 205 | RSM00023412 206 | RSM00023472 207 | RSM00023527 208 | RSM00023552 209 | RSM00023631 210 | RSM00023678 211 | RSM00023711 212 | RSM00023734 213 | RSM00023849 214 | RSM00023891 215 | RSM00023914 216 | RSM00023955 217 | RSM00023966 218 | RSM00024125 219 | RSM00024143 220 | RSM00024343 221 | RSM00024671 222 | RSM00024738 223 | RSM00024817 224 | RSM00024923 225 | RSM00024944 226 | RSM00024951 227 | RSM00025325 228 | RSM00025400 229 | RSM00025503 230 | RSM00025551 231 | RSM00025563 232 | RSM00025703 233 | RSM00025954 234 | RSM00026063 235 | KZ000035078 236 | KZ000035108 237 | KZ000035188 238 | KZ000035358 239 | KZ000035406 240 | KZ000035542 241 | KZ000035746 242 | KZ000036665 243 | KZ000036870 244 | RSM00034123 245 | RSM00034172 246 | RSM00034186 247 | RSM00034321 248 | RSM00034391 249 | RSM00034720 250 | RSM00034730 251 | RSM00034824 252 | RSM00034861 253 | RSM00036034 254 | RSM00036038 255 | RSM00036278 256 | RSM00037001 257 | UPM00033837 258 | UPM00033910 259 | BOM00033008 260 | RSM00031478 261 | RSM00031594 262 | RSM00031725 263 | RSM00031733 264 | RSM00031829 265 | RSM00031873 266 | RSM00031915 267 | RSM00032061 268 | RSM00032098 269 | RSM00032145 270 | RSM00032149 271 | RSM00032252 272 | RSM00032583 273 | RSM00032618 274 | UPM00033317 275 | UPM00033506 276 | UPM00033587 277 | AJ000037735 278 | AM000037686 279 | GG000037385 280 | GG000037545 281 | IS000002011 282 | IS000009972 283 | KZ000038198 284 | MG000044212 285 | MG000044288 286 | MG000044341 287 | RSM00037031 288 | RSM00037228 289 | RSM00037470 290 | RSM00037471 291 | SA000040438 292 | SY000040001 293 | TI000038599 294 | TI000038933 295 | TX000038763 296 | TX000038974 297 | TX000038987 298 | UZM00038262 299 | UZM00038413 300 | UZM00038457 301 | UZM00038618 302 | MP000061974 303 | MR000001403 304 | MR000001421 305 | MR000001437 306 | NG000001075 307 | NG000001080 308 | SHM00061902 309 | SU000062641 310 | TSE00147775 311 | JA000047401 312 | JA000047406 313 | JA000047407 314 | JA000047409 315 | JA000047413 316 | JA000047417 317 | JA000047420 318 | JA000047423 319 | JA000047428 320 | JA000047430 321 | JA000047433 322 | JA000047512 323 | JA000047570 324 | JA000047575 325 | JA000047582 326 | JA000047584 327 | JA000047588 328 | JA000047590 329 | JA000047592 330 | JA000047610 331 | JA000047615 332 | JA000047618 333 | JA000047636 334 | JA000047640 335 | JA000047662 336 | JA000047663 337 | JA000047666 338 | JA000047668 339 | JA000047670 340 | JA000047674 341 | JA000047677 342 | JA000047740 343 | JA000047742 344 | JA000047776 345 | JA000047777 346 | JA000047800 347 | JA000047814 348 | JA000047815 349 | JA000047817 350 | JA000047824 351 | JA000047831 352 | JA000047891 353 | JA000047897 354 | JA000047899 355 | JA000047912 356 | JA000047929 357 | TH000048327 358 | TH000048351 359 | TH000048354 360 | TH000048375 361 | TH000048405 362 | TH000048450 363 | CHM00054135 364 | CHM00054161 365 | CHM00054236 366 | CHM00054324 367 | CHM00054337 368 | CHM00054471 369 | CHM00054527 370 | CHM00054725 371 | CHM00054916 372 | CHM00055279 373 | CHM00056182 374 | CHM00056257 375 | CHM00056739 376 | CHM00056768 377 | CHM00056951 378 | CHM00056954 379 | CHM00057083 380 | CHM00057127 381 | CHM00057411 382 | CHM00057707 383 | CHM00057713 384 | CHM00057745 385 | CHM00058633 386 | CHM00058659 387 | CHM00058847 388 | CHM00058921 389 | CHM00059265 390 | CHM00059632 391 | CHM00059838 392 | SP000060010 393 | SP000060040 394 | SPE00120215 395 | SPE00120431 396 | CHM00050353 397 | CHM00050557 398 | CHM00050564 399 | CHM00050632 400 | CHM00050658 401 | CHM00050756 402 | CHM00050915 403 | CHM00050963 404 | CHM00050968 405 | CHM00051076 406 | CHM00051334 407 | CHM00051431 408 | CHM00051573 409 | CHM00051644 410 | CHM00051716 411 | CHM00051765 412 | CHM00051811 413 | CHM00051828 414 | CHM00052203 415 | CHM00052533 416 | CHM00052889 417 | CHM00053276 418 | CHM00053352 419 | CHM00053391 420 | CHM00053529 421 | CHM00053646 422 | CHM00053673 423 | CHM00053772 424 | CHM00053845 425 | CHM00053863 426 | CHM00053915 427 | CHM00054026 428 | CHM00054094 429 | TH000048501 430 | TH000048564 431 | CA001026270 432 | CA001031353 433 | CA001037650 434 | CA001057050 435 | CA001062745 436 | CA001098940 437 | CA001113540 438 | CA002300850 439 | CA002400654 440 | CA002402350 441 | CA003075040 442 | CA003081680 443 | CA005062922 444 | CA006010738 445 | CA006037775 446 | CA006075425 447 | CA006119500 448 | CA006158733 449 | CA007025250 450 | CA007040440 451 | CA007045400 452 | CA007047910 453 | CA007095480 454 | CA008204700 455 | CA008205700 456 | CA008403506 457 | CA008403800 458 | CA008501900 459 | BN000005335 460 | BN000005344 461 | CD000004750 462 | CD000004756 463 | CD000004758 464 | CF000004405 465 | CF000004453 466 | CF000004458 467 | CF000004459 468 | CT000004661 469 | GB000004510 470 | GB000004545 471 | GB000004550 472 | IV000005557 473 | IV000005562 474 | IV000005592 475 | IV000005599 476 | MZ000067237 477 | TO000005361 478 | TO000005380 479 | TO000005387 480 | UV000005502 481 | CA001054500 482 | CA001065010 483 | CA001077500 484 | CA001126510 485 | CA001183000 486 | CA001192940 487 | CA002100182 488 | CA002100700 489 | CA002200675 490 | CA002202400 491 | CA002203910 492 | CA002204100 493 | CA003012208 494 | CA003053520 495 | CA004016560 496 | CA004019035 497 | CA004056240 498 | CA004061861 499 | CA005023222 500 | CA005052880 501 | CA006034075 502 | CA007113534 503 | USW00012844 504 | USW00012944 505 | USW00013733 506 | USW00013882 507 | USW00013891 508 | USW00013894 509 | USW00013962 510 | USW00013984 511 | USW00014745 512 | USW00014848 513 | USW00023023 514 | USW00023065 515 | USW00024128 516 | USW00093822 517 | BC000068026 518 | BC000068032 519 | MZ000067323 520 | SF000068296 521 | SF000068712 522 | SHM00068906 523 | USW00025339 524 | WA007401540 525 | ZI000067761 526 | ZI000067765 527 | ZI000067781 528 | ZI000067843 529 | ZI000067867 530 | ZI000067975 531 | ZI000067977 532 | ZI000067983 533 | ZI000067991 534 | ASN00012065 535 | ASN00016065 536 | ASN00047007 537 | ASN00070080 538 | ASN00076047 539 | ASN00088043 540 | ASN00092038 541 | RP000098232 542 | RP000098430 543 | RP000098444 544 | RP000098836 545 | RP000098851 546 | AQW00061705 547 | ASN00003057 548 | ASN00009518 549 | ASN00014508 550 | ASN00015511 551 | ASN00027042 552 | ASN00028004 553 | ASN00035027 554 | ASN00035069 555 | ASN00037010 556 | ASN00038024 557 | ASN00039059 558 | ASN00039123 559 | ASN00043030 560 | ASN00056032 561 | BDM00078016 562 | CA002101300 563 | CA007016294 564 | MX000029030 565 | USC00244558 566 | USW00014742 567 | USW00014925 568 | USW00024157 569 | VE000080413 570 | WQW00041606 571 | -------------------------------------------------------------------------------- /gwgen/preproc.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Additional routines for preprocessing""" 3 | import tempfile 4 | import os.path as osp 5 | from collections import namedtuple 6 | import numpy as np 7 | import pandas as pd 8 | from psyplot.compat.pycompat import OrderedDict 9 | import gwgen.utils as utils 10 | from gwgen.utils import docstrings 11 | 12 | 13 | class CloudPreproc(utils.TaskBase): 14 | 15 | @property 16 | def task_data_dir(self): 17 | return osp.join(self.data_dir, 'eecra') 18 | 19 | _registry = [] 20 | 21 | 22 | class CloudInventory(CloudPreproc): 23 | """A task for computing the EECRA inventory for each station""" 24 | 25 | name = 'eecra_inventory' 26 | 27 | summary = 'Compute the inventory of the EECRA stations' 28 | 29 | http_xstall = 'http://cdiac.ornl.gov/ftp/ndp026c/XSTALL' 30 | 31 | _datafile = 'eecra_inventory.csv' 32 | 33 | dbname = 'eecra_inventory' 34 | 35 | has_run = True 36 | 37 | @property 38 | def setup_parallel(self): 39 | return self.setup_from == 'scratch' 40 | 41 | @property 42 | def default_config(self): 43 | return default_cloud_inventory_config()._replace( 44 | **super(CloudInventory, self).default_config._asdict()) 45 | 46 | @property 47 | def xstall_df(self): 48 | """The dataframe corresponding to the XSTALL stations""" 49 | use_xstall = self.task_config.xstall 50 | if utils.isstring(use_xstall): 51 | fname = self.task_config.no_xstall 52 | else: 53 | fname = tempfile.NamedTemporaryFile().name 54 | utils.download_file(self.http_xstall, fname) 55 | arr = np.loadtxt(fname, usecols=[1, 2, 3]) 56 | df = pd.DataFrame(arr, columns=['station_id', 'lat', 'lon']) 57 | df['station_id'] = df.station_id.astype(int) 58 | df.set_index('station_id', inplace=True) 59 | return df 60 | 61 | @classmethod 62 | def _modify_parser(cls, parser): 63 | parser.setup_args(default_cloud_inventory_config) 64 | cls.has_run = False 65 | parser, setup_grp, run_grp = super(CloudInventory, cls)._modify_parser( 66 | parser) 67 | parser.update_arg('xstall', group=setup_grp) 68 | cls.has_run = True 69 | return parser, setup_grp, run_grp 70 | 71 | def __init__(self, *args, **kwargs): 72 | super(CloudInventory, self).__init__(*args, **kwargs) 73 | self.__setup = False 74 | 75 | def setup(self, *args, **kwargs): 76 | self.__setup = True 77 | super(CloudInventory, self).setup(*args, **kwargs) 78 | 79 | def init_from_scratch(self): 80 | from gwgen.parameterization import HourlyCloud 81 | task = HourlyCloud.from_task(self) 82 | task.download_src(task.raw_dir) # make sure the source files exist 83 | 84 | def setup_from_file(self, **kwargs): 85 | """Set up the task from already stored files (and avoid locating the 86 | stations of this task)""" 87 | kwargs = self._split_kwargs(kwargs) 88 | for i, datafile in enumerate(utils.safe_list(self.datafile)): 89 | self._set_data(pd.read_csv(datafile, **kwargs[i]), i) 90 | 91 | def setup_from_db(self, **kwargs): 92 | """Set up the task from datatables already created (and avoid locating 93 | the stations of this task)""" 94 | kwargs = self._split_kwargs(kwargs) 95 | for i, dbname in enumerate(utils.safe_list(self.dbname)): 96 | self._set_data(pd.read_sql_query( 97 | "SELECT * FROM %s" % (self.dbname), 98 | self.engine, **kwargs[i]), i) 99 | 100 | def setup_from_scratch(self): 101 | from gwgen.parse_eecra import parse_file 102 | 103 | def compute(fname): 104 | g = parse_file(fname).groupby('station_id')[ 105 | ['lat', 'lon', 'year']] 106 | df = g.mean() 107 | df['counts'] = g.size() 108 | std = g.std() 109 | df['lon_std'] = std.lon 110 | df['lat_std'] = std.lat 111 | return df 112 | 113 | self.data = pd.concat(list(map(compute, self.stations))) 114 | 115 | def write2db(self, *args, **kwargs): 116 | if self.__setup: 117 | return 118 | kwargs.setdefault('if_exists', 'replace') 119 | super(CloudInventory, self).write2db(*args, **kwargs) 120 | 121 | def write2file(self, *args, **kwargs): 122 | if self.__setup: 123 | return 124 | super(CloudInventory, self).write2file(*args, **kwargs) 125 | 126 | def run(self, info): 127 | 128 | self.__setup = False 129 | 130 | if self.setup_from == 'scratch': 131 | df = self.data 132 | # we may use a parallel setup which requires a weighted average 133 | g = df.groupby(level='station_id') 134 | total_counts = g.counts.transform("sum") 135 | df['lat'] = df.counts / total_counts * df.lat 136 | df['lon'] = df.counts / total_counts * df.lon 137 | df['lat_std'] = (df.counts / total_counts) * df.lat_std ** 2 138 | df['lon_std'] = (df.counts / total_counts) * df.lon_std ** 2 139 | eecra = g.agg(OrderedDict([ 140 | ('lat', 'sum'), ('lon', 'sum'), ('lat_std', 'sum'), 141 | ('lon_std', 'sum'), 142 | ('year', ('min', 'max')), ('counts', 'sum')])) 143 | eecra.columns = ['lat', 'lon', 'lat_std', 'lon_std', 144 | 'firstyear', 'lastyear', 'counts'] 145 | eecra[['lat_std', 'lon_std']] **= 0.5 146 | 147 | use_xstall = self.task_config.xstall 148 | 149 | if use_xstall: 150 | to_replace = self.xstall_df 151 | # keep only matching stations 152 | to_replace = to_replace.join(eecra[[]], how='inner') 153 | eecra.loc[to_replace.index, ['lat', 'lon']] = to_replace 154 | self.data = eecra 155 | 156 | if self.task_config.to_csv: 157 | self.write2file() 158 | if self.task_config.to_db: 159 | self.write2db() 160 | 161 | 162 | class CloudGHCNMap(CloudPreproc): 163 | """A task for computing the EECRA inventory for each station""" 164 | 165 | name = 'eecra_ghcn_map' 166 | 167 | setup_requires = ['eecra_inventory'] 168 | 169 | summary = 'Compute the inventory of the EECRA stations' 170 | 171 | _datafile = 'eecra_ghcn_map.csv' 172 | 173 | dbname = 'eecra_ghcn_map' 174 | 175 | has_run = True 176 | 177 | @property 178 | def default_config(self): 179 | return default_cloud_ghcn_map_config()._replace( 180 | **super(CloudGHCNMap, self).default_config._asdict()) 181 | 182 | @classmethod 183 | def _modify_parser(cls, parser): 184 | parser.setup_args(default_cloud_ghcn_map_config) 185 | cls.has_run = False 186 | parser, setup_grp, run_grp = super(CloudGHCNMap, cls)._modify_parser( 187 | parser) 188 | parser.update_arg('max_distance', group=setup_grp, short='md') 189 | parser.pop_arg('to_db') 190 | parser.pop_arg('setup_from') 191 | cls.has_run = True 192 | return parser, setup_grp, run_grp 193 | 194 | def __init__(self, *args, **kwargs): 195 | super(CloudGHCNMap, self).__init__(*args, **kwargs) 196 | self.task_config = self.task_config._replace( 197 | setup_from='scratch', to_db=False) 198 | self.__setup = False 199 | 200 | def setup(self, *args, **kwargs): 201 | self.__setup = True 202 | super(CloudGHCNMap, self).setup(*args, **kwargs) 203 | 204 | def init_from_scratch(self): 205 | from gwgen.parameterization import HourlyCloud 206 | task = HourlyCloud.from_task(self) 207 | task.download_src(task.raw_dir) # make sure the source files exist 208 | 209 | def setup_from_scratch(self): 210 | """Does nothing but initializing an empty data frame. The real work is 211 | done in the :meth:`run` method""" 212 | self.data = pd.DataFrame([], columns=['station_id', 'dist'], 213 | index=pd.Index([], name='id')) 214 | 215 | def write2db(self, *args, **kwargs): 216 | raise NotImplementedError( 217 | 'The data is always written to the database!') 218 | 219 | def write2file(self, *args, **kwargs): 220 | if self.__setup: 221 | return 222 | super(CloudGHCNMap, self).write2file(*args, **kwargs) 223 | 224 | def run(self, info): 225 | def create_geog(table): 226 | conn.execute( 227 | 'ALTER TABLE %s ADD COLUMN geog geography(POINT,4326) ;' % ( 228 | table)) 229 | conn.execute(""" 230 | UPDATE %s 231 | SET geog = ST_SetSRID(ST_MakePoint(lon,lat),4326);""" % ( 232 | table)) 233 | conn.execute( 234 | 'CREATE INDEX {0}_geog ON {0} USING GIST (geog);'.format(table) 235 | ) 236 | from gwgen.evaluation import EvaluationPreparation 237 | self.__setup = False 238 | 239 | inv = self.eecra_inventory 240 | 241 | if not self.engine.has_table(inv.dbname): 242 | inv.write2db() 243 | conn = self.engine.connect() 244 | if 'geog' not in pd.read_sql_query('SELECT * FROM %s LIMIT 0' % ( 245 | inv.name), conn).columns: 246 | create_geog(inv.dbname) 247 | 248 | t = EvaluationPreparation.from_task(self) 249 | # download inventory 250 | t.download_src() 251 | ghcn = t.station_list 252 | ghcn = ghcn.ix[ghcn.vname == 'PRCP'].set_index('id') 253 | ghcn.to_sql('ghcn_inventory', self.engine, if_exists='replace') 254 | create_geog('ghcn_inventory') 255 | 256 | conn.execute("DROP TABLE IF EXISTS eecra_ghcn_map;") 257 | 258 | conn.execute(""" 259 | CREATE TABLE eecra_ghcn_map AS ( 260 | SELECT DISTINCT ON (id) id, station_id, dist FROM ( 261 | SELECT DISTINCT ON (a.station_id) 262 | b.id, a.station_id, ST_Distance(a.geog, b.geog) AS dist 263 | FROM eecra_inventory a 264 | INNER JOIN ghcn_inventory b ON ST_DWithin( 265 | a.geog, b.geog, 1000) 266 | ORDER BY a.station_id, ST_Distance(a.geog, b.geog)) foo 267 | ORDER BY id, dist);""") 268 | 269 | self.data = pd.read_sql('eecra_ghcn_map', self.engine, index_col='id') 270 | conn.close() 271 | 272 | if self.task_config.to_csv: 273 | self.write2file() 274 | 275 | 276 | CloudInventoryConfig = namedtuple( 277 | 'CloudInventoryConfig', ['xstall'] + list(utils.TaskConfig._fields)) 278 | 279 | 280 | # to_db is set to True by default because it is required 281 | docstrings.delete_params('TaskConfig.parameters', 'to_db', 'setup_from') 282 | 283 | 284 | CloudInventoryConfig = utils.append_doc( 285 | CloudInventoryConfig, docstrings.get_sections(docstrings.dedents(""" 286 | Parameters 287 | ---------- 288 | xstall: bool or str 289 | If True (default), download the XSTALL file from %s. 290 | This file contains some estimates of station longitude and latitude. 291 | If ``False`` or empty string, the file is not used, otherwise, if set 292 | with a string, it is interpreted as the path to the local file 293 | %%(TaskConfig.parameters.no_to_db|setup_from)s 294 | """ % CloudInventory.http_xstall), 'CloudInventoryConfig')) 295 | 296 | 297 | @docstrings.dedent 298 | def default_cloud_inventory_config(xstall=True, *args, **kwargs): 299 | """ 300 | Default config for :class:`CloudInventory` 301 | 302 | Parameters 303 | ---------- 304 | %(CloudInventoryConfig.parameters)s""" 305 | return CloudInventoryConfig( 306 | xstall, *utils.default_config(*args, **kwargs)) 307 | 308 | 309 | CloudGHCNMapConfig = namedtuple( 310 | 'CloudGHCNMapConfig', ['max_distance'] + list(utils.TaskConfig._fields)) 311 | 312 | 313 | # to_db is set to True by default because it is required 314 | docstrings.delete_params('TaskConfig.parameters', 'to_db', 'setup_from') 315 | 316 | 317 | CloudGHCNMapConfig = utils.append_doc( 318 | CloudGHCNMapConfig, docstrings.get_sections(docstrings.dedents(""" 319 | Parameters 320 | ---------- 321 | max_distance: float 322 | The maximum distance in meters for which we consider two stations as 323 | equal (Default: 1000m) 324 | %(TaskConfig.parameters.no_to_db|setup_from)s 325 | """), 'CloudGHCNMapConfig')) 326 | 327 | 328 | @docstrings.dedent 329 | def default_cloud_ghcn_map_config(max_distance=1000., *args, **kwargs): 330 | """ 331 | Default config for :class:`CloudGHCNMap` 332 | 333 | Parameters 334 | ---------- 335 | %(CloudGHCNMapConfig.parameters)s""" 336 | return CloudGHCNMapConfig( 337 | max_distance, *utils.default_config(*args, **kwargs)) 338 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GWGEN: A global weather generator for daily data 2 | Copyright (C) 2017 ARVE-Research 3 | 4 | This program is free software; you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation; either version 2 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | A copy of the GNU General Public License is provided down below. 15 | 16 | 17 | GNU GENERAL PUBLIC LICENSE 18 | Version 2, June 1991 19 | 20 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 21 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 22 | Everyone is permitted to copy and distribute verbatim copies 23 | of this license document, but changing it is not allowed. 24 | 25 | Preamble 26 | 27 | The licenses for most software are designed to take away your 28 | freedom to share and change it. By contrast, the GNU General Public 29 | License is intended to guarantee your freedom to share and change free 30 | software--to make sure the software is free for all its users. This 31 | General Public License applies to most of the Free Software 32 | Foundation's software and to any other program whose authors commit to 33 | using it. (Some other Free Software Foundation software is covered by 34 | the GNU Lesser General Public License instead.) You can apply it to 35 | your programs, too. 36 | 37 | When we speak of free software, we are referring to freedom, not 38 | price. Our General Public Licenses are designed to make sure that you 39 | have the freedom to distribute copies of free software (and charge for 40 | this service if you wish), that you receive source code or can get it 41 | if you want it, that you can change the software or use pieces of it 42 | in new free programs; and that you know you can do these things. 43 | 44 | To protect your rights, we need to make restrictions that forbid 45 | anyone to deny you these rights or to ask you to surrender the rights. 46 | These restrictions translate to certain responsibilities for you if you 47 | distribute copies of the software, or if you modify it. 48 | 49 | For example, if you distribute copies of such a program, whether 50 | gratis or for a fee, you must give the recipients all the rights that 51 | you have. You must make sure that they, too, receive or can get the 52 | source code. And you must show them these terms so they know their 53 | rights. 54 | 55 | We protect your rights with two steps: (1) copyright the software, and 56 | (2) offer you this license which gives you legal permission to copy, 57 | distribute and/or modify the software. 58 | 59 | Also, for each author's protection and ours, we want to make certain 60 | that everyone understands that there is no warranty for this free 61 | software. If the software is modified by someone else and passed on, we 62 | want its recipients to know that what they have is not the original, so 63 | that any problems introduced by others will not reflect on the original 64 | authors' reputations. 65 | 66 | Finally, any free program is threatened constantly by software 67 | patents. We wish to avoid the danger that redistributors of a free 68 | program will individually obtain patent licenses, in effect making the 69 | program proprietary. To prevent this, we have made it clear that any 70 | patent must be licensed for everyone's free use or not licensed at all. 71 | 72 | The precise terms and conditions for copying, distribution and 73 | modification follow. 74 | 75 | GNU GENERAL PUBLIC LICENSE 76 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 77 | 78 | 0. This License applies to any program or other work which contains 79 | a notice placed by the copyright holder saying it may be distributed 80 | under the terms of this General Public License. The "Program", below, 81 | refers to any such program or work, and a "work based on the Program" 82 | means either the Program or any derivative work under copyright law: 83 | that is to say, a work containing the Program or a portion of it, 84 | either verbatim or with modifications and/or translated into another 85 | language. (Hereinafter, translation is included without limitation in 86 | the term "modification".) Each licensee is addressed as "you". 87 | 88 | Activities other than copying, distribution and modification are not 89 | covered by this License; they are outside its scope. The act of 90 | running the Program is not restricted, and the output from the Program 91 | is covered only if its contents constitute a work based on the 92 | Program (independent of having been made by running the Program). 93 | Whether that is true depends on what the Program does. 94 | 95 | 1. You may copy and distribute verbatim copies of the Program's 96 | source code as you receive it, in any medium, provided that you 97 | conspicuously and appropriately publish on each copy an appropriate 98 | copyright notice and disclaimer of warranty; keep intact all the 99 | notices that refer to this License and to the absence of any warranty; 100 | and give any other recipients of the Program a copy of this License 101 | along with the Program. 102 | 103 | You may charge a fee for the physical act of transferring a copy, and 104 | you may at your option offer warranty protection in exchange for a fee. 105 | 106 | 2. You may modify your copy or copies of the Program or any portion 107 | of it, thus forming a work based on the Program, and copy and 108 | distribute such modifications or work under the terms of Section 1 109 | above, provided that you also meet all of these conditions: 110 | 111 | a) You must cause the modified files to carry prominent notices 112 | stating that you changed the files and the date of any change. 113 | 114 | b) You must cause any work that you distribute or publish, that in 115 | whole or in part contains or is derived from the Program or any 116 | part thereof, to be licensed as a whole at no charge to all third 117 | parties under the terms of this License. 118 | 119 | c) If the modified program normally reads commands interactively 120 | when run, you must cause it, when started running for such 121 | interactive use in the most ordinary way, to print or display an 122 | announcement including an appropriate copyright notice and a 123 | notice that there is no warranty (or else, saying that you provide 124 | a warranty) and that users may redistribute the program under 125 | these conditions, and telling the user how to view a copy of this 126 | License. (Exception: if the Program itself is interactive but 127 | does not normally print such an announcement, your work based on 128 | the Program is not required to print an announcement.) 129 | 130 | These requirements apply to the modified work as a whole. If 131 | identifiable sections of that work are not derived from the Program, 132 | and can be reasonably considered independent and separate works in 133 | themselves, then this License, and its terms, do not apply to those 134 | sections when you distribute them as separate works. But when you 135 | distribute the same sections as part of a whole which is a work based 136 | on the Program, the distribution of the whole must be on the terms of 137 | this License, whose permissions for other licensees extend to the 138 | entire whole, and thus to each and every part regardless of who wrote it. 139 | 140 | Thus, it is not the intent of this section to claim rights or contest 141 | your rights to work written entirely by you; rather, the intent is to 142 | exercise the right to control the distribution of derivative or 143 | collective works based on the Program. 144 | 145 | In addition, mere aggregation of another work not based on the Program 146 | with the Program (or with a work based on the Program) on a volume of 147 | a storage or distribution medium does not bring the other work under 148 | the scope of this License. 149 | 150 | 3. You may copy and distribute the Program (or a work based on it, 151 | under Section 2) in object code or executable form under the terms of 152 | Sections 1 and 2 above provided that you also do one of the following: 153 | 154 | a) Accompany it with the complete corresponding machine-readable 155 | source code, which must be distributed under the terms of Sections 156 | 1 and 2 above on a medium customarily used for software interchange; or, 157 | 158 | b) Accompany it with a written offer, valid for at least three 159 | years, to give any third party, for a charge no more than your 160 | cost of physically performing source distribution, a complete 161 | machine-readable copy of the corresponding source code, to be 162 | distributed under the terms of Sections 1 and 2 above on a medium 163 | customarily used for software interchange; or, 164 | 165 | c) Accompany it with the information you received as to the offer 166 | to distribute corresponding source code. (This alternative is 167 | allowed only for noncommercial distribution and only if you 168 | received the program in object code or executable form with such 169 | an offer, in accord with Subsection b above.) 170 | 171 | The source code for a work means the preferred form of the work for 172 | making modifications to it. For an executable work, complete source 173 | code means all the source code for all modules it contains, plus any 174 | associated interface definition files, plus the scripts used to 175 | control compilation and installation of the executable. However, as a 176 | special exception, the source code distributed need not include 177 | anything that is normally distributed (in either source or binary 178 | form) with the major components (compiler, kernel, and so on) of the 179 | operating system on which the executable runs, unless that component 180 | itself accompanies the executable. 181 | 182 | If distribution of executable or object code is made by offering 183 | access to copy from a designated place, then offering equivalent 184 | access to copy the source code from the same place counts as 185 | distribution of the source code, even though third parties are not 186 | compelled to copy the source along with the object code. 187 | 188 | 4. You may not copy, modify, sublicense, or distribute the Program 189 | except as expressly provided under this License. Any attempt 190 | otherwise to copy, modify, sublicense or distribute the Program is 191 | void, and will automatically terminate your rights under this License. 192 | However, parties who have received copies, or rights, from you under 193 | this License will not have their licenses terminated so long as such 194 | parties remain in full compliance. 195 | 196 | 5. You are not required to accept this License, since you have not 197 | signed it. However, nothing else grants you permission to modify or 198 | distribute the Program or its derivative works. These actions are 199 | prohibited by law if you do not accept this License. Therefore, by 200 | modifying or distributing the Program (or any work based on the 201 | Program), you indicate your acceptance of this License to do so, and 202 | all its terms and conditions for copying, distributing or modifying 203 | the Program or works based on it. 204 | 205 | 6. Each time you redistribute the Program (or any work based on the 206 | Program), the recipient automatically receives a license from the 207 | original licensor to copy, distribute or modify the Program subject to 208 | these terms and conditions. You may not impose any further 209 | restrictions on the recipients' exercise of the rights granted herein. 210 | You are not responsible for enforcing compliance by third parties to 211 | this License. 212 | 213 | 7. If, as a consequence of a court judgment or allegation of patent 214 | infringement or for any other reason (not limited to patent issues), 215 | conditions are imposed on you (whether by court order, agreement or 216 | otherwise) that contradict the conditions of this License, they do not 217 | excuse you from the conditions of this License. If you cannot 218 | distribute so as to satisfy simultaneously your obligations under this 219 | License and any other pertinent obligations, then as a consequence you 220 | may not distribute the Program at all. For example, if a patent 221 | license would not permit royalty-free redistribution of the Program by 222 | all those who receive copies directly or indirectly through you, then 223 | the only way you could satisfy both it and this License would be to 224 | refrain entirely from distribution of the Program. 225 | 226 | If any portion of this section is held invalid or unenforceable under 227 | any particular circumstance, the balance of the section is intended to 228 | apply and the section as a whole is intended to apply in other 229 | circumstances. 230 | 231 | It is not the purpose of this section to induce you to infringe any 232 | patents or other property right claims or to contest validity of any 233 | such claims; this section has the sole purpose of protecting the 234 | integrity of the free software distribution system, which is 235 | implemented by public license practices. Many people have made 236 | generous contributions to the wide range of software distributed 237 | through that system in reliance on consistent application of that 238 | system; it is up to the author/donor to decide if he or she is willing 239 | to distribute software through any other system and a licensee cannot 240 | impose that choice. 241 | 242 | This section is intended to make thoroughly clear what is believed to 243 | be a consequence of the rest of this License. 244 | 245 | 8. If the distribution and/or use of the Program is restricted in 246 | certain countries either by patents or by copyrighted interfaces, the 247 | original copyright holder who places the Program under this License 248 | may add an explicit geographical distribution limitation excluding 249 | those countries, so that distribution is permitted only in or among 250 | countries not thus excluded. In such case, this License incorporates 251 | the limitation as if written in the body of this License. 252 | 253 | 9. The Free Software Foundation may publish revised and/or new versions 254 | of the General Public License from time to time. Such new versions will 255 | be similar in spirit to the present version, but may differ in detail to 256 | address new problems or concerns. 257 | 258 | Each version is given a distinguishing version number. If the Program 259 | specifies a version number of this License which applies to it and "any 260 | later version", you have the option of following the terms and conditions 261 | either of that version or of any later version published by the Free 262 | Software Foundation. If the Program does not specify a version number of 263 | this License, you may choose any version ever published by the Free Software 264 | Foundation. 265 | 266 | 10. If you wish to incorporate parts of the Program into other free 267 | programs whose distribution conditions are different, write to the author 268 | to ask for permission. For software which is copyrighted by the Free 269 | Software Foundation, write to the Free Software Foundation; we sometimes 270 | make exceptions for this. Our decision will be guided by the two goals 271 | of preserving the free status of all derivatives of our free software and 272 | of promoting the sharing and reuse of software generally. 273 | 274 | NO WARRANTY 275 | 276 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 277 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 278 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 279 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 280 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 281 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 282 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 283 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 284 | REPAIR OR CORRECTION. 285 | 286 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 287 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 288 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 289 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 290 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 291 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 292 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 293 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 294 | POSSIBILITY OF SUCH DAMAGES. 295 | 296 | END OF TERMS AND CONDITIONS 297 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # psyplot documentation build configuration file, created by 4 | # sphinx-quickstart on Mon Jul 20 18:01:33 2015. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sphinx 16 | from docutils import nodes, utils 17 | from sphinx.util.nodes import split_explicit_title 18 | import os 19 | import re 20 | import six 21 | from itertools import product 22 | import gwgen 23 | 24 | # If extensions (or modules to document with autodoc) are in another directory, 25 | # add these directories to sys.path here. If the directory is relative to the 26 | # documentation root, use os.path.abspath to make it absolute, like shown here. 27 | #sys.path.insert(0, os.path.abspath('.')) 28 | 29 | # -- General configuration ------------------------------------------------ 30 | 31 | # If your documentation needs a minimal Sphinx version, state it here. 32 | #needs_sphinx = '1.0' 33 | 34 | # Add any Sphinx extension module names here, as strings. They can be 35 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 36 | # ones. 37 | extensions = [ 38 | 'sphinx.ext.doctest', 39 | 'sphinx.ext.intersphinx', 40 | 'sphinx.ext.autosummary', 41 | 'sphinx.ext.todo', 42 | 'sphinx.ext.viewcode', 43 | 'sphinx.ext.mathjax', 44 | 'sphinx.ext.napoleon', 45 | 'matplotlib.sphinxext.plot_directive', 46 | 'IPython.sphinxext.ipython_console_highlighting', 47 | 'IPython.sphinxext.ipython_directive', 48 | 'sphinxarg.ext', 49 | 'autodocsumm', 50 | 'sphinxfortran.fortran_domain', 51 | 'sphinxfortran.fortran_autodoc', 52 | ] 53 | 54 | fortran_src = '../gwgen/src' 55 | 56 | # Add any paths that contain templates here, relative to this directory. 57 | templates_path = ['_templates'] 58 | 59 | # on_rtd is whether we are on readthedocs.org, this line of code grabbed from 60 | # docs.readthedocs.org 61 | on_rtd = os.environ.get('READTHEDOCS', None) == 'True' 62 | 63 | # boolean controlling wether to calculate the examples or not 64 | # process_examples = False #not on_rtd 65 | 66 | # The cdo example would require the installation of climate data operators 67 | # which is a bit of an overkill 68 | example_gallery_config = {'dont_preprocess': ['../examples/example_cdo.ipynb']} 69 | 70 | napoleon_use_admonition_for_examples = True 71 | 72 | # The suffix(es) of source filenames. 73 | # You can specify multiple suffix as a list of string: 74 | source_suffix = '.rst' 75 | 76 | # The encoding of source files. 77 | # source_encoding = 'utf-8-sig' 78 | 79 | # The master toctree document. 80 | master_doc = 'index' 81 | 82 | autodoc_default_flags = ['show_inheritance', 'autosummary'] 83 | 84 | autoclass_content = 'both' 85 | 86 | # not_document_data = [] 87 | 88 | ipython_savefig_dir = os.path.join(os.path.dirname(__file__), '_static') 89 | 90 | # General information about the project. 91 | project = u'gwgen' 92 | copyright = u'2016, Philipp Sommer' 93 | author = u'Philipp Sommer, Jed Kaplan' 94 | 95 | # The version info for the project you're documenting, acts as replacement for 96 | # |version| and |release|, also used in various other places throughout the 97 | # built documents. 98 | # 99 | # The short X.Y version. 100 | version = re.match('\d+\.\d+\.\d+', gwgen.__version__).group() 101 | # The full version, including alpha/beta/rc tags. 102 | release = gwgen.__version__ 103 | 104 | # The language for content autogenerated by Sphinx. Refer to documentation 105 | # for a list of supported languages. 106 | # 107 | # This is also used if you do content translation via gettext catalogs. 108 | # Usually you set "language" from the command line for these cases. 109 | language = None 110 | 111 | # There are two options for replacing |today|: either, you set today to some 112 | # non-false value, then it is used: 113 | #today = '' 114 | # Else, today_fmt is used as the format for a strftime call. 115 | #today_fmt = '%B %d, %Y' 116 | 117 | # List of patterns, relative to source directory, that match files and 118 | # directories to ignore when looking for source files. 119 | exclude_patterns = ['_build'] 120 | 121 | # The reST default role (used for this markup: `text`) to use for all 122 | # documents. 123 | #default_role = None 124 | 125 | # If true, '()' will be appended to :func: etc. cross-reference text. 126 | #add_function_parentheses = True 127 | 128 | # If true, the current module name will be prepended to all description 129 | # unit titles (such as .. function::). 130 | #add_module_names = True 131 | 132 | # If true, sectionauthor and moduleauthor directives will be shown in the 133 | # output. They are ignored by default. 134 | #show_authors = False 135 | 136 | # The name of the Pygments (syntax highlighting) style to use. 137 | pygments_style = 'sphinx' 138 | 139 | # A list of ignored prefixes for module index sorting. 140 | #modindex_common_prefix = [] 141 | 142 | # If true, keep warnings as "system message" paragraphs in the built documents. 143 | #keep_warnings = False 144 | 145 | # If true, `todo` and `todoList` produce output, else they produce nothing. 146 | todo_include_todos = True 147 | 148 | 149 | # -- Options for HTML output ---------------------------------------------- 150 | 151 | # The theme to use for HTML and HTML Help pages. See the documentation for 152 | # a list of builtin themes. 153 | 154 | html_theme = 'alabaster' 155 | 156 | # Add any paths that contain custom static files (such as style sheets) here, 157 | # relative to this directory. They are copied after the builtin static files, 158 | # so a file named "default.css" will overwrite the builtin "default.css". 159 | html_static_path = ['_static'] 160 | 161 | # otherwise, readthedocs.org uses their theme by default, so no need to specify 162 | 163 | # Theme options are theme-specific and customize the look and feel of a theme 164 | # further. For a list of options available for each theme, see the 165 | # documentation. 166 | html_theme_options = {'travis_button': 'true', 167 | 'github_user': 'ARVE-Research', 168 | 'github_repo': 'gwgen', 169 | 'github_button': 'true'} 170 | 171 | # Add any paths that contain custom themes here, relative to this directory. 172 | #html_theme_path = [] 173 | 174 | # The name for this set of Sphinx documents. If None, it defaults to 175 | # " v documentation". 176 | #html_title = None 177 | 178 | # A shorter title for the navigation bar. Default is the same as html_title. 179 | #html_short_title = None 180 | 181 | # The name of an image file (relative to this directory) to place at the top 182 | # of the sidebar. 183 | #html_logo = None 184 | 185 | # The name of an image file (within the static path) to use as favicon of the 186 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 187 | # pixels large. 188 | #html_favicon = None 189 | 190 | # Add any extra paths that contain custom files (such as robots.txt or 191 | # .htaccess) here, relative to this directory. These files are copied 192 | # directly to the root of the documentation. 193 | #html_extra_path = [] 194 | 195 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 196 | # using the given strftime format. 197 | #html_last_updated_fmt = '%b %d, %Y' 198 | 199 | # If true, SmartyPants will be used to convert quotes and dashes to 200 | # typographically correct entities. 201 | #html_use_smartypants = True 202 | 203 | # Custom sidebar templates, maps document names to template names. 204 | html_sidebars = { 205 | '**': [ 206 | 'about.html', 207 | 'navigation.html', 208 | 'localtoc.html', 209 | 'relations.html', 210 | 'searchbox.html', 211 | 'sourcelink.html', 212 | ] 213 | } 214 | 215 | # Additional templates that should be rendered to pages, maps page names to 216 | # template names. 217 | #html_additional_pages = {} 218 | 219 | # If false, no module index is generated. 220 | #html_domain_indices = True 221 | 222 | # If false, no index is generated. 223 | #html_use_index = True 224 | 225 | # If true, the index is split into individual pages for each letter. 226 | #html_split_index = False 227 | 228 | # If true, links to the reST sources are added to the pages. 229 | #html_show_sourcelink = True 230 | 231 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 232 | #html_show_sphinx = True 233 | 234 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 235 | #html_show_copyright = True 236 | 237 | # If true, an OpenSearch description file will be output, and all pages will 238 | # contain a tag referring to it. The value of this option must be the 239 | # base URL from which the finished HTML is served. 240 | #html_use_opensearch = '' 241 | 242 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 243 | #html_file_suffix = None 244 | 245 | # Language to be used for generating the HTML full-text search index. 246 | # Sphinx supports the following languages: 247 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' 248 | # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' 249 | #html_search_language = 'en' 250 | 251 | # A dictionary with options for the search language support, empty by default. 252 | # Now only 'ja' uses this config value 253 | #html_search_options = {'type': 'default'} 254 | 255 | # The name of a javascript file (relative to the configuration directory) that 256 | # implements a search results scorer. If empty, the default will be used. 257 | #html_search_scorer = 'scorer.js' 258 | 259 | # Output file base name for HTML help builder. 260 | htmlhelp_basename = 'gwgendoc' 261 | 262 | # -- Options for LaTeX output --------------------------------------------- 263 | 264 | latex_elements = { 265 | # The paper size ('letterpaper' or 'a4paper'). 266 | #'papersize': 'letterpaper', 267 | 268 | # The font size ('10pt', '11pt' or '12pt'). 269 | #'pointsize': '10pt', 270 | 271 | # Additional stuff for the LaTeX preamble. 272 | #'preamble': '', 273 | 'preamble': '\setcounter{tocdepth}{10}' 274 | 275 | # Latex figure (float) alignment 276 | #'figure_align': 'htbp', 277 | } 278 | 279 | # Grouping the document tree into LaTeX files. List of tuples 280 | # (source start file, target name, title, 281 | # author, documentclass [howto, manual, or own class]). 282 | latex_documents = [ 283 | (master_doc, 'gwgen.tex', u'gwgen Documentation', 284 | u'Philipp Sommer', 'manual'), 285 | ] 286 | 287 | # The name of an image file (relative to this directory) to place at the top of 288 | # the title page. 289 | #latex_logo = None 290 | 291 | # For "manual" documents, if this is true, then toplevel headings are parts, 292 | # not chapters. 293 | #latex_use_parts = False 294 | 295 | # If true, show page references after internal links. 296 | #latex_show_pagerefs = False 297 | 298 | # If true, show URL addresses after external links. 299 | #latex_show_urls = False 300 | 301 | # Documents to append as an appendix to all manuals. 302 | #latex_appendices = [] 303 | 304 | # If false, no module index is generated. 305 | #latex_domain_indices = True 306 | 307 | 308 | # -- Options for manual page output --------------------------------------- 309 | 310 | # One entry per manual page. List of tuples 311 | # (source start file, name, description, authors, manual section). 312 | man_pages = [ 313 | (master_doc, 'gwgen', u'gwgen Documentation', 314 | [author], 1) 315 | ] 316 | 317 | # If true, show URL addresses after external links. 318 | #man_show_urls = False 319 | 320 | 321 | # -- Options for Texinfo output ------------------------------------------- 322 | 323 | # Grouping the document tree into Texinfo files. List of tuples 324 | # (source start file, target name, title, author, 325 | # dir menu entry, description, category) 326 | texinfo_documents = [ 327 | (master_doc, 'gwgen', u'gwgen Documentation', 328 | author, 'gwgen', 'A global weather generator for daily data', 329 | 'Miscellaneous'), 330 | ] 331 | 332 | # Documents to append as an appendix to all manuals. 333 | #texinfo_appendices = [] 334 | 335 | # If false, no module index is generated. 336 | #texinfo_domain_indices = True 337 | 338 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 339 | #texinfo_show_urls = 'footnote' 340 | 341 | # If true, do not generate a @detailmenu in the "Top" node's menu. 342 | #texinfo_no_detailmenu = False 343 | 344 | 345 | # -- Options for Epub output ---------------------------------------------- 346 | 347 | # Bibliographic Dublin Core info. 348 | epub_title = project 349 | epub_author = author 350 | epub_publisher = author 351 | epub_copyright = copyright 352 | 353 | # The basename for the epub file. It defaults to the project name. 354 | #epub_basename = project 355 | 356 | # The HTML theme for the epub output. Since the default themes are not optimized 357 | # for small screen space, using the same theme for HTML and epub output is 358 | # usually not wise. This defaults to 'epub', a theme designed to save visual 359 | # space. 360 | #epub_theme = 'epub' 361 | 362 | # The language of the text. It defaults to the language option 363 | # or 'en' if the language is not set. 364 | #epub_language = '' 365 | 366 | # The scheme of the identifier. Typical schemes are ISBN or URL. 367 | #epub_scheme = '' 368 | 369 | # The unique identifier of the text. This can be a ISBN number 370 | # or the project homepage. 371 | #epub_identifier = '' 372 | 373 | # A unique identification for the text. 374 | #epub_uid = '' 375 | 376 | # A tuple containing the cover image and cover page html template filenames. 377 | #epub_cover = () 378 | 379 | # A sequence of (type, uri, title) tuples for the guide element of content.opf. 380 | #epub_guide = () 381 | 382 | # HTML files that should be inserted before the pages created by sphinx. 383 | # The format is a list of tuples containing the path and title. 384 | #epub_pre_files = [] 385 | 386 | # HTML files shat should be inserted after the pages created by sphinx. 387 | # The format is a list of tuples containing the path and title. 388 | #epub_post_files = [] 389 | 390 | # A list of files that should not be packed into the epub file. 391 | epub_exclude_files = ['search.html'] 392 | 393 | # The depth of the table of contents in toc.ncx. 394 | #epub_tocdepth = 3 395 | 396 | # Allow duplicate toc entries. 397 | #epub_tocdup = True 398 | 399 | # Choose between 'default' and 'includehidden'. 400 | #epub_tocscope = 'default' 401 | 402 | # Fix unsupported image types using the Pillow. 403 | #epub_fix_images = False 404 | 405 | # Scale large images. 406 | #epub_max_image_width = 0 407 | 408 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 409 | #epub_show_urls = 'inline' 410 | 411 | # If false, no index is generated. 412 | #epub_use_index = True 413 | 414 | # Example configuration for intersphinx: refer to the Python standard library. 415 | intersphinx_mapping = { 416 | 'pandas': ('http://pandas.pydata.org/pandas-docs/stable/', None), 417 | 'numpy': ('http://docs.scipy.org/doc/numpy/', None), 418 | 'matplotlib': ('http://matplotlib.org/', None), 419 | 'sphinx': ('http://www.sphinx-doc.org/en/stable/', None), 420 | 'xarray': ('http://xarray.pydata.org/en/stable/', None), 421 | 'cartopy': ('http://scitools.org.uk/cartopy/docs/latest/', None), 422 | 'psyplot': ('http://psyplot.readthedocs.io/en/latest/', None), 423 | 'docrep': ('http://docrep.readthedocs.io/en/latest/', None), 424 | 'funcargparse': ('http://funcargparse.readthedocs.io/en/latest/', None), 425 | 'model_organization': ( 426 | 'http://model-organization.readthedocs.io/en/latest/', None), 427 | } 428 | if six.PY3: 429 | intersphinx_mapping['python'] = ('https://docs.python.org/3.4/', None) 430 | else: 431 | intersphinx_mapping['python'] = ('https://docs.python.org/2.7/', None) 432 | 433 | 434 | replacements = {} 435 | 436 | 437 | def link_aliases(app, what, name, obj, options, lines): 438 | for (key, val), (i, line) in product(six.iteritems(replacements), 439 | enumerate(lines)): 440 | lines[i] = line.replace(key, val) 441 | 442 | 443 | def doi_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): 444 | """Role for DOI. Copied from 445 | https://gist.github.com/jonls/b910afc46473b02597c4""" 446 | text = utils.unescape(text) 447 | has_explicit_title, title, part = split_explicit_title(text) 448 | full_url = 'https://doi.org/' + part 449 | if not has_explicit_title: 450 | title = 'doi:' + part 451 | pnode = nodes.reference(title, title, internal=False, refuri=full_url) 452 | return [pnode], [] 453 | 454 | 455 | def setup_link_role(app): 456 | app.add_role('doi', doi_role) 457 | 458 | 459 | def setup(app): 460 | app.connect('builder-inited', setup_link_role) 461 | app.connect('autodoc-process-docstring', link_aliases) 462 | return {'version': sphinx.__display_version__, 'parallel_read_safe': True} 463 | -------------------------------------------------------------------------------- /gwgen/mo_parseeecra.f90: -------------------------------------------------------------------------------- 1 | module parseeecra 2 | 3 | use omp_lib 4 | 5 | contains 6 | 7 | subroutine parse_file( & 8 | infile, year, nrecords, & 9 | yr,mn,dy,hr, & ! year,month,day,hour 10 | IB, & ! sky brightness indicator 11 | LAT, & ! latitude 12 | LON, & ! longitude 13 | station_id, & ! land: station number, ship: source deck, ship type 14 | LO, & ! land/ocean indicator 15 | ww, & ! present weather 16 | N, & ! total cloud cover 17 | Nh, & ! lower cloud amount 18 | h, & ! lower cloud base height 19 | CL, & ! low cloud type 20 | CM, & ! middle cloud type 21 | CH, & ! high cloud type 22 | AM, & ! middle cloud amount 23 | AH, & ! high cloud amount 24 | UM, & ! middle cloud amount 25 | UH, & ! high cloud amount 26 | IC, & ! change code 27 | SA, & ! solar altitude 28 | RI, & ! relative lunar illuminance 29 | SLP, & ! sea level pressure 30 | WS, & ! wind speed 31 | WD, & ! wind direction (degrees) 32 | AT, & ! air temperature 33 | DD, & ! dew point depression 34 | EL, & ! Land: station elevation (m) 35 | IW, & ! wind speed indicator 36 | IP) ! Land: sea level pressure flag 37 | ! year,month,day,hour yr,mn,dy,hr 8 51120100 96113021 none 38 | ! sky brightness indicator IB 1 0 1 none 39 | ! latitude x100 LAT 5 -9000 9000 none 40 | ! longitude x100 LON 5 0 36000 none 41 | ! land: station number ID 5 01000 98999 none 42 | ! ship: source deck, ship type 1100 9999 none,9 43 | ! land/ocean indicator LO 1 1 2 none 44 | ! present weather ww 2 0 99 -1 45 | ! total cloud cover N 1 0 8 none 46 | ! lower cloud amount Nh 2 -1 8 -1 47 | ! lower cloud base height h 2 -1 9 -1 48 | ! low cloud type CL 2 -1 11 -1 49 | ! middle cloud type CM 2 -1 12 -1 50 | ! high cloud type CH 2 -1 9 -1 51 | ! middle cloud amount x100 AM 3 0 800 900 52 | ! high cloud amount x100 AH 3 0 800 900 53 | ! middle cloud amount UM 1 0 8 9 54 | ! high cloud amount UH 1 0 8 9 55 | ! change code IC 2 0 9 none 56 | ! solar altitude (deg x10) SA 4 -900 900 none 57 | ! relative lunar illuminance x100 RI 4 -110 117 none 58 | ! sea level pressure (mb x10) SLP 5 9000,L 10999,L -1 59 | ! wind speed (ms-1 x10) WS 3 0 999 -1 60 | ! wind direction (degrees) WD 3 0 361 -1 61 | ! air temperature (C x10) AT 4 -949,L 599,L 900 62 | ! dew point depression (C x10) DD 3 0 700 900 63 | ! Land: station elevation (m) EL 4 -350 4877 9000 64 | ! wind speed indicator IW 1 0 1 9 65 | ! Land: sea level pressure flag IP 1 0) 66 | implicit none 67 | 68 | character(300), intent(in) :: infile 69 | integer, intent(in) :: nrecords, year 70 | integer, dimension(nrecords), intent(out) :: & 71 | yr,mn,dy,hr, & ! year,month,day,hour 72 | IB, & ! sky brightness indicator 73 | station_id, & ! land: station number, ship: source deck, ship type 74 | LO, & ! land/ocean indicator 75 | ww, & ! present weather 76 | N, & ! total cloud cover 77 | Nh, & ! lower cloud amount 78 | h, & ! lower cloud base height 79 | CL, & ! low cloud type 80 | CM, & ! middle cloud type 81 | CH, & ! high cloud type 82 | UM, & ! middle cloud amount 83 | UH, & ! high cloud amount 84 | IC, & ! change code 85 | WD, & ! wind direction (degrees) 86 | EL, & ! Land: station elevation (m) 87 | IW, & ! wind speed indicator 88 | IP ! Land: sea level pressure flag 89 | real, dimension(nrecords), intent(out) :: & 90 | LAT, & ! latitude 91 | LON, & ! longitude 92 | AM, & ! middle cloud amount 93 | AH, & ! high cloud amount 94 | SA, & ! solar altitude 95 | RI, & ! relative lunar illuminance 96 | SLP, & ! sea level pressure 97 | WS, & ! wind speed 98 | AT, & ! air temperature 99 | DD ! dew point depression 100 | integer :: i 101 | 102 | open(10,file=infile,status='old') 103 | 104 | 20 format(i2,i2,i2,i2, & ! yr,mn,dy,hr 105 | i1, & ! IB 106 | f5.0, & ! LAT 107 | f5.0, & ! LON 108 | i5, & ! station_id 109 | i1, & ! LO 110 | i2, & ! ww 111 | i1, & ! N 112 | i2, & ! Nh 113 | i2, & ! h 114 | i2, & ! CL 115 | i2, & ! CM 116 | i2, & ! CH 117 | f3.0, & ! AM 118 | f3.0, & ! AH 119 | i1, & ! UM 120 | i1, & ! UH 121 | i2, & ! IC 122 | f4.0, & ! SA 123 | f4.0, & ! RI 124 | f5.0, & ! SLP 125 | f3.0, & ! WS 126 | i3, & ! WD 127 | f4.0, & ! AT 128 | f3.0, & ! DD 129 | i4, & ! EL 130 | i1, & ! IW 131 | i1) ! IP 132 | 21 format(i4,i2,i2,i2, & ! yr,mn,dy,hr 133 | i1, & ! IB 134 | f5.0, & ! LAT 135 | f5.0, & ! LON 136 | i5, & ! station_id 137 | i1, & ! LO 138 | i2, & ! ww 139 | i1, & ! N 140 | i2, & ! Nh 141 | i2, & ! h 142 | i2, & ! CL 143 | i2, & ! CM 144 | i2, & ! CH 145 | f3.0, & ! AM 146 | f3.0, & ! AH 147 | i1, & ! UM 148 | i1, & ! UH 149 | i2, & ! IC 150 | f4.0, & ! SA 151 | f4.0, & ! RI 152 | f5.0, & ! SLP 153 | f3.0, & ! WS 154 | i3, & ! WD 155 | f4.0, & ! AT 156 | f3.0, & ! DD 157 | i4, & ! EL 158 | i1, & ! IW 159 | i1) ! IP 160 | do i=1,nrecords 161 | if (year < 1997) then 162 | read(10,20) yr(i),mn(i),dy(i),hr(i),IB(i),LAT(i),LON(i),station_id(i), & 163 | LO(i),ww(i),N(i),Nh(i),h(i),CL(i),CM(i),CH(i),AM(i),AH(i), & 164 | UM(i),UH(i),IC(i),SA(i),RI(i),SLP(i),WS(i),WD(i),AT(i),DD(i), & 165 | EL(i),IW(i),IP(i) 166 | yr(i) = yr(i) + 1900 167 | else 168 | read(10,21) yr(i),mn(i),dy(i),hr(i),IB(i),LAT(i),LON(i),station_id(i), & 169 | LO(i),ww(i),N(i),Nh(i),h(i),CL(i),CM(i),CH(i),AM(i),AH(i), & 170 | UM(i),UH(i),IC(i),SA(i),RI(i),SLP(i),WS(i),WD(i),AT(i),DD(i), & 171 | EL(i),IW(i),IP(i) 172 | endif 173 | if (LON(i) > 18000) LON(i) = LON(i) - 36000 174 | LAT(i) = LAT(i) * 0.01 175 | LON(i) = LON(i) * 0.01 176 | AM(i) = AM(i) * 0.01 177 | AH(i) = AH(i) * 0.01 178 | SA(i) = SA(i) * 0.01 179 | RI(i) = RI(i) * 0.01 180 | SLP(i) = SLP(i) * 0.01 181 | WS(i) = WS(i) * 0.1 182 | AT(i) = AT(i) * 0.1 183 | DD(i) = DD(i) * 0.1 184 | end do 185 | 186 | end subroutine parse_file 187 | 188 | subroutine extract_data(ids, src_dir, odir, years, imonths) 189 | 190 | implicit none 191 | 192 | integer, intent(in), dimension(:) :: ids, years, imonths 193 | character*300, intent(in) :: src_dir, odir 194 | 195 | INTEGER :: & 196 | yr,mn,dy,hr, & ! year,month,day,hour yr,mn,dy,hr 8 51120100 96113021 none 197 | IB, & ! sky brightness indicator IB 1 0 1 none 198 | LAT, & ! latitude x100 LAT 5 -9000 9000 none 199 | LON, & ! longitude x100 LON 5 0 36000 none 200 | station_id, & ! land: station number ID 5 01000 98999 none 201 | ! ship: source deck, ship type 1100 9999 none,9 202 | LO, & ! land/ocean indicator LO 1 1 2 none 203 | ww, & ! present weather ww 2 0 99 -1 204 | N, & ! total cloud cover N 1 0 8 none 205 | Nh, & ! lower cloud amount Nh 2 -1 8 -1 206 | h, & ! lower cloud base height h 2 -1 9 -1 207 | CL, & ! low cloud type CL 2 -1 11 -1 208 | CM, & ! middle cloud type CM 2 -1 12 -1 209 | CH, & ! high cloud type CH 2 -1 9 -1 210 | AM, & ! middle cloud amount x100 AM 3 0 800 900 211 | AH, & ! high cloud amount x100 AH 3 0 800 900 212 | UM, & ! middle cloud amount UM 1 0 8 9 213 | UH, & ! high cloud amount UH 1 0 8 9 214 | IC, & ! change code IC 2 0 9 none 215 | SA, & ! solar altitude (deg x10) SA 4 -900 900 none 216 | RI, & ! relative lunar illuminance x100 RI 4 -110 117 none 217 | SLP, & ! sea level pressure (mb x10) SLP 5 9000,L 10999,L -1 218 | WS, & ! wind speed (ms-1 x10) WS 3 0 999 -1 219 | WD, & ! wind direction (degrees) WD 3 0 361 -1 220 | AT, & ! air temperature (C x10) AT 4 -949,L 599,L 900 221 | DD, & ! dew point depression (C x10) DD 3 0 700 900 222 | EL, & ! Land: station elevation (m) EL 4 -350 4877 9000 223 | IW, & ! wind speed indicator IW 1 0 1 9 224 | IP ! Land: sea level pressure flag IP 1 0 2 9 225 | 226 | integer(kind = OMP_lock_kind) :: lck = 0 227 | integer(kind = OMP_lock_kind), dimension(size(ids)) :: locks 228 | integer :: ID 229 | integer :: j = 0, i = 0, nids = 0, keep = 0, old_id = 0, year = 0, & 230 | imonth = 0, iyear = 0 231 | integer, dimension(size(ids)) :: ounits 232 | integer, dimension(100) :: inunits = 0 233 | character*350, dimension(size(ids)) :: ofiles 234 | character*90 :: cfmt 235 | character*300 :: all_cfmt 236 | character*350 :: fname 237 | character*10 :: cid 238 | character*2 :: cyear 239 | character(len=3), dimension(12) :: months = (/ & 240 | 'JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN', 'JUL', 'AUG', 'SEP', 'OCT', & 241 | 'NOV', 'DEC' /) 242 | logical :: file_exists 243 | 244 | ! mn,dy,hr,IB,la,lo,id,LO,ww,N ,Nh,h ,CL,CM,CH,AM,AH,UM,UH,IC,SA,RI,SLP,WS,WD,AT,DD,EL,IW,IP 245 | cfmt = 'i2,i2,i2,i1,i5,i5,i5,i1,i2,i1,i2,i2,i2,i2,i2,i3,i3,i1,i1,i2,i4,i4,i5,i3 ,i3,i4,i3,i4,i1,i1' 246 | 247 | ! output format 248 | 25 format(i4,a,i0.2,a,i0.2,a,i0.2, & ! yr,mn,dy,hr 249 | a, & 250 | i1, & ! IB 251 | a, & 252 | f6.2, & ! LAT 253 | a, & 254 | f6.2, & ! LON 255 | a, & 256 | i5, & ! station_id 257 | a, & 258 | i1, & ! LO 259 | a, & 260 | i2, & ! ww 261 | a, & 262 | i1, & ! N 263 | a, & 264 | i2, & ! Nh 265 | a, & 266 | i2, & ! h 267 | a, & 268 | i2, & ! CL 269 | a, & 270 | i2, & ! CM 271 | a, & 272 | i2, & ! CH 273 | a, & 274 | f4.2, & ! AM 275 | a, & 276 | f5.2, & ! AH 277 | a, & 278 | i1, & ! UM 279 | a, & 280 | i1, & ! UH 281 | a, & 282 | i2, & ! IC 283 | a, & 284 | f5.1, & ! SA 285 | a, & 286 | f5.2, & ! RI 287 | a, & 288 | f6.1, & ! SLP 289 | a, & 290 | f4.1, & ! WS 291 | a, & 292 | i3, & ! WD 293 | a, & 294 | f5.1, & ! AT 295 | a, & 296 | f4.1, & ! DD 297 | a, & 298 | i4, & ! EL 299 | a, & 300 | i1, & ! IW 301 | a, & 302 | i1) ! IP 303 | 304 | nids = size(ids) 305 | 306 | j = 11 307 | do i = 1, nids 308 | locks(i) = i 309 | call OMP_init_lock(locks(i)) 310 | write(cid, '(I10)') ids(i) 311 | ofiles(i) = trim(odir)//trim(adjustl(cid))//'.csv' 312 | ounits(i) = j + 1000 313 | open(10, file=trim(ofiles(i)), status='unknown',action='write') 314 | write(10,'(a)') 'year,month,day,hour,IB,lat,lon,station_id,LO,ww,N,Nh,h,CL,CM,CH,AM,AH,UM,UH,IC,SA,RI,SLP,WS,WD,AT,DD,EL,IW,IP' 315 | j = j + 1 316 | close(10, status='keep') 317 | end do 318 | 319 | call OMP_init_lock(lck) 320 | 321 | do i = 1,size(inunits) 322 | inunits(i) = i + 50 323 | end do 324 | 325 | !$OMP PARALLEL DO SHARED(lck,locks,src_dir,odir,nids,ids,months,inunits,ounits, & 326 | !$OMP& ofiles,cfmt,imonths,years), & 327 | !$OMP FIRSTPRIVATE(old_id,keep), & 328 | !$OMP& PRIVATE(yr,mn,dy,hr,IB,LAT,LON,station_id,LO,ww,N,Nh, h,CL,CM,CH,AM,AH, & 329 | !$OMP& UM,UH,IC,SA,RI,SLP,WS,WD,AT, DD,EL,IW,IP,fname,cid,i,ID,imonth, & 330 | !$OMP& cyear,year,all_cfmt,iyear,j) 331 | do iyear = 1,size(years) 332 | year = years(iyear) 333 | ID = OMP_get_thread_num() + 1 334 | call OMP_set_lock(lck) 335 | write(*,*) "My thread is ", ID, '. Processing year ', year 336 | call OMP_unset_lock(lck) 337 | if (year < 1997) then 338 | all_cfmt = '(i2,'//cfmt//')' 339 | else 340 | all_cfmt = '(i4,'//cfmt//')' 341 | endif 342 | do imonth = 1,size(imonths) 343 | write(cyear,'(I0.2)') mod(year, 100) 344 | fname = trim(src_dir)//months(imonths(imonth))//cyear//'L' 345 | inquire(file=fname,EXIST=file_exists) 346 | if (.not. file_exists) goto 99 347 | open(inunits(ID), file=fname, status='old', action='read') 348 | j = 1 349 | do 350 | read(inunits(ID),all_cfmt,end=99,err=98) yr,mn,dy,hr,IB,LAT,LON, & 351 | station_id,LO,ww,N,Nh,h,CL,CM,CH,AM,AH,UM,UH, & 352 | IC,SA,RI,SLP,WS,WD,AT,DD,EL,IW,IP 353 | if (year < 1997) yr = yr + 1900 354 | if (old_id /= station_id) then 355 | if (keep /= 0) then 356 | close(ounits(keep)) 357 | call OMP_unset_lock(locks(keep)) 358 | endif 359 | keep = 0 360 | do i = 1, nids 361 | if (station_id == ids(i)) then 362 | keep = i 363 | call OMP_set_lock(locks(keep)) 364 | write(cid, '(I10)') ids(i) 365 | open(ounits(keep), file=trim(ofiles(keep)), & 366 | status='old',access='append',action='write') 367 | endif 368 | end do 369 | old_id = station_id 370 | endif 371 | if (keep > 0) then 372 | if (LON > 18000) LON = LON - 36000 373 | write(ounits(keep),25) yr,',',mn,',',dy,',',hr,',',IB,',', & 374 | 0.01*LAT,',',0.01*LON,',',station_id,',', & 375 | LO,',',ww,',',N,',',Nh,',',h,',',CL,',', & 376 | CM,',',CH,',',0.01*AM,',',0.01*AH,',',UM, & 377 | ',',UH,',',IC,',',0.1*SA,',',0.01*RI,',', & 378 | 0.1*SLP,',',0.1*WS,',',WD,',',0.1*AT,',', & 379 | 0.1*DD,',',EL,',',IW,',',IP 380 | end if 381 | j = j + 1 382 | goto 97 383 | 98 continue 384 | write(*,*) "Error occured at line ", j, "of file ", fname 385 | 97 continue 386 | end do 387 | 99 continue 388 | close(inunits(ID)) 389 | if (keep /= 0) then 390 | close(ounits(keep)) 391 | call OMP_unset_lock(locks(keep)) 392 | keep = 0 393 | endif 394 | end do 395 | end do 396 | !$OMP END PARALLEL DO 397 | 398 | call OMP_destroy_lock(lck) 399 | do i=1,nids 400 | call OMP_destroy_lock(locks(i)) 401 | end do 402 | end subroutine extract_data 403 | 404 | end module parseeecra 405 | -------------------------------------------------------------------------------- /tests/test_parameterization.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Test module for the parameterization. Note that the entire task framework is 3 | very sensible to internal errors , so if anything goes wrong, there should be 4 | an error within the setup""" 5 | import unittest 6 | import os 7 | import os.path as osp 8 | import shutil 9 | import numpy as np 10 | import pandas as pd 11 | # we use assert_frame_equal instead of pd.equal, because it is less strict 12 | from pandas.util.testing import assert_frame_equal 13 | import _base_testing as bt 14 | import gwgen.parameterization as param 15 | import gwgen.utils as utils 16 | from psyplot.config.rcsetup import safe_list 17 | 18 | try: 19 | from urllib.error import URLError 20 | except ImportError: 21 | URLError = IOError 22 | 23 | try: 24 | from pandas.tseries.offsets import MonthEnd 25 | except ImportError: 26 | from pandas.datetools import MonthEnd 27 | 28 | 29 | #: Message to provide a bit more information on the two data frames 30 | df_diff_msg = ( 31 | 'Task data and reference differ!\n{0}\n' 32 | 'Task\n%s\n{0}\n' 33 | 'Reference\n%s{0}').format('-' * 80) 34 | 35 | 36 | def df_equals(df, df_ref, *args, **kwargs): 37 | """Simple wrapper around assert_frame_equal to use unittests assertion 38 | 39 | Parameters 40 | ---------- 41 | df: pd.DataFrame 42 | The simulation data frame 43 | df_ref: pd.DataFrame 44 | The reference data frame 45 | 46 | Returns 47 | ------- 48 | None or Exception 49 | Either None if everything went fine, otherwise the raised Exception""" 50 | try: 51 | assert_frame_equal(df, df_ref, *args, **kwargs) 52 | except Exception as e: 53 | return e 54 | 55 | 56 | class _ParameterizerTestMixin(object): 57 | """A mixing for parameterizer tests""" 58 | 59 | param_cls = None 60 | 61 | @property 62 | def name(self): 63 | return self.param_cls.name 64 | 65 | def test_param(self, check_index_duplicates=True, 66 | check_data_duplicates=False, to_csv=True, **kwargs): 67 | def get_data(task): 68 | if not isinstance(task.data, pd.DataFrame): 69 | return task.data 70 | return [task.data] 71 | 72 | def no_duplicates(df): 73 | return df.ix[~df.index.duplicated(keep=False)] 74 | 75 | if self.param_cls is None: 76 | return 77 | to_db = self.use_db 78 | name = self.name 79 | self._test_init() 80 | kwargs.setdefault(name, {}) 81 | kwargs.setdefault('to_return', [name]) 82 | manager = self.organizer.param( 83 | stations=self.stations_file, to_csv=to_csv, to_db=to_db, 84 | **kwargs) 85 | task = manager.get_task(name) 86 | ref_data = get_data(task) 87 | 88 | # check if run worked 89 | if task.has_run: 90 | self.assertEqual( 91 | set(task.namelist_keys), 92 | set(self.organizer.exp_config['namelist']['weathergen_ctl'])) 93 | 94 | setup_from_file = False 95 | for fname in filter(None, safe_list(task.datafile)): 96 | setup_from_file = True 97 | self.assertTrue(osp.exists(fname), 98 | msg='Datafile %s of %s task does not exist!' % ( 99 | fname, name)) 100 | if check_index_duplicates: 101 | for df in get_data(task): 102 | idx = df.index 103 | self.assertFalse( 104 | idx.has_duplicates, 105 | msg='%s task index data has duplicates!\n%s' % ( 106 | name, idx.values[idx.duplicated(keep=False)])) 107 | if check_data_duplicates: 108 | self.assertFalse(task.data.duplicated().any(), 109 | msg='%s task data has duplicates!\n%s' % ( 110 | name, task.data[ 111 | task.data.duplicated(keep=False)])) 112 | engine = task.engine 113 | setup_from_db = False 114 | if engine is not None: 115 | sql_dtypes = task._split_kwargs(task.sql_dtypes) 116 | for i, table in enumerate(safe_list(task.dbname)): 117 | if table is not None: 118 | setup_from_db = True 119 | self.assertTrue( 120 | engine.has_table(table), 121 | msg='Database has no table %s of %s task' % (table, 122 | name)) 123 | data = task._get_data(i) 124 | data_cols = set(data.columns) | set(data.index.names) 125 | self.assertEqual(set(sql_dtypes[i]) & data_cols, 126 | data_cols, 127 | msg='Missing sql dtype for %s' % name) 128 | 129 | # check setup from file 130 | if setup_from_file: 131 | manager = self.organizer.param( 132 | stations=self.stations_file, **kwargs) 133 | new_task = manager.get_task(name) 134 | self.assertEqual(new_task.setup_from, 'file', 135 | msg='setup_from of %s task should be "file"!' % ( 136 | name)) 137 | new_data = get_data(new_task) 138 | self.assertEqual(len(new_data), len(ref_data), 139 | msg=('Number of dataframes for %s task are not ' 140 | 'equal after setup from file!') % name) 141 | for df, df_ref in zip(new_data, ref_data): 142 | df.sort_index(inplace=True) 143 | df_ref.sort_index(inplace=True) 144 | df = no_duplicates(df) 145 | df_ref = no_duplicates(df_ref) 146 | mask = (df != df_ref).values.any(axis=1) 147 | self.assertIsNone(df_equals(df, df_ref, check_dtype=False), 148 | msg=df_diff_msg % (df.ix[mask], 149 | df_ref.ix[mask])) 150 | # check setup from db 151 | if setup_from_db: 152 | for fname in filter(None, safe_list(task.datafile)): 153 | os.remove(fname) 154 | manager = self.organizer.param( 155 | stations=self.stations_file, **kwargs) 156 | new_task = manager.get_task(name) 157 | self.assertEqual(new_task.setup_from, 'db', 158 | msg='setup_from of %s task should be "db"!' % ( 159 | name)) 160 | new_data = get_data(new_task) 161 | self.assertEqual(len(new_data), len(ref_data), 162 | msg=('Number of dataframes for %s task are not ' 163 | 'equal after setup from db!') % name) 164 | for df, df_ref in zip(new_data, ref_data): 165 | df.sort_index(inplace=True) 166 | df_ref.sort_index(inplace=True) 167 | df = no_duplicates(df) 168 | df_ref = no_duplicates(df_ref) 169 | mask = (df != df_ref).values.any(axis=1) 170 | self.assertIsNone(df_equals(df, df_ref, check_dtype=False), 171 | msg=df_diff_msg % (df.ix[mask], 172 | df_ref.ix[mask])) 173 | return manager 174 | 175 | # the usage of distributed is not supported at the moment 176 | # def test_param_distributed(self, *args, **kwargs): 177 | # """Test the parameterization with the distributed package""" 178 | # if not self.organizer.global_config.get('serial'): 179 | # from distributed import LocalCluster 180 | # import multiprocessing as mp 181 | # c = LocalCluster(n_workers=mp.cpu_count(), diagnostics_port=None) 182 | # self.organizer.global_config['scheduler'] = c.scheduler_address 183 | # ret = self.test_param(*args, **kwargs) 184 | # c.close() 185 | # return ret 186 | 187 | 188 | class Test_DailyGHCNData(bt.BaseTest, _ParameterizerTestMixin): 189 | """Test case for the :class:`gwgen.parameterization.DailyGHCNData` task""" 190 | 191 | param_cls = param.DailyGHCNData 192 | 193 | stations_type = 'short' 194 | 195 | @unittest.skipIf(not bt.online, 'Only works with internet connection') 196 | def test_full_source_exists(self): 197 | """Test whether the url of the tar ball can be downloaded""" 198 | src = param.DailyGHCNData.http_source 199 | try: 200 | self._test_url(src) 201 | except (URLError, IOError) as e: 202 | if bt.on_travis: 203 | self.skipTest("Failed to load %s" % src) 204 | else: 205 | raise 206 | self.assert_(True) 207 | 208 | @unittest.skipIf(not bt.online, 'Only works with internet connection') 209 | def test_single_source_exists(self): 210 | """Test whether the url of the tar ball can be downloaded""" 211 | src = param.DailyGHCNData.http_single.format(self.stations[0]) + '.dly' 212 | try: 213 | self._test_url(src) 214 | except (URLError, IOError) as e: 215 | if bt.on_travis: 216 | self.skipTest("Failed to load %s" % src) 217 | else: 218 | raise 219 | self.assert_(True) 220 | 221 | 222 | class Test_CompleteDailyGHCNData(bt.BaseTest, _ParameterizerTestMixin): 223 | """Test case for the :class:`gwgen.parameterization.CompleteDailyGHCNData` 224 | task""" 225 | 226 | param_cls = param.CompleteDailyGHCNData 227 | 228 | stations_type = 'short' 229 | 230 | def test_param(self, **kwargs): 231 | manager = super(Test_CompleteDailyGHCNData, self).test_param(**kwargs) 232 | data = manager.get_task(self.name).data 233 | monthly = data.groupby(level=['id', 'year', 'month']).agg(len) 234 | monthly['day'] = 1 235 | s = pd.to_datetime(monthly.reset_index()[['year', 'month', 'day']]) 236 | s.index = monthly.index 237 | monthly['ndays'] = ((s + MonthEnd(0)) - s).dt.days + 1 238 | difference = monthly['prcp'] != monthly['ndays'] 239 | self.assertTrue( 240 | (monthly['prcp'] == monthly['ndays']).all(), 241 | msg='Prcp and number of days in month differ! %s' % ( 242 | monthly[['prcp', 'ndays']][difference])) 243 | 244 | 245 | class Test_YearlyCompleteDailyGHCNData(bt.BaseTest, _ParameterizerTestMixin): 246 | """Test case for the :class:`gwgen.parameterization.CompleteDailyGHCNData` 247 | task""" 248 | 249 | param_cls = param.YearlyCompleteDailyGHCNData 250 | 251 | stations_type = 'short' 252 | 253 | def test_param(self, **kwargs): 254 | import calendar 255 | manager = super(Test_YearlyCompleteDailyGHCNData, self).test_param( 256 | **kwargs) 257 | data = manager.get_task(self.name).data 258 | yearly = data.groupby(level=['id', 'year']).agg(len) 259 | years = yearly.index.get_level_values('year') 260 | ndays = 365 + np.vectorize(calendar.isleap)(years).astype(int) 261 | yearly['ndays'] = ndays 262 | difference = yearly['prcp'].values != ndays 263 | self.assertTrue( 264 | (yearly['prcp'].values == ndays).all(), 265 | msg='Prcp and number of days in month differ! %s' % ( 266 | yearly[['prcp', 'ndays']].iloc[difference])) 267 | 268 | 269 | class Test_MonthlyGHCNData(bt.BaseTest, _ParameterizerTestMixin): 270 | """Test case for the :class:`gwgen.parameterization.MonthlyGHCNData` task 271 | """ 272 | 273 | param_cls = param.MonthlyGHCNData 274 | 275 | stations_type = 'short' 276 | 277 | 278 | class Test_CompleteMonthlyGHCNData(bt.BaseTest, _ParameterizerTestMixin): 279 | """Test case for the 280 | :class:`gwgen.parameterization.CompleteMonthlyGHCNData` task""" 281 | 282 | param_cls = param.CompleteMonthlyGHCNData 283 | 284 | stations_type = 'short' 285 | 286 | 287 | class Test_PrcpDistParams(bt.BaseTest, _ParameterizerTestMixin): 288 | """Test case for the :class:`gwgen.parameterization.PrcpDistParams` task""" 289 | 290 | param_cls = param.PrcpDistParams 291 | 292 | 293 | class Test_TemperatureParameterizer(bt.BaseTest, _ParameterizerTestMixin): 294 | """Test case for the 295 | :class:`gwgen.parameterization.TemperatureParameterizer` task""" 296 | 297 | param_cls = param.TemperatureParameterizer 298 | 299 | def test_param(self, **kwargs): 300 | return super(Test_TemperatureParameterizer, self).test_param( 301 | temp=dict(cutoff=0, 302 | tmin_range1=(-40, -5), tmin_range2=(5, 20), 303 | tmax_range1=(-40, -5), tmax_range2=(5, 20)), **kwargs) 304 | 305 | 306 | class _CloudTestMixin(object): 307 | """A base class defining a test for using eecra stations directly""" 308 | 309 | def test_param_eecra(self, **kwargs): 310 | self.stations_file = self.eecra_stations_file 311 | kwargs.setdefault(self.param_cls.name, {}).setdefault( 312 | 'args_type', 'eecra') 313 | self.test_param(**kwargs) 314 | 315 | 316 | class Test_HourlyCloud(bt.BaseTest, _ParameterizerTestMixin, _CloudTestMixin): 317 | """Test case for the :class:`gwgen.parameterization.HourlyCloud` task""" 318 | 319 | param_cls = param.HourlyCloud 320 | 321 | stations_type = 'short' 322 | 323 | @unittest.skipIf(not bt.online, 'Only works with internet connection') 324 | def test_source_url(self): 325 | """Test whether the urls of the EECRA database work""" 326 | for key, url in self.param_cls.urls.items(): 327 | self._test_url(url) 328 | 329 | def test_extraction(self): 330 | self._test_init() 331 | orig_data = self.organizer.project_config['data'] 332 | self.organizer.project_config['data'] = self.test_dir 333 | eecra_dir = osp.join(self.test_dir, 'eecra') 334 | os.makedirs(eecra_dir) 335 | shutil.copytree(osp.join(orig_data, 'eecra', 'raw'), 336 | osp.join(eecra_dir, 'raw')) 337 | task = param.HourlyCloud.from_organizer(self.organizer, self.stations) 338 | task.years = [1996, 1997] # the years to use 339 | task.months = [1, 2] # JAN, FEB 340 | task.init_from_scratch() 341 | for station in task.eecra_stations: 342 | fname = osp.join(eecra_dir, 'stations', str(station) + '.csv') 343 | self.assertTrue(osp.exists(fname), 344 | msg='Missing file %s!') 345 | self.assertGreater(utils.file_len(fname), 1, 346 | msg='%s seems to be empty!' % fname) 347 | 348 | def test_param(self, **kwargs): 349 | """Unfortunately the raw EECRA data contains duplicates, so we don't 350 | check for them""" 351 | kwargs['check_index_duplicates'] = False 352 | super(Test_HourlyCloud, self).test_param(**kwargs) 353 | 354 | 355 | class Test_DailyCloud(bt.BaseTest, _ParameterizerTestMixin, _CloudTestMixin): 356 | """Test case for the :class:`gwgen.parameterization.DailyCloud` task""" 357 | 358 | param_cls = param.DailyCloud 359 | 360 | stations_type = 'short' 361 | 362 | 363 | class Test_MonthlyCloud(bt.BaseTest, _ParameterizerTestMixin, _CloudTestMixin): 364 | """Test case for the :class:`gwgen.parameterization.MonthlyCloud` task""" 365 | 366 | param_cls = param.MonthlyCloud 367 | 368 | stations_type = 'short' 369 | 370 | 371 | class Test_CompleteMonthlyCloud(bt.BaseTest, _ParameterizerTestMixin, 372 | _CloudTestMixin): 373 | """Test case for the :class:`gwgen.parameterization.CompleteMonthlyCloud` 374 | task""" 375 | 376 | param_cls = param.CompleteMonthlyCloud 377 | 378 | stations_type = 'short' 379 | 380 | 381 | class Test_YearlyCompleteMonthlyCloud(bt.BaseTest, _ParameterizerTestMixin, 382 | _CloudTestMixin): 383 | """Test case for the 384 | :class:`gwgen.parameterization.YearlyCompleteMonthlyCloud` task""" 385 | 386 | param_cls = param.YearlyCompleteMonthlyCloud 387 | 388 | stations_type = 'short' 389 | 390 | 391 | class Test_CompleteDailyCloud(bt.BaseTest, _ParameterizerTestMixin, 392 | _CloudTestMixin): 393 | """Test case for the :class:`gwgen.parameterization.CompleteDailyCloud` 394 | task""" 395 | 396 | param_cls = param.CompleteDailyCloud 397 | 398 | stations_type = 'short' 399 | 400 | def test_param(self, **kwargs): 401 | manager = super(Test_CompleteDailyCloud, self).test_param(**kwargs) 402 | # test whether all months are complete 403 | data = manager.get_task(self.name).data 404 | monthly = data.groupby(level=['id', 'year', 'month']).agg(len) 405 | monthly['day'] = 1 406 | s = pd.to_datetime(monthly.reset_index()[['year', 'month', 'day']]) 407 | s.index = monthly.index 408 | monthly['ndays'] = ((s + MonthEnd(0)) - s).dt.days + 1 409 | difference = monthly['mean_cloud'] != monthly['ndays'] 410 | self.assertTrue( 411 | (monthly['mean_cloud'] == monthly['ndays']).all(), 412 | msg='mean_cloud and number of days in month differ! %s' % ( 413 | monthly[['mean_cloud', 'ndays']][difference])) 414 | 415 | 416 | class Test_YearlyCompleteDailyCloud(bt.BaseTest, _ParameterizerTestMixin, 417 | _CloudTestMixin): 418 | """Test case for the 419 | :class:`gwgen.parameterization.YearlyCompleteDailyCloud` task""" 420 | 421 | param_cls = param.YearlyCompleteDailyCloud 422 | 423 | stations_type = 'short' 424 | 425 | def test_param(self, **kwargs): 426 | import calendar 427 | manager = super(Test_YearlyCompleteDailyCloud, self).test_param( 428 | **kwargs) 429 | data = manager.get_task(self.name).data 430 | yearly = data.groupby(level=['id', 'year']).agg(len) 431 | years = yearly.index.get_level_values('year') 432 | ndays = 365 + np.vectorize(calendar.isleap)(years).astype(int) 433 | yearly['ndays'] = ndays 434 | difference = yearly['mean_cloud'].values != ndays 435 | self.assertTrue( 436 | (yearly['mean_cloud'].values == ndays).all(), 437 | msg='mean_cloud and number of days in month differ! %s' % ( 438 | yearly[['mean_cloud', 'ndays']].iloc[difference])) 439 | 440 | 441 | class Test_CloudParameterizer(bt.BaseTest, _ParameterizerTestMixin, 442 | _CloudTestMixin): 443 | """Test case for the :class:`gwgen.parameterization.CloudParameterizer` 444 | task""" 445 | 446 | param_cls = param.CloudParameterizer 447 | 448 | 449 | class Test_CompleteMonthlyWind(bt.BaseTest, _ParameterizerTestMixin, 450 | _CloudTestMixin): 451 | """Test case for the :class:`gwgen.parameterization.CompleteMonthlyWind` 452 | task""" 453 | 454 | param_cls = param.CompleteMonthlyWind 455 | 456 | stations_type = 'short' 457 | 458 | 459 | class Test_YearlyCompleteMonthlyWind(bt.BaseTest, _ParameterizerTestMixin, 460 | _CloudTestMixin): 461 | """Test case for the 462 | :class:`gwgen.parameterization.YearlyCompleteMonthlyWind` task""" 463 | 464 | param_cls = param.YearlyCompleteMonthlyWind 465 | 466 | stations_type = 'short' 467 | 468 | 469 | class Test_WindParameterizer(bt.BaseTest, _ParameterizerTestMixin, 470 | _CloudTestMixin): 471 | """Test case for the :class:`gwgen.parameterization.WindParameterizer` 472 | task""" 473 | 474 | param_cls = param.WindParameterizer 475 | 476 | 477 | class Test_CrossCorrelation(bt.BaseTest, _ParameterizerTestMixin): 478 | """Test case for the 479 | :class:`gwgen.parameterization.CrossCorrelation` task""" 480 | 481 | param_cls = param.CrossCorrelation 482 | 483 | 484 | if __name__ == '__main__': 485 | unittest.main() 486 | --------------------------------------------------------------------------------