├── VERSION ├── CITATION.cff ├── setup.py ├── .gitignore ├── CONTRIBUTING.md ├── CODE_OF_CONDUCT.md ├── src ├── flex2traj.py ├── main.py ├── diagnosis.py ├── validation.py ├── biascorrection.py └── attribution.py ├── README.md └── LICENSE /VERSION: -------------------------------------------------------------------------------- 1 | 1.2.0 2 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "If you use this software, please cite the paper in discussion below." 3 | authors: 4 | - family-names: "Keune" 5 | given-names: "Jessica" 6 | orcid: "https://orcid.org/0000-0001-6104-2165" 7 | - family-names: "Schumacher" 8 | given-names: "Dominik L." 9 | orcid: "https://orcid.org/0000-0003-2699-2880" 10 | - family-names: "Miralles" 11 | given-names: "Diego G." 12 | orcid: "https://orcid.org/0000-0001-6186-5751" 13 | title: "Heat- And MoiSture Tracking framEwoRk - h-cel/hamster v1.2.0" 14 | version: 1.2.0 15 | doi: 10.5281/zenodo.5788506 16 | date-released: 2021-12-14 17 | url: "http://doi.org/10.5281/zenodo.5788506" 18 | preferred-citation: 19 | type: article 20 | authors: 21 | - family-names: "Keune" 22 | given-names: "Jessica" 23 | orcid: "https://orcid.org/0000-0001-6104-2165" 24 | - family-names: "Schumacher" 25 | given-names: "Dominik L." 26 | orcid: "https://orcid.org/0000-0003-2699-2880" 27 | - family-names: "Miralles" 28 | given-names: "Diego G." 29 | orcid: "https://orcid.org/0000-0001-6186-5751" 30 | doi: "10.5194/gmd-15-1875-2022" 31 | journal: "Geoscientific Model Development" 32 | title: "A unified framework to estimate the origins of atmospheric moisture and heat using Lagrangian models" 33 | year: 2022 34 | url: "https://gmd.copernicus.org/articles/15/1875/2022/" 35 | volume: 15 36 | year: 2022 37 | start: 1875 38 | end: 1898 39 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | from setuptools import find_packages 3 | 4 | setup( 5 | name='hamster-tracking', 6 | version='1.0.0', 7 | license='gpl-3.0', 8 | description = 'HAMSTER: a Heat And MoiSture Tracking framEwoRk for tracking air parcels through the atmosphere', 9 | author = 'Jessica Keune, Dominik L. Schumacher, Diego G. Miralles', 10 | author_email = 'jessica.keune@ugent.be', 11 | url = 'https://github.com/h-cel/hamster', 12 | download_url ='https://github.com/h-cel/hamster/archive/v1.0.0.tar.gz', 13 | keywords = ['Lagrangian models', 'moisture tracking', 'precipitation recycling', 'origins of precipitation', 14 | 'origins of heat', 'source regions of moisture', 'source regions of heat', 'air parcel tracking', 15 | 'land--atmosphere interactions'], # Keywords 16 | install_package_data = True, 17 | packages=find_packages("."), 18 | install_requires=['numpy','pandas','gzip','hdf5','netcdf4','argparse','calendar','os','fnmatch','imp','math','random','re','struct','sys','time','timeit','warnings','datetime','functools','dateutil'], 19 | long_description=open('README.md').read(), 20 | classifiers=[ 21 | 'Development Status :: 4 - Beta', 22 | 'Intended Audience :: Atmospheric sciences, Hydrology', 23 | 'Topic :: Tracking the origins of heat and moisture in the atmosphere', 24 | 'License :: gpl-3.0', 25 | 'Programming Language :: Python :: 3', 26 | ], 27 | package_dir={"": "src"}, 28 | packages=setuptools.find_packages(where="src"), 29 | python_requires=">=3.6", 30 | ) 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # job-related 2 | *.e* 3 | *.o* 4 | *.worker* 5 | *.log* 6 | *.sh* 7 | *.run* 8 | *.pbs* 9 | 10 | # Copy of mainpy to work-dir 11 | work/ 12 | paths.txt 13 | load_modules 14 | tmp/ 15 | 16 | # profiling output 17 | profile_output* 18 | 19 | # Byte-compiled / optimized / DLL files 20 | __pycache__/ 21 | *.py[cod] 22 | *$py.class 23 | 24 | # C extensions 25 | *.so 26 | 27 | # Distribution / packaging 28 | .Python 29 | build/ 30 | develop-eggs/ 31 | dist/ 32 | downloads/ 33 | eggs/ 34 | .eggs/ 35 | lib/ 36 | lib64/ 37 | parts/ 38 | sdist/ 39 | var/ 40 | wheels/ 41 | *.egg-info/ 42 | .installed.cfg 43 | *.egg 44 | MANIFEST 45 | 46 | # PyInstaller 47 | # Usually these files are written by a python script from a template 48 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 49 | *.manifest 50 | *.spec 51 | 52 | # Installer logs 53 | pip-log.txt 54 | pip-delete-this-directory.txt 55 | 56 | # Unit test / coverage reports 57 | htmlcov/ 58 | .tox/ 59 | .coverage 60 | .coverage.* 61 | .cache 62 | nosetests.xml 63 | coverage.xml 64 | *.cover 65 | .hypothesis/ 66 | .pytest_cache/ 67 | 68 | # Translations 69 | *.mo 70 | *.pot 71 | 72 | # Django stuff: 73 | *.log 74 | local_settings.py 75 | db.sqlite3 76 | 77 | # Flask stuff: 78 | instance/ 79 | .webassets-cache 80 | 81 | # Scrapy stuff: 82 | .scrapy 83 | 84 | # Sphinx documentation 85 | docs/_build/ 86 | 87 | # PyBuilder 88 | target/ 89 | 90 | # Jupyter Notebook 91 | .ipynb_checkpoints 92 | 93 | # pyenv 94 | .python-version 95 | 96 | # celery beat schedule file 97 | celerybeat-schedule 98 | 99 | # SageMath parsed files 100 | *.sage.py 101 | 102 | # Environments 103 | .env 104 | .venv 105 | env/ 106 | venv/ 107 | ENV/ 108 | env.bak/ 109 | venv.bak/ 110 | 111 | # Spyder project settings 112 | .spyderproject 113 | .spyproject 114 | 115 | # Rope project settings 116 | .ropeproject 117 | 118 | # mkdocs documentation 119 | /site 120 | 121 | # mypy 122 | .mypy_cache/ 123 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Welcome to the HAMSTER contributing guide 2 | 3 | Thank you for investing your time in contributing to our project! Any contribution will be highly appreciated and we will mention all contributors on [https://github.com/h-cel/hamster](https://github.com/h-cel/hamster). 4 | 5 | Before you continue, please read the [Code of Conduct](./CODE_OF_CONDUCT.md), which will help us to keep our community approachable and respectable. 6 | 7 | This guide will provide an overview of the contribution workflow from opening an issue, creating a pull request, reviewing, and merging the pull request. 8 | 9 | ### Issues 10 | 11 | #### Create a new issue 12 | 13 | If you spot a problem with the code or the documentation, [search if an issue already exists](https://github.com/h-cel/hamster/issues). 14 | If a related issue doesn't exist, you can open a [new issue](https://github.com/h-cel/hamster/issues/new). 15 | 16 | #### Solve an issue 17 | 18 | Scan through our [existing issues]((https://github.com/h-cel/hamster/issues) to find one that interests you. 19 | You can narrow down the search using `labels` as filters. 20 | As a general rule, we don’t assign issues to anyone. If you find an issue to work on, you are welcome to open a pull request with a fix. 21 | 22 | ### Make Changes 23 | 24 | #### Small changes in docs 25 | If you want to make small changes (fixing a typo or a sentence, updating broken links) in any of the `.md` files, you can do this using the user interface: 26 | just click on the **edit** button, make your changes, commit them to a new branch and [create a pull request](#pull-request) for a review. 27 | 28 | #### Bigger changes affecting HAMSTER 29 | If you want to contribute bigger changes that influence of the analysis with HAMSTER, we kindly ask you to do and test these on your local machine. Therefore, fork this repository and run some tests on your local computer. 30 | Make the necessary changes, test the changes, and only commit if they run successfully (we will include standard tests in the future). 31 | Once done, you can again [create a pull request](#pull-request) for a review. 32 | 33 | ### Pull Request 34 | 35 | When you're finished with the changes, create a pull request. 36 | - Please add a detailed description of the changes that will help us to review your pull request. 37 | - Don't forget to link any issue if you are solving one. 38 | - Once you submit your pull request, a HAMSTER team member will review your proposal. We may ask questions or request for additional information. 39 | - We may ask for changes to be made before a pull request can be merged via pull request comments. You can apply suggested changes directly through the user interface. You can make any other changes in your fork, then commit them to your branch. 40 | - As you update your pull request and apply changes, mark each conversation as resolved. 41 | - If you run into any merge issues, checkout this [git tutorial](https://lab.github.com/githubtraining/managing-merge-conflicts) to help you resolve merge conflicts and other issues. 42 | 43 | ### Your pull request is merged! 44 | 45 | Congratulations :tada::tada: The HAMSTER team thanks you :sparkles:. 46 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | jessica.keune@ugent.be. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /src/flex2traj.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Main script to extract trajectories from binary FLEXPART outputs 5 | # 6 | # This file is part of HAMSTER, 7 | # originally created by Dominik Schumacher, Jessica Keune, Diego G. Miralles 8 | # at the Hydro-Climate Extremes Lab, Department of Environment, Ghent University 9 | # 10 | # https://github.com/h-cel/hamster 11 | # 12 | # HAMSTER is free software: you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License as published by 14 | # the Free Software Foundation v3. 15 | # 16 | # HAMSTER is distributed in the hope that it will be useful, 17 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | # GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU General Public License 22 | # along with HAMSTER. If not, see . 23 | # 24 | 25 | import argparse 26 | import calendar 27 | import csv 28 | import fnmatch 29 | import gzip 30 | import imp 31 | import math 32 | import os 33 | import random 34 | import re 35 | import struct 36 | import sys 37 | import time 38 | import timeit 39 | import warnings 40 | from datetime import date, datetime, timedelta 41 | import datetime as datetime 42 | from functools import reduce 43 | from math import acos, atan, atan2, cos, floor, sin, sqrt 44 | 45 | import h5py 46 | import netCDF4 as nc4 47 | import numpy as np 48 | import pandas as pd 49 | from dateutil.relativedelta import relativedelta 50 | 51 | from hamsterfunctions import * 52 | 53 | 54 | def main_flex2traj( 55 | ryyyy, ayyyy, am, ad, tml, maskfile, maskval, idir, odir, fout, verbose 56 | ): 57 | 58 | ###--- MISC ---################################################################ 59 | logo = """ 60 | Hello, user. 61 | 62 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 63 | % __ _ ____ _ _ % 64 | % / _| | _____ _|___ \| |_ _ __ __ _ (_) % 65 | % | |_| |/ _ \ \/ / __) | __| '__/ _` || | % 66 | % | _| | __/> < / __/| |_| | | (_| || | % 67 | % |_| |_|\___/_/\_\_____|\__|_| \__,_|/ | % 68 | % |__/ % 69 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 70 | """ 71 | ############################################################################### 72 | ###--- SETUP ---############################################################### 73 | 74 | # ****************************************************************************** 75 | ## UNCOMMENT line ---> variable not saved 76 | selvars = np.asarray( 77 | [ 78 | 0, # pid | [ ALWAYS ] * 0 79 | 1, # x | [ ALWAYS ] * 1 80 | 2, # y | [ ALWAYS ] * 2 81 | 3, # z | [ ALWAYS ] * 3 82 | # 4, # itramem | [ NEVER ] 83 | 5, # oro | [ OPTIONAL ] # only needed for dry static energy 84 | # 6, # pv | [ OPTIONAL ] 85 | 7, # qq | [ ALWAYS ] * 4 86 | 8, # rho | [ ALWAYS ] * 5 87 | 9, # hmix | [ ALWAYS ] * 6 88 | # 10,# tropo | [ OPTIONAL ] * 7 # needed for droughtpropag 89 | 11, # temp | [ ALWAYS ] * 8 90 | # 12,# mass | [ NEVER! ] 91 | ] 92 | ) 93 | thevars = np.asarray( 94 | [ 95 | "pid", 96 | "x", 97 | "y", 98 | "z", 99 | "itramem", 100 | "oro", 101 | "pv", 102 | "qv", 103 | "rho", 104 | "hmix", 105 | "tropo", 106 | "temp", 107 | "mass", 108 | ] 109 | ) 110 | # ****************************************************************************** 111 | 112 | # last day of month 113 | ed = int(calendar.monthrange(ayyyy, am)[1]) 114 | 115 | dt_h = 6 # hardcoded, as further edits would be necessary if this was changed! 116 | time_bgn = datetime.datetime(year=ayyyy, month=am, day=ad, hour=6) 117 | # add 6 hours to handle end of month in same way as any other period 118 | time_end = datetime.datetime( 119 | year=ayyyy, month=am, day=ed, hour=18 120 | ) + datetime.timedelta(hours=dt_h) 121 | # convert trajectory length from day to dt_h (!=6); +2 needed ;) 122 | ntraj = tml * (24 // dt_h) + 2 123 | 124 | ############################################################################### 125 | ###--- MAIN ---################################################################ 126 | 127 | if verbose: 128 | print(logo) 129 | 130 | ##---0.) pepare directories 131 | outdir = odir + "/" + str(ryyyy) 132 | if not os.path.exists(outdir): # could use isdir too 133 | os.makedirs(outdir) 134 | 135 | ##---1.) load netCDF mask 136 | if maskfile is None or maskval == -999: 137 | mask = mlat = mlon = None 138 | else: 139 | mask, mlat, mlon = maskgrabber(maskfile) 140 | 141 | ##---2.) create datetime object (covering arrival period + trajectory length) 142 | fulltime_str = f2t_timelord(ntraj_d=tml, dt_h=dt_h, tbgn=time_bgn, tend=time_end) 143 | 144 | # ---3.) handle first step 145 | if verbose: 146 | print("\n---- Reading files to begin constructing trajectories ...\n") 147 | data, trajs = f2t_establisher( 148 | partdir=idir + "/" + str(ryyyy), 149 | selvars=selvars, 150 | time_str=fulltime_str[:ntraj], 151 | ryyyy=ryyyy, 152 | mask=mask, 153 | maskval=maskval, 154 | mlat=mlat, 155 | mlon=mlon, 156 | outdir=outdir, 157 | fout=fout, 158 | verbose=verbose, 159 | ) 160 | 161 | ##---4.) continue with next steps 162 | if verbose: 163 | print("\n\n---- Adding more files ... ") 164 | for ii in range(1, len(fulltime_str) - ntraj + 1): # CAUTION: INDEXING from 1! 165 | data, trajs = f2t_ascender( 166 | data=data, 167 | partdir=idir + "/" + str(ryyyy), 168 | selvars=selvars, 169 | time_str=fulltime_str[ii : ntraj + ii], 170 | ryyyy=ryyyy, 171 | mask=mask, 172 | maskval=maskval, 173 | mlat=mlat, 174 | mlon=mlon, 175 | outdir=outdir, 176 | fout=fout, 177 | verbose=verbose, 178 | ) 179 | 180 | ##---5.) done 181 | if verbose: 182 | print( 183 | "\n\n---- Done! \n Files with base '" + fout + "' written to:\n ", 184 | odir + "/" + str(ryyyy), 185 | ) 186 | print(" Dimensions: nstep x nparcel x nvar\n Var order: ", end="") 187 | print(*thevars[selvars].tolist(), sep=", ") 188 | print("\n All done!") 189 | -------------------------------------------------------------------------------- /src/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # MAIN SCRIPT OF HAMSTER 5 | # 6 | # This file is part of HAMSTER, 7 | # originally created by Dominik Schumacher, Jessica Keune, Diego G. Miralles 8 | # at the Hydro-Climate Extremes Lab, Department of Environment, Ghent University 9 | # 10 | # https://github.com/h-cel/hamster 11 | # 12 | # HAMSTER is free software: you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License as published by 14 | # the Free Software Foundation v3. 15 | # 16 | # HAMSTER is distributed in the hope that it will be useful, 17 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | # GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU General Public License 22 | # along with HAMSTER. If not, see . 23 | # 24 | 25 | ########################################################################### 26 | ##--- MODULES 27 | ########################################################################### 28 | 29 | import argparse 30 | import imp 31 | import os 32 | import time 33 | 34 | from attribution import main_attribution 35 | from biascorrection import main_biascorrection 36 | from diagnosis import main_diagnosis 37 | from flex2traj import main_flex2traj 38 | from hamsterfunctions import * 39 | 40 | ########################################################################### 41 | ##--- FUNCTIONS + COMMAND LINE ARGUMENTS 42 | ########################################################################### 43 | 44 | ## COMMAND LINE ARGUMENTS 45 | # read command line arguments (dates, thresholds and other flags) 46 | args = read_cmdargs() 47 | verbose = args.verbose 48 | print(printsettings(args)) 49 | 50 | # just waiting a random number of seconds (max. 30s) 51 | # to avoid overlap of path.exist and makedirs between parallel jobs (any better solution?) 52 | if args.waiter: 53 | waiter = random.randint(0, 30) 54 | time.sleep(waiter) 55 | 56 | ########################################################################### 57 | ##--- PATHS 58 | ########################################################################### 59 | 60 | ## determine working directory 61 | wpath = os.getcwd() 62 | os.chdir(wpath) 63 | 64 | ## load input and output paths & input file name base(s) 65 | print("Using paths from: " + wpath + "/" + args.pathfile) 66 | content = imp.load_source("", wpath + "/" + args.pathfile) # load like a python module 67 | path_refp = check_paths(content, "path_ref_p") 68 | path_refe = check_paths(content, "path_ref_e") 69 | path_refh = check_paths(content, "path_ref_h") 70 | path_reft = check_paths(content, "path_ref_t") 71 | path_orig = check_paths(content, "path_orig") 72 | path_diag = check_paths(content, "path_diag") 73 | path_attr = check_paths(content, "path_attr") 74 | path_bias = check_paths(content, "path_bias") 75 | maskfile = check_paths(content, "maskfile") 76 | path_f2t_diag = check_paths(content, "path_f2t_diag") 77 | base_f2t_diag = check_paths(content, "base_f2t_diag") 78 | path_f2t_traj = check_paths(content, "path_f2t_traj") 79 | base_f2t_traj = check_paths(content, "base_f2t_traj") 80 | 81 | # create output directories if they do not exist (in dependency of step) 82 | if args.steps == 0 and args.ctraj_len == 0 and not os.path.exists(path_f2t_diag): 83 | os.makedirs(path_f2t_diag) 84 | os.makedirs(path_f2t_diag + "/" + str(args.ryyyy)) 85 | if args.steps == 0 and args.ctraj_len > 0 and not os.path.exists(path_f2t_traj): 86 | os.makedirs(path_f2t_traj) 87 | os.makedirs(path_f2t_traj + "/" + str(args.ryyyy)) 88 | if args.steps == 1 and not os.path.exists(path_diag): 89 | os.makedirs(path_diag) 90 | if args.steps == 2 and not os.path.exists(path_attr): 91 | os.makedirs(path_attr) 92 | if args.steps == 3 and not os.path.exists(path_bias): 93 | os.makedirs(path_bias) 94 | 95 | ########################################################################### 96 | ##--- MAIN 97 | ########################################################################### 98 | ## (3) RUN main scripts with arguments 99 | if args.steps == 0: 100 | if args.ctraj_len == 0: 101 | path_f2t = path_f2t_diag 102 | base_f2t = base_f2t_diag 103 | elif args.ctraj_len > 0: 104 | path_f2t = path_f2t_traj 105 | base_f2t = base_f2t_traj 106 | main_flex2traj( 107 | ryyyy=args.ryyyy, 108 | ayyyy=args.ayyyy, 109 | am=args.am, 110 | ad=args.ad, 111 | tml=args.ctraj_len, 112 | maskfile=maskfile, 113 | maskval=args.maskval, 114 | idir=path_orig, 115 | odir=path_f2t, 116 | fout=base_f2t, 117 | verbose=args.verbose, 118 | ) 119 | 120 | if args.steps == 1: 121 | main_diagnosis( 122 | ryyyy=args.ryyyy, 123 | ayyyy=args.ayyyy, 124 | am=args.am, 125 | ad=args.ad, 126 | ipath=path_f2t_diag, 127 | ifile_base=base_f2t_diag, 128 | opath=path_diag, 129 | ofile_base=args.expid, 130 | mode=args.mode, 131 | gres=args.gres, 132 | verbose=args.verbose, 133 | veryverbose=args.veryverbose, 134 | fproc_npart=args.fproc_npart, 135 | # E criteria 136 | fevap=args.fevap, 137 | cevap_dqv=args.cevap_dqv, 138 | fevap_drh=args.fevap_drh, 139 | cevap_drh=args.cevap_drh, 140 | cevap_hgt=args.cevap_hgt, 141 | # P criteria 142 | fprec=args.fprec, 143 | cprec_dqv=args.cprec_dqv, 144 | cprec_rh=args.cprec_rh, 145 | # H criteria 146 | fheat=args.fheat, 147 | cheat_dtemp=args.cheat_dtemp, 148 | fheat_drh=args.fheat_drh, 149 | cheat_drh=args.cheat_drh, 150 | cheat_hgt=args.cheat_hgt, 151 | fheat_rdq=args.fheat_rdq, 152 | cheat_rdq=args.cheat_rdq, 153 | # pbl and height criteria 154 | cpbl_method=args.cpbl_method, 155 | cpbl_strict=args.cpbl_strict, 156 | cpbl_factor=args.cpbl_factor, 157 | refdate=args.refdate, 158 | fwrite_netcdf=args.write_netcdf, 159 | precision=args.precision, 160 | ftimethis=args.timethis, 161 | fvariable_mass=args.variable_mass, 162 | strargs=printsettings(args), 163 | ) 164 | 165 | if args.steps == 2: 166 | main_attribution( 167 | ryyyy=args.ryyyy, 168 | ayyyy=args.ayyyy, 169 | am=args.am, 170 | ad=args.ad, 171 | ipath=path_f2t_traj, 172 | ifile_base=base_f2t_traj, 173 | ipath_f2t=path_orig, 174 | opath=path_attr, 175 | ofile_base=args.expid, 176 | mode=args.mode, 177 | gres=args.gres, 178 | maskfile=maskfile, 179 | maskval=args.maskval, 180 | verbose=args.verbose, 181 | veryverbose=args.veryverbose, 182 | ctraj_len=args.ctraj_len, 183 | # E criteria 184 | cevap_dqv=args.cevap_dqv, 185 | fevap_drh=args.fevap_drh, 186 | cevap_drh=args.cevap_drh, 187 | cevap_hgt=args.cevap_hgt, 188 | # P criteria 189 | cprec_dqv=args.cprec_dqv, 190 | cprec_rh=args.cprec_rh, 191 | # H criteria 192 | cheat_dtemp=args.cheat_dtemp, 193 | fheat_drh=args.fheat_drh, 194 | cheat_drh=args.cheat_drh, 195 | cheat_hgt=args.cheat_hgt, 196 | fheat_rdq=args.fheat_rdq, 197 | cheat_rdq=args.cheat_rdq, 198 | # pbl and height criteria 199 | cpbl_method=args.cpbl_method, 200 | cpbl_strict=args.cpbl_strict, 201 | cpbl_factor=args.cpbl_factor, 202 | refdate=args.refdate, 203 | fwrite_netcdf=args.write_netcdf, 204 | precision=args.precision, 205 | ftimethis=args.timethis, 206 | fdry=args.fallingdry, 207 | fmemento=args.memento, 208 | mattribution=args.mattribution, 209 | crandomnit=args.ratt_nit, 210 | randatt_forcall=args.ratt_forcall, 211 | randatt_wloc=args.ratt_wloc, 212 | explainp=args.explainp, 213 | fdupscale=args.dupscale, 214 | fmupscale=args.mupscale, 215 | fvariable_mass=args.variable_mass, 216 | fwritestats=args.writestats, 217 | strargs=printsettings(args), 218 | ) 219 | 220 | if args.steps == 3: 221 | main_biascorrection( 222 | ryyyy=args.ryyyy, 223 | ayyyy=args.ayyyy, 224 | am=args.am, 225 | opath_attr=path_attr, 226 | opath_diag=path_diag, 227 | ipath_refp=path_refp, 228 | ipath_refe=path_refe, 229 | ipath_reft=path_reft, 230 | ipath_refh=path_refh, 231 | opath=path_bias, 232 | ofile_base=args.expid, # output 233 | mode=args.mode, 234 | maskfile=maskfile, 235 | maskval=args.maskval, 236 | verbose=args.verbose, 237 | veryverbose=args.veryverbose, 238 | fuseattp=args.bc_useattp, 239 | bcscale=args.bc_time, 240 | pref_data=args.pref_data, 241 | eref_data=args.eref_data, 242 | href_data=args.href_data, 243 | faggbwtime=args.bc_aggbwtime, 244 | fbc_e2p=args.bc_e2p, 245 | fbc_e2p_p=args.bc_e2p_p, 246 | fbc_e2p_e=args.bc_e2p_e, 247 | fbc_e2p_ep=args.bc_e2p_ep, 248 | fbc_t2p_ep=args.bc_t2p_ep, 249 | fbc_had=args.bc_had, 250 | fbc_had_h=args.bc_had_h, 251 | fdebug=args.debug, 252 | fwrite_netcdf=args.write_netcdf, 253 | fwrite_month=args.write_month, 254 | fwritestats=args.writestats, 255 | precision=args.precision, 256 | strargs=printsettings(args), 257 | ) 258 | -------------------------------------------------------------------------------- /src/diagnosis.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Main script to diagnose fluxes of two-step trajectories from a Lagrangian model, 5 | # such as FLEXPART. This script diagnoses evaporation (E) and precipitation (P) based 6 | # on the change in specific humidity (q), and sensible heat (H) based on the potential 7 | # temperature. Diagnosis is performed globally. 8 | # 9 | # This file is part of HAMSTER, 10 | # originally created by Dominik Schumacher, Jessica Keune, Diego G. Miralles 11 | # at the Hydro-Climate Extremes Lab, Department of Environment, Ghent University 12 | # 13 | # https://github.com/h-cel/hamster 14 | # 15 | # HAMSTER is free software: you can redistribute it and/or modify 16 | # it under the terms of the GNU General Public License as published by 17 | # the Free Software Foundation v3. 18 | # 19 | # HAMSTER is distributed in the hope that it will be useful, 20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 | # GNU General Public License for more details. 23 | # 24 | # You should have received a copy of the GNU General Public License 25 | # along with HAMSTER. If not, see . 26 | # 27 | 28 | import argparse 29 | import calendar 30 | import csv 31 | import fnmatch 32 | import gzip 33 | import imp 34 | import math 35 | import os 36 | import random 37 | import re 38 | import struct 39 | import sys 40 | import time 41 | import timeit 42 | import warnings 43 | from datetime import date, datetime, timedelta 44 | import datetime as datetime 45 | from functools import reduce 46 | from math import acos, atan, atan2, cos, floor, sin, sqrt 47 | 48 | import h5py 49 | import netCDF4 as nc4 50 | import numpy as np 51 | import pandas as pd 52 | from dateutil.relativedelta import relativedelta 53 | 54 | from hamsterfunctions import * 55 | 56 | 57 | def main_diagnosis( 58 | ryyyy, 59 | ayyyy, 60 | am, 61 | ad, 62 | ipath, 63 | ifile_base, 64 | opath, 65 | ofile_base, 66 | mode, 67 | gres, 68 | verbose, 69 | veryverbose, 70 | fproc_npart, 71 | # E criteria 72 | fevap, 73 | cevap_dqv, 74 | fevap_drh, 75 | cevap_drh, 76 | cevap_hgt, 77 | # P criteria 78 | fprec, 79 | cprec_dqv, 80 | cprec_rh, 81 | # H criteria 82 | fheat, 83 | cheat_dtemp, 84 | fheat_drh, 85 | cheat_drh, 86 | cheat_hgt, 87 | fheat_rdq, 88 | cheat_rdq, 89 | # pbl and height criteria 90 | cpbl_method, 91 | cpbl_strict, 92 | cpbl_factor, 93 | refdate, 94 | fwrite_netcdf, 95 | precision, 96 | ftimethis, 97 | fvariable_mass, 98 | strargs, 99 | ): 100 | 101 | ## Perform consistency checks 102 | if mode == "oper" and precision == "f4": 103 | precision = "f8" 104 | print( 105 | "Single precision should only be used for testing. Reset to double-precision." 106 | ) 107 | if fvariable_mass and not fproc_npart: 108 | fproc_npart = True 109 | print("Have to process all parcels for variable mass...") 110 | 111 | ## Construct precise input and storage paths 112 | mainpath = ipath + str(ryyyy) + "/" 113 | ofilename = ( 114 | str(ofile_base) 115 | + "_diag_r" 116 | + str(ryyyy)[-2:] 117 | + "_" 118 | + str(ayyyy) 119 | + "-" 120 | + str(am).zfill(2) 121 | + ".nc" 122 | ) 123 | ofile = opath + "/" + ofilename 124 | 125 | if verbose: 126 | disclaimer() 127 | print("\n PROCESSING: \t", ayyyy, "-", str(am).zfill(2)) 128 | print( 129 | "\n============================================================================================================" 130 | ) 131 | print(" ! using input path: \t", ipath) 132 | print(" ! using variable mass: \t" + str(fvariable_mass)) 133 | if fvariable_mass: 134 | print(" \t ! reference date for number of particles: \t" + str(refdate)) 135 | if fwrite_netcdf: 136 | print(" ! writing netcdf output: \t" + str(fwrite_netcdf)) 137 | print(" \t ! with grid resolution: \t", str(gres)) 138 | print(" \t ! output file: \t", opath + "/" + ofilename) 139 | print(" ! using internal timer: \t" + str(ftimethis)) 140 | print(" ! using mode: \t" + str(mode)) 141 | print( 142 | "\n============================================================================================================" 143 | ) 144 | print( 145 | "\n============================================================================================================" 146 | ) 147 | 148 | ## Start timer 149 | if ftimethis: 150 | megatic = timeit.default_timer() 151 | 152 | ## Prepare grid 153 | glon, glat, garea = makegrid(resolution=gres) 154 | 155 | ## Handle dates 156 | date_bgn = datetime.datetime.strptime( 157 | str(ayyyy) + "-" + str(am).zfill(2) + "-" + str(ad).zfill(2), "%Y-%m-%d" 158 | ) 159 | # get end date (always 00 UTC of the 1st of the next month) 160 | nayyyy = (date_bgn + relativedelta(months=1)).strftime("%Y") 161 | nam = (date_bgn + relativedelta(months=1)).strftime("%m") 162 | date_end = datetime.datetime.strptime( 163 | str(nayyyy) + "-" + str(nam).zfill(2) + "-01-00", "%Y-%m-%d-%H" 164 | ) 165 | timestep = datetime.timedelta(hours=6) 166 | date_seq = [] 167 | fdate_seq = [] 168 | mfdate_seq = [] 169 | idate = date_bgn + timestep 170 | while idate <= date_end: 171 | date_seq.append(idate.strftime("%Y%m%d%H")) 172 | fdate_seq.append(idate) 173 | mfdate_seq.append(idate - timestep / 2) # -dt/2 for backward run 174 | idate += timestep 175 | ntime = len(date_seq) 176 | 177 | ##-- TESTMODE 178 | if mode == "test": 179 | ntime = 12 180 | date_seq = date_seq[0:ntime] 181 | fdate_seq = fdate_seq[0:ntime] 182 | mfdate_seq = mfdate_seq[0:ntime] 183 | 184 | ## Create empty netcdf file (to be filled) 185 | if fwrite_netcdf: 186 | writeemptync( 187 | ofile, 188 | mfdate_seq, 189 | glon, 190 | glat, 191 | strargs, 192 | precision, 193 | fproc_npart, 194 | fprec, 195 | fevap, 196 | fheat, 197 | ) 198 | 199 | # Read in reference distribution of parcels 200 | if fvariable_mass: 201 | print( 202 | " \n !!! WARNING !!! With this version, variable mass can only be applied to 01_diagnosis -- it cannot be used consistently for all steps yet! \n" 203 | ) 204 | ary_rnpart = get_refnpart(refdate=refdate, ryyyy=ryyyy, glon=glon, glat=glat) 205 | 206 | ##-- LOOP THROUGH FILES 207 | if verbose: 208 | print("\n=== \t Start main program: 01_diagnosis...\n") 209 | 210 | for ix in range(ntime): 211 | 212 | if verbose: 213 | print( 214 | "--------------------------------------------------------------------------------------" 215 | ) 216 | print("Processing " + str(fdate_seq[ix])) 217 | 218 | ## Read date related trajectories -> ary is of dimension (ntrajlen x nparticles x nvars) 219 | ary = readtraj( 220 | idate=date_seq[ix], 221 | ipath=ipath + "/" + str(ryyyy), 222 | ifile_base=ifile_base, 223 | verbose=verbose, 224 | ) 225 | ary = calc_allvars(ary) 226 | dq = trajparceldiff(ary[:, :, 5], "diff") 227 | mrh = np.apply_over_axes(np.mean, ary[:, :, 10], 0) 228 | dTH = trajparceldiff(ary[:, :, 11], "diff") 229 | 230 | nparticle = ary.shape[1] 231 | if verbose: 232 | print( 233 | " TOTAL: " + str(date_seq[ix]) + " has " + str(nparticle) + " parcels" 234 | ) 235 | 236 | ## TESTMODE: less parcels 237 | if mode == "test": 238 | ntot = range(10000, 10100) 239 | else: 240 | ntot = range(nparticle) 241 | 242 | # smalltic = timeit.default_timer() 243 | 244 | ##-- LOOP OVER PARCELS TO DIAGNOSE P, E, H (and npart) and assign to grid 245 | if fproc_npart: 246 | # get midpoint indices on grid from ary 247 | imidi = get_all_midpindices(ary, glon, glat) 248 | ary_npart = gridall( 249 | imidi[:, 1], imidi[:, 0], np.repeat(1, nparticle), glon=glon, glat=glat 250 | ) 251 | elif not fproc_npart: 252 | # currently just writing empty array ... to be changed 253 | ary_npart = np.zeros(shape=(glat.size, glon.size)) 254 | 255 | ## Precipitation 256 | if fprec: 257 | fdqv = np.where(dq[0, :] < cprec_dqv) 258 | frh = np.where(mrh[0, :] > cprec_rh) 259 | isprec = np.intersect1d(fdqv, frh) 260 | p_ary = ary[:, isprec, :] 261 | pmidi = get_all_midpindices(p_ary, glon, glat) 262 | # grid 263 | ary_prec = gridall( 264 | pmidi[:, 1], pmidi[:, 0], dq[:, isprec][0], glon=glon, glat=glat 265 | ) 266 | ary_pnpart = gridall( 267 | pmidi[:, 1], 268 | pmidi[:, 0], 269 | np.repeat(1, isprec.size), 270 | glon=glon, 271 | glat=glat, 272 | ) 273 | 274 | ## Evaporation 275 | if fevap: 276 | isevap = filter_for_evap_parcels( 277 | ary, 278 | dq, 279 | cpbl_method, 280 | cpbl_strict, 281 | cpbl_factor, 282 | cevap_hgt, 283 | fevap_drh, 284 | cevap_drh, 285 | cevap_dqv, 286 | veryverbose, 287 | ) 288 | e_ary = ary[:, isevap, :] 289 | emidi = get_all_midpindices(e_ary, glon, glat) 290 | # grid 291 | ary_evap = gridall( 292 | emidi[:, 1], emidi[:, 0], dq[:, isevap][0], glon=glon, glat=glat 293 | ) 294 | ary_enpart = gridall( 295 | emidi[:, 1], 296 | emidi[:, 0], 297 | np.repeat(1, isevap.size), 298 | glon=glon, 299 | glat=glat, 300 | ) 301 | 302 | ## Sensible heat 303 | if fheat: 304 | isheat = filter_for_heat_parcels( 305 | ary, 306 | dTH, 307 | cpbl_method, 308 | cpbl_strict, 309 | cpbl_factor, 310 | cheat_hgt, 311 | fheat_drh, 312 | cheat_drh, 313 | cheat_dtemp, 314 | fheat_rdq, 315 | cheat_rdq, 316 | veryverbose, 317 | ) 318 | h_ary = ary[:, isheat, :] 319 | hmidi = get_all_midpindices(h_ary, glon, glat) 320 | # grid 321 | ary_heat = gridall( 322 | hmidi[:, 1], hmidi[:, 0], dTH[:, isheat][0], glon=glon, glat=glat 323 | ) 324 | ary_hnpart = gridall( 325 | hmidi[:, 1], 326 | hmidi[:, 0], 327 | np.repeat(1, isheat.size), 328 | glon=glon, 329 | glat=glat, 330 | ) 331 | 332 | # smalltoc = timeit.default_timer() 333 | # print("=== \t All parcels: ",str(round(smalltoc-smalltic, 2)),"seconds \n") 334 | 335 | ## Convert units 336 | if verbose: 337 | print(" * Converting units...") 338 | if fprec: 339 | ary_prec[:, :] = convertunits(ary_prec[:, :], garea, "P") 340 | if fevap: 341 | ary_evap[:, :] = convertunits(ary_evap[:, :], garea, "E") 342 | if fheat: 343 | ary_heat[:, :] = convertunits(ary_heat[:, :], garea, "H") 344 | 345 | ## Scale with parcel mass 346 | if fvariable_mass: 347 | print( 348 | " !!! WARNING !!! With this version, variable mass can only be applied to 01_diagnosis -- it cannot be used consistently for all steps yet!" 349 | ) 350 | if verbose: 351 | print(" * Applying variable mass...") 352 | if fprec: 353 | ary_prec[:, :] = scale_mass(ary_prec[:, :], ary_npart[:, :], ary_rnpart) 354 | if fevap: 355 | ary_evap[:, :] = scale_mass(ary_evap[:, :], ary_npart[:, :], ary_rnpart) 356 | if fheat: 357 | ary_heat[:, :] = scale_mass(ary_heat[:, :], ary_npart[:, :], ary_rnpart) 358 | 359 | # write to netcdf file 360 | if fwrite_netcdf: 361 | # writenc(ofile,ix,ary_prec[:,:],ary_evap[:,:],ary_heat[:,:],ary_npart[:,:],ary_pnpart[:,:],ary_enpart[:,:],ary_hnpart[:,:]) 362 | if fproc_npart: 363 | writenc(ofile, ix, ary_npart[:, :], "n_part") 364 | if fprec: 365 | writenc(ofile, ix, ary_prec[:, :], "P") 366 | writenc(ofile, ix, ary_pnpart[:, :], "P_n_part") 367 | if fevap: 368 | writenc(ofile, ix, ary_evap[:, :], "E") 369 | writenc(ofile, ix, ary_enpart[:, :], "E_n_part") 370 | if fheat: 371 | writenc(ofile, ix, ary_heat[:, :], "H") 372 | writenc(ofile, ix, ary_hnpart[:, :], "H_n_part") 373 | 374 | ## re-init. arrays 375 | if fproc_npart: 376 | ary_npart[:, :] = 0 377 | if fprec: 378 | ary_pnpart[:, :] = 0 379 | ary_prec[:, :] = 0 380 | if fevap: 381 | ary_enpart[:, :] = 0 382 | ary_evap[:, :] = 0 383 | if fheat: 384 | ary_hnpart[:, :] = 0 385 | ary_heat[:, :] = 0 386 | 387 | if ftimethis: 388 | megatoc = timeit.default_timer() 389 | if verbose: 390 | print( 391 | "\n=== \t End main program (total runtime so far: ", 392 | str(round(megatoc - megatic, 2)), 393 | "seconds) \n", 394 | ) 395 | 396 | if verbose: 397 | if fwrite_netcdf: 398 | print("\n Successfully written: " + ofile + " !") 399 | -------------------------------------------------------------------------------- /src/validation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | MAIN FUNCTION TO VALIDATE 5 | """ 6 | 7 | ########################################################################### 8 | ##--- MODULES 9 | ########################################################################### 10 | 11 | import argparse 12 | import calendar 13 | import csv 14 | import datetime 15 | import fnmatch 16 | import gzip 17 | import imp 18 | import math 19 | import os 20 | import random 21 | import re 22 | import struct 23 | import sys 24 | import time 25 | import timeit 26 | import warnings 27 | from datetime import date, datetime, timedelta 28 | from functools import reduce 29 | from math import acos, atan, atan2, cos, floor, sin, sqrt 30 | 31 | import h5py 32 | import netCDF4 as nc4 33 | import numpy as np 34 | import pandas as pd 35 | from dateutil.relativedelta import relativedelta 36 | 37 | ########################################################################### 38 | ##--- FUNCTIONS + COMMAND LINE ARGUMENTS 39 | ########################################################################### 40 | 41 | ## (1) LOADING FUNCTIONS 42 | exec(open("hamsterfunctions.py").read()) 43 | 44 | ## (2) COMMAND LINE ARGUMENTS 45 | # read command line arguments (dates, thresholds and other flags) 46 | args = read_cmdargs() 47 | verbose = args.verbose 48 | print(printsettings(args)) 49 | 50 | # just waiting a random number of seconds (max. 30s) 51 | # to avoid overlap of path.exist and makedirs between parallel jobs (any better solution?) 52 | if args.waiter: 53 | waiter = random.randint(0, 30) 54 | time.sleep(waiter) 55 | 56 | ########################################################################### 57 | ##--- PATHS 58 | ########################################################################### 59 | 60 | ## determine working directory 61 | wpath = os.getcwd() 62 | os.chdir(wpath) 63 | 64 | ## load input and output paths & input file name base(s) 65 | print("Using paths from: " + wpath + "/" + args.pathfile) 66 | content = imp.load_source("", wpath + "/" + args.pathfile) # load like a python module 67 | path_refp = content.path_ref_p 68 | path_refe = content.path_ref_e 69 | path_refh = content.path_ref_h 70 | path_diag = content.path_diag 71 | 72 | ########################################################################### 73 | ##--- ADDITIONAL VALIDATION FUNCTIONS 74 | ########################################################################### 75 | 76 | 77 | def contingency_table(ref, mod, thresh=0): 78 | # creates a contingency table based on 1D np.arrays 79 | # ATTN: mod is already binary data, i.e. 1 = detected; 0 = not detected 80 | ieventobs = np.where(ref > thresh)[0] 81 | ineventobs = np.where(ref <= thresh)[0] 82 | a = len(np.where(mod[ieventobs] >= 1)[0]) # hits 83 | b = len(np.where(mod[ineventobs] >= 1)[0]) # false alarms 84 | c = len(np.where(mod[ieventobs] < 1)[0]) # misses 85 | d = len(np.where(mod[ineventobs] < 1)[0]) # correct negatives 86 | return {"a": a, "b": b, "c": c, "d": d} 87 | 88 | 89 | def try_div(x, y): 90 | try: 91 | return x / y 92 | except ZeroDivisionError: 93 | return 0 94 | 95 | 96 | def calc_ctab_measures(cdict): 97 | # calculates common contingency table scores 98 | # scores following definitions from https://www.cawcr.gov.au/projects/verification/ 99 | a = cdict["a"] # hits 100 | b = cdict["b"] # false alarms 101 | c = cdict["c"] # misses 102 | d = cdict["d"] # correct negatives 103 | # calculate scores 104 | acc = try_div(a + d, a + b + c + d) # accuracy 105 | far = try_div(b, a + b) # false alarm ratio 106 | fbias = try_div(a + b, a + c) # frequency bias 107 | pod = try_div(a, a + c) # probability of detection (hit rate) 108 | pofd = try_div(b, b + d) # probability of false detection (false alarm rate) 109 | sr = try_div(a, a + b) # success ratio 110 | ts = try_div(a, a + c + b) # threat score (critical success index) 111 | a_random = try_div((a + c) * (a + b), a + b + c + d) 112 | ets = try_div( 113 | (a - a_random), (a + b + c + a_random) 114 | ) # equitable threat score (gilbert skill score) 115 | pss = pod - pofd # peirce's skill score (true skill statistic) 116 | odr = try_div(a * d, c * b) # odd's ratio 117 | return { 118 | "acc": acc, 119 | "far": far, 120 | "fbias": fbias, 121 | "pod": pod, 122 | "pofd": pofd, 123 | "sr": sr, 124 | "pss": pss, 125 | "odr": odr, 126 | } 127 | 128 | 129 | def init_netcdf(ofile, lat, lon): 130 | print(" Writing output file: " + str(ofile)) 131 | ncf = nc4.Dataset(ofile, "w", format="NETCDF4") 132 | ncf.createDimension("lat", lat.size) 133 | ncf.createDimension("lon", lon.size) 134 | latitudes = ncf.createVariable("lat", "f8", "lat") 135 | longitudes = ncf.createVariable("lon", "f8", "lon") 136 | ncf.title = "Validation statistics" 137 | ncf.description = "" 138 | today = datetime.datetime.now() 139 | ncf.history = "Created " + today.strftime("%d/%m/%Y %H:%M:%S") 140 | ncf.institution = ( 141 | "Hydro-Climate Extremes Laboratory (H-CEL), Ghent University, Ghent, Belgium" 142 | ) 143 | ncf.source = "Validation statistics for HAMSTER" 144 | latitudes.units = "degrees_north" 145 | longitudes.units = "degrees_east" 146 | latitudes[:] = lat[:] 147 | longitudes[:] = lon[:] 148 | ncf.close() 149 | 150 | 151 | def calc_stats(mdata, mndata, rdata, thresh=0.001): 152 | bias = np.zeros(shape=(mdata.shape[1], mdata.shape[2])) 153 | acc = np.zeros(shape=(mdata.shape[1], mdata.shape[2])) 154 | pod = np.zeros(shape=(mdata.shape[1], mdata.shape[2])) 155 | pofd = np.zeros(shape=(mdata.shape[1], mdata.shape[2])) 156 | pss = np.zeros(shape=(mdata.shape[1], mdata.shape[2])) 157 | fbias = np.zeros(shape=(mdata.shape[1], mdata.shape[2])) 158 | odr = np.zeros(shape=(mdata.shape[1], mdata.shape[2])) 159 | print(" Processing...") 160 | for y in range(rdata.shape[1]): 161 | progress = 100 * y / (rdata.shape[1] - 1) 162 | if verbose and round(progress) in range(10, 100, 5): 163 | print(" ..." + str(round(progress)) + "%...") 164 | for x in range(rdata.shape[2]): 165 | myctab = calc_ctab_measures( 166 | contingency_table(rdata[:, y, x], mndata[:, y, x], thresh=thresh) 167 | ) 168 | acc[y, x] = myctab["acc"] 169 | pod[y, x] = myctab["pod"] 170 | pofd[y, x] = myctab["pofd"] 171 | pss[y, x] = myctab["pss"] 172 | fbias[y, x] = myctab["fbias"] 173 | odr[y, x] = myctab["odr"] 174 | diff = mdata[:, y, x] - rdata[:, y, x] 175 | bias[y, x] = np.nanmean(diff) 176 | return { 177 | "acc": acc, 178 | "fbias": fbias, 179 | "pod": pod, 180 | "pofd": pofd, 181 | "pss": pss, 182 | "odr": odr, 183 | "diff": diff, 184 | "bias": bias, 185 | } 186 | 187 | 188 | def write_to_netcdf(ofile, vals, var="P"): 189 | print( 190 | " Writing " + str(var) + " validation statistics to output file: " + str(ofile) 191 | ) 192 | 193 | # create netCDF4 instance 194 | ncf = nc4.Dataset(ofile, "r+", format="NETCDF4") 195 | 196 | # create variables 197 | ncacc = ncf.createVariable(str(var) + "_acc", "f8", ("lat", "lon")) 198 | ncpod = ncf.createVariable(str(var) + "_pod", "f8", ("lat", "lon")) 199 | ncpofd = ncf.createVariable(str(var) + "_pofd", "f8", ("lat", "lon")) 200 | ncpss = ncf.createVariable(str(var) + "_pss", "f8", ("lat", "lon")) 201 | ncodr = ncf.createVariable(str(var) + "_odr", "f8", ("lat", "lon")) 202 | ncfbias = ncf.createVariable(str(var) + "_fbias", "f8", ("lat", "lon")) 203 | ncbias = ncf.createVariable(str(var) + "_bias", "f8", ("lat", "lon")) 204 | 205 | # set attributes 206 | ncpod.long_name = str(var) + ": probability of detection (hit rate)" 207 | ncacc.long_name = str(var) + ": accuracy" 208 | ncbias.long_name = str(var) + ": bias" 209 | ncfbias.long_name = str(var) + ": frequency bias" 210 | ncpofd.long_name = str(var) + ": probability of false detection (false alarm rate)" 211 | ncpss.long_name = str(var) + ": peirce´s skill score (true skill statistic)" 212 | ncodr.long_name = str(var) + ": odd`s ratio" 213 | 214 | # write to netcdf 215 | ncodr[:] = vals["odr"][:] 216 | ncpod[:] = vals["pod"][:] 217 | ncpss[:] = vals["pss"][:] 218 | ncpofd[:] = vals["pofd"][:] 219 | ncacc[:] = vals["acc"][:] 220 | ncpofd[:] = vals["pofd"][:] 221 | ncfbias[:] = vals["fbias"][:] 222 | ncbias[:] = vals["bias"][:] 223 | 224 | # close file 225 | ncf.close() 226 | 227 | 228 | ########################################################################### 229 | ########################################################################### 230 | ##--- MAIN 231 | ########################################################################### 232 | ########################################################################### 233 | 234 | 235 | def main_validation( 236 | ryyyy, 237 | ayyyy, 238 | am, 239 | opath_diag, # diagnosis (output) 240 | ipath_refp, 241 | ipath_refe, 242 | ipath_refh, 243 | opath, 244 | ofile_base, # output 245 | fprec, 246 | fevap, 247 | fheat, 248 | verbose, 249 | veryverbose, 250 | fwrite_netcdf, 251 | ): 252 | 253 | ## construct precise input and storage paths 254 | ofilename = ( 255 | str(ofile_base) 256 | + "_diag_r" 257 | + str(ryyyy)[-2:] 258 | + "_" 259 | + str(ayyyy) 260 | + "-" 261 | + str(am).zfill(2) 262 | + "_validation.nc" 263 | ) 264 | ofile = opath + "/" + ofilename 265 | 266 | #### DISCLAIMER 267 | if verbose: 268 | disclaimer() 269 | print("\n PROCESSING: \t", ayyyy, "-", str(am).zfill(2) + "\n") 270 | ## Resets & consistency checks 271 | if verbose: 272 | print(" ! using input paths: \t") 273 | print("\t" + str(opath_diag)) 274 | print(" ! using reference data from: \t") 275 | print("\t" + str(ipath_refp)) 276 | print("\t" + str(ipath_refe)) 277 | print("\t" + str(ipath_refh)) 278 | print(" ! writing netcdf output: \t") 279 | print("\t" + str(ofile)) 280 | print( 281 | "\n============================================================================================================" 282 | ) 283 | print("\n") 284 | 285 | ##--1. load diagnosis data #################################################### 286 | if verbose: 287 | print(" * Reading diagnosis data...") 288 | 289 | # read concatenated data 290 | ifilename = ( 291 | str(opath_diag) 292 | + "/" 293 | + str(ofile_base) 294 | + "_diag_r" 295 | + str(ryyyy)[-2:] 296 | + "_" 297 | + str(ayyyy) 298 | + "-" 299 | + str(am).zfill(2) 300 | + ".nc" 301 | ) 302 | with nc4.Dataset(ifilename, mode="r") as f: 303 | idate_seq = nc4.num2date(f["time"][:], f["time"].units, f["time"].calendar) 304 | totlats, totlons = read_diagdata( 305 | opath_diag, ofile_base, ryyyy, idate_seq, var="grid" 306 | ) 307 | glon, glat, garea = makegrid(resolution=abs(totlats[0] - totlats[1])) 308 | ftime = read_diagdata(opath_diag, ofile_base, ryyyy, idate_seq, var="time") 309 | fdays = np.unique(cal2date(ftime)) 310 | fyears = np.unique(date2year(ftime)) 311 | if fevap: 312 | E = read_diagdata(opath_diag, ofile_base, ryyyy, idate_seq, var="E") 313 | E_npart = read_diagdata( 314 | opath_diag, ofile_base, ryyyy, idate_seq, var="E_n_part" 315 | ) 316 | if fprec: 317 | P = -read_diagdata(opath_diag, ofile_base, ryyyy, idate_seq, var="P") 318 | P_npart = read_diagdata( 319 | opath_diag, ofile_base, ryyyy, idate_seq, var="P_n_part" 320 | ) 321 | if fheat: 322 | H = read_diagdata(opath_diag, ofile_base, ryyyy, idate_seq, var="H") 323 | H_npart = read_diagdata( 324 | opath_diag, ofile_base, ryyyy, idate_seq, var="H_n_part" 325 | ) 326 | 327 | # make sure we use daily aggregates 328 | if fdays.size != ftime.size: 329 | if fevap: 330 | Etot = convert2daily(E, ftime, fagg="sum") 331 | Enparttot = convert2daily(E_npart, ftime, fagg="sum") 332 | if fprec: 333 | Ptot = convert2daily(P, ftime, fagg="sum") 334 | Pnparttot = convert2daily(P_npart, ftime, fagg="sum") 335 | if fheat: 336 | Htot = convert2daily(H, ftime, fagg="mean") 337 | Hnparttot = convert2daily(H_npart, ftime, fagg="sum") 338 | else: 339 | if fevap: 340 | Etot = E 341 | Enparttot = E_npart 342 | if fprec: 343 | Ptot = P 344 | Pnparttot = P_npart 345 | if fheat: 346 | Htot = H 347 | Hnparttot = H_npart 348 | 349 | ##--2. load reference data #################################################### 350 | """ 351 | this part is STRICTLY CODED FOR (12-hourly) ERA-INTERIM only (so far), 352 | and HARDCODED too 353 | """ 354 | if verbose: 355 | print(" * Reading reference data...") 356 | 357 | if fevap: 358 | Eref, reflats, reflons = eraloader_12hourly( 359 | var="e", 360 | datapath=ipath_refe + "/E_1deg_", 361 | maskpos=True, 362 | maskneg=False, 363 | uptake_years=fyears, 364 | uptake_dates=fdays, 365 | lats=totlats, 366 | lons=totlons, 367 | ) 368 | gridcheck(totlats, reflats, totlons, reflons) 369 | 370 | if fheat: 371 | Href, reflats, reflons = eraloader_12hourly( 372 | var="sshf", 373 | datapath=ipath_refh + "/H_1deg_", 374 | maskpos=True, 375 | maskneg=False, 376 | uptake_years=fyears, 377 | uptake_dates=fdays, 378 | lats=totlats, 379 | lons=totlons, 380 | ) 381 | gridcheck(totlats, reflats, totlons, reflons) 382 | 383 | if fprec: 384 | Pref, reflats, reflons = eraloader_12hourly( 385 | var="tp", 386 | datapath=ipath_refp + "/P_1deg_", 387 | maskpos=False, # do NOT set this to True! 388 | maskneg=True, 389 | uptake_years=fyears, 390 | uptake_dates=fdays, 391 | lats=totlats, 392 | lons=totlons, 393 | ) 394 | gridcheck(totlats, reflats, totlons, reflons) 395 | 396 | ##--3. validation ######################################################### 397 | if verbose: 398 | print(" * Starting validation...") 399 | 400 | init_netcdf(ofile, totlats, totlons) 401 | 402 | if fprec: 403 | if verbose: 404 | print(" P") 405 | pstats = calc_stats(Ptot, Pnparttot, Pref, thresh=0.001) 406 | write_to_netcdf(ofile, pstats, var="P") 407 | if fevap: 408 | if verbose: 409 | print(" E") 410 | estats = calc_stats(Etot, Enparttot, Eref, thresh=0.001) 411 | write_to_netcdf(ofile, estats, var="E") 412 | if fheat: 413 | if verbose: 414 | print(" H") 415 | hstats = calc_stats(Htot, Hnparttot, Href, thresh=1) 416 | write_to_netcdf(ofile, hstats, var="H") 417 | 418 | 419 | ########################################################################### 420 | ##--- run main script 421 | ########################################################################### 422 | main_validation( 423 | ryyyy=args.ryyyy, 424 | ayyyy=args.ayyyy, 425 | am=args.am, 426 | opath_diag=path_diag, 427 | ipath_refp=path_refp, 428 | ipath_refe=path_refe, 429 | ipath_refh=path_refh, 430 | opath=path_diag, 431 | ofile_base=args.expid, 432 | fprec=args.fprec, 433 | fevap=args.fevap, 434 | fheat=args.fheat, 435 | verbose=args.verbose, 436 | veryverbose=args.veryverbose, 437 | fwrite_netcdf=args.write_netcdf, 438 | ) 439 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HAMSTER – a Heat And MoiSture Tracking framEwoRk 2 | 3 | **HAMSTER** is an open source software framework to trace heat and moisture through the atmosphere and establish (bias-corrected) source–receptor relationships, using output from a Lagrangian model. It has been developed within the DRY-2-DRY project at the Hydro-Climatic Extremes Laboratory (H-CEL) at Ghent University. 4 | 5 | [![DOI](https://zenodo.org/badge/372759352.svg)](https://zenodo.org/badge/latestdoi/372759352) 6 | 7 | - - - - 8 | ## What is HAMSTER? 9 | **HAMSTER** is a heat- and moisture tracking framwork to evaluate air parcel trajectories from a Lagrangian model, such as FLEXPART (Stohl et al., 2005) and to establish source–receptor relationships, such as the source regions of precipitation or heat. The current version of **HAMSTER** has been built using simulations from FLEXPART driven with ERA-Interim reanalysis data, but other simulations may be supported as well. 10 | 11 | **HAMSTER** consists of 4 modules, 12 | 13 | 0. flex2traj 14 | 1. Diagnosis 15 | 2. Attribution 16 | 3. Bias-correction 17 | 18 | which build up on each other. It is suggested to run them sequentially to obtain the most efficient and informative workflow. In the following, a short description of each module (step) is given. 19 | 20 | ### 0. flex2traj 21 | This module of **HAMSTER** reads in the instantaneous binary FLEXPART files, filters for a specific region (using a netcdf mask), constructs trajectories and writes them to a file. 22 | 23 | ### 1. Diagnosis 24 | The diagnosis part of **HAMSTER** identifies atmospheric fluxes of humidity (precipitation and evaporation) or heat (sensible heat flux) using trajectories constructed from FLEXPART binary data. There are several thresholds and criteria that can be set to reduce the bias and to increase the probability of detection for each flux. The output from this diagnosis step is used to bias correct source–receptor relationships. 25 | 26 | ### 2. Attribution 27 | The attribution part of **HAMSTER** constructs mass- and energy-conserving trajectories of heat and moisture (e.g. using a linear discounting of changes en route, or applying a random attribution for moisture), and establishes a first (biased) source–receptor relationship. The output of this step are, e.g., 4D netcdf files that illustrate the spatio-temporal origins of precipitation or *heat* arriving in the receptor region. Multiple options to construct these relationships are available. 28 | 29 | ### 3. Bias-correction 30 | The last module of **HAMSTER** uses information from the former two steps to bias-correct source–receptor relationships. Multiple options for bias-correction are available. 31 | 32 | - - - - 33 | ## What do I need to get and run HAMSTER? 34 | 35 | ### Prerequisites and Installation 36 | This section describes the prerequisites required to run HAMSTER, as well as the steps to install it. 37 | 38 | #### Prerequisites 39 | To run HAMSTER, you need 40 | * python 3 41 | * [git](https://git-scm.com/) 42 | 43 | and 44 | * [anaconda](https://www.anaconda.com/) (or similar to manage python packages) 45 | 46 | or 47 | * python 3 and the required modules on a cluster 48 | 49 | The main packages required to run **HAMSTER** are: 50 | ```bash 51 | import argparse 52 | import calendar 53 | import csv 54 | import fnmatch 55 | import gzip 56 | import imp 57 | import math 58 | import os 59 | import random 60 | import re 61 | import struct 62 | import sys 63 | import time 64 | import timeit 65 | import warnings 66 | import datetime 67 | import functools 68 | 69 | import h5py 70 | import netCDF4 71 | import numpy 72 | import pandas 73 | import dateutil 74 | ``` 75 | 76 | In addition, to execute the full chain (all 4 modules) of **HAMSTER**, the following data sets are needed: 77 | * **Output from a Lagrangian model** that traces air parcels and their properties (driven with a reanalysis or output from a GCM/RCM) 78 | * **Reference data**; e.g., the reanalysis used to run FLEXPART and track parcels, or any other reference data set used for bias-correction 79 | 80 | To run **HAMSTER**, you will also need to create one file: 81 | * `paths.txt` 82 | 83 | which lists the paths where the above data is found and where output will be stored. 84 | 85 | The file `paths.txt` is not part of **HAMSTER**. The order in this file is arbitrary, but it has to contain paths for diagnosis, attribution and biascorrection and reference data: 86 | ``` 87 | # This file contains all required paths and file names to run hamster; the order doesn't matter and paths can also be empty (if, e.g., not used) 88 | 89 | # MASK 90 | maskfile = "./flexpart_data/masks/mask.nc" 91 | 92 | # location of original flexpart files (untarred) 93 | ## (untar the original FLEXPART partposit_* files to this directory) 94 | path_orig = "./flexpart_data/orig" 95 | 96 | # location of the reference data used for bias correction (e.g., ERA-Interim) 97 | ## for each variable (P, E, H) 98 | path_ref_p = "./ERA-INTERIM/1x1/tp_12hourly" 99 | path_ref_e = "./ERA-INTERIM/1x1/evap_12hourly" 100 | path_ref_h = "./ERA-INTERIM/1x1/sshf_12hourly" 101 | 102 | # path and base name for global parcel diag data (t2) 103 | base_f2t_diag = "global" 104 | path_f2t_diag = "./flexpart_data/global/f2t_diag" 105 | 106 | # path and base name for parcel trajectory data 107 | base_f2t_traj = "bahamas_10d" 108 | path_f2t_traj = "./flexpart_data/bahamas/f2t_traj" 109 | 110 | # paths for processed data 111 | path_diag = "./flexpart_data/global/diag" 112 | path_attr = "./flexpart_data/bahamas/attr" 113 | path_bias = "./flexpart_data/bahamas/bias" 114 | ``` 115 | 116 | #### Installation 117 | To install **HAMSTER**, do the following: 118 | 119 | 1. Clone the repository 120 | ``` 121 | git clone https://github.com/h-cel/hamster 122 | cd hamster 123 | ``` 124 | 2. Make an anaconda environment with the necessary python packages 125 | ``` 126 | conda create -n _newenvironment_ --file requirements.txt 127 | ``` 128 | or install the packages listed in requirements.txt in your local environment. Note, however, that due to different versions, some errors might occur. It is thus recommended to load preinstalled environments, such as the one above. 129 | 130 | 131 | - - - - 132 | ## How do I run HAMSTER? 133 | 134 | To run **HAMSTER**, change into the `src` directory 135 | ``` 136 | cd src 137 | ``` 138 | and run 139 | ```python 140 | python main.py 141 | ``` 142 | 143 | Note that — without any flags — main.py is run with default values. Use 144 | ```python 145 | python main.py -h 146 | ``` 147 | for more details on setting dates, thresholds and other options. All user-specific paths are set in `paths.txt`. 148 | 149 | ### Quick start. 150 | To run all steps sequentially with the default settings, please proceed as follows. First, extract global 2-step trajectories (`steps--0 --maskval -999 --ctraj_len 0`) and perform the global diagnosis of fluxes (`--steps 1`), using 151 | ``` 152 | python main.py --steps 0 --maskval -999 --ctraj_len 0 153 | python main.py --steps 1 154 | ``` 155 | Then, extract 10-day trajectories for a specific region (using a default maskvalue of 1 for the given maskfile; `--steps 0`), which are required to diagnose source regions of heat and moisture (`--steps 2`), and finally bias-correct these source regions using the global diagnosis data from above (`--steps 3`): 156 | ``` 157 | python main.py --steps 0 158 | python main.py --steps 2 159 | python main.py --steps 3 160 | ``` 161 | 162 | **The most important settings are**: 163 | 164 | - `--steps` to select the part of hamster that is being executed (e.g., `--steps 0` runs flex2traj, `--steps 1` runs the diagnosis, `--steps 2` performs the attribution, ...) 165 | - `--ayyyy` and `--am` to select the analysis year and month (e.g., `--ayyyy 2002 --am 1`) 166 | - `--expid` to name a setting (e.g., `--expid "ALL-ABL"`) 167 | - `--ctraj_len` to determine the maximum length of a trajectory for evaluation (e.g., `--ctraj_len 15` to select 15 days; 10 is the default) 168 | - `--maskval` to filter for a value other than 1 using the maskfile from `paths.txt`(e.g., `--maskval 5001`) 169 | 170 | - - - - 171 | ## What flags can I set? 172 | 173 | Here, we provide a short description of all flags that be used when running **HAMSTER**. 174 | 175 | #### The detection of fluxes (P, E and H) is set via a set of flags. 176 | 177 | The detection of **precipitation** is set via `-–cprec_dqv` and `–-cprec_rh`: 178 | - `-–cprec_dqv` to set the minimum loss of specific humidity (unit: kg kg-1; default: 0 kg kg-1) 179 | - `–-cprec_rh` to the minimum average relative humidity (unit: %; default: 80% following the convection scheme from Emanuel, 1991) 180 | 181 | The detection of **evaporation** is set via `--cevap_dqv`, `--fevap_drh`, `--cevap_drh`, `--cevap_hgt`: 182 | - `--cevap_dqv` to set a minimum increase in specific humidity (unit: kg kg-1; default: 0.2 kg kg-1) 183 | - `--cevap_hgt` to filter for specific heights (unit: m; default: 0) 184 | - `--fevap_drh` to filter for relative humidity changes (False/True; default: False) employing a maximum change of `--cevap_drh` (unit: %, default: 15%) 185 | - `--fallingdry` to shorten the trajectory length if a parcel *falls dry* (q<0.00005 kg kg-1; default: False) 186 | 187 | The detection of **sensible heat fluxes** is set via `--cheat_dtemp`, `--cheat_hgt`, `--fheat_drh`, `--cheat_drh`, `--fheat_rdq` and `--cheat_rdq`: 188 | - `--cheat_dtemp` to set a minimum increase in potential temperature (unit: K; default: 1 K) 189 | - `--cheat_hgt` to filter for specific heights (unit: m; default: 0) 190 | - `--fheat_drh` to filter for relative humidity changes (False/True; default: False) employing a maximum change of `--cheat_drh` (unit: %, default: 15%) 191 | - `--fheat_rdq` to filter for relative specific humidity changes (False/True; default: False) employing a maximum change of `--cheat_rdq` (expressed as delta(qv)/qv; unit: %; default: 10%) 192 | 193 | For E and H, the detection of fluxes can be limited to the atmospheric boundary layer (ABL): 194 | - `--cpbl_method` describes the method used to determine the ABL height between two instances. Options are `actual`, `mean` and `max` (default: `max`). 195 | - `--cpbl_factor` sets a factor that is used to increase (>1) or decrease (<1) the ABL heights (default: 1). 196 | - `--cpbl_strict` determines the 'strictness' of the ABL criteria (`--cpbl_strict 2` requires both instances to be within the actual/maximum/mean ABL, `--cpbl_strict 1` requires only one instance to be within the actual/mean/maximum ABL; `--cpbl_strict 0` does not filter for the ABL at all). 197 | 198 | Note that the ABL criteria are set consistently for E and H. 199 | 200 | Using these flags, a lot of the settings used in FLEXPART publications can be mimicked. 201 | - **Sodemann et al., 2008** for the detection of E (minimum threshold for dqv/dt; parcel has be reside in the vicinity of the ABL; note, however, that minor differences exists, e.g. through the application of the ABL factor 1.5 everywhere as in Keune and Miralles (2019), etc.): 202 | ``` 203 | --cevap_dqv 0.0002 --fallingdry True --fevap_drh False --cpbl_method "mean" --cpbl_factor 1.5 204 | ``` 205 | - **Fremme and Sodemann, 2019** and **Sodemann, 2020** (minimum threshold for dqv/dt; parcel does not have to reside in the ABL): 206 | ``` 207 | --cevap_dqv 0.0001 --fevap_drh False --cpbl_strict 0 208 | ``` 209 | - **Schumacher et al., 2019** for the detection of H (minimum potential temperature increase; limitation by change in specific humidity content; parcel has to be within the maximum ABL at both time steps): 210 | ``` 211 | --cheat_dtemp 1 --cheat_rdq 10 --fheat_rdq True --fheat_drh False --cpbl_strict 2 --cpbl_method "max" 212 | ``` 213 | 214 | 215 | #### A few more notes on flags... 216 | - Short flags available! See `python main.py -h` for details (e.g., `-–ayyyy`can be replaced with `-ay` etc.) 217 | - Bias correction is, per default, performed using ERA-Interim data (i.e., the base for FLEXPART simulations). However, alternative data sets for P, E and/or H can be employed using `--pref_data others`, `--eref_data others` and `--href_data others`, respectively. The data has to be placed in the respective paths of `paths.txt` (`path_ref_p`, `path_ref_e`, `path_ref_h`) and **have to be in the correct format (netcdf), in the correct resolution on the correct (grid set via `--resolution`) with daily (or subdaily) time steps, and in the correct units (mm d-1 and W m-2)**. If `others` is employed, all files with the year (`--ayyyy`) from the respective directory are read in and used for bias correction. 218 | - Analysis is performed on a monthly basis: for an independent analysis of months, the flag `--memento` is incorporated (default: True). This flag requires additional data of the previous month, that is extracted from the trajectories or, if not available from the trajectories, read in from the binary FLEXPART files in a preloop. Hence, the smoothest workflow is obtained if flex2traj dumps trajectories that are one day longer than what is needed in 02_attribution (e.g., run `python main.py --steps 0 --ctraj_len 16` but `python main.py --steps 0 --ctraj_len 15` -- the preloop is not needed in this case). 219 | - The `expid` has to be used consistently for the settings between steps 1-2-3. Otherwise, source-sink relationships may be bias-corrected with other criteria (DANGER!). There is no proper check for this – the user has to make sure they are using everything correctly. Various regions or attribution methods can be run using separate directories. 220 | - There are quite a few flags for 02_attribution (e.g., refering to settings concerning the random attribution) and 03_biascorrection (e.g., refering to the applied time scale and the aggregation of the output) available. Please use the help option for details for now. 221 | - While the output of flex2traj could be adjusted through modifications in 00_flex2traj.py, currently, all other steps require the following 9 variables (and in that specific order): `parcel id`, `lon`, `lat`, `ztra1`, `topo`, `qvi`, `rhoi`, `hmixi`, `tti`. 222 | - If `--writestats True` is set for `--steps 2`, then the attribution statistics are written to a file `*_stats.csv` (absolute fraction of attributed precipitation, etc.). If `--writestats True` is set for `--steps 3`, then the validation statistics are written to a file `*_stats.csv` (bias in the sink region, the probability of detection etc.). 223 | - Use `--maskval -999` (or set maskfile=None in paths.txt) in combination with `--ctraj_len 0` to extract global 2-step trajectories for a global 'diagnosis' with flex2traj. 224 | 225 | - - - - 226 | ## An example. 227 | 228 | Here, we provide a very basic example. 229 | 230 | 1. Create a (global) netcdf file with a mask (value=1) for a specific region of interest, e.g., the Bahamas. 231 | 2. Adjust the maskfile in `paths.txt`. 232 | 3. Construct trajectories for parcels whose arrival+midpoints are over the Bahamas (don't forget to untar the binary FLEXPART simulations for this and the previous month and the first day of the following month): 233 | ```python 234 | python main.py --steps 0 --ayyyy 2000 --am 6 --ctraj_len 11 --maskval 1 235 | ``` 236 | 4. Perform a global analysis of fluxes (and the previous month), and evaluate the bias and the reliability of detection for your region of interest and its (potential) source region, possibly selecting various diagnosis methods and fine tuning detection criteria (using the already available global data set on the VO), e.g., 237 | ```python 238 | python main.py --steps 1 --ayyyy 2000 --am 6 --expid "DEFAULT" 239 | ... 240 | python main.py --steps 1 --ayyyy 2000 --am 6 --cpbl_strict 2 --cpbl_method "max" --cevap_dqv 0 --cheat_dtemp 0 --expid "ALL-ABL" 241 | ``` 242 | 5. Once you have fine-tuned your detection criteria, perform a first backward analysis considering a trajectory length of 10 days, e.g. 243 | ```python 244 | python main.py --steps 2 --ayyyy 2000 --am 6 --ctraj_len 10 --cpbl_strict 2 --cpbl_method "max" --cevap_dqv 0 --cheat_dtemp 0 --expid "ALL-ABL" 245 | ``` 246 | 6. Bias-correct the established source and aggregate the results over the backward time dimension 247 | ```python 248 | python main.py --steps 3 --ayyyy 2000 --am 6 --expid "ALL-ABL" --bc_aggbwtime True 249 | ``` 250 | The final netcdf file, `ALL-ABL_biascor-attr_r02_2002-06.nc` then contains all the source regions of heat and precipitation, both the raw and bias-corrected version (i.e., Had and Had_Hs, and E2P, E2P_Es, E2P_Ps, and E2P_EPs). 251 | 252 | - - - - 253 | ## Miscellaneous notes 254 | - Everything is coded for a **backward** analysis (Where does the heat come from? What is the source region of precipitation?). Adjustments for a forward analysis can be easily made, but require code changes. 255 | - Note that, however, flex2traj writes out data in a forward format (startdate --> enddate; but still filtering for the last step, i.e. in a backward manner), but that the time axis is swapped when reading this data in (enddate <-- startdate, see function `readtraj`) for all the remaining analysis steps. 256 | - Everything is more or less hard-coded for (global) FLEXPART–ERA-Interim simulations with a 6-hourly time step and a maximum of ~2 million parcels. Any changes in resolution or input data require code adjustments! 257 | - Note that regardless of the sink region size, 'flex2traj' reads in and temporarily stores data from all parcels during the backward analysis time period; in case of 15-day trajectories and 9 variables of interest, this translates to a numpy array with a size of ~ 7.2 GB (62 x 2e6 x 9 x 64 bit). For a small sink region with ~13'000 parcels (trajectory array: 62 x 13'000 x 9 x 64 bit ~ 0.5 GB), a total of 10 GB RAM is recommended to safely run flex2traj with a trajectory length of 15 days. 258 | - flex2traj-related directories are currently assumed to have an annual structure (e.g., path_f2t_diag + "/2002") - these are created automatically. 259 | - The minimum time scale for steps 1-2-3 is daily, which we assumed to be a reasonable limit for the FLEXPART–ERA-Interim simulations with 6-hourly time steps. 260 | - An additional file `*_warning.txt` is written, if a monthly bias-correction was required and daily data cannot be trusted (this is the case if, e.g., the reference data set contains precipitation for a specific day, but precipitation was not detected using FLEXPART and the selected detection criteria; and hence no trajectories were evaluated and no attribution for that specific day was performed, but the contribution of other precipitation days was upscaled to match the monthly precipitation amount). 261 | 262 | - - - - 263 | ## Known issues. 264 | - **Remote I/O error** when reading/writing a netcdf or h5 file: this only seems to happen with specific h5py / netCDF4 module versions (used in anaconda) and only on some clusters (victini). If you're on the UGent HPC cluster, please try to use the HPC modules listed above - then the error should disappear... 265 | - **Resolutions other than 1°**: the current version only supports grids of 1 degree - for ALL data (mask, reference data, hamster output) - the code should abort if that is not the case, stating that the grids are not identical, but it may not necessarily do so... so check your outputs carefully, if its runs through anyhow (at least step 3 - bias correction - should be erroneous!) 266 | 267 | - - - - 268 | ## Epilogue 269 | Keep in mind that... 270 | - **This code is not bug-free.** Please report any bugs through 'Issues' on https://github.com/h-cel/hamster/issues. 271 | - **This code is not intended to cover specific research-related tasks.** This code is intended to serve as a common base for the analysis of (FLEXPART) trajectories. Every user may create their own branch and adjust the code accordingly. Features of broad interest may be merged in future releases. 272 | 273 | ### Contact and support 274 | Dominik Schumacher (dominik.schumacher@ugent.be) and Jessica Keune (jessica.keune@ugent.be) 275 | 276 | ### Referencing 277 | If you use HAMSTER, please cite: 278 | Keune, J., Schumacher, D. L., & Miralles, D. G. (2022). A unified framework to estimate the origins of atmospheric moisture and heat using Lagrangian models. Geoscientific Model Development, 15, 1875–1898. https://doi.org/10.5194/gmd-15-1875-2022 279 | 280 | ### License 281 | Copyright 2021 Dominik Schumacher, Jessica Keune, Diego G. Miralles. 282 | 283 | This software is published under the GPLv3 license. This means: 284 | 1. Anyone can copy, modify and distribute this software. 285 | 2. You have to include the license and copyright notice with each and every distribution. 286 | 3. You can use this software privately. 287 | 4. You can use this software for commercial purposes. 288 | 5. If you dare build your business solely from this code, you risk open-sourcing the whole code base. 289 | 6. If you modify it, you have to indicate changes made to the code. 290 | 7. Any modifications of this code base MUST be distributed with the same license, GPLv3. 291 | 8. This software is provided without warranty. 292 | 9. The software author or license can not be held liable for any damages inflicted by the software. 293 | 294 | ## References 295 | - Fremme, A. and Sodemann, H.: The role of land and ocean evaporation on the variability of precipitation in the Yangtze River valley, Hydrol. Earth Syst. Sci., 23, 2525–2540, https://doi.org/10.5194/hess-23-2525-2019, 2019. 296 | - Keune, J., and Miralles, D. G.: A precipitation recycling network to assess freshwater vulnerability: Challenging the watershed convention, Water Resour. Res., 55, 9947– 9961, https://doi.org/10.1029/2019WR025310, 2019. 297 | - Schumacher, D.L., Keune, J., van Heerwaarden, C.C. et al.: Amplification of mega-heatwaves through heat torrents fuelled by upwind drought, Nat. Geosci. 12, 712–717, https://doi.org/10.1038/s41561-019-0431-6, 2019. 298 | - Schumacher, D.L., Keune, J. and Miralles, D.G.: Atmospheric heat and moisture transport to energy- and water-limited ecosystems. Ann. N.Y. Acad. Sci., 1472, 123-138, https://doi.org/10.1111/nyas.14357, 2020. 299 | - Sodemann, H., Schwierz, C., and Wernli, H.: Interannual variability of Greenland winter precipitation sources: Lagrangian moisture diagnostic and North Atlantic Oscillation influence. J. Geophys. Res. Atmos., 113(D3), https://doi.org/10.1029/2007JD008503, 2008. 300 | - Sodemann, H.: Beyond Turnover Time: Constraining the Lifetime Distribution of Water Vapor from Simple and Complex Approaches, J. Atmos. Sci., 77(2), 413-433, https://doi.org/10.1175/JAS-D-18-0336.1, 2020. 301 | 302 | -------------------------------------------------------------------------------- /src/biascorrection.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Main script to bias-correct source-receptor relationships established with HAMSTER; 5 | # e.g. using output from FLEXPART. Uses output from the (i) diagnosis and (ii) attribution 6 | # steps from HAMSTER. 7 | # 8 | # This file is part of HAMSTER, 9 | # originally created by Dominik Schumacher, Jessica Keune, Diego G. Miralles 10 | # at the Hydro-Climate Extremes Lab, Department of Environment, Ghent University 11 | # 12 | # https://github.com/h-cel/hamster 13 | # 14 | # HAMSTER is free software: you can redistribute it and/or modify 15 | # it under the terms of the GNU General Public License as published by 16 | # the Free Software Foundation v3. 17 | # 18 | # HAMSTER is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 | # GNU General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU General Public License 24 | # along with HAMSTER. If not, see . 25 | # 26 | 27 | import argparse 28 | import calendar 29 | import csv 30 | import fnmatch 31 | import gzip 32 | import imp 33 | import math 34 | import os 35 | import random 36 | import re 37 | import struct 38 | import sys 39 | import time 40 | import timeit 41 | import warnings 42 | from datetime import date, datetime, timedelta 43 | import datetime as datetime 44 | from functools import reduce 45 | from math import acos, atan, atan2, cos, floor, sin, sqrt 46 | 47 | import h5py 48 | import netCDF4 as nc4 49 | import numpy as np 50 | import pandas as pd 51 | from dateutil.relativedelta import relativedelta 52 | 53 | from hamsterfunctions import * 54 | 55 | 56 | def main_biascorrection( 57 | ryyyy, 58 | ayyyy, 59 | am, 60 | opath_attr, # attribution (output) 61 | opath_diag, # diagnosis (output) 62 | ipath_refp, 63 | ipath_refe, 64 | ipath_reft, 65 | ipath_refh, 66 | opath, 67 | ofile_base, # output 68 | mode, 69 | maskfile, 70 | maskval, 71 | verbose, 72 | veryverbose, 73 | fuseattp, 74 | bcscale, 75 | pref_data, 76 | eref_data, 77 | href_data, 78 | faggbwtime, 79 | fbc_e2p, 80 | fbc_e2p_p, 81 | fbc_e2p_e, 82 | fbc_e2p_ep, 83 | fbc_t2p_ep, 84 | fbc_had, 85 | fbc_had_h, 86 | fdebug, 87 | fwrite_netcdf, 88 | fwrite_month, 89 | fwritestats, 90 | precision, 91 | strargs, 92 | ): 93 | 94 | ## SOME PRELIMINARY SETTINGS TO REDUCE OUTPUT 95 | ## suppressing warnings, such as 96 | # invalid value encountered in true_divide 97 | # invalid value encountered in multiply 98 | if not fdebug: 99 | np.seterr(divide="ignore", invalid="ignore") 100 | # default values 101 | fwritewarning = False 102 | 103 | ## construct precise input and storage paths 104 | attrfile = ( 105 | opath_attr 106 | + "/" 107 | + str(ofile_base) 108 | + "_attr_r" 109 | + str(ryyyy)[-2:] 110 | + "_" 111 | + str(ayyyy) 112 | + "-" 113 | + str(am).zfill(2) 114 | + ".nc" 115 | ) 116 | ofilename = ( 117 | str(ofile_base) 118 | + "_biascor-attr_r" 119 | + str(ryyyy)[-2:] 120 | + "_" 121 | + str(ayyyy) 122 | + "-" 123 | + str(am).zfill(2) 124 | + ".nc" 125 | ) 126 | ofile = opath + "/" + ofilename 127 | ## additional statistic output files includes P validation data (*.csv) 128 | if fwritestats: 129 | sfilename = ( 130 | str(ofile_base) 131 | + "_biascor-attr_r" 132 | + str(ryyyy)[-2:] 133 | + "_" 134 | + str(ayyyy) 135 | + "-" 136 | + str(am).zfill(2) 137 | + "_stats.csv" 138 | ) 139 | sfile = opath + "/" + sfilename 140 | 141 | #### DISCLAIMER 142 | if verbose: 143 | disclaimer() 144 | print("\n PROCESSING: \t", ayyyy, "-", str(am).zfill(2) + "\n") 145 | ## Resets & consistency checks 146 | if mode == "oper" and precision == "f4": 147 | precision = "f8" 148 | print( 149 | " ! Single precision should only be used for testing. Reset to double-precision." 150 | ) 151 | if verbose: 152 | print(" ! using input paths: \t") 153 | print("\t" + str(opath_diag)) 154 | print("\t" + str(opath_attr)) 155 | print(" ! using reference data from: \t") 156 | print("\t" + str(ipath_refp)) 157 | print("\t" + str(ipath_refe)) 158 | print("\t" + str(ipath_refh)) 159 | if fbc_t2p_ep: 160 | print("\t" + str(ipath_reft)) 161 | print(" ! using mode: \t" + str(mode)) 162 | print(" ! using attribution data to bias-correct P: \t" + str(fuseattp)) 163 | print(" ! writing netcdf output: \t") 164 | print("\t" + str(ofile)) 165 | if fwritestats: 166 | print(" ! precipitation statistics in: \t") 167 | print("\t" + str(sfile)) 168 | print( 169 | "\n============================================================================================================" 170 | ) 171 | print("\n") 172 | 173 | ##--1. load attribution data; grab all uptake days ############################ 174 | if verbose: 175 | print(" * Reading attribution data...") 176 | if veryverbose: 177 | print(" --- file: " + str(attrfile)) 178 | 179 | with nc4.Dataset(attrfile, mode="r") as f: 180 | e2psrt = np.asarray(checknan(f["E2P"][:])) 181 | hadsrt = np.asarray(checknan(f["Had"][:])) 182 | arrival_time = nc4.num2date(f["time"][:], f["time"].units, f["time"].calendar) 183 | utime_srt = np.asarray(f["level"][:]) 184 | uptake_time = udays2udate(arrival_time, utime_srt) 185 | uptake_dates = cal2date(uptake_time) 186 | uyears = np.unique(date2year(uptake_time)) 187 | lats = np.asarray(f["lat"][:]) 188 | lons = np.asarray(f["lon"][:]) 189 | areas = ( 190 | 1e6 191 | * np.nan_to_num( 192 | gridded_area_exact(lats, res=abs(lats[1] - lats[0]), nlon=lons.size) 193 | )[:, 0] 194 | ) 195 | # expand uptake dimension to dates (instead of backward days) 196 | e2p = expand4Darray(e2psrt, arrival_time, utime_srt, veryverbose) 197 | had = expand4Darray(hadsrt, arrival_time, utime_srt, veryverbose) 198 | # convert water fluxes from mm-->m3 199 | e2p = convert_mm_m3(e2p, areas) 200 | 201 | # clean up 202 | del (e2psrt, hadsrt) 203 | 204 | ##--2. load diagnosis data #################################################### 205 | if verbose: 206 | print(" * Reading diagnosis data...") 207 | 208 | # read concatenated data 209 | totlats, totlons = read_diagdata( 210 | opath_diag, ofile_base, ryyyy, uptake_time, var="grid" 211 | ) 212 | gridcheck(lats, totlats, lons, totlons) 213 | ftime = read_diagdata(opath_diag, ofile_base, ryyyy, uptake_time, var="time") 214 | fdays = np.unique(cal2date(ftime)) 215 | E = read_diagdata(opath_diag, ofile_base, ryyyy, uptake_time, var="E") 216 | P = -read_diagdata(opath_diag, ofile_base, ryyyy, uptake_time, var="P") 217 | H = read_diagdata(opath_diag, ofile_base, ryyyy, uptake_time, var="H") 218 | # convert water fluxes from mm-->m3 to avoid area weighting in between 219 | E = convert_mm_m3(E, areas) 220 | P = convert_mm_m3(P, areas) 221 | 222 | # make sure we use daily aggregates 223 | if fdays.size != ftime.size: 224 | e_tot = convert2daily(E, ftime, fagg="sum") 225 | p_tot = convert2daily(P, ftime, fagg="sum") 226 | h_tot = convert2daily(H, ftime, fagg="mean") 227 | else: 228 | e_tot = E 229 | p_tot = P 230 | h_tot = H 231 | 232 | ## only keep what is really needed (P is stored analogous to E and H for consistency) 233 | datecheck(uptake_dates[0], fdays) 234 | ibgn = np.where(fdays == uptake_dates[0])[0][0] 235 | iend = np.where(fdays == uptake_dates[-1])[0][0] 236 | e_tot = e_tot[ibgn : iend + 1, :, :] 237 | p_tot = p_tot[ibgn : iend + 1, :, :] 238 | h_tot = h_tot[ibgn : iend + 1, :, :] 239 | fdates = fdays[ibgn : iend + 1] 240 | ## make sure we grabbed the right data 241 | if not np.array_equal(uptake_dates, fdates): 242 | raise SystemExit("---- hold your horses; datetime matching failed!") 243 | 244 | ## clean up 245 | del (E, P, H) 246 | 247 | ##--3. load reference data #################################################### 248 | if verbose: 249 | print(" * Reading reference data...") 250 | 251 | if eref_data == "eraint": 252 | e_ref, reflats, reflons = eraloader_12hourly( 253 | var="e", 254 | datapath=ipath_refe + "/E_1deg_", 255 | maskpos=True, 256 | maskneg=False, 257 | uptake_years=uyears, 258 | uptake_dates=uptake_dates, 259 | lats=lats, 260 | lons=lons, 261 | ) 262 | elif eref_data == "others": 263 | # attention: data has to be on the correct grid and daily (or subdaily that can be summed up) and with the correct sign (all positive) 264 | e_ref, reflats, reflons = get_reference_data( 265 | ipath_refe, "evaporation", uptake_dates 266 | ) 267 | gridcheck(totlats, reflats, totlons, reflons) 268 | 269 | # convert water fluxes from mm-->m3 to avoid area weighting in between 270 | e_ref = convert_mm_m3(e_ref, areas) 271 | 272 | if href_data == "eraint": 273 | h_ref, reflats, reflons = eraloader_12hourly( 274 | var="sshf", 275 | datapath=ipath_refh + "/H_1deg_", 276 | maskpos=True, 277 | maskneg=False, 278 | uptake_years=uyears, 279 | uptake_dates=uptake_dates, 280 | lats=lats, 281 | lons=lons, 282 | ) 283 | elif href_data == "others": 284 | # attention: data has to be on the correct grid and daily (or subdaily that can be summed up) and with the correct sign (all positive) 285 | h_ref, reflats, reflons = get_reference_data( 286 | ipath_refh, "sensible heat flux", uptake_dates 287 | ) 288 | gridcheck(totlats, reflats, totlons, reflons) 289 | 290 | if pref_data == "eraint": 291 | p_ref, reflats, reflons = eraloader_12hourly( 292 | var="tp", 293 | datapath=ipath_refp + "/P_1deg_", 294 | maskpos=False, # do NOT set this to True! 295 | maskneg=True, 296 | uptake_years=uyears, 297 | uptake_dates=uptake_dates, 298 | lats=lats, 299 | lons=lons, 300 | ) 301 | elif pref_data == "others": 302 | # attention: data has to be on the correct grid and daily (or subdaily that can be summed up) and with the correct sign (all positive) 303 | p_ref, reflats, reflons = get_reference_data( 304 | ipath_refp, "precipitation", uptake_dates 305 | ) 306 | gridcheck(totlats, reflats, totlons, reflons) 307 | 308 | # convert water fluxes from mm-->m3 to avoid area weighting in between 309 | p_ref = convert_mm_m3(p_ref, areas) 310 | 311 | if fbc_t2p_ep: 312 | print("Reading T") 313 | # attention: data has to be on the correct grid and daily (or subdaily that can be summed up) and with the correct sign (all positive) 314 | t_ref, reflats, reflons = get_reference_data( 315 | ipath_reft, "transpiration", uptake_dates 316 | ) 317 | gridcheck(totlats, reflats, totlons, reflons) 318 | # convert water fluxes from mm-->m3 to avoid area weighting in between 319 | t_ref = convert_mm_m3(t_ref, areas) 320 | 321 | # calculate T/E 322 | t_over_e = t_ref / e_ref 323 | # requires adjustments... 324 | t_over_e[t_over_e > 1] = 1 325 | t_over_e[t_over_e == "inf"] = 1 326 | t_over_e[np.isnan(t_over_e)] = 0 327 | 328 | ##--4. biascorrection ######################################################### 329 | if verbose: 330 | print(" * Starting bias correction...") 331 | 332 | ## P-scaling requires arrival region mask 333 | mask, mlat, mlon = maskgrabber(maskfile) 334 | # currently, only identical grids (mask = attribution = reference data) are supported... 335 | gridcheck(mlat, totlats, mlon, totlons) 336 | 337 | xla, xlo = np.where(mask == maskval) # P[:,xla,xlo] is merely a 2D array... ;) 338 | ibgn = np.where(uptake_time == arrival_time[0])[0][0] # only arrival days! 339 | 340 | ## preliminary checks 341 | if not fuseattp: 342 | # re-evaluate precip. data to check if it can be used (need daily data here because of upscaling in 02) 343 | fuseattp = check_attributedp( 344 | pdiag=p_tot[ibgn:, xla, xlo], pattr=e2p, veryverbose=veryverbose 345 | ) 346 | 347 | # ****************************************************************************** 348 | ## (i) BIAS CORRECTING THE SOURCE 349 | # ****************************************************************************** 350 | if verbose: 351 | print(" --- Bias correction using source data...") 352 | # quick consistency check 353 | consistencycheck(had, h_tot, bcscale, fdebug) 354 | consistencycheck(e2p, e_tot, bcscale, fdebug) 355 | # calculate bias correction factor 356 | alpha_h = calc_sourcebcf(ref=h_ref, diag=h_tot, tscale=bcscale) 357 | alpha_e = calc_sourcebcf(ref=e_ref, diag=e_tot, tscale=bcscale) 358 | # apply bias correction factor 359 | had_hcorrtd = np.multiply(alpha_h, had) 360 | e2p_ecorrtd = np.multiply(alpha_e, e2p) 361 | 362 | # ****************************************************************************** 363 | ## (ii) BIAS CORRECTING THE SINK (P only) 364 | # ****************************************************************************** 365 | if verbose: 366 | print(" --- Bias correction using sink data...") 367 | # calculate (daily) bias correction factor 368 | if fuseattp: 369 | alpha_p = calc_sinkbcf(ref=p_ref[ibgn:, xla, xlo], att=e2p, tscale=bcscale) 370 | # perform monthly bias correction if necessary 371 | if np.all(np.nan_to_num(alpha_p) == 0): 372 | print( 373 | " * Monthly bias correction needed to match reference precipitation..." 374 | ) 375 | alpha_p = calc_sinkbcf( 376 | ref=p_ref[ibgn:, xla, xlo], att=e2p, tscale="monthly" 377 | ) 378 | fwritewarning = True 379 | else: 380 | alpha_p = calc_sinkbcf( 381 | ref=p_ref[ibgn:, xla, xlo], att=p_tot[ibgn:, xla, xlo], tscale=bcscale 382 | ) 383 | # perform monthly bias correction if necessary 384 | if np.all(np.nan_to_num(alpha_p) == 0): 385 | print( 386 | " * Monthly bias correction needed to match reference precipitation..." 387 | ) 388 | alpha_p = calc_sinkbcf( 389 | ref=p_ref[ibgn:, xla, xlo], att=p_tot[ibgn:, xla, xlo], tscale="monthly" 390 | ) 391 | fwritewarning = True 392 | # apply bias correction factor 393 | e2p_pcorrtd = np.swapaxes(alpha_p * np.swapaxes(e2p, 0, 3), 0, 3) 394 | 395 | # additionally perform monthly bias correction of P if necessary 396 | if not checkpsum(p_ref[ibgn:, xla, xlo], e2p_pcorrtd, verbose=False): 397 | print( 398 | " * Additional monthly bias correction needed to match reference precipitation..." 399 | ) 400 | alpha_p = calc_sinkbcf( 401 | ref=p_ref[ibgn:, xla, xlo], att=e2p_pcorrtd, tscale="monthly" 402 | ) 403 | e2p_pcorrtd = np.swapaxes(alpha_p * np.swapaxes(e2p_pcorrtd, 0, 3), 0, 3) 404 | fwritewarning = True 405 | checkpsum(p_ref[ibgn:, xla, xlo], e2p_pcorrtd, verbose=verbose) 406 | 407 | # ****************************************************************************** 408 | ## (iii) BIAS CORRECTING THE SOURCE AND THE SINK (P only) 409 | # ****************************************************************************** 410 | if verbose: 411 | print(" --- Bias correction using source and sink data...") 412 | # step 1: check how much e2p changed due to source-correction already 413 | alpha_p_ecor = calc_sinkbcf(ref=e2p_ecorrtd, att=e2p, tscale=bcscale) 414 | # step 2: calculate how much more correction is needed to match sink 415 | alpha_p = calc_sinkbcf(ref=p_ref[ibgn:, xla, xlo], att=e2p_pcorrtd, tscale=bcscale) 416 | # perform monthly bias correction if necessary 417 | if np.all(np.nan_to_num(alpha_p) == 0): 418 | print( 419 | " * Monthly bias correction needed to match reference precipitation..." 420 | ) 421 | alpha_p = calc_sinkbcf( 422 | ref=p_ref[ibgn:, xla, xlo], att=e2p_pcorrtd, tscale="monthly" 423 | ) 424 | fwritewarning = True 425 | # step 3: calculate adjusted bias correction factor 426 | alpha_p_res = np.divide(alpha_p, alpha_p_ecor) 427 | e2p_epcorrtd = np.swapaxes(alpha_p_res * np.swapaxes(e2p_ecorrtd, 0, 3), 0, 3) 428 | 429 | # additionally perform monthly bias correction of P if necessary 430 | if not checkpsum(p_ref[ibgn:, xla, xlo], e2p_epcorrtd, verbose=False): 431 | print( 432 | " * Additional monthly bias correction needed to match reference precipitation..." 433 | ) 434 | alpha_p_res = calc_sinkbcf( 435 | ref=p_ref[ibgn:, xla, xlo], att=e2p_epcorrtd, tscale="monthly" 436 | ) 437 | e2p_epcorrtd = np.swapaxes(alpha_p_res * np.swapaxes(e2p_epcorrtd, 0, 3), 0, 3) 438 | fwritewarning = True 439 | checkpsum(p_ref[ibgn:, xla, xlo], e2p_epcorrtd, verbose=verbose) 440 | 441 | # save some data in case debugging is needed 442 | if fdebug: 443 | frac_e2p = calc_alpha(e2p, e_tot) 444 | frac_had = calc_alpha(had, h_tot) 445 | 446 | # T2P; transpiration fraction 447 | if fbc_t2p_ep: 448 | t2p_epcorrtd = t_over_e * e2p_epcorrtd 449 | else: 450 | t2p_epcorrtd = np.zeros(shape=e2p_epcorrtd.shape) 451 | 452 | ##--5. aggregate ############################################################## 453 | ## aggregate over uptake time (uptake time dimension is no longer needed!) 454 | ahad = np.nansum(had, axis=1) 455 | ahad_hcorrtd = np.nansum(had_hcorrtd, axis=1) 456 | ae2p = np.nansum(e2p, axis=1) 457 | ae2p_ecorrtd = np.nansum(e2p_ecorrtd, axis=1) 458 | ae2p_pcorrtd = np.nansum(e2p_pcorrtd, axis=1) 459 | ae2p_epcorrtd = np.nansum(e2p_epcorrtd, axis=1) 460 | at2p_epcorrtd = np.nansum(t2p_epcorrtd, axis=1) 461 | # free up memory if backward time not needed anymore... 462 | if faggbwtime: 463 | del ( 464 | had, 465 | had_hcorrtd, 466 | e2p, 467 | e2p_ecorrtd, 468 | e2p_pcorrtd, 469 | e2p_epcorrtd, 470 | t2p_epcorrtd, 471 | ) 472 | 473 | if fwritestats: 474 | # write some additional statistics about P-biascorrection before converting back to mm 475 | writestats_03( 476 | sfile, 477 | p_ref, 478 | ae2p, 479 | ae2p_ecorrtd, 480 | ae2p_pcorrtd, 481 | ae2p_epcorrtd, 482 | ahad, 483 | ahad_hcorrtd, 484 | xla, 485 | xlo, 486 | ibgn, 487 | ) 488 | 489 | ##--6. unit conversion ############################################################## 490 | # and convert water fluxes back from m3 --> mm 491 | if not faggbwtime: 492 | e2p = convert_m3_mm(e2p, areas) 493 | e2p_ecorrtd = convert_m3_mm(e2p_ecorrtd, areas) 494 | e2p_pcorrtd = convert_m3_mm(e2p_pcorrtd, areas) 495 | e2p_epcorrtd = convert_m3_mm(e2p_epcorrtd, areas) 496 | t2p_epcorrtd = convert_m3_mm(t2p_epcorrtd, areas) 497 | if fdebug or faggbwtime: 498 | ae2p = convert_m3_mm(ae2p, areas) 499 | ae2p_ecorrtd = convert_m3_mm(ae2p_ecorrtd, areas) 500 | ae2p_pcorrtd = convert_m3_mm(ae2p_pcorrtd, areas) 501 | ae2p_epcorrtd = convert_m3_mm(ae2p_epcorrtd, areas) 502 | at2p_epcorrtd = convert_m3_mm(at2p_epcorrtd, areas) 503 | 504 | ##--7. debugging needed? ###################################################### 505 | if fdebug: 506 | print(" * Creating debugging file") 507 | writedebugnc( 508 | opath + "/debug.nc", 509 | arrival_time, 510 | uptake_time, 511 | lons, 512 | lats, 513 | maskbymaskval(mask, maskval), 514 | mask3darray(p_ref[ibgn:, :, :], xla, xlo), 515 | mask3darray(p_tot[ibgn:, :, :], xla, xlo), 516 | convert_mm_m3(ae2p, areas), 517 | convert_mm_m3(ae2p_ecorrtd, areas), 518 | convert_mm_m3(ae2p_pcorrtd, areas), 519 | convert_mm_m3(ae2p_epcorrtd, areas), 520 | np.nan_to_num(frac_e2p), 521 | np.nan_to_num(frac_had), 522 | alpha_p, 523 | np.nan_to_num(alpha_p_ecor), 524 | np.nan_to_num(alpha_p_res), 525 | np.nan_to_num(alpha_e), 526 | np.nan_to_num(alpha_h), 527 | strargs, 528 | precision, 529 | ) 530 | 531 | ##--8. write final output ############################################################ 532 | if verbose: 533 | print(" * Writing final output... ") 534 | 535 | if fwrite_netcdf: 536 | # get attributes from attribution file and modify 537 | attrdesc = getattr(nc4.Dataset(attrfile), "description") + "; " + strargs 538 | biasdesc = attrdesc.replace("02_attribution", "03_biascorrection") 539 | 540 | # write to netcdf 541 | if faggbwtime: 542 | writefinalnc( 543 | ofile=ofile, 544 | fdate_seq=arrival_time, 545 | udate_seq=np.nan, 546 | glon=lons, 547 | glat=lats, 548 | Had=ahad, 549 | Had_Hs=ahad_hcorrtd, 550 | E2P=ae2p, 551 | E2P_Es=ae2p_ecorrtd, 552 | E2P_Ps=ae2p_pcorrtd, 553 | E2P_EPs=ae2p_epcorrtd, 554 | T2P_EPs=at2p_epcorrtd, 555 | strargs=biasdesc, 556 | precision=precision, 557 | fwrite_month=fwrite_month, 558 | fbc_had=fbc_had, 559 | fbc_had_h=fbc_had_h, 560 | fbc_e2p=fbc_e2p, 561 | fbc_e2p_p=fbc_e2p_p, 562 | fbc_e2p_e=fbc_e2p_e, 563 | fbc_e2p_ep=fbc_e2p_ep, 564 | fbc_t2p_ep=fbc_t2p_ep, 565 | ) 566 | if not faggbwtime: 567 | writefinalnc( 568 | ofile=ofile, 569 | fdate_seq=arrival_time, 570 | udate_seq=utime_srt, 571 | glon=lons, 572 | glat=lats, 573 | Had=reduce4Darray(had, veryverbose), 574 | Had_Hs=reduce4Darray(had_hcorrtd, veryverbose), 575 | E2P=reduce4Darray(e2p, veryverbose), 576 | E2P_Es=reduce4Darray(e2p_ecorrtd, veryverbose), 577 | E2P_Ps=reduce4Darray(e2p_pcorrtd, veryverbose), 578 | E2P_EPs=reduce4Darray(e2p_epcorrtd, veryverbose), 579 | T2P_EPs=reduce4Darray(t2p_epcorrtd, veryverbose), 580 | strargs=biasdesc, 581 | precision=precision, 582 | fwrite_month=fwrite_month, 583 | fbc_had=fbc_had, 584 | fbc_had_h=fbc_had_h, 585 | fbc_e2p=fbc_e2p, 586 | fbc_e2p_p=fbc_e2p_p, 587 | fbc_e2p_e=fbc_e2p_e, 588 | fbc_e2p_ep=fbc_e2p_ep, 589 | fbc_t2p_ep=fbc_t2p_ep, 590 | ) 591 | if fwritewarning: 592 | wfile = ( 593 | opath 594 | + "/" 595 | + str(ofile_base) 596 | + "_biascor-attr_r" 597 | + str(ryyyy)[-2:] 598 | + "_" 599 | + str(ayyyy) 600 | + "-" 601 | + str(am).zfill(2) 602 | + "_WARNING.csv" 603 | ) 604 | writewarning(wfile) 605 | 606 | if os.path.exists(ofile): 607 | print("Removing " + str(attrfile) + " ...") 608 | os.remove(attrfile) 609 | -------------------------------------------------------------------------------- /src/attribution.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Main script to establish source-receptor relationships based on backward trajectories, 5 | # from e.g. FLEXPART. Two source regions are established: moisture and heat. 6 | # 7 | # This file is part of HAMSTER, 8 | # originally created by Dominik Schumacher, Jessica Keune, Diego G. Miralles 9 | # at the Hydro-Climate Extremes Lab, Department of Environment, Ghent University 10 | # 11 | # https://github.com/h-cel/hamster 12 | # 13 | # HAMSTER is free software: you can redistribute it and/or modify 14 | # it under the terms of the GNU General Public License as published by 15 | # the Free Software Foundation v3. 16 | # 17 | # HAMSTER is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20 | # GNU General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU General Public License 23 | # along with HAMSTER. If not, see . 24 | # 25 | 26 | import argparse 27 | import calendar 28 | import csv 29 | import fnmatch 30 | import gzip 31 | import imp 32 | import math 33 | import os 34 | import random 35 | import re 36 | import struct 37 | import sys 38 | import time 39 | import timeit 40 | import warnings 41 | from datetime import date, datetime, timedelta 42 | import datetime as datetime 43 | from functools import reduce 44 | from math import acos, atan, atan2, cos, floor, sin, sqrt 45 | 46 | import h5py 47 | import netCDF4 as nc4 48 | import numpy as np 49 | import pandas as pd 50 | from dateutil.relativedelta import relativedelta 51 | 52 | from hamsterfunctions import * 53 | 54 | 55 | def main_attribution( 56 | ryyyy, 57 | ayyyy, 58 | am, 59 | ad, 60 | ipath, 61 | ifile_base, 62 | ipath_f2t, 63 | opath, 64 | ofile_base, 65 | mode, 66 | gres, 67 | maskfile, 68 | maskval, 69 | verbose, 70 | veryverbose, 71 | ctraj_len, 72 | # E criteria 73 | cevap_dqv, 74 | fevap_drh, 75 | cevap_drh, 76 | cevap_hgt, 77 | # P criteria 78 | cprec_dqv, 79 | cprec_rh, 80 | # H criteria 81 | cheat_dtemp, 82 | fheat_drh, 83 | cheat_drh, 84 | cheat_hgt, 85 | fheat_rdq, 86 | cheat_rdq, 87 | # pbl and height criteria 88 | cpbl_method, 89 | cpbl_strict, 90 | cpbl_factor, 91 | refdate, 92 | fwrite_netcdf, 93 | precision, 94 | ftimethis, 95 | fdry, 96 | fmemento, 97 | mattribution, 98 | crandomnit, 99 | randatt_forcall, 100 | randatt_wloc, 101 | explainp, 102 | fdupscale, 103 | fmupscale, 104 | fvariable_mass, 105 | fwritestats, 106 | strargs, 107 | ): 108 | 109 | # TODO: add missing features 110 | if fvariable_mass: 111 | raise SystemExit("---- ABORTED: no can do, not implemented!") 112 | 113 | #### INPUT PATHS (incl. year) 114 | ipath_pp = os.path.join(ipath_f2t, str(ryyyy)) # raw partposit data 115 | ipath_tr = os.path.join(ipath, str(ryyyy)) # trajectory data (h5 files) 116 | 117 | #### OUTPUT FILES 118 | ## main netcdf output 119 | ofilename = ( 120 | str(ofile_base) 121 | + "_attr_r" 122 | + str(ryyyy)[-2:] 123 | + "_" 124 | + str(ayyyy) 125 | + "-" 126 | + str(am).zfill(2) 127 | + ".nc" 128 | ) 129 | ofile = opath + "/" + ofilename 130 | ## additional statistic output files (*.csv) 131 | # monthly statistics 132 | sfilename = ( 133 | str(ofile_base) 134 | + "_attr_r" 135 | + str(ryyyy)[-2:] 136 | + "_" 137 | + str(ayyyy) 138 | + "-" 139 | + str(am).zfill(2) 140 | + "_stats.csv" 141 | ) 142 | statfile = opath + "/" + sfilename 143 | # trajectory-based precipitation statistics 144 | if fwritestats and mattribution == "linear": 145 | pfilename = ( 146 | str(ofile_base) 147 | + "_attr_r" 148 | + str(ryyyy)[-2:] 149 | + "_" 150 | + str(ayyyy) 151 | + "-" 152 | + str(am).zfill(2) 153 | + "_p-linear-attribution.csv" 154 | ) 155 | pattfile = opath + "/" + pfilename 156 | with open(pattfile, "w") as pfile: 157 | writer = csv.writer(pfile, delimiter="\t", lineterminator="\n",) 158 | writer.writerow(["DATE", "F_ATT", "F_POT", "P_DQDT"]) 159 | 160 | #### INPUT FILES 161 | ## read netcdf mask 162 | mask, mlat, mlon = maskgrabber(maskfile) 163 | 164 | #### DISCLAIMER 165 | if verbose: 166 | disclaimer() 167 | print("\n PROCESSING: \t", ayyyy, "-", str(am).zfill(2)) 168 | print( 169 | "\n============================================================================================================\n" 170 | ) 171 | ## Resets & consistency checks 172 | if fwritestats and mattribution == "random": 173 | print( 174 | " ! Option not yet available for attribution method random. Continuing anyhow..." 175 | ) 176 | if mode == "oper" and precision == "f4": 177 | precision = "f8" 178 | print( 179 | " ! Single precision should only be used for testing. Reset to double-precision." 180 | ) 181 | if verbose: 182 | print(" ! using raw partposit input path: \t", ipath_pp) 183 | print(" ! using trajectory data input path: \t", ipath_tr) 184 | print(" ! using internal timer: \t" + str(ftimethis)) 185 | print(" ! using mode: \t" + str(mode)) 186 | print(" ! using attribution method (P): \t" + str(mattribution)) 187 | print(" ! using trajectory-based upscaling: \t" + str(explainp)) 188 | if mattribution == "random": 189 | print(" ! using minimum iterations: \t" + str(crandomnit)) 190 | print(" ! using daily upscaling: \t" + str(fdupscale)) 191 | print(" ! using monthly upscaling: \t" + str(fmupscale)) 192 | if fwrite_netcdf: 193 | print(" ! writing netcdf output: \t" + str(fwrite_netcdf)) 194 | print(" \t ! with grid resolution: \t", str(gres)) 195 | print(" \t ! output file: \t", opath + "/" + ofilename) 196 | print(" ! additional statistics in: \t" + str(statfile)) 197 | if fwritestats and mattribution == "linear": 198 | print(" ! precipitation statistics in: \t" + str(pattfile)) 199 | print( 200 | "\n============================================================================================================" 201 | ) 202 | print( 203 | "\n============================================================================================================" 204 | ) 205 | 206 | #### SETTINGS 207 | ## start timer 208 | if ftimethis: 209 | megatic = timeit.default_timer() 210 | 211 | ## grids 212 | glon, glat, garea = makegrid(resolution=gres) 213 | gridcheck(glat, mlat, glon, mlon) 214 | 215 | ## -- DATES 216 | dt = 6 # hardcoded for FLEXPART ERA-INTERIM with 6h 217 | timestep = datetime.timedelta(hours=dt) 218 | 219 | # get start date (to read trajectories) - NOTE: we begin at 06 UTC... 220 | sdate_bgn = ( 221 | str(ayyyy) 222 | + "-" 223 | + str(am).zfill(2) 224 | + "-" 225 | + str(ad).zfill(2) 226 | + "-" 227 | + str(dt).zfill(2) 228 | ) 229 | datetime_bgn = datetime.datetime.strptime(sdate_bgn, "%Y-%m-%d-%H") 230 | # get end date (to read trajectories) - NOTE: always 00 UTC of the 1st of the next month 231 | nayyyy, nam = nextmonth(datetime_bgn) 232 | sdate_end = str(nayyyy) + "-" + str(nam).zfill(2) + "-01-00" 233 | datetime_end = datetime.datetime.strptime(sdate_end, "%Y-%m-%d-%H") 234 | 235 | # file dates (arrival, 6h seq) 236 | datetime_seq, fdatetime_seq, ffdatetime_seq = timelord( 237 | datetime_bgn, datetime_end, timestep 238 | ) 239 | # daily arrival dates (24h seq for netCDF writing) 240 | fdate_seq = timelord( 241 | datetime_bgn - timestep, 242 | datetime_end - timestep, 243 | datetime.timedelta(hours=24), 244 | ret="datetime", 245 | ) 246 | fdateasdate = datetime2date(fdate_seq) 247 | 248 | # NOTE: better to keep these as lists to maintain consistency 249 | 250 | # calculate number of time steps, also aggregated to daily resolution 251 | ntime = len(fdatetime_seq) 252 | ndaytime = len(fdate_seq) 253 | 254 | ## TESTMODE 255 | if mode == "test": 256 | 257 | ctraj_len_orig = ctraj_len 258 | ctraj_len = ( 259 | 2 # NOTE: changes here must be accompanied by changes in 01_diagnosis! 260 | ) 261 | 262 | ntime = 4 # NOTE: use multiples of 4 only, else output is not saved 263 | datetime_seq = datetime_seq[(4 * ctraj_len) : (4 * ctraj_len + ntime)] 264 | fdatetime_seq = fdatetime_seq[(4 * ctraj_len) : (4 * ctraj_len + ntime)] 265 | ndaytime = int(ntime / 4) 266 | fdate_seq = fdate_seq[ctraj_len : ctraj_len + ndaytime] 267 | fdateasdate = fdateasdate[ctraj_len : ctraj_len + ndaytime] 268 | 269 | ## -- WRITE NETCDF OUTPUT (empty, to be filled) 270 | if fwrite_netcdf: 271 | writeemptync4D( 272 | ofile, fdate_seq, np.arange(-ctraj_len, 1), glat, glon, strargs, precision 273 | ) 274 | 275 | # traj max len, expressed in input data (6-hourly) steps 276 | tml = int(4 * ctraj_len) # hardcoded for 6-hourly input 277 | # compact form of max traj len in days (used for array filling w/ shortened uptake dim) 278 | ctl = ctraj_len 279 | 280 | ### MEMENTO --- pre-loop to produce independent monthly output 281 | ## NOTE: this is irrelevant for E2P, but crucial for Had (& Ead) 282 | ## NOTE: we only need to know if some parcel makes it to the ABL, that's it! 283 | ## NOTE: must fill array with negative number whose abs exceeds max traj len 284 | if fmemento: 285 | pidlog = -999 * np.ones(shape=2100000).astype(int) 286 | 287 | if mode == "oper": # skip if multi-counting somehow desired and/or if testing 288 | ###--- PRELOOP v2 289 | # NOTE: code further below this function call here could also be moved to 290 | # first main loop iteration, so that no dim checking necessary 291 | ntrajstep = readtraj( 292 | idate=datetime_seq[0], 293 | ipath=ipath_tr, 294 | ifile_base=ifile_base, 295 | verbose=False, 296 | ).shape[0] 297 | 298 | # I believe this could be done for parcels of interest / pot. conflict only... but we'll leave it like this for now 299 | if ntrajstep < tml + 2 + 4: 300 | # only do this if data really isn't already 'there' 301 | preloop_dates = timelord( 302 | fdatetime_seq[0] - (tml + 5) * timestep, 303 | fdatetime_seq[0] - timestep, 304 | timestep, 305 | ret="fileformat", 306 | ) 307 | preloop_files = [ 308 | ipath_pp + "/partposit_" + idfile + ".gz" 309 | for idfile in preloop_dates 310 | ] 311 | extendarchive = grabmesomehpbl(filelist=preloop_files, verbose=verbose) 312 | else: 313 | if verbose: 314 | print( 315 | "\n=== \t INFO: no pre-loop needed, trajectories are long enough" 316 | ) 317 | 318 | ###--- MAIN LOOP 319 | 320 | ## prepare STATS 321 | # number of parcels 322 | tneval = tnnevala = tnnevalm = tnevalp = tnnevalp = tnevalh = tnnevalh = 0 323 | # precip. statistics 324 | psum = patt = punatt = pmiss = 0 325 | 326 | ## loop over time to read in files 327 | if verbose: 328 | print("\n=== \t Start main program...\n") 329 | for ix in range(ntime): 330 | if verbose: 331 | print( 332 | "--------------------------------------------------------------------------------------" 333 | ) 334 | print("Processing " + str(fdatetime_seq[ix])) 335 | 336 | # safety exit hardcoded for FLEXPART–ERA-Interim runs 337 | if ryyyy == ayyyy and am == 12 and ix == range(ntime)[-1]: 338 | continue 339 | 340 | ## 1) read in all files associated with data --> ary is of dimension (ntrajlen x nparcels x nvars) 341 | ary = readtraj( 342 | idate=datetime_seq[ix], 343 | ipath=ipath_tr, 344 | ifile_base=ifile_base, 345 | verbose=verbose, 346 | ) 347 | 348 | nparcel = ary.shape[1] 349 | ntrajleng = ary.shape[0] 350 | if verbose: 351 | print( 352 | " TOTAL: " + str(datetime_seq[ix]) + " has " + str(nparcel) + " parcels" 353 | ) 354 | 355 | if mode == "test": 356 | ntot = range(1000) 357 | else: 358 | ntot = range(nparcel) 359 | 360 | # figure out where to store data (on which arriving day) 361 | arv_idx = np.where( 362 | np.asarray(fdateasdate) 363 | == (fdatetime_seq[ix] - relativedelta(hours=3)).date() 364 | )[0][0] 365 | 366 | # pre-allocate arrays (repeat at every 4th step) 367 | if ix % 4 == 0: 368 | ary_heat = np.zeros(shape=(ctl + 1, glat.size, glon.size)) 369 | ary_etop = np.zeros(shape=(ctl + 1, glat.size, glon.size)) 370 | # upscaling measures (currently has to be per day as well) 371 | if fdupscale: 372 | ipatt = ipmiss = 0 373 | 374 | # STATS: number of parcels per file 375 | neval = nnevala = nnevalm = nevalp = nnevalp = nevalh = nnevalh = 0 376 | 377 | # grab extended trajectory data 378 | ### CHECK/CLEANUP: CAN ALL OF THIS GO? 379 | # (i) POM data not supported anymore 380 | # (ii) if trajectories too short: abort (let's not fix everything right away for the user) 381 | if fmemento: 382 | # NOTE: pom data can come with duplicate IDs; remove to avoid (some) trouble 383 | if not np.unique(ary[0, :, 0]).size == ary[0, :, 0].size: 384 | print( 385 | "\t INFO: duplicates detected, original pom array shape=", ary.shape 386 | ) 387 | _, ikeep = np.unique(ary[0, :, 0], return_index=True) 388 | ary = ary[:, ikeep, :] 389 | print("\t INFO: duplicates eliminated, new pom array shape=", ary.shape) 390 | nparcel = ary.shape[1] # update 391 | ntot = range(nparcel) 392 | # NOTE: yet another ******* pom fix ... to be removed! 393 | # (flex2traj output is not affected by this) 394 | IDs = ary[0, :, 0] 395 | thresidx = int((9 / 10) * ary.shape[1]) # this should do the trick 396 | # simply shift to indices > 2e6 397 | IDs[thresidx:][IDs[thresidx:] < 3000] += 2e6 398 | 399 | # extract what is needed from extendarchive if trajs 'too short' 400 | if ary.shape[0] < tml + 2 + 4 and ix < ctraj_len * 4: 401 | extendtrajs = np.empty(shape=(4, nparcel, 2)) 402 | for pp in range(4): 403 | allIDs = extendarchive[-(4 - pp + ix)][:, 0] 404 | extendtrajs[pp, :, 0] = extendarchive[-(4 - pp + ix)][:, 0][ 405 | np.where(np.isin(allIDs, ary[0, :, 0])) 406 | ] # ID 407 | extendtrajs[pp, :, 1] = extendarchive[-(4 - pp + ix)][:, 1][ 408 | np.where(np.isin(allIDs, ary[0, :, 0])) 409 | ] # hpbl 410 | 411 | ## 2) establish source–receptor relationships 412 | for i in ntot: 413 | 414 | ## - 2.0) only evaluate if the parcel is in target region (midpoint or arrival point) 415 | mlat_ind, mlon_ind = midpindex(ary[:2, i, :], glon=mlon, glat=mlat) 416 | alat_ind, alon_ind = arrpindex(ary[0, i, :], glon=mlon, glat=mlat) 417 | if ( 418 | not mask[alat_ind, alon_ind] == maskval 419 | and not mask[mlat_ind, mlon_ind] == maskval 420 | ): 421 | nnevalm += 1 422 | nnevala += 1 423 | continue 424 | 425 | ## - 2.1) check how far back trajectory should be evaluated 426 | if mask[alat_ind, alon_ind] == maskval: 427 | ID = int(ary[0, i, 0]) 428 | if fmemento and ary[0, i, 3] < np.max(ary[:4, i, 7]): 429 | if ix < ctraj_len * 4: # rely on (extended) traj data 430 | 431 | # check if parcel has been inside before 432 | is_inmask = whereinmask( 433 | mask=mask, 434 | maskval=maskval, 435 | masklat=mlat, 436 | masklon=mlon, 437 | trajlat=ary[: (tml + 2), i, 2], 438 | trajlon=ary[: (tml + 2), i, 1], 439 | ) 440 | 441 | ### CHECK/CLEANUP: CAN ALL OF THIS GO? (if only trajs > length allowed)? 442 | # check if parcel was 'in PBL' 443 | hgt = ary[: (tml + 2), i, 3] # consistent with max traj len 444 | if ary.shape[0] < tml + 2 + 4: 445 | longhpbl = np.concatenate( 446 | (ary[: (tml + 2), i, 7], extendtrajs[:, i, 1]) 447 | ) 448 | else: 449 | longhpbl = ary[: (tml + 2 + 4), i, 7] 450 | is_inpbl = np.where(hgt < maxlastn(longhpbl, n=4)[:-4])[ 451 | 0 452 | ] # omit last 4 453 | 454 | # check where parcel inside and 'in PBL' 455 | is_arrv = np.intersect1d(is_inmask, is_inpbl) 456 | 457 | # now determine ihf_H 458 | if is_arrv.size > 1: 459 | ihf_H = is_arrv[is_arrv > 0].min() + 1 460 | elif is_arrv == 0: 461 | ihf_H = tml + 2 # use max traj len 462 | else: 463 | raise RuntimeError( 464 | "--- FATAL ERROR: Schrödingers cat situation; is parcel inside and in PBL, or not?" 465 | ) 466 | 467 | else: # fully rely on log from now 468 | istep = pidlog[ID] 469 | ihf_H = min((ix - istep + 1), tml + 2) 470 | else: 471 | ihf_H = tml + 2 472 | 473 | ## - 2.2) read only the most basic parcel information 474 | # NOTE: this could easily be done more efficiently 475 | hgt, hpbl, temp, qv, dens, pres = glanceparcel(ary[:4, i, :]) 476 | 477 | # sorry, yet another date for writing the P date to the csv (preliminary). 478 | # because i wanted to have the hours in there as wel (not assign to day only) 479 | pdate = str( 480 | (fdatetime_seq[ix] - relativedelta(hours=3)).strftime("%Y%m%d%H") 481 | ) 482 | 483 | ## - 2.3) diagnose fluxes 484 | 485 | ## (a) E2P, evaporation resulting in precipitation 486 | if not mask[mlat_ind, mlon_ind] == maskval: 487 | nnevalm += 1 488 | else: 489 | if (qv[0] - qv[1]) < cprec_dqv and ( 490 | (q2rh(qv[0], pres[0], temp[0]) + q2rh(qv[1], pres[1], temp[1])) / 2 491 | ) > cprec_rh: 492 | 493 | # prec 494 | prec = abs(qv[0] - qv[1]) 495 | # log some statistics 496 | nevalp += 1 497 | psum += prec 498 | 499 | # read full parcel information 500 | ( 501 | lons, 502 | lats, 503 | temp, 504 | hgt, 505 | qv, 506 | hpbl, 507 | dens, 508 | pres, 509 | pottemp, 510 | epottemp, 511 | ) = readparcel(ary[: tml + 2, i, :]) 512 | rh = q2rh(qv, pres, temp) 513 | 514 | # calculate all required changes along trajectory 515 | dq = trajparceldiff(qv[:], "diff") 516 | # evaluate only until trajectory falls dry 517 | ihf_E = tml + 2 518 | if fdry and np.any(qv[1:ihf_E] <= 0.00005): 519 | ihf_E = np.min(np.where(qv[1:ihf_E] <= 0.00005)[0] + 1) 520 | 521 | # identify uptake locations 522 | is_inpbl = pblcheck( 523 | np.dstack((hgt[:ihf_E], hpbl[:ihf_E]))[0, :, :], 524 | cpbl_strict, 525 | minh=cheat_hgt, 526 | fpbl=cpbl_factor, 527 | method=cpbl_method, 528 | ) 529 | is_drh = drhcheck(rh[:ihf_E], checkit=fevap_drh, maxdrh=cevap_drh) 530 | is_uptk = dq[: ihf_E - 1] > cevap_dqv 531 | evap_idx = np.where( 532 | np.logical_and(is_inpbl, np.logical_and(is_drh, is_uptk)) 533 | )[0] 534 | 535 | # log some stats if trajectory is without any uptakes (for upscaling) 536 | if evap_idx.size == 0: 537 | nnevalp += 1 538 | pmiss += prec 539 | if fdupscale: 540 | ipmiss += prec 541 | 542 | # ATTRIBUTION 543 | if evap_idx.size > 0: 544 | if mattribution == "linear": 545 | etop = linear_attribution_p( 546 | qv[:ihf_E], iupt=evap_idx, explainp=explainp 547 | ) 548 | elif mattribution == "random": 549 | etop = random_attribution_p( 550 | qtot=qv[:ihf_E], 551 | iupt=evap_idx, 552 | explainp=explainp, 553 | nmin=crandomnit, 554 | forc_all=randatt_forcall, 555 | weight_locations=randatt_wloc, 556 | verbose=verbose, 557 | veryverbose=veryverbose, 558 | ) 559 | # write to grid 560 | for itj in evap_idx: 561 | ary_etop[ctl - (itj + 3 - ix % 4) // 4, :, :] += gridder( 562 | plon=lons[itj : itj + 2], 563 | plat=lats[itj : itj + 2], 564 | pval=etop[itj], 565 | glon=glon, 566 | glat=glat, 567 | ) 568 | 569 | # write additional stats to csv-file (currently: ALWAYS explain="none"; also: why only linear?) 570 | if fwritestats and mattribution == "linear": 571 | if evap_idx.size == 0: 572 | pattdata = [pdate, str(0), str(0), str(prec)] 573 | elif evap_idx.size > 0: 574 | etop = linear_attribution_p( 575 | qv[:ihf_E], iupt=evap_idx, explainp="none" 576 | ) 577 | pattdata = [ 578 | pdate, 579 | str(np.sum(etop[evap_idx] / prec)), 580 | str(1 - etop[-1] / prec), 581 | str(prec), 582 | ] 583 | append2csv(pattfile, pattdata) 584 | 585 | # log some statistics (for upscaling) 586 | patt += np.sum(etop[evap_idx]) 587 | punatt += prec - np.sum(etop[evap_idx]) 588 | if fdupscale: 589 | ipatt += np.sum(etop[evap_idx]) 590 | ipmiss += prec - np.sum(etop[evap_idx]) 591 | 592 | ## (b) H, surface sensible heat arriving in PBL (or nocturnal layer) 593 | if not mask[alat_ind, alon_ind] == maskval: 594 | nnevala += 1 595 | else: 596 | if ihf_H >= 2 and hgt[0] < np.max(hpbl[:4]): 597 | 598 | # log some statistics 599 | nevalh += 1 600 | 601 | # read full parcel information 602 | ( 603 | lons, 604 | lats, 605 | temp, 606 | hgt, 607 | qv, 608 | hpbl, 609 | dens, 610 | pres, 611 | pottemp, 612 | epottemp, 613 | ) = readparcel(ary[:ihf_H, i, :]) 614 | rh = q2rh(qv, pres, temp) 615 | 616 | # calculate all required changes along trajectory 617 | dTH = trajparceldiff(pottemp[:], "diff") 618 | 619 | # identify sensible heat uptakes 620 | is_inpbl = pblcheck( 621 | np.dstack((hgt[:ihf_H], hpbl[:ihf_H]))[0, :, :], 622 | cpbl_strict, 623 | minh=cheat_hgt, 624 | fpbl=cpbl_factor, 625 | method=cpbl_method, 626 | ) 627 | is_drh = drhcheck(rh[:ihf_H], checkit=fheat_drh, maxdrh=cheat_drh) 628 | is_rdqv = rdqvcheck( 629 | qv[:ihf_H], checkit=fheat_rdq, maxrdqv=cheat_rdq 630 | ) 631 | is_uptk = dTH[: ihf_H - 1] > cheat_dtemp 632 | heat_idx = np.where( 633 | np.logical_and( 634 | is_inpbl, 635 | np.logical_and(is_drh, np.logical_and(is_rdqv, is_uptk)), 636 | ) 637 | )[0] 638 | 639 | # discount uptakes linearly 640 | if heat_idx.size == 0: 641 | # log some statistics 642 | nnevalh += 1 643 | if heat_idx.size > 0: 644 | dTH_disc = linear_discounter(v=pottemp[:ihf_H], min_gain=0) 645 | 646 | # loop through sensible heat uptakes 647 | for itj in heat_idx: 648 | # NOTE: hardcoded for writing daily data 649 | ary_heat[ctl - (itj + 3 - ix % 4) // 4, :, :] += ( 650 | gridder( 651 | plon=lons[itj : itj + 2], 652 | plat=lats[itj : itj + 2], 653 | pval=dTH_disc[itj], 654 | glon=glon, 655 | glat=glat, 656 | ) 657 | / 4 658 | ) 659 | 660 | # update parcel log 661 | if fmemento: 662 | pidlog[ID] = ix # NOTE: double-check 663 | 664 | neval = len(ntot) 665 | if verbose: 666 | print( 667 | " STATS: Evaluated " 668 | + str(neval - nnevala) 669 | + " ({:.2f}".format(100 * (neval - nnevala) / (neval)) 670 | + "%) arriving parcels inside mask (advection)." 671 | ) 672 | if nnevalh != 0: 673 | print( 674 | " --- ATTENTION: " 675 | + str(nnevalh) 676 | + "/" 677 | + str(neval - nnevala) 678 | + " arriving parcels are not associated with any heat uptakes..." 679 | ) 680 | print( 681 | " STATS: Evaluated " 682 | + str(neval - nnevalm) 683 | + " ({:.2f}".format(100 * (neval - nnevalm) / (neval)) 684 | + "%) midpoint parcels inside mask (precipitation)." 685 | ) 686 | if nnevalm == neval: 687 | print(" STATS: Evaluated " + str(nevalp) + " precipitating parcels.") 688 | else: 689 | print( 690 | " STATS: Evaluated " 691 | + str(nevalp) 692 | + " ({:.2f}".format(100 * (nevalp) / (neval - nnevalm)) 693 | + "%) precipitating parcels." 694 | ) 695 | if nnevalp != 0: 696 | print( 697 | " --- ATTENTION: " 698 | + str(nnevalp) 699 | + "/" 700 | + str(nevalp) 701 | + " precipitating parcels are not associated with any evap uptakes..." 702 | ) 703 | 704 | ## SOME DAILY CALCULATIONS 705 | if (ix + 1) % 4 == 0: 706 | # DAILY UPSCALING of E2P, taking into account the missing trajectories (i.e. the ones without any uptakes) 707 | if fdupscale and (nnevalp != 0 or ipmiss != 0): 708 | if ipatt == 0: 709 | print( 710 | " \n--- WARNING: there were no trajectories with uptakes, so upscaling is impossible...\n " 711 | ) 712 | else: 713 | upsfac = 1 + (ipmiss / ipatt) 714 | ary_etop[:, :, :] = upsfac * ary_etop[:, :, :] 715 | # corrections for final statistics 716 | patt += -np.sum(ipatt) + np.sum(ary_etop) 717 | pmiss += -np.sum(ipmiss) 718 | if verbose: 719 | print(" * Upscaling... (factor: {:.4f}".format(upsfac) + ")") 720 | # Convert units 721 | if verbose: 722 | print(" * Converting units...") 723 | ary_etop[:, :, :] = convertunits(ary_etop[:, :, :], garea, "E") 724 | ary_heat[:, :, :] = convertunits(ary_heat[:, :, :], garea, "H") 725 | 726 | if fwrite_netcdf: 727 | writenc4D(ofile, arv_idx, ary_etop, ary_heat, verbose) 728 | 729 | ## STATS summary 730 | tneval += neval 731 | tnnevala += nnevala 732 | tnnevalm += nnevalm 733 | tnevalp += nevalp 734 | tnnevalp += nnevalp 735 | tnevalh += nevalh 736 | tnnevalh += nnevalh 737 | 738 | # MONTHLY UPSCALING of E2P, taking into account the missing trajectories (i.e. the ones without any uptakes) 739 | if fmupscale and (pmiss != 0 or punatt != 0): 740 | if patt == 0: 741 | print( 742 | " \n--- WARNING: there were no trajectories with uptakes, so upscaling is impossible...\n" 743 | ) 744 | else: 745 | upsfac = 1 + ((pmiss + punatt) / patt) 746 | # load full etop array and upscale 747 | fdata = nc4.Dataset(ofile, "r+") 748 | uns_etop = fdata.variables["E2P"][:] 749 | ups_etop = upsfac * uns_etop 750 | fdata.variables["E2P"][:] = ups_etop 751 | fdata.close() 752 | # corrections for final statistics 753 | patt += np.sum(pmiss) + np.sum(punatt) 754 | pmiss += -np.sum(pmiss) 755 | punatt += -np.sum(punatt) 756 | if verbose: 757 | print( 758 | " * Monthly upscaling for unattributed precipitation... (factor: {:.4f}".format( 759 | upsfac 760 | ) 761 | + ")" 762 | ) 763 | 764 | if ftimethis: 765 | megatoc = timeit.default_timer() 766 | if verbose: 767 | print( 768 | "\n=== \t End main program (total runtime so far: ", 769 | str(round(megatoc - megatic, 2)), 770 | "seconds) \n", 771 | ) 772 | 773 | if verbose: 774 | if fwrite_netcdf: 775 | if psum != 0: 776 | # adding attributed fraction to netcdf file description 777 | append_attrfrac_netcdf(ofile, "{:.2f}".format(patt / psum)) 778 | print("\n Successfully written: " + ofile + " !\n") 779 | 780 | writestats_02( 781 | statfile, 782 | tneval, 783 | tnnevala, 784 | tnevalh, 785 | tnnevalh, 786 | tnnevalm, 787 | tnevalp, 788 | tnnevalp, 789 | patt, 790 | psum, 791 | punatt, 792 | pmiss, 793 | ) 794 | if verbose: 795 | with open(statfile, "r") as sfile: 796 | print(sfile.read()) 797 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | --------------------------------------------------------------------------------