├── eoldas ├── Identity.py ├── config_files │ ├── eoldas_config.conf │ └── default.conf ├── eoldas_Kernel_Operator.py ├── __init__.py ├── eoldas_ParamStorage.py ├── eoldas_Parser.py ├── eoldas_Observation_Operator.py ├── eoldas_Spectral.py ├── eoldas_DModel_Operator.py ├── eoldas_Solver.py ├── eoldas_SpecialVariable.py └── eoldas_ConfFile.py ├── MANIFEST.in ├── setup.cfg ├── .gitignore ├── README.rst ├── setup.py └── scripts └── eoldas_run.py /eoldas/Identity.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.txt 2 | recursive-include eoldas *py *conf 3 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [egg_info] 2 | tag_build = 3 | tag_svn_revision = true 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | 3 | # Packages 4 | *.egg 5 | *.egg-info 6 | dist 7 | build 8 | eggs 9 | parts 10 | bin 11 | var 12 | sdist 13 | develop-eggs 14 | .installed.cfg 15 | 16 | # Installer logs 17 | pip-log.txt 18 | 19 | # Unit test / coverage reports 20 | .coverage 21 | .tox 22 | 23 | #Translations 24 | *.mo 25 | 26 | #Mr Developer 27 | .mr.developer.cfg 28 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | eoldas 2 | ====== 3 | 4 | :Info: Earth Observation Land Data Assimilation System (EO-LDAS) 5 | :Author: P Lewis & J Gomez-Dans , 6 | :Date: $Date: 2012-05-21 16:00:00 +0000 $ 7 | :Description: README file 8 | 9 | Please see `the EOLDAS documentation webpage `_ 10 | 11 | An Earth Observation Land Data Assimilation System (EO-LDAS). This package blah 12 | blah blah 13 | -------------------------------------------------------------------------------- /eoldas/config_files/eoldas_config.conf: -------------------------------------------------------------------------------- 1 | # configuration file 2 | # note that some words are protected: values, keys 3 | # so dont use them 4 | # other than that, you can set any level of hierarchy in the 5 | # representation 6 | # this next part serves no purpose other than 7 | # to show how different options can be set 8 | 9 | # It is a requirement that parameter.names be defined 10 | 11 | [parameter] 12 | location = ['time'] 13 | limits = [[1,365,1]] 14 | help_limits = Specify the limits for location variables. Of the form [[min,max,step],[...]]. e.g. [[70,170,1]] for location ['time'] 15 | name=EOLDAS 16 | help_solve = 'Solver switch for state variables parameter.names 0 : leave at default 1 : solve for each location 2 : solve a single value for all locations e.g. [0, 1, 1, 1]' 17 | datatypes = x 18 | 19 | [parameter.result] 20 | format = 'PARAMETERS' 21 | 22 | [parameter.x] 23 | datatype = x 24 | apply_grid = True 25 | bounds = [[0.01,0.99]]*len($parameter.names) 26 | 27 | [general] 28 | datadir = .,~/.eoldas 29 | help_datadir ="Specify where the data and or conf files are" 30 | here = os.getcwdu() 31 | grid = True 32 | is_spectral = True 33 | calc_posterior_unc=False 34 | help_calc_posterior_unc ="Switch to calculate the posterior uncertainty" 35 | write_results=True 36 | help_write_results="Flag to make eoldas write its results to files" 37 | 38 | [general.optimisation] 39 | # These are the default values 40 | iprint=1 41 | gtol=1e-3 42 | maxIter=1e4 43 | maxFunEvals=2e4 44 | plot=0 45 | # see http://openopt.org/NLP#Box-bound_constrained 46 | solverfn=scipy_lbfgsb 47 | no_df = False 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | import sys, os 3 | 4 | DISTNAME = "eoldas" 5 | DESCRIPTION = "An Earth Observation Land Data Assimilation System (EO-LDAS)" 6 | LONG_DESCRIPTION = open('README.txt').read() 7 | MAINTAINER = 'Jose Gomez-Dans/NCEO & University College London' 8 | MAINTAINER_EMAIL = "j.gomez-dans@ucl.ac.uk" 9 | URL = 'http://github.com/jgomezdans/eoldas' 10 | LICENSE = 'Undecided' 11 | VERSION = "1.1.0" 12 | DOWNLOAD_URL="https://github.com/jgomezdans/eoldas/zipball/master" 13 | 14 | setup(name='eoldas', 15 | version=VERSION, 16 | description=DESCRIPTION, 17 | long_description=LONG_DESCRIPTION, 18 | keywords='', 19 | maintainer=MAINTAINER, 20 | maintainer_email=MAINTAINER_EMAIL, 21 | url='http://www.assimila.eu/eoldas', 22 | download_url=DOWNLOAD_URL, 23 | license='', 24 | packages=['eoldas'], 25 | package_dir={'eoldas': 'eoldas'}, 26 | package_data={'eoldas': ['config_files/*.conf']}, 27 | #packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), 28 | include_package_data=True, 29 | zip_safe=False, 30 | install_requires=["semidiscrete1>0.9", 31 | "semidiscrete2>0.9", 32 | "semidiscrete3>0.9", 33 | "semidiscrete4>0.9"], 34 | # "OpenOpt>=0.39"], 35 | # "FuncDesigner>=0.39", 36 | # "DerApproximator>=0.39", 37 | # "SpaceFuncs>=0.39"], 38 | # -*- Extra requirements: -*- 39 | # It require the RT codes, but the package names are still in flux 40 | #], 41 | classifiers=[ 42 | 'Intended Audience :: Science/Research', 43 | 'Intended Audience :: Developers', 44 | 'License :: OSI Approved', 45 | 'Programming Language :: Fortran', 46 | 'Programming Language :: Python', 47 | 'Topic :: Software Development', 48 | 'Topic :: Scientific/Engineering', 49 | 'Operating System :: Microsoft :: Windows', 50 | 'Operating System :: POSIX', 51 | 'Operating System :: Unix', 52 | 'Operating System :: MacOS' 53 | ], 54 | 55 | scripts=['scripts/eoldas_run.py'], 56 | ) 57 | -------------------------------------------------------------------------------- /eoldas/config_files/default.conf: -------------------------------------------------------------------------------- 1 | # NB case is ignored on keys 2 | 3 | 4 | [parameter] 5 | location = ['time'] 6 | limits = [[1,365,1]] 7 | #limits = [[70,170,1]] 8 | help_limits = Specify the limits for location variables. Of the form [[min,max,step],[...]]. e.g. [[70,170,1]] for location ['time'] 9 | names = 'gamma_time Isotropic RossThick LiSparseModis'.split() 10 | solve = [1]*len($parameter.names) 11 | help_solve = 'Solver switch for state variables parameter.names 0 : leave at default 1 : solve for each location 2 : solve a single value for all locations e.g. [0, 1, 1, 1]' 12 | datatypes = x 13 | 14 | [parameter.result] 15 | filename = 'test/data_type/output/test.params' 16 | format = 'PARAMETERS' 17 | 18 | [parameter.assoc_solve] 19 | gamma_time = 2 20 | 21 | # This doesnt work with eg Isotropic.465.6 = 0 22 | # for the moment 23 | 24 | [parameter.x] 25 | datatype = x 26 | names = $parameter.names 27 | default = [0]*len($parameter.names) 28 | apply_grid = True 29 | sd = [1.]*len($parameter.names) 30 | bounds = [[0,1]]*len($parameter.names) 31 | 32 | [parameter.x.assoc_default] 33 | gamma_time = 100.0 34 | gamma_time_helper = 'gamma' 35 | 36 | [parameter.x.assoc_bounds] 37 | gamma_time = [0.0001,1000.] 38 | 39 | [general] 40 | datadir = test/data_type/input/,.,~/.eoldas 41 | help_datadir ="Specify where the data and or conf files are" 42 | here = os.getcwdu() 43 | grid = True 44 | is_spectral = False 45 | 46 | [general.optimisation] 47 | # These are the default values 48 | iprint=1 49 | gtol=1e-3 50 | maxIter=2e4 51 | maxFunEvals=2e4 52 | name='solver' 53 | plot=0 54 | # see http://openopt.org/NLP#Box-bound_constrained 55 | solverfn=scipy_lbfgsb 56 | randomise=False 57 | no_df = False 58 | 59 | [operator] 60 | modelt.name=DModel_Operator 61 | modelt.datatypes = x 62 | obs.name=Kernel_Operator 63 | obs.datatypes = x,y 64 | 65 | [operator.modelt.x] 66 | names = $parameter.names 67 | sd = [1.]*len($operator.modelt.x.names) 68 | datatype = x 69 | 70 | [operator.obs.x] 71 | names = $parameter.names[1:] 72 | sd = [1.0]*len($operator.obs.x.names) 73 | datatype = x 74 | 75 | [operator.obs.y] 76 | control = 'mask vza vaa sza saa'.split() 77 | #names = "465.6 553.6 645.5 856.5 1241.6 1629.1 2114.1".split() 78 | #sd = "0.003 0.004 0.004 0.015 0.013 0.01 0.006".split() 79 | names = "645.5 856.5".split() 80 | sd = "0.004 0.015".split() 81 | datatype = y 82 | state = test/data_type/input/test.brf 83 | 84 | [operator.obs.y.result] 85 | filename = 'test/data_type/output/test-fwd.brf' 86 | format = 'PARAMETERS' 87 | 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /eoldas/eoldas_Kernel_Operator.py: -------------------------------------------------------------------------------- 1 | from kernels import * 2 | #from eoldas_obs import * 3 | from eoldas_Lib import * 4 | from eoldas_Operator import Operator 5 | ''' 6 | Kernels model interface to eoldas 7 | ''' 8 | class Kernel_Operator(Operator): 9 | 10 | def preload_prepare(self): 11 | ''' 12 | Here , we use preload_prepare to make sure 13 | the x & any y data are NOT gridded for this 14 | operator. 15 | 16 | This method is called before any data are loaded, 17 | so ensures they are not loaded as a grid. 18 | ''' 19 | # mimic setting the apply_grid flag in options 20 | self.y_state.options.y.apply_grid = False 21 | 22 | def postload_prepare(self): 23 | ''' 24 | This is called on initialisation, after data have been read in. 25 | 26 | In this method, we have a linear model, so we can 27 | pre-calculate the kernels (stored in self.K) 28 | 29 | ''' 30 | # This is not gridded data, so we have explicit 31 | # information on self.y.control 32 | self.mask = self.y.control\ 33 | [:,self.y_meta.control=='mask'].astype(bool) 34 | vza = self.y.control[:,self.y_meta.control=='vza'] 35 | vaa = self.y.control[:,self.y_meta.control=='vaa'] 36 | sza = self.y.control[:,self.y_meta.control=='sza'] 37 | saa = self.y.control[:,self.y_meta.control=='saa'] 38 | self.kernels = Kernels(vza,sza,vaa-saa,RossHS=False,MODISSPARSE=True,RecipFlag=True,normalise=1,doIntegrals=False,LiType='Sparse',RossType='Thick') 39 | K = numpy.ones([len(vza),3]) 40 | K[:,1] = self.kernels.Ross[:] 41 | K[:,2] = self.kernels.Li[:] 42 | # here, we know the full number of states and their names 43 | names = self.x_meta.state 44 | bands = self.y_meta.state 45 | nb = len(bands) 46 | nn = len(names) 47 | ns = len(vza) 48 | M = np.eye(nb).astype(bool) 49 | I = np.eye(nb).astype(float) 50 | M1 = np.zeros((3,3*nb,nb)).astype(bool) 51 | I1 = np.zeros((3*nb,nb)).astype(float) 52 | for i in xrange(3): 53 | M1[i,i*nb:(i+1)*nb,:] = True 54 | 55 | self.K = np.zeros((ns,nn,ns,nb)) 56 | for i in xrange(ns): 57 | this = I1.copy() 58 | for jj in xrange(3): 59 | that = this[M1[jj]].reshape((nb,nb)) 60 | that[M] = K[i,jj] 61 | this[M1[jj]] = that.flatten() 62 | self.K[i,:,i,:] = this 63 | self.K = np.matrix(self.K.reshape((ns*nn,ns*nb))) 64 | self.nb = nb 65 | self.nn = nn 66 | self.ns = ns 67 | # we can form self.H so that y = self.H x 68 | # to do this, we need to know the location data 69 | # in self.x 70 | testh = self.H(self.x.state) 71 | self.linear.H_prime = np.matrix(self.K) 72 | self.isLinear = True 73 | 74 | 75 | def H(self,x): 76 | ''' 77 | The reflectance from the kernels 78 | ''' 79 | x = x.flatten() 80 | self.Hx = np.array((x * self.K).reshape(self.ns,self.nb)) 81 | return self.Hx 82 | 83 | 84 | if __name__ == "__main__": 85 | self = tester() 86 | 87 | -------------------------------------------------------------------------------- /eoldas/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from eoldas_ParamStorage import * 4 | from eoldas_Files import * 5 | from eoldas_Lib import * 6 | from eoldas_Parser import * 7 | from eoldas_ConfFile import * 8 | from eoldas_SpecialVariable import * 9 | from eoldas_State import * 10 | from eoldas_Operator import * 11 | from eoldas_Spectral import * 12 | from eoldas_DModel_Operator import * 13 | from eoldas_Observation_Operator import * 14 | from eoldas_Kernel_Operator import * 15 | from eoldas_Solver import * 16 | from eoldas_ParamStorage import ParamStorage 17 | from eoldas_Parser import Parser 18 | #from eoldas_Solver import eoldas_Solver as Solver 19 | 20 | class eoldas(Parser): 21 | 22 | ''' 23 | The Earth Observation Land Data Assimilation System: EOLDAS 24 | 25 | This tool is designed primarily to be used from a command 26 | line prompt. 27 | 28 | The operation of the EOLDAS is controlled by these main mechanisms: 29 | 30 | 1. The command line 31 | ========================== 32 | A parser is invoked. This has a set of default options 33 | but can be extended through the configuration file. 34 | Type: 35 | 36 | eoldas.py --help 37 | 38 | to see this, or: 39 | 40 | eoldas.py --conf=default.conf --help 41 | 42 | to see how new command line options have been added. 43 | You should notice that you can now specify e.g.: 44 | 45 | --parameter.limits=PARAMETER.LIMITS 46 | --parameter.solve=PARAMETER.SOLVE 47 | 48 | This allows the user a good deal of flexibility in 49 | setting up EOLDAS experiments, as it shouldn't involve 50 | much writing code, just setting things up in a configuration 51 | file or files. 52 | 53 | 2. The configuration file 54 | ========================== 55 | This is the main way of controlling an EOLDAS experiment. 56 | 57 | 58 | 3. Calling the eoldas class 59 | ========================== 60 | e.g. from python: 61 | 62 | this = eoldas(args) 63 | 64 | where args is a list or a string containing the 65 | equivalent of command line arguments 66 | 67 | e.g. 68 | 69 | this = eoldas('eoldas --help') 70 | 71 | 72 | ''' 73 | def __init__(self,argv,name='eoldas',logger=None): 74 | from eoldas.eoldas_Lib import sortopt, sortlog 75 | 76 | argv = argv or sys.argv 77 | here = os.getcwd() 78 | self.thisname = name 79 | Parser.__init__(self,argv,name=self.thisname,logger=logger,\ 80 | general=None,outdir=".",getopdir=False,parse=True) 81 | os.chdir(here) 82 | if not hasattr(self,'configs'): 83 | self.logger.error('No configration file specfied') 84 | help(eoldas) 85 | return 86 | 87 | self.thisname = name 88 | solver = eoldas_Solver(self,logger=self.logger,name=self.thisname+'.solver') 89 | self.general = sortopt(self.root[0],'general',ParamStorage()) 90 | self.general.write_results = sortopt(self.general,'write_results',True) 91 | self.general.calc_posterior_unc = sortopt(self.general,'calc_posterior_unc',False) 92 | self.general.passer = sortopt(self.general,'passer',False) 93 | self.solver = solver 94 | self.logger.info('testing full cost functions') 95 | for i in xrange(len(solver.confs.infos)): 96 | self.logger.info('%d/%d ...'%(i+1,len(solver.confs.infos))) 97 | # try an initial solver.prep(i) 98 | J = solver.cost(None) 99 | J_prime = solver.cost_df(None) 100 | self.logger.info('done') 101 | 102 | # give the user some info on where the log file is 103 | # in case theyve forgotten 104 | print 'logging to',self.general.logfile 105 | 106 | def solve(self,unc=None,write=None): 107 | ''' 108 | Run the solver 109 | 110 | Options: 111 | 112 | unc : Set to True to calculate posterior uncertainty 113 | write : Set to True to write out datafiles 114 | 115 | ''' 116 | if unc == None: 117 | unc = self.general.calc_posterior_unc 118 | if write == None: 119 | write = self.general.write_results 120 | solver = self.solver 121 | for i in xrange(len(solver.confs.infos)): 122 | solver.prep(i) 123 | J = solver.cost(None) 124 | J_prime = solver.cost_df(None) 125 | # run the solver 126 | if not self.general.passer: 127 | solver.solver() 128 | # Hessian 129 | if unc: 130 | solver.uncertainty() 131 | # write out the state 132 | if write: 133 | solver.write() 134 | # write out any fwd modelling of observations 135 | solver.writeHx() 136 | 137 | def uncertainty(self,write=None): 138 | ''' 139 | Calculate uncertainty 140 | 141 | Options: 142 | 143 | write : Set to True to write out datafiles 144 | 145 | ''' 146 | solver = self.solver 147 | if write == None: 148 | write = self.general.write_results 149 | 150 | for i in xrange(len(solver.confs.infos)): 151 | solver.prep(i) 152 | J = solver.cost(None) 153 | J_prime = solver.cost_df(None) 154 | # run the solver 155 | # write out the state 156 | if write: 157 | solver.write() 158 | # write out any fwd modelling of observations 159 | solver.writeHx() 160 | 161 | def write(self): 162 | ''' 163 | Write out datafiles 164 | 165 | ''' 166 | solver = self.solver 167 | for i in xrange(len(solver.confs.infos)): 168 | solver.prep(i) 169 | J = solver.cost(None) 170 | J_prime = solver.cost_df(None) 171 | # write out the state 172 | solver.write() 173 | # write out any fwd modelling of observations 174 | solver.writeHx() 175 | 176 | -------------------------------------------------------------------------------- /scripts/eoldas_run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import os 4 | import logging 5 | import numpy as np 6 | import pdb 7 | from eoldas import * 8 | from eoldas.eoldas_ParamStorage import ParamStorage 9 | from eoldas.eoldas_Parser import Parser 10 | from eoldas.eoldas_Solver import eoldas_Solver as Solver 11 | 12 | class eoldas(Parser): 13 | ''' 14 | The Earth Observation Land Data Assimilation System: EOLDAS 15 | 16 | This tool is designed primarily to be used from a command 17 | line prompt. 18 | 19 | The operation of the EOLDAS is controlled by these main mechanisms: 20 | 21 | 1. The command line 22 | ========================== 23 | A parser is invoked. This has a set of default options 24 | but can be extended through the configuration file. 25 | Type: 26 | 27 | eoldas.py --help 28 | 29 | to see this, or: 30 | 31 | eoldas.py --conf=default.conf --help 32 | 33 | to see how new command line options have been added. 34 | You should notice that you can now specify e.g.: 35 | 36 | --parameter.limits=PARAMETER.LIMITS 37 | --parameter.solve=PARAMETER.SOLVE 38 | 39 | This allows the user a good deal of flexibility in 40 | setting up EOLDAS experiments, as it shouldn't involve 41 | much writing code, just setting things up in a configuration 42 | file or files. 43 | 44 | 2. The configuration file 45 | ========================== 46 | This is the main way of controlling an EOLDAS experiment. 47 | 48 | 49 | 3. Calling the eoldas class 50 | ========================== 51 | e.g. from python: 52 | 53 | this = eoldas(args) 54 | 55 | where args is a list or a string containing the 56 | equivalent of command line arguments 57 | 58 | e.g. 59 | 60 | this = eoldas('eoldas --help') 61 | 62 | 63 | ''' 64 | def __init__(self,argv,name='eoldas',logger=None): 65 | from eoldas.eoldas_Lib import sortopt, sortlog 66 | 67 | argv = argv or sys.argv 68 | here = os.getcwd() 69 | self.thisname = name 70 | Parser.__init__(self,argv,name=self.thisname,logger=logger,\ 71 | general=None,outdir=".",getopdir=False,parse=True) 72 | os.chdir(here) 73 | if not hasattr(self,'configs'): 74 | self.logger.error('No configration file specfied') 75 | help(eoldas) 76 | return 77 | 78 | self.thisname = name 79 | solver = Solver(self,logger=self.logger,name=self.thisname+'.solver') 80 | self.general = sortopt(self.root[0],'general',ParamStorage()) 81 | self.general.write_results = sortopt(self.general,'write_results',True) 82 | self.general.calc_posterior_unc = sortopt(self.general,'calc_posterior_unc',False) 83 | self.general.passer = sortopt(self.general,'passer',False) 84 | self.solver = solver 85 | self.logger.info('testing full cost functions') 86 | for i in xrange(len(solver.confs.infos)): 87 | self.logger.info('%d/%d ...'%(i+1,len(solver.confs.infos))) 88 | # try an initial solver.prep(i) 89 | J = solver.cost(None) 90 | J_prime = solver.cost_df(None) 91 | self.logger.info('done') 92 | 93 | # give the user some info on where the log file is 94 | # in case theyve forgotten 95 | print 'logging to',self.general.logfile 96 | def solve(self,unc=None,write=None): 97 | ''' 98 | Run the solver 99 | 100 | Options: 101 | 102 | unc : Set to True to calculate posterior uncertainty 103 | write : Set to True to write out datafiles 104 | 105 | ''' 106 | if unc == None: 107 | unc = self.general.calc_posterior_unc 108 | if write == None: 109 | write = self.general.write_results 110 | solver = self.solver 111 | for i in xrange(len(solver.confs.infos)): 112 | solver.prep(i) 113 | J = solver.cost(None) 114 | J_prime = solver.cost_df(None) 115 | # run the solver 116 | if not self.general.passer: 117 | solver.solver() 118 | # Hessian 119 | if unc: 120 | solver.uncertainty() 121 | # write out the state 122 | if write: 123 | solver.write() 124 | # write out any fwd modelling of observations 125 | solver.writeHx() 126 | 127 | def uncertainty(self,write=None): 128 | ''' 129 | Calculate uncertainty 130 | 131 | Options: 132 | 133 | write : Set to True to write out datafiles 134 | 135 | ''' 136 | solver = self.solver 137 | if write == None: 138 | write = self.general.write_results 139 | 140 | for i in xrange(len(solver.confs.infos)): 141 | solver.prep(i) 142 | J = solver.cost(None) 143 | J_prime = solver.cost_df(None) 144 | # run the solver 145 | # write out the state 146 | if write: 147 | solver.write() 148 | # write out any fwd modelling of observations 149 | solver.writeHx() 150 | 151 | def write(self): 152 | ''' 153 | Write out datafiles 154 | 155 | ''' 156 | solver = self.solver 157 | for i in xrange(len(solver.confs.infos)): 158 | solver.prep(i) 159 | J = solver.cost(None) 160 | J_prime = solver.cost_df(None) 161 | # write out the state 162 | solver.write() 163 | # write out any fwd modelling of observations 164 | solver.writeHx() 165 | 166 | 167 | def eoldas_demo(): 168 | ''' 169 | Standard running of eoldas 170 | 171 | e.g. 172 | 173 | eoldas.py --conf=default.conf --calc_posterior_unc --write_results 174 | 175 | ''' 176 | argv = sys.argv 177 | this = [argv[0].replace('.py','')] 178 | argv[0] = '--conf=eoldas_config.conf' 179 | [this.append(i) for i in argv] 180 | #pdb.set_trace() 181 | self = eoldas(this) 182 | self.solve() 183 | 184 | if __name__ == "__main__": 185 | eoldas_demo() 186 | 187 | -------------------------------------------------------------------------------- /eoldas/eoldas_ParamStorage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import pdb 3 | 4 | class ParamStorage( dict): 5 | """ 6 | A class to store parameters as a dictionary, but to being able to retrieve 7 | them using point notation. 8 | 9 | """ 10 | def __init__(self,name=None,doc=None,logger=None): 11 | ''' 12 | Initialise ParamStorage instance 13 | 14 | (just a call to dict initialisation for self.__dict__) 15 | 16 | ''' 17 | if logger: 18 | self.logger = logger 19 | #from eoldas_Lib import sortlog 20 | #sortlog(self,None,logger,debug=True) 21 | dict.__init__(self) 22 | # needed for pickling 23 | if name: 24 | self.__name__ = str(name) 25 | if doc: 26 | self.__doc__ = str(doc) 27 | #def __getstate__(self): 28 | # return self.__dict__ 29 | #def __setstate__(self): 30 | # return self.__dict__ 31 | 32 | 33 | 34 | def dict(self): 35 | ''' 36 | Return the dictionary (self.__dict__) 37 | ''' 38 | return self.__dict__ 39 | 40 | def update ( self, other_dict,combine=False,copy=False ): 41 | ''' 42 | Hierarchically update the ParamStorage self with 43 | another ParamStorage other_dict. 44 | 45 | If combine is True, then a full Hierarchical copy is made. 46 | If not true, then any sub-members in other_dict overwrite those 47 | in self. 48 | 49 | If copy is True, then a .copy() is applied to all elements when 50 | updating. 51 | 52 | ''' 53 | for ( k, v ) in other_dict.iteritems(): 54 | vtype = type(v) != dict and type(v) != ParamStorage 55 | if k in self.dict(): 56 | # k exists in self as well as other_dict 57 | if combine: 58 | if type(self[k]) == ParamStorage: 59 | self[k].update(other_dict[k],copy=copy,combine=combine) 60 | else: 61 | self[k] = other_dict[k] 62 | else: 63 | if copy and vtype: 64 | try: 65 | vv = v.copy() 66 | except: 67 | vv = v 68 | self[ k ] = vv 69 | self.__dict__.__setitem__ ( k, v) 70 | else: 71 | self[ k ] = v 72 | self.__dict__.__setitem__ ( k, v) 73 | else: 74 | if copy and vtype: 75 | try: 76 | vv = v.copy() 77 | except: 78 | vv = v 79 | super( ParamStorage, self ).__setattr__( k, vv) 80 | else: 81 | super( ParamStorage, self ).__setattr__( k, v ) 82 | 83 | 84 | def log(self,this=None,logger=None,name=None,supername='',logfile="log.dat",n=0,logdir="logs",debug=True,print_helper=False): 85 | ''' 86 | Dump the whole contents to the a log 87 | 88 | Useful for inspecting data 89 | 90 | If self.logger is set, this is the log that is used, otherwise 91 | one is set up from logfile (and possibly logdir). 92 | In this case, name is used as the identifier in the log. You 93 | have to set name to enable log initialisation. 94 | 95 | If print_helper is True, then print_helper fields are logged (default False) 96 | 97 | ''' 98 | from eoldas_Lib import sortlog 99 | self.logger = sortlog(self,logfile,logger,name=name,logdir=logdir,debug=debug) 100 | if this == None: 101 | this = self 102 | try: 103 | self.logger.info("**"*20) 104 | self.logger.info("logging parameters ...") 105 | self.logger.info("**"*20) 106 | except: 107 | pass 108 | for ( k, v ) in this.iteritems(): 109 | if supername != '': 110 | thisname = "%s.%s" % (supername,str(k)) 111 | else: 112 | thisname = str(k) 113 | strk = str(k) 114 | doit = True 115 | if strk.replace('__','') != strk or str(k) == 'helper' or str(k)[:5] == "help_": 116 | doit = False 117 | if doit or print_helper: 118 | try: 119 | self.logger.info("%s = %s" % (thisname,str(v))) 120 | except: 121 | pass 122 | if type(v) == ParamStorage and k != 'logger': 123 | self.log(this=this[k],name=None,n=n+2,supername=thisname,print_helper=print_helper) 124 | 125 | def __getitem__(self, key): 126 | val = self.__dict__.__getitem__ ( key ) 127 | return val 128 | 129 | def __setitem__(self, key, val): 130 | self.__dict__.__setitem__ ( key, val ) 131 | 132 | def __getattr__( self, name ): 133 | #return self[name] 134 | #if name in self.__dict__: 135 | return self.__dict__.__getitem__ ( name ) 136 | #return None 137 | 138 | def __setattr__(self, name, value): 139 | if name in self: 140 | self[ name ] = value 141 | self.__dict__.__setitem__ ( name, value ) 142 | else: 143 | super( ParamStorage, self ).__setattr__( name, value ) 144 | 145 | def iteritems(self): 146 | for k in self: 147 | yield (k, self[k]) 148 | def __iter__(self): 149 | for k in self.__dict__.keys(): 150 | yield k 151 | 152 | def to_dict(self,no_instance=False): 153 | ''' 154 | Convert the contents of a ParamStorage to a dict type 155 | using hierarchical interpretation of ParamStorage elements. 156 | You can access the dictionary directly as self.__dict__ but this 157 | might contain other ParamStorage elements. 158 | 159 | Returns the dictionary. 160 | 161 | This is useful for portability, pickling etc. 162 | 163 | If no_instance is set, instancemethod types 164 | are treated as strings. 165 | ''' 166 | this = {} 167 | for ( k, v ) in self.iteritems(): 168 | t = type(v) 169 | tt = str(t)[:6] 170 | if k != 'logger' and k != 'self': 171 | if type(v) == ParamStorage: 172 | this[k] = v.to_dict(no_instance=no_instance) 173 | # a hck-y bit to not dump methods & classes 174 | elif not (no_instance and t.__name__ == 'instancemethod') \ 175 | and tt != ' 2: 80 | conf = theseargs[0][2:] 81 | else: 82 | conf = self.args[i+1] 83 | self.top.general.conf.append(conf) 84 | elif theseargs[0] == "--datadir": 85 | datadir1 = theseargs[1].replace('[','').\ 86 | replace(']','').split() 87 | [datadir1.append(datadir[i]) for i in \ 88 | xrange(len(datadir))] 89 | datadir = datadir1 90 | elif theseargs[0] == "--logfile": 91 | logfile= theseargs[1] 92 | elif theseargs[0] == "--logdir": 93 | logdir = theseargs[1] 94 | elif theseargs[0] == "--outdir": 95 | outdir = theseargs[1] 96 | if self.top.general.conf == []: 97 | self.top.general.conf = conf 98 | if logfile == None: 99 | logfile = conf.replace('conf','log') 100 | self.top.general.here = os.getcwd() 101 | self.top.general.datadir = datadir 102 | self.top.general.logfile = logfile 103 | self.top.general.logdir = logdir 104 | self.top.general.outdir = outdir 105 | # add here to datadir 106 | # in addition to '.' to take account of the change of directory 107 | self.top.general.datadir = self.__add_here_to_datadir(\ 108 | self.top.general.here,self.top.general.datadir) 109 | self.top.general.datadir = self.__add_here_to_datadir(\ 110 | self.top.general.outdir,self.top.general.datadir) 111 | # cd to where the output is to be 112 | self.__cd(self.top.general.outdir) 113 | # set up the default command line options 114 | self.default_loader() 115 | # update with anything passed here 116 | if general and type(general) == ParamStorage: self.top.update(\ 117 | self.__unload(general),combine=True) 118 | # read the conf files to get any cmd line options 119 | self.logger = sortlog(self,self.top.general.logfile,logger,name=self.thisname,\ 120 | logdir=self.top.general.logdir) 121 | self.config = ConfFile(self.top.general.conf,name=self.thisname+'.config',\ 122 | loaders=self.loaders,datadir=self.top.\ 123 | general.datadir,logger=self.logger,logdir=self.top.general.logdir,\ 124 | logfile=self.top.general.logfile) 125 | if len(self.config.configs) == 0: 126 | this = "Warning: Nothing doing ... you haven't set any configuration",\ 127 | self.config.storelog 128 | try: 129 | self.logger(this) 130 | except: 131 | print "Called with args:" 132 | print "eoldas",self.args 133 | pass 134 | raise Exception(this) 135 | 136 | # now loaders contains all of the defaults set here 137 | # plus those from the config (config opver-rides defaults here) 138 | self.loaders = self.config.loaders 139 | # now convert loaders into parser information 140 | self.parseLoader(self.loaders) 141 | self.parse(self.fullargs) 142 | if general and type(general) == ParamStorage: 143 | self.top.update(self.__unload(general),combine=True) 144 | if general and type(general) == str: 145 | self.parse(general.split()) 146 | if general and type(general) == list: 147 | self.parse(general) 148 | # now update the info in self.config 149 | for i in self.config.infos: 150 | i.update(self.top,combine=True) 151 | # so now all terms in self.config.infos 152 | # contain information from the config file, updated by 153 | # the cmd line 154 | i.logger = self.logger 155 | i.log() 156 | # move the information up a level 157 | self.infos = self.config.infos 158 | self.configs = self.config.configs 159 | self.config_log = self.config.storelog 160 | 161 | #if getopdir: 162 | # self.sortnames() 163 | self.config.loglist(self.top) 164 | 165 | #del(self.config.infos) 166 | #del(self.config.configs) 167 | #del(self.config) 168 | self.__cd(self.top.general.here) 169 | 170 | def __add_here_to_datadir(self,here,datadir): 171 | from os import sep,curdir,pardir 172 | if type(datadir) == str: 173 | datadir = [datadir] 174 | iadd = 0 175 | for i in xrange(len(datadir)): 176 | j = i + iadd 177 | if datadir[j] == curdir or datadir[j] == pardir: 178 | tmp = datadir[:j] 179 | rest = datadir[j+1:] 180 | tmp.append("%s%s%s" % (here,sep,datadir[j])) 181 | tmp.append(datadir[j]) 182 | iadd += 1 183 | for k in xrange(len(rest)): 184 | tmp.append(rest[k]) 185 | datadir = tmp 186 | return datadir 187 | 188 | def default_loader(self): 189 | """ 190 | Load up parser information for first pass 191 | """ 192 | self.loaders = [] 193 | self.top.general.here = os.getcwd() 194 | self.loaders.append(["datadir",['.',self.top.general.here],\ 195 | "Specify where the data and or conf files are"]) 196 | self.loaders.append(["passer",False,\ 197 | "Pass over optimisation (i.e. report and plot the initial values)"]) 198 | self.loaders.append(["outdir",None,\ 199 | "Explicitly mspecify the results and processing output directory"]) 200 | self.loaders.append(["verbose",False,"Switch ON verbose mode","v"]) 201 | self.loaders.append(["debug",False,optparse.SUPPRESS_HELP,"d"]) 202 | self.loaders.append(["conf","default.conf",\ 203 | "Specify configuration file. Set multiple files by using the flag multiple times.","c"]) 204 | self.loaders.append(["logdir","logs",\ 205 | "Subdirectory to put log file in"]) 206 | self.loaders.append(["logfile","logfile.logs","Log file name"]) 207 | 208 | 209 | def parseLoader(self,loaders): 210 | """ 211 | Utility to load a set of terms from the list loaders 212 | into the ParamStorage general 213 | 214 | If there are 3 terms in each loaders element, they refer to: 215 | 1. name 216 | 2. default value 217 | 3. helper text 218 | If there is a fourth, it is associated with extras 219 | (short parser option) 220 | """ 221 | general = ParamStorage () 222 | general.__default__ = ParamStorage () 223 | general.__extras__ = ParamStorage () 224 | general.__helper__ = ParamStorage () 225 | 226 | for this in loaders: 227 | if len(this) > 1: 228 | general[this[0]] = this[1] 229 | general.__default__[this[0]] = this[1] 230 | else: 231 | general[this[0]] = None 232 | general.__default__[this[0]] = None 233 | if len(this) > 2: 234 | general.__helper__[this[0]] = this[2] 235 | else: 236 | general.__helper__[this[0]] = optparse.SUPPRESS_HELP 237 | if len(this) > 3: 238 | general.__extras__[this[0]] = "%s" % this[3] 239 | else: 240 | general.__extras__[this[0]] = None 241 | # make sure arrays arent numpy.ndarray 242 | if type(general.__default__[this[0]]) == np.ndarray: 243 | general.__default__[this[0]] = \ 244 | list(general.__default__[this[0]]) 245 | 246 | self.top.update(self.__unload(general),combine=True) 247 | 248 | 249 | def __list_to_string__(self,thisstr): 250 | """ 251 | Utility to convert a list to some useable string 252 | """ 253 | return(str(thisstr).replace('[','_').strip("']").\ 254 | replace('.dat','').replace("_'","_").replace(",","").\ 255 | replace(" ","").replace("''","_").replace("___","_").\ 256 | replace("__","_")); 257 | 258 | 259 | def __cd(self,outdir): 260 | if not os.path.exists(outdir): 261 | try: 262 | os.makedirs(outdir) 263 | except OSerror: 264 | print "Fatal: Prevented from creating",outdir 265 | sys.exit(-1) 266 | try: 267 | os.chdir(outdir) 268 | except: 269 | print "Fatal: unable to cd to",outdir 270 | raise Exception("Fatal: unable to cd to %s"%outdir) 271 | 272 | def sortnames(self): 273 | """ 274 | Utility code to sort out some useful filenames & directories 275 | """ 276 | if self.top.general.outdir == None: 277 | basename = self.top.general.basename 278 | confnames = self.__list_to_string__(self.top.general.conf) 279 | self.top.general.outdir = basename + "_conf_" + confnames 280 | 281 | self.__cd(self.top.general.outdir) 282 | 283 | def parse(self,args,log=False): 284 | ''' 285 | Given a list such as sys.argv (of that form) 286 | or an equivalent string parse the general and store 287 | in self.parser 288 | ''' 289 | self.dolog = log or self.dolog 290 | if type(args) == type(""): 291 | args = args.split() 292 | args = args[1:] 293 | self.top.general.cmdline = str(args) 294 | usage = "usage: %prog [general] arg1 arg2" 295 | parser = OptionParser(usage,version="%prog 0.1") 296 | 297 | # we go through items in self.top.general and set up 298 | # parser general for each 299 | for this in sorted(self.top.general.__helper__.__dict__.keys()): 300 | # sort out the action 301 | # based on type of the default 302 | default=self.top.general.__default__[this] 303 | action="store" 304 | thistype=type(default) 305 | if thistype == type(None): 306 | thistype = type("") 307 | 308 | argss = '--%s'%this 309 | dest = "%s"%this 310 | helper = self.top.general.__helper__[this] 311 | if type(default) == type([]): 312 | # list, so append 313 | action="store" 314 | elif type(default) == type(True): 315 | action="store_true" 316 | typer = "string" 317 | if thistype != type([]) and thistype != type(True): 318 | typer='%s' % str(thistype).split("'")[1] 319 | # has it got extras? 320 | if self.top.general.__extras__[this] != None: 321 | parser.add_option('-%s'%self.top.general.__extras__[\ 322 | this].lower(),argss,type="string",action=action,\ 323 | help=helper,default=str(default)) 324 | else: 325 | parser.add_option(argss,action=action,help=helper,\ 326 | type="string",default=str(default)) 327 | elif ( thistype != type(True)): 328 | if self.top.general.__extras__[this] != None: 329 | parser.add_option('-%s'%self.top.general.__extras__[\ 330 | this].lower(),argss,type="string",action=action,\ 331 | help=helper,default=str(default)) 332 | else: 333 | parser.add_option(argss,action=action,help=helper,\ 334 | default=str(default)) 335 | if thistype == type(True): 336 | if self.top.general.__extras__[this] != None: 337 | parser.add_option('-%s'%self.top.general.__extras__[\ 338 | this].lower(),argss,dest=dest,action=action,\ 339 | help=helper,default=default) 340 | else: 341 | parser.add_option(argss,action=action,help=helper,\ 342 | dest=dest,default=default) 343 | that = this.split('.') 344 | argss = '--' 345 | for i in xrange(len(that)-1): 346 | argss = argss + "%s." % that[i] 347 | argss = argss + 'no_%s' % that[-1] 348 | helper='The opposite of --%s'%this 349 | action='store_false' 350 | typer='%s' % str(thistype).split("'")[1] 351 | # has it got extras? 352 | if self.top.general.__extras__[this] != None: 353 | parser.add_option('-%s'%self.top.general.__extras__[\ 354 | this].capitalize(),argss,dest=dest,action=action,\ 355 | help=helper) 356 | else: 357 | parser.add_option(argss,action=action,dest=dest,\ 358 | help=helper) 359 | 360 | 361 | # we have set all option types as str, so we need to interpret 362 | # them in__unload 363 | (general, args) = parser.parse_args(args) 364 | #for data_file in args: 365 | # general.data_file.append(data_file) 366 | #general.data_file = list(np.array(general.data_file).flatten()) 367 | #general.brf= list(np.array(general.brf).flatten()) 368 | # load these into self.general 369 | self.top.update(self.__unload(general.__dict__),combine=True) 370 | #self.sortnames() 371 | if self.dolog: 372 | self.log = set_up_logfile(self.top.general.logfile,\ 373 | logdir=self.top.general.logdir) 374 | self.log_report() 375 | 376 | def __unload(self,options): 377 | from eoldas_ConfFile import array_type_convert 378 | this = ParamStorage() 379 | this.general = ParamStorage() 380 | for (k,v) in options.iteritems(): 381 | ps = this 382 | that = k.split('.') 383 | if len(that) == 1: 384 | ps = this.general 385 | else: 386 | for i in xrange(len(that)-1): 387 | if not hasattr(ps,that[i]): 388 | ps[that[i]] = ParamStorage() 389 | ps = ps[that[i]] 390 | # set the value v which needs to 391 | # to be interpreted 392 | ps[that[-1]] = array_type_convert(self.top,v) 393 | return this 394 | 395 | 396 | 397 | def demonstration(): 398 | # this will read a conf file & over-ride with cmd line options 399 | here = os.getcwd() 400 | print "Parser help" 401 | #help(Parser) 402 | # start parser 403 | print "Testing Parser class with conf file default.conf" 404 | self = Parser("%s --conf=default.conf --outdir=test/parser --logfile=log.dat --logdir=logs" % 'test') 405 | os.chdir(here) 406 | print "outdir:",self.top.general.outdir 407 | print "log file in test/parser/logs/log.dat" 408 | del(self) 409 | print "Testing Parser class with conf file default.conf and --help" 410 | self = Parser("%s --no_log --conf=default.conf --outdir=test/parser --logfile=log.dat \ 411 | --logdir=logs --help" % 'test',log=False) 412 | # normally called as Parser(sys.argv) 413 | 414 | 415 | if __name__ == "__main__": 416 | from eoldas_Parser import Parser 417 | help(Parser) 418 | demonstration() 419 | -------------------------------------------------------------------------------- /eoldas/eoldas_Observation_Operator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import pdb 3 | import numpy as np 4 | #from eoldas_model import subset_vector_matrix 5 | from eoldas_State import State 6 | from eoldas_ParamStorage import ParamStorage 7 | from eoldas_Operator import * 8 | 9 | class Observation_Operator ( Operator ): 10 | ''' 11 | The EOLDAS Observation Operator Class 12 | 13 | This is designed to allow 'plug-n-play' operators provided 14 | they are spectral models, i.e. model reflectance or radiance 15 | as a function of the control variables and wavelength. 16 | Control variables are terms such as vza, vaa, but could for instance 17 | include polarisation state. Alternatively, terms such as polarisation 18 | could be included as pseudowavelengths (see Kernels_Operator for 19 | non spectral models) 20 | 21 | To use the Observation_Operator class, the user must declare 22 | 23 | operator.obs.name=Observation_Operator 24 | 25 | where obs is an arbitrary name for this operator. 26 | 27 | There should also be a section [operator.obs.rt_model] 28 | 29 | This must at least declare `model` e.g. 30 | 31 | operator.obs.rt_model.model = rtmodel_ad_trans2 32 | 33 | where rtmodel_ad_trans2 is a python class or a shared object library 34 | (i.e. rtmodel_ad_trans2.so in unix). Other terms pertinent to 35 | running the rt model can also be declared, e.g.: 36 | 37 | operator.obs.rt_model.use_median=True 38 | 39 | which means that the full band pass functions are not used, but 40 | the median of the bandpass, which is generally much faster. 41 | 42 | operator.obs.rt_model.spectral_interval = 1 43 | 44 | which declares the spectral interval of the model. This is assumed to 45 | be in nm. 46 | 47 | operator.obs.rt_model.ignore_derivative = False 48 | 49 | Set to True to override loading any defined derivative functions in the library and use numerical approximations instead 50 | 51 | The declared class (e.g. rtmodel_ad_trans2) is loaded in the method postload_prepare, which happens after all data and configuration information have been loaded into the operator (at the end of __init__). If this fails for any reason (e.g. it does not exist in the PYTHONPATH) then an exception is raised as the eoldas cannot continue. For those interested in where items are stored, this will be in self.options.rt_model.model. 52 | 53 | At this point, we also load methods from the rt_model. This is done by the (private) method _load_rt_library() and loads the methods in this class as `self.rt_library.` 54 | 55 | These must include: 56 | rt_model 57 | 58 | It may also include: 59 | 60 | rt_modelpre, rt_modelpost 61 | 62 | which are setup functions (e.g. memory allocation/deallocation) for the rt 63 | model that are called in the class constructir and destructor 64 | respectively. 65 | 66 | If the rt model can calculate adjoints, this is declared via the methods: 67 | 68 | rt_modeld, rt_modeldpre, rt_modeldpost, rt_modelpred 69 | 70 | where `rt_modeld` calculated the adjoint, `rt_modeldpre` is a method for setting up the mode (e.g. memory allocation) that is called after `rt_modelpre`, `rt_modeldpost` is called after `rt_modelpost`. 71 | 72 | `rt_modelpred` is normally equivalent to rt_model but may be called in its place if adjoints are used. 73 | 74 | ''' 75 | 76 | def H( self, x): 77 | ''' 78 | The forward model of the operator 79 | 80 | Load the state vector from self.state.x and calculate BRF for all 81 | observations and wavebands, either using a full bandpass function or 82 | a single wavelength (it's median), and maybe initialising the adjoint 83 | code if that's required. 84 | 85 | ''' 86 | 87 | self.state = x.reshape(self.x.state.shape) 88 | # initialise to 0 89 | self.linear.H = self.linear.H*0. 90 | if not self.isLoaded: 91 | # we only have to load these once 92 | try: 93 | self.doy = self.y.location[:,self.y_meta.location == 'time'] 94 | except: 95 | pass 96 | try: 97 | self.mask = self.y.control\ 98 | [:,self.y_meta.control=='mask'].astype(bool) 99 | self.vza = self.y.control[:,self.y_meta.control=='vza'] 100 | self.vaa = self.y.control[:,self.y_meta.control=='vaa'] 101 | self.sza = self.y.control[:,self.y_meta.control=='sza'] 102 | self.saa = self.y.control[:,self.y_meta.control=='saa'] 103 | self.isLoaded = True 104 | except: 105 | self.logger.error('error loading control information in %s'%\ 106 | 'Observation_Operator') 107 | self.logger.error(\ 108 | 'Check the configuration for mask,vza,sza,vaa,saa') 109 | raise Exception('error loading control information') 110 | 111 | # bands_to_use is of dimensions (nbands,nl) 112 | # and contains the normalised bandpass functions 113 | # All spectral information is contained in 114 | # self.y_meta.spectral self.rt_library.rt_modelpred 115 | # rt_modelpred 116 | for i_obs in np.where(self.mask)[0]: 117 | self.linear.H[i_obs] = self.model_reflectance(i_obs) 118 | return self.linear.H 119 | 120 | def J(self): 121 | ''' 122 | The cost function for the observation operator 123 | ''' 124 | x,Cx1,xshape,y,Cy1,yshape = self.getxy() 125 | self.Hx = self.H(x) 126 | diff = (y - self.Hx.flatten()) 127 | part1 = diff * Cy1 128 | self.linear.brf_ad = (-part1).reshape(self.Hx.shape) 129 | self.linear.J = 0.5*np.dot(part1,diff) 130 | return self.linear.J 131 | 132 | def J_prime_prime(self): 133 | ''' 134 | The second order differntial 135 | 136 | For this operator, this is independent for each observation 137 | so we can do it numerically, but changing each observation 138 | 139 | ''' 140 | x,Cx1,xshape,y,Cy1,yshape = self.getxy() 141 | J,J_prime0 = self.J_prime() 142 | #pdb.set_trace() 143 | # switch this off tmp if its on 144 | isLinear = self.isLinear 145 | self.isLinear = False 146 | # numerical calculations of H_prime 147 | # unsetting isLinear forces a calculation 148 | # at this x 149 | nparam = self.x.state.shape[-1] 150 | nb = self.y.state.shape[-1] 151 | ns = self.x.state.size / nparam 152 | #pdb.set_trace() 153 | H_prime = self.H_prime(x) #.reshape(nparam*ns,nb) 154 | 155 | # H_prime is dimensioned (yshape.prod(),ynparam) 156 | M = np.matrix(H_prime) 157 | C = np.matrix(np.diag(Cy1)) 158 | # JPP = M.T * C * M 159 | JPP = M * C * M.T 160 | # this should be of dimension nparam*ns ^2 161 | self.isLinear = isLinear 162 | return J,J_prime0,JPP 163 | 164 | def J_prime_full( self): 165 | ''' 166 | The derivative of the cost function for the observation 167 | operator. 168 | 169 | This is achieved with an adjoint if available 170 | ''' 171 | J = self.J() 172 | x,Cx1,xshape,y,Cy1,yshape = self.getxy() 173 | state = x.reshape(self.x.state.shape) 174 | for i_obs in np.where(self.mask)[0]: 175 | self.linear.J_prime[i_obs] = self.rt_library.rt_modeld(i_obs+1,\ 176 | state[i_obs], [self.vza[i_obs]], [self.vaa[i_obs]], \ 177 | self.sza[i_obs], self.saa[i_obs], self.linear.brf_ad[i_obs],\ 178 | self.y_meta.spectral.bands_to_use ) 179 | return J,self.linear.J_prime 180 | 181 | def preload_prepare(self): 182 | ''' 183 | Here , we use preload_prepare to make sure 184 | the x & any y data are NOT gridded for this 185 | operator. 186 | 187 | This method is called before any data are loaded, 188 | so ensures they are not loaded as a grid. 189 | ''' 190 | # mimic setting the apply_grid flag in options 191 | self.y_state.options.y.apply_grid = False 192 | # method J_prime_approx_1 is appropriate here 193 | self.J_prime_approx = self.J_prime_approx_1 194 | 195 | def postload_prepare(self): 196 | ''' 197 | This is called on initialisation, after data have been read in 198 | 199 | ''' 200 | try: 201 | self.rt_model = self.options.rt_model 202 | self.rt_class = self.rt_model.model 203 | self._load_rt_library ( rt_library=self.rt_class) 204 | except: 205 | raise Exception('rt library %s could not be loaded in %s'%\ 206 | (self.options.rt_model.model,'Observation_Operator')) 207 | try: 208 | self.setup_rt_model() 209 | except: 210 | raise Exception('Error initialising the rt model %s'%\ 211 | self.options.rt_model.model) 212 | 213 | def setup_rt_model ( self ): 214 | """ 215 | This sets up the RT model (and adjoint if available) 216 | by calling any preparation methods. 217 | 218 | """ 219 | if not 'linear' in self.dict(): 220 | self.linear = ParamStorage() 221 | if 'y' in self.dict(): 222 | self.linear.H = np.zeros(self.y.state.shape) 223 | else: 224 | self.linear.H = np.zeros(self.x.state.shape) 225 | self.nv = 1 226 | self.npt = len(self.y.state) 227 | 228 | self.linear.J_prime = np.zeros(self.x.state.shape) 229 | 230 | if not self.rt_model.use_median: 231 | self.bandIndex = self.y_meta.spectral.all_bands 232 | else: 233 | self.bandIndex = self.y_meta.spectral.median_bands 234 | self.linear.brf_ad = np.ones((self.npt,len(self.bandIndex))) 235 | 236 | self.rt_library.rt_modelpre(np.array(self.bandIndex) + 1 ) 237 | self.rt_library.rt_modeldpre (self.npt ) 238 | 239 | self.x_orig = self.x.state.copy() 240 | if self.rt_model.use_median: 241 | bands_to_use = self.y_meta.spectral.median_bands_to_use 242 | bandpass_library = self.y_meta.spectral.median_bandpass_library 243 | index = self.y_meta.spectral.median_bandpass_index 244 | else: 245 | bands_to_use = self.y_meta.spectral.bands_to_use 246 | bandpass_library = self.y_meta.spectral.bandpass_library 247 | index = self.y_meta.spectral.bandpass_index 248 | 249 | self.y_meta.spectral.bands_to_use = \ 250 | np.zeros((len(bands_to_use),len(self.bandIndex))) 251 | for (i,bandname) in enumerate(self.y_meta.spectral.bandnames): 252 | fullb = bandpass_library[bandname] 253 | this = fullb[index[bandname]] 254 | this = this/this.sum() 255 | ww = np.where(np.in1d(self.bandIndex,index[bandname]))[0] 256 | self.y_meta.spectral.bands_to_use[i,ww] = this 257 | 258 | _nowt = lambda self, *args : None 259 | 260 | def _load_rt_library ( self, rt_library ): 261 | """ 262 | A method that loads up the compiled RT library code and sets some 263 | configuration options. This method tries to import all the methods and 264 | make them available through `self.rt_library.`. It is also 265 | a safe importer: if some functions are not available in the library, 266 | it will provide safe methods from them. Additionally, while importing, 267 | a number of configuration options in the class are also updated or set 268 | to default values. 269 | 270 | Parameters 271 | ----------- 272 | rt_library : string 273 | This is the name of the library object (.so file) that will be 274 | loaded 275 | 276 | """ 277 | from eoldas_Lib import sortopt 278 | import_string = "from %s import " % ( rt_library ) 279 | self.logger.debug ("Using %s..." % rt_library ) 280 | 281 | self.rt_library = sortopt(self,'rt_library',ParamStorage()) 282 | # 1. Import main functionality 283 | try: 284 | self.logger.debug ("Loading rt_model") 285 | exec ( import_string + "rt_model" ) 286 | self.rt_library.rt_model = rt_model 287 | except ImportError: 288 | self.logger.info(\ 289 | "Could not import basic RT functionality: rt_model") 290 | self.logger.info(\ 291 | "Check library paths, and whether %s.so is available" % \ 292 | rt_library) 293 | raise Exception('error importing library %s'%rt_library) 294 | 295 | # 1a. Import conditioning methods that can be ignored 296 | try: 297 | exec ( import_string + 'rt_modelpre' ) 298 | self.rt_library.rt_modelpre = rt_modelpre 299 | except: 300 | self.rt_library.rt_modelpre = self._nowt 301 | try: 302 | exec ( import_string + 'rt_modelpre' ) 303 | self.rt_library.rt_modelpost = rt_modelpre 304 | except: 305 | self.rt_library.rt_modelpost = self._nowt 306 | 307 | # 2. Try to import derivative 308 | self.rt_model.ignore_derivative = sortopt(self.rt_model,\ 309 | 'ignore_derivative',False) 310 | 311 | if self.rt_model.ignore_derivative == False: 312 | try: 313 | exec ( import_string + \ 314 | "rt_modeld, rt_modeldpre, rt_modeldpost" + \ 315 | ", rt_modelpred" ) 316 | self.rt_model.have_adjoint = True 317 | self.J_prime = self.J_prime_full 318 | self.rt_library.rt_modeld = rt_modeld 319 | self.rt_library.rt_modeldpre = rt_modeldpre 320 | self.rt_library.rt_modeldpost = rt_modeldpost 321 | self.rt_library.rt_modelpred = rt_modelpred 322 | except ImportError: 323 | self.logger.info("No adjoint. Using finite differences approximation.") 324 | self.rt_model.have_adjoint = False 325 | else: 326 | self.logger.info("ignoring adjoint. Using finite differences approximation.") 327 | self.rt_model.have_adjoint = False 328 | 329 | self._configure_adjoint () 330 | 331 | try: 332 | exec( import_string + "rt_model_deriv") 333 | self.rt_library.rt_model_deriv = rt_model_deriv 334 | except ImportError: 335 | self.rt_library.rt_model_deriv = None 336 | 337 | 338 | def _configure_adjoint ( self ): 339 | """ 340 | A method to configure the adjoint code, whether the _true_ adjoint or 341 | its numerical approximation is used. 342 | """ 343 | if not self.rt_model.have_adjoint: 344 | self.rt_library.rt_modeld = None 345 | self.J_prime = self.J_prime_approx_1 346 | self.rt_library.rt_modeldpre = lambda x: None 347 | self.rt_library.rt_modeldpost = lambda : None 348 | self.rt_library.rt_modelpred = self.rt_library.rt_model 349 | 350 | model_reflectance = lambda self,i_obs : self.rt_library.rt_modelpred( i_obs+1, \ 351 | self.state[i_obs], [self.vza[i_obs]], [self.vaa[i_obs]], \ 352 | self.sza[i_obs], self.saa[i_obs],self.y_meta.spectral.bands_to_use ) 353 | model_reflectance.__name__ = 'model_reflectance' 354 | model_reflectance.__doc__ = """ 355 | Calculate the cost of a single observation\ 356 | 357 | i_obs is the observation index (1-based) 358 | 359 | It is in this function that we interface with 360 | any rt libraries 361 | 362 | """ 363 | 364 | def hessian(self,use_median=True,epsilon=1.e-15,linear_approx=False): 365 | """ 366 | observation operator hessian 367 | """ 368 | self.logger.info ("Starting Hessian ") 369 | nparams = self.setup.config.params.n_params 370 | if not hasattr ( self, "hessian_m"): 371 | self.hessian_m = np.zeros([self.setup.grid_n_obs*self.setup.grid_n_params,self.setup.grid_n_obs*self.setup.grid_n_params]) 372 | else: 373 | self.hessian_m[:,:] = 0 374 | for i_obs in xrange( self.obs.npt ): 375 | nbands = self.obs.nbands[i_obs] 376 | if self.obs.qa[i_obs] > 0: 377 | # get the parameters (full set) 378 | x = self.setup.store_params[self.setup.obs_shift[i_obs],:] 379 | # form a mask of which of these vary 380 | mask = (self.setup.fix_params[self.setup.obs_shift[i_obs], :] == 1) + (self.setup.fix_params[self.setup.obs_shift[i_obs], :] == 2) 381 | dobs_dx = np.matrix(self.dobs_dx_single_obs (x[mask], i_obs, use_median=use_median, epsilon=epsilon,linear_approx=linear_approx)) 382 | # this is the same as self.hprime[i_obs,:.nbands,mask] 383 | # so it has dimensions [nbands,n_params] 384 | # The call to self.dobs_dx_single_obs also calculates the brf for that sample, in 385 | # self.store_brf[i_obs, :nbands] 386 | thism = dobs_dx * self.obs.real_obsinvcovar[i_obs] * dobs_dx.T 387 | #thism = dobs_dx * self.obs.obsinvcovar[i_obs] * dobs_dx.T 388 | # try to put in the second order term 389 | #try: 390 | # thatm = self.obs.real_obsinvcovar[i_obs] * np.matrix(self.h_prime_prime[i_obs,:nbands,mask]).T 391 | # diff = np.matrix(self.obs.observations[ i_obs, :nbands] - self.store_brf[i_obs, :nbands]) 392 | # thism = thism - diff * thatm 393 | #except: 394 | # pass 395 | # thism is a n_params x n_params matrix 396 | # pack it into the big one 397 | # so, which time does this obs belong to? 398 | this = self.setup.obs_shift[i_obs] 399 | nv = thism.shape[0] 400 | self.hessian_m[this*nv:(this+1)*nv,this*nv:(this+1)*nv] = self.hessian_m[this*nv:(this+1)*nv,this*nv:(this+1)*nv] + thism 401 | oksamps = np.where(self.setup.fix_params.flatten()>0)[0] 402 | xx = self.hessian_m[oksamps] 403 | return xx[:,oksamps] 404 | 405 | def set_crossval ( self, leave_out = 0.2 ): 406 | """ 407 | This method allows to set a cross validation strategy in the data by 408 | selecting a fraction of the samples that can be left out from those 409 | that have a good QA. This is expressed as a fraction, by default we 410 | use 20%. The location of these samples is random. 411 | 412 | Parameters 413 | ------------ 414 | leave_out : float 415 | The fraction of samples to "leave out" 416 | """ 417 | from random import sample 418 | ndoys = self.obs.qa.sum() 419 | leave_out_xval = int ( np.ceil(ndoys*leave_out) ) 420 | self.logger.info ("Cross validation: Leaving %d samples out" % \ 421 | leave_out_xval ) 422 | xval_locs = np.array([np.where ( self.obs.doys==i) \ 423 | for i in sample(self.obs.doys[ self.obs.qa==1], \ 424 | leave_out_xval)]).squeeze() 425 | self.xval[ xval_locs ] = -1 426 | 427 | 428 | def set_crossval_single ( self, xval_loc ): 429 | """ 430 | This method allows to set a cross validation strategy in the data by 431 | selecting a single observation to be "left out". Looping over all the 432 | observations is required for Generalised cross validation strategies. 433 | 434 | Parameters 435 | ----------- 436 | xval_loc : integer 437 | location of the sample left out. 438 | """ 439 | self.logger.info ("Cross validation: Leaving %d-th sample out" % \ 440 | xval_loc ) 441 | self.xval = self.obs.qa.copy() 442 | self.xval[ xval_loc ] = -1 443 | 444 | 445 | 446 | def calculate_crossval_err ( self ): 447 | """ 448 | Calcualte the crossvalidation error. Once the DA process has completed, 449 | this method calculates the mismatch between the observations that were 450 | left out of the DA optimisation and their estimates. They are weighted 451 | by the observational uncertainty in each case. 452 | 453 | """ 454 | xval_err = 0.0 455 | for i in xrange( self.obs.npt ): 456 | if self.xval[i] == -1: 457 | difference = self.obs.observations[ i, :self.obs.nbands[i] ] - \ 458 | self.rt.brf[ i, :self.obs.nbands[i] ] 459 | inv_obs_covar = self.obs.obsinvcovar[i] 460 | part1 = np.dot ( inv_obs_covar, difference ) 461 | xval_err += np.dot( part1, difference ) 462 | 463 | for j in xrange ( self.obs.nbands[i] ): 464 | info_str = "Doy: %d Obs #%d: " % ( self.obs.doys[i], i ) 465 | info_str = info_str + "B%d Obs: %g Model: %g Dif: %g" % \ 466 | ( j+1, self.obs.observations[ i, j ], \ 467 | self.rt.brf [i,j], self.obs.observations[ i, j ] - \ 468 | self.rt.brf [i,j] ) 469 | self.logger.info ( info_str ) 470 | return ( xval_err, self.params_x, self.rt.brf, self.obs.observations ) 471 | 472 | 473 | -------------------------------------------------------------------------------- /eoldas/eoldas_Spectral.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import pdb 3 | import numpy as np 4 | #from eoldas_model import subset_vector_matrix 5 | from eoldas_State import State 6 | from eoldas_ParamStorage import ParamStorage 7 | #from eoldas_Operator import * 8 | import logging 9 | 10 | class Spectral ( ParamStorage ): 11 | ''' 12 | The purpose of this class is to perform operations on 13 | spectral data, such as waveband selection, spectral library loading etc. 14 | 15 | The following methods are available: 16 | 17 | 1. normalise_wavebands (self.bandnames) 18 | 2. load_bandpass_from_pulses ( wavelengths, bandwidth, nlw, 19 | bandpass_library={}, spectral_interval=1 ) 20 | 3. load_bandpass_library ( bandpass_filename, 21 | bandpass_library={}, pad_edges=True ) 22 | 4. setup_spectral_config ( self, wavelengths, nbands ) 23 | ''' 24 | 25 | def __init__(self,options,name=None): 26 | ''' 27 | The class initialiser 28 | 29 | This sets up the problem, i.e. loads the spectral information from 30 | an options structure. 31 | 32 | The options structure will contain: 33 | 34 | y_meta.names: 35 | -------------------- 36 | The names of the state vector elements for a y State as string list. 37 | These are interpreted as wavebands associated with the y state. 38 | They may be: 39 | 1. the names of bandpass functions e.g. MODIS-b2 40 | 2. Wavelength intervals e.g. "450-550: 41 | 3. Single wavelength e.g. 451 42 | In the case of 2 or 3, we interpret the wavelengths 43 | directly from the string. For defined bandpass functions 44 | we look in a library to interpret the functions. 45 | 46 | 47 | Optional: 48 | options.operator_spectral 49 | -------------------- 50 | (lmin,lmax,lstep) for the operator. If this is *not* specified 51 | then the model is assumed to be 'non spectral' and a flag 52 | self.is_spectral is set False. In this case, we form a new set 53 | of state variables to solve for and don't bother setting up 54 | all of the bandpass information. This means that we have to inform 55 | the parameters operator to set up a new set of state vectors. 56 | These are based on the bandpass names self.bandnames and 57 | the names in options.x_meta.names. 58 | 59 | options.datadir 60 | -------------------- 61 | Data directory for bandpass files 62 | 63 | options.bandpass_libraries 64 | -------------------- 65 | A list of filenames containing band pass libraries 66 | 67 | options.bandpass_library 68 | -------------------- 69 | A dictionary of existing bandpass functions 70 | 71 | This makes calls to: 72 | self.setup_spectral_config() 73 | self.load_bandpass_library() 74 | self.load_bandpass_from_pulses() 75 | self.normalise_wavebands() 76 | 77 | 78 | ''' 79 | from eoldas_Lib import isfloat,sortopt 80 | if name == None: 81 | import time 82 | thistime = str(time.time()) 83 | name = type(self).__name__ 84 | name = "%s.%s" % (name,thistime) 85 | self.thisname = name 86 | 87 | self.options = options 88 | self.logger = logging.getLogger (self.thisname+'.spectral') 89 | # set self.nlw, self.bandnames & associated size terms 90 | # we need this to discretise the bandpass functions 91 | self.setup_spectral_config(self.options.y_meta.state) 92 | 93 | self.is_spectral=sortopt(options.general,'is_spectral',True) 94 | 95 | if not self.is_spectral: 96 | self.logger.info("Non spectral operator specified") 97 | return 98 | 99 | self.logger.info("Spectral operator specified") 100 | if 'datadir' in self.options.dict(): 101 | self.datadir = self.options.datadir 102 | else: 103 | self.datadir = ['.','~/.eoldas','..'] 104 | self.logger.info("Data directories: %s"%str(self.datadir)) 105 | if 'bandpass_library' in self.options.dict(): 106 | self.logger.info("Receiving existing bandpass library") 107 | self.bandpass_library = self.options.bandpass_library 108 | else: 109 | self.logger.info("Starting new bandpass library") 110 | self.bandpass_library={} 111 | if 'bandpass_libraries' in self.options.dict(): 112 | self.bandpass_libraries = self.options.bandpass_libraries 113 | for i in self.bandpass_libraries: 114 | self.logger.info("Attempting to load bandpass library %s"%i) 115 | self.bandpass_library,is_error = \ 116 | self.load_bandpass_library(i,self.bandpass_library) 117 | if is_error[0]: 118 | self.logger.error\ 119 | ('Cannot load file %s loaded into bandpass library'%i) 120 | self.logger.error(is_error[1]) 121 | else: 122 | self.logger.info("No bandpass libraries specified") 123 | 124 | self.lstep = sortopt(self,'lstep',1) 125 | self.nlw = sortopt(self,'nlw',None) 126 | 127 | # if you havent defined the wavelength range, use 0-10000 128 | if self.nlw == None: 129 | self.logger.info("No wavelength bounds defined") 130 | self.logger.info("setting as [0,10000,1]") 131 | self.nlw = np.arange(10000) 132 | self.pseudowavelengths = [] 133 | # Now we can start to load the band names 134 | for i in self.bandnames: 135 | # check to see if its in the library: 136 | if not i in self.bandpass_library: 137 | # try to interpret it 138 | ok,this = isfloat(i) 139 | if ok: 140 | self.bandpass_library, bandpass_name = \ 141 | self.load_bandpass_from_pulses(str(i),\ 142 | this,0.5*self.lstep,self.nlw,\ 143 | self.bandpass_library,\ 144 | self.lstep) 145 | else: 146 | # Try splitting it 147 | that = i.split('-') 148 | if len(that) == 2 and isfloat(that[0])[0] \ 149 | and isfloat(that[1])[0]: 150 | f0 = float(that[0]) 151 | f1 = float(that[1]) 152 | self.bandpass_library, bandpass_name = \ 153 | self.load_bandpass_from_pulses(i,\ 154 | 0.5*(f1+f0),0.5*(f1-f0),self.nlw,\ 155 | self.bandpass_library,\ 156 | self.lstep) 157 | else: 158 | f0 = len(self.pseudowavelengths) + self.nlw[0] 159 | bandpass_name = i 160 | self.logger.info("******************************************************") 161 | self.logger.info("requested waveband %s is not in the spectral library"%i) 162 | self.logger.info("and cannot be interpreted as a wavelength or wavelength range") 163 | self.logger.info("so we will set it up as a pseudowavelength and hope that") 164 | self.logger.info("any operator you use does not need to interpret its value") 165 | self.logger.info("for reference:") 166 | self.logger.info(" band %s"%i) 167 | self.logger.info("is interpreted here as:") 168 | self.bandpass_library, dummy = \ 169 | self.load_bandpass_from_pulses(str(f0),\ 170 | f0,self.lstep*0.5,self.nlw,\ 171 | self.bandpass_library,\ 172 | self.lstep) 173 | self.logger.info(" %d nm"%f0) 174 | self.logger.info("******************************************************") 175 | self.pseudowavelengths.append(i) 176 | try: 177 | self.bandpass_library[i] = self.bandpass_library[dummy] 178 | except: 179 | self.bandpass_library[i] = self.bandpass_library[str(f0)] 180 | # Now post-process the bandpass library 181 | self.normalise_wavebands(self.bandnames) 182 | 183 | def load_bandpass_library ( bandpass_filename, bandpass_library={}, \ 184 | pad_edges=True ): 185 | """Loads bandpass files. 186 | 187 | Loads bandpass files in ASCII format. The data are read from 188 | `bandpass_filename`, the spectral range of interest is `nlw` 189 | (usually in nm) and if you have already a dictionary with named 190 | bandpass functions, you can give it as `bandpass_library`. 191 | 192 | Note that bands names that are already present 193 | in the library will be ignored. If the bandpass functions aren't 194 | padded to 0 at the edges of the band, the `pad_edges` option will 195 | set them to 0 at the edges. Otherwise, the interpolation goes a 196 | bit crazy. 197 | 198 | A bandpass file contains something like: 199 | 200 | [begin MODIS-b2] 201 | 450 0.0 202 | 550 1.0 203 | 650 2.0 204 | 750 1.0 205 | 850 0.0 206 | [end MODIS-b2] 207 | 208 | Npte that it doesn't need to be normalised when defined. 209 | 210 | """ 211 | from eoldas_Lib import get_filename 212 | fname,fname_err = get_filename(bandpass_filename,datadir=self.datadir) 213 | if fname_err[0] == 0: 214 | bp_fp = open ( fname, 'r' ) 215 | else: 216 | return bandpass_library,fname_err 217 | 218 | self.logger.info('file %s loaded into bandpass library'%fname) 219 | 220 | nlw = self.nlw 221 | 222 | data = bp_fp.readlines() 223 | bp_fp.close() 224 | bands = {} 225 | for ( i, dataline ) in enumerate ( data ): 226 | 227 | if dataline.find ( "[begin" ) >= 0: 228 | name_start = dataline.replace("[begin", "").strip().\ 229 | replace("]", "") 230 | bands[ name_start ] = [] 231 | 232 | elif dataline.find ( "[end" ) >= 0: 233 | name_end = dataline.replace("[end", "").strip().replace("]", "") 234 | # Make sure the file is consistent 235 | if name_end != name_start: 236 | fname_err[0] = True 237 | fname_err[1] = \ 238 | "Inconsistent '[begin ]' and " + \ 239 | "'[end]' names (%s != %s) at line %d" \ 240 | % (name_start, name_end, i+1 ) 241 | return bandpass_library,fname_err 242 | name_start = None 243 | else: 244 | ( x, y ) = dataline.strip().split() 245 | # Double check ehtat we have a good band name 246 | if name_start is not None: 247 | fname_err[0] = True 248 | fname_err[1] = \ 249 | "Bandpass file appears corrupt " + \ 250 | "at line " % (i+1) 251 | return bandpass_library,fname_err 252 | bands[name_start].append ( [float(x), float(y)] ) 253 | 254 | for ( band_name, filter_fncn ) in bands.iteritems() : 255 | if not bandpass_library.has_key( band_name ): 256 | filter_fncn = np.array ( filter_fncn ) 257 | if pad_edges: 258 | bandpass_library [ band_name ] = np.interp ( nlw, \ 259 | filter_fncn[:,0], filter_fncn[:, 1], left=0.0, right=0.0) 260 | 261 | else: 262 | bandpass_library [ band_name ] = np.interp ( nlw, \ 263 | filter_fncn[:,0], filter_fncn[:, 1] ) 264 | return bandpass_library,True 265 | 266 | 267 | def setup_spectral_config ( self, bandnames ): 268 | """ 269 | Observation operators that consider spectral information 270 | store sample spectra (e.g. of chlorophyll absoption) at some given 271 | spectral interval and over some defined range. 272 | 273 | To work with the operator, we need to sample out spectral 274 | information to this grid. 275 | 276 | We assume that the operator spectral limits are defined in: 277 | 278 | options.operator_spectral = (min,max,step) 279 | 280 | The strings with the waveband names are contained in the 281 | list (or array) bandnames. 282 | 283 | This method sets up: 284 | self.nlw : sampled wavelengths, e.g. [400.,401., ...2500.] 285 | self.bandnames 286 | : bandnames e.g. ['451','MODIS-b2'] 287 | 288 | some size stores: 289 | self.nbands : len(self.bandnames) 290 | self.lmin : min(self.nlw) 291 | self.lmax : max(self.nlw) 292 | self.lstep : self.nlw[1]-self.nlw[0] 293 | self.nl : len(self.nlw) 294 | 295 | """ 296 | self.bandnames = np.atleast_1d(bandnames) 297 | self.nbands = len(self.bandnames) 298 | try: 299 | if not 'bounds' in self.options.options.rt_model.dict().keys(): 300 | self.is_spectral = False 301 | return 302 | else: 303 | self.is_spectral = True 304 | except: 305 | self.is_spectral = False 306 | return 307 | self.lmin = self.options.options.rt_model.bounds[0] 308 | self.lmax = self.options.options.rt_model.bounds[1] 309 | self.lstep = self.options.options.rt_model.bounds[2] 310 | 311 | # In case we want to set up bandpass functions, we need to know how 312 | # the spectral functions are sampled 313 | # nl is the number of samples. Usually 2101 :) 314 | self.nl = int((self.lmax - self.lmin + 1 )/self.lstep + 0.5 ) 315 | 316 | # nlw is the sampled wavelength of each band. 317 | #Usually ranges from 400 to 2500nm 318 | self.nlw = np.arange(self.nl) * self.lstep + self.lmin 319 | 320 | medianbad = lambda self,w,f : w[np.where(np.abs(np.cumsum(f)/f.sum() - 0.5)\ 321 | == np.min(np.abs(np.cumsum(f)/f.sum() - 0.5)))[0][0]] 322 | 323 | median = lambda self,w,f : np.where(np.min(np.abs((w*f).sum() - w)) == np.abs((w*f).sum() - w))[0][0] 324 | 325 | def normalise_wavebands (self,bandnames): 326 | """Process and normalise wavebands, as well as select wv flags 327 | 328 | Parameters 329 | ------------ 330 | 331 | bandnames : array-like 332 | Bandpass names that should be in self.bandpass_library. 333 | 334 | 335 | Outputs 336 | ------------ 337 | 338 | This method sets up, for each k in bandnames: 339 | 340 | self.bandpass_library[k]: 341 | Normalised bandpass response 342 | 343 | self.median_bandpass_library[k]: 344 | Normalised median bandpass response 345 | 346 | self.median_bandpass_index[k]: 347 | The index of the median wavelength 348 | 349 | self.bandpass_index[k]: 350 | The indices of the bandpass samples 351 | 352 | self.all_bands: 353 | Set to True if a particular wavelength is used 354 | 355 | self.median_bands: 356 | Set to True if a particular wavelength sample is used for the median 357 | 358 | How to use 359 | ------------ 360 | We use this information to request that an observation operator 361 | only calculates terms for the particular wavelengths we need. 362 | To do that, it requires an array of flags specifying which 363 | bands to use. That is contained in the mask self.all_bands (or 364 | self.median_bands). The observation operator then only calculates 365 | e.g. reflectance in these bands, so we can load to the full 366 | spectral array array sp_data with information returned by the 367 | operator, data with: 368 | 369 | sp_data[self.all_bands] = data 370 | 371 | The integral over a waveband then is: 372 | 373 | for (i,k) in enumerate(bandnames): 374 | refl[i] = np.dot(sp_data,self.bandpass_library[k]) 375 | median_refl[i] = np.dot(sp_data,self.median_bandpass_library[k]) 376 | 377 | Or, a slightly faster access if the arrays are large: 378 | 379 | for (i,k) in enumerate(bandnames): 380 | ww = self.bandpass_index[k] 381 | refl[i] = np.dot(sp_data[ww],self.bandpass_library[k][ww]) 382 | 383 | """ 384 | self.median_bandpass_library = {} 385 | self.median_bandpass_index = {} 386 | self.bandpass_index = {} 387 | self.all_bands = np.zeros_like(self.nlw).astype(float) 388 | self.median_bands = np.zeros_like(self.nlw).astype(int) 389 | self.bands_to_use = [] 390 | self.median_bands_to_use = [] 391 | for k in bandnames: 392 | self.logger.info('Loading band %s'%str(k)) 393 | try: 394 | v = self.bandpass_library[k] 395 | except: 396 | # some confusion wrt strings and numbers so fix it 397 | v = self.bandpass_library[str(k)] 398 | self.bandpass_library[k] = v 399 | # normalise 400 | self.bandpass_library[k] = v/v.sum() 401 | v = self.bandpass_library[k] 402 | # find median 403 | median = self.median(self.nlw,v) 404 | # specify the index of the median 405 | self.median_bandpass_index[k] = median 406 | self.median_bandpass_library[k] = 0.*v 407 | self.median_bandpass_library[k][self.median_bandpass_index[k]] = 1 408 | self.median_bands[self.median_bandpass_index[k]] = 1 409 | # specify the indices of the full bandpass 410 | self.bandpass_index[k] = np.where(self.bandpass_library[k]>0)[0] 411 | self.all_bands = self.all_bands + self.bandpass_library[k] 412 | self.bands_to_use.append(self.bandpass_library[k]) 413 | self.median_bands_to_use.append(self.median_bands) 414 | 415 | ww = np.where(self.all_bands > 0)[0] 416 | self.all_bands = ww 417 | ww = np.where(self.median_bands > 0)[0] 418 | self.median_bands = ww 419 | self.bands_to_use = np.array(self.bands_to_use) 420 | self.median_bands_to_use = np.array(self.median_bands_to_use) 421 | 422 | 423 | 424 | 425 | def load_bandpass_from_pulses ( self,thisname,wavelengths, bandwidth, nlw, \ 426 | bandpass_library,spectral_interval ): 427 | """Bandpass functions from center and bandwidth 428 | 429 | This function calculates bandpass functions from the centre 430 | wavelength and the bandwdith specified by the user. 431 | 432 | The results are stored in a spectral (the configuration storage 433 | for all things spectral!). This function returns a boxcar-type 434 | spectral passband function, with edges specified by half 435 | the bandwidth. 436 | 437 | Parameters 438 | ---------- 439 | bandwidth: array-like 440 | wavelengths: array-like 441 | spectral: ParamStorage 442 | 443 | 444 | """ 445 | wavelengths = np.array(wavelengths) 446 | bandmin = np.array(wavelengths - bandwidth) 447 | bandmax = np.array(wavelengths + bandwidth) 448 | if bandmin.size ==1 and bandmin < 0: 449 | bandmin = 0 450 | elif bandmin.size >1: 451 | ww = np.where(bandmin<0) 452 | bandmin[ww] = 0 453 | bandpass_names = [] 454 | for indice, band_min in np.ndenumerate ( bandmin ): 455 | if band_min == bandmax[ indice ]: # NULL wavelength 456 | bandpass_names.append ( "NULL" ) 457 | else: 458 | bandpass_names.append ( "%f-%f" % \ 459 | ( band_min, bandmax[indice] ) ) 460 | 461 | 462 | magnitude = np.array ( [0, 1, 1, 0] ) 463 | for (i, bandpass_name ) in enumerate ( bandpass_names ): 464 | if bandpass_name != "NULL" and \ 465 | (not bandpass_library.has_key ( bandpass_name ) ): 466 | limits = [float(wv) for wv in bandpass_name.split("-")] 467 | mini = limits[0] 468 | maxi = limits[1] 469 | x = np.array ( [ mini - spectral_interval*0.5, mini, \ 470 | maxi, maxi + spectral_interval*0.5] ) 471 | xx = np.interp \ 472 | ( nlw, x, magnitude ) 473 | bandpass_library[thisname] = xx 474 | bandpass_library[bandpass_name] = xx 475 | return bandpass_library,bandpass_name 476 | 477 | 478 | if __name__ == "__main__": 479 | demonstration() 480 | 481 | 482 | 483 | -------------------------------------------------------------------------------- /eoldas/eoldas_DModel_Operator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import pdb 3 | import numpy as np 4 | #from eoldas_model import subset_vector_matrix 5 | from eoldas_State import State 6 | from eoldas_ParamStorage import ParamStorage 7 | from eoldas_Operator import * 8 | 9 | class DModel_Operator ( Operator ): 10 | 11 | def preload_prepare(self): 12 | ''' 13 | Here , we use preload_prepare to make sure 14 | the x & any y data are gridded for this 15 | operator. This greatly simplifies the 16 | application of the differential operator. 17 | 18 | This method is called before any data are loaded, 19 | so ensures they are loaded as a grid. 20 | ''' 21 | from eoldas_Lib import sortopt 22 | for i in np.array(self.options.datatypes).flatten(): 23 | # mimic setting the apply_grid flag in options 24 | if self.dict().has_key('%s_state'%i): 25 | self['%s_state'%i].options[i].apply_grid = True 26 | self.novar = sortopt(self,'novar',False) 27 | self.gamma_col = sortopt(self,'gamma_col',None) 28 | self.beenHere =False 29 | 30 | def postload_prepare(self): 31 | ''' 32 | This is called on initialisation, after data have been read in 33 | 34 | Here, we load parameters specifically associated with the 35 | model H(x). 36 | 37 | In the case of this differential operator, there are: 38 | 39 | model_order : order of the differential operator 40 | (integer) 41 | wraparound : edge conditions 42 | Can be: 43 | periodic 44 | none 45 | reflexive 46 | lag : The (time/space) lag at which 47 | the finite difference is calculated 48 | in the differential operator here. 49 | If this is 1, then we take the difference 50 | between each sample point and its neighbour. 51 | This is what we normally use. The main 52 | purpose of this mechanism is to allow 53 | differences at multiple lags to be 54 | calculated (fuller autocorrelation function 55 | constraints as in kriging) 56 | 57 | Multiple lags can be specified (which you could 58 | use to perform kriging), in which case lag 59 | weight should also be specified. 60 | 61 | lag_weight : The weight associated with each lag. This will 62 | generally be decreasing with increasing lag for 63 | a 'usual' autocorrelation function. There is no point 64 | specifying this if only a single lag is specified 65 | as the function is normalised. 66 | 67 | If the conditions are specified as periodic 68 | the period of the function can also be specified, e.g. 69 | for time varying data, you could specify 365 for the 70 | periodic period. 71 | 72 | These are specified in the configuration file as 73 | 74 | operator.modelt.rt_model.model_order 75 | operator.modelt.rt_model.wraparound 76 | operator.modelt.rt_model.lag 77 | operator.modelt.rt_model.lag_weight 78 | 79 | The default values (set here) are 1, 'none', 1 and 1 respectively. 80 | 81 | To specify the period for `periodic` specify e.g.: 82 | 83 | [operator.modelt.rt_model] 84 | wraparound=periodic,365 85 | 86 | The default period is set to 0, which implies that it is periodic 87 | on whatever the data extent is. 88 | 89 | Or for multiple lags: 90 | 91 | [operator.modelt.rt_model] 92 | lag=1,2,3,4,5 93 | lag_weight=1,0.7,0.5,0.35,0.2 94 | 95 | NB this lag mechanism has not yet been fully tested 96 | and should be used with caution. It is intended more 97 | as a placeholder for future developments. 98 | 99 | Finally, we can also decide to work with 100 | inverse gamma (i.e. an uncertainty-based measure) 101 | 102 | This is achieved by setting the flag 103 | 104 | operator.modelt.rt_model.inverse_gamma=True 105 | 106 | This flag should be set if you intend to estimate gamma in the 107 | Data Assimilation. Again, the is experimental and should be used 108 | with caution. 109 | 110 | ''' 111 | from eoldas_Lib import sortopt 112 | self.rt_model = sortopt(self.options,'rt_model',ParamStorage()) 113 | self.rt_model.lag = sortopt(self.rt_model,'lag',1) 114 | self.rt_model.inverse_gamma= \ 115 | sortopt(self.rt_model,'inverse_gamma',False) 116 | self.rt_model.model_order = \ 117 | sortopt(self.rt_model,'model_order',1) 118 | self.rt_model.wraparound = \ 119 | sortopt(self.rt_model,'wraparound','none') 120 | self.rt_model.wraparound_mod = 0 121 | if np.array(self.rt_model.wraparound).size == 2 and \ 122 | np.array(self.rt_model.wraparound)[0] == 'periodic': 123 | self.rt_model.wraparound_mod = \ 124 | np.array(self.rt_model.wraparound)[1] 125 | self.rt_model.wraparound = \ 126 | np.array(self.rt_model.wraparound)[0] 127 | self.rt_model.lag = \ 128 | sortopt(self.rt_model,'lag',[1]) 129 | self.rt_model.lag = np.array(self.rt_model.lag).flatten() 130 | 131 | self.rt_model.lag_weight = \ 132 | sortopt(self.rt_model,'lag_weight',[1.]*\ 133 | self.rt_model.lag.size) 134 | self.rt_model.lag_weight = np.array(\ 135 | self.rt_model.lag_weight).flatten().astype(float) 136 | 137 | if self.rt_model.lag_weight.sum() == 0: 138 | self.rt_model.lag_weight[:] = np.ones(self.rt_model.lag_weight.size) 139 | 140 | self.rt_model.lag_weight = self.rt_model.lag_weight\ 141 | / self.rt_model.lag_weight.sum() 142 | 143 | def setH(self): 144 | ''' 145 | This method sets up the matrices required for the model. 146 | 147 | This operator is written so that it can apply smoothing 148 | in different dimensions. This is controlled by the model 149 | state vector. 150 | 151 | The names of the states are stored in self.x_meta.location 152 | and the associated location information in self.x_meta.location. 153 | So, we look through these looking for matches, e.g. 'row' in 154 | location and 'gamma_row' in names would mean that we want 155 | to apply the model over the row dimension. There should be only 156 | one gamma term in the state vectors for this operator. If you 157 | give more than one, only the last one will be used. 158 | 159 | NOT YET IMPLEMENTED: 160 | The model can be applied to multiple dimensions by specifying 161 | e.g. gamma_time_row. If you want separate gammas for e.g. 162 | time and row, then you should use separate operators. If 163 | gamma_roccol is specified, then the model applies to 164 | Euclidean distance in row/col space. 165 | 166 | Formally, the problem can be stated most simply as a matrix 167 | D so that gamma D x is the rate of change of x with respect 168 | to the target location variable (time, row, col etc). 169 | The job of this method then is to form and store D. 170 | 171 | The main complication to this is we have to split up x into 172 | those terms that we will apply D to (x2 here) and separately 173 | pull out the gamma terms. The resultant matrix D then needs to 174 | be re-formed so as to apply to the whole vector x, rather than 175 | just x2. We do this with masks. 176 | 177 | On input, x is a 1D vector. 178 | 179 | ''' 180 | x,Cx1,xshape,y,Cy1,yshape = self.getxy() 181 | # the names of the variables in x 182 | names = np.array(self.x_meta.state) 183 | # the names of the location information (e.g. time, row, col) 184 | location = self.x_meta.location 185 | self.logger.info('Setting up model matrices...') 186 | 187 | if self.x_meta.is_grid: 188 | try: 189 | self.x.location = self.x_state.ungridder.location 190 | self.x.qlocation = self.x_state.ungridder.qlocation 191 | except: 192 | raise Exception("You are trying to ungrid a dataset that wasn't gridded using State.regrid()" +\ 193 | " so the ungridder information is not available. Either load the data using State.grid " +\ 194 | " or set it up some other way or avoid calling this method with this type of data") 195 | 196 | # first, reshape x from its 1-D form to 197 | # have the same shape as self.x.state. We store 198 | # this shape as xshape. 199 | xshape = self.x.state.shape 200 | 201 | # we can't change the tuple directly, so need a 202 | # vector representation that we can manipulate 203 | # This is xshaper 204 | xshaper = np.array(xshape) 205 | 206 | # the data are assumed loaded into x 207 | 208 | # At this point, x2 is just a copy of the full input vector x 209 | # mask then is a mask of the same size as self.x_meta.state 210 | # by deafult, this mask is True. We will modify it to 211 | # take out bits we dont want later. 212 | x2 = x.reshape(xshape) 213 | mask = np.ones_like(x2).astype(bool) 214 | 215 | # We now need to recognise any gamma terms that might be in 216 | # the state vector. Candidates are 'gamma_%s'%(location) 217 | # e.g. gamma_time. 218 | 219 | # The number of dimensions of x can vary, depending on how many 220 | # loaction terms are used, so its a little tricky to 221 | # pull the information out. 222 | # We loop over the locations, indexed as i 223 | self.linear.datamask = np.ones(xshape[-1]).astype(bool) 224 | for i in xrange(len(location)): 225 | # and form the name of the candidate term in the variable 'this' 226 | this = 'gamma_%s'%location[i] 227 | ww = np.where(this == names)[0] 228 | # Then we see if it appears in the names of the state variables 229 | if len(ww): 230 | # form a mask so we dont apply the operator to gamma 231 | # terms. Note that *all* gamma terms are masked 232 | # even though we only actually use the last one we 233 | # come across. 234 | # we use [...,ww[0]] because the identifier for the 235 | # state is always in the final dimension. 236 | mask[...,ww[0]] = False 237 | # We store ww[0] as it will alllow us to access gamma 238 | # in subsequent calls in this same way. This is 239 | # self.linear.gamma_col 240 | self.linear.gamma_col = ww[0] 241 | # and is used as ... 242 | gammas = x2[...,self.linear.gamma_col] 243 | self.linear.datamask[self.linear.gamma_col] = False 244 | # We want to store an index into which of the 245 | # location vector terms we are dealing with here. 246 | # This is 247 | self.linear.gamma_loc = i 248 | # Once we apply the mask to get rid of the gamma columns 249 | # we need to keep track of the new shape for x2 250 | # This will be x2shape 251 | xshaper[-1] -= 1 252 | 253 | self.linear.x2shape = tuple(xshaper) 254 | self.linear.x2mask = mask.flatten() 255 | # so, apply the mask to take out the gamma columns 256 | x2 = x[self.linear.x2mask].reshape(self.linear.x2shape) 257 | 258 | # We next need access to the location information 259 | # for the selected dimension self.linear.gamma_loc. 260 | # If the data are gridded, we need to form the relevant information 261 | # Ungridded data we can access location directly as it is explicitly 262 | # stored. We store the location vector as 'locations' 263 | try: 264 | locshape = gammas.shape 265 | except: 266 | # If no gamma term is given, it is implicit that it is 267 | # the first dimension of location, but we have no data to mask 268 | self.linear.gamma_col = None 269 | self.linear.gamma_loc = 0 270 | locshape = (0) 271 | gammas = x2[...,0]*0.+1.0 272 | #if self.x_meta.is_grid: 273 | # the locational variable of interest is self.linear.gamma_loc 274 | # the grid is dimensioned e.g. [t,r,c,p] 275 | # so we need e.g. locations which is of dimension 276 | # e.g. [t,r,c] 277 | # locations = self.x.location 278 | # access the ungridded location data 279 | lim = self.x_meta.qlocation[self.linear.gamma_loc] 280 | nloc = lim[1] - lim[0] + 1 281 | locations = self.x.location[...,self.linear.gamma_loc] 282 | locshape = tuple(np.array(self.x.location.shape)[:-1]) 283 | 284 | for (i,lag) in enumerate(self.rt_model.lag): 285 | wt = self.rt_model.lag_weight[i] 286 | 287 | slocations = wt*(np.roll(locations,lag,\ 288 | axis=self.linear.gamma_loc) - locations).astype(float) 289 | slocations2 = (locations - np.roll(locations,-lag,\ 290 | axis=self.linear.gamma_loc)).astype(float) 291 | 292 | # If there is no variation, it is a waste of time to calculate 293 | # the derivative 294 | if i == 0 and np.abs(slocations).sum() + np.abs(slocations2).sum() == 0: 295 | # there is no variation here 296 | self.novar = True 297 | return 0 298 | self.novar = False 299 | ww = np.where(slocations > 0) 300 | mod = int(float(self.rt_model.wraparound_mod)/lim[-1]+0.5) or slocations.shape[self.linear.gamma_loc] 301 | if self.rt_model.wraparound == 'reflexive': 302 | slocations[ww] = 0. 303 | #slocations[ww] = -np.fmod(mod - slocations[ww],mod) 304 | elif self.rt_model.wraparound == 'periodic': 305 | if self.rt_model.wraparound_mod == 0: 306 | slocations[ww] = slocations2[ww] 307 | else: 308 | slocations[ww] = -lim[-1] * np.fmod( mod - slocations[ww]/lim[-1],mod) 309 | else: # none 310 | slocations[ww] = 0. 311 | ww = np.where(slocations != 0) 312 | slocations[ww] = 1./slocations[ww] 313 | 314 | if i == 0: 315 | # Form the D matrix. This is of the size required to 316 | # process the x2 data, and this is the most convenient 317 | # form to use it in 318 | m = np.zeros(slocations.shape * 2) 319 | ww = np.where(slocations != 0) 320 | ww2 = np.array(ww).copy() 321 | ww2[self.linear.gamma_loc] = ww2[self.linear.gamma_loc] - lag 322 | ww2 = tuple(ww2) 323 | m[ww*2] = m[ww*2] - slocations[ww] 324 | if False and self.rt_model.wraparound == 'reflexive': 325 | ww2 = np.abs(ww-lag) 326 | # this is looped as there might be multiple elements with the 327 | # same index for the reflecxive case 328 | if m.ndim > 2: 329 | raise Exception("Not yet implemented: Can't use reflexive mode for multi-dimensions yet") 330 | for (c,j) in enumerate(ww2): 331 | m[j,ww[c]] = m[j,ww[c]] + slocations[ww[c]] 332 | else: 333 | ww = tuple(ww) 334 | ww2 = tuple(ww2) 335 | m[ww2+ww] = m[ww2+ww] + slocations[ww] 336 | # fix for edge conditions 337 | dd = m.copy() 338 | #import pdb; pdb.set_trace() 339 | dd = dd.reshape(tuple([np.array(self.linear.x2shape[:-1]).prod()])*2) 340 | ddw = np.where(dd.diagonal() == 0)[0] 341 | for d in (ddw): 342 | ds = -dd[d,:].sum() 343 | dd[d,:] += dd[d,:] 344 | dd[d,d] = ds 345 | m = dd.reshape(m.shape) 346 | self.logger.info('Caching model matrices...') 347 | # 348 | if np.array(xshape).prod() == Cy1.size: 349 | self.linear.C1 = Cy1.reshape(xshape)[mask]\ 350 | .reshape( self.linear.x2shape ) 351 | elif xshape[1] == Cy1.size: 352 | self.linear.C1 = np.tile(Cy1,xshape[0])[mask.flatten()].reshape( self.linear.x2shape ) 353 | else: 354 | raise Exception("Can't deal with full covar matrix in DModel yet") 355 | nn = slocations.flatten().size 356 | m = m.reshape(nn,nn) 357 | self.linear.D1 = np.matrix(m).T 358 | for i in xrange(1,self.rt_model.model_order): 359 | m = np.matrix(self.linear.D1).T * m 360 | self.linear.D1 = m 361 | self.logger.info('... Done') 362 | return True 363 | 364 | def J(self): 365 | ''' 366 | A slightly modified J as its efficient to 367 | precalculate things for this model 368 | 369 | J = 0.5 * x.T D1.T gamma^2 D1 x 370 | 371 | ''' 372 | x,Cx1,xshape,y,Cy1,yshape = self.getxy() 373 | self.Hsetup() 374 | if self.novar: 375 | return 0 376 | xshape = self.x.state.shape 377 | try: 378 | if self.linear.gamma_col != None: 379 | gamma = x.reshape(self.x.state.shape)\ 380 | [...,self.linear.gamma_col].flatten() 381 | else: 382 | # no gamma variable, so use 1.0 383 | gamma = x.reshape(self.x.state.shape)\ 384 | [...,0].flatten()*0.+1. 385 | except: 386 | self.logger.error('gamma_col not set ... recovering and assuming no variation here') 387 | self.linear.gamma_col = None 388 | gamma = x.reshape(self.x.state.shape)[...,0].flatten()*0.+1. 389 | self.novar = True 390 | self.Hsetup() 391 | return 0 392 | 393 | x2 = x[self.linear.x2mask].reshape(self.linear.x2shape) 394 | J = 0. 395 | i = 0 396 | if self.rt_model.inverse_gamma: 397 | tgamma = 1./gamma 398 | else: 399 | tgamma = gamma 400 | for count in xrange(self.x.state.shape[-1]): 401 | if count != self.linear.gamma_col: 402 | C1 = np.diag(self.linear.C1[...,i].\ 403 | reshape(self.linear.D1.shape[0])) 404 | x2a = x2[...,i].reshape(self.linear.D1.shape[0]) 405 | xg = np.matrix(x2a*tgamma).T 406 | dxg = self.linear.D1.T * xg 407 | 408 | J += np.array(0.5 * dxg.T * C1 * dxg)[0][0] 409 | i += 1 410 | #print x[0],J 411 | return np.array(J).flatten()[0] 412 | 413 | def J_prime_prime(self): 414 | ''' 415 | Calculation of J'' 416 | 417 | We already have the differntial operator 418 | self.linear.D1 and self.gamma 419 | after we call self.J_prime() 420 | 421 | Here, J'' = D1.T gamma^2 D1 422 | 423 | J' is of shape (nobs,nstates) 424 | which is the same as the shape of x 425 | 426 | D1 is of shape (nobs,nobs) 427 | which needs to be expanded to 428 | (nobs,nstates,nobs,nstates) 429 | 430 | ''' 431 | x,Cx1,xshape,y,Cy1,yshape = self.getxy() 432 | J,J_prime = self.J_prime() 433 | xshape = self.x.state.shape 434 | 435 | if not 'linear' in self.dict(): 436 | self.linear = ParamStorage() 437 | if not 'J_prime_prime' in self.linear.dict(): 438 | self.linear.J_prime_prime = \ 439 | np.zeros(xshape*2) 440 | else: 441 | self.linear.J_prime_prime[:] = 0 442 | # we need an indexing system in case of multiple 443 | # nobs columns 444 | x2a = np.diag(np.ones(self.linear.x2shape[:-1]).flatten()) 445 | try: 446 | gamma = self.linear.gamma.flatten() 447 | except: 448 | if self.linear.gamma_col != None: 449 | gamma = x.reshape(self.x.state.shape)\ 450 | [...,self.linear.gamma_col].flatten() 451 | else: 452 | # no gamma variable, so use 1.0 453 | gamma = x.reshape(self.x.state.shape)\ 454 | [...,0].flatten()*0.+1. 455 | gamma = self.linear.gamma.flatten() 456 | if self.rt_model.inverse_gamma: 457 | tgamma = 1./gamma 458 | dg = 2./(gamma*gamma*gamma) 459 | else: 460 | tgamma = gamma 461 | dg = 1.0 462 | nshape = tuple([np.array(self.linear.x2shape[:-1]).prod()]) 463 | D1 = np.matrix(self.linear.D1.reshape(nshape*2)) 464 | i = 0 465 | # so, e.g. we have xshape as (50, 100, 2) 466 | # because one of those columns refers to the gamma value 467 | # self.linear.gamma_col will typically be 0 468 | for count in xrange(xshape[-1]): 469 | if count != self.linear.gamma_col: 470 | # we only want to process the non gamma col 471 | C1 = np.diag(self.linear.C1[...,i].\ 472 | reshape(self.linear.D1.shape[0])) 473 | xg = np.matrix(x2a*tgamma*tgamma) 474 | dxg = D1 * xg 475 | deriv = np.array(dxg.T * C1 * D1) 476 | # so we have gamma^2 D^2 which is the Hessian 477 | # we just have to put it in the right place now 478 | # the technical issue is indexing an array of eg 479 | # (50, 100, 2, 50, 100, 2) 480 | # but it might have more or fewer dimensions 481 | nd = len(np.array(xshape)[:-1]) 482 | nshape = tuple(np.array(xshape)[:-1]) 483 | if nd == 1: 484 | self.linear.J_prime_prime[:,count,:,count] = deriv.reshape(nshape*2) 485 | elif nd == 2: 486 | self.linear.J_prime_prime[:,:,count,:,:,count] = deriv.reshape(nshape*2) 487 | elif nd == 3: 488 | self.linear.J_prime_prime[:,:,:,count,:,:,:,count] = deriv.reshape(nshape*2) 489 | else: 490 | self.logger.error("Can't calculate Hessian for %d dimensions ... I can only do up to 3"%nd) 491 | 492 | #ww = np.where(deriv) 493 | #ww2 = tuple([ww[0]]) + tuple([ww[0]*0+count]) \ 494 | # + tuple([ww[1]] )+ tuple([ww[0]*0+count]) 495 | #x1 = deriv.shape[0] 496 | #x2 = self.linear.J_prime_prime.shape[-1] 497 | #xx = self.linear.J_prime_prime.copy() 498 | #xx = xx.reshape(x1,x2,x1,x2) 499 | #xx[ww2] = deriv[ww] 500 | #self.linear.J_prime_prime = xx.reshape(self.linear.J_prime_prime.shape) 501 | i += 1 502 | if self.linear.gamma_col != None: 503 | c = self.linear.gamma_col 504 | nd = len(np.array(xshape)[:-1]) 505 | nshape = tuple(np.array(xshape)[:-1]) 506 | deriv = np.diag(dg*2*J/(tgamma*tgamma)).reshape(nshape*2) 507 | if nd == 1: 508 | self.linear.J_prime_prime[:,c,:,c] = deriv 509 | elif nd == 2: 510 | self.linear.J_prime_prime[:,:,c,:,:,c] = deriv 511 | elif nd == 3: 512 | self.linear.J_prime_prime[:,:,:,c,:,:,:,c] = deriv 513 | else: 514 | self.logger.error("Can't calculate Hessian for %d dimensions ... I can only do up to 3"%nd) 515 | 516 | #dd = np.arange(nshape[0]) 517 | #x1 = dd.shape[0] 518 | #x2 = self.linear.J_prime_prime.shape[-1] 519 | #xx = self.linear.J_prime_prime.copy() 520 | #xx = xx.reshape(x1,x2,x1,x2) 521 | #xx[dd,dd*0+self.linear.gamma_col,\ 522 | # dd,dd*0+self.linear.gamma_col] = dg*2*J/(tgamma*tgamma) 523 | #self.linear.J_prime_prime = xx.reshape(self.linear.J_prime_prime.shape) 524 | n = np.array(xshape).prod() 525 | return J,J_prime,self.linear.J_prime_prime.reshape(n,n) 526 | 527 | def J_prime(self): 528 | ''' 529 | A slightly modified J as its efficient to 530 | precalculate things for this model 531 | 532 | J' = D.T gamma^2 D x 533 | 534 | ''' 535 | J = self.J() 536 | if self.novar: 537 | return 0,self.nowt 538 | x,Cx1,xshape,y,Cy1,yshape = self.getxy() 539 | x2 = x[self.linear.x2mask].reshape(self.linear.x2shape) 540 | if self.linear.gamma_col != None: 541 | gamma = x.reshape(self.x.state.shape)\ 542 | [...,self.linear.gamma_col].flatten() 543 | else: 544 | # no gamma variable, so use 1.0 545 | gamma = x.reshape(self.x.state.shape)\ 546 | [...,0].flatten()*0.+1. 547 | #gamma = self.linear.gamma.flatten() 548 | 549 | if self.rt_model.inverse_gamma: 550 | tgamma = 1./gamma 551 | dg = -1./(gamma*gamma) 552 | else: 553 | tgamma = gamma 554 | dg = 1.0 555 | 556 | g2 = tgamma * tgamma 557 | xshape = self.x.state.shape 558 | J_prime = np.zeros((x.shape[0]/xshape[-1],xshape[-1])) 559 | D2x_sum = 0. 560 | # loop over the non gamma variables 561 | i = 0 562 | # store gamma in case needed elsewhere 563 | self.linear.gamma = gamma 564 | for count in xrange(self.x.state.shape[-1]): 565 | if count != self.linear.gamma_col: 566 | C1 = np.diag(self.linear.C1[...,i].\ 567 | reshape(self.linear.D1.shape[0])) 568 | x2a = x2[...,i].reshape(self.linear.D1.shape[0]) 569 | xg = np.matrix(x2a*tgamma).T 570 | dxg = self.linear.D1 * xg 571 | deriv = np.array(dxg.T * C1 * self.linear.D1)[0] 572 | J_prime[...,count] = deriv * tgamma 573 | #if self.linear.gamma_col != None: 574 | # J_prime_gamma = deriv * x2a 575 | # D2x_sum = D2x_sum + J_prime_gamma 576 | i += 1 577 | if self.linear.gamma_col != None: 578 | J_prime[...,self.linear.gamma_col] = dg*2*J/tgamma 579 | 580 | return J,J_prime 581 | 582 | def Hsetup(self): 583 | ''' 584 | setup for the differential operator H(x) 585 | 586 | ''' 587 | if not self.beenHere and not 'H_prime' in self.linear.dict(): 588 | self.logger.info('Setting up storage for efficient model operator') 589 | if 'y' in self.dict(): 590 | self.linear.H = np.zeros(self.y.state.shape) 591 | self.linear.H_prime = np.zeros(self.y.state.shape*2) 592 | else: 593 | self.linear.H = np.zeros(self.x.state.shape) 594 | self.linear.H_prime = np.zeros(self.x.state.shape*2) 595 | self.setH() 596 | if self.novar: 597 | self.nowt = 0. * self.x.state 598 | #del self.linear.H_prime, self.linear.H 599 | self.beenHere = True 600 | 601 | 602 | -------------------------------------------------------------------------------- /eoldas/eoldas_Solver.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import pdb 3 | import numpy as np 4 | from eoldas_State import State 5 | from eoldas_ParamStorage import ParamStorage 6 | from eoldas_Operator import * 7 | from eoldas_Lib import sort_non_spectral_model,sortlog 8 | 9 | class eoldas_Solver(ParamStorage): 10 | ''' 11 | Eoldas solver class 12 | 13 | ''' 14 | def __init__(self,confs,logger=None,logfile=None,thisname=None,name=None,datadir=None,logdir=None): 15 | ''' 16 | Initialise the solver. 17 | 18 | This does the following: 19 | 1. Read configuration file(s) 20 | 2. Load operators 21 | 3. Test the call to the cost function 22 | 23 | There can be multiple groups of configuration files, so 24 | self.confs, that holds the core information setup here 25 | can contain multiple configurations. 26 | 27 | The number of configurations is len(confs.infos) and 28 | the ith configuration is conf = self.confs.infos[i]. 29 | 30 | Various loggers are available throughout the classes 31 | used, but the top level logger is self.confs.logger, 32 | so you can log with e.g. 33 | 34 | self.confs.logger.info('this is some info') 35 | 36 | The root operator is stored in self.confs.root[i] 37 | for the ith configuration, so the basic call to the cost 38 | function is: 39 | 40 | J,J_prime = self.confs[i].parameter.cost(None) 41 | 42 | ''' 43 | from eoldas_ConfFile import ConfFile 44 | from eoldas_Lib import sortopt 45 | name = name or thisname 46 | if name == None: 47 | import time 48 | thistime = str(time.time()) 49 | name = type(self).__name__ 50 | name = "%s.%s" % (name,thistime) 51 | self.thisname = name 52 | 53 | self.confs = confs 54 | self.top = sortopt(self,'top',ParamStorage()) 55 | self.top.general = sortopt(self.top,'general',ParamStorage()) 56 | thisname = sortopt(self.top.general,'name',thisname or self.thisname) 57 | logfile = sortopt(self.top.general,'logfile',logfile or 'log.dat') 58 | logdir = sortopt(self.top.general,'logdir',logfile or 'logs') 59 | datadir = sortopt(self.top.general,'datadir',datadir or ['.']) 60 | self.logger = sortlog(self,logfile,logger or self.confs.logger,\ 61 | name=self.thisname,logdir=logdir,debug=True) 62 | n_configs = len(self.confs.infos) 63 | self.confs.root = [] 64 | self.have_unc = False 65 | try: 66 | logdir = logdir 67 | except: 68 | logdir = self.top.general.logdir 69 | 70 | # first set up parameter 71 | conf = confs.infos[0] 72 | 73 | general = conf.general 74 | op = conf.parameter 75 | if not 'parameter' in conf.dict(): 76 | raise Exception('No parameter field found in %s item %d'%\ 77 | (conf.__doc__,0)) 78 | general.is_spectral = sortopt(general,'is_spectral',True) 79 | if not general.is_spectral: 80 | sort_non_spectral_model(op,conf.operator,logger=confs.logger) 81 | general.init_test = sortopt(general,'init_test',False) 82 | confs.logger.info('loading parameter state') 83 | conf.parameter.name = 'Operator' 84 | parameter = eval(op.name)(op,general,\ 85 | parameter=None,\ 86 | logger=confs.logger,\ 87 | name=name+".parameter",\ 88 | datatype=list(conf.parameter.datatypes),\ 89 | logdir=logdir,\ 90 | logfile=logfile,\ 91 | datadir=datadir) 92 | try: 93 | parameter.transform = parameter.options.x.transform 94 | parameter.invtransform = parameter.options.x.invtransform 95 | except: 96 | parameter.transform = parameter.options.x.names 97 | parameter.invtransform = parameter.options.x.names 98 | 99 | # we now have access to parameter.x.state, parameter.x.sd etc 100 | # and possibly parameter.y.state etc. 101 | operators = [] 102 | for (opname,op) in conf.operator.dict().iteritems(): 103 | if opname != 'helper': 104 | #pdb.set_trace() 105 | exec('from eoldas_%s import %s'%(op.name,op.name)) 106 | # make sure the data limits and x bounds are the same 107 | # ... inherit from parameter 108 | op.limits = parameter.options.limits 109 | if not 'datatypes' in op.dict(): 110 | op.datatypes = 'x' 111 | thisop = eval(op.name)(op,general,parameter=parameter,\ 112 | logger=confs.logger,\ 113 | name=name+".%s-%s"%(thisname,opname),\ 114 | datatype=list(op.datatypes),\ 115 | logdir=logdir,\ 116 | logfile=logfile,\ 117 | datadir=datadir) 118 | # load from parameter 119 | thisop.loader(parameter) 120 | operators.append(thisop) 121 | try: 122 | thisop.transform = parameter.options.x.transform 123 | thisop.invtransform = parameter.options.x.invtransform 124 | except: 125 | thisop.transform = parameter.options.x.names 126 | thisop.invtransform = parameter.options.x.names 127 | thisop.ploaderMask = np.in1d(parameter.x_meta.state,thisop.x_meta.state) 128 | try: 129 | thisop.invtransform = np.array(thisop.invtransform[thisop.ploaderMask]) 130 | thisop.transform = np.array(thisop.transform[thisop.ploaderMask]) 131 | except: 132 | ww = thisop.ploaderMask 133 | thisop.invtransform = np.array(thisop.transform)[ww] 134 | thisop.transform = np.array(thisop.transform)[ww] 135 | # sort the loaders 136 | parameter.operators = operators 137 | self.confs.root.append(parameter) 138 | # Now we have set up the operators 139 | # try out the cost function 140 | if general.init_test: 141 | self.logger.info('testing cost function calls') 142 | J,J_prime = self.confs.root[0].cost() 143 | self.logger.info('done') 144 | self.confs.root = np.array(self.confs.root) 145 | 146 | def cost(self,xopt): 147 | ''' 148 | Load xopt into the full state vector into root.x.state 149 | 150 | and calculate the cost J and J_prime 151 | 152 | The J_prime stored in self.J_prime is of dimension 153 | of the number of state variables that are targeted 154 | for estimation. 155 | 156 | ''' 157 | if not xopt == None: 158 | self.loader(xopt,self.root.x.state) 159 | else: 160 | if not 'nmask' in self.dict(): 161 | for i in xrange(len(self.confs.infos)): 162 | self.prep(i) 163 | xopt = np.zeros(self.nmask1+self.nmask2) 164 | J,J_prime = self.root.cost() 165 | self.J_prime = xopt*0. 166 | try: 167 | self.unloader(self.J_prime,J_prime.reshape(self.root.x.state.shape)) 168 | except: 169 | self.unloader(self.J_prime,J_prime) 170 | return np.array(J).flatten()[0] 171 | 172 | cost_df = lambda self,x:self.J_prime 173 | cost_df.__name__ = 'cost_df' 174 | cost_df.__doc__ = ''' 175 | This method returns the cost function derivative 176 | J_prime, assuming that self.cost(xopt) has 177 | already been called. 178 | ''' 179 | 180 | def approx_cost_df(self,xopt): 181 | ''' 182 | A discrete approximation to the cost fn and its derivative 183 | 184 | Mainly useful for testing as it can be done faster 185 | 186 | If DerApproximator is not available, the 'full' 187 | cost function is returned, which doesn't allow a test. 188 | Check log file or try: 189 | 190 | from DerApproximator import get_d1 191 | 192 | if you are concerned about that. 193 | 194 | ''' 195 | try: 196 | from DerApproximator import get_d1 197 | except: 198 | self.confs.logger.error(\ 199 | "Cannot import DerApproximator for derivative approx'") 200 | J = self.cost(xopt) 201 | J_prime = self.cost_df(xopt) 202 | self.J_prime_approx = J_prime.flatten() 203 | return self.J_prime_approx 204 | 205 | self.J_prime_approx = get_d1(self.cost,xopt) 206 | return self.J_prime_approx 207 | 208 | def prep(self,thisconf): 209 | ''' 210 | A method to prepare the solver 211 | 212 | ''' 213 | self.root = self.confs.root[thisconf] 214 | root = self.confs.root[thisconf] 215 | 216 | self.sortMask() 217 | 218 | self.op = sortopt(root.general,'optimisation',ParamStorage()) 219 | self.op.plot = sortopt(self.op,'plot',0) 220 | self.op.name = sortopt(self.op,'name','solver') 221 | self.op.maxfunevals = sortopt(self.op,'maxfunevals',2e4) 222 | self.op.maxiter = sortopt(self.op,'maxiter',1e4) 223 | self.op.gtol = sortopt(self.op,'gtol',1e-3) 224 | self.op.iprint = sortopt(self.op,'iprint',1) 225 | self.op.solverfn = sortopt(self.op,'solverfn','scipy_lbfgsb') 226 | self.op.randomise = sortopt(self.op,'randomise',False) 227 | self.op.no_df = sortopt(self.op,'no_df',False) 228 | 229 | self.result = sortopt(root.options,'result',ParamStorage()) 230 | self.result.filename = sortopt(root.options.result,'filename',\ 231 | 'results.pkl') 232 | self.result.fmt = sortopt(root.options.result,'format','pickle') 233 | try: 234 | self.transform = self.root.options.x.transform 235 | self.invtransform = self.root.options.x.transform 236 | except: 237 | self.transform = None 238 | self.invtransform = None 239 | 240 | # descend into the operators and identify any observation ops 241 | # we then want to be able to write out files (or plot data) 242 | # that are H(x). 243 | # We only do this for Operators that have both 'x' and 'y' 244 | # terms as we store the filename under 'y.result' 245 | self.Hx_ops = [] 246 | for i,op in enumerate(root.operators): 247 | op.loader(root) 248 | if 'y_meta' in op.dict() and op.options.y.datatype == 'y': 249 | # There is a potential observation 250 | op.y_state.options.y.result = \ 251 | sortopt(op.y_state.options.y,'result',ParamStorage()) 252 | op.y_state.options.y.result.filename = \ 253 | sortopt(op.y_state.options.y.result,\ 254 | 'filename',op.y_state.thisname) 255 | op.y_state.options.y.result.format = \ 256 | sortopt(op.y_state.options.y.result,\ 257 | 'format','PARAMETERS') 258 | state = op.y_state._state 259 | this = { \ 260 | 'filename':op.y_state.options.y.result.filename,\ 261 | 'format':op.y_state.options.y.result.format,\ 262 | 'state':state,\ 263 | 'y':op.y_state,\ 264 | 'op':op,\ 265 | 'transform':self.transform,\ 266 | 'invtransform':self.invtransform,\ 267 | 'Hx':op.linear.H} 268 | op.Hx_info = this 269 | self.Hx_ops.append(this) 270 | else: 271 | this = { \ 272 | 'transform':self.transform,\ 273 | 'invtransform':self.invtransform,\ 274 | } 275 | op.Hx_info = this 276 | 277 | 278 | def solver(self,thisconf=0): 279 | ''' 280 | The solver. Use this method to run the optimisation 281 | code to minimse the cost function. 282 | 283 | Options: 284 | -------- 285 | thisconf : Index of which configuration set to use. 286 | By default, this is the first of the set. 287 | 288 | 289 | ''' 290 | from eoldas_Lib import sortopt 291 | self.logger.info('importing optimisation modules') 292 | try: 293 | from openopt import NLP 294 | self.isNLP = True 295 | self.confs.logger.info("NLP imported from openopt") 296 | import scipy.optimize 297 | self.confs.logger.info("scipy.optimize imported") 298 | except: 299 | self.isScipy = False 300 | self.confs.logger.error("scipy.optimize NOT imported") 301 | self.confs.logger.error("Maybe you don't have scipy ...") 302 | self.confs.logger.error("***************************") 303 | self.confs.logger.error("Failed to find an optimizer") 304 | self.confs.logger.error(\ 305 | "... get a proper python installation") 306 | self.confs.logger.error("with scipy and/or openopt") 307 | self.confs.logger.error("NLP NOT imported from openopt") 308 | self.confs.logger.error("Maybe you don't have openopt ...") 309 | self.isNLP = False 310 | 311 | root = self.root 312 | self.logger.info('done') 313 | self.bounds = root.x_meta.bounds 314 | self.names = root.x_meta.state 315 | self.xorig = root.x.state.copy() 316 | 317 | self.lb = np.array(([ self.bounds[i][0] \ 318 | for i in xrange(len(self.bounds))])\ 319 | *(self.xorig.size/len(self.bounds)))\ 320 | .reshape(self.xorig.shape) 321 | self.ub = np.array(([ self.bounds[i][1] \ 322 | for i in xrange(len(self.bounds))])\ 323 | *(self.xorig.size/len(self.bounds)))\ 324 | .reshape(self.xorig.shape) 325 | 326 | xopt = np.zeros(self.nmask1+self.nmask2) 327 | lb = np.zeros(self.nmask1+self.nmask2) 328 | ub = np.zeros(self.nmask1+self.nmask2) 329 | 330 | self.unloader(xopt,root.x.state) 331 | self.unloader(lb,self.lb) 332 | self.unloader(ub,self.ub) 333 | if self.op.randomise: 334 | xopt = np.random.rand(xopt.size)*(ub-lb)+lb 335 | self.confs.logger.info('Randomising initial x') 336 | self.loader(xopt,root.x.state) 337 | # now we have to subset self.xorig, self.lb, self.ub 338 | # for the solver here 339 | if self.op.no_df: 340 | pp = NLP(self.cost, xopt.flatten(), iprint=self.op.iprint, \ 341 | goal='min', name=self.op.name, show=0,\ 342 | lb = lb, ub = ub, \ 343 | plot = int(self.op.plot) ,\ 344 | maxFunEvals = int(self.op.maxFunEvals), \ 345 | maxIter = int(self.op.maxIter), gtol = self.op.gtol) 346 | else: 347 | pp = NLP(self.cost, xopt.flatten(), iprint=self.op.iprint, \ 348 | goal='min', name=self.op.name, show=0,\ 349 | lb = lb, ub = ub, df=self.cost_df,\ 350 | plot = int(self.op.plot) ,\ 351 | maxFunEvals = int(self.op.maxfunevals), \ 352 | maxIter = int(self.op.maxiter), gtol = self.op.gtol) 353 | # df=self.cost_df 354 | r = pp.solve(self.op.solverfn) 355 | self.loader(r.xf,root.x.state) 356 | self.min_cost = r.ff 357 | self.confs.logger.info('Min cost = %s'%str(self.min_cost)) 358 | 359 | 360 | def sortMask(self): 361 | ''' 362 | Sort masks for loading and unloading data 363 | ''' 364 | self.root.options.solve = \ 365 | sortopt(self.root.options,\ 366 | 'solve',np.array([1]*len(self.root.x_meta.state))) 367 | self.solve = np.array(self.root.options.solve).copy() 368 | # set up masks for loading and unloading 369 | self.mask1 = np.zeros_like(self.root.x.state).astype(bool) 370 | ww = np.where(self.solve == 1)[0] 371 | self.mask1[...,ww] = True 372 | self.mask2 = np.zeros_like(self.root.x.state[0]).astype(bool) 373 | ww = np.where(self.solve == 2)[0] 374 | self.mask2[ww] = True 375 | self.nmask1 = self.mask1.sum() 376 | self.nmask2 = self.mask2.sum() 377 | self.wmask2 = np.where(self.mask2)[0] 378 | 379 | 380 | def loader(self,xopt,xstate,M=False): 381 | ''' 382 | From xopt, that being optimized, load the full xstate 383 | 384 | ''' 385 | if M: 386 | mask1 = self.mask1.flatten() 387 | count = 0 388 | for i in np.where(mask1)[0]: 389 | xstate[i,mask1] = np.array(xopt[count]).flatten() 390 | count += 1 391 | return 392 | 393 | xstate[self.mask1] = xopt[:self.nmask1].flatten() 394 | for n,i in enumerate(self.wmask2): 395 | xstate[...,i] = xopt[self.nmask1+n] 396 | 397 | def unloader(self,xopt,xstate,M=False): 398 | ''' 399 | From xstate, load xopt, that being optimized 400 | ''' 401 | if M: 402 | n = self.nmask1 403 | mask1 = np.matrix(self.mask1.flatten()) 404 | MM = mask1.T * mask1 405 | out = xstate[MM].reshape(n,n) 406 | return out 407 | 408 | xstate = (xstate.copy()).reshape(self.mask1.shape) 409 | 410 | xopt[:self.nmask1] = xstate[self.mask1] 411 | #xopt[:self.nmask1] = xstate[self.mask1].flatten() 412 | for n,i in enumerate(self.wmask2): 413 | xopt[self.nmask1+n] = xstate[...,i][0] 414 | 415 | def writeHx(self,filename=None,format=None,fwd=True): 416 | ''' 417 | Writer function for observations ('y' states) 418 | 419 | Assumes filename in self.result.filename 420 | and format in self.result.format 421 | 422 | These can be specified in the config file as: 423 | 424 | parameter.result.filename 425 | parameter.result.format 426 | 427 | These can of course be over-ridden using the options 428 | filename and format. 429 | 430 | ''' 431 | for this in self.Hx_ops: 432 | filename = this['filename'] 433 | format = this['format'] 434 | state = this['state'] 435 | op = this['op'] 436 | # ensure Hx is up to date 437 | J = op.J() 438 | Hx = op.Hx 439 | filenamey = filename+ '_orig' 440 | self.logger.info('Writing H(x) data to %s'%this['filename']) 441 | self.logger.info('Writing y data to %s'%filenamey) 442 | try: 443 | # dont stop just because it messes up plotting 444 | op.plot(ploterr=self.have_unc) 445 | except: 446 | pass 447 | # write the observation data 448 | state.name.fmt = format 449 | state.write(filenamey,format) 450 | 451 | # to write Hx data we have to mimic 452 | # the y data 453 | # first make a copy of the dataset 454 | self.datacopy = state.data.state.copy() 455 | self.sdcopy = state.data.sd.copy() 456 | self.C1copy = state.data.C1.copy() 457 | 458 | # try fwdSd 459 | try: 460 | state.data.sd = op.fwdSd 461 | state.data.C1 = op.fwdUncert 462 | except: 463 | state.data.sd = self.sdcopy*0. 464 | state.data.C1 = self.C1copy*0 465 | 466 | # insert the Hx data 467 | # into state 468 | state.data.state = Hx.reshape(state.data.state.shape) 469 | 470 | # write the file 471 | state.write(filename,format) 472 | #then copy back the original data to tidy up 473 | state.data.state = self.datacopy.copy() 474 | state.data.sd = self.sdcopy.copy() 475 | state.data.C1 = self.C1copy.copy() 476 | #state.name.fmt = self.fmtcopy 477 | 478 | def uncertainty(self): 479 | ''' 480 | Calculate the inverse Hessian of all operators 481 | 482 | ''' 483 | from sys import stderr 484 | J,J_prime,J_prime_prime = self.root.hessian() 485 | # reduce the rank 486 | Hsmall = self.unloader(None,J_prime_prime,M=True) 487 | self.have_unc = False 488 | try: 489 | IHsmall = np.matrix(Hsmall).I 490 | self.have_unc = True 491 | #U, ss, V = np.linalg.svd(Hsmall) 492 | #ww = np.where(ss>ss[0]*0.01) 493 | #sss = ss*0 494 | #sss[ww] = 1./ss[ww] 495 | #IHsmall = np.dot(U, np.dot(np.diag(sss), V)) 496 | except: 497 | IHsmall = np.matrix(Hsmall) 498 | J_prime_prime = J_prime_prime*0 499 | self.loader(IHsmall,J_prime_prime,M=True) 500 | self.Ihessian = J_prime_prime 501 | self.IHsmall = IHsmall 502 | dd = self.IHsmall.diagonal() 503 | 504 | #print >> stderr, "WARNING: ... something " 505 | try: 506 | self.root.x.sd = np.sqrt(np.array(self.Ihessian.diagonal()).flatten()) 507 | self.root.Ihessian = self.Ihessian 508 | nfwd = self.root.fwdError() 509 | # if this works you should get uncertainty if fwd modelling in 510 | # op.fwdUncert for each operator self.operators 511 | except: 512 | # then you have -ve values 513 | self.logger.error("WARNING: ill-conditioned system with unstable estimates of uncertainty") 514 | for i,op in enumerate(self.root.operators): 515 | # propagate the sd data down 516 | op.x.sd = self.root.x.sd.reshape(op.loaderMask.shape)[op.loaderMask] 517 | 518 | def write(self,filename=None,format=None): 519 | ''' 520 | Writer function for the state variable 521 | 522 | Assumes filename in self.result.filename 523 | and format in self.result.format 524 | 525 | These can be specified in the config file as: 526 | 527 | parameter.result.filename 528 | parameter.result.format 529 | 530 | These can of course be over-ridden using the options 531 | filename and format. 532 | 533 | ''' 534 | filename = filename or self.result.filename 535 | format = format or self.result.format 536 | self.logger.info('writing results to %s'%filename) 537 | try: 538 | np.savez(filename.replace('.dat','.npz'),Ihessian=self.Ihessian,IHsmall=self.IHsmall) 539 | except: 540 | pass 541 | try: 542 | self.root.plot(noy=True,ploterr=self.have_unc) 543 | except: 544 | pass 545 | self.root.x_state.write(filename,None,fmt=format) 546 | 547 | def tester(): 548 | ''' 549 | Derivative test for total J_prime 550 | 551 | It should plot a scatterplot of derivatives calculated 552 | by independent methods. 553 | 554 | They should lie on a 1:1 line, or if not, there 555 | is a problem with the derivative calculation implemented. 556 | 557 | In this case, you should check the individual operator 558 | derivatives carefully, using e.g. tester() in 559 | eoldas_Operator.py 560 | ''' 561 | solver = eoldas_Solver() 562 | print "See logfile for results of test" 563 | 564 | for i in xrange(len(solver.confs.infos)): 565 | solver.prep(i) 566 | xopt = np.zeros(solver.nmask1+solver.nmask2) 567 | 568 | # randomise, so we get a good signal to look at 569 | # Make the xstate random here, just as a good test 570 | #xopt = np.random.rand(solver.nmask1+solver.nmask2) 571 | 572 | solver.loader(xopt,solver.root.x.state) 573 | J = solver.cost(xopt) 574 | J_prime = solver.cost_df(xopt) 575 | J_prime_approx = solver.approx_cost_df(xopt) 576 | #ww = np.where(J_prime>0) 577 | #J_prime = J_prime[ww] 578 | #J_prime_approx = J_prime_approx[ww] 579 | try: 580 | import pylab 581 | max = np.max([np.max(J_prime),np.max(J_prime_approx)]) 582 | min = np.min([np.min(J_prime),np.min(J_prime_approx)]) 583 | pylab.plot(min,max,'b-') 584 | pylab.plot(J_prime,J_prime_approx,'o') 585 | pylab.show() 586 | except: 587 | pass 588 | 589 | def demonstration(): 590 | ''' 591 | An example of running EOLDAS 592 | ''' 593 | from eoldas_ConfFile import ConfFile 594 | print "Testing ConfFile class with conf file eoldas_Test1" 595 | logdir = 'test/eoldas_Test/log' 596 | logfile = 'log.dat' 597 | thisname = 'eoldas_Test1' 598 | conffile = ['semid_default.conf'] 599 | 600 | #['Obs1.conf'] 601 | datadir = ['.'] 602 | confs = ConfFile(conffile,\ 603 | logdir=logdir,\ 604 | logfile=logfile,\ 605 | datadir=datadir) 606 | 607 | solver = eoldas_Solver(confs,thisname=thisname,\ 608 | logdir=logdir,\ 609 | logfile=logfile,\ 610 | datadir=datadir) 611 | 612 | for i in xrange(len(solver.confs.infos)): 613 | solver.prep(i) 614 | # try an initial calculation 615 | J = solver.cost() 616 | J_prime = solver.cost_df(None) 617 | # run the solver 618 | solver.solver() 619 | # Hessian 620 | solver.uncertainty() 621 | # write out the state 622 | solver.write() 623 | # write out any fwd modelling of observations 624 | solver.writeHx() 625 | 626 | if __name__ == "__main__": 627 | from eoldas_Solver import * 628 | help(eoldas_Solver) 629 | help(tester) 630 | help(demonstration) 631 | demonstration() 632 | -------------------------------------------------------------------------------- /eoldas/eoldas_SpecialVariable.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from eoldas_ParamStorage import ParamStorage 3 | import pdb 4 | import numpy as np 5 | from eoldas_Files import writer,reader,init_read_write 6 | from eoldas_Lib import sortlog 7 | 8 | class SpecialVariable(ParamStorage): 9 | ''' 10 | A class that can deal with the datatypes needed for eoldas 11 | 12 | It allows a variable to be set to various data types 13 | and interprets these into the data structure. 14 | 15 | The data structure imposed here is: 16 | 17 | self.data.this 18 | self.name.this 19 | 20 | to store some item called 'this'. Other information can be stored as well 21 | but part of the idea here is to have some imposed constraints on the 22 | data structure so that we can sensibly load up odfferent datasets. 23 | 24 | The idea is that data will be stored in self.data and associated 25 | metadata in self.name. If items are given the same name in both 26 | sub-structures, we can easily keep track of them. There is no actual 27 | requirement that this is adhered to, but it is certainy permitted and 28 | encouraged for the indended use of this class. 29 | 30 | Probably the most important thing then about this class is that is 31 | a SpecialVariable is assigned different data types, then it can do sensible 32 | things with them in the context of the EOLDAS (and wider applications). 33 | 34 | When an assignment takes place (either of ther form 35 | 36 | self.state = foo 37 | 38 | or 39 | 40 | self['state'] = foo 41 | 42 | then what is actually stored depends on the type and nature of foo. The 43 | main features as follows: 44 | 45 | If foo is a string: 46 | A guess is made that it is a filename, and an attempt is made to 47 | read the file. All directories in the list self.dirnames are 48 | searched for the filename, and any readable files found are 49 | considered candidates, Each of these is read in turn. A set of 50 | potential data formats, specified by the readers in readers 51 | (self.reader_functions) is considered, as if a sucessful interpretation 52 | takes place the data is returned and stiored in the derired variable. 53 | 54 | So, for example, if we have self.state = foo as above and foo is a valid, 55 | readable file in the list of directories specified, and it is 56 | interprable with one of the formats defined, then the main dataset 57 | is loaded into: 58 | 59 | self.data.state 60 | 61 | (alternatively known as self.data['state']). 62 | 63 | If foo is a ParamStorage, it should have the same structure as that here 64 | (i.e. self.data and self.name) and these structures are then loaded. 65 | 66 | 67 | If foo is a dictionary (type dict) 68 | It is first converted to a ParamStorage and then loaded as above. 69 | 70 | If foo is any other datatype, it is left pretty much as it, except that 71 | an attempt to convert to a np.array is made. 72 | 73 | Depending on the format, there might be data other than the main 74 | dataset (e.g. locational information) and these are loaded by the 75 | loaders into relevant oarts of self.data and self.name. 76 | 77 | For classes that use this class for EOLDAS, we will typically use: 78 | 79 | self.data.state : state variable data 80 | self.data.sd : uncertainty information as sd 81 | (or a similar fuller representation) 82 | self.data.control : control information (e.g. view angles) 83 | self.data.location : location information 84 | 85 | with associated descriptor data in the relevant parts of self.name. 86 | 87 | The idea for simple use of the data structure then is for all of these 88 | datasets represented as 2D datasets, where the number of rows in 89 | each of the self.data.state etc field will be the same, but 90 | the number of columns will tend to vary (e.g different numbers of state 91 | variables). 92 | 93 | The reason for considering such a 2D 'flat'(ish) representation is 94 | that it is easy to tabulate and understand. In fact the data will be of 95 | quite high dimension. E.g. is the data vary by time, x and y, then 96 | we would have 3 columns for self.data.location, with descriptors for 97 | the columns in self.name.location, and corresponding state data in 98 | self.data.state (with the number of state variables determining the 99 | number of columns in that table). 100 | 101 | As mentioned, at the pooint of this class, there is no strict 102 | requirement for any such structure ti the data loaded or used, but 103 | that is the plan for EOLDAS use, so worth documenting at this point. 104 | 105 | ''' 106 | 107 | def __init__(self,info=[],name=None,thisname=None,readers=[],log_terms={},\ 108 | datadir=["."],env=None,\ 109 | header=None,writers={}, 110 | simple=False,logger=None): 111 | ''' 112 | Class SpecialVariable initialisation. 113 | 114 | Sets up the class as a ParamStorage and calls self.init() 115 | 116 | See init() fort a fuller descripotion of the options. 117 | 118 | ''' 119 | ParamStorage.__init__(self,logger=logger) 120 | name = name or thisname 121 | if name == None: 122 | import time 123 | thistime = str(time.time()) 124 | name = type(self).__name__ 125 | name = "%s.%s" % (name,thistime) 126 | self.thisname = name 127 | 128 | self.init(info=[],name=self.thisname,readers=readers,log_terms={},\ 129 | datadir=datadir,env=env,\ 130 | header=header,writers=writers,\ 131 | simple=False,logger=logger) 132 | 133 | 134 | 135 | def init(self,info=[],name=None,readers=[],log_terms={},\ 136 | datadir=None,env=None,\ 137 | header=None,writers={},\ 138 | simple=False,logger=None): 139 | ''' 140 | Initialise information in a SpecialVariable instance. 141 | 142 | Options: 143 | info : Information tthat can be passed through to reader 144 | methods (a list). 145 | thisname : a name to use to identify this instance in any 146 | logging. By default this is None. If thisname is set 147 | to True, then logging is to stdout. 148 | readers : A list of reader methods that are pre-pended to 149 | those already contained in the class. 150 | log_terms : A dictionary of log options. By default 151 | {'logfile':None,'logdir':'log','debug':True} 152 | If thisname is set, and logfile specified, then 153 | logs are logged to that file. If thisname is set 154 | to True, then logging is to stdout. 155 | datadir : A list of directories to search for data files 156 | to interpret if the SpecialVariable is set to a 157 | string. 158 | env : An environment variable that can be used to extend 159 | the datadir variable. 160 | header : A header string to use to identify pickle files. 161 | By default, this is set to 162 | "EOLDAS -- plewis -- UCL -- V0.1" 163 | simple : A flag to swicth off the 'complicated' 164 | interpretation methods, i.e. just set and return 165 | variables literally, do not try to interpret them. 166 | 167 | 168 | ''' 169 | self.set('simple',True) 170 | if name == None: 171 | import time 172 | thistime = str(time.time()) 173 | name = type(self).__name__ 174 | name = "%s.%s" % (name,thistime) 175 | self.thisname = name 176 | 177 | # this is where we will put any data 178 | self.data = ParamStorage() 179 | self.name = ParamStorage() 180 | 181 | self.info = info 182 | 183 | self.datadir = datadir or ['.'] 184 | self.env = env 185 | 186 | init_read_write(self,header,readers,writers) 187 | 188 | # sort logging and log if thisname != None 189 | self.log_terms = {'logfile':None,'logdir':'log','debug':True} 190 | # override logging info 191 | for (key,value) in log_terms.iteritems(): 192 | self.log_terms[key] = value 193 | self.logger= sortlog(self,self.log_terms['logfile'],logger,name=self.thisname,\ 194 | logdir=self.log_terms['logdir'],\ 195 | debug=self.log_terms['debug']) 196 | self.simple = simple 197 | 198 | set = lambda self,this,value :ParamStorage.__setattr__(self,this,value) 199 | set.__name__ = 'set' 200 | set.__doc__ = """ 201 | A method to set the literal value of this, rather than attempt 202 | an interpretation (e.g. used when self.simple is True) 203 | """ 204 | get = lambda self,this :ParamStorage.__getattr__(self,this) 205 | get.__name__ = 'get' 206 | get.__doc__ = """ 207 | A method to get the literal value of this, rather than attempt 208 | an interpretation (e.g. used when self.simple is True) 209 | """ 210 | 211 | def __setitem__(self,this,value): 212 | ''' 213 | Variable setting method for style self['this']. 214 | 215 | Interpreted the same as via __setattr__. 216 | 217 | ''' 218 | # always set the item 219 | self.__setattr__(this,value) 220 | 221 | def __setattr__(self,this,value): 222 | ''' 223 | Variable setting method for style self.this 224 | 225 | Varies what it does depending on the type of value. 226 | 227 | The method interprets and sets the SpecialVariable value: 228 | 229 | 1. ParamStorage or SpecialVariable. The data are directly loaded. 230 | This is one of the most flexible formats for input. It expects 231 | fields 'data' and/or 'name', which are loaded into self. 232 | There will normally be a field data.this, where this is 233 | the variable name passed here. 234 | 2. A dictionary, same format as the ParamStorage. 235 | 3. A tuple, interpreted as (data,name) and loaded accordingly. 236 | 4. *string* as filename (various formats). An attempt to read the 237 | string as a file (of a set of formats) is made. If none pass 238 | then it it maintained as a string. 239 | 5. A numpy array (np.array) that is loaded into self.data.this. 240 | 6. Anything else. Loaded into self.data.this as a numpy array. 241 | 242 | ''' 243 | if self.simple: 244 | self.set(this,value) 245 | return 246 | t = type(value) 247 | try: 248 | if t == ParamStorage or t == SpecialVariable: 249 | # update the whole structure 250 | #self.__set_if_unset('data',ParamStorage()) 251 | #self.__set_if_unset('name',ParamStorage()) 252 | self.data.update(value.data,combine=True) 253 | self.name.update(value.name,combine=True) 254 | elif t == dict: 255 | n_value = ParamStorage().from_dict(value) 256 | self.__setattr__(this,n_value) 257 | elif t == tuple or t == list: 258 | # assumed to be (data,name) or [data,name] 259 | #self.__set_if_unset('data',ParamStorage()) 260 | #self.__set_if_unset('name',ParamStorage()) 261 | #ParamStorage.__setattr__(self['data'],this,value[0]) 262 | #ParamStorage.__setattr__(self['name'],this,value[1]) 263 | ParamStorage.__setattr__(self['data'],this,np.array(value)) 264 | elif t == str: 265 | # set the term 266 | #self.__set_if_unset('data',ParamStorage()) 267 | #self.__set_if_unset('name',ParamStorage()) 268 | ParamStorage.__setattr__(self['data'],this,value) 269 | # interpret as a file read if possible 270 | self.process_data_string(this,info=self.info) 271 | elif t == np.ndarray: 272 | #self.__set_if_unset('data',ParamStorage()) 273 | #self.__set_if_unset('name',ParamStorage()) 274 | ParamStorage.__setattr__(self['data'],this,value) 275 | else: 276 | ParamStorage.__setattr__(self['data'],this,\ 277 | np.array(value)) 278 | except: 279 | if self.logger: 280 | self.logger.info("Failed to set SpecialVariable %s from %s %s"\ 281 | %(this,t.__name__,value)) 282 | return 283 | if self.logger: 284 | self.logger.info("Set variable %s from type %s"%(this,t.__name__)) 285 | 286 | def __getattr__(self,name): 287 | ''' 288 | Variable getting method for style self.this 289 | 290 | If the field 'data' exists in self.__dict__ and 'name' 291 | is in the dictionary, then the field self.data.this is returned. 292 | 293 | Otherwise, if the field 'name' is in self.__dict__, self.name 294 | is returned. 295 | 296 | Otherwise return None. 297 | 298 | ''' 299 | if 'data' in self.__dict__ and name in self.data.__dict__: 300 | return self.data.__dict__.__getitem__ ( name ) 301 | elif name in self.__dict__: 302 | return self.__dict__.__getitem__ ( name ) 303 | else: 304 | return None 305 | 306 | def __getitem__(self,name): 307 | ''' 308 | Variable getting method for style self['this']. 309 | 310 | Interpreted the same as via __getattr__. 311 | 312 | ''' 313 | # first look in data 314 | if 'data' in self.__dict__ and name in self.data.__dict__: 315 | return self.data.__dict__.__getitem__ ( name ) 316 | elif name in self.__dict__: 317 | return self.__dict__.__getitem__ ( name ) 318 | else: 319 | return None 320 | 321 | 322 | def process_data_string(self,name,info=[],fmt=None): 323 | ''' 324 | Attempt to load data from a string, assuming the string is a filename. 325 | 326 | The array self.datadir is searched for readable files with the 327 | string 'name' (also self.env), and a list of potential files 328 | considered for reading. 329 | 330 | Each readable file is passed to self.read, and if it is interpretable, 331 | it is loaded according to the read method. 332 | 333 | Note tha the format can be specified. If not, then all formats 334 | are attempted until a sucessful read is made. 335 | 336 | ''' 337 | from eoldas_Lib import get_filename 338 | 339 | orig = self.data[name] 340 | if self.logger: 341 | self.logger.debug('%s is a string ... see if its a readable file ...' \ 342 | % name) 343 | # find a list of potential files 344 | goodfiles, is_error = get_filename(orig,datadir=self.datadir,\ 345 | env=self.env,multiple=True) 346 | if is_error[0] and self.logger: 347 | self.logger.debug(str(is_error[1])) 348 | return 349 | if self.logger: 350 | self.logger.debug("*** looking at potential files %s"%str(goodfiles)) 351 | # loop over all files that it might be 352 | for goodfile in goodfiles: 353 | stuff,is_error = reader(self,goodfile,name,fmt=fmt,info=info) 354 | if not is_error[0] and self.logger: 355 | self.logger.info("Read file %s "%goodfile) 356 | return 357 | if self.logger: 358 | self.logger.debug(self.error_msg) 359 | return 360 | 361 | 362 | write = lambda self,filename,fmt : writer(self,filename,None,fmt=fmt) 363 | read = lambda self,filename,fmt : reader(self,filename,None,fmt=fmt,info=[]) 364 | 365 | class DemoClass(ParamStorage): 366 | ''' 367 | A demonstration class using SpecialVariable 368 | 369 | The behaviour we desire is that a SpecialVariable 370 | acts like a ParamStorage (i.e. we can get or set 371 | by attribute or item 372 | 373 | e.g. 374 | 375 | x.state = 3 and 376 | x['state'] = 3 377 | 378 | give the same result, and 379 | 380 | print x['state'] and 381 | print x.state 382 | 383 | give the same result. This is easy enough to achieve 384 | for all cases other than getting from x.state. It turns out 385 | that __getattr__ does not override the default method 386 | for state *if* state is set in the class instance. 387 | 388 | To get around that, we have to use a fake name (fakes here) 389 | and instead of storing state, we store _state. This makes daling with 390 | with all of the conditions a little more complicated and a little 391 | slower, but it allows a much more consistent interface. 392 | 393 | At any time, a SpecialVariable can simply be over-written by using 394 | assigning to its fake name. 395 | 396 | e.g. 397 | 398 | instance the class 399 | 400 | x = demonstration() 401 | 402 | set a non-special value 'cheese' 403 | 404 | x.foo = 'bar' 405 | 406 | we can use this as x.foo or x['foo'] 407 | 408 | print x.foo,x['foo'] 409 | 410 | which should give bar bar 411 | 412 | now use the SpecialVariable. There are many way to load this up, 413 | but an easy one is via a dictionary. 414 | 415 | data = {'state':np.ones(2)*5. ,'foo':np.ones(10)} 416 | name = {'state':'of the nation','foo':'bar'} 417 | this = {'data':data,'name':name} 418 | x.state = this 419 | 420 | print x.state,x['state'] 421 | 422 | which gives [ 5. 5.] [ 5. 5.], so we get the same from either 423 | approach. Note that what is returned from the SpecialVariable is 424 | only what is in this['data']['state'], and that is fully the intention 425 | of the SpecialVariable class. It can be loaded with rich information 426 | from a range of sources, but if you want a quick interpretation of 427 | the data (i.e. x.state) you only get what is in x.state, or more fully, 428 | 429 | x._state.data.state 430 | 431 | The other data that we passed to the SpecialVariable is as it was 432 | when read in, but relative to x._state, i.e. we have: 433 | 434 | x._state.name.foo 435 | 436 | which is bar. 437 | 438 | If you want to directly access the SpecialVariable, you can use: 439 | 440 | x.get(x.fakes['state']) 441 | 442 | which is the same as 443 | 444 | x._state or x[x.fakes['state']] 445 | 446 | It is not adviseable to directly use the underscore access as the fakes 447 | lookup dictionary can be changed. It is best to always use 448 | x.fakes['state']. Indeed, if you want to override the 'special' nature 449 | of a term such as 'state', you can simply remove their entry from the 450 | table: 451 | 452 | old_dict = x.fakes.copy() 453 | del x.fakes['state'] 454 | 455 | Now, if you type: 456 | 457 | print x.state 458 | 459 | You get a KeyError for state, so it would have been better to: 460 | 461 | x.fakes = old_dict.copy() 462 | del x.fakes['state'] 463 | x['state'] = x[old_dict['state']] 464 | print x.state 465 | 466 | which should give [ 5. 5.], but the type of x.state will have 467 | changed from SpecialVariable to np.ndarray. 468 | 469 | If you want to convert the SpecialVariable back to a dictionary 470 | you can do: 471 | 472 | print x[x.fakes['state']].to_dict() 473 | 474 | or a little less verbosely: 475 | 476 | print x._state.to_dict() 477 | 478 | 479 | ''' 480 | 481 | 482 | def __init__(self,info=[],thisname=None,readers=[],\ 483 | datadir=["."],\ 484 | env=None,\ 485 | header=None,\ 486 | logger=None, 487 | log_terms={},simple=False): 488 | ''' 489 | Class initialisation. 490 | 491 | Set up self.state and self.other as SpecialVariables 492 | and initialise them to None. 493 | 494 | ''' 495 | 496 | self.set('fakes',{'state':'_state','other':'_other'}) 497 | 498 | nSpecial = len(self.get('fakes')) 499 | for i in self.fakes: 500 | thatname = thisname and "%s.%s"%(thisname,i) 501 | 502 | self[i] = SpecialVariable(logger=logger,info=info,thisname=thatname,\ 503 | readers=readers,datadir=datadir,\ 504 | env=env,\ 505 | header=header,\ 506 | log_terms=log_terms,\ 507 | simple=False) 508 | self[i] = None 509 | 510 | 511 | get = lambda self,this :ParamStorage.__getattr__(self,this) 512 | get.__name__ = 'get' 513 | get.__doc__ = ''' 514 | An alternative interface to get the value of a class member 515 | that by-passes any more complex mechanisms. This returns the 'true' 516 | value of a class member, as opposed to an interpreted value. 517 | ''' 518 | set = lambda self,this,that :ParamStorage.__setattr__(self,this,that) 519 | set.__name__ = 'set' 520 | set.__doc__ = ''' 521 | An alternative interface to set the value of a class member 522 | that by-passes any more complex mechanisms. This sets the 'true' 523 | value of a class member, as opposed to an interpreted value. 524 | ''' 525 | 526 | var = lambda self,this : self[self['fakes'][this]] 527 | var.__name__='var' 528 | var.__doc__ = ''' 529 | Return the data associated with SpecialVariable this, 530 | rather than an interpretation of it 531 | ''' 532 | 533 | 534 | def __set_if_unset(self,name,value): 535 | ''' 536 | A utility to check if the requested attribute 537 | is not currently set, and to set it if so. 538 | ''' 539 | if name in self.fakes: 540 | fname = self.fakes[name] 541 | if not fname in self.__dict__: 542 | ParamStorage.__setattr__(self,fname,value) 543 | return True 544 | else: 545 | if not name in self.__dict__: 546 | ParamStorage.__setattr__(self,name,value) 547 | return True 548 | return False 549 | 550 | 551 | def __getattr__(self,name): 552 | ''' 553 | get attribute, e.g. return self.state 554 | ''' 555 | return self.__getitem__(name) 556 | 557 | def __setattr__(self,name,value): 558 | ''' 559 | set attribute, e.g. self.state = 3 560 | ''' 561 | if not self.__set_if_unset(name,value): 562 | self.__setitem__(name,value,nocheck=True) 563 | 564 | def __getitem__(self,name): 565 | ''' 566 | get item for class, e.g. x = self['state'] 567 | ''' 568 | if name in ['Data','Name']: 569 | return self._state[name.lower()] 570 | elif name in ['Control','Location']: 571 | return self._state[name.lower()] 572 | elif name in self.fakes: 573 | this = self.get(self.fakes[name]) 574 | return SpecialVariable.__getitem__(this,name) 575 | else: 576 | this = self.get(name) 577 | return self.__dict__.__getitem__ ( name ) 578 | 579 | def __setitem__(self,name,value,nocheck=False): 580 | ''' 581 | set item for class e.g. self['state'] = 3 582 | ''' 583 | if nocheck or not self.__set_if_unset(name,value): 584 | if name in ['Data','Name']: 585 | self._state[name.lower()] = value 586 | elif name in ['Control','Location']: 587 | self._state[name.lower()] = value 588 | elif name in self.fakes: 589 | this = self.get(self.fakes[name]) 590 | SpecialVariable.__setattr__(this,name,value) 591 | else: 592 | this = self.get(name) 593 | ParamStorage.__setattr__(self,name,value) 594 | 595 | 596 | def demonstration(): 597 | # set state to a filename 598 | # and it will be loaded with the data 599 | x = DemoClass() 600 | 601 | data = {'state':np.ones(2)*5. ,'foo':np.ones(10)} 602 | name = {'state':'of the nation','foo':'bar'} 603 | this = {'data':data,'name':name} 604 | x.state = this 605 | print 1,x.state,x['state'] 606 | 607 | 608 | x.oats = 'beans and barley-o' 609 | # nothing set so far 610 | print 2,x['state'] 611 | # should return the same 612 | print 3,x.state 613 | x.state = 'test/data_type/input/test.brf' 614 | print 4,x.state 615 | print 5,x.Name.fmt 616 | 617 | # set state to a dict and 618 | # it will load from that 619 | data = {'state':np.zeros(10)} 620 | name = {'state':'foo'} 621 | x.state = {'data':data,'name':name} 622 | print 6,x.state 623 | 624 | # set from a ParamStorage 625 | # and it will be loaded 626 | this = ParamStorage() 627 | this.data = ParamStorage() 628 | this.name = ParamStorage() 629 | this.data.state = np.ones(10) 630 | this.name.state = 'bar' 631 | this.data.sd = np.ones(10)*2. 632 | this.name.sd = 'sd info' 633 | # assign the data 634 | x.state = this 635 | # access the data 636 | print 7,x.state 637 | # access another member 638 | # Data, Name == implicitly .state 639 | print 8,x.Data.sd 640 | print 9,x.Name.sd 641 | # set directly 642 | x.Name.sd = 'bar' 643 | print 10,x.Name.sd 644 | 645 | # set from a tuple (data,name) 646 | # or a list [data,name] 647 | data = 'foo' 648 | name = 'bar' 649 | x.state = (data,name) 650 | print 11,x.state 651 | x.state = [name,data] 652 | print 12,x.state 653 | 654 | # set from a numpy array 655 | x.state = np.array(np.arange(10)) 656 | print 13,x.state 657 | 658 | # set from another state 659 | y = DemoClass() 660 | y.state = x.state 661 | x.state = x.state * 2 662 | print 'x state',x.state 663 | print 'y state',y.state 664 | 665 | # set from a float 666 | x.state = 100. 667 | print 14,x.state 668 | 669 | # another interesting feature 670 | # we have 2 special terms in demonstration 671 | # state and other 672 | # if we set up some strcture for data 673 | # for other 674 | this = ParamStorage() 675 | this.data = ParamStorage() 676 | this.name = ParamStorage() 677 | this.data.other = np.ones(10) 678 | this.name.other = 'bar' 679 | # and the assign it to state 680 | x.state = this 681 | print 15,'state',x.state 682 | # we see state is unchanged 683 | # but other is also not set. 684 | print 16,'other',x.other 685 | # we load into other using: 686 | x.other = this 687 | print 'other',x.other 688 | # but if you look at the information contained 689 | print 17,x._other.to_dict() 690 | print 18,x._state.to_dict() 691 | 692 | # or better writtem as: 693 | print 19,x.var('state').to_dict() 694 | 695 | # you will see that state contains the other data that was loaded 696 | 697 | # a simple way to write out the data is to a pickle 698 | # x.write_pickle('xstate','x_state.pkl') 699 | # but try to avoid using the underscores 700 | print 20,"x state in pickle:",x.state 701 | SpecialVariable.write(x._state,'x_state.pkl',fmt='pickle') 702 | 703 | # which we can reload: 704 | z = DemoClass() 705 | z.state = 'x_state.pkl' 706 | print 21,"z state read from pickle",z.state 707 | 708 | # which is the same as a forced read ... 709 | zz = DemoClass() 710 | zz.state = 'x_state.pkl' 711 | print 22,zz.state 712 | 713 | # read a brf file 714 | zz.Name.qlocation = [[170,365,1],[0,500,1],[200,200,1]] 715 | zz.state = 'test/data_type/input/interpolated_data.npz' 716 | print zz.state 717 | SpecialVariable.write(zz._state,'test/data_type/output/interpolated_data.pkl',fmt='pickle') 718 | zz.state = 'test/data_type/input/test.brf' 719 | print zz.state 720 | 721 | # so we can convenirntly use pickle format as an interchange 722 | 723 | 724 | if __name__ == "__main__": 725 | demonstration() 726 | help(SpecialVariable) 727 | help(DemoClass) 728 | 729 | 730 | 731 | -------------------------------------------------------------------------------- /eoldas/eoldas_ConfFile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import pdb 3 | import sys 4 | import numpy as np 5 | import os 6 | from eoldas_ParamStorage import ParamStorage 7 | 8 | def type_convert(info,this): 9 | """ 10 | Function to make a guess at the type of a variable 11 | defined as a string 12 | 13 | 1. Variables from within the config file can be referred to via $ 14 | e.g.: 15 | [model] 16 | names = eeny,meeny,miney 17 | [parameter] 18 | names = $model.names,moe 19 | 20 | so: 21 | 22 | model.names = ['eeny','meeny','miney'] 23 | parameter.names = [['eeny','meeny','miney'],'moe'] 24 | 25 | 2. Any parameter names that begin 'assoc_' are 26 | special lists/arrays where the parameter values 27 | are defined associated with a parameter name. If this 28 | mechanism is used, parameter.names *must* be defined. 29 | 30 | For example: 31 | 32 | [parameter] 33 | names= gamma,xlai, xhc, rpl, xkab, scen, xkw, xkm, 34 | xleafn, xs1,xs2,xs3,xs4,lad 35 | 36 | 37 | [parameter.assoc_bounds] 38 | gamma = 0.01,None 39 | xlai = 0.01,0.99 40 | xhc = 0.01,10.0 41 | rpl = 0.001,0.10 42 | xkab = 0.1,0.99 43 | scen = 0.0,1.0 44 | xkw = 0.01,0.99 45 | xkm = 0.3,0.9 46 | xleafn = 0.9,2.5 47 | xs1 = 0.0, 4. 48 | xs2 = 0.0, 5. 49 | xs3 = None, None 50 | xs4 = None, None 51 | lad = None, None 52 | 53 | 54 | Statements in the config file go through an attempt at 55 | evaluation. This is partly to set the type (i.e. int, float etc) 56 | to something reasonable, but has the by-product of allowing 57 | quite a flexible statement definition. 58 | 59 | For example, we can apply a logical statement 60 | 61 | [general] 62 | rule = True 63 | unrule = not $general.rule 64 | 65 | which sets: 66 | 67 | general.rule = True 68 | general.unrule = False 69 | 70 | By default, you have several python modules that 71 | you can call from within the config file. 72 | 73 | These are those resulting from: 74 | 75 | import sys 76 | import numpy as np 77 | import os 78 | from eoldas_parser import Parser 79 | from eoldas_params_storage import ParamStorage 80 | 81 | so, for example you can call: 82 | 83 | [general] 84 | x = np.random.rand(len($parameter.names)) 85 | datadir = .,sys.exec_prefix 86 | here = os.getcwdu() 87 | nice = os.nice(19) 88 | 89 | which results in: 90 | 91 | general.x = [ 0.40663379 0.93222149 0.95141211 92 | 0.06559793 0.71743235 0.22486385 93 | 0.57224872 0.73858977 0.19203518 94 | 0.70350535 0.7228549 0.86616254 95 | 0.34699597 0.78502996] 96 | general.datadir = ['.', '/usr/local/epd-7.0-2-rh5-x86_64'] 97 | general.here = /data/somewhere/eoldaslib 98 | general.nice = 19 99 | 100 | Finally, an attempt is made at executing a statement. This might 101 | prove a little dangerous (e.g. you could (intentionally or maliciously) 102 | use it to delete files when parsing a conf file, but if you want to 103 | do that there are plenty of other ways to achieve it). An example of 104 | when this might be useful might be if you want to import a python class 105 | whilst parsing the config file (for some reason). 106 | 107 | [general] 108 | np = import numpy as np 109 | npp = import somethingThatDoesntExist 110 | 111 | which sets: 112 | 113 | general.np = import numpy as np : exec True 114 | general.npp = import somethingThatDoesntExist 115 | 116 | Note that if a statement has been executed (i.e. run sucessfully 117 | through exec()). 118 | 119 | Then the string ' : exec True' is added to the end of 120 | its value as stored. 121 | 122 | In the case of 'import somethingThatDoesntExist', this was 123 | not sucessfully interpreted via an exec() and so is simply assumed to be 124 | a string. 125 | 126 | You might need to be a little careful in using strings. I suppose 127 | it is vaguely conceivable that you might actually want to set 128 | general.np to 'import numpy as np'. Note that items such as those 129 | you have imported are not available to other lines of the config, 130 | so import serves little purpose, other than testing perhaps. 131 | 132 | There might come a point where you might be doing so much coding 133 | in the config file that you might be better off writing some 134 | new methods/classes, but some degree of processing is of value, and 135 | at no great computing complexity or processing cost. 136 | 137 | """ 138 | # first, try to eval it 139 | # for this to work, we need to put info. 140 | # on anything that looks like a variable 141 | # which are flagged by using a $ 142 | try: 143 | orig_this = this 144 | this = this.replace('$','info.') 145 | try: 146 | return eval(this) 147 | except: 148 | try: 149 | exec(this) 150 | #this += ' : exec True' 151 | return this 152 | except: 153 | pass 154 | if this != orig_this: 155 | # then we have failed to interpret something 156 | # log an error and return None 157 | raise Exception( 'Failed to interpret %s' % orig_this) 158 | except: 159 | pass # dont worry if it doesnt work 160 | try: 161 | this_value = eval(this) 162 | return this_value 163 | except: 164 | try: 165 | exec(this) 166 | this += ' : exec True' 167 | return this 168 | except: 169 | pass 170 | # otherwise just return it, with blanks stripped 171 | return this.strip() 172 | 173 | 174 | def array_type_convert(info,this): 175 | """ 176 | 177 | Parameters: 178 | info : a dictionary 179 | this : a string 180 | 181 | The string this is split on CSV and each element 182 | passed through to be interpreted by the method 183 | type_convert. A 'safe' split is performed, which 184 | doesn't split on [] or (). 185 | 186 | """ 187 | # we dont want to split commas inside [] or () 188 | if type(this) != str: 189 | return this 190 | x = safesplit(this,',') 191 | n = len(x) 192 | if n == 1: 193 | return type_convert(info,x[0]) 194 | else: 195 | that = [] 196 | for i in xrange(n): 197 | that.append(type_convert(info,x[i])) 198 | # youd think np.flatten would be 199 | # fine here, but it isnt 200 | other = [] 201 | for i in that: 202 | if type(i) == list: 203 | for j in i: 204 | other.append(j) 205 | else: 206 | other.append(i) 207 | that = np.array(other) 208 | return that 209 | 210 | def safesplit(text, character): 211 | ''' 212 | A function to split a string, taking account of [] and () 213 | and quotes 214 | ''' 215 | lent = len(text) 216 | intoken = ParamStorage() 217 | qtoken = ParamStorage() 218 | for i in ["'",'"']: 219 | qtoken[i] = False 220 | for i in ["()","[]","{}"]: 221 | intoken[i] = 0 222 | 223 | start = 0 224 | lst = [] 225 | i = 0 226 | while i < lent: 227 | # are any of intoken, qtoken open 228 | isopen = False 229 | for (j,k) in intoken.iteritems(): 230 | if text[i] == j[0]: 231 | intoken[j] += 1 232 | elif text[i] == j[1]: 233 | intoken[j] -= 1 234 | isopen = isopen or (intoken[j]!=0) 235 | for (j,k) in qtoken.iteritems(): 236 | if text[i] == j[0]: 237 | qtoken[j] = not qtoken[j] 238 | isopen = isopen or qtoken[j] 239 | if text[i] == character and not isopen: 240 | lst.append(text[start:i]) 241 | start = i+1 242 | elif text[i] == '\\': 243 | i += 2 244 | continue 245 | i += 1 246 | lst.append(text[start:]) 247 | return lst 248 | 249 | def assoc_to_flat(names,data,prev): 250 | ''' 251 | Given an array of names (names) 252 | and a dataset (data) 253 | convert to a flat array 254 | 255 | Parameters: 256 | names: a list of names 257 | data: a list of names OR 258 | a dictionary 259 | prev: prev array to load into 260 | 261 | If data is an array of names, a boolean numpy array is returned 262 | of size len(names) which is True where an element of data appears 263 | in names and False elsewhere. This is useful to subset an array 264 | of the same size as names into one associated with the data array. 265 | 266 | If data is a dictionary then the items associated with the keys 267 | in the list names is returned as a numpy array. If an item doesnt 268 | exist in data, then None is returned. 269 | 270 | This latter use is the main purpose of assoc_to_flat(). In effect 271 | it loads items from a dictionary into an array, based on the keys 272 | in the list names. 273 | 274 | In the class ConfFile it is used to translate any data structure 275 | which has a name starting assoc_ into a numpy array of the same name 276 | (without the assoc_) at the same level of the hierarchy. 277 | 278 | ''' 279 | out = [] 280 | for (i,n) in enumerate(names): 281 | try: 282 | # try to pull from a key 283 | this = data[n] 284 | except: 285 | if type(data) == dict or type(data) == ParamStorage: 286 | this = prev[i] 287 | else: 288 | try: 289 | # array 290 | ndata = np.array(data) 291 | this = bool((ndata == n).sum()) 292 | except: 293 | this = None 294 | out.append(this) 295 | return np.array(out) 296 | 297 | class ConfFile(object): 298 | ''' 299 | A configuration file parser class. 300 | 301 | Parameters: 302 | options: The class is initialised with "options", 303 | which may be of type ParamStorage 304 | or a list or a string. 305 | 306 | If options is a list or a string, it is assumed to contain the names 307 | of one or more configuration files. The list may contain lists 308 | that also contain the names of configuration files. 309 | 310 | The parsed information is put into self.infos (a list containing 311 | elements of ParamStorage type). 312 | 313 | The length of self.infos will normally be the same as the length of the 314 | list "options". 315 | 316 | If "options" is a string (the name of a configuration file) this is the 317 | same as a list with a single element. 318 | 319 | If the list contains sub-lists, these are all read into the 320 | same part of self.infos. 321 | 322 | The raw configuration information associated with each element 323 | of self.infos is in a list self.configs. 324 | 325 | If a configration file is not found or is invalid, there is a 326 | fatal error ONLY of fatal=True. 327 | 328 | Configuration files are searched for by absolute path name, 329 | if one is given. If there is a failure to read the file referred 330 | to by absolute pathname, the local name of the file is assumed, and the 331 | search continues for that file through the directory list datadir. 332 | 333 | If the path name is relative, the file is searched for 334 | through the directory list datadir. 335 | 336 | The class is derived from the object class. 337 | information via the command line. 338 | 339 | See type_convert() for some rules on defining a configration file. 340 | 341 | 342 | ''' 343 | def __init__(self,options,name=None,loaders=[],logfile=None,\ 344 | logdir=None,datadir=['.','~/.eoldas'],logger=None\ 345 | ,env="EOLDAS",fatal=False): 346 | ''' 347 | Parse the configuration file(s) referred to in the (string or list) 348 | "options". 349 | 350 | Parameters: 351 | 352 | options : The class is initialised with "options", 353 | which may be of type ParamStorage 354 | or a list or a string. 355 | "options" may also be of type ParamStorage, 356 | in which case it is simply loaded here. 357 | 358 | Options: 359 | 360 | log_name : name of log. If none is specified, 361 | logging goes to stdout. 362 | log : boolean, specifying whether or not to do 363 | logging. 364 | datadir : a list of strings, specifying where to 365 | look for configuration files. 366 | env : an environment variable that may contain 367 | more places to search for configuration files 368 | (after datadir) (default "EOLDAS"). 369 | fatal : specify whether not finding a configuration 370 | file should be fatal or not (default False) 371 | 372 | See type_convert() for some rules on defining a configration file. 373 | 374 | ''' 375 | from eoldas_Lib import sortlog 376 | if name == None: 377 | import time 378 | thistime = str(time.time()) 379 | name = type(self).__name__ 380 | name = "%s.%s" % (name,thistime) 381 | self.thisname = name 382 | self.logdir = logdir 383 | self.logfile = logfile 384 | self.storelog = "" 385 | self.datadir=datadir 386 | self.env=env 387 | self.fatal=fatal 388 | self.logger = sortlog(self,self.logfile,logger,name=self.thisname,logdir=self.logdir,debug=True) 389 | self.configs = [] 390 | self.infos = [] 391 | self.loaders = loaders 392 | if type(options) == ParamStorage: 393 | self.options.update(options,combine=True) 394 | if type(options)== list or type(options) == str: 395 | self.read_conf_file(options) 396 | #if log == True: 397 | # self.loglist(self.info) 398 | 399 | def loglist(self,info): 400 | ''' 401 | Utility to send log items to the logger 402 | ''' 403 | 404 | # check to see if info is of type ParamStorage 405 | if type(info) != ParamStorage: 406 | return 407 | # check to see if the ParamStorage item has 408 | # name and/or doc defined and log as appropriate 409 | if hasattr(info,'__name__'): 410 | if hasattr(info,'__doc__'): 411 | self.logger.info("%s: %s" % (str(info.__name__),\ 412 | str(info.__doc__))) 413 | if hasattr(info,'__name__'): 414 | self.logger.info("%s: " % (str(info.__name__))) 415 | if 'helper' in info.dict(): 416 | # log might be a string or a list 417 | if type(info.helper) == str: 418 | this = info.helper.split('\n') 419 | else: 420 | this = info.helper 421 | if this != None: 422 | for i in this: 423 | self.logger.info("%s" % i) 424 | 425 | # loop over items in this ParamStorage and recursively call 426 | # loglist for them 427 | for (k,v) in info.iteritems(): 428 | if k[:2] != '__': 429 | self.loglist(info[k]) 430 | return 431 | 432 | def read_conf_file (self, conf_files,options=None): 433 | """ 434 | This method reads one or more configuration files into 435 | self.infos and self.configs. 436 | 437 | Parameters: 438 | conf_files : list of one or more config files 439 | 440 | """ 441 | if type(conf_files) == str: 442 | conf_files = [conf_files] 443 | # give up the idea of multiple sets of conf files 444 | 445 | fname = np.array(conf_files).flatten().tolist() 446 | for (i,this) in enumerate(fname): 447 | fname[i] = os.path.expanduser(this) 448 | self.logger.info ("Trying config files: %s" % fname ) 449 | config,info,config_error = self.read_single_conf_file( \ 450 | fname,options=options) 451 | if config == False: 452 | self.logger.error(config_error) 453 | else: 454 | self.loglist(info) 455 | self.configs.append(config) 456 | self.infos.append(info) 457 | return self.configs 458 | 459 | def rescan_info(self,config,this,thisinfo,fullthis,info,d): 460 | ''' 461 | Try to eval all terms 462 | ''' 463 | if d > 10: 464 | return 465 | for i in thisinfo.dict().keys(): 466 | if type(thisinfo[i]) == ParamStorage: 467 | self.rescan_info(config,this,thisinfo[i],fullthis,info,d+1) 468 | else: 469 | try: 470 | # only do strings that have 'info' in them 471 | if thisinfo[i].count('info.'): 472 | thisinfo[i] = eval('%s'%str(thisinfo[i])) 473 | except: 474 | pass 475 | 476 | def scan_info(self,config,this,info,fullthis,fullinfo): 477 | """ 478 | Take a ConfigParser instance config and scan info into 479 | config.info. This is called recursively if needed. 480 | 481 | Parameters: 482 | config : the configuration object 483 | this : the current item to be parsed 484 | info : where this item is to go 485 | fullthis : the full name of this 486 | fullinfo : the full (top level) version of info. 487 | 488 | """ 489 | from eoldas_ConfFile import assoc_to_flat 490 | # find the keys in the top level 491 | # loop over 492 | thiss = np.array(this.split('.')) 493 | # just in case .. is used as separator 494 | ww = np.where(thiss != '') 495 | thiss = thiss[ww] 496 | nextone = '' 497 | for i in xrange(1,len(thiss)-1): 498 | nextone = nextone + thiss[i] + '.' 499 | if len(thiss) > 1: 500 | nextone = nextone + thiss[-1] 501 | # first, check if its already there 502 | if not hasattr(info,thiss[0]): 503 | info[thiss[0]] = ParamStorage() 504 | info[thiss[0]].helper = [] 505 | # load up the info 506 | if len(thiss) == 1: 507 | for option in config.options(fullthis): 508 | fulloption = option 509 | # option may have a '.' separated term as well 510 | options = np.array(option.split('.')) 511 | # tidy up any double dot stuff 512 | ww = np.where(options != '') 513 | options = options[ww] 514 | # need to iterate to make sure it is loaded 515 | # at the right level 516 | # of the hierachy 517 | this_info = info[this] 518 | # so now this_info is at the base 519 | for i in xrange(len(options)-1): 520 | if not hasattr(this_info,options[i]): 521 | this_info[options[i]] = ParamStorage() 522 | this_info[options[i]].helper = [] 523 | this_info = this_info[options[i]] 524 | option = options[-1] 525 | this_info[option] = array_type_convert(fullinfo,\ 526 | config.get(fullthis,fulloption)) 527 | if option[:6] == 'assoc_': 528 | noption = option[6:] 529 | this_info[noption] = assoc_to_flat(\ 530 | fullinfo.parameter.names,this_info[option],\ 531 | this_info[noption]) 532 | is_assoc = True 533 | else: 534 | is_assoc = False 535 | if not hasattr(this_info,'helper'): 536 | this_info.helper = [] 537 | ndot = len(fullthis.split('.')) 538 | pres = '' 539 | for i in xrange(1,ndot): 540 | pres += ' ' 541 | if type(this_info.helper) == str: 542 | this_info.helper += "\n%s%s.%-8s = %-8s" % \ 543 | (pres,fullthis,fulloption,str(this_info[option])) 544 | elif type(this_info.helper) == list: 545 | this_info.helper.append("%s%s.%-8s = %-8s" % \ 546 | (pres,fullthis,fulloption,\ 547 | str(this_info[option]))) 548 | if is_assoc: 549 | if type(this_info.helper) == str: 550 | this_info.helper += "\n%s%s.%-8s = %-8s" % \ 551 | (pres,fullthis,fulloption.replace\ 552 | ('assoc_',''),str(this_info[noption])) 553 | elif type(this_info.helper) == list: 554 | this_info.helper.append("%s%s.%-8s = %-8s" % \ 555 | (pres,fullthis,fulloption.replace\ 556 | ('assoc_',''),str(this_info[noption]))) 557 | else: 558 | self.scan_info(config,nextone,info[thiss[0]],fullthis,fullinfo) 559 | if thiss[-1][:6] == 'assoc_' and thiss[0] in fullinfo.dict(): 560 | # only do this operation when at the top level 561 | noption = thiss[-1][6:] 562 | option = thiss[-1] 563 | this_info = info 564 | fulloption = thiss[0] 565 | this_info = this_info[thiss[0]] 566 | for i in xrange(1,len(thiss)-1): 567 | this_info = this_info[thiss[i]] 568 | fulloption = '%s.%s' % (fulloption,thiss[i]) 569 | fulloption = '%s.%s' % (fulloption,noption) 570 | #this_info[noption] = assoc_to_flat(fullinfo.parameter.names\ 571 | # ,this_info[option],\ 572 | # this_info[noption]) 573 | if not 'names' in this_info.dict(): 574 | this_info.names = fullinfo.parameter.names 575 | 576 | if not option in this_info.dict(): 577 | this_info[option] = [0]*len(this_info.names) 578 | if not noption in this_info.dict(): 579 | this_info[noption] = [0]*len(this_info.names) 580 | this_info[noption] = assoc_to_flat(this_info.names\ 581 | ,this_info[option],\ 582 | this_info[noption]) 583 | 584 | ndot = len(fullthis.split('.')) 585 | pres = '' 586 | for i in xrange(1,ndot): 587 | pres += ' ' 588 | if type(this_info.helper) == str: 589 | this_info.helper += "\n%s%-8s = %-8s" % (pres,\ 590 | fulloption,str(this_info[noption])) 591 | elif type(this_info.helper) == list: 592 | this_info.helper.append("%s%-8s = %-8s" % (pres,\ 593 | fulloption,str(this_info[noption]))) 594 | 595 | def read_single_conf_file (self, conf_files,options=None): 596 | """ 597 | 598 | Purpose: 599 | parse the information from conf_files into a 600 | ConfigParser class instance and return this. 601 | 602 | Parameters: 603 | conf_files : list of one or more config files 604 | 605 | Options: 606 | options=None : pass an options structure through 607 | 608 | Uses: 609 | self.datadir=['.',,'~/.eoldas'] 610 | : list of directories to look 611 | for config files 612 | self.env=None : name of an environment variable 613 | where config files 614 | can be searched for if not found 615 | in datadir (or absolute 616 | path name not given) 617 | self.fatal=False : flag to state whether the 618 | call should fail if 619 | a requested config file is not found. 620 | 621 | 622 | Returns: 623 | tuple : (config, config_error) 624 | where: 625 | config : ConfigParser class instance 626 | or False if an error occurs 627 | config_error : string giving information on error 628 | """ 629 | import ConfigParser 630 | from eoldas_Lib import get_filename 631 | # Instantiate a parser 632 | config = ConfigParser.ConfigParser() 633 | # Read the config files. If it doesn't exist, raise exception. 634 | # 635 | if type(conf_files) == str: 636 | conf_files = [conf_files] 637 | all_conf_files = [] 638 | 639 | for fname in conf_files: 640 | fname,fname_err = get_filename(fname,datadir=self.datadir,\ 641 | env=self.env) 642 | if fname_err[0] != 0: 643 | if self.fatal: 644 | return False,False,\ 645 | "Cannot find configuration file %s\n%s" \ 646 | % (fname,fname_err[1]) 647 | else: 648 | all_conf_files.append(fname) 649 | thisdir = os.path.dirname(fname) 650 | if not thisdir in self.datadir: 651 | self.datadir.append(thisdir) 652 | if len(all_conf_files) == 0: 653 | return False,False,\ 654 | "%s: No valid conf files found in list %s in dirs %s" \ 655 | % (os.getcwd(),conf_files,self.datadir) 656 | config.config_files = config.read(all_conf_files) 657 | if len(config.config_files) == 0: 658 | return False,False,\ 659 | "%s: No valid conf files found in list %s in dirs %s" \ 660 | % (os.getcwd(),conf_files,self.datadir) 661 | 662 | # from here on, we attempt to pull specific information from 663 | # the conf files 664 | info = ParamStorage(name='info',doc=\ 665 | 'Configuration information for %s' % \ 666 | str(config.config_files)) 667 | 668 | # scan everything into config.info 669 | # but it helps to sort it to get the info in the right order 670 | sections = config.sections() 671 | #sections.sort() 672 | firstsections = [] 673 | secondsections = [] 674 | for this in sections: 675 | if this[:7] == 'general' or this[:9] == 'parameter': 676 | firstsections.append(this) 677 | else: 678 | secondsections.append(this) 679 | firstsections.sort() 680 | sections = firstsections 681 | [sections.append(i) for i in secondsections] 682 | for this in sections: 683 | self.logger.debug('...Section %s'%this) 684 | self.scan_info(config,this,info,this,info) 685 | self.rescan_info(config,this,info,this,info,0) 686 | 687 | 688 | self.config = config 689 | self.info = info 690 | if options != None and type(options) == ParamStorage: 691 | self.info.update(options,combine=True) 692 | 693 | # sort any helper text looping over self.info 694 | # into self.loaders 695 | self.__sort_help(self.info,"") 696 | try: 697 | self.logger.info("Config: %s read correctly" \ 698 | % str(all_conf_files)) 699 | except: 700 | pass 701 | return self.config,self.info,"Config: %s read correctly" \ 702 | % str(all_conf_files) 703 | 704 | def __sort_help(self,info,name): 705 | ''' 706 | sort any helper_ options into the list 707 | self.loaders 708 | 709 | loader format is of the form: 710 | 711 | self.loaders.append(["datadir",['.',self.options.here], 712 | "Specify where the data and or conf files are"]) 713 | 714 | ''' 715 | for (key,item) in info.iteritems(): 716 | if key[:5] == 'help_': 717 | hkey = key[5:] 718 | # this is potentially help text for hkey 719 | # If hitem exists and is of type ParamStorage 720 | # then its internal text 721 | if hkey in info.dict().keys(): 722 | hitem = info[hkey] 723 | thisname = '%s%s' % (name,hkey) 724 | hthisname = '%s%s' % (name,key) 725 | if type(item) != ParamStorage: 726 | # if hkey starts with 'general.' get rid of that 727 | if thisname[:8] == 'general.': 728 | thisname = thisname[8:] 729 | # insert into self.loaders 730 | isloaded = False 731 | for i in xrange(len(self.loaders)): 732 | if self.loaders[i][0] == thisname: 733 | self.loaders[i][1] = hitem 734 | self.loaders[i][2] = item 735 | isloaded = True 736 | if not isloaded: 737 | self.loaders.append([thisname,hitem,item]) 738 | elif type(item) == ParamStorage and key[:2] != '__': 739 | # recurse 740 | self.__sort_help(item,'%s%s.'%(name,key)) 741 | 742 | def demonstration(): 743 | ''' 744 | A test call to use ConfFile. 745 | 746 | We import the class. 747 | Then we initialise a instance of ConfFile with 748 | the configuration file "default.conf" 749 | 750 | ''' 751 | from eoldas_ConfFile import ConfFile 752 | print "Testing ConfFile class with conf file default.conf" 753 | self = ConfFile('default.conf') 754 | # no log has been set up, so logging info 755 | # is stored in self.storelog 756 | print "logger info:" 757 | print self.storelog 758 | 759 | if __name__ == "__main__": 760 | demonstration() 761 | help(type_convert) 762 | help(array_type_convert) 763 | help(safesplit) 764 | help(assoc_to_flat) 765 | help(ConfFile) 766 | --------------------------------------------------------------------------------