├── .github └── workflows │ └── python-package.yml ├── .gitignore ├── CHANGES.txt ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── bin ├── README.md └── test.py ├── docs ├── documentation.md └── img │ ├── all_estimates.png │ ├── err_evo.png │ ├── ga_evolution.png │ ├── logo_src │ ├── modest-logo.png │ ├── modest-logo.xcf │ └── pop_evo_4.png │ └── modest-logo.png ├── examples ├── lin │ ├── README.md │ ├── model.png │ ├── resources │ │ └── lin_model.mo │ └── showcase │ │ ├── GA_search_path.png │ │ ├── PS_search_path.png │ │ └── RMSE_grid.png ├── simple │ ├── README.md │ ├── resources │ │ ├── Simple2R1C_ic.mo │ │ ├── Simple2R1C_ic_linux64.fmu │ │ ├── Simple2R1C_ic_win32.fmu │ │ ├── Simple2R1C_ic_win64.fmu │ │ ├── est.json │ │ ├── est_validate_hj.json │ │ ├── inputs.csv │ │ ├── known.json │ │ ├── known_validate_hj.json │ │ ├── result.csv │ │ └── true_parameters.csv │ ├── simple-slsqp.py │ ├── simple.py │ ├── simple_1param.py │ ├── simple_ga_vs_ga_parallel.py │ ├── simple_legacy_ga.py │ └── validate_hj.py └── sin │ ├── README.md │ ├── model.png │ ├── resources │ └── sin_model.mo │ └── showcase │ ├── GA_PS_search_path.png │ ├── GA_search_path.png │ ├── PS_search_path_bad_initial.png │ ├── PS_search_path_good_initial.png │ └── RMSE_grid.png ├── modestpy ├── __init__.py ├── estim │ ├── __init__.py │ ├── error.py │ ├── estpar.py │ ├── ga │ │ ├── __init__.py │ │ ├── algorithm.py │ │ ├── ga.py │ │ ├── individual.py │ │ └── population.py │ ├── ga_parallel │ │ ├── __init__.py │ │ └── ga_parallel.py │ ├── make_param_file.py │ ├── plots.py │ ├── ps │ │ ├── __init__.py │ │ └── ps.py │ └── scipy │ │ ├── __init__.py │ │ └── scipy.py ├── estimation.py ├── fmi │ ├── __init__.py │ ├── compiler.py │ ├── fmpy_test.py │ ├── fmpy_warning_example.py │ ├── model.py │ └── model_pyfmi.py ├── loginit.py ├── test │ ├── README.md │ ├── __init__.py │ ├── resources │ │ ├── __init__.py │ │ ├── simple2R1C │ │ │ ├── Simple2R1C.mo │ │ │ ├── Simple2R1C_linux64.fmu │ │ │ ├── Simple2R1C_win32.fmu │ │ │ ├── Simple2R1C_win64.fmu │ │ │ ├── est.json │ │ │ ├── inputs.csv │ │ │ ├── known.json │ │ │ ├── parameters.csv │ │ │ └── result.csv │ │ └── simple2R1C_ic │ │ │ ├── Simple2R1C_ic.mo │ │ │ ├── Simple2R1C_ic_linux64.fmu │ │ │ ├── Simple2R1C_ic_win32.fmu │ │ │ ├── Simple2R1C_ic_win64.fmu │ │ │ ├── est.json │ │ │ ├── inputs.csv │ │ │ ├── known.json │ │ │ ├── result.csv │ │ │ └── true_parameters.csv │ ├── run.py │ ├── test_estimation.py │ ├── test_fmpy.py │ ├── test_ga.py │ ├── test_ga_parallel.py │ ├── test_ps.py │ ├── test_scipy.py │ └── test_utilities.py └── utilities │ ├── __init__.py │ ├── delete_logs.py │ ├── figures.py │ ├── parameters.py │ └── sysarch.py ├── requirements.txt └── setup.py /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Ubuntu18.04 5 | 6 | on: 7 | push: 8 | branches: [ "master" ] 9 | pull_request: 10 | branches: [ "master" ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-18.04 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | python-version: ["3.6", "3.7"] 20 | 21 | steps: 22 | - uses: actions/checkout@v3 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v3 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | - name: Install dependencies 28 | run: | 29 | sudo apt-get install -y libgfortran3 gcc g++ 30 | python -m pip install --upgrade pip 31 | python -m pip install . 32 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 33 | - name: Lint with flake8 34 | run: | 35 | # stop the build if there are Python syntax errors or undefined names 36 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 37 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 38 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 39 | - name: Test with unittest 40 | run: | 41 | python modestpy/test/run.py 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | venv/ 2 | .idea/ 3 | *.ipynb 4 | *.pyc 5 | *log.txt 6 | *.log 7 | **/workdir/** 8 | build/ 9 | .vscode/ 10 | *.swp 11 | 12 | # Setuptools distribution folder. 13 | dist/ 14 | 15 | # Python egg metadata, regenerated from source files by setuptools. 16 | *.egg-info 17 | -------------------------------------------------------------------------------- /CHANGES.txt: -------------------------------------------------------------------------------- 1 | 2 | Changes in v.0.1.1 3 | ==================== 4 | - temporary directories are now automatically removed 5 | - added Dockerfile with exemplary setup working with ModestPy 6 | 7 | Changes in v.0.1 8 | ==================== 9 | - parallel genetic algorithm added (based on modestga) 10 | - FMPy instead of pyFMI 11 | 12 | Changes in v.0.0.9: 13 | ==================== 14 | - it is possible now to estimate just 1 parameter (fixed bug in plot_pop_evo()) 15 | 16 | Changes in v.0.0.8: 17 | ==================== 18 | - Version used in the ModestPy paper 19 | - Added interface to SciPy algorithms 20 | 21 | Changes in v.0.0.7: 22 | ==================== 23 | - added SQP method 24 | - modified interface of the Estimation class to facilitate multi-algorithm pipelines 25 | 26 | Changes in v.0.0.6: 27 | ==================== 28 | - LHS initialization of GA 29 | - random seed 30 | - many small bug fixes 31 | 32 | Changes in v.0.0.5: 33 | ==================== 34 | - Decreased tolerance of CVode solver in PyFMI 35 | 36 | Changes in v.0.0.4: 37 | ==================== 38 | - New pattern search plot (parameter evolution) added to Estimation.py 39 | - GA/PS default parameters tuned 40 | 41 | Changes in v.0.0.3: 42 | ==================== 43 | - Tolerance criteria for GA and PS exposed in the Estimation API. 44 | 45 | Changes in v.0.0.2: 46 | ==================== 47 | - Estimation class imported directly in __init__.py to allow imports like "from modestpy import Estimation". 48 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:18.04 2 | 3 | WORKDIR /modestpy 4 | 5 | # System 6 | ARG DEBIAN_FRONTEND=noninteractive 7 | 8 | RUN apt-get update 9 | RUN apt-get install -y libgfortran3 gcc g++ 10 | RUN apt-get install -y python3 python3-pip 11 | RUN apt-get install -y libjpeg8-dev zlib1g-dev 12 | 13 | # Modestpy 14 | WORKDIR /modestpy 15 | COPY . . 16 | RUN python3 -m pip install -U pip 17 | RUN python3 -m pip install . 18 | RUN python3 -m pip install -r requirements.txt 19 | ENTRYPOINT ["/bin/bash"] 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2017, University of Southern Denmark 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include LICENSE 3 | recursive-include modestpy/test/resources/simple2R1C * 4 | recursive-include modestpy/test/resources/simple2R1C_ic * 5 | recursive-include docs * -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build stop rm run bash test 2 | 3 | build: stop rm 4 | DOCKER_BUILDKIT=1 docker build -t modestpy . 5 | 6 | stop: 7 | -docker stop modestpy_container 8 | 9 | rm: 10 | -docker rm modestpy_container 11 | 12 | run: 13 | docker run \ 14 | -t \ 15 | -d \ 16 | --name modestpy_container \ 17 | modestpy 18 | 19 | bash: 20 | docker exec -ti modestpy_container bash 21 | 22 | test: 23 | docker exec -ti modestpy_container python3 modestpy/test/run.py 24 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | FMI-compliant Model Estimation in Python 2 | ======================================== 3 | 4 | .. figure:: /docs/img/modest-logo.png 5 | :alt: modestpy 6 | 7 | .. figure:: https://github.com/sdu-cfei/modest-py/actions/workflows/python-package.yml/badge.svg?branch=master 8 | :alt: unit_test_status 9 | :target: https://github.com/sdu-cfei/modest-py/actions/workflows/python-package.yml 10 | 11 | .. figure:: https://img.shields.io/pypi/dm/modestpy.svg 12 | :alt: downloads 13 | :target: https://pypistats.org/packages/modestpy 14 | 15 | Description 16 | ----------- 17 | 18 | **ModestPy** facilitates parameter estimation in models compliant with 19 | `Functional Mock-up Interface `__. 20 | 21 | Features: 22 | 23 | - combination of global and local search methods (genetic algorithm, pattern search, truncated Newton method, L-BFGS-B, sequential least squares), 24 | - suitable also for non-continuous and non-differentiable models, 25 | - scalable to multiple cores (genetic algorithm from `modestga `_), 26 | - Python 3. 27 | 28 | **Due to time constraints, ModestPy is no longer actively developed. The last system known to work well was Ubuntu 18.04.** 29 | Unit tests in GitHub Actions are run on Ubuntu 18.04 and Python 3.6/3.7. 30 | It does not mean it will not work on other systems, but it is not guaranteed. 31 | Use Docker (as described below) if you want to run ModestPy on a tested platform. 32 | 33 | Installation with pip (recommended) 34 | ----------------------------------- 35 | 36 | It is now possible to install ModestPy with a single command: 37 | 38 | :: 39 | 40 | pip install modestpy 41 | 42 | Alternatively: 43 | 44 | :: 45 | 46 | pip install https://github.com/sdu-cfei/modest-py/archive/master.zip 47 | 48 | Installation with conda 49 | ----------------------- 50 | 51 | Conda is installation is less frequently tested, but should work: 52 | 53 | :: 54 | 55 | conda config --add channels conda-forge 56 | conda install modestpy 57 | 58 | Docker 59 | ------------ 60 | 61 | **Due to time constraints, Modestpy is no longer actively developed. 62 | The last system known to work well was Ubuntu 18.04.** 63 | If you encounter any issues with running ModestPy on your system (e.g. some libs missing), try using Docker. 64 | 65 | I prepared a ``Dockerfile`` and some initial ``make`` commands: 66 | 67 | - ``make build`` - build an image with ModestPy, based on Ubuntu 18.04 (tag = ``modestpy``) 68 | - ``make run`` - run the container (name = ``modestpy_container``) 69 | - ``make test`` - run unit tests in the running container and print output to terminal 70 | - ``make bash`` - run Bash in the running container 71 | 72 | Most likely you will like to modify ``Dockerfile`` and ``Makefile`` to your needs, e.g. by adding bind volumes with your FMUs. 73 | 74 | Test your installation 75 | ---------------------- 76 | 77 | The unit tests will work only if you installed ModestPy with conda or cloned the project from GitHub. To run tests: 78 | 79 | .. code:: python 80 | 81 | >>> from modestpy.test import run 82 | >>> run.tests() 83 | 84 | or 85 | 86 | :: 87 | 88 | cd 89 | python ./bin/test.py 90 | 91 | 92 | Usage 93 | ----- 94 | 95 | Users are supposed to call only the high level API included in 96 | ``modestpy.Estimation``. The API is fully discussed in the `docs `__. 97 | You can also check out this `simple example `__. 98 | The basic usage is as follows: 99 | 100 | .. code:: python 101 | 102 | from modestpy import Estimation 103 | 104 | if __name__ == "__main__": 105 | session = Estimation(workdir, fmu_path, inp, known, est, ideal) 106 | estimates = session.estimate() 107 | err, res = session.validate() 108 | 109 | More control is possible via optional arguments, as discussed in the `documentation 110 | `__. 111 | 112 | The ``if __name__ == "__main__":`` wrapper is needed on Windows, because ``modestpy`` 113 | relies on ``multiprocessing``. You can find more explanation on why this is needed 114 | `here `__. 115 | 116 | ``modestpy`` automatically saves results in the working 117 | directory including csv files with estimates and some useful plots, 118 | e.g.: 119 | 120 | 1) Error evolution in combined GA+PS estimation (dots represent switch 121 | from GA to PS): |Error-evolution| 122 | 123 | 2) Visualization of GA evolution: |GA-evolution| 124 | 125 | 3) Scatter matrix plot for interdependencies between parameters: 126 | |Intedependencies| 127 | 128 | Cite 129 | ---- 130 | 131 | To cite ModestPy, please use: 132 | 133 | \K. Arendt, M. Jradi, M. Wetter, C.T. Veje, ModestPy: An Open-Source Python Tool for Parameter Estimation in Functional Mock-up Units, *Proceedings of the American Modelica Conference 2018*, Cambridge, MA, USA, October 9-10, 2018. 134 | 135 | The preprint version of the conference paper presenting ModestPy is available `here 136 | `__. The paper was based on v.0.0.8. 137 | 138 | License 139 | ------- 140 | 141 | Copyright (c) 2017-2019, University of Southern Denmark. All rights reserved. 142 | 143 | This code is licensed under BSD 2-clause license. See 144 | `LICENSE `__ file in the project root for license terms. 145 | 146 | .. |Error-evolution| image:: /docs/img/err_evo.png 147 | .. |GA-evolution| image:: /docs/img/ga_evolution.png 148 | .. |Intedependencies| image:: /docs/img/all_estimates.png 149 | 150 | -------------------------------------------------------------------------------- /bin/README.md: -------------------------------------------------------------------------------- 1 | # Executable scripts 2 | 3 | Current list of scripts: 4 | 5 | * `test.py` - runs all tests -------------------------------------------------------------------------------- /bin/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from modestpy.loginit import config_logger 3 | from modestpy.test import run 4 | 5 | if __name__ == "__main__": 6 | config_logger(filename="unit_tests.log", level="DEBUG") 7 | run.tests() 8 | -------------------------------------------------------------------------------- /docs/documentation.md: -------------------------------------------------------------------------------- 1 | # modestpy 2 | ## Introduction 3 | 4 | Users are supposed to use only `modestpy.Estimation` class and its two 5 | methods `estimate()` and `validate()`. The class defines a single interface 6 | for different optimization algorithms. Currently, the available algorithms are: 7 | - parallel genetic algorithm (MODESTGA) - recommended, 8 | - legacy single-process genetic algorithm (GA), 9 | - pattern search (PS), 10 | - SciPy solvers (e.g. 'TNC', 'L-BFGS-B', 'SLSQP'). 11 | 12 | The methods can be used in a sequence, e.g. MODESTGA+PS (default), 13 | using the argument `methods`. All estimation settings are set during instantiation. 14 | Results of estimation and validation are saved in the working directory `workdir` 15 | (it must exist). 16 | 17 | ## Learn by examples 18 | 19 | First define the following variables: 20 | 21 | * `workdir` (`str`) - path to the working directory (it must exist) 22 | * `fmu_path` (`str`) - path to the FMU compiled for your platform 23 | * `inp` (`pandas.DataFrame`) - inputs, index given in seconds and named `time` 24 | * `est` (`dict(str : tuple(float, float, float))`) - dictionary mapping parameter names to tuples (initial guess, lower bound, upper bound) 25 | * `known` (`dict(str : float)`) - dictionary mapping parameter names to known values 26 | * `ideal` (`pandas.DataFrame`) - ideal solution (usually measurements), index given in seconds and named `time` 27 | 28 | Indexes of `inp` and `ideal` must be equal, i.e. `inp.index == ideal.index` must be `True`. 29 | Columns in `inp` and `ideal` must have the same names as model inputs and outputs, respectively. 30 | All model inputs must be present in `inp`, but only chosen outputs may be included in `ideal`. 31 | Data for each variable present in `ideal` are used to calculate the error function that is minimized by **modestpy**. 32 | 33 | Now the parameters can be estimated using default settings: 34 | 35 | ``` 36 | python 37 | >>> session = Estimation(workdir, fmu_path, inp, known, est, ideal) 38 | >>> estimates = session.estimate() # Returns dict(str: float) 39 | >>> err, res = session.validate() # Returns tuple(dict(str: float), pandas.DataFrame) 40 | ``` 41 | 42 | All results are also saved in `workdir`. 43 | 44 | By default all data from `inp` and `ideal` (all rows) are used in both estimation and validation. 45 | To slice the data into separate learning and validation periods, additional arguments need to be defined: 46 | 47 | * `lp_n` (`int`) - number of learning periods, randomly selected within `lp_frame` 48 | * `lp_len` (`float`) - length of single learning period 49 | * `lp_frame` (`tuple(float, float)`) - beginning and end of learning time frame 50 | * `vp` (`tuple(float, float)`) - validation period 51 | 52 | Often model parameters are used to define the initial conditions in the model, 53 | in example initial temperature. The initial values have to be read from the measured data stored in `ideal`. 54 | You can do this with the optional argument `ic_param`: 55 | 56 | * `ic_param` (`dict(str : str)`) - maps model parameters to column names in `ideal` 57 | 58 | Estimation algorithms (MODESTGA, PS, SQP) can be tuned by overwriting specific keys in `modestga_opts`, `ps_opts` and `scipy_opts`. 59 | The default options are: 60 | 61 | ``` 62 | # Default MODESTGA options 63 | MODESTGA_OPTS = { 64 | 'workers': 3, # CPU cores to use 65 | 'generations': 50, # Max. number of generations 66 | 'pop_size': 30, # Population size 67 | 'mut_rate': 0.01, # Mutation rate 68 | 'trm_size': 3, # Tournament size 69 | 'tol': 1e-3, # Solution tolerance 70 | 'inertia': 100, # Max. number of non-improving generations 71 | 'ftype': 'RMSE' 72 | } 73 | 74 | # Default PS options 75 | self.PS_OPTS = { 76 | 'maxiter': 500, 77 | 'rel_step': 0.02, 78 | 'tol': 1e-11, 79 | 'try_lim': 1000, 80 | 'ftype': 'RMSE' 81 | } 82 | 83 | # Default SCIPY options 84 | SCIPY_OPTS = { 85 | 'solver': 'L-BFGS-B', 86 | 'options': {'disp': True, 87 | 'iprint': 2, 88 | 'maxiter': 150, 89 | 'full_output': True}, 90 | 'ftype': 'RMSE' 91 | } 92 | ``` 93 | 94 | ## Docstrings 95 | 96 | ```python 97 | class Estimation(object): 98 | """Public interface of `modestpy`. 99 | 100 | Index in DataFrames `inp` and `ideal` must be named 'time' 101 | and given in seconds. The index name assertion check is 102 | implemented to avoid situations in which a user reads DataFrame 103 | from a csv and forgets to use `DataFrame.set_index(column_name)` 104 | (it happens quite often...). 105 | 106 | Currently available estimation methods: 107 | - MODESTGA - parallel genetic algorithm (default GA in modestpy) 108 | - GA_LEGACY - single-process genetic algorithm (legacy implementation, discouraged) 109 | - PS - pattern search (Hooke-Jeeves) 110 | - SCIPY - interface to algorithms available through 111 | scipy.optimize.minimize() 112 | 113 | Parameters: 114 | ----------- 115 | workdir: str 116 | Output directory, must exist 117 | fmu_path: str 118 | Absolute path to the FMU 119 | inp: pandas.DataFrame 120 | Input data, index given in seconds and named 'time' 121 | known: dict(str: float) 122 | Dictionary with known parameters (`parameter_name: value`) 123 | est: dict(str: tuple(float, float, float)) 124 | Dictionary defining estimated parameters, 125 | (`par_name: (guess value, lo limit, hi limit)`) 126 | ideal: pandas.DataFrame 127 | Ideal solution (usually measurements), 128 | index in seconds and named `time` 129 | lp_n: int or None 130 | Number of learning periods, one if `None` 131 | lp_len: float or None 132 | Length of a single learning period, entire `lp_frame` if `None` 133 | lp_frame: tuple of floats or None 134 | Learning period time frame, entire data set if `None` 135 | vp: tuple(float, float) or None 136 | Validation period, entire data set if `None` 137 | ic_param: dict(str, str) or None 138 | Mapping between model parameters used for IC and variables from 139 | `ideal` 140 | methods: tuple(str, str) 141 | List of methods to be used in the pipeline 142 | ga_opts: dict 143 | Genetic algorithm options 144 | ps_opts: dict 145 | Pattern search options 146 | scipy_opts: dict 147 | SciPy solver options 148 | ftype: string 149 | Cost function type. Currently 'NRMSE' (advised for multi-objective 150 | estimation) or 'RMSE'. 151 | seed: None or int 152 | Random number seed. If None, current time or OS specific 153 | randomness is used. 154 | default_log: bool 155 | If true, use default logging settings. Use false if you want to 156 | use own logging. 157 | logfile: str 158 | If default_log=True, this argument can be used to specify the log 159 | file name 160 | """ 161 | ``` 162 | -------------------------------------------------------------------------------- /docs/img/all_estimates.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/docs/img/all_estimates.png -------------------------------------------------------------------------------- /docs/img/err_evo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/docs/img/err_evo.png -------------------------------------------------------------------------------- /docs/img/ga_evolution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/docs/img/ga_evolution.png -------------------------------------------------------------------------------- /docs/img/logo_src/modest-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/docs/img/logo_src/modest-logo.png -------------------------------------------------------------------------------- /docs/img/logo_src/modest-logo.xcf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/docs/img/logo_src/modest-logo.xcf -------------------------------------------------------------------------------- /docs/img/logo_src/pop_evo_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/docs/img/logo_src/pop_evo_4.png -------------------------------------------------------------------------------- /docs/img/modest-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/docs/img/modest-logo.png -------------------------------------------------------------------------------- /examples/lin/README.md: -------------------------------------------------------------------------------- 1 | The charts in `showcase/` show the behavior of GA and PS when the cost function is convex. The charts were generated by an finding the parameters of the model `resources/lin_model.mo`, but the Python code used to generate these charts is no longer here. 2 | 3 | See `examples/simple/` for an example with code. 4 | -------------------------------------------------------------------------------- /examples/lin/model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/examples/lin/model.png -------------------------------------------------------------------------------- /examples/lin/resources/lin_model.mo: -------------------------------------------------------------------------------- 1 | within ; 2 | model lin_model 3 | Modelica.Blocks.Interfaces.RealInput u2 4 | annotation (Placement(transformation(extent={{-128,-46},{-88,-6}}))); 5 | Modelica.Blocks.Math.Gain gain(k=b) 6 | annotation (Placement(transformation(extent={{-56,-36},{-36,-16}}))); 7 | Modelica.Blocks.Math.Gain gain1(k=a) 8 | annotation (Placement(transformation(extent={{-56,14},{-36,34}}))); 9 | Modelica.Blocks.Math.Add add 10 | annotation (Placement(transformation(extent={{64,-10},{84,10}}))); 11 | Modelica.Blocks.Interfaces.RealOutput y 12 | annotation (Placement(transformation(extent={{96,-10},{116,10}}))); 13 | parameter Real b=1 "Gain value multiplied with input signal"; 14 | parameter Real a=1 "Gain value multiplied with input signal"; 15 | Modelica.Blocks.Interfaces.RealInput u1 16 | annotation (Placement(transformation(extent={{-128,4},{-88,44}}))); 17 | equation 18 | connect(add.y, y) 19 | annotation (Line(points={{85,0},{85,0},{106,0}}, color={0,0,127})); 20 | connect(u1, gain1.u) 21 | annotation (Line(points={{-108,24},{-58,24}}, color={0,0,127})); 22 | connect(u2, gain.u) annotation (Line(points={{-108,-26},{-84,-26},{-58,-26}}, 23 | color={0,0,127})); 24 | connect(gain1.y, add.u1) annotation (Line(points={{-35,24},{14,24},{14,6},{62, 25 | 6}}, color={0,0,127})); 26 | connect(gain.y, add.u2) annotation (Line(points={{-35,-26},{14,-26},{14,-6},{ 27 | 62,-6}}, color={0,0,127})); 28 | annotation (uses(Modelica(version="3.2.2"))); 29 | end lin_model; 30 | -------------------------------------------------------------------------------- /examples/lin/showcase/GA_search_path.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/examples/lin/showcase/GA_search_path.png -------------------------------------------------------------------------------- /examples/lin/showcase/PS_search_path.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/examples/lin/showcase/PS_search_path.png -------------------------------------------------------------------------------- /examples/lin/showcase/RMSE_grid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/examples/lin/showcase/RMSE_grid.png -------------------------------------------------------------------------------- /examples/simple/README.md: -------------------------------------------------------------------------------- 1 | # Simple example: estimation of 3 parameters in R2C1 thermal network 2 | 3 | This example shows how to set up a learning session using ``Estimation`` class. 4 | 5 | Simulation results are saved in ``modest-py/examples/simple/workdir``. 6 | 7 | Notice that the parameters are interdependent and the same model behavior is 8 | achieved with different sets of estimates (e.g. compare estimates vs. error on 9 | `scatter.png`). Hence, it is likely that you get different parameters, than 10 | the ones used to produce the measured data (``ideal``). Compare your result 11 | with ``resources/true_parameters.csv`` and look at the validation results. 12 | -------------------------------------------------------------------------------- /examples/simple/resources/Simple2R1C_ic.mo: -------------------------------------------------------------------------------- 1 | within ; 2 | model Simple2R1C 3 | parameter Modelica.SIunits.ThermalResistance R1=1 4 | "Constant thermal resistance of material"; 5 | parameter Modelica.SIunits.ThermalResistance R2=1 6 | "Constant thermal resistance of material"; 7 | parameter Modelica.SIunits.HeatCapacity C=1000 8 | "Heat capacity of element (= cp*m)"; 9 | parameter Modelica.SIunits.Temperature Tstart=20 10 | "Initial temperature"; 11 | 12 | 13 | Modelica.Thermal.HeatTransfer.Components.HeatCapacitor heatCapacitor(C=C, T(fixed= 14 | true, start=Tstart)) 15 | annotation (Placement(transformation(extent={{2,0},{22,20}}))); 16 | Modelica.Thermal.HeatTransfer.Components.ThermalResistor thermalResistor2(R= 17 | R2) annotation (Placement(transformation(extent={{-40,-10},{-20,10}}))); 18 | Modelica.Blocks.Interfaces.RealInput Ti2 19 | annotation (Placement(transformation(extent={{-130,-20},{-90,20}}))); 20 | Modelica.Blocks.Interfaces.RealOutput T 21 | annotation (Placement(transformation(extent={{96,-10},{116,10}}))); 22 | Modelica.Thermal.HeatTransfer.Components.ThermalResistor thermalResistor1(R= 23 | R1) annotation (Placement(transformation(extent={{-42,36},{-22,56}}))); 24 | Modelica.Blocks.Interfaces.RealInput Ti1 25 | annotation (Placement(transformation(extent={{-130,26},{-90,66}}))); 26 | Modelica.Thermal.HeatTransfer.Sensors.TemperatureSensor temperatureSensor 27 | annotation (Placement(transformation(extent={{42,-10},{62,10}}))); 28 | Modelica.Thermal.HeatTransfer.Sources.PrescribedTemperature 29 | prescribedTemperature1 30 | annotation (Placement(transformation(extent={{-78,36},{-58,56}}))); 31 | Modelica.Thermal.HeatTransfer.Sources.PrescribedTemperature 32 | prescribedTemperature2 33 | annotation (Placement(transformation(extent={{-76,-10},{-56,10}}))); 34 | equation 35 | connect(heatCapacitor.port, thermalResistor2.port_b) 36 | annotation (Line(points={{12,0},{-20,0}}, color={191,0,0})); 37 | connect(thermalResistor1.port_b, heatCapacitor.port) annotation (Line( 38 | points={{-22,46},{-8,46},{-8,0},{12,0}}, color={191,0,0})); 39 | connect(heatCapacitor.port, temperatureSensor.port) 40 | annotation (Line(points={{12,0},{12,0},{42,0}}, color={191,0,0})); 41 | connect(T, temperatureSensor.T) 42 | annotation (Line(points={{106,0},{62,0}}, color={0,0,127})); 43 | connect(Ti1, prescribedTemperature1.T) 44 | annotation (Line(points={{-110,46},{-96,46},{-80,46}}, color={0,0,127})); 45 | connect(thermalResistor1.port_a, prescribedTemperature1.port) 46 | annotation (Line(points={{-42,46},{-50,46},{-58,46}}, color={191,0,0})); 47 | connect(Ti2, prescribedTemperature2.T) 48 | annotation (Line(points={{-110,0},{-94,0},{-78,0}}, color={0,0,127})); 49 | connect(thermalResistor2.port_a, prescribedTemperature2.port) 50 | annotation (Line(points={{-40,0},{-48,0},{-56,0}}, color={191,0,0})); 51 | annotation (Icon(coordinateSystem(preserveAspectRatio=false)), Diagram( 52 | coordinateSystem(preserveAspectRatio=false)), 53 | uses(Modelica(version="3.2.2"))); 54 | end Simple2R1C; 55 | -------------------------------------------------------------------------------- /examples/simple/resources/Simple2R1C_ic_linux64.fmu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/examples/simple/resources/Simple2R1C_ic_linux64.fmu -------------------------------------------------------------------------------- /examples/simple/resources/Simple2R1C_ic_win32.fmu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/examples/simple/resources/Simple2R1C_ic_win32.fmu -------------------------------------------------------------------------------- /examples/simple/resources/Simple2R1C_ic_win64.fmu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/examples/simple/resources/Simple2R1C_ic_win64.fmu -------------------------------------------------------------------------------- /examples/simple/resources/est.json: -------------------------------------------------------------------------------- 1 | { 2 | "R1": [0.08, 0.001, 0.5], 3 | "R2": [0.08, 0.001, 0.5], 4 | "C": [1000.0, 100.0, 10000.0] 5 | } 6 | -------------------------------------------------------------------------------- /examples/simple/resources/est_validate_hj.json: -------------------------------------------------------------------------------- 1 | { 2 | "R1": [0.08, 0.001, 0.5], 3 | "R2": [0.08, 0.001, 0.5] 4 | } 5 | -------------------------------------------------------------------------------- /examples/simple/resources/known.json: -------------------------------------------------------------------------------- 1 | { 2 | "Tstart": 293.15 3 | } -------------------------------------------------------------------------------- /examples/simple/resources/known_validate_hj.json: -------------------------------------------------------------------------------- 1 | { 2 | "Tstart": 293.15, 3 | "C": 2000.0 4 | } 5 | -------------------------------------------------------------------------------- /examples/simple/resources/true_parameters.csv: -------------------------------------------------------------------------------- 1 | R1,R2,C 2 | 0.1,0.25,2000 3 | -------------------------------------------------------------------------------- /examples/simple/simple-slsqp.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import json 8 | import os 9 | 10 | import matplotlib.pyplot as plt 11 | import pandas as pd 12 | 13 | from modestpy.estim.scipy.scipy import SCIPY 14 | from modestpy.utilities.sysarch import get_sys_arch 15 | 16 | if __name__ == "__main__": 17 | """ 18 | This file is supposed to be run from the root directory. 19 | Otherwise the paths have to be corrected. 20 | """ 21 | 22 | # DATA PREPARATION ============================================== 23 | # Resources 24 | platform = get_sys_arch() 25 | assert platform, "Unsupported platform type!" 26 | fmu_file = "Simple2R1C_ic_" + platform + ".fmu" 27 | 28 | fmu_path = os.path.join("examples", "simple", "resources", fmu_file) 29 | inp_path = os.path.join("examples", "simple", "resources", "inputs.csv") 30 | ideal_path = os.path.join("examples", "simple", "resources", "result.csv") 31 | est_path = os.path.join("examples", "simple", "resources", "est_validate_hj.json") 32 | known_path = os.path.join( 33 | "examples", "simple", "resources", "known_validate_hj.json" 34 | ) 35 | 36 | # Working directory 37 | workdir = os.path.join("examples", "simple", "workdir") 38 | if not os.path.exists(workdir): 39 | os.mkdir(workdir) 40 | assert os.path.exists(workdir), "Work directory does not exist" 41 | 42 | # Load inputs 43 | inp = pd.read_csv(inp_path).set_index("time") 44 | 45 | # Load measurements (ideal results) 46 | ideal = pd.read_csv(ideal_path).set_index("time") 47 | 48 | # Load definition of estimated parameters (name, initial value, bounds) 49 | with open(est_path) as f: 50 | est = json.load(f) 51 | 52 | # Load definition of known parameters (name, value) 53 | with open(known_path) as f: 54 | known = json.load(f) 55 | 56 | # MODEL IDENTIFICATION ========================================== 57 | # session = Estimation(workdir, fmu_path, inp, known, est, ideal, 58 | # lp_n=3, lp_len=25000, lp_frame=(0, 25000), 59 | # vp = (150000, 215940), ic_param={'Tstart': 'T'}, 60 | # ga_pop=20, ga_iter=20, ps_iter=30, ga_tol=0.001, 61 | # ps_tol=0.0001, ftype='RMSE', lhs=True) 62 | 63 | # estimates = session.estimate() 64 | # err, res = session.validate() 65 | scipy = SCIPY(fmu_path, inp, known, est, ideal, ftype="RMSE", solver="SLSQP") 66 | 67 | par = scipy.estimate() 68 | print(par) 69 | 70 | res = scipy.res 71 | res["ideal"] = ideal["T"] 72 | res.plot() 73 | plt.show() 74 | 75 | print("ERROR={}".format(scipy.best_err)) 76 | -------------------------------------------------------------------------------- /examples/simple/simple.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import json 8 | import logging 9 | import os 10 | 11 | import pandas as pd 12 | 13 | from modestpy import Estimation 14 | from modestpy.utilities.sysarch import get_sys_arch 15 | 16 | logging.basicConfig(level="INFO", filename="test.log", filemode="w") 17 | 18 | if __name__ == "__main__": 19 | """ 20 | This file is supposed to be run from the root directory. 21 | Otherwise the paths have to be corrected. 22 | """ 23 | 24 | # DATA PREPARATION ============================================== 25 | # Resources 26 | platform = get_sys_arch() 27 | assert platform, "Unsupported platform type!" 28 | fmu_file = "Simple2R1C_ic_" + platform + ".fmu" 29 | 30 | fmu_path = os.path.join("examples", "simple", "resources", fmu_file) 31 | inp_path = os.path.join("examples", "simple", "resources", "inputs.csv") 32 | ideal_path = os.path.join("examples", "simple", "resources", "result.csv") 33 | est_path = os.path.join("examples", "simple", "resources", "est.json") 34 | known_path = os.path.join("examples", "simple", "resources", "known.json") 35 | 36 | # Working directory 37 | workdir = os.path.join("examples", "simple", "workdir") 38 | if not os.path.exists(workdir): 39 | os.mkdir(workdir) 40 | assert os.path.exists(workdir), "Work directory does not exist" 41 | 42 | # Load inputs 43 | inp = pd.read_csv(inp_path).set_index("time") 44 | 45 | # Load measurements (ideal results) 46 | ideal = pd.read_csv(ideal_path).set_index("time") 47 | 48 | # Load definition of estimated parameters (name, initial value, bounds) 49 | with open(est_path) as f: 50 | est = json.load(f) 51 | 52 | # Load definition of known parameters (name, value) 53 | with open(known_path) as f: 54 | known = json.load(f) 55 | 56 | # MODEL IDENTIFICATION ========================================== 57 | # Comparing parallel GA against GA using different population sizes 58 | case_workdir = os.path.join(workdir, "modestga") 59 | if not os.path.exists(case_workdir): 60 | os.mkdir(case_workdir) 61 | 62 | session = Estimation( 63 | case_workdir, 64 | fmu_path, 65 | inp, 66 | known, 67 | est, 68 | ideal, 69 | lp_n=1, 70 | lp_len=50000, 71 | lp_frame=(0, 50000), 72 | vp=(0, 50000), 73 | ic_param={"Tstart": "T"}, 74 | methods=("MODESTGA",), 75 | modestga_opts={ 76 | "generations": 20, # Max. number of generations 77 | "pop_size": 60, # Population size 78 | "trm_size": 7, # Tournament size 79 | "tol": 1e-3, # Absolute tolerance 80 | "workers": 3, # Number of CPUs to use 81 | }, 82 | ftype="RMSE", 83 | default_log=True, 84 | logfile="simple.log", 85 | ) 86 | estimates = session.estimate() 87 | err, res = session.validate() 88 | -------------------------------------------------------------------------------- /examples/simple/simple_1param.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import json 8 | import os 9 | 10 | import pandas as pd 11 | 12 | from modestpy import Estimation 13 | from modestpy.utilities.sysarch import get_sys_arch 14 | 15 | if __name__ == "__main__": 16 | """ 17 | This file is supposed to be run from the root directory. 18 | Otherwise the paths have to be corrected. 19 | """ 20 | 21 | # DATA PREPARATION ============================================== 22 | # Resources 23 | platform = get_sys_arch() 24 | assert platform, "Unsupported platform type!" 25 | fmu_file = "Simple2R1C_ic_" + platform + ".fmu" 26 | 27 | fmu_path = os.path.join("examples", "simple", "resources", fmu_file) 28 | inp_path = os.path.join("examples", "simple", "resources", "inputs.csv") 29 | ideal_path = os.path.join("examples", "simple", "resources", "result.csv") 30 | est_path = os.path.join("examples", "simple", "resources", "est.json") 31 | known_path = os.path.join("examples", "simple", "resources", "known.json") 32 | 33 | # Working directory 34 | workdir = os.path.join("examples", "simple", "workdir") 35 | if not os.path.exists(workdir): 36 | os.mkdir(workdir) 37 | assert os.path.exists(workdir), "Work directory does not exist" 38 | 39 | # Load inputs 40 | inp = pd.read_csv(inp_path).set_index("time") 41 | 42 | # Load measurements (ideal results) 43 | ideal = pd.read_csv(ideal_path).set_index("time") 44 | 45 | # Load definition of estimated parameters (name, initial value, bounds) 46 | with open(est_path) as f: 47 | est = json.load(f) 48 | 49 | del est["R1"] # We want to estimate only C 50 | del est["R2"] # We want to estimate only C 51 | 52 | # Load definition of known parameters (name, value) 53 | with open(known_path) as f: 54 | known = json.load(f) 55 | 56 | known["R1"] = 0.1 57 | known["R2"] = 0.25 58 | 59 | # MODEL IDENTIFICATION ========================================== 60 | session = Estimation( 61 | workdir, 62 | fmu_path, 63 | inp, 64 | known, 65 | est, 66 | ideal, 67 | lp_n=2, 68 | lp_len=50000, 69 | lp_frame=(0, 50000), 70 | vp=(0, 50000), 71 | ic_param={"Tstart": "T"}, 72 | methods=("MODESTGA", "PS"), 73 | ps_opts={"maxiter": 500, "tol": 1e-6}, 74 | scipy_opts={}, 75 | ftype="RMSE", 76 | default_log=True, 77 | logfile="simple.log", 78 | ) 79 | 80 | estimates = session.estimate() 81 | err, res = session.validate() 82 | -------------------------------------------------------------------------------- /examples/simple/simple_ga_vs_ga_parallel.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import json 8 | import logging 9 | import os 10 | 11 | import pandas as pd 12 | 13 | from modestpy import Estimation 14 | from modestpy.utilities.sysarch import get_sys_arch 15 | 16 | logging.basicConfig(level="INFO", filename="test.log", filemode="w") 17 | 18 | if __name__ == "__main__": 19 | """ 20 | This file is supposed to be run from the root directory. 21 | Otherwise the paths have to be corrected. 22 | """ 23 | 24 | # DATA PREPARATION ============================================== 25 | # Resources 26 | platform = get_sys_arch() 27 | assert platform, "Unsupported platform type!" 28 | fmu_file = "Simple2R1C_ic_" + platform + ".fmu" 29 | 30 | fmu_path = os.path.join("examples", "simple", "resources", fmu_file) 31 | inp_path = os.path.join("examples", "simple", "resources", "inputs.csv") 32 | ideal_path = os.path.join("examples", "simple", "resources", "result.csv") 33 | est_path = os.path.join("examples", "simple", "resources", "est.json") 34 | known_path = os.path.join("examples", "simple", "resources", "known.json") 35 | 36 | # Working directory 37 | workdir = os.path.join("examples", "simple", "workdir") 38 | if not os.path.exists(workdir): 39 | os.mkdir(workdir) 40 | assert os.path.exists(workdir), "Work directory does not exist" 41 | 42 | # Load inputs 43 | inp = pd.read_csv(inp_path).set_index("time") 44 | 45 | # Load measurements (ideal results) 46 | ideal = pd.read_csv(ideal_path).set_index("time") 47 | 48 | # Load definition of estimated parameters (name, initial value, bounds) 49 | with open(est_path) as f: 50 | est = json.load(f) 51 | 52 | # Load definition of known parameters (name, value) 53 | with open(known_path) as f: 54 | known = json.load(f) 55 | 56 | # MODEL IDENTIFICATION ========================================== 57 | # Comparing parallel GA against GA using different population sizes 58 | for method in ["MODESTGA", "GA_LEGACY"]: 59 | for pop in [40, 60, 80]: 60 | case_workdir = os.path.join(workdir, f"{method}-{pop}") 61 | if not os.path.exists(case_workdir): 62 | os.mkdir(case_workdir) 63 | 64 | session = Estimation( 65 | case_workdir, 66 | fmu_path, 67 | inp, 68 | known, 69 | est, 70 | ideal, 71 | lp_n=1, 72 | lp_len=50000, 73 | lp_frame=(0, 50000), 74 | vp=(0, 50000), 75 | ic_param={"Tstart": "T"}, 76 | methods=((method,)), 77 | ga_opts={ 78 | "maxiter": 20, 79 | "pop_size": pop, 80 | "trm_size": 7, 81 | "tol": 1e-3, 82 | "lhs": True, 83 | }, 84 | modestga_opts={ 85 | "generations": 20, 86 | "pop_size": pop, 87 | "trm_size": 7, 88 | "tol": 1e-3, 89 | "workers": 2, 90 | }, 91 | ftype="RMSE", 92 | default_log=True, 93 | logfile="simple.log", 94 | ) 95 | estimates = session.estimate() 96 | err, res = session.validate() 97 | -------------------------------------------------------------------------------- /examples/simple/simple_legacy_ga.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import json 8 | import os 9 | 10 | import pandas as pd 11 | 12 | from modestpy import Estimation 13 | from modestpy.loginit import config_logger 14 | from modestpy.utilities.sysarch import get_sys_arch 15 | 16 | if __name__ == "__main__": 17 | """ 18 | This file is supposed to be run from the root directory. 19 | Otherwise the paths have to be corrected. 20 | """ 21 | config_logger(filename="simple.log", level="DEBUG") 22 | 23 | # DATA PREPARATION ============================================== 24 | # Resources 25 | platform = get_sys_arch() 26 | assert platform, "Unsupported platform type!" 27 | fmu_file = "Simple2R1C_ic_" + platform + ".fmu" 28 | 29 | fmu_path = os.path.join("examples", "simple", "resources", fmu_file) 30 | inp_path = os.path.join("examples", "simple", "resources", "inputs.csv") 31 | ideal_path = os.path.join("examples", "simple", "resources", "result.csv") 32 | est_path = os.path.join("examples", "simple", "resources", "est.json") 33 | known_path = os.path.join("examples", "simple", "resources", "known.json") 34 | 35 | # Working directory 36 | workdir = os.path.join("examples", "simple", "workdir") 37 | if not os.path.exists(workdir): 38 | os.mkdir(workdir) 39 | assert os.path.exists(workdir), "Work directory does not exist" 40 | 41 | # Load inputs 42 | inp = pd.read_csv(inp_path).set_index("time") 43 | 44 | # Load measurements (ideal results) 45 | ideal = pd.read_csv(ideal_path).set_index("time") 46 | 47 | # Load definition of estimated parameters (name, initial value, bounds) 48 | with open(est_path) as f: 49 | est = json.load(f) 50 | 51 | # Load definition of known parameters (name, value) 52 | with open(known_path) as f: 53 | known = json.load(f) 54 | 55 | # MODEL IDENTIFICATION ========================================== 56 | session = Estimation( 57 | workdir, 58 | fmu_path, 59 | inp, 60 | known, 61 | est, 62 | ideal, 63 | lp_n=2, 64 | lp_len=50000, 65 | lp_frame=(0, 50000), 66 | vp=(0, 50000), 67 | ic_param={"Tstart": "T"}, 68 | methods=("GA_LEGACY", "PS"), 69 | ga_opts={"maxiter": 5, "tol": 0.001, "lhs": True}, 70 | ps_opts={"maxiter": 500, "tol": 1e-6}, 71 | scipy_opts={}, 72 | ftype="RMSE", 73 | default_log=True, 74 | logfile="simple.log", 75 | ) 76 | 77 | estimates = session.estimate() 78 | err, res = session.validate() 79 | -------------------------------------------------------------------------------- /examples/simple/validate_hj.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import json 8 | import os 9 | 10 | import pandas as pd 11 | 12 | from modestpy import Estimation 13 | from modestpy.utilities.sysarch import get_sys_arch 14 | 15 | if __name__ == "__main__": 16 | """ 17 | This file is supposed to be run from the root directory. 18 | Otherwise the paths have to be corrected. 19 | """ 20 | 21 | # DATA PREPARATION ============================================== 22 | # Resources 23 | platform = get_sys_arch() 24 | assert platform, "Unsupported platform type!" 25 | fmu_file = "Simple2R1C_ic_" + platform + ".fmu" 26 | 27 | fmu_path = os.path.join("examples", "simple", "resources", fmu_file) 28 | inp_path = os.path.join("examples", "simple", "resources", "inputs.csv") 29 | ideal_path = os.path.join("examples", "simple", "resources", "result.csv") 30 | est_path = os.path.join("examples", "simple", "resources", "est_validate_hj.json") 31 | known_path = os.path.join( 32 | "examples", "simple", "resources", "known_validate_hj.json" 33 | ) 34 | 35 | # Working directory 36 | workdir = os.path.join("examples", "simple", "workdir") 37 | if not os.path.exists(workdir): 38 | os.mkdir(workdir) 39 | assert os.path.exists(workdir), "Work directory does not exist" 40 | 41 | # Load inputs 42 | inp = pd.read_csv(inp_path).set_index("time") 43 | 44 | # Load measurements (ideal results) 45 | ideal = pd.read_csv(ideal_path).set_index("time") 46 | 47 | # Load definition of estimated parameters (name, initial value, bounds) 48 | with open(est_path) as f: 49 | est = json.load(f) 50 | 51 | # Load definition of known parameters (name, value) 52 | with open(known_path) as f: 53 | known = json.load(f) 54 | 55 | # MODEL IDENTIFICATION ========================================== 56 | session = Estimation( 57 | workdir, 58 | fmu_path, 59 | inp, 60 | known, 61 | est, 62 | ideal, 63 | lp_n=2, 64 | lp_len=25000, 65 | lp_frame=(0, 25000), 66 | vp=(150000, 215940), 67 | ic_param={"Tstart": "T"}, 68 | methods=("PS",), 69 | ps_opts={"maxiter": 300, "tol": 1e-6}, 70 | ftype="RMSE", 71 | ) 72 | 73 | estimates = session.estimate() 74 | err, res = session.validate() 75 | -------------------------------------------------------------------------------- /examples/sin/README.md: -------------------------------------------------------------------------------- 1 | The charts in `showcase/` show the behavior of GA and PS when the cost function is non-convex. The charts were generated by an finding the parameters of the model `resources/sin_model.mo`, but the Python code used to generate these charts is no longer here. 2 | 3 | See `examples/simple/` for an example with code. 4 | 5 | -------------------------------------------------------------------------------- /examples/sin/model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/examples/sin/model.png -------------------------------------------------------------------------------- /examples/sin/resources/sin_model.mo: -------------------------------------------------------------------------------- 1 | within ; 2 | model sin_model 3 | Modelica.Blocks.Math.Sin sin 4 | annotation (Placement(transformation(extent={{0,-4},{20,16}}))); 5 | Modelica.Blocks.Interfaces.RealInput u 6 | annotation (Placement(transformation(extent={{-128,-50},{-88,-10}}))); 7 | Modelica.Blocks.Math.Gain gain(k=a) 8 | annotation (Placement(transformation(extent={{30,-4},{50,16}}))); 9 | Modelica.Blocks.Sources.Clock clock 10 | annotation (Placement(transformation(extent={{-60,-4},{-40,16}}))); 11 | Modelica.Blocks.Math.Gain gain1(k=b/10000) 12 | annotation (Placement(transformation(extent={{-30,-4},{-10,16}}))); 13 | Modelica.Blocks.Math.Add add 14 | annotation (Placement(transformation(extent={{64,-10},{84,10}}))); 15 | Modelica.Blocks.Interfaces.RealOutput y 16 | annotation (Placement(transformation(extent={{96,-10},{116,10}}))); 17 | parameter Real b=1 "Gain value multiplied with input signal"; 18 | parameter Real a=1 "Gain value multiplied with input signal"; 19 | equation 20 | connect(sin.y, gain.u) 21 | annotation (Line(points={{21,6},{21,6},{28,6}}, color={0,0,127})); 22 | connect(clock.y, gain1.u) 23 | annotation (Line(points={{-39,6},{-39,6},{-32,6}}, color={0,0,127})); 24 | connect(gain1.y, sin.u) 25 | annotation (Line(points={{-9,6},{-9,6},{-2,6}}, color={0,0,127})); 26 | connect(gain.y, add.u1) 27 | annotation (Line(points={{51,6},{62,6}}, color={0,0,127})); 28 | connect(add.y, y) 29 | annotation (Line(points={{85,0},{85,0},{106,0}}, color={0,0,127})); 30 | connect(u, add.u2) annotation (Line(points={{-108,-30},{52,-30},{52,-6},{62, 31 | -6}}, color={0,0,127})); 32 | annotation (uses(Modelica(version="3.2.2"))); 33 | end sin_model; 34 | -------------------------------------------------------------------------------- /examples/sin/showcase/GA_PS_search_path.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/examples/sin/showcase/GA_PS_search_path.png -------------------------------------------------------------------------------- /examples/sin/showcase/GA_search_path.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/examples/sin/showcase/GA_search_path.png -------------------------------------------------------------------------------- /examples/sin/showcase/PS_search_path_bad_initial.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/examples/sin/showcase/PS_search_path_bad_initial.png -------------------------------------------------------------------------------- /examples/sin/showcase/PS_search_path_good_initial.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/examples/sin/showcase/PS_search_path_good_initial.png -------------------------------------------------------------------------------- /examples/sin/showcase/RMSE_grid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/examples/sin/showcase/RMSE_grid.png -------------------------------------------------------------------------------- /modestpy/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | 5 | This code is licensed under BSD 2-clause license. 6 | See LICENSE file in the project root for license terms. 7 | """ 8 | 9 | from modestpy.estimation import Estimation 10 | -------------------------------------------------------------------------------- /modestpy/estim/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | 5 | This code is licensed under BSD 2-clause license. 6 | See LICENSE file in the project root for license terms. 7 | """ 8 | -------------------------------------------------------------------------------- /modestpy/estim/error.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import logging 8 | 9 | import numpy as np 10 | import pandas as pd 11 | 12 | 13 | def calc_err(result, ideal, forgetting=False, ftype="RMSE"): 14 | """ 15 | Returns a dictionary with Normalised Root Mean Square Errors 16 | for each variable in ideal. The dictionary contains also a key 17 | ``tot`` with a sum of all errors 18 | 19 | If ``forgetting`` = ``True``, the error function is multiplied 20 | by a linear function with y=0 for the first time step and y=1 21 | for the last time step. In other words, the newer the error, 22 | the most important it is. It is used in the UKF tuning, 23 | since UKF needs some time to converge and the path it takes 24 | to converge should not be taken into account. 25 | 26 | :param result: DataFrame 27 | :param ideal: DataFrame 28 | :param forgetting: bool, if True, the older the error the lower weight 29 | :param string ftype: Cost function type, currently 'RMSE' or 'NRMSE' 30 | :return: dictionary 31 | """ 32 | logger = logging.getLogger("error") 33 | 34 | for v in ideal.columns: 35 | assert ( 36 | v in result.columns 37 | ), "Columns in ideal and model solution not matching: {} vs. {}".format( 38 | ideal.columns, result.columns 39 | ) 40 | 41 | # Get original variable names 42 | variables = list(ideal.columns) 43 | 44 | # Rename columns 45 | ideal = ideal.rename(columns=lambda x: x + "_ideal") 46 | result = result.rename(columns=lambda x: x + "_model") 47 | 48 | # Concatenate and interpolate 49 | comp = pd.concat([ideal, result], sort=False) 50 | comp = comp.sort_index().interpolate().bfill() 51 | 52 | if forgetting: 53 | forget_weights = np.linspace(0.0, 1.0, len(comp.index)) 54 | else: 55 | forget_weights = None 56 | 57 | # Calculate error 58 | error = dict() 59 | for v in variables: 60 | comp[v + "_se"] = np.square(comp[v + "_ideal"] - comp[v + "_model"]) 61 | 62 | if forgetting: 63 | # Cost function multiplied by a linear function 64 | # (0 for the oldest timestep, 1 for the newest) 65 | comp[v + "_se"] = comp[v + "_se"] * forget_weights 66 | 67 | mse = comp[v + "_se"].mean() # Mean square error 68 | rmse = mse**0.5 # Root mean square error 69 | 70 | ideal_mean = comp[v + "_ideal"].abs().mean() 71 | 72 | if ideal_mean != 0.0: 73 | nrmse = rmse / ideal_mean # Normalized root mean square error 74 | else: 75 | msg = ( 76 | "Ideal solution for variable '{}' is null, " 77 | "so the error cannot be normalized.".format(v) 78 | ) 79 | logger.error(msg) 80 | raise ZeroDivisionError(msg) 81 | 82 | # Choose error function type 83 | if ftype == "NRMSE": 84 | error[v] = nrmse 85 | elif ftype == "RMSE": 86 | error[v] = rmse 87 | else: 88 | raise ValueError("Cost function type unknown: {}".format(ftype)) 89 | 90 | logger.debug("Calculated partial error ({}) = {}".format(ftype, error[v])) 91 | 92 | # Calculate total error (sum of partial errors) 93 | assert "tot" not in error, "'tot' is not an allowed name " "for output variables..." 94 | error["tot"] = 0 95 | for v in variables: 96 | error["tot"] += error[v] 97 | 98 | logger.debug("Calculated total error ({}) = {}".format(ftype, error["tot"])) 99 | 100 | return error 101 | -------------------------------------------------------------------------------- /modestpy/estim/estpar.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import numpy as np 8 | import pandas as pd 9 | 10 | 11 | class EstPar: 12 | """ 13 | Estimated parameter for PS. 14 | """ 15 | 16 | def __init__(self, name, lo=None, hi=None, value=None): 17 | self.name = name 18 | self.value = value 19 | self.lo = lo # Lower limit 20 | self.hi = hi # Upper limit 21 | 22 | def __str__(self): 23 | s = ( 24 | self.name 25 | + "={:.4f}".format(self.value) 26 | + " (" 27 | + str(self.lo) 28 | + "-" 29 | + str(self.hi) 30 | + ")" 31 | ) 32 | return s 33 | 34 | 35 | def estpars_2_df(est_pars): 36 | """ 37 | Converts list of EstPar instances into DataFrame. 38 | 39 | :param est_pars: list of EstPar instances 40 | :return: DataFrame 41 | """ 42 | df = pd.DataFrame() 43 | for p in est_pars: 44 | df[p.name] = np.array([p.value]) 45 | return df 46 | 47 | 48 | def df_2_estpars(df): 49 | """Converts single-row data frame into a list of EstPar instances. 50 | hi/lo limits are unknown and assumed to be +/- inf. 51 | 52 | :param df: DataFrame with parameters (single row) 53 | :return: list of EstPar instances""" 54 | estpars = [] 55 | for p in df: 56 | ep = EstPar(p, float("-inf"), float("+inf"), df[p][0]) 57 | estpars.append(ep) 58 | return estpars 59 | -------------------------------------------------------------------------------- /modestpy/estim/ga/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | 5 | This code is licensed under BSD 2-clause license. 6 | See LICENSE file in the project root for license terms. 7 | """ 8 | -------------------------------------------------------------------------------- /modestpy/estim/ga/algorithm.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import logging 8 | import random 9 | 10 | from modestpy.estim.ga.population import Population 11 | 12 | # Constants controlling the evolution 13 | UNIFORM_RATE = 0.5 # affects crossover 14 | MUT_RATE = 0.10 # standard mutation rate 15 | MUT_RATE_INC = 0.33 # increased mutation rate when population diversity is low 16 | INC_MUT_PROP = 0.33 # proportion of population undergoing increased mutation 17 | MAX_CHANGE = 10 # [%] maximum change of gene in increased mutation 18 | TOURNAMENT_SIZE = 10 # number of individuals in the tournament 19 | DIVERSITY_LIM = 0.33 # controls increased mutation activation 20 | ELITISM = True # if True, saves the fittest individual 21 | 22 | # Printing on the screen 23 | VERBOSE = True 24 | 25 | 26 | def evolve(pop): 27 | """ 28 | Evolves the population. 29 | 30 | :param pop: Population 31 | :return: Population 32 | """ 33 | logger = logging.getLogger("ga.algorithm.evolve") 34 | 35 | new_pop = Population( 36 | fmu_path=pop.fmu_path, 37 | pop_size=pop.size(), 38 | inp=pop.inputs, 39 | known=pop.known_pars, 40 | est=pop.get_estpars(), 41 | ideal=pop.ideal, 42 | init=False, 43 | ) 44 | 45 | elite_offset = 0 46 | if ELITISM: 47 | new_pop.add_individual(pop.get_fittest()) 48 | elite_offset = 1 49 | 50 | # Crossover 51 | for i in range(elite_offset, new_pop.size()): 52 | ind1 = tournament_selection(pop, TOURNAMENT_SIZE) 53 | ind2 = tournament_selection(pop, TOURNAMENT_SIZE) 54 | child = crossover(ind1, ind2, UNIFORM_RATE) 55 | new_pop.add_individual(child) 56 | logger.debug("Crossover: ({}) x ({}) -> ({})".format(ind1, ind2, child)) 57 | 58 | # Mutation 59 | # Check population diversity 60 | if is_population_diverse(new_pop, DIVERSITY_LIM): 61 | # Low mutation rate, completely random new values 62 | logger.debug("Population diversity is OK -> standard mutation") 63 | for i in range(elite_offset, new_pop.size()): 64 | mutation(new_pop.individuals[i], MUT_RATE) 65 | else: 66 | # Population is not diverse 67 | logger.debug("Population diversity is LOW -> increased mutation") 68 | for i in range(elite_offset, new_pop.size()): 69 | if random.random() < INC_MUT_PROP: 70 | # Increased mutation rate, slightly changed values 71 | slight_mutation(new_pop.individuals[i], MUT_RATE_INC, MAX_CHANGE) 72 | else: 73 | # Increased mutation rate, completely random new values 74 | mutation(new_pop.individuals[i], MUT_RATE_INC) 75 | 76 | # Calculate 77 | new_pop.calculate() 78 | 79 | # Return 80 | return new_pop 81 | 82 | 83 | def is_population_diverse(pop, diversity_lim): 84 | """ 85 | Check if the population is diverse. Returns true if the share 86 | of identical individuals in the population is higher than 87 | ``diversity_lim``. 88 | 89 | :param pop: Population 90 | :param diversity_lim: float (0-1), minimum share of identical individuals 91 | defining non-diverse population 92 | :return: boolean 93 | """ 94 | identical_count = 0 95 | total_count = len(pop.individuals) 96 | genes = list() 97 | 98 | for ind in pop.individuals: 99 | genes.append(ind.genes) 100 | 101 | # Count duplicates 102 | for row in genes: 103 | dup = genes.count(row) 104 | if dup > identical_count: 105 | identical_count = dup 106 | 107 | # Check against the limit 108 | is_diverse = True 109 | if (float(identical_count) / float(total_count)) > diversity_lim: 110 | is_diverse = False 111 | return is_diverse 112 | 113 | 114 | def crossover(ind1, ind2, uniformity): 115 | """ 116 | Crossover operation. The child takes genes from both parents. 117 | 118 | :param ind1: Individual (parent 1) 119 | :param ind2: Individual (parent 2) 120 | :param uniformity: float, uniformity rate 121 | :return: Individual (child) 122 | """ 123 | logger = logging.getLogger("ga.algorithm.crossover") 124 | 125 | # Avoid working on the same objects! 126 | # Otherwise, changing child's genes 127 | # affects parent's genes (since genes are stored in a dict). 128 | # The parent might be used again in another crossover operation, 129 | # so it must not be modified. 130 | child = ind1.get_clone() 131 | i1_clone = ind1.get_clone() 132 | i2_clone = ind2.get_clone() 133 | 134 | logger.debug("Ind1: {}".format(i1_clone)) 135 | logger.debug("Ind2: {}".format(i2_clone)) 136 | 137 | for name in child.get_sorted_gene_names(): 138 | randnum = random.random() 139 | if randnum <= uniformity: 140 | logger.debug("Take '{}' from ind1".format(name)) 141 | child.set_gene(name, i1_clone.genes[name]) 142 | else: 143 | logger.debug("Take '{}' from ind2".format(name)) 144 | child.set_gene(name, i2_clone.genes[name]) 145 | 146 | return child 147 | 148 | 149 | def mutation(ind, mut_rate): 150 | """ 151 | Standard mutation. Mutates genes in place. 152 | 153 | :param ind: Individual 154 | :param mut_rate: mutation rate 155 | :return: None 156 | """ 157 | logger = logging.getLogger("ga.algorithm.mutation") 158 | 159 | for g_name in ind.get_sorted_gene_names(): 160 | if random.random() < mut_rate: 161 | logger.debug("Mutate gene for parameter {}".format(g_name)) 162 | ind.set_gene(g_name, random.random()) 163 | 164 | 165 | def slight_mutation(ind, mut_rate, max_change): 166 | """ 167 | Slightly mutates the genes. 168 | 169 | :param ind: Individual 170 | :param mut_rate: float, mutation rate 171 | :param max_change: float (0-100), maximum allowed percentage 172 | change of genes 173 | :return: None 174 | """ 175 | logger = logging.getLogger("ga.algorithm.slight_mutation") 176 | 177 | for g_name in ind.get_sorted_gene_names(): 178 | if random.random() < mut_rate: 179 | logger.debug("Mutate gene for parameter {}".format(g_name)) 180 | value = ind.genes[g_name] 181 | new_value = value + random.uniform(-1.0, 1.0) * max_change / 100.0 182 | if new_value > 1.0: 183 | new_value = 1.0 184 | if new_value < 0.0: 185 | new_value = 0.0 186 | ind.set_gene(g_name, new_value) 187 | 188 | 189 | def tournament_selection(pop, tournament_size): 190 | # Create tournament population 191 | t_pop = Population( 192 | pop.fmu_path, 193 | tournament_size, 194 | pop.inputs, 195 | pop.known_pars, 196 | pop.get_estpars(), 197 | pop.ideal, 198 | init=False, 199 | ) 200 | # For each place in the tournament get a random individual 201 | for i in range(tournament_size): 202 | rand_index = random.randint(0, pop.size() - 1) 203 | t_pop.individuals.append(pop.individuals[rand_index]) 204 | 205 | return t_pop.get_fittest() 206 | 207 | 208 | def info(txt): 209 | if VERBOSE: 210 | if isinstance(txt, str): 211 | print("[ALGORITHM]", txt) 212 | else: 213 | print("[ALGORITHM]", repr(txt)) 214 | -------------------------------------------------------------------------------- /modestpy/estim/ga/ga.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import logging 8 | import os 9 | import random 10 | 11 | import matplotlib.pyplot as plt 12 | import numpy as np 13 | import pandas as pd 14 | import pyDOE as doe 15 | 16 | import modestpy.estim.plots as plots 17 | from modestpy.estim.estpar import EstPar 18 | from modestpy.estim.ga import algorithm 19 | from modestpy.estim.ga.population import Population 20 | 21 | 22 | class GA(object): 23 | """DEPRECATED. Use MODESTGA instead. 24 | 25 | Genetic algorithm for FMU parameter estimation. 26 | This is the main class of the package, containing the high-level 27 | algorithm and some result plotting methods. 28 | """ 29 | 30 | # Ploting settings 31 | FIG_DPI = 150 32 | FIG_SIZE = (15, 10) 33 | 34 | NAME = "GA" 35 | METHOD = "_method_" 36 | ITER = "_iter_" 37 | ERR = "_error_" 38 | 39 | def __init__( 40 | self, 41 | fmu_path, 42 | inp, 43 | known, 44 | est, 45 | ideal, 46 | maxiter=100, 47 | tol=0.001, 48 | look_back=10, 49 | pop_size=40, 50 | uniformity=0.5, 51 | mut=0.05, 52 | mut_inc=0.3, 53 | trm_size=6, 54 | ftype="RMSE", 55 | init_pop=None, 56 | lhs=False, 57 | ): 58 | """ 59 | The population can be initialized in various ways: 60 | - if `init_pop` is None, one individual is initialized using 61 | initial guess from `est` 62 | - if `init_pop` contains less individuals than `pop_size`, 63 | then the rest is random 64 | - if `init_pop` == `pop_size` then no random individuals are generated 65 | 66 | :param fmu_path: string, absolute path to the FMU 67 | :param inp: DataFrame, columns with input timeseries, index in seconds 68 | :param known: Dictionary, key=parameter_name, value=value 69 | :param est: Dictionary, key=parameter_name, value=tuple 70 | (guess value, lo limit, hi limit), guess can be None 71 | :param ideal: DataFrame, ideal solution to be compared with model 72 | outputs (variable names must match) 73 | :param maxiter: int, maximum number of generations 74 | :param tol: float, when error does not decrease by more than 75 | ``tol`` for the last ``lookback`` generations, 76 | simulation stops 77 | :param look_back: int, number of past generations to track 78 | the error decrease (see ``tol``) 79 | :param pop_size: int, size of the population 80 | :param uniformity: float (0.-1.), uniformity rate, affects gene 81 | exchange in the crossover operation 82 | :param mut: float (0.-1.), mutation rate, specifies how often genes 83 | are to be mutated to a random value, 84 | helps to reach the global optimum 85 | :param mut_inc: float (0.-1.), increased mutation rate, specifies 86 | how often genes are to be mutated by a 87 | small amount, used when the population diversity 88 | is low, helps to reach a local optimum 89 | :param trm_size: int, size of the tournament 90 | :param string ftype: Cost function type. Currently 'NRMSE' 91 | (advised for multi-objective estimation) 92 | or 'RMSE'. 93 | :param DataFrame init_pop: Initial population. DataFrame with 94 | estimated parameters. If None, takes 95 | initial guess from est. 96 | :param bool lhs: If True, init_pop and initial guess in est are 97 | neglected, and the population is chosen using 98 | Lating Hypercube Sampling. 99 | """ 100 | self.logger = logging.getLogger(type(self).__name__) 101 | 102 | deprecated_msg = "This GA implementation is deprecated. Use MODESTGA instead." 103 | print(deprecated_msg) 104 | self.logger.warning( 105 | "This GA implementation is deprecated. Use MODESTGA instead." 106 | ) 107 | 108 | self.logger.info("GA constructor invoked") 109 | 110 | assert inp.index.equals(ideal.index), "inp and ideal indexes are not matching" 111 | 112 | # Evolution parameters 113 | algorithm.UNIFORM_RATE = uniformity 114 | algorithm.MUT_RATE = mut 115 | algorithm.MUT_RATE_INC = mut_inc 116 | algorithm.TOURNAMENT_SIZE = int(trm_size) 117 | 118 | self.max_generations = maxiter 119 | self.tol = tol 120 | self.look_back = look_back 121 | 122 | # History of fittest errors from each generation (list of floats) 123 | self.fittest_errors = list() 124 | 125 | # History of all estimates and errors from all individuals 126 | self.all_estim_and_err = pd.DataFrame() 127 | 128 | # Initiliaze EstPar objects 129 | estpars = list() 130 | for key in sorted(est.keys()): 131 | self.logger.info( 132 | "Add {} (initial guess={}) to estimated parameters".format( 133 | key, est[key][0] 134 | ) 135 | ) 136 | estpars.append( 137 | EstPar(name=key, value=est[key][0], lo=est[key][1], hi=est[key][2]) 138 | ) 139 | 140 | # Put known into DataFrame 141 | known_df = pd.DataFrame() 142 | for key in known: 143 | assert ( 144 | known[key] is not None 145 | ), "None is not allowed in known parameters (parameter {})".format(key) 146 | known_df[key] = [known[key]] 147 | self.logger.info("Known parameters:\n{}".format(str(known_df))) 148 | 149 | # If LHS initialization, init_pop is disregarded 150 | if lhs: 151 | self.logger.info("LHS initialization") 152 | init_pop = GA._lhs_init( 153 | par_names=[p.name for p in estpars], 154 | bounds=[(p.lo, p.hi) for p in estpars], 155 | samples=pop_size, 156 | criterion="c", 157 | ) 158 | self.logger.debug("Current population:\n{}".format(str(init_pop))) 159 | # Else, if no init_pop provided, generate one individual 160 | # based on initial guess from `est` 161 | elif init_pop is None: 162 | self.logger.info( 163 | "No initial population provided, one individual will be based " 164 | "on the initial guess and the other will be random" 165 | ) 166 | init_pop = pd.DataFrame({k: [est[k][0]] for k in est}) 167 | self.logger.debug("Current population:\n{}".format(str(init_pop))) 168 | 169 | # Take individuals from init_pop and add random individuals 170 | # until pop_size == len(init_pop) 171 | # (the number of individuals in init_pop can be lower than 172 | # the desired pop_size) 173 | if init_pop is not None: 174 | missing = pop_size - init_pop.index.size 175 | self.logger.debug("Missing individuals = {}".format(missing)) 176 | if missing > 0: 177 | self.logger.debug("Add missing individuals (random)...") 178 | while missing > 0: 179 | init_pop = init_pop.append( 180 | { 181 | key: random.random() * (est[key][2] - est[key][1]) 182 | + est[key][1] 183 | for key in sorted(est.keys()) 184 | }, 185 | ignore_index=True, 186 | ) 187 | missing -= 1 188 | self.logger.debug("Current population:\n{}".format(str(init_pop))) 189 | 190 | # Initialize population 191 | self.logger.debug("Instantiate Population ") 192 | self.pop = Population( 193 | fmu_path=fmu_path, 194 | pop_size=pop_size, 195 | inp=inp, 196 | known=known_df, 197 | est=estpars, 198 | ideal=ideal, 199 | init=True, 200 | ftype=ftype, 201 | init_pop=init_pop, 202 | ) 203 | 204 | def estimate(self): 205 | """ 206 | Proxy method. Each algorithm from ``estim`` 207 | package should have this method 208 | 209 | :return: DataFrame 210 | """ 211 | self.evolution() 212 | return self.get_estimates() 213 | 214 | def evolution(self): 215 | 216 | gen_count = 1 217 | err_decreasing = True 218 | 219 | # Generation 1 (initialized population) 220 | self.logger.info("Generation " + str(gen_count)) 221 | self.logger.info(str(self.pop)) 222 | 223 | # Update results 224 | self._update_res(gen_count) 225 | 226 | gen_count += 1 227 | 228 | # Next generations (evolution) 229 | while (gen_count <= self.max_generations) and err_decreasing: 230 | 231 | # Evolve 232 | self.pop = algorithm.evolve(self.pop) 233 | 234 | # Update results 235 | self._update_res(gen_count) 236 | 237 | # Print info 238 | self.logger.info("Generation " + str(gen_count)) 239 | self.logger.info(str(self.pop)) 240 | 241 | # Look back 242 | if len(self.fittest_errors) > self.look_back: 243 | err_past = self.fittest_errors[-self.look_back] 244 | err_now = self.fittest_errors[-1] 245 | err_decrease = err_past - err_now 246 | if err_decrease < self.tol: 247 | self.logger.info( 248 | "Error decrease smaller than tol: {0:.5f} < {1:.5f}".format( 249 | err_decrease, self.tol 250 | ) 251 | ) 252 | self.logger.info("Stopping evolution...") 253 | err_decreasing = False 254 | else: 255 | self.logger.info( 256 | "'Look back' error decrease = {0:.5f} > " 257 | "tol = {1:.5f}\n".format(err_decrease, self.tol) 258 | ) 259 | # Increase generation count 260 | gen_count += 1 261 | 262 | # Print summary 263 | self.logger.info("FITTEST PARAMETERS:\n{}".format(self.get_estimates())) 264 | 265 | # Return 266 | return self.pop.get_fittest() 267 | 268 | def get_estimates(self, as_dict=False): 269 | """ 270 | Gets estimated parameters of the best (fittest) individual. 271 | 272 | :param as_dict: boolean (True to get dictionary instead DataFrame) 273 | :return: DataFrame 274 | """ 275 | return self.pop.get_fittest_estimates() 276 | 277 | def get_error(self): 278 | """ 279 | :return: float, last error 280 | """ 281 | return self.pop.get_fittest_error() 282 | 283 | def get_errors(self): 284 | """ 285 | :return: list, all errors from all generations 286 | """ 287 | return self.fittest_errors 288 | 289 | def get_sim_res(self): 290 | """ 291 | Gets simulation result of the best individual. 292 | 293 | :return: DataFrame 294 | """ 295 | return self.pop.get_fittest().result.copy() 296 | 297 | def get_full_solution_trajectory(self): 298 | """ 299 | Returns all parameters and errors from all iterations. 300 | The returned DataFrame contains columns with parameter names, 301 | additional column '_error_' for the error and the index 302 | named '_iter_'. 303 | 304 | :return: DataFrame 305 | """ 306 | df = self.all_estim_and_err.copy() 307 | summary = pd.DataFrame() 308 | for i in range(1, df[GA.ITER].max() + 1): 309 | summary = summary.append(self._get_best_from_gen(i)) 310 | 311 | summary[GA.ITER] = summary[GA.ITER].astype(int) 312 | summary = summary.set_index(GA.ITER) 313 | 314 | summary[GA.METHOD] = GA.NAME 315 | 316 | return summary 317 | 318 | def get_plots(self): 319 | """ 320 | Returns a list with important plots produced by this estimation method. 321 | Each list element is a dictionary with keys 'name' and 'axes'. The name 322 | should be given as a string, while axes as matplotlib.Axes instance. 323 | 324 | :return: list(dict) 325 | """ 326 | plots = list() 327 | plots.append({"name": "GA", "axes": self.plot_pop_evo()}) 328 | return plots 329 | 330 | def save_plots(self, workdir): 331 | self.plot_comparison(os.path.join(workdir, "ga_comparison.png")) 332 | self.plot_error_evo(os.path.join(workdir, "ga_error_evo.png")) 333 | self.plot_parameter_evo(os.path.join(workdir, "ga_param_evo.png")) 334 | self.plot_pop_evo(os.path.join(workdir, "ga_pop_evo.png")) 335 | 336 | def plot_error_evo(self, file=None): 337 | """Returns a plot of the error evolution. 338 | 339 | :param file: string (path to the file, if None, file not created) 340 | :return: Axes 341 | """ 342 | fig, ax = plt.subplots() 343 | ax.plot(self.fittest_errors) 344 | ax.set_xlabel("Generation") 345 | ax.set_ylabel("Error (NRMSE)") 346 | if file: 347 | fig = ax.get_figure() 348 | fig.set_size_inches(GA.FIG_SIZE) 349 | fig.savefig(file, dpi=GA.FIG_DPI) 350 | return ax 351 | 352 | def plot_comparison(self, file=None): 353 | """ 354 | Creates a plot with a comparison of simulation results 355 | (fittest individual) vs. measured result. 356 | 357 | :param file: string, path to the file. If ``None``, file not created. 358 | :return: Axes 359 | """ 360 | simulated = self.get_sim_res() 361 | measured = self.pop.ideal.copy() 362 | return plots.plot_comparison(simulated, measured, file) 363 | 364 | def plot_parameter_evo(self, file=None): 365 | """ 366 | Returns a plot of the parameter evolution. 367 | 368 | :param file: string (path to the file, if None, file not created) 369 | :return: Axes 370 | """ 371 | parameters = self.get_full_solution_trajectory() 372 | parameters = parameters.drop("generation", axis=1) 373 | return plots.plot_parameter_evo(parameters, file) 374 | 375 | def plot_inputs(self, file=None): 376 | """ 377 | Returns a plot with inputs. 378 | 379 | :param file: string 380 | :return: axes 381 | """ 382 | inputs = self.pop.inputs 383 | return plots.plot_inputs(inputs, file) 384 | 385 | def plot_pop_evo(self, file=None): 386 | """ 387 | Creates a plot with the evolution of all parameters as a scatter plot. 388 | Can be interpreted as the *population diversity*. 389 | The color of the points is darker for higher accuracy. 390 | 391 | :param file: string, path to the file. If ``None``, file not created. 392 | :return: Axes 393 | """ 394 | estimates = self.all_estim_and_err 395 | pars = list(estimates.columns) 396 | pars.remove("individual") 397 | pars.remove(GA.ITER) 398 | pars.remove(GA.ERR) 399 | assert len(pars) > 0, "No parameters found" 400 | 401 | fig, axes = plt.subplots(nrows=len(pars), sharex=True, squeeze=False) 402 | fig.subplots_adjust(right=0.75) 403 | i = 0 404 | 405 | last_err = self.fittest_errors[-1] 406 | first_err = self.fittest_errors[0] 407 | 408 | for v in pars: 409 | ax = axes[i, 0] 410 | scatter = ax.scatter( 411 | x=estimates[GA.ITER], 412 | y=estimates[v], 413 | c=estimates[GA.ERR], 414 | cmap="viridis", 415 | edgecolors="none", 416 | vmin=last_err, 417 | vmax=first_err, 418 | alpha=0.25, 419 | ) 420 | ax.set_xlim([0, estimates[GA.ITER].max() + 1]) 421 | ax.text( 422 | x=1.05, 423 | y=0.5, 424 | s=v, 425 | transform=ax.transAxes, 426 | fontweight="bold", 427 | horizontalalignment="center", 428 | verticalalignment="center", 429 | ) 430 | i += 1 431 | axes[-1, 0].set_xlabel("Generation") 432 | 433 | # Color bar on the side 434 | cbar_ax = fig.add_axes([0.85, 0.10, 0.05, 0.8]) 435 | fig.colorbar(scatter, cax=cbar_ax, label="Error") 436 | 437 | if file: 438 | fig.set_size_inches(GA.FIG_SIZE) 439 | fig.savefig(file, dpi=GA.FIG_DPI) 440 | return axes 441 | 442 | def _update_res(self, gen_count): 443 | # Save estimates 444 | generation_estimates = self.pop.get_all_estimates_and_errors() 445 | generation_estimates[GA.ITER] = gen_count 446 | self.all_estim_and_err = pd.concat( 447 | [self.all_estim_and_err, generation_estimates] 448 | ) 449 | 450 | # Append error lists 451 | self.fittest_errors.append(self.pop.get_fittest_error()) 452 | 453 | def _get_best_from_gen(self, generation): 454 | """ 455 | Gets fittest individuals (parameter sets) from the chosen generation. 456 | 457 | :param generation: int (generation number) 458 | :return: DataFrame 459 | """ 460 | df = self.all_estim_and_err.copy() 461 | df.index = df[GA.ITER] 462 | # Select individuals with minimum error from the chosen individuals 463 | fittest = df.loc[df[GA.ERR] == df.loc[generation][GA.ERR].min()].loc[generation] 464 | # Check how many individuals found 465 | if isinstance(fittest, pd.DataFrame): 466 | # More than 1 found... 467 | # Get the first one 468 | fittest = fittest.iloc[0] 469 | elif isinstance(fittest, pd.Series): 470 | # Only 1 found... 471 | pass 472 | # Drop column 'individual' 473 | fittest = fittest.drop("individual") 474 | 475 | return fittest 476 | 477 | def _get_n_param(self): 478 | """ 479 | Returns number of estimated parameters 480 | 481 | :return: int 482 | """ 483 | return len(self.get_estimates()) 484 | 485 | @staticmethod 486 | def _lhs_init(par_names, bounds, samples, criterion="c"): 487 | """ 488 | Returns LHS samples. 489 | 490 | :param par_names: List of parameter names 491 | :type par_names: list(str) 492 | :param bounds: List of lower/upper bounds, 493 | must be of the same length as par_names 494 | :type bounds: list(tuple(float, float)) 495 | :param int samples: Number of samples 496 | :param str criterion: A string that tells lhs how to sample the 497 | points. See docs for pyDOE.lhs(). 498 | :return: DataFrame 499 | """ 500 | lhs = doe.lhs(len(par_names), samples=samples, criterion="c") 501 | par_vals = {} 502 | for par, i in zip(par_names, range(len(par_names))): 503 | par_min = bounds[i][0] 504 | par_max = bounds[i][1] 505 | par_vals[par] = lhs[:, i] * (par_max - par_min) + par_min 506 | 507 | # Convert dict(str: np.ndarray) to pd.DataFrame 508 | par_df = pd.DataFrame(columns=par_names, index=np.arange(samples)) 509 | for i in range(samples): 510 | for p in par_names: 511 | par_df.loc[i, p] = par_vals[p][i] 512 | 513 | logger = logging.getLogger(GA.__name__) 514 | logger.info("Initial guess based on LHS:\n{}".format(par_df)) 515 | return par_df 516 | -------------------------------------------------------------------------------- /modestpy/estim/ga/individual.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import copy 8 | import logging 9 | import random 10 | 11 | import numpy as np 12 | import pandas as pd 13 | 14 | from modestpy.estim.error import calc_err 15 | 16 | 17 | class Individual(object): 18 | def __init__( 19 | self, est_objects, population, genes=None, use_init_guess=False, ftype="NRMSE" 20 | ): 21 | """ 22 | Individual can be initialized using `genes` OR initial guess 23 | in `est_objects` (genes are inferred from parameters and vice versa). 24 | Otherwise, random genes are assumed. 25 | 26 | :param est_objects: List of EstPar objects with estimated parameters 27 | :type est_objects: list(EstPar) 28 | :param Population population: Population instance 29 | :param genes: Genes (can be also inferred from `parameters`) 30 | :type genes: dict(str: float) 31 | :param bool use_init_guess: If True, use initial guess from 32 | `est_objects` 33 | :param str ftype: Cost function type, 'RMSE' or 'NRMSE' 34 | """ 35 | 36 | self.logger = logging.getLogger(type(self).__name__) 37 | 38 | # Reference to the population object 39 | self.population = population 40 | 41 | # Assign variables shared across the population 42 | self.ideal = population.ideal 43 | self.model = population.model 44 | 45 | # Cost function type 46 | self.ftype = ftype 47 | 48 | # Deep copy EstPar instances to avoid sharing between individuals 49 | self.est_par_objects = copy.deepcopy(est_objects) 50 | 51 | # Generate genes 52 | if not genes and not use_init_guess: 53 | # Generate random genes 54 | est_names = Individual._get_names(self.est_par_objects) 55 | self.genes = Individual._random_genes(est_names) 56 | elif genes and not use_init_guess: 57 | # Use provided genes 58 | self.genes = copy.deepcopy(genes) 59 | elif use_init_guess and not genes: 60 | # Infer genes from parameters 61 | self.genes = dict() 62 | for p in self.est_par_objects: 63 | self.genes[p.name] = (p.value - p.lo) / (p.hi - p.lo) 64 | assert ( 65 | self.genes[p.name] >= 0.0 and self.genes[p.name] <= 1.0 66 | ), "Initial guess outside the bounds" 67 | else: 68 | msg = "Either genes or parameters have to be None" 69 | self.logger.error(msg) 70 | raise ValueError(msg) 71 | 72 | # Update parameters 73 | self._update_parameters() 74 | 75 | # Individual result 76 | self.result = None 77 | self.error = None 78 | 79 | # Main methods ------------------------------ 80 | def calculate(self): 81 | # Just in case, individual result and error 82 | # are cleared before simulation 83 | self.reset() 84 | # Important to set estimated parameters just before simulation, 85 | # because all individuals share the same model instance 86 | self.model.set_param(self.est_par_df) 87 | # Simulation 88 | self.result = self.model.simulate() 89 | # Make sure the returned result is not empty 90 | assert ( 91 | self.result.empty is False 92 | ), "Empty result returned from simulation... (?)" 93 | # Calculate error 94 | self.logger.debug( 95 | "Calculating error ({}) in individual {}".format(self.ftype, self.genes) 96 | ) 97 | self.error = calc_err(self.result, self.ideal, ftype=self.ftype) 98 | 99 | def reset(self): 100 | self.result = None 101 | self.error = None 102 | self.est_par_objects = copy.deepcopy(self.est_par_objects) 103 | 104 | def set_gene(self, name, value): 105 | self.genes[name] = value 106 | self._update_parameters() 107 | 108 | def get_gene(self, name): 109 | return self.genes[name] 110 | 111 | def get_sorted_gene_names(self): 112 | return sorted(self.genes.keys()) 113 | 114 | def get_estimates(self, as_dict=False): 115 | """ 116 | :param as_dict: boolean (True to get dictionary instead DataFrame) 117 | :return: DataFrame with estimated parameters 118 | """ 119 | df = pd.DataFrame() 120 | for par in self.est_par_objects: 121 | df[par.name] = np.array([par.value]) 122 | if as_dict: 123 | return df.to_dict() 124 | else: 125 | return df 126 | 127 | def get_estimates_and_error(self): 128 | estimates = self.get_estimates() 129 | estimates["_error_"] = self.error["tot"] 130 | return estimates 131 | 132 | def get_clone(self): 133 | clone = Individual( 134 | self.est_par_objects, self.population, self.genes, ftype=self.ftype 135 | ) 136 | return clone 137 | 138 | # Private methods --------------------------- 139 | def _update_parameters(self): 140 | # Calculate parameter values 141 | self.est_par_objects = self._calc_parameters(self.genes) 142 | # Convert estimated parameters to dataframe 143 | self.est_par_df = Individual._est_pars_2_df(self.est_par_objects) 144 | 145 | @staticmethod 146 | def _est_pars_2_df(est_pars): 147 | df = pd.DataFrame() 148 | for p in est_pars: 149 | df[p.name] = np.array([p.value]) 150 | return df 151 | 152 | def _calc_parameters(self, genes): 153 | """ 154 | Calculates parameters based on genes and limits. 155 | :return: None 156 | """ 157 | for par in self.est_par_objects: 158 | gene = genes[par.name] 159 | par.value = par.lo + gene * (par.hi - par.lo) 160 | return self.est_par_objects 161 | 162 | @staticmethod 163 | def _random_genes(par_names): 164 | """ 165 | Generates random genes. 166 | :return: dict(str: float) 167 | """ 168 | genes = dict() 169 | for par in par_names: 170 | g = 0 171 | while g == 0: # Because random.random() can return 0 172 | g = random.random() 173 | genes[par] = g 174 | return genes 175 | 176 | @staticmethod 177 | def _get_names(est_params): 178 | names = list() 179 | for par in est_params: 180 | names.append(par.name) 181 | return names 182 | 183 | # Overriden methods -------------------------- 184 | def __str__(self): 185 | s = "Individual (" 186 | for par in self.est_par_objects: 187 | s += par.name + "={0:.3f}".format(par.value) 188 | s += ", " 189 | # Delete trailing comma 190 | s = s[:-2] 191 | s += "), err=" 192 | if self.error: 193 | s += "{:.4f} ".format(self.error["tot"]) 194 | else: 195 | s += "None" 196 | return s 197 | -------------------------------------------------------------------------------- /modestpy/estim/ga/population.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import copy 8 | import logging 9 | 10 | import pandas as pd 11 | 12 | from modestpy.estim.ga.individual import Individual 13 | from modestpy.fmi.model import Model 14 | 15 | 16 | class Population(object): 17 | def __init__( 18 | self, 19 | fmu_path, 20 | pop_size, 21 | inp, 22 | known, 23 | est, 24 | ideal, 25 | init=True, 26 | opts=None, 27 | ftype="NRMSE", 28 | init_pop=None, 29 | ): 30 | """ 31 | :param fmu_path: string 32 | :param pop_size: int 33 | :param inp: DataFrame 34 | :param known: DataFrame 35 | :param est: List of EstPar objects 36 | :param ideal: DataFrame 37 | :param init: bool 38 | :param dict opts: Additional FMI options to be passed to the simulator 39 | :param string ftype: Cost function type. Currently 'NRMSE' or 'RMSE'. 40 | :param DataFrame init_pop: Initial population, DataFrame with initial 41 | guesses for estimated parameters 42 | """ 43 | self.logger = logging.getLogger(type(self).__name__) 44 | 45 | # Initialize list of individuals 46 | self.individuals = list() 47 | 48 | # Assign attributes 49 | self.fmu_path = fmu_path 50 | self.pop_size = pop_size 51 | self.inputs = inp 52 | self.known_pars = known 53 | self.estpar = est 54 | self.outputs = [var for var in ideal] 55 | self.ideal = ideal 56 | self.ftype = ftype 57 | 58 | # Instantiate model 59 | self.model = None 60 | 61 | if init: 62 | # Instiate individuals before initialization 63 | self.instantiate_model(opts=opts) 64 | self._initialize(init_pop) 65 | self.calculate() 66 | 67 | def instantiate_model(self, opts): 68 | self.model = Model(self.fmu_path, opts=opts) 69 | self.model.set_input(self.inputs) 70 | self.model.set_param(self.known_pars) 71 | self.model.set_outputs(self.outputs) 72 | 73 | def add_individual(self, indiv): 74 | assert isinstance(indiv, Individual), "Only Individual instances allowed..." 75 | indiv.reset() 76 | self.individuals.append(indiv) 77 | 78 | def calculate(self): 79 | for i in self.individuals: 80 | i.calculate() 81 | 82 | def size(self): 83 | return self.pop_size 84 | 85 | def get_fittest(self): 86 | fittest = self.individuals[0] 87 | for ind in self.individuals: 88 | if ind.error["tot"] < fittest.error["tot"]: 89 | fittest = ind 90 | fittest = copy.copy(fittest) 91 | return fittest 92 | 93 | def get_fittest_error(self): 94 | return self.get_fittest().error["tot"] 95 | 96 | def get_population_errors(self): 97 | err = list() 98 | for i in self.individuals: 99 | err.append(i.error["tot"]) 100 | return err 101 | 102 | def get_fittest_estimates(self): 103 | return self.get_fittest().get_estimates() 104 | 105 | def get_all_estimates_and_errors(self): 106 | all_estim = pd.DataFrame() 107 | i = 1 108 | for ind in self.individuals: 109 | i_estim = ind.get_estimates_and_error() 110 | i_estim["individual"] = i 111 | all_estim = pd.concat([all_estim, i_estim]) 112 | i += 1 113 | return all_estim 114 | 115 | def get_estpars(self): 116 | """Returns EstPar list""" 117 | return self.estpar 118 | 119 | def _initialize(self, init_pop=None): 120 | self.logger.debug("Initialize population with init_pop=\n{}".format(init_pop)) 121 | 122 | self.individuals = list() 123 | 124 | # How to initialize? Random or explicit initial guess? 125 | if init_pop is not None: 126 | assert ( 127 | len(init_pop.index) == self.pop_size 128 | ), "Population size does not match initial guess {} != {}".format( 129 | init_pop.index.size, self.pop_size 130 | ) 131 | init_guess = True 132 | else: 133 | init_guess = False 134 | 135 | for i in range(self.pop_size): 136 | # --------------------------------------------------> BUG HERE 137 | if init_guess: 138 | # Update value in EstPar objects with the next initial guess 139 | for n in range(len(self.estpar)): 140 | self.estpar[n].value = init_pop.loc[i, self.estpar[n].name] 141 | self.logger.debug("Individual #{} <- {}".format(i, self.estpar[n])) 142 | 143 | self.add_individual( 144 | Individual( 145 | est_objects=self.estpar, 146 | population=self, 147 | ftype=self.ftype, 148 | use_init_guess=init_guess, 149 | ) 150 | ) 151 | # <-------------------------------------------------- BUG HERE 152 | 153 | def __str__(self): 154 | fittest = self.get_fittest() 155 | s = repr(self) 156 | s += "\n" 157 | s += "Number of individuals: " + str(len(self.individuals)) + "\n" 158 | if len(self.individuals) > 0: 159 | for i in self.individuals: 160 | s += str(i) + "\n" 161 | s += "-" * 110 + "\n" 162 | s += "Fittest: " + str(fittest) + "\n" 163 | s += "-" * 110 + "\n" 164 | else: 165 | s += "EMPTY" 166 | return s 167 | -------------------------------------------------------------------------------- /modestpy/estim/ga_parallel/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | 5 | This code is licensed under BSD 2-clause license. 6 | See LICENSE file in the project root for license terms. 7 | """ 8 | -------------------------------------------------------------------------------- /modestpy/estim/ga_parallel/ga_parallel.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import logging 8 | import os 9 | from multiprocessing import Manager 10 | from multiprocessing.managers import BaseManager 11 | from random import random 12 | 13 | import numpy as np 14 | import pandas as pd 15 | from modestga import minimize 16 | 17 | import modestpy.estim.plots as plots 18 | import modestpy.utilities.figures as figures 19 | from modestpy.estim.error import calc_err 20 | from modestpy.estim.estpar import EstPar 21 | from modestpy.estim.estpar import estpars_2_df 22 | from modestpy.fmi.model import Model 23 | 24 | 25 | class ObjectiveFun: 26 | def __init__(self, fmu_path, inp, known, est, ideal, ftype="RMSE"): 27 | self.logger = logging.getLogger(type(self).__name__) 28 | self.model = None 29 | self.fmu_path = fmu_path 30 | self.inp = inp 31 | 32 | # Known parameters to DataFrame 33 | known_df = pd.DataFrame() 34 | for key in known: 35 | assert ( 36 | known[key] is not None 37 | ), "None is not allowed in known parameters (parameter {})".format(key) 38 | known_df[key] = [known[key]] 39 | self.known = known_df 40 | 41 | self.est = est 42 | self.ideal = ideal 43 | self.output_names = [var for var in ideal] 44 | self.ftype = ftype 45 | self.best_err = 1e7 46 | self.res = pd.DataFrame() 47 | 48 | self.logger.debug(f"fmu_path = {fmu_path}") 49 | self.logger.debug(f"inp = {inp}") 50 | self.logger.debug(f"known = {known}") 51 | self.logger.debug(f"est = {est}") 52 | self.logger.debug(f"ideal = {ideal}") 53 | self.logger.debug(f"output_names = {self.output_names}") 54 | 55 | def rescale(self, v, lo, hi): 56 | return lo + v * (hi - lo) 57 | 58 | def __call__(self, x, *args): 59 | # Instantiate the model 60 | self.logger.debug(f"x = {x}") 61 | if self.model is None: 62 | self.model = self._get_model_instance( 63 | self.fmu_path, self.inp, self.known, self.est, self.output_names 64 | ) 65 | logging.debug(f"Model instance returned: {self.model}") 66 | 67 | # Updated parameters are stored in x. Need to update the model. 68 | parameters = pd.DataFrame(index=[0]) 69 | try: 70 | for v, ep in zip(x, self.est): 71 | parameters[ep.name] = self.rescale(v, ep.lo, ep.hi) 72 | except TypeError as e: 73 | raise e 74 | self.logger.debug(f"parameters = {parameters}") 75 | self.model.parameters_from_df(parameters) 76 | self.logger.debug(f"est: {self.est}") 77 | self.logger.debug(f"parameters: {parameters}") 78 | self.logger.debug(f"model: {self.model}") 79 | self.logger.debug("Calling simulation...") 80 | result = self.model.simulate() 81 | self.logger.debug(f"result: {result}") 82 | err = calc_err(result, self.ideal, ftype=self.ftype)["tot"] 83 | # Update best error and result 84 | if err < self.best_err: 85 | self.best_err = err 86 | self.res = result 87 | 88 | return err 89 | 90 | def _get_model_instance(self, fmu_path, inputs, known_pars, est, output_names): 91 | self.logger.debug("Getting model instance...") 92 | self.logger.debug(f"inputs = {inputs}") 93 | self.logger.debug(f"known_pars = {known_pars}") 94 | self.logger.debug(f"est = {est}") 95 | self.logger.debug(f"estpars_2_df(est) = {estpars_2_df(est)}") 96 | self.logger.debug(f"output_names = {output_names}") 97 | model = Model(fmu_path) 98 | model.inputs_from_df(inputs) 99 | model.parameters_from_df(known_pars) 100 | model.parameters_from_df(estpars_2_df(est)) 101 | model.specify_outputs(output_names) 102 | self.logger.debug(f"Model instance initialized: {model}") 103 | self.logger.debug(f"Model instance initialized: {model.model}") 104 | res = model.simulate() 105 | self.logger.debug(f"test result: {res}") 106 | return model 107 | 108 | 109 | class MODESTGA(object): 110 | """ 111 | Parallel Genetic Algorithm based on modestga 112 | (https://github.com/krzysztofarendt/modestga). 113 | 114 | Main features of modestga: 115 | - parallel, 116 | - adaptive mutation, 117 | - suitable for large-scale non-convex problems, 118 | - pure Python, so easy to adjust to your own needs. 119 | 120 | The number of CPU cores to use can be controlled 121 | with the argument `workers` (default: all CPUs). If run with `workers=1`, 122 | the algorithm works in a single process mode and is similar to the legacy 123 | GA implementation in modestpy. 124 | 125 | If `workers > 1`, the number of processes is equal to `workers + 1`. 126 | In this parallel mode, the population is divided into `workers` subpopulations, 127 | each of which evolves within a single process, while the main process 128 | is responsible for gene exchange between subpopulations at the end 129 | of each generation. 130 | 131 | Since the population is divided into smaller subpopulations, the user 132 | should remember to adjust the tournament size to not set it larger 133 | than the subpopulation size. 134 | 135 | The full list of options which can be used to control the optimization 136 | process is as follows: 137 | - `workers` - number of CPUs (default all CPUs), 138 | - `generations` - number of GA iterations (default 50), 139 | - `pop_size` - population size (default 50), is divided into `workers` subpopulations, 140 | - `mut_rate` - mutation rate (default 0.01), 141 | - `trm_size` - tournament size (default 20), should be lower than the subpopulation size, 142 | - `tol` - solution absolute tolerance (default 1e-3), 143 | - `inertia` - maximum number of non-improving generations (default 100), 144 | - `xover_ratio` - crossover ratio (default 0.5). 145 | """ 146 | 147 | # Summary placeholder 148 | TMP_SUMMARY = pd.DataFrame() 149 | 150 | # Ploting settings 151 | FIG_DPI = 150 152 | FIG_SIZE = (10, 6) 153 | 154 | NAME = "MODESTGA" 155 | METHOD = "_method_" 156 | ITER = "_iter_" 157 | ERR = "_error_" 158 | 159 | def __init__( 160 | self, 161 | fmu_path, 162 | inp, 163 | known, 164 | est, 165 | ideal, 166 | options={}, 167 | ftype="RMSE", 168 | generations=None, 169 | pop_size=None, 170 | mut_rate=None, 171 | trm_size=None, 172 | tol=None, 173 | inertia=None, 174 | workers=None, 175 | ): 176 | """ 177 | :param fmu_path: string, absolute path to the FMU 178 | :param inp: DataFrame, columns with input timeseries, index in seconds 179 | :param known: Dictionary, key=parameter_name, value=value 180 | :param est: Dictionary, key=parameter_name, value=tuple 181 | (guess value, lo limit, hi limit), guess can be None 182 | :param ideal: DataFrame, ideal solution to be compared with model 183 | outputs (variable names must match) 184 | :param options: dict, additional options passed to the solver (not used here) 185 | :param ftype: str, cost function type. Currently 'NRMSE' (advised 186 | for multi-objective estimation) or 'RMSE'. 187 | :param generations: int, max. number of generations 188 | :param pop_size: int, population size 189 | :param mut_rate: float, mutation rate 190 | :param trm_size: int, tournament size 191 | :param tol: float, absolute solution tolerance 192 | :param inertia: int, maximum number of non-improving generations 193 | :param workers: int, number of CPUs to use 194 | """ 195 | self.logger = logging.getLogger(type(self).__name__) 196 | 197 | assert inp.index.equals(ideal.index), "inp and ideal indexes are not matching" 198 | 199 | self.fmu_path = fmu_path 200 | self.inp = inp 201 | self.known = known 202 | self.ideal = ideal 203 | self.ftype = ftype 204 | 205 | # Default solver options 206 | self.workers = os.cpu_count() # CPU cores to use 207 | self.options = { 208 | "generations": 50, # Max. number of generations 209 | "pop_size": 30, # Population size 210 | "mut_rate": 0.01, # Mutation rate 211 | "trm_size": 3, # Tournament size 212 | "tol": 1e-3, # Solution tolerance 213 | "inertia": 100, # Max. number of non-improving generations 214 | "xover_ratio": 0.5, # Crossover ratio 215 | } 216 | 217 | # User options 218 | if workers is not None: 219 | self.workers = workers 220 | if generations is not None: 221 | self.options["generations"] = generations 222 | if pop_size is not None: 223 | self.options["pop_size"] = pop_size 224 | if mut_rate is not None: 225 | self.options["mut_rate"] = mut_rate 226 | if trm_size is not None: 227 | self.options["trm_size"] = trm_size 228 | if tol is not None: 229 | self.options["tol"] = tol 230 | if inertia is not None: 231 | self.options["inertia"] = inertia 232 | 233 | # Adjust trm_size if population size is too small 234 | if self.options["trm_size"] >= (self.options["pop_size"] // (self.workers * 2)): 235 | new_trm_size = self.options["pop_size"] // (self.workers * 4) 236 | new_pop_size = self.options["pop_size"] 237 | if new_trm_size < 2: 238 | new_trm_size = 2 239 | new_pop_size = new_trm_size * self.workers * 4 240 | self.logger.warning( 241 | "Tournament size has to be lower than pop_size // (workers * 2). " 242 | f"Re-adjusting to trm_size = {new_trm_size}, pop_size = {new_pop_size}" 243 | ) 244 | self.options["trm_size"] = new_trm_size 245 | self.options["pop_size"] = new_pop_size 246 | 247 | # Warn the user about a possible mistake in the chosen options 248 | if self.options["trm_size"] <= 1: 249 | self.logger.warning( 250 | "Tournament size equals 1. The possible reasons are:\n" 251 | " - too small population size leading to readjusted tournament size\n" 252 | " - too many workers (population is divided among workers)\n" 253 | " - you chose tournament size equal to 1 by mistake\n" 254 | "The optimization will proceed, but the performance " 255 | "might be suboptimal..." 256 | ) 257 | 258 | self.logger.debug(f"MODESTGA options: {self.options}") 259 | self.logger.debug(f"MODESTGA workers = {self.workers}") 260 | 261 | # Known parameters to DataFrame 262 | known_df = pd.DataFrame() 263 | for key in known: 264 | assert ( 265 | known[key] is not None 266 | ), "None is not allowed in known parameters (parameter {})".format(key) 267 | known_df[key] = [known[key]] 268 | 269 | # est: dictionary to a list with EstPar instances 270 | self.est = list() 271 | for key in est: 272 | lo = est[key][1] 273 | hi = est[key][2] 274 | if est[key][0] is None: # If guess is None, assume random guess 275 | v = lo + random() * (hi - lo) 276 | else: # Else, take the guess passed in est 277 | v = est[key][0] 278 | self.est.append(EstPar(name=key, value=v, lo=lo, hi=hi)) 279 | est = self.est 280 | 281 | # Model 282 | output_names = [var for var in ideal] 283 | 284 | # Outputs 285 | self.summary = pd.DataFrame() 286 | self.res = pd.DataFrame() 287 | self.best_err = 1e7 288 | 289 | # Temporary placeholder for summary 290 | # It needs to be stored as class variable, because it has to be updated 291 | # from a static method used as callback 292 | self.summary_cols = [x.name for x in self.est] + [MODESTGA.ERR, MODESTGA.METHOD] 293 | MODESTGA.TMP_SUMMARY = pd.DataFrame(columns=self.summary_cols) 294 | 295 | # Log 296 | self.logger.info("MODESTGA initialized... =========================") 297 | 298 | def estimate(self): 299 | # Objective function 300 | self.logger.debug("Instantiating ObjectiveFun") 301 | objective_fun = ObjectiveFun( 302 | self.fmu_path, self.inp, self.known, self.est, self.ideal, self.ftype 303 | ) 304 | self.logger.debug(f"ObjectiveFun: {objective_fun}") 305 | 306 | # Initial guess 307 | x0 = [MODESTGA.scale(x.value, x.lo, x.hi) for x in self.est] 308 | self.logger.debug("modestga x0 = {}".format(x0)) 309 | 310 | # Save initial guess in summary 311 | row = pd.DataFrame(index=[0]) 312 | for x, c in zip(x0, MODESTGA.TMP_SUMMARY.columns): 313 | row[c] = x 314 | row[MODESTGA.ERR] = np.nan 315 | row[MODESTGA.METHOD] = MODESTGA.NAME 316 | MODESTGA.TMP_SUMMARY = MODESTGA.TMP_SUMMARY.append(row, ignore_index=True) 317 | 318 | # Parameter bounds 319 | b = [(0.0, 1.0) for x in self.est] 320 | self.logger.debug(f"bounds = {b}") 321 | 322 | out = minimize( 323 | objective_fun, 324 | bounds=b, 325 | x0=x0, 326 | args=(), 327 | callback=MODESTGA._callback, 328 | options=self.options, 329 | workers=self.workers, 330 | ) 331 | 332 | self.logger.debug(f"out = {out}") 333 | outx = [ 334 | MODESTGA.rescale(x, ep.lo, ep.hi) for x, ep in zip(out.x.tolist(), self.est) 335 | ] 336 | 337 | self.logger.debug("modestga x = {}".format(outx)) 338 | 339 | # Update summary 340 | self.summary = MODESTGA.TMP_SUMMARY.copy() 341 | self.summary.index += 1 # Adjust iteration counter 342 | self.summary.index.name = MODESTGA.ITER # Rename index 343 | 344 | # Update error 345 | self.summary[MODESTGA.ERR] = list( 346 | map(objective_fun, self.summary[[x.name for x in self.est]].values) 347 | ) 348 | 349 | for ep in self.est: 350 | name = ep.name 351 | # list(map(...)) - for Python 2/3 compatibility 352 | self.summary[name] = list( 353 | map(lambda x: MODESTGA.rescale(x, ep.lo, ep.hi), self.summary[name]) 354 | ) # Rescale 355 | 356 | # Reset temp placeholder 357 | MODESTGA.TMP_SUMMARY = pd.DataFrame(columns=self.summary_cols) 358 | 359 | # Return DataFrame with estimates 360 | par_vec = outx 361 | par_df = pd.DataFrame(columns=[x.name for x in self.est], index=[0]) 362 | for col, x in zip(par_df.columns, par_vec): 363 | par_df[col] = x 364 | 365 | return par_df 366 | 367 | @staticmethod 368 | def scale(v, lo, hi): 369 | # scaled = (rescaled - lo) / (hi - lo) 370 | return (v - lo) / (hi - lo) 371 | 372 | @staticmethod 373 | def rescale(v, lo, hi): 374 | # rescaled = lo + scaled * (hi - lo) 375 | return lo + v * (hi - lo) 376 | 377 | def get_plots(self): 378 | """ 379 | Returns a list with important plots produced by this estimation method. 380 | Each list element is a dictionary with keys 'name' and 'axes'. The name 381 | should be given as a string, while axes as matplotlib.Axes instance. 382 | 383 | :return: list(dict) 384 | """ 385 | plots = list() 386 | plots.append({"name": "MODESTGA", "axes": self.plot_parameter_evo()}) 387 | return plots 388 | 389 | def save_plots(self, workdir): 390 | self.plot_comparison(os.path.join(workdir, "ps_comparison.png")) 391 | self.plot_error_evo(os.path.join(workdir, "ps_error_evo.png")) 392 | self.plot_parameter_evo(os.path.join(workdir, "ps_param_evo.png")) 393 | 394 | def plot_comparison(self, file=None): 395 | return plots.plot_comparison(self.res, self.ideal, file) 396 | 397 | def plot_error_evo(self, file=None): 398 | err_df = pd.DataFrame(self.summary[MODESTGA.ERR]) 399 | return plots.plot_error_evo(err_df, file) 400 | 401 | def plot_parameter_evo(self, file=None): 402 | par_df = self.summary.drop([MODESTGA.METHOD], axis=1) 403 | par_df = par_df.rename( 404 | columns={x: "error" if x == MODESTGA.ERR else x for x in par_df.columns} 405 | ) 406 | 407 | # Get axes 408 | axes = par_df.plot(subplots=True) 409 | fig = figures.get_figure(axes) 410 | # x label 411 | axes[-1].set_xlabel("Iteration") 412 | # ylim for error 413 | axes[-1].set_ylim(0, None) 414 | 415 | if file: 416 | fig.set_size_inches(MODESTGA.FIG_SIZE) 417 | fig.savefig(file, dpi=MODESTGA.FIG_DPI) 418 | return axes 419 | 420 | def get_full_solution_trajectory(self): 421 | """ 422 | Returns all parameters and errors from all iterations. 423 | The returned DataFrame contains columns with parameter names, 424 | additional column '_error_' for the error and the index 425 | named '_iter_'. 426 | 427 | :return: DataFrame 428 | """ 429 | return self.summary 430 | 431 | def get_error(self): 432 | """ 433 | :return: float, last error 434 | """ 435 | return float(self.summary[MODESTGA.ERR].iloc[-1]) 436 | 437 | def get_errors(self): 438 | """ 439 | :return: list, all errors from all iterations 440 | """ 441 | return self.summary[MODESTGA.ERR].tolist() 442 | 443 | @staticmethod 444 | def _callback(xk, fx, ng, *args): # TODO: it must be pickable for multiprocessing 445 | # New row 446 | row = pd.DataFrame(index=[0]) 447 | for x, c in zip(xk, MODESTGA.TMP_SUMMARY.columns): 448 | row[c] = x 449 | 450 | row[MODESTGA.ERR] = np.nan 451 | row[MODESTGA.METHOD] = MODESTGA.NAME 452 | 453 | # Append 454 | MODESTGA.TMP_SUMMARY = MODESTGA.TMP_SUMMARY.append(row, ignore_index=True) 455 | -------------------------------------------------------------------------------- /modestpy/estim/make_param_file.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import copy 8 | 9 | import pandas as pd 10 | 11 | 12 | def make_param_file(est, known, path): 13 | """ 14 | Saves parameter file from ``est`` and ``known`` dictionaries. 15 | 16 | :param est: Dictionary, key=parameter_name, value=tuple 17 | (guess value, lo limit, hi limit), guess can be None 18 | :param known: Dictionary, key=parameter_name, value=value 19 | :param path: string, path to the file 20 | :return: None 21 | """ 22 | par = copy.copy(known) 23 | for p in est: 24 | par[p] = est[p][0] 25 | for p in par: 26 | par[p] = [par[p]] 27 | par = pd.DataFrame.from_dict(par) 28 | par.to_csv(path, index=False) 29 | -------------------------------------------------------------------------------- /modestpy/estim/plots.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import matplotlib.pyplot as plt 8 | import numpy as np 9 | 10 | 11 | def plot_comparison(sim_res, ideal_res, f=None): 12 | """Plots comparison: simulation vs. ideal results. 13 | 14 | :param sim_res: DataFrame 15 | :param ideal_res: DataFrame 16 | :param f: string 17 | :return: axes 18 | """ 19 | simulated = sim_res.copy() 20 | measured = ideal_res.copy() 21 | measured.columns = [x + "_meas" for x in measured.columns] 22 | 23 | variables = list(simulated.columns) 24 | assert len(variables) > 0, "No output variables to be compared" 25 | 26 | fig, axes = plt.subplots(nrows=len(variables), ncols=1) 27 | i = 0 28 | for var in variables: 29 | if len(variables) > 1: 30 | ax = axes[i] 31 | else: 32 | ax = axes 33 | var_meas = var + "_meas" 34 | ax.plot( 35 | measured.index / 3600.0, measured[var_meas], label="$" + var + "_{meas}$" 36 | ) 37 | ax.plot(simulated.index / 3600.0, simulated[var], label="$" + var + "$") 38 | ax.legend() 39 | ax.set_xlim(measured.index[0] / 3600) 40 | i += 1 41 | if len(variables) > 1: 42 | ax_last = axes[-1] 43 | else: 44 | ax_last = axes 45 | ax_last.set_xlabel("time [h]") 46 | 47 | if f: 48 | fig.savefig(f) 49 | 50 | return axes 51 | 52 | 53 | def plot_error_evo(errors, f=None): 54 | """Plots evolution of errors. 55 | 56 | :param errors: DataFrame 57 | :param f: string 58 | :return: axes 59 | """ 60 | fig, ax = plt.subplots() 61 | ax.plot(errors) 62 | ax.set_xlabel("Iteration") 63 | ax.set_ylabel("Error (NRMSE)") 64 | if f: 65 | fig = ax.get_figure() 66 | fig.savefig(f) 67 | return ax 68 | 69 | 70 | def plot_parameter_evo(parameters, file=None): 71 | """Plots parameter evolution. 72 | 73 | :param parameters: DataFrame 74 | :param file: string 75 | :return: axes 76 | """ 77 | par_evo = parameters.copy() 78 | # Get axes 79 | axes = par_evo.plot(subplots=True) 80 | fig = axes[0].get_figure() 81 | # Extend y lim 82 | axes = _extend_ylim(axes, par_evo) 83 | # x label 84 | axes[-1].set_xlabel("Iteration") 85 | 86 | if file: 87 | fig.savefig(file) 88 | return axes 89 | 90 | 91 | def plot_inputs(inputs, file=None): 92 | """Plots inputs. 93 | 94 | :param inputs: DataFrame 95 | :param file: string 96 | :return: axes 97 | """ 98 | axes = inputs.plot(subplots=True) 99 | fig = axes[0].get_figure() 100 | # x label 101 | axes[-1].set_xlabel("Time [s]") 102 | 103 | if file: 104 | fig.savefig(file) 105 | return axes 106 | 107 | 108 | def _extend_ylim(axes, df): 109 | # Extend y lim a bit and assign 3 y ticks in each subplot 110 | i = 0 111 | for p in list(df.columns): 112 | minimum = float(df[p].min()) 113 | maximum = float(df[p].max()) 114 | if maximum - minimum > 0.0001: 115 | # Varying values 116 | y_range = maximum - minimum 117 | ext = y_range * 0.2 118 | y_ext_range = y_range + 2 * ext 119 | axes[i].set_ylim([minimum - ext, maximum + ext]) 120 | axes[i].set_yticks( 121 | np.arange(minimum - ext, maximum + ext * 1.1, y_ext_range / 2) 122 | ) 123 | else: 124 | # Probably constant value 125 | # ...but just in case take the average 126 | avg = (maximum + minimum) / 2 127 | # set_ylim and set_yticks wouldn't work if average == 0 128 | if avg != 0.0: 129 | ext = avg * 0.2 130 | axes[i].set_ylim([minimum - ext, maximum + ext]) 131 | axes[i].set_yticks(np.arange(avg - ext, avg + ext * 1.1, ext)) 132 | i += 1 133 | return axes 134 | -------------------------------------------------------------------------------- /modestpy/estim/ps/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | 5 | This code is licensed under BSD 2-clause license. 6 | See LICENSE file in the project root for license terms. 7 | """ 8 | -------------------------------------------------------------------------------- /modestpy/estim/ps/ps.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import copy 8 | import logging 9 | import os 10 | from random import random 11 | 12 | import pandas as pd 13 | 14 | import modestpy.estim.plots as plots 15 | import modestpy.utilities.figures as figures 16 | from modestpy.estim.error import calc_err 17 | from modestpy.estim.estpar import EstPar 18 | from modestpy.estim.estpar import estpars_2_df 19 | from modestpy.fmi.model import Model 20 | 21 | 22 | class PS(object): 23 | """ 24 | Pattern search (Hooke-Jeeves) algorithm for FMU parameter estimation. 25 | """ 26 | 27 | # Ploting settings 28 | FIG_DPI = 150 29 | FIG_SIZE = (10, 6) 30 | 31 | NAME = "PS" 32 | METHOD = "_method_" 33 | ITER = "_iter_" 34 | ERR = "_error_" 35 | 36 | # Maximum allowed relative step 37 | STEP_CEILING = 1.00 38 | 39 | # Step is multiplied by this factor if solution improves 40 | STEP_INC = 1.0 41 | 42 | # Step is divided by this factor if solution does not improve 43 | STEP_DEC = 1.5 44 | 45 | def __init__( 46 | self, 47 | fmu_path, 48 | inp, 49 | known, 50 | est, 51 | ideal, 52 | rel_step=0.01, 53 | tol=0.0001, 54 | try_lim=30, 55 | maxiter=300, 56 | ftype="RMSE", 57 | ): 58 | """ 59 | :param fmu_path: string, absolute path to the FMU 60 | :param inp: DataFrame, columns with input timeseries, index in seconds 61 | :param known: Dictionary, key=parameter_name, value=value 62 | :param est: Dictionary, key=parameter_name, value=tuple 63 | (guess value, lo limit, hi limit), guess can be None 64 | :param ideal: DataFrame, ideal solution to be compared 65 | with model outputs (variable names must match) 66 | :param rel_step: float, initial relative step when modifying parameters 67 | :param tol: float, stopping criterion, when rel_step 68 | becomes smaller than tol algorithm stops 69 | :param try_lim: integer, maximum number of tries to decrease rel_step 70 | :param maxiter: integer, maximum number of iterations 71 | :param string ftype: Cost function type. Currently 'NRMSE' or 'RMSE' 72 | """ 73 | self.logger = logging.getLogger(type(self).__name__) 74 | 75 | assert inp.index.equals(ideal.index), "inp and ideal indexes are not matching" 76 | assert ( 77 | rel_step > tol 78 | ), "Relative step must not be smaller than the stop criterion" 79 | 80 | # Cost function type 81 | self.ftype = ftype 82 | 83 | # Ideal solution 84 | self.ideal = ideal 85 | 86 | # Adjust COM_POINTS 87 | # CVODE solver complains without "-1" 88 | PS.COM_POINTS = len(self.ideal) - 1 89 | 90 | # Inputs 91 | self.inputs = inp 92 | 93 | # Known parameters to DataFrame 94 | known_df = pd.DataFrame() 95 | for key in known: 96 | assert ( 97 | known[key] is not None 98 | ), "None is not allowed in known parameters " "(parameter {})".format(key) 99 | known_df[key] = [known[key]] 100 | 101 | # est: dictionary to a list with EstPar instances 102 | self.est = list() 103 | for key in est: 104 | lo = est[key][1] 105 | hi = est[key][2] 106 | if est[key][0] is None: # If guess is None, assume random guess 107 | v = lo + random() * (hi - lo) 108 | else: # Else, take the guess passed in est 109 | v = est[key][0] 110 | self.est.append(EstPar(name=key, value=v, lo=lo, hi=hi)) 111 | est = self.est 112 | 113 | # Model 114 | output_names = [var for var in ideal] 115 | self.model = PS._get_model_instance(fmu_path, inp, known_df, est, output_names) 116 | 117 | # Initial value for relative parameter step (0-1) 118 | self.rel_step = rel_step 119 | 120 | # Min. allowed relative parameter change (0-1) 121 | # PS stops when self.max_change < tol 122 | self.tol = tol 123 | 124 | # Max. number of iterations without moving to a new point 125 | self.try_lim = try_lim 126 | 127 | # Max. number of iterations in total 128 | self.max_iter = maxiter 129 | 130 | # Outputs 131 | self.summary = pd.DataFrame() 132 | self.res = pd.DataFrame() 133 | 134 | self.logger.info("Pattern Search initialized... =========================") 135 | 136 | def estimate(self): 137 | """ 138 | Proxy method. Each algorithm from ``estim`` package should 139 | have this method. 140 | 141 | :return: DataFrame 142 | """ 143 | return self._search() 144 | 145 | def get_error(self): 146 | """ 147 | :return: float, last error 148 | """ 149 | return float(self.summary[PS.ERR].iloc[-1]) 150 | 151 | def get_errors(self): 152 | """ 153 | :return: list, all errors from all iterations 154 | """ 155 | return self.summary[PS.ERR].tolist() 156 | 157 | def get_full_solution_trajectory(self): 158 | """ 159 | Returns all parameters and errors from all iterations. 160 | The returned DataFrame contains columns with parameter names, 161 | additional column '_error_' for the error and the index 162 | named '_iter_'. 163 | 164 | :return: DataFrame 165 | """ 166 | return self.summary 167 | 168 | def save_plots(self, workdir): 169 | self.plot_comparison(os.path.join(workdir, "ps_comparison.png")) 170 | self.plot_error_evo(os.path.join(workdir, "ps_error_evo.png")) 171 | self.plot_parameter_evo(os.path.join(workdir, "ps_param_evo.png")) 172 | 173 | def plot_comparison(self, file=None): 174 | return plots.plot_comparison(self.res, self.ideal, file) 175 | 176 | def plot_error_evo(self, file=None): 177 | err_df = pd.DataFrame(self.summary[PS.ERR]) 178 | return plots.plot_error_evo(err_df, file) 179 | 180 | def plot_parameter_evo(self, file=None): 181 | par_df = self.summary.drop([PS.METHOD], axis=1) 182 | par_df = par_df.rename( 183 | columns={x: "error" if x == PS.ERR else x for x in par_df.columns} 184 | ) 185 | 186 | # Get axes 187 | axes = par_df.plot(subplots=True) 188 | fig = figures.get_figure(axes) 189 | # x label 190 | axes[-1].set_xlabel("Iteration") 191 | # ylim for error 192 | axes[-1].set_ylim(0, None) 193 | 194 | if file: 195 | fig.set_size_inches(PS.FIG_SIZE) 196 | fig.savefig(file, dpi=PS.FIG_DPI) 197 | return axes 198 | 199 | def plot_inputs(self, file=None): 200 | return plots.plot_inputs(self.inputs, file) 201 | 202 | def _search(self): 203 | """ 204 | Pattern _search loop. 205 | 206 | :return: DataFrame with estimates 207 | """ 208 | initial_estimates = copy.deepcopy(self.est) 209 | best_estimates = copy.deepcopy(initial_estimates) 210 | current_estimates = copy.deepcopy(initial_estimates) 211 | 212 | initial_result = self.model.simulate() 213 | self.res = initial_result 214 | initial_error = calc_err(initial_result, self.ideal, ftype=self.ftype)["tot"] 215 | best_err = initial_error 216 | 217 | # First line of the summary 218 | summary = estpars_2_df(current_estimates) 219 | summary[PS.ERR] = [initial_error] 220 | 221 | # Counters 222 | n_try = 0 223 | iteration = 0 224 | 225 | # Search loop 226 | while ( 227 | (n_try < self.try_lim) 228 | and (iteration < self.max_iter) 229 | and (self.rel_step > self.tol) 230 | ): 231 | iteration += 1 232 | self.logger.info( 233 | "Iteration no. {} " "=========================".format(iteration) 234 | ) 235 | improved = False 236 | 237 | # Iterate over all parameters 238 | for par in current_estimates: 239 | for sign in ["+", "-"]: 240 | # Calculate new parameter 241 | new_par = self._get_new_estpar(par, self.rel_step, sign) 242 | 243 | # Simulate and calculate error 244 | self.model.set_param(estpars_2_df([new_par])) 245 | result = self.model.simulate() 246 | err = calc_err(result, self.ideal, ftype=self.ftype)["tot"] 247 | 248 | # Save point if solution improved 249 | if err < best_err: 250 | self.res = result 251 | best_err = err 252 | 253 | # Shortest path search 254 | # best_estimates = PS._replace_par(best_estimates, 255 | # new_par) 256 | 257 | # Orthogonal search 258 | best_estimates = PS._replace_par(current_estimates, new_par) 259 | 260 | improved = True 261 | 262 | # Reset model parameters 263 | self.model.set_param(estpars_2_df(current_estimates)) 264 | 265 | # Go to the new point 266 | current_estimates = copy.deepcopy(best_estimates) 267 | 268 | # Update summary 269 | current_estimates_df = estpars_2_df(current_estimates) 270 | current_estimates_df.index = [iteration] 271 | summary = pd.concat([summary, current_estimates_df]) 272 | summary[PS.ERR][iteration] = best_err 273 | 274 | if not improved: 275 | n_try += 1 276 | self.rel_step /= PS.STEP_DEC 277 | self.logger.info("Solution did not improve...") 278 | self.logger.debug("Step reduced to {}".format(self.rel_step)) 279 | self.logger.debug("Tries left: {}".format(self.try_lim - n_try)) 280 | else: 281 | # Solution improved, reset n_try counter 282 | n_try = 0 283 | self.rel_step *= PS.STEP_INC 284 | if self.rel_step > PS.STEP_CEILING: 285 | self.rel_step = PS.STEP_CEILING 286 | self.logger.info("Solution improved") 287 | self.logger.debug("Current step is {}".format(self.rel_step)) 288 | self.logger.info("New error: {}".format(best_err)) 289 | self.logger.debug( 290 | "New estimates:\n{}".format(estpars_2_df(current_estimates)) 291 | ) 292 | 293 | # Reorder columns in summary 294 | s_cols = summary.columns.tolist() 295 | s_cols.remove(PS.ERR) 296 | s_cols.append(PS.ERR) 297 | summary = summary[s_cols] 298 | 299 | # Start iterations from 1 300 | summary.index += 1 301 | 302 | # Rename index in summary 303 | summary.index = summary.index.rename(PS.ITER) 304 | 305 | # Add column with method name 306 | summary[PS.METHOD] = PS.NAME 307 | 308 | # Print summary 309 | reason = "Unknown" 310 | if n_try >= self.try_lim: 311 | reason = "Maximum number of tries to decrease the step reached" 312 | elif iteration >= self.max_iter: 313 | reason = "Maximum number of iterations reached" 314 | elif self.rel_step <= self.tol: 315 | reason = "Relative step smaller than the stoping criterion" 316 | 317 | self.logger.info("Pattern search finished. Reason: {}".format(reason)) 318 | self.logger.info("Summary:\n{}".format(summary)) 319 | 320 | # Final assignments 321 | self.summary = summary 322 | final_estimates = estpars_2_df(best_estimates) 323 | 324 | return final_estimates 325 | 326 | def get_plots(self): 327 | """ 328 | Returns a list with important plots produced by this estimation method. 329 | Each list element is a dictionary with keys 'name' and 'axes'. The name 330 | should be given as a string, while axes as matplotlib.Axes instance. 331 | 332 | :return: list(dict) 333 | """ 334 | plots = list() 335 | plots.append({"name": "PS", "axes": self.plot_parameter_evo()}) 336 | return plots 337 | 338 | def _get_new_estpar(self, estpar, rel_step, sign): 339 | """ 340 | Returns new ``EstPar`` object with modified value, 341 | according to ``sign`` and ``max_change``. 342 | 343 | :param estpar: EstPar 344 | :param rel_step: float, (0-1) 345 | :param sign: string, '+' or '-' 346 | :return: EstPar 347 | """ 348 | sign_mltp = None 349 | if sign == "+": 350 | sign_mltp = 1.0 351 | elif sign == "-": 352 | sign_mltp = -1.0 353 | else: 354 | print("Unrecognized sign ({})".format(sign)) 355 | 356 | new_value = estpar.value * (1 + rel_step * sign_mltp) 357 | 358 | if new_value > estpar.hi: 359 | new_value = estpar.hi 360 | if new_value < estpar.lo: 361 | new_value = estpar.lo 362 | 363 | return EstPar(estpar.name, estpar.lo, estpar.hi, new_value) 364 | 365 | @staticmethod 366 | def _get_model_instance(fmu_path, inputs, known_pars, est, output_names): 367 | model = Model(fmu_path) 368 | model.set_input(inputs) 369 | model.set_param(known_pars) 370 | model.set_param(estpars_2_df(est)) 371 | model.set_outputs(output_names) 372 | return model 373 | 374 | @staticmethod 375 | def _replace_par(estpar_list, estpar): 376 | """ 377 | Puts ``estpar`` in ``estpar_list``, replacing object 378 | with the same ``name``. 379 | 380 | :param estpar_list: list of EstPar objects 381 | :param estpar: EstPar 382 | :return: list of EstPar objects 383 | """ 384 | new_list = copy.deepcopy(estpar_list) 385 | for i in range(len(new_list)): 386 | if new_list[i].name == estpar.name: 387 | new_list[i] = copy.deepcopy(estpar) 388 | return new_list 389 | -------------------------------------------------------------------------------- /modestpy/estim/scipy/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | 5 | This code is licensed under BSD 2-clause license. 6 | See LICENSE file in the project root for license terms. 7 | """ 8 | -------------------------------------------------------------------------------- /modestpy/estim/scipy/scipy.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import logging 8 | import os 9 | from random import random 10 | 11 | import numpy as np 12 | import pandas as pd 13 | from scipy.optimize import minimize 14 | 15 | import modestpy.estim.plots as plots 16 | import modestpy.utilities.figures as figures 17 | from modestpy.estim.error import calc_err 18 | from modestpy.estim.estpar import EstPar 19 | from modestpy.estim.estpar import estpars_2_df 20 | from modestpy.fmi.model import Model 21 | 22 | 23 | class SCIPY(object): 24 | """ 25 | Interface to `scipy.optimize.minimize()`. 26 | """ 27 | 28 | # Summary placeholder 29 | TMP_SUMMARY = pd.DataFrame() 30 | 31 | # Ploting settings 32 | FIG_DPI = 150 33 | FIG_SIZE = (10, 6) 34 | 35 | NAME = "SCIPY" 36 | METHOD = "_method_" 37 | ITER = "_iter_" 38 | ERR = "_error_" 39 | 40 | def __init__( 41 | self, fmu_path, inp, known, est, ideal, solver, options={}, ftype="RMSE" 42 | ): 43 | """ 44 | :param fmu_path: string, absolute path to the FMU 45 | :param inp: DataFrame, columns with input timeseries, index in seconds 46 | :param known: Dictionary, key=parameter_name, value=value 47 | :param est: Dictionary, key=parameter_name, value=tuple 48 | (guess value, lo limit, hi limit), guess can be None 49 | :param ideal: DataFrame, ideal solution to be compared with model 50 | outputs (variable names must match) 51 | :param solver: str, solver type (e.g. 'TNC', 'L-BFGS-B', 'SLSQP') 52 | :param options: dict, additional options passed to the SciPy's solver 53 | :param ftype: str, cost function type. Currently 'NRMSE' (advised 54 | for multi-objective estimation) or 'RMSE'. 55 | """ 56 | self.logger = logging.getLogger(type(self).__name__) 57 | 58 | assert inp.index.equals(ideal.index), "inp and ideal indexes are not matching" 59 | 60 | # Solver type 61 | self.solver = solver 62 | 63 | # Default solver options 64 | self.options = {"disp": True, "iprint": 2, "maxiter": 500} 65 | 66 | if len(options) > 0: 67 | for key in options: 68 | self.options[key] = options[key] 69 | 70 | # Cost function type 71 | self.ftype = ftype 72 | 73 | # Ideal solution 74 | self.ideal = ideal 75 | 76 | # Adjust COM_POINTS 77 | # CVODE solver complains without "-1" 78 | SCIPY.COM_POINTS = len(self.ideal) - 1 79 | 80 | # Inputs 81 | self.inputs = inp 82 | 83 | # Known parameters to DataFrame 84 | known_df = pd.DataFrame() 85 | for key in known: 86 | assert ( 87 | known[key] is not None 88 | ), "None is not allowed in known parameters (parameter {})".format(key) 89 | known_df[key] = [known[key]] 90 | 91 | # est: dictionary to a list with EstPar instances 92 | self.est = list() 93 | for key in est: 94 | lo = est[key][1] 95 | hi = est[key][2] 96 | if est[key][0] is None: # If guess is None, assume random guess 97 | v = lo + random() * (hi - lo) 98 | else: # Else, take the guess passed in est 99 | v = est[key][0] 100 | self.est.append(EstPar(name=key, value=v, lo=lo, hi=hi)) 101 | est = self.est 102 | 103 | # Model 104 | output_names = [var for var in ideal] 105 | self.model = SCIPY._get_model_instance( 106 | fmu_path, inp, known_df, est, output_names 107 | ) 108 | 109 | # Outputs 110 | self.summary = pd.DataFrame() 111 | self.res = pd.DataFrame() 112 | self.best_err = 1e7 113 | 114 | # Temporary placeholder for summary 115 | # It needs to be stored as class variable, because it has to be updated 116 | # from a static method used as callback 117 | self.summary_cols = [x.name for x in self.est] + [SCIPY.ERR, SCIPY.METHOD] 118 | SCIPY.TMP_SUMMARY = pd.DataFrame(columns=self.summary_cols) 119 | 120 | # Log 121 | self.logger.info("SCIPY initialized... =========================") 122 | 123 | def estimate(self): 124 | 125 | # Initial error 126 | initial_result = self.model.simulate() 127 | self.res = initial_result 128 | initial_error = calc_err(initial_result, self.ideal, ftype=self.ftype)["tot"] 129 | self.best_err = initial_error 130 | 131 | def objective(x): 132 | """Returns model error""" 133 | # Updated parameters are stored in x. Need to update the model. 134 | self.logger.debug("objective(x={})".format(x)) 135 | 136 | parameters = pd.DataFrame(index=[0]) 137 | try: 138 | for v, ep in zip(x, self.est): 139 | parameters[ep.name] = SCIPY.rescale(v, ep.lo, ep.hi) 140 | except TypeError as e: 141 | print(x) 142 | raise e 143 | self.model.set_param(parameters) 144 | result = self.model.simulate() 145 | err = calc_err(result, self.ideal, ftype=self.ftype)["tot"] 146 | # Update best error and result 147 | if err < self.best_err: 148 | self.best_err = err 149 | self.res = result 150 | 151 | return err 152 | 153 | # Initial guess 154 | x0 = [SCIPY.scale(x.value, x.lo, x.hi) for x in self.est] 155 | self.logger.debug("SciPy x0 = {}".format(x0)) 156 | 157 | # Save initial guess in summary 158 | row = pd.DataFrame(index=[0]) 159 | for x, c in zip(x0, SCIPY.TMP_SUMMARY.columns): 160 | row[c] = x 161 | row[SCIPY.ERR] = np.nan 162 | row[SCIPY.METHOD] = SCIPY.NAME 163 | SCIPY.TMP_SUMMARY = SCIPY.TMP_SUMMARY.append(row, ignore_index=True) 164 | 165 | # Parameter bounds 166 | b = [(0.0, 1.0) for x in self.est] 167 | 168 | out = minimize( 169 | objective, 170 | x0, 171 | bounds=b, 172 | constraints=[], 173 | method=self.solver, 174 | callback=SCIPY._callback, 175 | options=self.options, 176 | ) 177 | 178 | outx = [ 179 | SCIPY.rescale(x, ep.lo, ep.hi) for x, ep in zip(out.x.tolist(), self.est) 180 | ] 181 | 182 | self.logger.debug("SciPy x = {}".format(outx)) 183 | 184 | # Update summary 185 | self.summary = SCIPY.TMP_SUMMARY.copy() 186 | self.summary.index += 1 # Adjust iteration counter 187 | self.summary.index.name = SCIPY.ITER # Rename index 188 | 189 | # Update error 190 | self.summary[SCIPY.ERR] = list( 191 | map(objective, self.summary[[x.name for x in self.est]].values) 192 | ) 193 | 194 | for ep in self.est: 195 | name = ep.name 196 | # list(map(...)) - for Python 2/3 compatibility 197 | self.summary[name] = list( 198 | map(lambda x: SCIPY.rescale(x, ep.lo, ep.hi), self.summary[name]) 199 | ) # Rescale 200 | 201 | # Add solver name to column `method` 202 | self.summary[SCIPY.METHOD] += "[" + self.solver + "]" 203 | 204 | # Reset temp placeholder 205 | SCIPY.TMP_SUMMARY = pd.DataFrame(columns=self.summary_cols) 206 | 207 | # Return DataFrame with estimates 208 | par_vec = outx 209 | par_df = pd.DataFrame(columns=[x.name for x in self.est], index=[0]) 210 | for col, x in zip(par_df.columns, par_vec): 211 | par_df[col] = x 212 | 213 | return par_df 214 | 215 | @staticmethod 216 | def scale(v, lo, hi): 217 | # scaled = (rescaled - lo) / (hi - lo) 218 | return (v - lo) / (hi - lo) 219 | 220 | @staticmethod 221 | def rescale(v, lo, hi): 222 | # rescaled = lo + scaled * (hi - lo) 223 | return lo + v * (hi - lo) 224 | 225 | def get_plots(self): 226 | """ 227 | Returns a list with important plots produced by this estimation method. 228 | Each list element is a dictionary with keys 'name' and 'axes'. The name 229 | should be given as a string, while axes as matplotlib.Axes instance. 230 | 231 | :return: list(dict) 232 | """ 233 | plots = list() 234 | plots.append( 235 | {"name": "SCIPY-{}".format(self.solver), "axes": self.plot_parameter_evo()} 236 | ) 237 | return plots 238 | 239 | def save_plots(self, workdir): 240 | self.plot_comparison(os.path.join(workdir, "ps_comparison.png")) 241 | self.plot_error_evo(os.path.join(workdir, "ps_error_evo.png")) 242 | self.plot_parameter_evo(os.path.join(workdir, "ps_param_evo.png")) 243 | 244 | def plot_comparison(self, file=None): 245 | return plots.plot_comparison(self.res, self.ideal, file) 246 | 247 | def plot_error_evo(self, file=None): 248 | err_df = pd.DataFrame(self.summary[SCIPY.ERR]) 249 | return plots.plot_error_evo(err_df, file) 250 | 251 | def plot_parameter_evo(self, file=None): 252 | par_df = self.summary.drop([SCIPY.METHOD], axis=1) 253 | par_df = par_df.rename( 254 | columns={x: "error" if x == SCIPY.ERR else x for x in par_df.columns} 255 | ) 256 | 257 | # Get axes 258 | axes = par_df.plot(subplots=True) 259 | fig = figures.get_figure(axes) 260 | # x label 261 | axes[-1].set_xlabel("Iteration") 262 | # ylim for error 263 | axes[-1].set_ylim(0, None) 264 | 265 | if file: 266 | fig.set_size_inches(SCIPY.FIG_SIZE) 267 | fig.savefig(file, dpi=SCIPY.FIG_DPI) 268 | return axes 269 | 270 | def get_full_solution_trajectory(self): 271 | """ 272 | Returns all parameters and errors from all iterations. 273 | The returned DataFrame contains columns with parameter names, 274 | additional column '_error_' for the error and the index 275 | named '_iter_'. 276 | 277 | :return: DataFrame 278 | """ 279 | return self.summary 280 | 281 | def get_error(self): 282 | """ 283 | :return: float, last error 284 | """ 285 | return float(self.summary[SCIPY.ERR].iloc[-1]) 286 | 287 | def get_errors(self): 288 | """ 289 | :return: list, all errors from all iterations 290 | """ 291 | return self.summary[SCIPY.ERR].tolist() 292 | 293 | # PRIVATE METHODS 294 | 295 | @staticmethod 296 | def _callback(xk): 297 | # New row 298 | row = pd.DataFrame(index=[0]) 299 | for x, c in zip(xk, SCIPY.TMP_SUMMARY.columns): 300 | row[c] = x 301 | 302 | row[SCIPY.ERR] = np.nan 303 | row[SCIPY.METHOD] = SCIPY.NAME 304 | 305 | # Append 306 | SCIPY.TMP_SUMMARY = SCIPY.TMP_SUMMARY.append(row, ignore_index=True) 307 | 308 | @staticmethod 309 | def _get_model_instance(fmu_path, inputs, known_pars, est, output_names): 310 | model = Model(fmu_path) 311 | model.set_input(inputs) 312 | model.set_param(known_pars) 313 | model.set_param(estpars_2_df(est)) 314 | model.set_outputs(output_names) 315 | return model 316 | -------------------------------------------------------------------------------- /modestpy/fmi/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | 5 | This code is licensed under BSD 2-clause license. 6 | See LICENSE file in the project root for license terms. 7 | """ 8 | -------------------------------------------------------------------------------- /modestpy/fmi/compiler.py: -------------------------------------------------------------------------------- 1 | """DEPRECATED. 2 | 3 | WARNING, THIS FILE RELIES ON PYMODELICA WHICH WILL NOT BE 4 | INCLUDED IN THE DEPENDENCIES IN THE NEXT RELEASE. 5 | 6 | Copyright (c) 2017, University of Southern Denmark 7 | All rights reserved. 8 | This code is licensed under BSD 2-clause license. 9 | See LICENSE file in the project root for license terms. 10 | """ 11 | 12 | from __future__ import absolute_import 13 | from __future__ import division 14 | from __future__ import print_function 15 | 16 | import os 17 | import shutil 18 | 19 | from modestpy.utilities.sysarch import get_sys_arch 20 | 21 | try: 22 | from pymodelica import compile_fmu 23 | except ImportError as e: 24 | raise ImportError("pymodelica is required to run this script!") 25 | 26 | 27 | def mo_2_fmu(model_name, mo_path, fmu_path=None): 28 | """ 29 | Compiles FMU 2.0 CS from a MO file (Modelica model). 30 | 31 | :param model_name: string, path to the model in the MO file 32 | :param mos_path: string, path to the input MO file 33 | :param fmu_path: string or None, path to the output FMU file; 34 | "CWD/model_name.fmu" if None 35 | :return: string, path to the resulting FMU 36 | """ 37 | # opts = {'nle_solver_min_tol': 1e-10} 38 | opts = {} 39 | 40 | compile_fmu(model_name, mo_path, target="cs", version="2.0", compiler_options=opts) 41 | 42 | std_fmu_path = os.path.join(os.getcwd(), model_name.replace(".", "_") + ".fmu") 43 | if fmu_path is not None: 44 | print("Moving FMU to: {}".format(fmu_path)) 45 | shutil.move(std_fmu_path, fmu_path) 46 | return fmu_path 47 | return std_fmu_path 48 | 49 | 50 | # Example 51 | if __name__ == "__main__": 52 | 53 | platform = get_sys_arch() 54 | 55 | # FMU from examples 56 | # mo_path = os.path.join('.', 'examples', 'simple', 'resources', 57 | # 'Simple2R1C.mo') 58 | # fmu_path = os.path.join('.', 'examples', 'simple', 'resources', 59 | # 'Simple2R1C_{}.fmu'.format(platform)) 60 | # model_name = "Simple2R1C" 61 | 62 | # FMU from tests 63 | mo_path = os.path.join(".", "tests", "resources", "simple2R1C", "Simple2R1C.mo") 64 | fmu_path = os.path.join( 65 | ".", "tests", "resources", "simple2R1C", "Simple2R1C_{}.fmu".format(platform) 66 | ) 67 | model_name = "Simple2R1C" 68 | 69 | # Compilation 70 | mo_2_fmu(model_name, mo_path, fmu_path) 71 | -------------------------------------------------------------------------------- /modestpy/fmi/fmpy_test.py: -------------------------------------------------------------------------------- 1 | """Example of FMPy-based simulation.""" 2 | import json 3 | 4 | import matplotlib.pyplot as plt 5 | import numpy as np 6 | import pandas as pd 7 | from fmpy import dump 8 | from fmpy import extract 9 | from fmpy import instantiate_fmu 10 | from fmpy import read_model_description 11 | from fmpy import simulate_fmu 12 | from fmpy.util import read_csv 13 | from fmpy.util import write_csv 14 | 15 | from modestpy.utilities.sysarch import get_sys_arch 16 | 17 | 18 | def df_to_struct_arr(df): 19 | """Converts a DataFrame to structured array.""" 20 | struct_arr = np.rec.fromrecords(df, names=df.columns.tolist()) 21 | 22 | return struct_arr 23 | 24 | 25 | def struct_arr_to_df(arr): 26 | """Converts a structured array to DataFrame.""" 27 | df = pd.DataFrame(arr).set_index("time") 28 | 29 | return df 30 | 31 | 32 | # Paths 33 | fmu_path = f"examples/simple/resources/Simple2R1C_ic_{get_sys_arch()}.fmu" 34 | input_path = "examples/simple/resources/inputs.csv" 35 | known_path = "examples/simple/resources/known.json" 36 | est_path = "examples/simple/resources/est.json" 37 | 38 | # Print some info about the FMU 39 | dump(fmu_path) 40 | 41 | # Instantiate FMU 42 | model_desc = read_model_description(fmu_path) 43 | unzipdir = extract(fmu_path) 44 | fmu = instantiate_fmu(unzipdir, model_desc) 45 | 46 | # Input 47 | inp_df = pd.read_csv(input_path) 48 | inp_struct = df_to_struct_arr(inp_df) 49 | 50 | # Parameters 51 | with open(known_path, "r") as f: 52 | start_values = json.load(f) 53 | 54 | # Declare output names 55 | # output = [] 56 | 57 | # Start and stop time 58 | start_time = inp_df["time"].iloc[0] 59 | stop_time = inp_df["time"].iloc[-1] 60 | output_interval = inp_df["time"].iloc[1] - inp_df["time"].iloc[0] 61 | 62 | # Reset the FMU instance instead of creating a new one 63 | fmu.reset() 64 | 65 | # Simulate 66 | result = simulate_fmu( 67 | filename=fmu_path, 68 | start_values=start_values, 69 | start_time=start_time, 70 | stop_time=stop_time, 71 | input=inp_struct, 72 | output=None, 73 | output_interval=output_interval, 74 | fmu_instance=fmu, 75 | ) 76 | 77 | # Free the FMU instance and free the shared library 78 | fmu.freeInstance() 79 | 80 | # Result to DataFrame 81 | result = struct_arr_to_df(result) 82 | print(result) 83 | plt.plot(result) 84 | plt.show() 85 | -------------------------------------------------------------------------------- /modestpy/fmi/fmpy_warning_example.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exemplary code triggering [WARNING] fmi2DoStep: currentCommunicationPoint = ..., expected ... 3 | """ 4 | import json 5 | 6 | import matplotlib.pyplot as plt 7 | import numpy as np 8 | import pandas as pd 9 | from fmpy import dump 10 | from fmpy import extract 11 | from fmpy import instantiate_fmu 12 | from fmpy import read_model_description 13 | from fmpy import simulate_fmu 14 | from fmpy.util import read_csv 15 | from fmpy.util import write_csv 16 | 17 | from modestpy.utilities.sysarch import get_sys_arch 18 | 19 | 20 | def df_to_struct_arr(df): 21 | """Converts a DataFrame to structured array.""" 22 | struct_arr = np.rec.fromrecords(df, names=df.columns.tolist()) 23 | 24 | return struct_arr 25 | 26 | 27 | def struct_arr_to_df(arr): 28 | """Converts a structured array to DataFrame.""" 29 | df = pd.DataFrame(arr).set_index("time") 30 | 31 | return df 32 | 33 | 34 | # Paths 35 | fmu_path = f"examples/simple/resources/Simple2R1C_ic_{get_sys_arch()}.fmu" 36 | input_path = "examples/simple/resources/inputs.csv" 37 | known_path = "examples/simple/resources/known.json" 38 | est_path = "examples/simple/resources/est.json" 39 | 40 | # Print some info about the FMU 41 | dump(fmu_path) 42 | 43 | # Instantiate FMU 44 | model_desc = read_model_description(fmu_path) 45 | unzipdir = extract(fmu_path) 46 | fmu = instantiate_fmu(unzipdir, model_desc) 47 | 48 | # Input 49 | inp_df = pd.read_csv(input_path) 50 | inp_struct = df_to_struct_arr(inp_df) 51 | 52 | # Parameters 53 | with open(known_path, "r") as f: 54 | start_values = json.load(f) 55 | 56 | # Declare output names 57 | # output = [] 58 | 59 | # Start and stop time 60 | start_time = 360 # THIS TRIGGERS THE WARNINGS 61 | stop_time = inp_df["time"].iloc[-1] 62 | output_interval = inp_df["time"].iloc[1] - inp_df["time"].iloc[0] 63 | 64 | # Reset the FMU instance instead of creating a new one 65 | fmu.reset() 66 | 67 | # Simulate 68 | result = simulate_fmu( 69 | filename=fmu_path, 70 | start_values=start_values, 71 | start_time=start_time, 72 | stop_time=stop_time, 73 | input=inp_struct, 74 | output=None, 75 | output_interval=output_interval, 76 | fmu_instance=fmu, 77 | ) 78 | 79 | # Free the FMU instance and free the shared library 80 | fmu.freeInstance() 81 | 82 | # Result to DataFrame 83 | result = struct_arr_to_df(result) 84 | print(result) 85 | plt.plot(result) 86 | plt.show() 87 | -------------------------------------------------------------------------------- /modestpy/fmi/model.py: -------------------------------------------------------------------------------- 1 | """FMI wrapper for the model. 2 | 3 | Copyright (c) 2017, University of Southern Denmark 4 | All rights reserved. 5 | This code is licensed under BSD 2-clause license.L 6 | See LICENSE file in the project root for license terms. 7 | """ 8 | import logging 9 | from pathlib import Path 10 | 11 | import numpy as np 12 | import pandas as pd 13 | from fmpy import extract 14 | from fmpy import instantiate_fmu 15 | from fmpy import read_model_description 16 | from fmpy import simulate_fmu 17 | 18 | 19 | def df_to_struct_arr(df): 20 | """Converts a DataFrame to structured array.""" 21 | # time index must be reset to pass it to the struct_arr 22 | df = df.reset_index() 23 | struct_arr = np.rec.fromrecords(df, names=df.columns.tolist()) 24 | 25 | return struct_arr 26 | 27 | 28 | def struct_arr_to_df(arr): 29 | """Converts a structured array to DataFrame.""" 30 | df = pd.DataFrame(arr).set_index("time") 31 | 32 | return df 33 | 34 | 35 | class Model(object): 36 | """FMU model to be simulated with inputs and parameters provided from 37 | files or dataframes. 38 | """ 39 | 40 | tmpdir_file = "tmp_dirs.txt" 41 | 42 | def __init__(self, fmu_path, opts=None): 43 | self.logger = logging.getLogger(type(self).__name__) 44 | 45 | try: 46 | self.logger.debug("Loading FMU") 47 | # Load FMU 48 | model_desc = read_model_description(fmu_path) 49 | self.unzipdir = extract(fmu_path) 50 | self._save_tmpdir(self.unzipdir) 51 | self.model = instantiate_fmu(self.unzipdir, model_desc) 52 | 53 | except Exception as e: 54 | self.logger.error(type(e).__name__) 55 | self.logger.error(str(e)) 56 | raise e 57 | 58 | self.start = None 59 | self.end = None 60 | self.timeline = None 61 | self.opts = opts # TODO: Not used at the moment 62 | 63 | self.input_arr = None 64 | self.output_names = list() 65 | self.parameters = dict() 66 | 67 | def _save_tmpdir(self, d): 68 | if not Path(Model.tmpdir_file).exists(): 69 | open(str(Model.tmpdir_file), "w").close() 70 | 71 | with open(Model.tmpdir_file, "a") as f: 72 | f.write(str(d) + "\n") 73 | 74 | def inputs_from_csv(self, csv, sep=",", exclude=list()): 75 | """Reads inputs from a CSV file. 76 | 77 | Time (column `time`) should be given in seconds. 78 | 79 | :param csv: Path to the CSV file 80 | :param exclude: list of strings, columns to be excluded 81 | :return: None 82 | """ 83 | df = pd.read_csv(csv, sep=sep) 84 | assert "time" in df.columns, "'time' not present in csv..." 85 | df = df.set_index("time") 86 | self.inputs_from_df(df, exclude) 87 | 88 | def set_input(self, df, exclude=list()): 89 | return self.inputs_from_df(df, exclude) 90 | 91 | def inputs_from_df(self, df, exclude=list()): 92 | """Reads inputs from dataframe. 93 | 94 | Index must be named 'time' and given in seconds. 95 | The index name assertion check is implemented to avoid 96 | situations in which a user read DataFrame from csv 97 | and forgot to use ``DataFrame.set_index(column_name)`` 98 | (it happens quite often...). 99 | 100 | :param df: DataFrame 101 | :param exclude: list of strings, names of columns to be omitted 102 | :return: 103 | """ 104 | assert df.index.name == "time", ( 105 | "Index name ('{}') different " 106 | "than 'time'! " 107 | "Are you sure you assigned index " 108 | "correctly?".format(df.index.name) 109 | ) 110 | if len(exclude) > 0: 111 | df = df.drop(exclude, axis="columns") 112 | self.timeline = df.index.values 113 | self.start = self.timeline[0] 114 | self.end = self.timeline[-1] 115 | self.input_arr = df_to_struct_arr(df) 116 | 117 | def set_outputs(self, outputs): 118 | self.specify_outputs(outputs) 119 | 120 | def specify_outputs(self, outputs): 121 | """Specifies names of output variables. 122 | 123 | :param outputs: List of strings, names of output variables 124 | :return: None 125 | """ 126 | for name in outputs: 127 | if name not in self.output_names: 128 | self.output_names.append(name) 129 | 130 | def parameters_from_csv(self, csv, sep=","): 131 | """Read parameters from a CSV file.""" 132 | df = pd.read_csv(csv, sep=sep) 133 | self.parameters_from_df(df) 134 | 135 | def set_param(self, df): 136 | self.parameters_from_df(df) 137 | 138 | def parameters_from_df(self, df): 139 | """Get parameters from a DataFrame.""" 140 | self.logger.debug(f"parameters_from_df = {df}") 141 | if df is not None: 142 | df = df.copy() 143 | for col in df: 144 | self.parameters[col] = df[col] 145 | self.logger.debug(f"Updated parameters: {self.parameters}") 146 | 147 | def simulate(self, reset=True): 148 | """Performs a simulation. 149 | 150 | :param bool reset: if True, the model will be resetted after 151 | simulation (use False with E+ FMU) 152 | :return: Dataframe with results 153 | """ 154 | # Calculate output interval (in seconds) 155 | assert self.input_arr is not None, "No inputs assigned" 156 | output_interval = self.input_arr[1][0] - self.input_arr[0][0] 157 | 158 | # Initial condition 159 | start_values = dict() 160 | input_names = self.input_arr.dtype.names 161 | for name in input_names: 162 | if name != "time": 163 | start_values[name] = self.input_arr[name][0] 164 | 165 | assert "time" in input_names, "time must be the first input" 166 | 167 | # Set parameters 168 | for name, value in self.parameters.items(): 169 | if name != "time": 170 | start_values[name] = value 171 | 172 | # Inputs 173 | assert self.input_arr is not None, "Inputs not assigned!" 174 | 175 | # Options (fixed) 176 | pass # TODO 177 | 178 | # Options (user) 179 | pass # TODO 180 | 181 | # Simulation 182 | self.logger.debug("Starting simulation") 183 | result = simulate_fmu( 184 | self.unzipdir, 185 | start_values=start_values, 186 | start_time=self.start, 187 | stop_time=self.end, 188 | input=self.input_arr, 189 | output=self.output_names, 190 | output_interval=output_interval, 191 | fmu_instance=self.model 192 | # solver='Euler', # TODO: It might be useful to add solver/step to options 193 | # step_size=0.005 194 | ) 195 | 196 | # Convert result to DataFrame 197 | res_df = struct_arr_to_df(result) 198 | 199 | # Reset model 200 | if reset: 201 | try: 202 | self.model.reset() 203 | except Exception as e: 204 | self.logger.warning(str(e)) 205 | self.logger.warning( 206 | "If you try to simulate an EnergyPlus FMU, " "use reset=False" 207 | ) 208 | # Return 209 | return res_df 210 | -------------------------------------------------------------------------------- /modestpy/fmi/model_pyfmi.py: -------------------------------------------------------------------------------- 1 | """DEPRECATED. PyFMI-based model. Superseded by FMPy-based class. 2 | 3 | Copyright (c) 2017, University of Southern Denmark 4 | All rights reserved. 5 | This code is licensed under BSD 2-clause license. 6 | See LICENSE file in the project root for license terms. 7 | """ 8 | import logging 9 | 10 | import numpy as np 11 | import pandas as pd 12 | from pyfmi import load_fmu 13 | from pyfmi.fmi import FMUException 14 | 15 | 16 | class Model(object): 17 | """ 18 | FMU model to be simulated with inputs and parameters provided from 19 | files or dataframes. 20 | """ 21 | 22 | # Number of tries to simulate a model 23 | # (sometimes the solver can't converge at first) 24 | TRIES = 15 25 | 26 | def __init__(self, fmu_path, opts=None): 27 | self.logger = logging.getLogger(type(self).__name__) 28 | 29 | try: 30 | self.logger.debug("Loading FMU") 31 | self.model = load_fmu(fmu_path) 32 | except FMUException as e: 33 | self.logger.error(type(e).__name__) 34 | self.logger.error(e.message) 35 | 36 | self.start = None 37 | self.end = None 38 | self.timeline = None 39 | self.opts = opts 40 | 41 | self.input_names = list() 42 | self.input_values = list() 43 | self.output_names = list() 44 | self.parameter_names = list() 45 | # self.full_io = list() 46 | 47 | self.parameter_df = pd.DataFrame() 48 | 49 | self.res = None 50 | 51 | def inputs_from_csv(self, csv, sep=",", exclude=list()): 52 | """ 53 | Reads inputs from a CSV file (format of the standard input file 54 | in ModelManager). It is assumed that time is given in seconds. 55 | :param csv: Path to the CSV file 56 | :param exclude: list of strings, columns to be excluded 57 | :return: None 58 | """ 59 | df = pd.read_csv(csv, sep=sep) 60 | assert "time" in df.columns, "'time' not present in csv..." 61 | df = df.set_index("time") 62 | self.inputs_from_df(df, exclude) 63 | 64 | def inputs_from_df(self, df, exclude=list()): 65 | """ 66 | Reads inputs from dataframe. 67 | 68 | Index must be named 'time' and given in seconds. 69 | The index name assertion check is implemented to avoid 70 | situations in which a user read DataFrame from csv 71 | and forgot to use ``DataFrame.set_index(column_name)`` 72 | (it happens quite often...). 73 | 74 | :param df: DataFrame 75 | :param exclude: list of strings, names of columns to be omitted 76 | :return: 77 | """ 78 | assert df.index.name == "time", ( 79 | "Index name ('{}') different " 80 | "than 'time'! " 81 | "Are you sure you assigned index " 82 | "correctly?".format(df.index.name) 83 | ) 84 | self.timeline = df.index.values 85 | self.start = self.timeline[0] 86 | self.end = self.timeline[-1] 87 | 88 | for col in df: 89 | if col not in exclude: 90 | if col not in self.input_names: 91 | self.input_names.append(col) 92 | self.input_values.append(df[col].values) 93 | 94 | def specify_outputs(self, outputs): 95 | """ 96 | Specifies names of output variables 97 | :param outputs: List of strings, names of output variables 98 | :return: None 99 | """ 100 | for name in outputs: 101 | if name not in self.output_names: 102 | self.output_names.append(name) 103 | 104 | def parameters_from_csv(self, csv, sep=","): 105 | df = pd.read_csv(csv, sep=sep) 106 | self.parameters_from_df(df) 107 | 108 | def parameters_from_df(self, df): 109 | self.logger.debug(f"parameters_from_df = {df}") 110 | if df is not None: 111 | df = df.copy() 112 | for col in df: 113 | self.parameter_df[col] = df[col] 114 | self.logger.debug(f"Updated parameters: {self.parameter_df}") 115 | 116 | def simulate(self, com_points=None, reset=True): 117 | """ 118 | Performs a simulation. 119 | :param int com_points: number of communication points, if None, 120 | standard value is used (500) 121 | :param bool reset: if True, the model will be resetted after 122 | simulation (use False with E+ FMU) 123 | :param dict opts: Additional FMI options to be passed to the simulator 124 | (consult FMI specification) 125 | :return: Dataframe with results 126 | """ 127 | 128 | if com_points is None: 129 | self.logger.warning( 130 | "[fmi\\model] Warning! Default number " 131 | "of communication points assumed (500)" 132 | ) 133 | com_points = 500 134 | 135 | # IC 136 | self._set_ic() 137 | 138 | # Set parameters 139 | if not self.parameter_df.empty: 140 | self._set_all_parameters() 141 | 142 | # Inputs 143 | i = list() 144 | i.append(self.timeline) 145 | i.extend(self.input_values) 146 | i = Model._merge_inputs(i) 147 | input_obj = [self.input_names, i] 148 | 149 | # Options (fixed) 150 | fmi_opts = self.model.simulate_options() 151 | 152 | # Prevents saving results to a file 153 | fmi_opts["result_handling"] = "memory" 154 | fmi_opts["result_handler"] = "ResultHandlerMemory" 155 | 156 | # No output <- works only with CVode solver 157 | # fmi_opts['CVode_options']['verbosity'] = 50 158 | 159 | # Options (provided by the user) 160 | fmi_opts["ncp"] = com_points 161 | 162 | if (self.opts is not None) and (type(self.opts) is dict): 163 | self.logger.debug("User-defined FMI options found: {}".format(self.opts)) 164 | for k in self.opts: 165 | if type(self.opts[k]) is not dict: 166 | fmi_opts[k] = self.opts[k] 167 | self.logger.debug( 168 | "Setting FMI option: [{}] = {}".format(k, self.opts[k]) 169 | ) 170 | elif type(self.opts[k]) is dict: 171 | for subkey in self.opts[k]: 172 | # It works only for single nested sub-dictionaries 173 | fmi_opts[k][subkey] = self.opts[k][subkey] 174 | self.logger.debug( 175 | "Setting FMI option: [{}][{}] = {}".format( 176 | k, subkey, self.opts[k][subkey] 177 | ) 178 | ) 179 | else: 180 | raise TypeError("Wrong type of values in 'opts' dictionary") 181 | 182 | # Save all options to log 183 | self.logger.debug("All FMI options: {}".format(fmi_opts)) 184 | 185 | # Simulation 186 | tries = 0 187 | while tries < Model.TRIES: 188 | try: 189 | assert (self.start is not None) and ( 190 | self.end is not None 191 | ), "start and stop cannot be None" # Shouldn't it be OR? 192 | self.logger.debug("Starting simulation") 193 | self.res = self.model.simulate( 194 | start_time=self.start, 195 | final_time=self.end, 196 | input=input_obj, 197 | options=fmi_opts, 198 | ) 199 | break 200 | except FMUException as e: 201 | tries += 1 202 | self.logger.warning("Simulation failed, failure no. {}".format(tries)) 203 | self.logger.warning(type(e).__name__) 204 | self.logger.warning(str(e)) 205 | if tries >= Model.TRIES: 206 | self.logger.error( 207 | "Maximum number of failures " 208 | "reached ({}). " 209 | "Won't try again...".format(Model.TRIES) 210 | ) 211 | raise e 212 | 213 | # Convert result to dataframe 214 | df = pd.DataFrame() 215 | df["time"] = self.res["time"] 216 | df = df.set_index("time") 217 | for var in self.output_names: 218 | df[var] = self.res[var] 219 | 220 | # Reset model 221 | if reset: 222 | try: 223 | self.reset() 224 | except FMUException as e: 225 | self.logger.warning(e.message) 226 | self.logger.warning( 227 | "If you try to simulate an EnergyPlus FMU, " "use reset=False" 228 | ) 229 | 230 | # Return 231 | return df 232 | 233 | def reset(self): 234 | """ 235 | Resets model. After resetting inputs, parameters and outputs 236 | must be set again! 237 | :return: None 238 | """ 239 | self.model.reset() 240 | 241 | def _set_ic(self): 242 | """Sets initial condition (ic).""" 243 | ic = dict() 244 | for i in range(len(self.input_names)): 245 | ic[self.input_names[i]] = self.input_values[i][0] 246 | # Call PyFMI method 247 | for var in ic: 248 | self.model.set(var, ic[var]) 249 | 250 | def _set_parameter(self, name, value): 251 | if name not in self.parameter_names: 252 | self.parameter_names.append(name) 253 | self.model.set(name, value) 254 | 255 | def _set_all_parameters(self): 256 | for var in self.parameter_df: 257 | self._set_parameter(var, self.parameter_df[var]) 258 | 259 | @staticmethod 260 | def _merge_inputs(inputs): 261 | return np.transpose(np.vstack(inputs)) 262 | 263 | @staticmethod 264 | def _create_timeline(end, intervals): 265 | t = np.linspace(0, end, intervals + 1) 266 | return t 267 | -------------------------------------------------------------------------------- /modestpy/loginit.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | 8 | 9 | def config_logger(filename="modestpy.log", level="DEBUG"): 10 | """ 11 | Configure logger using logging.basicConfig. Use only if you don't 12 | have your own logger in your application. 13 | :param str filename: Log file name 14 | :param str level: Logging level ('DEBUG', 'WARNING', 'ERROR', 'INFO') 15 | """ 16 | import logging 17 | 18 | logging.basicConfig( 19 | filename=filename, 20 | filemode="w", 21 | level=level, 22 | format="[%(asctime)s][%(name)s][%(levelname)s] " "%(message)s", 23 | ) 24 | -------------------------------------------------------------------------------- /modestpy/test/README.md: -------------------------------------------------------------------------------- 1 | # Tests 2 | 3 | To run all tests simply run the following command from the root directory (Replace ``/`` with backslash ``\`` on Windows): 4 | 5 | ``` 6 | python modestpy/test/run.py 7 | ``` 8 | 9 | Tests can be run also directly from the interpreter: 10 | 11 | ```python 12 | >>> from modestpy.test import run 13 | >>> run.tests() 14 | ``` 15 | -------------------------------------------------------------------------------- /modestpy/test/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | 5 | This code is licensed under BSD 2-clause license. 6 | See LICENSE file in the project root for license terms. 7 | """ 8 | -------------------------------------------------------------------------------- /modestpy/test/resources/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/modestpy/test/resources/__init__.py -------------------------------------------------------------------------------- /modestpy/test/resources/simple2R1C/Simple2R1C.mo: -------------------------------------------------------------------------------- 1 | within ; 2 | model Simple2R1C 3 | parameter Modelica.SIunits.ThermalResistance R1=1 4 | "Constant thermal resistance of material"; 5 | parameter Modelica.SIunits.ThermalResistance R2=1 6 | "Constant thermal resistance of material"; 7 | parameter Modelica.SIunits.HeatCapacity C=1000 8 | "Heat capacity of element (= cp*m)"; 9 | 10 | 11 | Modelica.Thermal.HeatTransfer.Components.HeatCapacitor heatCapacitor(C=C, T(fixed= 12 | true, start=293.15)) 13 | annotation (Placement(transformation(extent={{2,0},{22,20}}))); 14 | Modelica.Thermal.HeatTransfer.Components.ThermalResistor thermalResistor2(R= 15 | R2) annotation (Placement(transformation(extent={{-40,-10},{-20,10}}))); 16 | Modelica.Blocks.Interfaces.RealInput Ti2 17 | annotation (Placement(transformation(extent={{-130,-20},{-90,20}}))); 18 | Modelica.Blocks.Interfaces.RealOutput T 19 | annotation (Placement(transformation(extent={{96,-10},{116,10}}))); 20 | Modelica.Thermal.HeatTransfer.Components.ThermalResistor thermalResistor1(R= 21 | R1) annotation (Placement(transformation(extent={{-42,36},{-22,56}}))); 22 | Modelica.Blocks.Interfaces.RealInput Ti1 23 | annotation (Placement(transformation(extent={{-130,26},{-90,66}}))); 24 | Modelica.Thermal.HeatTransfer.Sensors.TemperatureSensor temperatureSensor 25 | annotation (Placement(transformation(extent={{42,-10},{62,10}}))); 26 | Modelica.Thermal.HeatTransfer.Sources.PrescribedTemperature 27 | prescribedTemperature1 28 | annotation (Placement(transformation(extent={{-78,36},{-58,56}}))); 29 | Modelica.Thermal.HeatTransfer.Sources.PrescribedTemperature 30 | prescribedTemperature2 31 | annotation (Placement(transformation(extent={{-76,-10},{-56,10}}))); 32 | equation 33 | connect(heatCapacitor.port, thermalResistor2.port_b) 34 | annotation (Line(points={{12,0},{-20,0}}, color={191,0,0})); 35 | connect(thermalResistor1.port_b, heatCapacitor.port) annotation (Line( 36 | points={{-22,46},{-8,46},{-8,0},{12,0}}, color={191,0,0})); 37 | connect(heatCapacitor.port, temperatureSensor.port) 38 | annotation (Line(points={{12,0},{12,0},{42,0}}, color={191,0,0})); 39 | connect(T, temperatureSensor.T) 40 | annotation (Line(points={{106,0},{62,0}}, color={0,0,127})); 41 | connect(Ti1, prescribedTemperature1.T) 42 | annotation (Line(points={{-110,46},{-96,46},{-80,46}}, color={0,0,127})); 43 | connect(thermalResistor1.port_a, prescribedTemperature1.port) 44 | annotation (Line(points={{-42,46},{-50,46},{-58,46}}, color={191,0,0})); 45 | connect(Ti2, prescribedTemperature2.T) 46 | annotation (Line(points={{-110,0},{-94,0},{-78,0}}, color={0,0,127})); 47 | connect(thermalResistor2.port_a, prescribedTemperature2.port) 48 | annotation (Line(points={{-40,0},{-48,0},{-56,0}}, color={191,0,0})); 49 | annotation (Icon(coordinateSystem(preserveAspectRatio=false)), Diagram( 50 | coordinateSystem(preserveAspectRatio=false)), 51 | uses(Modelica(version="3.2.2"))); 52 | end Simple2R1C; 53 | -------------------------------------------------------------------------------- /modestpy/test/resources/simple2R1C/Simple2R1C_linux64.fmu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/modestpy/test/resources/simple2R1C/Simple2R1C_linux64.fmu -------------------------------------------------------------------------------- /modestpy/test/resources/simple2R1C/Simple2R1C_win32.fmu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/modestpy/test/resources/simple2R1C/Simple2R1C_win32.fmu -------------------------------------------------------------------------------- /modestpy/test/resources/simple2R1C/Simple2R1C_win64.fmu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/modestpy/test/resources/simple2R1C/Simple2R1C_win64.fmu -------------------------------------------------------------------------------- /modestpy/test/resources/simple2R1C/est.json: -------------------------------------------------------------------------------- 1 | { 2 | "R1": [ 3 | 0.08, 4 | 0.001, 5 | 0.5 6 | ], 7 | "R2": [ 8 | 0.08, 9 | 0.001, 10 | 0.5 11 | ], 12 | "C": [ 13 | 1000.0, 14 | 500.0, 15 | 10000.0 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /modestpy/test/resources/simple2R1C/known.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | } 4 | -------------------------------------------------------------------------------- /modestpy/test/resources/simple2R1C/parameters.csv: -------------------------------------------------------------------------------- 1 | R1,R2,C 2 | 0.1,0.25,2000 3 | -------------------------------------------------------------------------------- /modestpy/test/resources/simple2R1C_ic/Simple2R1C_ic.mo: -------------------------------------------------------------------------------- 1 | within ; 2 | model Simple2R1C 3 | parameter Modelica.SIunits.ThermalResistance R1=1 4 | "Constant thermal resistance of material"; 5 | parameter Modelica.SIunits.ThermalResistance R2=1 6 | "Constant thermal resistance of material"; 7 | parameter Modelica.SIunits.HeatCapacity C=1000 8 | "Heat capacity of element (= cp*m)"; 9 | parameter Modelica.SIunits.Temperature Tstart=20 10 | "Initial temperature"; 11 | 12 | 13 | Modelica.Thermal.HeatTransfer.Components.HeatCapacitor heatCapacitor(C=C, T(fixed= 14 | true, start=Tstart)) 15 | annotation (Placement(transformation(extent={{2,0},{22,20}}))); 16 | Modelica.Thermal.HeatTransfer.Components.ThermalResistor thermalResistor2(R= 17 | R2) annotation (Placement(transformation(extent={{-40,-10},{-20,10}}))); 18 | Modelica.Blocks.Interfaces.RealInput Ti2 19 | annotation (Placement(transformation(extent={{-130,-20},{-90,20}}))); 20 | Modelica.Blocks.Interfaces.RealOutput T 21 | annotation (Placement(transformation(extent={{96,-10},{116,10}}))); 22 | Modelica.Thermal.HeatTransfer.Components.ThermalResistor thermalResistor1(R= 23 | R1) annotation (Placement(transformation(extent={{-42,36},{-22,56}}))); 24 | Modelica.Blocks.Interfaces.RealInput Ti1 25 | annotation (Placement(transformation(extent={{-130,26},{-90,66}}))); 26 | Modelica.Thermal.HeatTransfer.Sensors.TemperatureSensor temperatureSensor 27 | annotation (Placement(transformation(extent={{42,-10},{62,10}}))); 28 | Modelica.Thermal.HeatTransfer.Sources.PrescribedTemperature 29 | prescribedTemperature1 30 | annotation (Placement(transformation(extent={{-78,36},{-58,56}}))); 31 | Modelica.Thermal.HeatTransfer.Sources.PrescribedTemperature 32 | prescribedTemperature2 33 | annotation (Placement(transformation(extent={{-76,-10},{-56,10}}))); 34 | equation 35 | connect(heatCapacitor.port, thermalResistor2.port_b) 36 | annotation (Line(points={{12,0},{-20,0}}, color={191,0,0})); 37 | connect(thermalResistor1.port_b, heatCapacitor.port) annotation (Line( 38 | points={{-22,46},{-8,46},{-8,0},{12,0}}, color={191,0,0})); 39 | connect(heatCapacitor.port, temperatureSensor.port) 40 | annotation (Line(points={{12,0},{12,0},{42,0}}, color={191,0,0})); 41 | connect(T, temperatureSensor.T) 42 | annotation (Line(points={{106,0},{62,0}}, color={0,0,127})); 43 | connect(Ti1, prescribedTemperature1.T) 44 | annotation (Line(points={{-110,46},{-96,46},{-80,46}}, color={0,0,127})); 45 | connect(thermalResistor1.port_a, prescribedTemperature1.port) 46 | annotation (Line(points={{-42,46},{-50,46},{-58,46}}, color={191,0,0})); 47 | connect(Ti2, prescribedTemperature2.T) 48 | annotation (Line(points={{-110,0},{-94,0},{-78,0}}, color={0,0,127})); 49 | connect(thermalResistor2.port_a, prescribedTemperature2.port) 50 | annotation (Line(points={{-40,0},{-48,0},{-56,0}}, color={191,0,0})); 51 | annotation (Icon(coordinateSystem(preserveAspectRatio=false)), Diagram( 52 | coordinateSystem(preserveAspectRatio=false)), 53 | uses(Modelica(version="3.2.2"))); 54 | end Simple2R1C; 55 | -------------------------------------------------------------------------------- /modestpy/test/resources/simple2R1C_ic/Simple2R1C_ic_linux64.fmu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/modestpy/test/resources/simple2R1C_ic/Simple2R1C_ic_linux64.fmu -------------------------------------------------------------------------------- /modestpy/test/resources/simple2R1C_ic/Simple2R1C_ic_win32.fmu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/modestpy/test/resources/simple2R1C_ic/Simple2R1C_ic_win32.fmu -------------------------------------------------------------------------------- /modestpy/test/resources/simple2R1C_ic/Simple2R1C_ic_win64.fmu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdu-cfei/modest-py/b374431672d57e85824b839d3f444ea53d004388/modestpy/test/resources/simple2R1C_ic/Simple2R1C_ic_win64.fmu -------------------------------------------------------------------------------- /modestpy/test/resources/simple2R1C_ic/est.json: -------------------------------------------------------------------------------- 1 | { 2 | "R1": [0.08, 0.001, 0.5], 3 | "R2": [0.08, 0.001, 0.5], 4 | "C": [1000.0, 100.0, 10000.0] 5 | } 6 | -------------------------------------------------------------------------------- /modestpy/test/resources/simple2R1C_ic/known.json: -------------------------------------------------------------------------------- 1 | { 2 | "Tstart": 293.15 3 | } -------------------------------------------------------------------------------- /modestpy/test/resources/simple2R1C_ic/true_parameters.csv: -------------------------------------------------------------------------------- 1 | R1,R2,C 2 | 0.1,0.25,2000 3 | -------------------------------------------------------------------------------- /modestpy/test/run.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import unittest 8 | 9 | from modestpy.loginit import config_logger 10 | from modestpy.test import test_estimation 11 | from modestpy.test import test_fmpy 12 | from modestpy.test import test_ga 13 | from modestpy.test import test_ps 14 | from modestpy.test import test_scipy 15 | from modestpy.test import test_utilities 16 | 17 | 18 | def all_suites(): 19 | 20 | suites = [ 21 | test_fmpy.suite(), 22 | # test_ga.suite(), 23 | test_ps.suite(), 24 | test_scipy.suite(), 25 | test_estimation.suite(), 26 | test_utilities.suite(), 27 | ] 28 | 29 | all_suites = unittest.TestSuite(suites) 30 | return all_suites 31 | 32 | 33 | def tests(): 34 | runner = unittest.TextTestRunner() 35 | test_suite = all_suites() 36 | runner.run(test_suite) 37 | 38 | 39 | if __name__ == "__main__": 40 | config_logger(filename="unit_tests.log", level="DEBUG") 41 | tests() 42 | -------------------------------------------------------------------------------- /modestpy/test/test_estimation.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import json 8 | import os 9 | import shutil 10 | import tempfile 11 | import unittest 12 | 13 | import pandas as pd 14 | 15 | from modestpy import Estimation 16 | from modestpy.utilities.sysarch import get_sys_arch 17 | 18 | 19 | class TestEstimation(unittest.TestCase): 20 | def setUp(self): 21 | 22 | # Platform (win32, win64, linux32, linux 64) 23 | platform = get_sys_arch() 24 | assert platform, "Unsupported platform type!" 25 | 26 | # Temp directory 27 | self.tmpdir = tempfile.mkdtemp() 28 | 29 | # Parent directory 30 | parent = os.path.dirname(__file__) 31 | 32 | # Resources 33 | self.fmu_path = os.path.join( 34 | parent, 35 | "resources", 36 | "simple2R1C_ic", 37 | "Simple2R1C_ic_{}.fmu".format(platform), 38 | ) 39 | inp_path = os.path.join(parent, "resources", "simple2R1C_ic", "inputs.csv") 40 | ideal_path = os.path.join(parent, "resources", "simple2R1C_ic", "result.csv") 41 | est_path = os.path.join(parent, "resources", "simple2R1C_ic", "est.json") 42 | known_path = os.path.join(parent, "resources", "simple2R1C_ic", "known.json") 43 | 44 | # Assert there is an FMU for this platform 45 | assert os.path.exists( 46 | self.fmu_path 47 | ), "FMU for this platform ({}) doesn't exist.\n".format( 48 | platform 49 | ) + "No such file: {}".format( 50 | self.fmu_path 51 | ) 52 | 53 | self.inp = pd.read_csv(inp_path).set_index("time") 54 | self.ideal = pd.read_csv(ideal_path).set_index("time") 55 | 56 | with open(est_path) as f: 57 | self.est = json.load(f) 58 | with open(known_path) as f: 59 | self.known = json.load(f) 60 | 61 | def tearDown(self): 62 | shutil.rmtree(self.tmpdir) 63 | 64 | def test_estimation_basic(self): 65 | """Will use default methods ('MODESTGA', 'PS')""" 66 | modestga_opts = {"generations": 2, "workers": 1, "pop_size": 8, "trm_size": 3} 67 | ps_opts = {"maxiter": 2} 68 | session = Estimation( 69 | self.tmpdir, 70 | self.fmu_path, 71 | self.inp, 72 | self.known, 73 | self.est, 74 | self.ideal, 75 | modestga_opts=modestga_opts, 76 | ps_opts=ps_opts, 77 | default_log=False, 78 | ) 79 | estimates = session.estimate() 80 | err, res = session.validate() 81 | 82 | self.assertIsNotNone(estimates) 83 | self.assertGreater(len(estimates), 0) 84 | self.assertIsNotNone(err) 85 | self.assertIsNotNone(res) 86 | self.assertGreater(len(res.index), 1) 87 | self.assertGreater(len(res.columns), 0) 88 | 89 | def test_estimation_basic_parallel(self): 90 | """Will use default methods ('MODESTGA', 'PS')""" 91 | modestga_opts = {"generations": 2, "workers": 2, "pop_size": 16, "trm_size": 3} 92 | ps_opts = {"maxiter": 2} 93 | session = Estimation( 94 | self.tmpdir, 95 | self.fmu_path, 96 | self.inp, 97 | self.known, 98 | self.est, 99 | self.ideal, 100 | modestga_opts=modestga_opts, 101 | ps_opts=ps_opts, 102 | default_log=False, 103 | ) 104 | estimates = session.estimate() 105 | err, res = session.validate() 106 | 107 | self.assertIsNotNone(estimates) 108 | self.assertGreater(len(estimates), 0) 109 | self.assertIsNotNone(err) 110 | self.assertIsNotNone(res) 111 | self.assertGreater(len(res.index), 1) 112 | self.assertGreater(len(res.columns), 0) 113 | 114 | def test_estimation_all_args(self): 115 | modestga_opts = {"generations": 2, "workers": 2, "pop_size": 16, "trm_size": 3} 116 | ps_opts = {"maxiter": 3} 117 | session = Estimation( 118 | self.tmpdir, 119 | self.fmu_path, 120 | self.inp, 121 | self.known, 122 | self.est, 123 | self.ideal, 124 | lp_n=2, 125 | lp_len=3600, 126 | lp_frame=(0, 3600), 127 | vp=(20000, 40000), 128 | ic_param={"Tstart": "T"}, 129 | methods=("MODESTGA", "PS"), 130 | modestga_opts=modestga_opts, 131 | ps_opts=ps_opts, 132 | ftype="NRMSE", 133 | default_log=False, 134 | ) 135 | 136 | estimates = session.estimate() 137 | err, res = session.validate() # Standard validation period 138 | err2, res2 = session.validate(vp=(25000, 28600)) 139 | 140 | self.assertIsNotNone(estimates) 141 | self.assertGreater(len(estimates), 0) 142 | self.assertIsNotNone(err) 143 | self.assertIsNotNone(res) 144 | self.assertIsNotNone(err2) 145 | self.assertIsNotNone(res2) 146 | self.assertGreater(len(res.index), 1) 147 | self.assertGreater(len(res.columns), 0) 148 | self.assertGreater(len(res2.index), 1) 149 | self.assertGreater(len(res2.columns), 0) 150 | self.assertEqual(session.lp[0][0], 0) 151 | self.assertEqual(session.lp[0][1], 3600) 152 | self.assertLess(err["tot"], 1.7) # NRMSE 153 | 154 | def test_estimation_rmse(self): 155 | modestga_opts = {"generations": 8} 156 | ps_opts = {"maxiter": 16} 157 | 158 | session = Estimation( 159 | self.tmpdir, 160 | self.fmu_path, 161 | self.inp, 162 | self.known, 163 | self.est, 164 | self.ideal, 165 | lp_n=1, 166 | lp_len=3600, 167 | lp_frame=(0, 3600), 168 | vp=(20000, 40000), 169 | ic_param={"Tstart": "T"}, 170 | methods=("MODESTGA", "PS"), 171 | modestga_opts=modestga_opts, 172 | ps_opts=ps_opts, 173 | ftype="RMSE", 174 | default_log=False, 175 | ) 176 | 177 | estimates = session.estimate() 178 | err, res = session.validate() 179 | 180 | self.assertIsNotNone(estimates) 181 | self.assertGreater(len(estimates), 0) 182 | self.assertIsNotNone(err) 183 | self.assertIsNotNone(res) 184 | self.assertGreater(len(res.index), 1) 185 | self.assertGreater(len(res.columns), 0) 186 | self.assertLess(err["tot"], 1.48) 187 | 188 | def test_ga_only(self): 189 | modestga_opts = {"generations": 1} 190 | ps_opts = {"maxiter": 0} 191 | session = Estimation( 192 | self.tmpdir, 193 | self.fmu_path, 194 | self.inp, 195 | self.known, 196 | self.est, 197 | self.ideal, 198 | lp_n=1, 199 | lp_len=3600, 200 | lp_frame=(0, 3600), 201 | vp=(20000, 40000), 202 | ic_param={"Tstart": "T"}, 203 | methods=("MODESTGA",), 204 | modestga_opts=modestga_opts, 205 | ps_opts=ps_opts, 206 | ftype="RMSE", 207 | default_log=False, 208 | ) 209 | session.estimate() 210 | 211 | def test_ps_only(self): 212 | modestga_opts = {"generations": 0} 213 | ps_opts = {"maxiter": 1} 214 | session = Estimation( 215 | self.tmpdir, 216 | self.fmu_path, 217 | self.inp, 218 | self.known, 219 | self.est, 220 | self.ideal, 221 | lp_n=1, 222 | lp_len=3600, 223 | lp_frame=(0, 3600), 224 | vp=(20000, 40000), 225 | ic_param={"Tstart": "T"}, 226 | methods=("PS",), 227 | modestga_opts=modestga_opts, 228 | ps_opts=ps_opts, 229 | ftype="RMSE", 230 | default_log=False, 231 | ) 232 | session.estimate() 233 | 234 | def test_opts(self): 235 | modestga_opts = { 236 | "workers": 2, # CPU cores to use 237 | "generations": 10, # Max. number of generations 238 | "pop_size": 40, # Population size 239 | "mut_rate": 0.05, # Mutation rate 240 | "trm_size": 10, # Tournament size 241 | "tol": 1e-4, # Solution tolerance 242 | "inertia": 20, # Max. number of non-improving generations 243 | } 244 | ps_opts = {"maxiter": 10, "rel_step": 0.1, "tol": 0.001, "try_lim": 10} 245 | session = Estimation( 246 | self.tmpdir, 247 | self.fmu_path, 248 | self.inp, 249 | self.known, 250 | self.est, 251 | self.ideal, 252 | methods=("MODESTGA", "PS"), 253 | modestga_opts=modestga_opts, 254 | ps_opts=ps_opts, 255 | default_log=False, 256 | ) 257 | modestga_return = session.MODESTGA_OPTS 258 | ps_return = session.PS_OPTS 259 | 260 | def extractDictAFromB(A, B): 261 | return dict([(k, B[k]) for k in A.keys() if k in B.keys()]) 262 | 263 | self.assertEqual( 264 | modestga_opts, extractDictAFromB(modestga_opts, modestga_return) 265 | ) 266 | self.assertEqual(ps_opts, extractDictAFromB(ps_opts, ps_return)) 267 | 268 | 269 | def suite(): 270 | suite = unittest.TestSuite() 271 | suite.addTest(TestEstimation("test_estimation_basic")) 272 | suite.addTest(TestEstimation("test_estimation_basic_parallel")) 273 | suite.addTest(TestEstimation("test_estimation_all_args")) 274 | suite.addTest(TestEstimation("test_estimation_rmse")) 275 | suite.addTest(TestEstimation("test_ga_only")) 276 | suite.addTest(TestEstimation("test_ps_only")) 277 | suite.addTest(TestEstimation("test_opts")) 278 | 279 | return suite 280 | 281 | 282 | if __name__ == "__main__": 283 | unittest.main() 284 | -------------------------------------------------------------------------------- /modestpy/test/test_fmpy.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import json 8 | import os 9 | import shutil 10 | import tempfile 11 | import unittest 12 | 13 | import pandas as pd 14 | 15 | from modestpy.fmi.model import Model 16 | from modestpy.loginit import config_logger 17 | from modestpy.utilities.sysarch import get_sys_arch 18 | 19 | 20 | class TestFMPy(unittest.TestCase): 21 | def setUp(self): 22 | 23 | # Platform (win32, win64, linux32, linix64) 24 | platform = get_sys_arch() 25 | assert platform, "Unsupported platform type!" 26 | 27 | # Temp directory 28 | self.tmpdir = tempfile.mkdtemp() 29 | 30 | # Parent directory 31 | parent = os.path.dirname(__file__) 32 | 33 | # Resources 34 | self.fmu_path = os.path.join( 35 | parent, "resources", "simple2R1C", "Simple2R1C_{}.fmu".format(platform) 36 | ) 37 | inp_path = os.path.join(parent, "resources", "simple2R1C", "inputs.csv") 38 | ideal_path = os.path.join(parent, "resources", "simple2R1C", "result.csv") 39 | est_path = os.path.join(parent, "resources", "simple2R1C", "est.json") 40 | known_path = os.path.join(parent, "resources", "simple2R1C", "known.json") 41 | 42 | # Assert there is an FMU for this platform 43 | assert os.path.exists( 44 | self.fmu_path 45 | ), "FMU for this platform ({}) doesn't exist.\n".format( 46 | platform 47 | ) + "No such file: {}".format( 48 | self.fmu_path 49 | ) 50 | 51 | self.inp = pd.read_csv(inp_path).set_index("time") 52 | self.ideal = pd.read_csv(ideal_path).set_index("time") 53 | 54 | with open(est_path) as f: 55 | self.est = json.load(f) 56 | with open(known_path) as f: 57 | known_dict = json.load(f) 58 | known_records = dict() 59 | for k, v in known_dict.items(): 60 | known_records[k] = [v] 61 | self.known_df = pd.DataFrame.from_dict(known_records) 62 | 63 | def tearDown(self): 64 | shutil.rmtree(self.tmpdir) 65 | 66 | def test_simulation(self): 67 | output_names = self.ideal.columns.tolist() 68 | model = Model(self.fmu_path) 69 | model.inputs_from_df(self.inp) 70 | model.specify_outputs(output_names) 71 | model.parameters_from_df(self.known_df) 72 | 73 | res1 = model.simulate(reset=True) 74 | res2 = model.simulate(reset=False) 75 | 76 | self.assertTrue(res1.equals(res2), "Dataframes not equal") 77 | 78 | input_size = self.inp.index.size 79 | result_size = res1.index.size 80 | self.assertTrue(input_size == result_size, "Result size different than input") 81 | 82 | 83 | def suite(): 84 | suite = unittest.TestSuite() 85 | suite.addTest(TestFMPy("test_simulation")) 86 | 87 | return suite 88 | 89 | 90 | if __name__ == "__main__": 91 | config_logger(filename="unit_tests.log", level="DEBUG") 92 | unittest.main() 93 | -------------------------------------------------------------------------------- /modestpy/test/test_ga.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import json 8 | import os 9 | import random 10 | import shutil 11 | import tempfile 12 | import unittest 13 | 14 | import numpy as np 15 | import pandas as pd 16 | 17 | from modestpy.estim.ga.ga import GA 18 | from modestpy.loginit import config_logger 19 | from modestpy.utilities.sysarch import get_sys_arch 20 | 21 | 22 | class TestGA(unittest.TestCase): 23 | def setUp(self): 24 | 25 | # Platform (win32, win64, linux32, linux64) 26 | platform = get_sys_arch() 27 | assert platform, "Unsupported platform type!" 28 | 29 | # Temp directory 30 | self.tmpdir = tempfile.mkdtemp() 31 | 32 | # Parent directory 33 | parent = os.path.dirname(__file__) 34 | 35 | # Resources 36 | self.fmu_path = os.path.join( 37 | parent, "resources", "simple2R1C", "Simple2R1C_{}.fmu".format(platform) 38 | ) 39 | inp_path = os.path.join(parent, "resources", "simple2R1C", "inputs.csv") 40 | ideal_path = os.path.join(parent, "resources", "simple2R1C", "result.csv") 41 | est_path = os.path.join(parent, "resources", "simple2R1C", "est.json") 42 | known_path = os.path.join(parent, "resources", "simple2R1C", "known.json") 43 | 44 | # Assert there is an FMU for this platform 45 | assert os.path.exists( 46 | self.fmu_path 47 | ), "FMU for this platform ({}) doesn't exist.\n".format( 48 | platform 49 | ) + "No such file: {}".format( 50 | self.fmu_path 51 | ) 52 | 53 | self.inp = pd.read_csv(inp_path).set_index("time") 54 | self.ideal = pd.read_csv(ideal_path).set_index("time") 55 | 56 | with open(est_path) as f: 57 | self.est = json.load(f) 58 | with open(known_path) as f: 59 | self.known = json.load(f) 60 | 61 | # GA settings 62 | self.gen = 4 63 | self.pop = 8 64 | self.trm = 3 65 | 66 | def tearDown(self): 67 | shutil.rmtree(self.tmpdir) 68 | 69 | def test_ga(self): 70 | random.seed(1) 71 | ga = GA( 72 | self.fmu_path, 73 | self.inp, 74 | self.known, 75 | self.est, 76 | self.ideal, 77 | maxiter=self.gen, 78 | pop_size=self.pop, 79 | trm_size=self.trm, 80 | ) 81 | self.estimates = ga.estimate() 82 | 83 | # Generate plot 84 | plot_path = os.path.join(self.tmpdir, "popevo.png") 85 | ga.plot_pop_evo(plot_path) 86 | 87 | # Make sure plot is created 88 | self.assertTrue(os.path.exists(plot_path)) 89 | 90 | # Make sure errors do not increase 91 | errors = ga.get_errors() 92 | for i in range(1, len(errors)): 93 | prev_err = errors[i - 1] 94 | next_err = errors[i] 95 | self.assertGreaterEqual(prev_err, next_err) 96 | 97 | def test_init_pop(self): 98 | random.seed(1) 99 | init_pop = pd.DataFrame( 100 | { 101 | "R1": [0.1, 0.2, 0.3], 102 | "R2": [0.15, 0.25, 0.35], 103 | "C": [1000.0, 1100.0, 1200.0], 104 | } 105 | ) 106 | pop_size = 3 107 | ga = GA( 108 | self.fmu_path, 109 | self.inp, 110 | self.known, 111 | self.est, 112 | self.ideal, 113 | maxiter=self.gen, 114 | pop_size=pop_size, 115 | trm_size=self.trm, 116 | init_pop=init_pop, 117 | ) 118 | i1 = ga.pop.individuals[0] 119 | i2 = ga.pop.individuals[1] 120 | i3 = ga.pop.individuals[2] 121 | R1_lo = self.est["R1"][1] 122 | R1_hi = self.est["R1"][2] 123 | R2_lo = self.est["R2"][1] 124 | R2_hi = self.est["R2"][2] 125 | C_lo = self.est["C"][1] 126 | C_hi = self.est["C"][2] 127 | assert i1.genes == { 128 | "C": (1000.0 - C_lo) / (C_hi - C_lo), 129 | "R1": (0.1 - R1_lo) / (R1_hi - R1_lo), 130 | "R2": (0.15 - R2_lo) / (R2_hi - R2_lo), 131 | } 132 | assert i2.genes == { 133 | "C": (1100.0 - C_lo) / (C_hi - C_lo), 134 | "R1": (0.2 - R1_lo) / (R1_hi - R1_lo), 135 | "R2": (0.25 - R2_lo) / (R2_hi - R2_lo), 136 | } 137 | assert i3.genes == { 138 | "C": (1200.0 - C_lo) / (C_hi - C_lo), 139 | "R1": (0.3 - R1_lo) / (R1_hi - R1_lo), 140 | "R2": (0.35 - R2_lo) / (R2_hi - R2_lo), 141 | } 142 | 143 | def test_lhs(self): 144 | """ 145 | Tests if populations of two instances with lhs=True 146 | and the same seed are identical. 147 | """ 148 | random.seed(1) 149 | np.random.seed(1) 150 | ga = GA( 151 | self.fmu_path, 152 | self.inp, 153 | self.known, 154 | self.est, 155 | self.ideal, 156 | maxiter=self.gen, 157 | lhs=True, 158 | ) 159 | indiv = ga.pop.individuals 160 | par1 = list() 161 | for i in indiv: 162 | par1.append(i.get_estimates(as_dict=True)) 163 | 164 | random.seed(1) 165 | np.random.seed(1) 166 | ga = GA( 167 | self.fmu_path, 168 | self.inp, 169 | self.known, 170 | self.est, 171 | self.ideal, 172 | maxiter=self.gen, 173 | lhs=True, 174 | ) 175 | indiv = ga.pop.individuals 176 | par2 = list() 177 | for i in indiv: 178 | par2.append(i.get_estimates(as_dict=True)) 179 | 180 | for d1, d2 in zip(par1, par2): 181 | self.assertDictEqual(d1, d2) 182 | 183 | 184 | def suite(): 185 | suite = unittest.TestSuite() 186 | suite.addTest(TestGA("test_ga")) 187 | suite.addTest(TestGA("test_init_pop")) 188 | return suite 189 | 190 | 191 | if __name__ == "__main__": 192 | config_logger(filename="unit_tests.log", level="DEBUG") 193 | unittest.main() 194 | -------------------------------------------------------------------------------- /modestpy/test/test_ga_parallel.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import json 8 | import os 9 | import random 10 | import shutil 11 | import tempfile 12 | import unittest 13 | 14 | import numpy as np 15 | import pandas as pd 16 | 17 | from modestpy.estim.ga_parallel.ga_parallel import MODESTGA 18 | from modestpy.loginit import config_logger 19 | from modestpy.utilities.sysarch import get_sys_arch 20 | 21 | 22 | class TestMODESTGA(unittest.TestCase): 23 | def setUp(self): 24 | 25 | # Platform (win32, win64, linux32, linux64) 26 | platform = get_sys_arch() 27 | assert platform, "Unsupported platform type!" 28 | 29 | # Temp directory 30 | self.tmpdir = tempfile.mkdtemp() 31 | 32 | # Parent directory 33 | parent = os.path.dirname(__file__) 34 | 35 | # Resources 36 | self.fmu_path = os.path.join( 37 | parent, "resources", "simple2R1C", "Simple2R1C_{}.fmu".format(platform) 38 | ) 39 | inp_path = os.path.join(parent, "resources", "simple2R1C", "inputs.csv") 40 | ideal_path = os.path.join(parent, "resources", "simple2R1C", "result.csv") 41 | est_path = os.path.join(parent, "resources", "simple2R1C", "est.json") 42 | known_path = os.path.join(parent, "resources", "simple2R1C", "known.json") 43 | 44 | # Assert there is an FMU for this platform 45 | assert os.path.exists( 46 | self.fmu_path 47 | ), "FMU for this platform ({}) doesn't exist.\n".format( 48 | platform 49 | ) + "No such file: {}".format( 50 | self.fmu_path 51 | ) 52 | 53 | self.inp = pd.read_csv(inp_path).set_index("time") 54 | self.ideal = pd.read_csv(ideal_path).set_index("time") 55 | 56 | with open(est_path) as f: 57 | self.est = json.load(f) 58 | with open(known_path) as f: 59 | self.known = json.load(f) 60 | 61 | # MODESTGA settings 62 | self.gen = 2 63 | self.pop = None # Set individually 64 | self.trm = None # Set individually 65 | self.workers = None # Set individually 66 | 67 | def tearDown(self): 68 | shutil.rmtree(self.tmpdir) 69 | 70 | def test_modestga_default(self): 71 | ga = MODESTGA( 72 | self.fmu_path, 73 | self.inp, 74 | self.known, 75 | self.est, 76 | self.ideal, 77 | generations=self.gen, 78 | ) 79 | par_df = ga.estimate() 80 | assert type(par_df) is pd.DataFrame 81 | 82 | def test_modestga_1_worker(self): 83 | ga = MODESTGA( 84 | self.fmu_path, 85 | self.inp, 86 | self.known, 87 | self.est, 88 | self.ideal, 89 | generations=self.gen, 90 | pop_size=6, 91 | trm_size=self.trm, 92 | tol=1e-3, 93 | inertia=5, 94 | workers=1, 95 | ) 96 | par_df = ga.estimate() 97 | assert type(par_df) is pd.DataFrame 98 | 99 | def test_modestga_2_workers_small_pop(self): 100 | ga = MODESTGA( 101 | self.fmu_path, 102 | self.inp, 103 | self.known, 104 | self.est, 105 | self.ideal, 106 | generations=self.gen, 107 | pop_size=2, 108 | trm_size=1, 109 | tol=1e-3, 110 | inertia=5, 111 | workers=2, 112 | ) 113 | par_df = ga.estimate() 114 | assert type(par_df) is pd.DataFrame 115 | 116 | def test_modestga_2_workers_large_pop(self): 117 | ga = MODESTGA( 118 | self.fmu_path, 119 | self.inp, 120 | self.known, 121 | self.est, 122 | self.ideal, 123 | generations=self.gen, 124 | pop_size=32, 125 | trm_size=3, 126 | tol=1e-3, 127 | inertia=5, 128 | workers=2, 129 | ) 130 | par_df = ga.estimate() 131 | assert type(par_df) is pd.DataFrame 132 | 133 | 134 | def suite(): 135 | suite = unittest.TestSuite() 136 | suite.addTest(TestMODESTGA("test_modestga_default")) 137 | suite.addTest(TestMODESTGA("test_modestga_1_worker")) 138 | suite.addTest(TestMODESTGA("test_modestga_2_workers_small_pop")) 139 | suite.addTest(TestMODESTGA("test_modestga_2_workers_large_pop")) 140 | return suite 141 | 142 | 143 | if __name__ == "__main__": 144 | config_logger(filename="unit_tests.log", level="DEBUG") 145 | unittest.main() 146 | -------------------------------------------------------------------------------- /modestpy/test/test_ps.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import json 8 | import os 9 | import shutil 10 | import tempfile 11 | import unittest 12 | 13 | import pandas as pd 14 | 15 | from modestpy.estim.ps.ps import PS 16 | from modestpy.loginit import config_logger 17 | from modestpy.utilities.sysarch import get_sys_arch 18 | 19 | 20 | class TestPS(unittest.TestCase): 21 | def setUp(self): 22 | 23 | # Platform (win32, win64, linux32, linix64) 24 | platform = get_sys_arch() 25 | assert platform, "Unsupported platform type!" 26 | 27 | # Temp directory 28 | self.tmpdir = tempfile.mkdtemp() 29 | 30 | # Parent directory 31 | parent = os.path.dirname(__file__) 32 | 33 | # Resources 34 | self.fmu_path = os.path.join( 35 | parent, "resources", "simple2R1C", "Simple2R1C_{}.fmu".format(platform) 36 | ) 37 | inp_path = os.path.join(parent, "resources", "simple2R1C", "inputs.csv") 38 | ideal_path = os.path.join(parent, "resources", "simple2R1C", "result.csv") 39 | est_path = os.path.join(parent, "resources", "simple2R1C", "est.json") 40 | known_path = os.path.join(parent, "resources", "simple2R1C", "known.json") 41 | 42 | # Assert there is an FMU for this platform 43 | assert os.path.exists( 44 | self.fmu_path 45 | ), "FMU for this platform ({}) doesn't exist.\n".format( 46 | platform 47 | ) + "No such file: {}".format( 48 | self.fmu_path 49 | ) 50 | 51 | self.inp = pd.read_csv(inp_path).set_index("time") 52 | self.ideal = pd.read_csv(ideal_path).set_index("time") 53 | 54 | with open(est_path) as f: 55 | self.est = json.load(f) 56 | with open(known_path) as f: 57 | self.known = json.load(f) 58 | 59 | # PS settings 60 | self.max_iter = 3 61 | self.try_lim = 2 62 | 63 | def tearDown(self): 64 | shutil.rmtree(self.tmpdir) 65 | 66 | def test_ps(self): 67 | self.ps = PS( 68 | self.fmu_path, 69 | self.inp, 70 | self.known, 71 | self.est, 72 | self.ideal, 73 | maxiter=self.max_iter, 74 | try_lim=self.try_lim, 75 | ) 76 | self.estimates = self.ps.estimate() 77 | 78 | # Generate plots 79 | self.ps.plot_comparison(os.path.join(self.tmpdir, "ps_comparison.png")) 80 | self.ps.plot_error_evo(os.path.join(self.tmpdir, "ps_error_evo.png")) 81 | self.ps.plot_parameter_evo(os.path.join(self.tmpdir, "ps_param_evo.png")) 82 | 83 | # Make sure plots are created 84 | self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "ps_comparison.png"))) 85 | self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "ps_error_evo.png"))) 86 | self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "ps_param_evo.png"))) 87 | 88 | # Make sure errors do not increase 89 | errors = self.ps.get_errors() 90 | for i in range(1, len(errors)): 91 | prev_err = errors[i - 1] 92 | next_err = errors[i] 93 | self.assertGreaterEqual(prev_err, next_err) 94 | 95 | 96 | def suite(): 97 | suite = unittest.TestSuite() 98 | suite.addTest(TestPS("test_ps")) 99 | 100 | return suite 101 | 102 | 103 | if __name__ == "__main__": 104 | config_logger(filename="unit_tests.log", level="DEBUG") 105 | unittest.main() 106 | -------------------------------------------------------------------------------- /modestpy/test/test_scipy.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import json 8 | import os 9 | import shutil 10 | import tempfile 11 | import unittest 12 | 13 | import pandas as pd 14 | 15 | from modestpy.estim.scipy.scipy import SCIPY 16 | from modestpy.loginit import config_logger 17 | from modestpy.utilities.sysarch import get_sys_arch 18 | 19 | 20 | class TestSCIPY(unittest.TestCase): 21 | def setUp(self): 22 | 23 | # Platform (win32, win64, linux32, linix64) 24 | platform = get_sys_arch() 25 | assert platform, "Unsupported platform type!" 26 | 27 | # Temp directory 28 | self.tmpdir = tempfile.mkdtemp() 29 | 30 | # Parent directory 31 | parent = os.path.dirname(__file__) 32 | 33 | # Resources 34 | self.fmu_path = os.path.join( 35 | parent, "resources", "simple2R1C", "Simple2R1C_{}.fmu".format(platform) 36 | ) 37 | inp_path = os.path.join(parent, "resources", "simple2R1C", "inputs.csv") 38 | ideal_path = os.path.join(parent, "resources", "simple2R1C", "result.csv") 39 | est_path = os.path.join(parent, "resources", "simple2R1C", "est.json") 40 | known_path = os.path.join(parent, "resources", "simple2R1C", "known.json") 41 | 42 | # Assert there is an FMU for this platform 43 | assert os.path.exists( 44 | self.fmu_path 45 | ), "FMU for this platform ({}) doesn't exist.\n".format( 46 | platform 47 | ) + "No such file: {}".format( 48 | self.fmu_path 49 | ) 50 | 51 | self.inp = pd.read_csv(inp_path).set_index("time") 52 | self.ideal = pd.read_csv(ideal_path).set_index("time") 53 | 54 | with open(est_path) as f: 55 | self.est = json.load(f) 56 | with open(known_path) as f: 57 | self.known = json.load(f) 58 | 59 | # PS settings 60 | self.max_iter = 3 61 | self.try_lim = 2 62 | 63 | def tearDown(self): 64 | shutil.rmtree(self.tmpdir) 65 | 66 | def test_scipy(self): 67 | self.scipy = SCIPY( 68 | self.fmu_path, self.inp, self.known, self.est, self.ideal, solver="L-BFGS-B" 69 | ) 70 | self.estimates = self.scipy.estimate() 71 | 72 | # Generate plots 73 | self.scipy.plot_comparison(os.path.join(self.tmpdir, "scipy_comparison.png")) 74 | self.scipy.plot_error_evo(os.path.join(self.tmpdir, "scipy_error_evo.png")) 75 | self.scipy.plot_parameter_evo(os.path.join(self.tmpdir, "scipy_param_evo.png")) 76 | 77 | # Make sure plots are created 78 | self.assertTrue( 79 | os.path.exists(os.path.join(self.tmpdir, "scipy_comparison.png")) 80 | ) 81 | self.assertTrue( 82 | os.path.exists(os.path.join(self.tmpdir, "scipy_error_evo.png")) 83 | ) 84 | self.assertTrue( 85 | os.path.exists(os.path.join(self.tmpdir, "scipy_param_evo.png")) 86 | ) 87 | 88 | # Make last error is lower than initial 89 | errors = self.scipy.get_errors() 90 | self.assertGreaterEqual(errors[0], errors[-1]) 91 | 92 | 93 | def suite(): 94 | suite = unittest.TestSuite() 95 | suite.addTest(TestSCIPY("test_scipy")) 96 | 97 | return suite 98 | 99 | 100 | if __name__ == "__main__": 101 | config_logger(filename="unit_tests.log", level="DEBUG") 102 | unittest.main() 103 | -------------------------------------------------------------------------------- /modestpy/test/test_utilities.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | import os 8 | import tempfile 9 | import unittest 10 | 11 | from modestpy.loginit import config_logger 12 | from modestpy.utilities.delete_logs import delete_logs 13 | 14 | 15 | class TestUtilities(unittest.TestCase): 16 | def setUp(self): 17 | # Temp directory 18 | self.temp_dir = tempfile.mkdtemp() 19 | # Temp log file 20 | self.log_path = os.path.join(self.temp_dir, "test.log") 21 | log_file = open(self.log_path, "w") 22 | log_file.close() 23 | 24 | def tearDown(self): 25 | try: 26 | os.remove(self.log_path) 27 | except OSError: 28 | pass # File already removed 29 | os.rmdir(self.temp_dir) 30 | 31 | def test_delete_logs(self): 32 | delete_logs(self.temp_dir) 33 | content = os.listdir(self.temp_dir) 34 | self.assertEqual(len(content), 0) 35 | 36 | 37 | def suite(): 38 | suite = unittest.TestSuite() 39 | suite.addTest(TestUtilities("test_delete_logs")) 40 | 41 | return suite 42 | 43 | 44 | if __name__ == "__main__": 45 | config_logger(filename="unit_tests.log", level="DEBUG") 46 | unittest.main() 47 | -------------------------------------------------------------------------------- /modestpy/utilities/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Various utility classes and functions 3 | """ 4 | -------------------------------------------------------------------------------- /modestpy/utilities/delete_logs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | 8 | from __future__ import absolute_import 9 | from __future__ import division 10 | from __future__ import print_function 11 | 12 | import os 13 | 14 | 15 | def delete_logs(directory=os.getcwd()): 16 | """ 17 | Deletes log files in the directory. 18 | 19 | :param directory: string, path to the directory 20 | :return: None 21 | """ 22 | content = os.listdir(directory) 23 | for el in content: 24 | if el.split(".")[-1] == "log": 25 | # This is a log file 26 | fpath = os.path.join(directory, el) 27 | print("Removing {}".format(fpath)) 28 | try: 29 | os.remove(fpath) 30 | except WindowsError as e: 31 | print(e.message) 32 | return 33 | -------------------------------------------------------------------------------- /modestpy/utilities/figures.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | 8 | 9 | def get_figure(ax): 10 | """ 11 | Retrieves figure from axes. Axes can be either an instance 12 | of Matplotlib.Axes or a 1D/2D array of Matplotlib.Axes. 13 | 14 | :param ax: Axes or vector/array of Axes 15 | :return: Matplotlib.Figure 16 | """ 17 | fig = None 18 | try: 19 | # Single plot 20 | fig = ax.get_figure() 21 | except AttributeError: 22 | # Subplots 23 | try: 24 | # 1D grid 25 | fig = ax[0].get_figure() 26 | except AttributeError: 27 | # 2D grid 28 | fig = ax[0][0].get_figure() 29 | 30 | return fig 31 | -------------------------------------------------------------------------------- /modestpy/utilities/parameters.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | 8 | from __future__ import absolute_import 9 | from __future__ import division 10 | from __future__ import print_function 11 | 12 | import pandas as pd 13 | 14 | 15 | class Parameters: 16 | """ 17 | Enables to: 18 | 19 | * read csv with parameters 20 | * assign new values 21 | * save csv with updated parameters 22 | 23 | """ 24 | 25 | def __init__(self, f=None): 26 | self.pars = pd.DataFrame() 27 | self.f = None 28 | if f is not None: 29 | self.read(f) 30 | 31 | def read(self, f): 32 | self.pars = pd.read_csv(f) 33 | self.f = f 34 | 35 | def assign(self, **kwargs): 36 | """ 37 | Assigns new parameters from **kwargs. 38 | Does not automatically save the file. 39 | 40 | :param kwargs: name = value 41 | :return: None 42 | """ 43 | for key in kwargs: 44 | self.pars[key] = [kwargs[key]] 45 | 46 | def update_and_save(self, new_par): 47 | """ 48 | Updates csv with parameters with new parameters from 49 | new_par DataFrame. Automatically saves the file. 50 | 51 | :param new_par: DataFrame 52 | :return: None 53 | """ 54 | for key in new_par: 55 | self.pars[key] = new_par[key] 56 | self.save() 57 | 58 | def save_template(self, dictionary, f): 59 | self.f = f 60 | # Convert scalars to unit vectors 61 | for key in dictionary: 62 | dictionary[key] = [dictionary[key]] 63 | # Generate and save DataFrame 64 | df = pd.DataFrame.from_dict(dictionary) 65 | df.to_csv(f, index=False) 66 | # Update self.pars 67 | self.pars = pd.read_csv(f) 68 | 69 | def show(self): 70 | print(self.pars) 71 | 72 | def save(self, f=None): 73 | if f: 74 | self.pars.to_csv(f, index=False) 75 | else: 76 | self.pars.to_csv(self.f, index=False) 77 | -------------------------------------------------------------------------------- /modestpy/utilities/sysarch.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017, University of Southern Denmark 3 | All rights reserved. 4 | This code is licensed under BSD 2-clause license. 5 | See LICENSE file in the project root for license terms. 6 | """ 7 | 8 | from __future__ import absolute_import 9 | from __future__ import division 10 | from __future__ import print_function 11 | 12 | import platform 13 | 14 | 15 | def get_sys_arch(): 16 | """ 17 | Returns system architecture: 18 | 'win32', 'win64', 'linux32', 'linux64' or 'unknown'. 19 | 20 | Modestpy supports only windows and linux at the moment, 21 | so other platforms are not recognized. 22 | 23 | .. warning:: It relies on the ``platform.architecture`` function 24 | which on Windows 64bit with 32bit Python interpereter 25 | returns 32bit. However since JModelica on Windows 26 | works by default as 32bit and requires Python 32bit, 27 | for the time being this solution is accepted. 28 | 29 | Returns 30 | ------- 31 | str or None 32 | """ 33 | sys_type = platform.system() 34 | bit_arch = platform.architecture()[0] 35 | 36 | sys = None 37 | bits = None 38 | 39 | if ("win" in sys_type) or ("Win" in sys_type) or ("WIN" in sys_type): 40 | sys = "win" 41 | elif ("linux" in sys_type) or ("Linux" in sys_type) or ("LINUX" in sys_type): 42 | 43 | sys = "linux" 44 | 45 | if "32" in bit_arch: 46 | bits = "32" 47 | elif "64" in bit_arch: 48 | bits = "64" 49 | 50 | if sys and bits: 51 | sys_bits = sys + bits 52 | else: 53 | sys_bits = None 54 | 55 | return sys_bits 56 | 57 | 58 | if __name__ == "__main__": 59 | # Test 60 | print(get_sys_arch()) 61 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | black 3 | isort 4 | flake8 5 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="modestpy", 5 | version="0.1.1", 6 | description="FMI-compliant model identification package", 7 | url="https://github.com/sdu-cfei/modest-py", 8 | keywords="fmi fmu optimization model identification estimation", 9 | author="Krzysztof Arendt, Center for Energy Informatics SDU", 10 | author_email="krzysztof.arendt@gmail.com, veje@mmmi.sdu.dk", 11 | license="BSD", 12 | platforms=["Windows", "Linux"], 13 | packages=[ 14 | "modestpy", 15 | "modestpy.estim", 16 | "modestpy.estim.ga_parallel", 17 | "modestpy.estim.ga", 18 | "modestpy.estim.ps", 19 | "modestpy.estim.scipy", 20 | "modestpy.fmi", 21 | "modestpy.utilities", 22 | "modestpy.test", 23 | ], 24 | include_package_data=True, 25 | install_requires=[ 26 | "fmpy", 27 | "scipy", 28 | "pandas", 29 | "matplotlib", 30 | "numpy", 31 | "pyDOE", 32 | "modestga", 33 | ], 34 | classifiers=["Programming Language :: Python :: 3"], 35 | ) 36 | --------------------------------------------------------------------------------