├── popt ├── cost_functions │ ├── __init__.py │ ├── rosenbrock.py │ ├── epf.py │ ├── quadratic.py │ ├── npv.py │ ├── ren_npv.py │ ├── ren_npv_co2.py │ ├── ecalc_npv.py │ └── ecalc_pareto_npv.py ├── loop │ ├── __init__.py │ ├── ensemble_base.py │ ├── optimize.py │ └── extensions.py ├── misc_tools │ ├── __init__.py │ └── basic_tools.py ├── update_schemes │ ├── __init__.py │ ├── subroutines │ │ ├── __init__.py │ │ └── cma.py │ ├── smcopt.py │ └── genopt.py ├── __init__.py └── README.md ├── misc ├── __init__.py ├── system_tools │ └── __init__.py ├── ecl_common.py ├── grid │ ├── __init__.py │ ├── unstruct.py │ └── sector.py └── read_input_csv.py ├── pipt ├── misc_tools │ └── __init__.py ├── loop │ └── __init__.py ├── update_schemes │ ├── gies │ │ ├── __init__.py │ │ └── gies_rlmmac.py │ ├── update_methods_ns │ │ ├── __init__.py │ │ ├── margIS_update.py │ │ ├── full_update.py │ │ ├── hybrid_update.py │ │ └── subspace_update.py │ ├── __init__.py │ ├── es.py │ └── enkf.py ├── README.md ├── __init__.py └── pipt_init.py ├── ensemble └── __init__.py ├── input_output ├── __init__.py ├── get_ecl_key_val.py └── organize.py ├── simulator ├── __init__.py └── README.md ├── docs ├── tutorials │ ├── pipt │ │ ├── permx.png │ │ ├── priormean.npz │ │ ├── jupyter_kernel.png │ │ ├── var.csv │ │ ├── true_data.csv │ │ ├── Schdl.sch │ │ ├── 3D_ESMDA.toml │ │ └── RUNFILE.mako │ ├── popt │ │ ├── permx.png │ │ ├── jupyter_kernel.png │ │ ├── init_optim.toml │ │ └── 3WELL.mako │ └── README.md ├── manual_inv_prob_python.pdf ├── index.md ├── stylesheets │ ├── code_select.css │ ├── extra.css │ └── jupyter.css ├── javascripts │ ├── mathjax.js │ └── extra.js ├── overrides │ └── main.html ├── bib │ ├── bib2md.py │ └── refs.bib ├── references.md ├── gen_ref_pages.py └── dev_guide.md ├── tests ├── conftest.py ├── parser_input.pipt ├── test_linear.py ├── test_quadratic.py └── test_pipt_file_parser.py ├── .github └── workflows │ ├── requirements.txt │ ├── deploy-docs.yml │ └── tests.yml ├── CITATION.cff ├── setup.py ├── .gitignore ├── README.md └── mkdocs.yml /popt/cost_functions/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /misc/__init__.py: -------------------------------------------------------------------------------- 1 | """More tools.""" 2 | -------------------------------------------------------------------------------- /pipt/misc_tools/__init__.py: -------------------------------------------------------------------------------- 1 | """More tools.""" 2 | -------------------------------------------------------------------------------- /ensemble/__init__.py: -------------------------------------------------------------------------------- 1 | """Multiple realisations management.""" -------------------------------------------------------------------------------- /popt/loop/__init__.py: -------------------------------------------------------------------------------- 1 | """Main loop for running optimization.""" -------------------------------------------------------------------------------- /popt/misc_tools/__init__.py: -------------------------------------------------------------------------------- 1 | """Optimisation helpers.""" 2 | -------------------------------------------------------------------------------- /popt/update_schemes/__init__.py: -------------------------------------------------------------------------------- 1 | """Iterative steppers.""" 2 | -------------------------------------------------------------------------------- /input_output/__init__.py: -------------------------------------------------------------------------------- 1 | """File-based communication and storage.""" 2 | -------------------------------------------------------------------------------- /pipt/loop/__init__.py: -------------------------------------------------------------------------------- 1 | """Main loop for running data assimilation.""" 2 | -------------------------------------------------------------------------------- /pipt/update_schemes/gies/__init__.py: -------------------------------------------------------------------------------- 1 | """Descriptive description.""" 2 | -------------------------------------------------------------------------------- /misc/system_tools/__init__.py: -------------------------------------------------------------------------------- 1 | """Multiprocessing and environment management.""" 2 | -------------------------------------------------------------------------------- /pipt/update_schemes/update_methods_ns/__init__.py: -------------------------------------------------------------------------------- 1 | """Descriptive description.""" 2 | -------------------------------------------------------------------------------- /popt/__init__.py: -------------------------------------------------------------------------------- 1 | """Optimisation methods. 2 | 3 | --8<-- "popt/README.md" 4 | """ 5 | -------------------------------------------------------------------------------- /simulator/__init__.py: -------------------------------------------------------------------------------- 1 | """Model wrappers. 2 | 3 | --8<-- "simulator/README.md" 4 | """ 5 | -------------------------------------------------------------------------------- /docs/tutorials/pipt/permx.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Python-Ensemble-Toolbox/PET/HEAD/docs/tutorials/pipt/permx.png -------------------------------------------------------------------------------- /docs/tutorials/popt/permx.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Python-Ensemble-Toolbox/PET/HEAD/docs/tutorials/popt/permx.png -------------------------------------------------------------------------------- /popt/update_schemes/subroutines/__init__.py: -------------------------------------------------------------------------------- 1 | from .subroutines import * 2 | from .cma import * 3 | from .optimizers import * -------------------------------------------------------------------------------- /docs/manual_inv_prob_python.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Python-Ensemble-Toolbox/PET/HEAD/docs/manual_inv_prob_python.pdf -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | hide: 3 | - navigation 4 | --- 5 | 6 |

Home

7 | 8 | --8<-- "README.md" 9 | -------------------------------------------------------------------------------- /docs/tutorials/pipt/priormean.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Python-Ensemble-Toolbox/PET/HEAD/docs/tutorials/pipt/priormean.npz -------------------------------------------------------------------------------- /docs/tutorials/pipt/jupyter_kernel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Python-Ensemble-Toolbox/PET/HEAD/docs/tutorials/pipt/jupyter_kernel.png -------------------------------------------------------------------------------- /docs/tutorials/popt/jupyter_kernel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Python-Ensemble-Toolbox/PET/HEAD/docs/tutorials/popt/jupyter_kernel.png -------------------------------------------------------------------------------- /docs/stylesheets/code_select.css: -------------------------------------------------------------------------------- 1 | .language-pycon .gp, .language-pycon .go { /* Generic.Prompt, Generic.Output */ 2 | user-select: none; 3 | } 4 | -------------------------------------------------------------------------------- /pipt/README.md: -------------------------------------------------------------------------------- 1 | ### Python Inverse Problem Toolbox (PIPT) 2 | 3 | PIPT is one part of the PET application. Here we solve Data-Assimilation and inverse problems. 4 | -------------------------------------------------------------------------------- /docs/stylesheets/extra.css: -------------------------------------------------------------------------------- 1 | /* From NORCE color palette */ 2 | :root { 3 | --md-primary-fg-color: #007075; 4 | --md-primary-fg-color--light: #c6dddf; 5 | --md-primary-fg-color--dark: #060654; 6 | } 7 | -------------------------------------------------------------------------------- /docs/tutorials/README.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | Here are some tutorials. 4 | 5 | - [`tutorial_pipt.ipynb`](pipt/tutorial_pipt): Tutorial for running PIPT 6 | - [`tutorial_pipt.ipynb`](popt/tutorial_popt): Tutorial for running POPT 7 | -------------------------------------------------------------------------------- /pipt/update_schemes/gies/gies_rlmmac.py: -------------------------------------------------------------------------------- 1 | 2 | from pipt.update_schemes.gies.gies_base import GIESMixIn 3 | from pipt.update_schemes.gies.rlmmac_update import rlmmac_update 4 | 5 | 6 | class gies_rlmmac(GIESMixIn, rlmmac_update): 7 | pass -------------------------------------------------------------------------------- /pipt/update_schemes/__init__.py: -------------------------------------------------------------------------------- 1 | """Ensemble analysis/conditioning/inversion schemes.""" 2 | # Add path to private repo to access the private methods 3 | # import os 4 | # home = os.path.expanduser("~") # os independent home 5 | # __path__.append(os.path.join(home,'4DSEIS_private/4DSEIS-packages/update_schemes')) 6 | -------------------------------------------------------------------------------- /simulator/README.md: -------------------------------------------------------------------------------- 1 | # Simulator package 2 | 3 | This folder contains simple models intended for testing the PET code. For wrappers for Eclipse, OPM, seismic, and rock physics, and information about how to add wrappers for new simulators, see the repo: 4 | 5 | https://github.com/Python-Ensemble-Toolbox/SimulatorWrap 6 | -------------------------------------------------------------------------------- /pipt/__init__.py: -------------------------------------------------------------------------------- 1 | """Inversion (estimation, data assimilation) 2 | 3 | --8<-- "pipt/README.md" 4 | """ 5 | 6 | __pdoc__ = {} 7 | 8 | # import replacement module and override submodule in pipt 9 | # import sys 10 | # from #replacement_package import #replacement_submodule 11 | # sys.modules["pipt.#module.#submodule"] = #replacement_submodule 12 | -------------------------------------------------------------------------------- /popt/cost_functions/rosenbrock.py: -------------------------------------------------------------------------------- 1 | """Rosenbrock objective function.""" 2 | 3 | 4 | def rosenbrock(state, *args, **kwargs): 5 | """ 6 | Rosenbrock: http://en.wikipedia.org/wiki/Rosenbrock_function 7 | """ 8 | x = state[0]['vector'] 9 | x0 = x[:-1] 10 | x1 = x[1:] 11 | f = sum((1 - x0) ** 2) + 100 * sum((x1 - x0 ** 2) ** 2) 12 | return f 13 | -------------------------------------------------------------------------------- /popt/cost_functions/epf.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def epf(r, c_eq=0, c_iq=0): 4 | r""" 5 | The external penalty function, given as 6 | 7 | $$ 0.5r(\sum_i c_{eq}^2 + \sum(\max(c_{iq},0)^2) $$ 8 | 9 | We assume that the ensemble members are stacked as columns. 10 | """ 11 | 12 | return r*0.5*( np.sum(c_eq**2, axis=0) + np.sum(np.maximum(-c_iq,0)**2, axis=0) ) 13 | -------------------------------------------------------------------------------- /docs/stylesheets/jupyter.css: -------------------------------------------------------------------------------- 1 | /* No need for these margins in the rendered jupyter notebooks */ 2 | .document-is-notebook h1 { margin-bottom: 0; } 3 | .jp-Notebook h2:first-of-type { margin-top: 0; } 4 | 5 | /* stderr is used by tqdm progressbar so should not be red */ 6 | .jupyter-wrapper .jp-RenderedText[data-mime-type="application/vnd.jupyter.stderr"] { 7 | background-color: #cdd6d6ff !important; 8 | } 9 | -------------------------------------------------------------------------------- /docs/javascripts/mathjax.js: -------------------------------------------------------------------------------- 1 | window.MathJax = { 2 | tex: { 3 | inlineMath: [["\\(", "\\)"]], 4 | displayMath: [["\\[", "\\]"]], 5 | processEscapes: true, 6 | processEnvironments: true 7 | }, 8 | options: { 9 | ignoreHtmlClass: ".*|", 10 | processHtmlClass: "arithmatex" 11 | } 12 | }; 13 | 14 | document$.subscribe(() => { 15 | MathJax.startup.output.clearCache() 16 | MathJax.typesetClear() 17 | MathJax.texReset() 18 | MathJax.typesetPromise() 19 | }) 20 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | import pytest 4 | import shutil 5 | 6 | @pytest.fixture(scope="session") 7 | def temp_examples_dir(request, tmp_path_factory): 8 | """Clone PET Examples repo to a temp dir. Return its path.""" 9 | pth = tmp_path_factory.mktemp("temp_dir") 10 | subprocess.run(["git", "clone", "--depth", "1", "--branch", "examples-dev", 11 | "https://github.com/Python-Ensemble-Toolbox/Examples.git", pth], 12 | check=True) 13 | yield pth 14 | shutil.rmtree(str(pth)) 15 | -------------------------------------------------------------------------------- /tests/parser_input.pipt: -------------------------------------------------------------------------------- 1 | # Single input 2 | KEYWORD1 3 | STRING1 4 | 5 | KEYWORD2 6 | 1 7 | 8 | # Multiple input in single row 9 | KEYWORD3 10 | STRING1 STRING2 STRING3 11 | 12 | KEYWORD4 13 | 1 2 3 4 14 | 15 | # Multiple input on multiple rows 16 | KEYWORD5 17 | STRING1 STRING2 18 | STRING3 19 | 20 | KEYWORD6 21 | 1 2 3 22 | 4 5 23 | 24 | # Combining strings and numbers, single row 25 | KEYWORD7 26 | STRING1 1 2 STRING3 27 | 28 | # Combining strings and numbers, multiple rows 29 | KEYWORD8 30 | STRING1 1 2 3 4 STRING2 5 31 | STRING3 6 7 8 32 | 33 | # String without tab 34 | KEYWORD9 35 | STRING1 STRING2 36 | 37 | -------------------------------------------------------------------------------- /docs/overrides/main.html: -------------------------------------------------------------------------------- 1 | 2 | {% extends "base.html" %} 3 | 4 | {% block extrahead %} 5 | 6 | 7 | {% endblock %} 8 | 9 | {% block scripts %} 10 | 11 | {{ super() }} 12 | 13 | {% endblock %} 14 | 15 | {% block content %} 16 | {{ super() }} 17 | 18 | {# 19 | Hello World! 20 | {{ page.url }} 21 | {{ "relative/link" | url }} 22 | {{ "/absolute/link" | url }} 23 | {{ "https://www.google.com/" | url }} 24 | #} 25 | 26 | {% endblock %} 27 | -------------------------------------------------------------------------------- /docs/tutorials/pipt/var.csv: -------------------------------------------------------------------------------- 1 | ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64 2 | ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64 3 | ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64 4 | ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64 5 | ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64 6 | ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64 7 | ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64 8 | ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64 9 | ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64 10 | ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64,ABS,64 11 | -------------------------------------------------------------------------------- /.github/workflows/requirements.txt: -------------------------------------------------------------------------------- 1 | # These are dependencies necessary to build docs. 2 | # Since we don't need compatibility with other dependencies when only producing the docs, 3 | # we can here afford the luxury of pinning them. 4 | jupytext==1.16.4 5 | latexcodec==3.0.0 6 | markdown==3.7 7 | markdown-it-py==3.0.0 8 | mkdocs==1.6.1 9 | mkdocs-autorefs==1.2.0 10 | mkdocs-gen-files==0.5.0 11 | mkdocs-get-deps==0.2.0 12 | mkdocs-glightbox==0.4.0 13 | mkdocs-jupyter==0.25.1 14 | mkdocs-literate-nav==0.6.1 15 | mkdocs-material==9.5.43 16 | mkdocs-material-extensions==1.3.1 17 | mkdocs-section-index==0.3.9 18 | mkdocstrings==0.26.2 19 | mkdocstrings-python==1.12.2 20 | nbclient==0.10.0 21 | nbconvert==7.16.5 22 | nbformat==5.10.4 23 | -------------------------------------------------------------------------------- /popt/README.md: -------------------------------------------------------------------------------- 1 | ### Python Optimization Problem Toolbox (POPT) 2 | 3 | POPT is one part of the PET application. Here we solve the optimization problem. 4 | Currently, the following methods are implemented: 5 | 6 | - EnOpt: The standard ensemble optimization method 7 | - GenOpt: Generalized ensemble optimization (using non-Gaussian distributions) 8 | - SmcOpt: Gradient-free optimization based on sequential Monte Carlo 9 | - LineSearch: Gradient based method satisfying the strong Wolfie conditions 10 | 11 | The gradient and Hessian methods are compatible with SciPy, and can be used as input to scipy.optimize.minimize. 12 | A POPT tutorial is found [here](https://python-ensemble-toolbox.github.io/PET/tutorials/popt/tutorial_popt). 13 | -------------------------------------------------------------------------------- /pipt/pipt_init.py: -------------------------------------------------------------------------------- 1 | """Descriptive description.""" 2 | 3 | # External imports 4 | from fnmatch import filter # to check if wildcard name is in list 5 | from importlib import import_module 6 | 7 | 8 | def init_da(da_input, en_input, sim): 9 | "initialize the ensemble object based on the DA inputs" 10 | 11 | assert len( 12 | da_input['daalg']) == 2, f"Need to input assimilation type and update method, got {da_input['daalg']}" 13 | 14 | da_import = getattr(import_module('pipt.update_schemes.' + 15 | da_input['daalg'][0]), f'{da_input["daalg"][1]}_{da_input["analysis"]}') 16 | 17 | # Init. update scheme class, and get an object of that class 18 | return da_import(da_input, en_input, sim) 19 | -------------------------------------------------------------------------------- /.github/workflows/deploy-docs.yml: -------------------------------------------------------------------------------- 1 | # From https://github.com/mhausenblas/mkdocs-deploy-gh-pages 2 | name: Publish mkDocs via GitHub Pages 3 | # build the documentation whenever there are new commits on main 4 | on: 5 | push: 6 | branches: 7 | - main 8 | # Alternative: only build for tags. 9 | # tags: 10 | # - '*' 11 | 12 | jobs: 13 | build: 14 | name: Deploy MkDocs 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout main 18 | uses: actions/checkout@v2 19 | 20 | - name: Deploy docs 21 | uses: mhausenblas/mkdocs-deploy-gh-pages@master 22 | env: 23 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 24 | CONFIG_FILE: mkdocs.yml 25 | REQUIREMENTS: .github/workflows/requirements.txt 26 | # CUSTOM_DOMAIN: optionaldomain.com 27 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | # This CITATION.cff file was generated with cffinit. 2 | # Visit https://bit.ly/cffinit to generate yours today! 3 | 4 | cff-version: 1.2.0 5 | title: Python Ensemble Toolbox (PET) 6 | message: >- 7 | If you use this software, please cite it using the 8 | metadata from this file. 9 | type: software 10 | authors: 11 | - given-names: Data assimilation and optimization group 12 | family-names: NORCE Energy 13 | email: info@data-assimilation.no 14 | repository-code: 'https://github.com/Python-Ensemble-Toolbox/PET' 15 | abstract: >- 16 | PET is a toolbox for ensemble based Data-Assimilation 17 | developed and maintained by the data-assimilation and 18 | optimization group at NORCE Norwegian Research Centre AS. 19 | keywords: 20 | - Data assimlation 21 | - Optimization 22 | - Ensemble methods 23 | license: GPL-3.0 24 | -------------------------------------------------------------------------------- /docs/javascripts/extra.js: -------------------------------------------------------------------------------- 1 | // https://squidfunk.github.io/mkdocs-material/customization/#additional-javascript 2 | console.log("This message appears on every page") 3 | 4 | // Tag converted notebooks (to facilitate custom styling) 5 | document.addEventListener("DOMContentLoaded", function() { 6 | if (document.querySelector('.jp-Notebook')) { 7 | document.body.classList.add('document-is-notebook'); 8 | } 9 | }); 10 | 11 | // Using the document$ observable from mkdocs-material to get notified of page "reload" also if using `navigation.instant` (SSA) 12 | // https://github.com/danielfrg/mkdocs-jupyter/issues/99#issuecomment-2455307893 13 | document$.subscribe(function() { 14 | // First check if the page contains a notebook-related class 15 | if (document.querySelector('.jp-Notebook')) { 16 | document.querySelector("div.md-sidebar.md-sidebar--primary").remove(); 17 | } 18 | }); 19 | -------------------------------------------------------------------------------- /popt/cost_functions/quadratic.py: -------------------------------------------------------------------------------- 1 | """Descriptive description.""" 2 | 3 | import numpy as np 4 | from popt.cost_functions.epf import epf 5 | 6 | 7 | def quadratic(x, *args, **kwargs): 8 | r"""Quadratic objective function 9 | 10 | $$ f(x) = ||x - b||^2_A $$ 11 | """ 12 | 13 | r = kwargs.get('r', -1) 14 | 15 | x = x[0]['vector'] 16 | dim, ne = x.shape 17 | A = 0.5*np.diag(np.ones(dim)) 18 | b = 1.0*np.ones(dim) 19 | f = np.zeros(ne) 20 | for i in range(ne): 21 | u = x[:, i] - b 22 | f[i] = u.T@A@u 23 | 24 | # check for contraints 25 | if r >= 0: 26 | c_eq = g(x[:, i]) 27 | c_iq = h(x[:, i]) 28 | f[i] += epf(r, c_eq=c_eq, c_iq=c_iq) 29 | 30 | return f 31 | 32 | # Equality constraint saying that sum of x should be equal to dimention + 1 33 | def g(x): 34 | return sum(x) - (x.size + 1) 35 | 36 | # Inequality constrint saying that x_1 should be equal or less than 0 37 | def h(x): 38 | return -x[0] 39 | 40 | -------------------------------------------------------------------------------- /docs/tutorials/pipt/true_data.csv: -------------------------------------------------------------------------------- 1 | 3206.1643,1987.9631,1065.9841,0.13173291,3.8683615e-06,1.1263041e-07,5623.6016,1885.6818,1257.3524 2 | 3266.9717,2012.8158,1047.4318,2.4454727,0.000111172936,2.884899e-06,6115.081,1974.2863,1251.7607 3 | 3289.8975,2104.009,1104.5723,18.57857,0.0014210797,3.2323213e-05,6162.225,2003.3813,1260.366 4 | 3153.4924,2199.685,1172.2677,84.85923,0.01113214,0.00021175515,6067.3267,1989.0044,1247.8177 5 | 2804.8997,2298.32,1227.9575,302.24408,0.06683203,0.0009664455,5910.1406,1955.4485,1228.9327 6 | 2377.446,2370.2407,1263.4275,692.1582,0.32635704,0.0034318906,5794.84,1931.1086,1212.0724 7 | 1959.8289,2413.9614,1283.7205,1123.388,1.2904354,0.01007478,5721.0337,1922.3533,1202.4875 8 | 1585.4658,2429.8596,1294.185,1587.4271,4.191758,0.025624914,5686.736,1923.3035,1197.3948 9 | 1308.9434,2411.0825,1296.4686,2011.7366,11.3020115,0.058260355,5696.123,1937.3766,1200.2947 10 | 1097.5681,2360.5327,1295.3679,2390.4883,25.935778,0.120828636,5716.646,1954.3308,1205.7493 11 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | # From https://github.com/actions/starter-workflows 2 | # 3 | # Potential todos (ref DAPPER) 4 | # - Compute test coverage and submit to coveralls.io 5 | # - Also config for macOS and/or Windows 6 | # - Also config for conda 7 | # - Lint 8 | 9 | name: CI tests 10 | 11 | on: 12 | push: 13 | branches: 14 | - '*' 15 | # - main 16 | # - master 17 | pull_request: 18 | branches: 19 | - '*' 20 | 21 | jobs: 22 | bundled: 23 | 24 | runs-on: ubuntu-latest 25 | strategy: 26 | fail-fast: false 27 | matrix: 28 | python-version: ["3.10", 3.11, 3.12] 29 | 30 | steps: 31 | - uses: actions/checkout@v2 32 | - name: Set up Python ${{ matrix.python-version }} 33 | uses: actions/setup-python@v2 34 | with: 35 | python-version: ${{ matrix.python-version }} 36 | - name: Install dependencies 37 | run: | 38 | python -m pip install --upgrade pip setuptools 39 | python -m pip install -e . 40 | - name: Launch tests 41 | run: | 42 | pytest 43 | -------------------------------------------------------------------------------- /misc/ecl_common.py: -------------------------------------------------------------------------------- 1 | """Common definitions for all import filters 2 | 3 | Examples 4 | -------- 5 | >>> from .ecl_common import Phase, Prop 6 | """ 7 | 8 | 9 | # Enumeration of the phases that can exist in a simulation. 10 | # 11 | # The names are abbreviations and should really be called oleic, aqueous and 12 | # gaseous, if we wanted to be correct and type more. 13 | # 14 | # The value that is returned is the Eclipse moniker, for backwards 15 | # compatibility with code that uses this directly. 16 | Phase = type('Phase', (object,), { 17 | 'oil': 'OIL', 18 | 'wat': 'WAT', 19 | 'gas': 'GAS', 20 | }) 21 | 22 | 23 | # Properties that can be queried from an output file. 24 | # 25 | # The property forms the first part of a "selector", which is a tuple 26 | # containing the necessary information to setup the name of the property 27 | # to read. 28 | Prop = type('Prop', (object,), { 29 | 'pres': 'P', # pressure 30 | 'sat': 'S', # saturation 31 | 'mole': 'x', # mole fraction 32 | 'dens': 'D', # density 33 | 'temp': 'T', # temperature 34 | 'leak': 'L', # leaky well 35 | }) 36 | -------------------------------------------------------------------------------- /docs/tutorials/pipt/Schdl.sch: -------------------------------------------------------------------------------- 1 | 2 | ------------------- WELL SPECIFICATION DATA -------------------------- 3 | WELSPECS 4 | INJ1 G 1 1 2357 WATER 1* STD 1* 1* 1* / 5 | INJ2 G 5 1 2357 WATER 1* STD 1* 1* 1* / 6 | INJ3 G 10 1 2357 WATER 1* STD 1* 1* 1* / 7 | PRO1 G 1 10 2357 WATER 1* STD 1* 1* 1* / 8 | PRO2 G 5 10 2357 WATER 1* STD 1* 1* 1* / 9 | PRO3 G 10 10 2357 WATER 1* STD 1* 1* 1* / 10 | / 11 | 12 | COMPDAT 13 | INJ1 1 1 1 1 OPEN 1* 1* 0.15 1* 5.0 / 14 | INJ1 1 1 2 2 OPEN 1* 1* 0.15 1* 5.0 / 15 | INJ2 5 1 1 1 OPEN 1* 1* 0.15 1* 5.0 / 16 | INJ2 5 1 2 2 OPEN 1* 1* 0.15 1* 5.0 / 17 | INJ3 10 1 1 1 OPEN 1* 1* 0.15 1* 5.0 / 18 | INJ3 10 1 2 2 OPEN 1* 1* 0.15 1* 5.0 / 19 | PRO1 1 10 1 1 OPEN 1* 1* 0.15 1* 5.0 / 20 | PRO1 1 10 2 2 OPEN 1* 1* 0.15 1* 5.0 / 21 | PRO2 5 10 1 1 OPEN 1* 1* 0.15 1* 5.0 / 22 | PRO2 5 10 2 2 OPEN 1* 1* 0.15 1* 5.0 / 23 | PRO3 10 10 1 1 OPEN 1* 1* 0.15 1* 5.0 / 24 | PRO3 10 10 2 2 OPEN 1* 1* 0.15 1* 5.0 / 25 | / 26 | 27 | -------------------------------------------------------------------------------- /docs/tutorials/pipt/3D_ESMDA.toml: -------------------------------------------------------------------------------- 1 | [ensemble] 2 | ne = 50.0 3 | state = "permx" 4 | prior_permx = [["vario", "sph"], ["mean", "priormean.npz"], ["var", 1.0], ["range", 10.0], ["aniso", 1.0], 5 | ["angle", 0.0], ["grid", [10.0, 10.0, 2.0]]] 6 | 7 | [dataassim] 8 | daalg = ["esmda", "esmda"] 9 | analysis = "approx" 10 | energy = 98.0 11 | obsvarsave = "yes" 12 | restartsave = "no" 13 | analysisdebug = ["pred_data", "state", "data_misfit", "prev_data_misfit"] 14 | restart = "no" 15 | obsname = "days" 16 | truedataindex = [400, 800, 1200, 1600, 2000, 2400, 2800, 3200, 3600, 4000] 17 | truedata = "true_data.csv" 18 | assimindex = [0,1,2,3,4,5,6,7,8,9] 19 | datatype = ["WOPR PRO1", "WOPR PRO2", "WOPR PRO3", "WWPR PRO1", "WWPR PRO2", 20 | "WWPR PRO3", "WWIR INJ1", "WWIR INJ2", "WWIR INJ3"] 21 | staticvar = "permx" 22 | datavar = "var.csv" 23 | mda = [ ["tot_assim_steps", 3], ['inflation_param', [2, 4, 4]] ] 24 | 25 | [fwdsim] 26 | reporttype = "days" 27 | reportpoint = [400, 800, 1200, 1600, 2000, 2400, 2800, 3200, 3600, 4000] 28 | replace = "yes" 29 | saveforecast = "yes" 30 | sim_limit = 300.0 31 | rerun = 1 32 | runfile = "runfile" 33 | datatype = ["WOPR PRO1", "WOPR PRO2", "WOPR PRO3", "WWPR PRO1", "WWPR PRO2", 34 | "WWPR PRO3", "WWIR INJ1", "WWIR INJ2", "WWIR INJ3"] 35 | parallel = 4 36 | startdate = "1/1/2022" 37 | -------------------------------------------------------------------------------- /tests/test_linear.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from pathlib import Path, PosixPath 4 | 5 | import numpy as np 6 | import subprocess 7 | 8 | # Logger (since we cannot print during testing) 9 | # -- there is probably a more official way to do this. 10 | logfile = Path.cwd() / "PET-test-log" 11 | with open(logfile, "w") as file: 12 | pass 13 | def prnt(*args, **kwargs): 14 | with open(logfile, "a") as file: 15 | print(*args, **kwargs, file=file) 16 | 17 | 18 | def test_git_clone(temp_examples_dir): 19 | # prnt(cwd) 20 | # prnt(os.listdir(cwd)) 21 | assert (temp_examples_dir / "3Spot").is_dir() 22 | 23 | 24 | def test_mod(temp_examples_dir: PosixPath): 25 | """Validate a few values of the result of the `LinearModel` example.""" 26 | cwd = temp_examples_dir / "LinearModel" 27 | old = Path.cwd() 28 | 29 | try: 30 | os.chdir(cwd) 31 | sys.path.append(str(cwd)) 32 | subprocess.run(["python", "write_true_and_data.py"], cwd=temp_examples_dir) 33 | import run_script 34 | finally: 35 | os.chdir(old) 36 | 37 | result = run_script.assimilation.ensemble.state['permx'].mean(axis=1) 38 | np.testing.assert_array_almost_equal( 39 | result[[1, 2, 3, -3, -2, -1]], 40 | [-0.07294738, 0.00353635, -0.06393236, 0.45394362, 0.44388684, 0.37096157], 41 | decimal=5) 42 | -------------------------------------------------------------------------------- /docs/tutorials/popt/init_optim.toml: -------------------------------------------------------------------------------- 1 | [ensemble] 2 | disable_tqdm = true 3 | ne = 10 4 | state = ["injbhp","prodbhp"] 5 | prior_injbhp = [ 6 | ["mean","init_injbhp.npz"], 7 | ["var",6250.0], 8 | ["limits",100.0,500.0] 9 | ] 10 | prior_prodbhp = [ 11 | ["mean","init_prodbhp.npz"], 12 | ["var",6250.0,], 13 | ["limits",20.0,300.0] 14 | ] 15 | num_models = 1 16 | transform = true 17 | 18 | [optim] 19 | maxiter = 5 20 | tol = 1e-06 21 | alpha = 0.2 22 | beta = 0.1 23 | alpha_maxiter = 3 24 | resample = 0 25 | optimizer = 'GA' 26 | nesterov = true 27 | restartsave = true 28 | restart = false 29 | hessian = false 30 | inflation_factor = 10 31 | savedata = ["alpha","obj_func_values"] 32 | 33 | [fwdsim] 34 | npv_const = [ 35 | ["wop",283.05], 36 | ["wgp",0.0], 37 | ["wwp",37.74], 38 | ["wwi",12.58], 39 | ["disc",0.08], 40 | ["obj_scaling",-1.0e6] 41 | ] 42 | parallel = 2 43 | simoptions = [ 44 | ['mpi', 'mpirun -np 3'], 45 | ['sim_path', '/usr/bin/'], 46 | ['sim_flag', '--tolerance-mb=1e-5 --parsing-strictness=low'] 47 | ] 48 | sim_limit = 5.0 49 | runfile = "3well" 50 | reportpoint = [ 51 | 1994-02-09 00:00:00, 52 | 1995-01-01 00:00:00, 53 | 1996-01-01 00:00:00, 54 | 1997-01-01 00:00:00, 55 | 1998-01-01 00:00:00, 56 | 1999-01-01 00:00:00, 57 | ] 58 | reporttype = "dates" 59 | datatype = ["fopt","fgpt","fwpt","fwit"] 60 | -------------------------------------------------------------------------------- /docs/bib/bib2md.py: -------------------------------------------------------------------------------- 1 | """Render bibtex file as markdown with keys as headings that can be cross-referenced.""" 2 | 3 | from pathlib import Path 4 | 5 | import pybtex.backends.markdown as fmt 6 | from pybtex.database import parse_string 7 | from pybtex.richtext import Tag, Text 8 | from pybtex.style.formatting.unsrt import Style as UnsrtStyle 9 | from pybtex.style.formatting.unsrt import field, sentence 10 | 11 | fmt.SPECIAL_CHARS.remove("[") 12 | fmt.SPECIAL_CHARS.remove("]") 13 | fmt.SPECIAL_CHARS.remove("(") 14 | fmt.SPECIAL_CHARS.remove(")") 15 | fmt.SPECIAL_CHARS.remove("-") 16 | 17 | HERE = Path(__file__).parent 18 | 19 | 20 | class MyStyle(UnsrtStyle): 21 | # default_sorting_style = "author_year_title" 22 | def format_title(self, e, which_field, as_sentence=True): 23 | formatted_title = field( 24 | which_field, 25 | apply_func=lambda text: Tag("tt", Text('"', text.capitalize(), '"')), 26 | ) 27 | if as_sentence: 28 | return sentence[formatted_title] 29 | else: 30 | return formatted_title 31 | 32 | 33 | bib = parse_string((HERE / "refs.bib").read_text(), "bibtex") 34 | 35 | formatted = MyStyle().format_entries(bib.entries.values()) 36 | 37 | md = "" 38 | for entry in formatted: 39 | md += f"### `{entry.key}`\n\n" 40 | md += entry.text.render_as("markdown") + "\n\n" 41 | 42 | (HERE.parent / "references.md").write_text(md) 43 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | EXTRAS = { 4 | "doc": [ 5 | "mkdocs-material", 6 | "mkdocstrings", 7 | "mkdocstrings-python", 8 | "mkdocs-gen-files", 9 | "mkdocs-literate-nav", 10 | "mkdocs-section-index", 11 | "mkdocs-glightbox", 12 | "mkdocs-jupyter", 13 | "pybtex", 14 | ], 15 | } 16 | 17 | setup( 18 | name='PET', 19 | version='1.0', 20 | packages=['pipt', 'popt', 'ensemble', 'simulator', 'input_output', 'misc'], 21 | url='https://github.com/Python-Ensemble-Toolbox/PET', 22 | license_files=('LICENSE.txt',), 23 | author='', 24 | author_email='krfo@norceresearch.no', 25 | description='Python Ensemble Toolbox', 26 | install_requires=[ 27 | 'numpy', 28 | 'scipy', 29 | 'matplotlib', 30 | 'h5py', 31 | 'mako', 32 | 'tqdm', 33 | 'PyWavelets', 34 | 'psutil', 35 | 'geostat @ git+https://github.com/Python-Ensemble-Toolbox/Geostatistics@main', 36 | 'pytest', 37 | 'pandas', # libecalc 8.9.0 has requirement pandas<2,>=1 38 | 'p_tqdm', 39 | 'mat73', 40 | 'opencv-python', 41 | 'rips', 42 | 'tomli', 43 | 'tomli-w', 44 | 'pyyaml', 45 | 'libecalc==8.23.1', # pin version to avoid frequent modifications 46 | 'scikit-learn', 47 | 'pylops' 48 | 49 | ] + EXTRAS['doc'], 50 | ) 51 | -------------------------------------------------------------------------------- /tests/test_quadratic.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from pathlib import Path, PosixPath 4 | 5 | import numpy as np 6 | import subprocess 7 | 8 | # Logger (since we cannot print during testing) 9 | # -- there is probably a more official way to do this. 10 | logfile = Path.cwd() / "PET-test-log" 11 | with open(logfile, "w") as file: 12 | pass 13 | 14 | 15 | def prnt(*args, **kwargs): 16 | with open(logfile, "a") as file: 17 | print(*args, **kwargs, file=file) 18 | 19 | 20 | def test_git_clone(temp_examples_dir): 21 | # prnt(cwd) 22 | # prnt(os.listdir(cwd)) 23 | assert (temp_examples_dir / "Quadratic").is_dir() 24 | 25 | 26 | def test_mod(temp_examples_dir: PosixPath): 27 | """Validate a few values of the result of the `Quadratic` example.""" 28 | cwd = temp_examples_dir / "Quadratic" 29 | old = Path.cwd() 30 | 31 | try: 32 | os.chdir(cwd) 33 | sys.path.append(str(cwd)) 34 | import run_opt 35 | run_opt.main() 36 | files = os.listdir('./') 37 | results = [name for name in files if "optimize_result" in name] 38 | num_iter = len(results) - 1 39 | state = np.load(f'optimize_result_{num_iter}.npz', allow_pickle=True)['x'] 40 | obj = np.load(f'optimize_result_{num_iter}.npz', allow_pickle=True)['obj_func_values'] 41 | finally: 42 | os.chdir(old) 43 | 44 | np.testing.assert_array_almost_equal(state, [0.5, 0.5], decimal=1) 45 | np.testing.assert_array_almost_equal(obj, [0.0], decimal=0) 46 | -------------------------------------------------------------------------------- /input_output/get_ecl_key_val.py: -------------------------------------------------------------------------------- 1 | """Descriptive description.""" 2 | 3 | import numpy as np 4 | import sys 5 | 6 | 7 | def read_file(val_type, filename): 8 | 9 | file = open(filename, 'r') 10 | lines = file.readlines() 11 | key = '' 12 | line_idx = 0 13 | while key != val_type: 14 | line = lines[line_idx] 15 | if not line: 16 | print('Error: Keyword not found') 17 | sys.exit(1) 18 | 19 | line_idx += 1 20 | if len(line): 21 | key = line.split() 22 | if key: 23 | key = key[0] 24 | data = [] 25 | finished = False 26 | while line_idx < len(lines) and not finished: 27 | line = lines[line_idx] 28 | line_idx += 1 29 | if line == '\n' or line[:2] == '--': 30 | continue 31 | if line == '': 32 | break 33 | if line.strip() == '/': 34 | finished = True 35 | sub_str = line.split() 36 | for s in sub_str: 37 | if '*' in s: 38 | num_val = s.split('*') 39 | v = float(num_val[1]) * np.ones(int(num_val[0])) 40 | data.append(v) 41 | elif '/' in s: 42 | finished = True 43 | break 44 | else: 45 | data.append(float(s)) 46 | 47 | values = np.hstack(data) 48 | return values 49 | 50 | def write_file(filename, val_type, data): 51 | 52 | file = open(filename, 'w') 53 | file.writelines(val_type + '\n') 54 | if data.dtype == 'int64': 55 | np.savetxt(file, data, fmt='%i') 56 | else: 57 | np.savetxt(file, data) 58 | file.writelines('/' + '\n') 59 | file.close() 60 | -------------------------------------------------------------------------------- /docs/references.md: -------------------------------------------------------------------------------- 1 | ### `emerick2016a` 2 | 3 | Alexandre A\. Emerick\. 4 | `"Analysis of the performance of ensemble-based assimilation of production and seismic data"`\. 5 | *Journal of Petroleum Science and Engineering*, 139:219–239, 2016\. 6 | 7 | ### `evensen2009a` 8 | 9 | Geir Evensen\. 10 | *Data Assimilation*\. 11 | Springer, 2 edition, 2009\. 12 | 13 | ### `emerick2013a` 14 | 15 | Alexandre A\. Emerick and Albert C\. Reynolds\. 16 | `"Ensemble smoother with multiple data assimilation"`\. 17 | *Computers & Geosciences*, 55:3–15, 2013\. 18 | 19 | ### `rafiee2017` 20 | 21 | Javad Rafiee and Albert C Reynolds\. 22 | `"Theoretical and efficient practical procedures for the generation of inflation factors for es-mda"`\. 23 | *Inverse Problems*, 33(11):115003, 2017\. 24 | 25 | ### `chen2013` 26 | 27 | Yan Chen and Dean S\. Oliver\. 28 | `"Levenberg–Marquardt forms of the iterative ensemble smoother for efficient history matching and uncertainty quantification"`\. 29 | *Computational Geosciences*, 17(4):689–703, 2013\. 30 | 31 | ### `raanes2019` 32 | 33 | Patrick Nima Raanes, Andreas Størksen Stordal, and Geir Evensen\. 34 | `"Revising the stochastic iterative ensemble smoother"`\. 35 | *Nonlinear Processes in Geophysics*, 26(3):325–338, 2019\. 36 | [doi:10\.5194/npg-26-325-2019](https://doi.org/10.5194/npg-26-325-2019)\. 37 | 38 | ### `evensen2019` 39 | 40 | Geir Evensen, Patrick N\. Raanes, Andreas S\. Stordal, and Joakim Hove\. 41 | `"Efficient implementation of an iterative ensemble smoother for data assimilation and reservoir history matching"`\. 42 | *Frontiers in Applied Mathematics and Statistics*, 5:47, 2019\. 43 | [doi:10\.3389/fams\.2019\.00047](https://doi.org/10.3389/fams.2019.00047)\. 44 | 45 | ### `kingma2014` 46 | 47 | Diederik P Kingma\. 48 | `"Adam: a method for stochastic optimization"`\. 49 | *arXiv preprint arXiv:1412\.6980*, 2014\. 50 | 51 | ### `hansen2006` 52 | 53 | Nikolaus Hansen\. 54 | `"The CMA evolution strategy: a comparing review"`\. 55 | *Towards a new evolutionary computation: Advances in the estimation of distribution algorithms*, pages 75–102, 2006\. 56 | 57 | -------------------------------------------------------------------------------- /popt/cost_functions/npv.py: -------------------------------------------------------------------------------- 1 | """Net present value.""" 2 | import numpy as np 3 | 4 | 5 | def npv(pred_data, **kwargs): 6 | """ 7 | Net present value cost function 8 | 9 | Parameters 10 | ---------- 11 | pred_data : array_like 12 | Ensemble of predicted data. 13 | 14 | **kwargs : dict 15 | Other arguments sent to the npv function 16 | 17 | keys_opt : list 18 | Keys with economic data. 19 | 20 | - wop: oil price 21 | - wgp: gas price 22 | - wwp: water production cost 23 | - wwi: water injection cost 24 | - disc: discount factor 25 | - obj_scaling: used to scale the objective function (negative since all methods are minimizers) 26 | 27 | report : list 28 | Report dates. 29 | 30 | Returns 31 | ------- 32 | objective_values : numpy.ndarray 33 | Objective function values (NPV) for all ensemble members. 34 | """ 35 | 36 | # Get the necessary input 37 | keys_opt = kwargs.get('input_dict',{}) 38 | report = kwargs.get('true_order', []) 39 | 40 | # Economic values 41 | npv_const = dict(keys_opt['npv_const']) 42 | 43 | values = [] 44 | for i in np.arange(1, len(pred_data)): 45 | 46 | Qop = np.squeeze(pred_data[i]['fopt']) - np.squeeze(pred_data[i - 1]['fopt']) 47 | Qgp = np.squeeze(pred_data[i]['fgpt']) - np.squeeze(pred_data[i - 1]['fgpt']) 48 | Qwp = np.squeeze(pred_data[i]['fwpt']) - np.squeeze(pred_data[i - 1]['fwpt']) 49 | Qwi = np.squeeze(pred_data[i]['fwit']) - np.squeeze(pred_data[i - 1]['fwit']) 50 | delta_days = (report[1][i] - report[1][0]).days 51 | 52 | val = (Qop * npv_const['wop'] + Qgp * npv_const['wgp'] - Qwp * npv_const['wwp'] - Qwi * npv_const['wwi']) / ( 53 | (1 + npv_const['disc']) ** (delta_days / 365)) 54 | 55 | values.append(val) 56 | 57 | if 'obj_scaling' in npv_const: 58 | return np.array(sum(values)) / npv_const['obj_scaling'] 59 | else: 60 | return np.array(sum(values)) 61 | 62 | -------------------------------------------------------------------------------- /misc/grid/__init__.py: -------------------------------------------------------------------------------- 1 | """\ 2 | Generic read module which determines format from extension. 3 | """ 4 | import logging 5 | import os.path as pth 6 | 7 | 8 | # module specific log; add a null handler so that we won't get an 9 | # error if the main program hasn't set up a log 10 | log = logging.getLogger(__name__) # pylint: disable=invalid-name 11 | log.addHandler(logging.NullHandler()) 12 | 13 | 14 | def read_grid(filename, cache_dir=None): 15 | """ 16 | Read a grid file and optionally cache it for faster future reads. 17 | 18 | Parameters 19 | ---------- 20 | filename : str 21 | Name of the grid file to read, including path. 22 | cache_dir : str 23 | Path to a directory where a cache of the grid may be stored to ensure faster read next time. 24 | """ 25 | # allow shortcut to home directories to be used in paths 26 | fullname = pth.expanduser(filename) 27 | 28 | # split the filename into directory, name and extension 29 | base, ext = pth.splitext(fullname) 30 | 31 | if ext.lower() == '.grdecl': 32 | from misc import grdecl as grdecl 33 | log.info("Reading corner point grid from \"%s\"", fullname) 34 | grid = grdecl.read(fullname) 35 | 36 | elif ext.lower() == '.egrid': 37 | # in case we only have a simulation available, with the binary 38 | # output from the restart, we can read this directly 39 | from misc import ecl as ecl 40 | log.info("Reading binary Eclipse grid from \"%s\"", fullname) 41 | egrid = ecl.EclipseGrid(base) 42 | grid = {'DIMENS': egrid.shape[::-1], 43 | 'COORD': egrid.coord, 44 | 'ZCORN': egrid.zcorn, 45 | 'ACTNUM': egrid.actnum} 46 | 47 | elif ext.lower() == '.pickle': 48 | # direct import of pickled file (should not do this, prefer to read 49 | # it through a cache directory 50 | import pickle 51 | log.info("Reading binary grid dump from \"%s\"", fullname) 52 | with open(fullname, 'rb') as f: 53 | grid = pickle.load(f) 54 | 55 | else: 56 | raise ValueError( 57 | "File format with extension \"{0}\" is unknown".format(ext)) 58 | 59 | return grid 60 | -------------------------------------------------------------------------------- /pipt/update_schemes/update_methods_ns/margIS_update.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.linalg import solve 3 | import copy as cp 4 | from pipt.misc_tools import analysis_tools as at 5 | 6 | class margIS_update(): 7 | 8 | """ 9 | Placeholder for private margIS method 10 | """ 11 | def update(self): 12 | if self.iteration == 1: # method requires some initiallization 13 | self.aug_prior = cp.deepcopy(at.aug_state(self.prior_state, self.list_states)) 14 | self.mean_prior = self.aug_prior.mean(axis=1) 15 | self.X = (self.aug_prior - np.dot(np.resize(self.mean_prior, (len(self.mean_prior), 1)), 16 | np.ones((1, self.ne)))) 17 | self.W = np.eye(self.ne) 18 | self.current_w = np.zeros((self.ne,)) 19 | self.E = np.dot(self.real_obs_data, self.proj) 20 | 21 | M = len(self.real_obs_data) 22 | Ytmp = solve(self.W, self.proj) 23 | if len(self.scale_data.shape) == 1: 24 | Y = np.dot(np.expand_dims(self.scale_data ** (-1), axis=1), np.ones((1, self.ne))) * \ 25 | np.dot(self.aug_pred_data, Ytmp) 26 | else: 27 | Y = solve(self.scale_data, np.dot(self.aug_pred_data, Ytmp)) 28 | 29 | pred_data_mean = np.mean(self.aug_pred_data, 1) 30 | delta_d = (self.obs_data_vector - pred_data_mean) 31 | 32 | if len(self.cov_data.shape) == 1: 33 | S = np.dot(delta_d, (self.cov_data**(-1)) * delta_d) 34 | Ratio = M / S 35 | grad_lklhd = np.dot(Y.T * Ratio, (self.cov_data**(-1)) * delta_d) 36 | grad_prior = (self.ne - 1) * self.current_w 37 | self.C_w = (np.dot(Ratio * Y.T, np.dot(np.diag(self.cov_data ** (-1)), Y)) + (self.ne - 1) * np.eye(self.ne)) 38 | else: 39 | S = np.dot(delta_d, solve(self.cov_data, delta_d)) 40 | Ratio = M / S 41 | grad_lklhd = np.dot(Y.T * Ratio, solve(self.cov_data, delta_d)) 42 | grad_prior = (self.ne - 1) * self.current_w 43 | self.C_w = (np.dot(Ratio * Y.T, solve(self.cov_data, Y)) + (self.ne - 1) * np.eye(self.ne)) 44 | 45 | self.sqrt_w_step = solve(self.C_w, grad_prior + grad_lklhd) 46 | -------------------------------------------------------------------------------- /popt/cost_functions/ren_npv.py: -------------------------------------------------------------------------------- 1 | "Net present value cost function with injection from RENewable energy" 2 | 3 | import numpy as np 4 | 5 | 6 | def ren_npv(pred_data, kwargs): 7 | """ 8 | Net present value cost function with injection from RENewable energy 9 | 10 | Parameters 11 | ---------- 12 | pred_data : ndarray 13 | Ensemble of predicted data. 14 | 15 | **kwargs : dict 16 | Other arguments sent to the npv function 17 | 18 | - keys_opt (list) 19 | Keys with economic data. 20 | 21 | - report (list) 22 | Report dates. 23 | 24 | Returns 25 | ------- 26 | objective_values : ndarray 27 | Objective function values (NPV) for all ensemble members. 28 | """ 29 | 30 | # Get the necessary input 31 | keys_opt = kwargs.get('input_dict', {}) 32 | report = kwargs.get('true_order', []) 33 | 34 | # Economic values 35 | npv_const = dict(keys_opt['npv_const']) 36 | 37 | # Loop over timesteps 38 | values = [] 39 | for i in np.arange(1, len(pred_data)): 40 | 41 | Qop = np.squeeze(pred_data[i]['fopt']) - np.squeeze(pred_data[i - 1]['fopt']) 42 | Qgp = np.squeeze(pred_data[i]['fgpt']) - np.squeeze(pred_data[i - 1]['fgpt']) 43 | Qwp = np.squeeze(pred_data[i]['fwpt']) - np.squeeze(pred_data[i - 1]['fwpt']) 44 | 45 | Qrenwi = [] 46 | Qwi = [] 47 | for key in keys_opt['datatype']: 48 | if 'wwit' in key: 49 | if 'ren' in key: 50 | Qrenwi.append(np.squeeze( 51 | pred_data[i][key]) - np.squeeze(pred_data[i - 1][key])) 52 | else: 53 | Qwi.append(np.squeeze(pred_data[i][key]) - 54 | np.squeeze(pred_data[i - 1][key])) 55 | Qrenwi = np.sum(Qrenwi, axis=0) 56 | Qwi = np.sum(Qwi, axis=0) 57 | 58 | delta_days = (report[1][i] - report[1][0]).days 59 | val = (Qop * npv_const['wop'] + Qgp * npv_const['wgp'] - Qwp * npv_const['wwp'] - Qwi * npv_const['wwi'] - 60 | Qrenwi * npv_const['wrenwi']) / ( 61 | (1 + npv_const['disc']) ** (delta_days / 365)) 62 | 63 | values.append(val) 64 | 65 | if 'obj_scaling' in npv_const: 66 | return sum(values) / npv_const['obj_scaling'] 67 | else: 68 | return sum(values) 69 | 70 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | DEADJOE 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | cover/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | docs/templates/search.js 75 | 76 | # PyBuilder 77 | .pybuilder/ 78 | target/ 79 | 80 | # Jupyter Notebook 81 | .ipynb_checkpoints 82 | 83 | # IPython 84 | profile_default/ 85 | ipython_config.py 86 | 87 | # pyenv 88 | # For a library or package, you might want to ignore these files since the code is 89 | # intended to run in multiple environments; otherwise, check them in: 90 | # .python-version 91 | 92 | # pipenv 93 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 94 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 95 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 96 | # install all needed dependencies. 97 | #Pipfile.lock 98 | 99 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 100 | __pypackages__/ 101 | 102 | # Celery stuff 103 | celerybeat-schedule 104 | celerybeat.pid 105 | 106 | # SageMath parsed files 107 | *.sage.py 108 | 109 | # Environments 110 | .env 111 | .venv 112 | env/ 113 | venv/ 114 | ENV/ 115 | env.bak/ 116 | venv.bak/ 117 | 118 | # Spyder project settings 119 | .spyderproject 120 | .spyproject 121 | 122 | # Rope project settings 123 | .ropeproject 124 | 125 | # mkdocs documentation 126 | /site 127 | 128 | # mypy 129 | .mypy_cache/ 130 | .dmypy.json 131 | dmypy.json 132 | 133 | # Pyre type checker 134 | .pyre/ 135 | 136 | # pytype static type analyzer 137 | .pytype/ 138 | 139 | # Cython debug symbols 140 | cython_debug/ 141 | 142 | # Pycharm 143 | .idea/ 144 | 145 | #documentation 146 | docs-generated/ 147 | 148 | # mergetoolfiles 149 | *.orig 150 | -------------------------------------------------------------------------------- /docs/bib/refs.bib: -------------------------------------------------------------------------------- 1 | @article{emerick2016a, 2 | title={Analysis of the performance of ensemble-based assimilation of production and seismic data}, 3 | author={Emerick, Alexandre A.}, 4 | journal={Journal of Petroleum Science and Engineering}, 5 | volume={139}, 6 | pages={219--239}, 7 | year={2016}, 8 | publisher={Elsevier} 9 | } 10 | 11 | @book{evensen2009a, 12 | title={Data Assimilation}, 13 | author={Evensen, Geir}, 14 | year={2009}, 15 | edition={2}, 16 | pages={307}, 17 | publisher={Springer} 18 | } 19 | 20 | @article{emerick2013a, 21 | title={Ensemble smoother with multiple data assimilation}, 22 | author={Emerick, Alexandre A. and Reynolds, Albert C.}, 23 | journal={Computers \& Geosciences}, 24 | volume={55}, 25 | pages={3--15}, 26 | year={2013}, 27 | publisher={Elsevier} 28 | } 29 | 30 | @article{rafiee2017, 31 | title={Theoretical and efficient practical procedures for the generation of inflation factors for ES-MDA}, 32 | author={Rafiee, Javad and Reynolds, Albert C}, 33 | journal={Inverse Problems}, 34 | volume={33}, 35 | number={11}, 36 | pages={115003}, 37 | year={2017}, 38 | publisher={IOP Publishing} 39 | } 40 | 41 | @article{chen2013, 42 | title={Levenberg--{M}arquardt forms of the iterative ensemble smoother for efficient history matching and uncertainty quantification}, 43 | author={Chen, Yan and Oliver, Dean S.}, 44 | journal={Computational Geosciences}, 45 | volume={17}, 46 | number={4}, 47 | pages={689--703}, 48 | year={2013}, 49 | publisher={Springer} 50 | } 51 | 52 | @article{raanes2019, 53 | title={Revising the stochastic iterative ensemble smoother}, 54 | author={Raanes, Patrick Nima and Stordal, Andreas St{\o}rksen and Evensen, Geir}, 55 | doi={10.5194/npg-26-325-2019}, 56 | journal={Nonlinear Processes in Geophysics}, 57 | volume={26}, 58 | number={3}, 59 | pages={325--338}, 60 | year={2019}, 61 | publisher={Copernicus GmbH} 62 | } 63 | 64 | @article{evensen2019, 65 | title={Efficient implementation of an iterative ensemble smoother for data assimilation and reservoir history matching}, 66 | author={Evensen, Geir and Raanes, Patrick N. and Stordal, Andreas S. and Hove, Joakim}, 67 | doi={10.3389/fams.2019.00047}, 68 | journal={Frontiers in Applied Mathematics and Statistics}, 69 | volume={5}, 70 | pages={47}, 71 | year={2019}, 72 | publisher={Frontiers} 73 | } 74 | 75 | @article{kingma2014, 76 | title={Adam: A method for stochastic optimization}, 77 | author={Kingma, Diederik P}, 78 | journal={arXiv preprint arXiv:1412.6980}, 79 | year={2014} 80 | } 81 | 82 | @article{hansen2006, 83 | title={The {CMA} evolution strategy: a comparing review}, 84 | author={Hansen, Nikolaus}, 85 | journal={Towards a new evolutionary computation: Advances in the estimation of distribution algorithms}, 86 | pages={75--102}, 87 | year={2006}, 88 | publisher={Springer} 89 | } 90 | -------------------------------------------------------------------------------- /docs/tutorials/pipt/RUNFILE.mako: -------------------------------------------------------------------------------- 1 | <%! 2 | import numpy as np 3 | %> 4 | 5 | -- *------------------------------------------* 6 | -- * * 7 | -- * base grid model with input parameters * 8 | -- * * 9 | -- *------------------------------------------* 10 | RUNSPEC 11 | 12 | TITLE 13 | INVERTED 5 SPOT MODEL 14 | 15 | --DIMENS 16 | -- NDIVIX NDIVIY NDIVIZ 17 | -- 60 60 5 / 18 | 19 | --BLACKOIL 20 | OIL 21 | WATER 22 | GAS 23 | DISGAS 24 | 25 | METRIC 26 | 27 | TABDIMS 28 | -- NTSFUN NTPVT NSSFUN NPPVT NTFIP NRPVT NTENDP 29 | 1 1 35 30 5 30 1 / 30 | 31 | EQLDIMS 32 | -- NTEQUL NDRXVD NDPRVD 33 | 1 5 100 / 34 | 35 | WELLDIMS 36 | -- NWMAXZ NCWMAX NGMAXZ MWGMAX 37 | 15 15 2 20 / 38 | 39 | VFPPDIMS 40 | -- MXMFLO MXMTHP MXMWFR MXMGFR MXMALQ NMMVFT 41 | 10 10 10 10 1 1 / 42 | 43 | VFPIDIMS 44 | -- MXSFLO MXSTHP NMSVFT 45 | 10 10 1 / 46 | 47 | AQUDIMS 48 | -- MXNAQN MXNAQC NIFTBL NRIFTB NANAQU NCAMAX 49 | 0 0 1 36 2 200/ 50 | 51 | FAULTDIM 52 | 500 / 53 | 54 | START 55 | 01 JAN 2022 / 56 | 57 | NSTACK 58 | 25 / 59 | 60 | 61 | NOECHO 62 | 63 | GRID 64 | INIT 65 | 66 | INCLUDE 67 | '../Grid.grdecl' / 68 | / 69 | 70 | PERMX 71 | % for i in range(0, len(permx)): 72 | % if permx[i] < 6: 73 | ${"%.3f" %(np.exp(permx[i]))} 74 | % else: 75 | ${"%.3f" %(np.exp(6))} 76 | % endif 77 | % endfor 78 | / 79 | 80 | COPY 81 | 'PERMX' 'PERMY' / 82 | 'PERMX' 'PERMZ' / 83 | / 84 | 85 | 86 | PROPS =============================================================== 87 | 88 | INCLUDE 89 | '../pvt.txt' / 90 | / 91 | 92 | REGIONS =============================================================== 93 | 94 | ENDBOX 95 | 96 | SOLUTION =============================================================== 97 | 98 | 99 | -- DATUM DATUM OWC OWC GOC GOC RSVD RVVD SOLN 100 | -- DEPTH PRESS DEPTH PCOW DEPTH PCOG TABLE TABLE METH 101 | EQUIL 102 | 2355.00 200.46 3000 0.00 2355.0 0.000 / 103 | 104 | 105 | RPTSOL 106 | 'PRES' 'SWAT' / 107 | 108 | RPTRST 109 | BASIC=2 / 110 | 111 | 112 | 113 | SUMMARY ================================================================ 114 | 115 | RUNSUM 116 | 117 | 118 | RPTONLY 119 | 120 | WWIR 121 | 'INJ1' 122 | 'INJ2' 123 | 'INJ3' 124 | / 125 | 126 | WOPR 127 | 'PRO1' 128 | 'PRO2' 129 | 'PRO3' 130 | / 131 | 132 | WWPR 133 | 'PRO1' 134 | 'PRO2' 135 | 'PRO3' 136 | / 137 | 138 | ELAPSED 139 | 140 | SCHEDULE ============================================================= 141 | 142 | 143 | RPTSCHED 144 | 'NEWTON=2' / 145 | 146 | RPTRST 147 | BASIC=2 FIP RPORV / 148 | 149 | ------------------- WELL SPECIFICATION DATA -------------------------- 150 | 151 | INCLUDE 152 | '../Schdl.sch' / 153 | / 154 | 155 | 156 | WCONINJE 157 | 'INJ1' WATER 'OPEN' BHP 2* 300/ 158 | 'INJ2' WATER 'OPEN' BHP 2* 300/ 159 | 'INJ3' WATER 'OPEN' BHP 2* 300/ 160 | / 161 | 162 | WCONPROD 163 | 'PRO1' 'OPEN' BHP 5* 90 / 164 | 'PRO2' 'OPEN' BHP 5* 90 / 165 | 'PRO3' 'OPEN' BHP 5* 90 / 166 | / 167 | --------------------- PRODUCTION SCHEDULE ---------------------------- 168 | 169 | 170 | 171 | TSTEP 172 | 10*400 / 173 | / 174 | 175 | END 176 | -------------------------------------------------------------------------------- /popt/misc_tools/basic_tools.py: -------------------------------------------------------------------------------- 1 | """ 2 | Collection of simple, yet useful Python tools 3 | """ 4 | 5 | 6 | import numpy as np 7 | import sys 8 | 9 | def index2d(list2d, value): 10 | """ 11 | Search in a 2D list for pattern or value and return is (i, j) index. If the 12 | pattern/value is not found, (None, None) is returned 13 | 14 | Examples 15 | -------- 16 | 17 | >>> l = [['string1', 1], ['string2', 2]] 18 | >>> print index2d(l, 'string1') 19 | (0, 0) 20 | 21 | Parameters 22 | ---------- 23 | list2d : list of lists 24 | 2D list. 25 | 26 | value : object 27 | Pattern or value to search for. 28 | 29 | Returns 30 | ------- 31 | ind : tuple 32 | Indices (i, j) of the value. 33 | """ 34 | return next(((i, j) for i, lst in enumerate(list2d) for j, x in enumerate(lst) if x == value), None) 35 | 36 | 37 | def read_file(val_type, filename): 38 | """ 39 | Read an eclipse file with specified keyword. 40 | 41 | Examples 42 | -------- 43 | >>> read_file('PERMX','filename.permx') 44 | 45 | Parameters 46 | ---------- 47 | val_type : 48 | keyword or property 49 | filename : 50 | the file that is read 51 | 52 | Returns 53 | ------- 54 | values : 55 | a vector with values for each cell 56 | """ 57 | 58 | file = open(filename, 'r') 59 | lines = file.readlines() 60 | key = '' 61 | line_idx = 0 62 | while key != val_type: 63 | line = lines[line_idx] 64 | if not line: 65 | print('Error: Keyword not found') 66 | sys.exit(1) 67 | 68 | line_idx += 1 69 | if len(line): 70 | key = line.split() 71 | if key: 72 | key = key[0] 73 | data = [] 74 | finished = False 75 | while line_idx < len(lines) and not finished: 76 | line = lines[line_idx] 77 | line_idx += 1 78 | if line == '\n' or line[:2] == '--': 79 | continue 80 | if line == '': 81 | break 82 | if line.strip() == '/': 83 | finished = True 84 | sub_str = line.split() 85 | for s in sub_str: 86 | if '*' in s: 87 | num_val = s.split('*') 88 | v = float(num_val[1]) * np.ones(int(num_val[0])) 89 | data.append(v) 90 | elif '/' in s: 91 | finished = True 92 | break 93 | else: 94 | data.append(float(s)) 95 | 96 | values = np.hstack(data) 97 | return values 98 | 99 | 100 | def write_file(filename, val_type, data): 101 | """Write an eclipse file with specified keyword. 102 | 103 | Examples 104 | -------- 105 | >>> write_file('filename.permx','PERMX',data_vec) 106 | 107 | Parameters 108 | ---------- 109 | filename : 110 | the file that is read 111 | val_type: 112 | keyword or property 113 | data : 114 | data written to file 115 | """ 116 | 117 | file = open(filename, 'w') 118 | file.writelines(val_type + '\n') 119 | if data.dtype == 'int64': 120 | np.savetxt(file, data, fmt='%i') 121 | else: 122 | np.savetxt(file, data) 123 | file.writelines('/' + '\n') 124 | file.close() 125 | -------------------------------------------------------------------------------- /docs/gen_ref_pages.py: -------------------------------------------------------------------------------- 1 | """Generate reference pages (md) from code (py). 2 | 3 | Based on `https://mkdocstrings.github.io/recipes/` 4 | 5 | Note that the generated markdown files have almost no content, 6 | merely contain a reference to the corresponding `mkdocstring` identifier. 7 | """ 8 | 9 | from pathlib import Path 10 | 11 | import mkdocs_gen_files 12 | 13 | nav = mkdocs_gen_files.Nav() 14 | 15 | root = Path(__file__).parent.parent 16 | 17 | src = root 18 | for path in sorted(src.rglob("*.py")): 19 | 20 | # Skip "venv" and other similarly named directories 21 | if path.name.startswith("venv") or path.name.startswith(".venv") or path.name.startswith(".") or "site-packages" in path.parts: 22 | continue 23 | 24 | # Skip and print "cause" 25 | cause = None 26 | # Skip if read issue 27 | try: 28 | txt = path.read_text() 29 | except UnicodeDecodeError: 30 | cause = f"Warning: Skipping (due to read error) {path.relative_to(root)}" 31 | # Skip files that don't have docstrings (to avoid build abortion) 32 | # More elaborate solution: https://github.com/mkdocstrings/mkdocstrings/discussions/412 33 | if txt and (lines := txt.splitlines()): 34 | for line in lines: 35 | line = line.strip() 36 | if not line or line.startswith("#"): 37 | continue 38 | if not line.startswith(('"', "'")): 39 | cause = f"Warning: Skipping (due to missing docstring) {path.relative_to(root)}" 40 | break 41 | # namespace packages not unsupported. 42 | # Fix? https://github.com/mkdocstrings/mkdocstrings/discussions/563 43 | if path.name != "__init__.py": 44 | parent_has_init = (path.parent / "__init__.py").exists() 45 | if not parent_has_init: 46 | cause = f"Warning: Skipping namespace package file: {path.relative_to(root)}" 47 | if cause: 48 | print(cause) 49 | continue 50 | 51 | parts = tuple(path.relative_to(src).with_suffix("").parts) 52 | path_md = Path("reference", path.relative_to(src).with_suffix(".md")) 53 | 54 | if parts[-1] == "__init__": 55 | # Generate index.md 56 | parts = parts[:-1] # name of parent dir 57 | path_md = path_md.with_name("index.md") 58 | elif parts[0] == "docs": 59 | continue 60 | 61 | # PS: Uncomment (replace `mkdocs_gen_files.open`) to view actual .md files 62 | # path_md = Path("docs", path_md) 63 | # if not path_md.parent.exists(): 64 | # path_md.parent.mkdir(parents=True) 65 | # with open(path_md, "w") as fd: 66 | 67 | with mkdocs_gen_files.open(path_md, "w") as fd: 68 | # Explicitly set the title to avoid mkdocs capitalizing 69 | # names and removing underscores (only applies to files) 70 | print(f"# {parts[-1]}", file=fd) 71 | 72 | identifier = ".".join(parts) 73 | print("::: " + identifier, file=fd) 74 | 75 | mkdocs_gen_files.set_edit_path(path_md, ".." / path.relative_to(root)) 76 | 77 | # > So basically, you can use the literate-nav plugin just for its ability to 78 | # > infer only sub-directories, without ever writing any actual "literate navs". 79 | with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file: 80 | # nav_file.writelines(nav.build_literate_nav()) 81 | nav_file.writelines( 82 | "# Code reference\nUse links in sidebar to navigate the code docstrings.\n" 83 | + "".join(list(nav.build_literate_nav())) 84 | ) 85 | -------------------------------------------------------------------------------- /tests/test_pipt_file_parser.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from input_output.read_config import read_clean_file, remove_empty_lines, parse_keywords 4 | 5 | 6 | class TestPiptInit(unittest.TestCase): 7 | """ 8 | Test core methods in read_txt() which parser .pipt files. 9 | """ 10 | 11 | def setUp(self): 12 | # Read "parser_input.pipt" and parse with core methods in read_txt 13 | lines = read_clean_file('tests/parser_input.pipt') 14 | clean_lines = remove_empty_lines(lines) 15 | self.keys = parse_keywords(clean_lines) 16 | 17 | def test_single_input(self): 18 | # String 19 | self.assertIsInstance(self.keys['keyword1'], str) 20 | self.assertEqual(self.keys['keyword1'], 'string1') 21 | 22 | # Float 23 | self.assertIsInstance(self.keys['keyword2'], float) 24 | self.assertEqual(self.keys['keyword2'], 1.0) 25 | 26 | def test_multiple_input_single_row(self): 27 | # String 28 | self.assertIsInstance(self.keys['keyword3'], list) 29 | self.assertListEqual(self.keys['keyword3'], ['string1', 'string2', 'string3']) 30 | 31 | # Float 32 | self.assertIsInstance(self.keys['keyword4'], list) 33 | self.assertListEqual(self.keys['keyword4'], [1.0, 2.0, 3.0, 4.0]) 34 | 35 | def test_multiple_input_multiple_rows(self): 36 | # String 37 | self.assertIsInstance(self.keys['keyword5'], list) 38 | self.assertIsInstance(self.keys['keyword5'][0], list) 39 | self.assertIsInstance(self.keys['keyword5'][1], list) 40 | self.assertListEqual(self.keys['keyword5'], [['string1', 'string2'], ['string3']]) 41 | 42 | # Float 43 | self.assertIsInstance(self.keys['keyword6'], list) 44 | self.assertIsInstance(self.keys['keyword6'][0], list) 45 | self.assertIsInstance(self.keys['keyword6'][1], list) 46 | self.assertListEqual(self.keys['keyword6'], [[1.0, 2.0, 3.0], [4.0, 5.0]]) 47 | 48 | def test_combinations_single_row(self): 49 | # Combination of strings and floats 50 | self.assertIsInstance(self.keys['keyword7'], list) 51 | self.assertIsInstance(self.keys['keyword7'][0], str) 52 | self.assertIsInstance(self.keys['keyword7'][1], list) 53 | self.assertIsInstance(self.keys['keyword7'][2], str) 54 | 55 | self.assertListEqual(self.keys['keyword7'], ['string1', [1.0, 2.0], 'string3']) 56 | 57 | def test_combinations_multiple_rows(self): 58 | # Combination of strings and floats 59 | self.assertIsInstance(self.keys['keyword8'], list) 60 | self.assertIsInstance(self.keys['keyword8'][0], list) 61 | self.assertIsInstance(self.keys['keyword8'][0][0], str) 62 | self.assertIsInstance(self.keys['keyword8'][0][1], list) 63 | self.assertIsInstance(self.keys['keyword8'][0][2], list) 64 | self.assertIsInstance(self.keys['keyword8'][0][3], str) 65 | self.assertIsInstance(self.keys['keyword8'][0][4], float) 66 | self.assertIsInstance(self.keys['keyword8'][1], list) 67 | self.assertIsInstance(self.keys['keyword8'][1][0], str) 68 | self.assertIsInstance(self.keys['keyword8'][1][1], list) 69 | 70 | self.assertListEqual(self.keys['keyword8'], [['string1', [1.0, 2.0], [3.0, 4.0], 'string2', 5.0], 71 | ['string3', [6.0, 7.0, 8.0]]]) 72 | 73 | def test_string_without_tab(self): 74 | # String with whitespaces instead of \t are parsed as single string 75 | self.assertIsInstance(self.keys['keyword9'], str) 76 | self.assertEqual(self.keys['keyword9'], 'string1 string2') 77 | -------------------------------------------------------------------------------- /pipt/update_schemes/update_methods_ns/full_update.py: -------------------------------------------------------------------------------- 1 | """EnRML (IES) as in 2013.""" 2 | 3 | import numpy as np 4 | from copy import deepcopy 5 | import copy as cp 6 | from scipy.linalg import solve, solve_banded, cholesky, lu_solve, lu_factor, inv 7 | import pickle 8 | import pipt.misc_tools.analysis_tools as at 9 | 10 | 11 | class full_update(): 12 | """ 13 | Full LM Update scheme as defined in "Chen, Y., & Oliver, D. S. (2013). Levenberg–Marquardt forms of the iterative ensemble 14 | smoother for efficient history matching and uncertainty quantification. Computational Geosciences, 17(4), 689–703. 15 | https://doi.org/10.1007/s10596-013-9351-5". Note that for a EnKF or ES update, or for update within GN scheme, lambda = 0. 16 | 17 | !!! note 18 | no localization is implemented for this method yet. 19 | """ 20 | 21 | def ext_Am(self, *args, **kwargs): 22 | """ 23 | The class is initialized by calculating the required Am matrix. 24 | """ 25 | 26 | delta_scaled_prior = self.state_scaling[:, None] * \ 27 | np.dot(at.aug_state(self.prior_state, self.list_states), self.proj) 28 | 29 | u_d, s_d, v_d = np.linalg.svd(delta_scaled_prior, full_matrices=False) 30 | 31 | # remove the last singular value/vector. This is because numpy returns all ne values, while the last is actually 32 | # zero. This part is a good place to include eventual additional truncation. 33 | energy = 0 34 | trunc_index = len(s_d) - 1 # inititallize 35 | for c, elem in enumerate(s_d): 36 | energy += elem 37 | if energy / sum(s_d) >= self.trunc_energy: 38 | trunc_index = c # take the index where all energy is preserved 39 | break 40 | u_d, s_d, v_d = u_d[:, :trunc_index + 41 | 1], s_d[:trunc_index + 1], v_d[:trunc_index + 1, :] 42 | self.Am = np.dot(u_d, np.eye(trunc_index + 1) * 43 | ((s_d ** (-1))[:, None])) # notation from paper 44 | 45 | 46 | def update(self): 47 | 48 | if self.Am is None: 49 | self.ext_Am() # do this only once 50 | 51 | aug_state = at.aug_state(self.current_state, self.list_states) 52 | aug_prior_state = at.aug_state(self.prior_state, self.list_states) 53 | 54 | delta_state = (self.state_scaling**(-1))[:, None]*np.dot(aug_state, self.proj) 55 | 56 | u_d, s_d, v_d = np.linalg.svd(self.pert_preddata, full_matrices=False) 57 | if self.trunc_energy < 1: 58 | ti = (np.cumsum(s_d) / sum(s_d)) <= self.trunc_energy 59 | u_d, s_d, v_d = u_d[:, ti].copy(), s_d[ti].copy(), v_d[ti, :].copy() 60 | 61 | if len(self.scale_data.shape) == 1: 62 | x_1 = np.dot(u_d.T, np.dot(np.expand_dims(self.scale_data ** (-1), axis=1), np.ones((1, self.ne))) * 63 | (self.real_obs_data - self.aug_pred_data)) 64 | else: 65 | x_1 = np.dot(u_d.T, solve(self.scale_data, 66 | (self.real_obs_data - self.aug_pred_data))) 67 | x_2 = solve(((self.lam + 1) * np.eye(len(s_d)) + np.diag(s_d ** 2)), x_1) 68 | x_3 = np.dot(np.dot(v_d.T, np.diag(s_d)), x_2) 69 | delta_m1 = np.dot((self.state_scaling[:, None]*delta_state), x_3) 70 | 71 | x_4 = np.dot(self.Am.T, (self.state_scaling**(-1)) 72 | [:, None]*(aug_state - aug_prior_state)) 73 | x_5 = np.dot(self.Am, x_4) 74 | x_6 = np.dot(delta_state.T, x_5) 75 | x_7 = np.dot(v_d.T, solve( 76 | ((self.lam + 1) * np.eye(len(s_d)) + np.diag(s_d ** 2)), np.dot(v_d, x_6))) 77 | delta_m2 = -np.dot((self.state_scaling[:, None]*delta_state), x_7) 78 | 79 | self.step = delta_m1 + delta_m2 80 | -------------------------------------------------------------------------------- /pipt/update_schemes/update_methods_ns/hybrid_update.py: -------------------------------------------------------------------------------- 1 | """ 2 | ES, and Iterative ES updates with hybrid update matrix calculated from multi-fidelity runs. 3 | """ 4 | 5 | import numpy as np 6 | from scipy.linalg import solve 7 | from pipt.misc_tools import analysis_tools as at 8 | 9 | class hybrid_update: 10 | ''' 11 | Class for hybrid update schemes as described in: Fossum, K., Mannseth, T., & Stordal, A. S. (2020). Assessment of 12 | multilevel ensemble-based data assimilation for reservoir history matching. Computational Geosciences, 24(1), 13 | 217–239. https://doi.org/10.1007/s10596-019-09911-x 14 | 15 | Note that the scheme is slightly modified to be inline with the standard (I)ES approximate update scheme. This 16 | enables the scheme to efficiently be coupled with multiple updating strategies via class MixIn 17 | ''' 18 | 19 | def update(self): 20 | x_3 = [] 21 | pert_state = [] 22 | for l in range(self.tot_level): 23 | aug_state = at.aug_state(self.current_state[l], self.list_states, self.cell_index) 24 | mean_state = np.mean(aug_state, 1) 25 | if 'emp_cov' in self.keys_da and self.keys_da['emp_cov'] == 'yes': 26 | pert_state.append((self.state_scaling**(-1))[:, None] * (aug_state - np.dot(np.resize(mean_state, (len(mean_state), 1)), 27 | np.ones((1, self.ml_ne[l]))))) 28 | else: 29 | pert_state.append((self.state_scaling**(-1) 30 | )[:, None] * np.dot(aug_state, self.proj[l])) 31 | 32 | u_d, s_d, v_d = np.linalg.svd(self.pert_preddata[l], full_matrices=False) 33 | if self.trunc_energy < 1: 34 | ti = (np.cumsum(s_d) / sum(s_d)) <= self.trunc_energy 35 | u_d, s_d, v_d = u_d[:, ti].copy(), s_d[ti].copy(), v_d[ti, :].copy() 36 | 37 | # x_1 = np.dot(u_d.T, solve(self.scale_data[l], 38 | # (self.real_obs_data[l] - self.aug_pred_data[l]))) 39 | 40 | x_2 = solve(((self.lam + 1) * np.eye(len(s_d)) + np.diag(s_d ** 2)), u_d.T) 41 | x_3.append(np.dot(np.dot(v_d.T, np.diag(s_d)), x_2)) 42 | 43 | # Calculate each row of self.step individually to avoid memory issues. 44 | self.step = [np.empty(pert_state[l].shape) for l in range(self.tot_level)] 45 | 46 | # do maximum 1000 rows at a time. 47 | step_size = min(1000, int(self.state_scaling.shape[0]/2)) 48 | row_step = [np.arange(start, start+step_size) for start in 49 | np.arange(0, self.state_scaling.shape[0]-step_size, step_size)] 50 | #add the last rows 51 | row_step.append(np.arange(row_step[-1][-1]+1, self.state_scaling.shape[0])) 52 | 53 | for row in row_step: 54 | kg = sum([self.cov_wgt[indx_l]*np.dot(pert_state[indx_l][row, :], x_3[indx_l]) for indx_l in 55 | range(self.tot_level)]) 56 | for l in range(self.tot_level): 57 | if len(self.scale_data[l].shape) == 1: 58 | self.step[l][row, :] = np.dot(self.state_scaling[row, None] * kg, 59 | np.dot(np.expand_dims(self.scale_data[l] ** (-1), axis=1), 60 | np.ones((1, self.ml_ne[l]))) * 61 | (self.real_obs_data[l] - self.aug_pred_data[l])) 62 | else: 63 | self.step[l][row, :] = np.dot(self.state_scaling[row, None] * kg, solve(self.scale_data[l], 64 | (self.real_obs_data[l] - 65 | self.aug_pred_data[l]))) 66 | -------------------------------------------------------------------------------- /pipt/update_schemes/update_methods_ns/subspace_update.py: -------------------------------------------------------------------------------- 1 | """Stochastic iterative ensemble smoother (IES, i.e. EnRML) with *subspace* implementation.""" 2 | 3 | import numpy as np 4 | from copy import deepcopy 5 | import copy as cp 6 | from scipy.linalg import solve, solve_banded, cholesky, lu_solve, lu_factor, inv 7 | import pickle 8 | import pipt.misc_tools.analysis_tools as at 9 | 10 | 11 | class subspace_update(): 12 | """ 13 | Ensemble subspace update, as described in Raanes, P. N., Stordal, A. S., & 14 | Evensen, G. (2019). Revising the stochastic iterative ensemble smoother. 15 | Nonlinear Processes in Geophysics, 26(3), 325–338. https://doi.org/10.5194/npg-26-325-2019 16 | More information about the method is found in Evensen, G., Raanes, P. N., Stordal, A. S., & Hove, J. (2019). 17 | Efficient Implementation of an Iterative Ensemble Smoother for Data Assimilation and Reservoir History Matching. 18 | Frontiers in Applied Mathematics and Statistics, 5(October), 114. https://doi.org/10.3389/fams.2019.00047 19 | """ 20 | 21 | def update(self): 22 | if self.iteration == 1: # method requires some initiallization 23 | self.current_W = np.zeros((self.ne, self.ne)) 24 | self.E = np.dot(self.real_obs_data, self.proj) 25 | Y = np.dot(self.aug_pred_data, self.proj) 26 | # Y = self.pert_preddata 27 | 28 | omega = np.eye(self.ne) + np.dot(self.current_W, self.proj) 29 | LU = lu_factor(omega.T) 30 | S = lu_solve(LU, Y.T).T 31 | 32 | # scaled_misfit = (self.aug_pred_data - self.real_obs_data) 33 | if len(self.scale_data.shape) == 1: 34 | scaled_misfit = (self.scale_data ** (-1) 35 | )[:, None] * (self.aug_pred_data - self.real_obs_data) 36 | else: 37 | scaled_misfit = solve( 38 | self.scale_data, (self.aug_pred_data - self.real_obs_data)) 39 | 40 | u, s, v = np.linalg.svd(S, full_matrices=False) 41 | if self.trunc_energy < 1: 42 | ti = (np.cumsum(s) / sum(s)) <= self.trunc_energy 43 | if sum(ti) == 0: 44 | # the first singular value contains more than the prescibed trucation energy. 45 | ti[0] = True 46 | u, s, v = u[:, ti].copy(), s[ti].copy(), v[ti, :].copy() 47 | 48 | ps_inv = np.diag([el_s ** (-1) for el_s in s]) 49 | # if 'emp_cov' in self.keys_da and self.keys_da['emp_cov'] == 'yes': 50 | X = np.dot(ps_inv, np.dot(u.T, self.E)) 51 | if len(self.scale_data.shape) == 1: 52 | X = np.dot(ps_inv, np.dot(u.T, (self.scale_data ** (-1))[:, None]*self.E)) 53 | else: 54 | X = np.dot(ps_inv, np.dot(u.T, solve(self.scale_data, self.E))) 55 | Lam, z = np.linalg.eig(np.dot(X, X.T)) 56 | # else: 57 | # X = np.dot(np.dot(ps_inv, np.dot(u.T, np.diag(self.cov_data))),np.dot(u,ps_inv)) 58 | # Lam, z = np.linalg.eig(X) 59 | # Lam = s**2 60 | # z = np.eye(len(s)) 61 | 62 | X2 = np.dot(u, np.dot(ps_inv.T, z)) 63 | X3 = np.dot(S.T, X2) 64 | 65 | # X3_old = np.dot(X2, np.linalg.solve(np.eye(len(Lam)) + np.diag(Lam), X2.T)) 66 | step_m = np.dot(X3, solve(np.eye(len(Lam)) + (1+self.lam) * 67 | np.diag(Lam), np.dot(X3.T, self.current_W))) 68 | 69 | step_d = np.dot(X3, solve(np.eye(len(Lam)) + (1+self.lam) * 70 | np.diag(Lam), np.dot(X2.T, scaled_misfit))) 71 | 72 | # step_d = np.dot(np.linalg.inv(omega).T, np.dot(np.dot(Y.T, X2), 73 | # solve((np.eye(len(Lam)) + (self.lam+1)*np.diag(Lam)), 74 | # np.dot(X2.T, scaled_misfit)))) 75 | self.w_step = -self.current_W/(1+self.lam) - (step_d - step_m/(1+self.lam)) 76 | -------------------------------------------------------------------------------- /pipt/update_schemes/es.py: -------------------------------------------------------------------------------- 1 | """ 2 | ES type schemes 3 | """ 4 | from pipt.update_schemes.enkf import enkf_approx 5 | from pipt.update_schemes.enkf import enkf_full 6 | from pipt.update_schemes.enkf import enkf_subspace 7 | 8 | import numpy as np 9 | from copy import deepcopy 10 | from pipt.misc_tools import analysis_tools as at 11 | 12 | 13 | class esMixIn(): 14 | """ 15 | This is the straightforward ES analysis scheme. We treat this as a all-data-at-once EnKF step, hence the 16 | calc_analysis method here is identical to that in the `enkf` class. Since, for the moment, ASSIMINDEX is parsed in a 17 | specific manner (or more precise, single rows and columns in the PIPT init. file is parsed to a 1D list), a 18 | `Simultaneous` 'loop' had to be implemented, and `es` will use this to do the inversion. Maybe in the future, we can 19 | make the `enkf` class do simultaneous updating also. The consequence of all this is that we inherit BOTH `enkf` and 20 | `Simultaneous` classes, which is convenient. The `Simultaneous` class is inherited to set up the correct inversion 21 | structure and `enkf` is inherited to get `calc_analysis`, so we do not have to implement it again. 22 | """ 23 | 24 | def __init__(self, keys_da, keys_en, sim): 25 | """ 26 | The class is initialized by passing the PIPT init. file upwards in the hierarchy to be read and parsed in 27 | `pipt.input_output.pipt_init.ReadInitFile`. 28 | """ 29 | # Pass init. file to Simultaneous parent class (Python searches parent classes from left to right). 30 | super().__init__(keys_da, keys_en, sim) 31 | 32 | if self.restart is False: 33 | # At the moment, the iterative loop is threated as an iterative smoother an thus we check if assim. indices 34 | # are given as in the Simultaneous loop. 35 | self.check_assimindex_simultaneous() 36 | 37 | # Extract no. assimilation steps from MDA keyword in DATAASSIM part of init. file and set this equal to 38 | # the number of iterations pluss one. Need one additional because the iter=0 is the prior run. 39 | self.max_iter = 2 40 | 41 | def check_convergence(self): 42 | """ 43 | Calculate the "convergence" of the method. Important to 44 | """ 45 | self.prev_data_misfit = self.prior_data_misfit 46 | # only calulate for the final (posterior) estimate 47 | if self.iteration == len(self.keys_da['assimindex']): 48 | assim_index = [self.keys_da['obsname'], list( 49 | np.concatenate(self.keys_da['assimindex']))] 50 | list_datatypes = self.list_datatypes 51 | obs_data_vector, pred_data = at.aug_obs_pred_data(self.obs_data, self.pred_data, assim_index, 52 | list_datatypes) 53 | 54 | data_misfit = at.calc_objectivefun( 55 | self.full_real_obs_data, pred_data, self.full_cov_data) 56 | self.data_misfit = np.mean(data_misfit) 57 | self.data_misfit_std = np.std(data_misfit) 58 | 59 | else: # sequential updates not finished. Misfit is not relevant 60 | self.data_misfit = self.prior_data_misfit 61 | 62 | # Logical variables for conv. criteria 63 | why_stop = {'rel_data_misfit': 1 - (self.data_misfit / self.prev_data_misfit), 64 | 'data_misfit': self.data_misfit, 65 | 'prev_data_misfit': self.prev_data_misfit} 66 | 67 | if self.data_misfit == self.prev_data_misfit: 68 | self.logger.info( 69 | f'ES update {self.iteration} complete!') 70 | self.current_state = deepcopy(self.state) 71 | else: 72 | if self.data_misfit < self.prior_data_misfit: 73 | self.logger.info( 74 | f'ES update complete! Objective function decreased from {self.prior_data_misfit:0.1f} to {self.data_misfit:0.1f}.') 75 | else: 76 | self.logger.info( 77 | f'ES update complete! Objective function increased from {self.prior_data_misfit:0.1f} to {self.data_misfit:0.1f}.') 78 | # Return conv = False, why_stop var. 79 | return False, True, why_stop 80 | 81 | 82 | class es_approx(esMixIn, enkf_approx): 83 | """ 84 | Mixin of ES class and approximate update 85 | """ 86 | pass 87 | 88 | 89 | class es_full(esMixIn, enkf_full): 90 | """ 91 | mixin of ES class and full update. 92 | Note that since we do not iterate there is no difference between is full and approx. 93 | """ 94 | pass 95 | 96 | 97 | class es_subspace(esMixIn, enkf_subspace): 98 | """ 99 | mixin of ES class and subspace update. 100 | """ 101 | pass 102 | -------------------------------------------------------------------------------- /popt/update_schemes/subroutines/cma.py: -------------------------------------------------------------------------------- 1 | """Covariance matrix adaptation (CMA).""" 2 | import numpy as np 3 | from popt.misc_tools import optim_tools as ot 4 | 5 | __all__ = ['CMA'] 6 | 7 | class CMA: 8 | 9 | def __init__(self, ne, dim, alpha_mu=None, n_mu=None, alpha_1=None, alpha_c=None, corr_update=False, equal_weights=True): 10 | ''' 11 | This is a rather simple simple CMA class [`hansen2006`][]. 12 | 13 | Parameters 14 | ---------------------------------------------------------------------------------------------------------- 15 | ne : int 16 | Ensemble size 17 | 18 | dim : int 19 | Dimensions of control vector 20 | 21 | alpha_mu : float 22 | Learning rate for rank-mu update. If None, value proposed in [1] is used. 23 | 24 | n_mu : int, `n_mu < ne` 25 | Number of best samples of ne, to be used for rank-mu update. 26 | Default is int(ne/2). 27 | 28 | alpha_1 : float 29 | Learning rate fro rank-one update. If None, value proposed in [1] is used. 30 | 31 | alpha_c : float 32 | Parameter (inverse if backwards time horizen)for evolution path update 33 | in the rank-one update. See [1] for more info. If None, value proposed in [1] is used. 34 | 35 | corr_update : bool 36 | If True, CMA is used to update a correlation matrix. Default is False. 37 | 38 | equal_weights : bool 39 | If True, all n_mu members are assign equal weighting, `w_i = 1/n_mu`. 40 | If False, the weighting scheme proposed in [1], where `w_i = log(n_mu + 1)-log(i)`, 41 | and normalized such that they sum to one. Defualt is True. 42 | ''' 43 | self.alpha_mu = alpha_mu 44 | self.n_mu = n_mu 45 | self.alpha_1 = alpha_1 46 | self.alpha_c = alpha_c 47 | self.ne = ne 48 | self.dim = dim 49 | self.evo_path = 0 50 | self.corr_update = corr_update 51 | 52 | #If None is given, default values are used 53 | if self.n_mu is None: 54 | self.n_mu = int(self.ne/2) 55 | 56 | if equal_weights: 57 | self.weights = np.ones(self.n_mu)/self.n_mu 58 | else: 59 | self.weights = np.array([np.log(self.n_mu + 1)-np.log(i+1) for i in range(self.n_mu)]) 60 | self.weights = self.weights/np.sum(self.weights) 61 | 62 | self.mu_eff = 1/np.sum(self.weights**2) 63 | self.c_cov = 1/self.mu_eff * 2/(dim+2**0.5)**2 +\ 64 | (1-1/self.mu_eff)*min(1, (2*self.mu_eff-1)/((dim+2)**2+self.mu_eff)) 65 | 66 | if self.alpha_1 is None: 67 | self.alpha_1 = self.c_cov/self.mu_eff 68 | if self.alpha_mu is None: 69 | self.alpha_mu = self.c_cov*(1-1/self.mu_eff) 70 | if self.alpha_c is None: 71 | self.alpha_c = 4/(dim+4) 72 | 73 | def _rank_mu(self, X, J): 74 | ''' 75 | Calculates the rank-mu matrix of CMA-ES. 76 | ''' 77 | index = J.argsort() # lowest (best) to highest (worst) 78 | Xsorted = (X[index[:self.n_mu]] - np.mean(X, axis=0)).T # shape (d, ne) 79 | weights = self.weights 80 | Cmu = (Xsorted*weights)@Xsorted.T 81 | 82 | if self.corr_update: 83 | Cmu = ot.cov2corr(Cmu) 84 | 85 | return Cmu 86 | 87 | def _rank_one(self, step): 88 | ''' 89 | Calculates the rank-one matrix of CMA-ES. 90 | ''' 91 | s = self.alpha_c 92 | self.evo_path = (1-s)*self.evo_path + np.sqrt(s*(2-s)*self.mu_eff)*step 93 | C1 = np.outer(self.evo_path, self.evo_path) 94 | 95 | if self.corr_update: 96 | C1 = ot.cov2corr(C1) 97 | 98 | return C1 99 | 100 | def __call__(self, cov, step, X, J): 101 | ''' 102 | Performs the CMA update. 103 | 104 | Parameters 105 | -------------------------------------------------- 106 | cov : array_like, of shape (d, d) 107 | Current covariance or correlation matrix. 108 | 109 | step : array_like, of shape (d,) 110 | New step of control vector. 111 | Used to update the evolution path. 112 | 113 | X : array_like, of shape (n, d) 114 | Control ensemble of size n. 115 | 116 | J : array_like, of shape (n,) 117 | Objective ensemble of size n. 118 | 119 | Returns 120 | -------------------------------------------------- 121 | out : array_like, of shape (d, d) 122 | CMA updated covariance (correlation) matrix. 123 | ''' 124 | a_mu = self.alpha_mu 125 | a_one = self.alpha_1 126 | C_mu = self._rank_mu(X, J) 127 | C_one = self._rank_one(step) 128 | 129 | cov = (1 - a_one - a_mu)*cov + a_one*C_one + a_mu*C_mu 130 | return cov 131 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PET: Python Ensemble Toolbox 2 | 3 |

4 | 5 |


6 | 7 | PET is a toolbox for ensemble-based Data Assimilation and Optimisation. 8 | It is developed and maintained by the eponymous group 9 | at NORCE Norwegian Research Centre AS. 10 | 11 | [![CI status](https://github.com/Python-Ensemble-Toolbox/PET/actions/workflows/tests.yml/badge.svg)](https://github.com/Python-Ensemble-Toolbox/PET/actions/workflows/tests.yml) 12 | 13 | 14 | ## Installation 15 | 16 | Before installing ensure you have python3 pre-requisites. On a Debian system run: 17 | 18 | ``` 19 | sudo upt-get update 20 | sudo apt-get install python3 21 | sudo apt-get install python3-pip 22 | sudo apt-get install python3-venv 23 | ``` 24 | 25 | To install PET, first clone the repo (assuming you have added the SSH key) 26 | 27 | ```sh 28 | git clone git@github.com:Python-Ensemble-Toolbox/PET.git PET 29 | ``` 30 | 31 | Make sure you have the latest version of `pip` and `setuptools`: 32 | 33 | ```sh 34 | python3 -m pip install --upgrade pip setuptools 35 | ``` 36 | 37 | Optionally (but recommended): Create and activate a virtual environment: 38 | 39 | ```sh 40 | python3 -m venv venv-PET 41 | source venv-PET/bin/activate 42 | ``` 43 | 44 | Some additional features might be not part of your default installation and need to be set in the Python (virtual) environment manually: 45 | 46 | ``` 47 | python3 -m pip install wheel 48 | python3 setup.py bdist_wheel 49 | ``` 50 | 51 | If you do not install PET inside a virtual environment, 52 | you may have to include the `--user` option in the following 53 | (to install to your local Python site packages, usually located in `~/.local`). 54 | 55 | Inside the PET folder, run 56 | 57 | ```sh 58 | python3 -m pip install -e . 59 | ``` 60 | 61 | - The dot is needed to point to the current directory. 62 | - The `-e` option installs PET such that changes to it take effect immediately 63 | (without re-installation). 64 | 65 | ## Examples 66 | 67 | PET needs to be set up with a configuration file. See the example [repository](https://github.com/Python-Ensemble-Toolbox/Examples) for inspiration. 68 | 69 | ## Tutorials 70 | 71 | - A PIPT tutorial is found [here](https://python-ensemble-toolbox.github.io/PET/tutorials/pipt/tutorial_pipt) 72 | - A POPT tutorial is found [here](https://python-ensemble-toolbox.github.io/PET/tutorials/popt/tutorial_popt) 73 | 74 | ## Suggested readings: 75 | 76 | If you use PET in a scientific publication, we would appreciate it if you cited one of the first papers where the PET was introduced. Each of them describes some of the PET's functionalities: 77 | 78 | ### Bayesian data assimilation with EnRML and ES-MDA for History-Matching Workflow with AI-Geomodeling 79 | #### Cite as 80 | Fossum, Kristian, Sergey Alyaev, and Ahmed H. Elsheikh. "Ensemble history-matching workflow using interpretable SPADE-GAN geomodel." First Break 42.2 (2024): 57-63. https://doi.org/10.3997/1365-2397.fb2024014 81 | 82 | ``` 83 | @article{fossum2024ensemble, 84 | title={Ensemble history-matching workflow using interpretable SPADE-GAN geomodel}, 85 | author={Fossum, Kristian and Alyaev, Sergey and Elsheikh, Ahmed H}, 86 | journal={First Break}, 87 | volume={42}, 88 | number={2}, 89 | pages={57--63}, 90 | year={2024}, 91 | publisher={European Association of Geoscientists \& Engineers}, 92 | url = {https://doi.org/10.3997/1365-2397.fb2024014} 93 | } 94 | ``` 95 | 96 | ### Bayesian inversion technique, localization, and data compression for history matching of the Edvard Grieg field using 4D seismic data 97 | #### Cite as 98 | 99 | Lorentzen, R.J., Bhakta, T., Fossum, K. et al. Ensemble-based history matching of the Edvard Grieg field using 4D seismic data. Comput Geosci 28, 129–156 (2024). https://doi.org/10.1007/s10596-024-10275-0 100 | 101 | 102 | ``` 103 | @article{lorentzen2024ensemble, 104 | title={Ensemble-based history matching of the Edvard Grieg field using 4D seismic data}, 105 | author={Lorentzen, Rolf J and Bhakta, Tuhin and Fossum, Kristian and Haugen, Jon Andr{\'e} and Lie, Espen Oen and Ndingwan, Abel Onana and Straith, Knut Richard}, 106 | journal={Computational Geosciences}, 107 | volume={28}, 108 | number={1}, 109 | pages={129--156}, 110 | year={2024}, 111 | publisher={Springer}, 112 | url={https://doi.org/10.1007/s10596-024-10275-0} 113 | } 114 | ``` 115 | 116 | ### Offshore wind farm layout optimization using ensemble methods 117 | #### Cite as 118 | 119 | Eikrem, K.S., Lorentzen, R.J., Faria, R. et al. Offshore wind farm layout optimization using ensemble methods. Renewable Energy 216, 119061 (2023). https://www.sciencedirect.com/science/article/pii/S0960148123009758 120 | 121 | ``` 122 | @article{Eikrem2023offshore, 123 | title = {Offshore wind farm layout optimization using ensemble methods}, 124 | journal = {Renewable Energy}, 125 | volume = {216}, 126 | pages = {119061}, 127 | year = {2023}, 128 | issn = {0960-1481}, 129 | doi = {https://doi.org/10.1016/j.renene.2023.119061}, 130 | url = {https://www.sciencedirect.com/science/article/pii/S0960148123009758}, 131 | author = {Kjersti Solberg Eikrem and Rolf Johan Lorentzen and Ricardo Faria and Andreas St{\o}rksen Stordal and Alexandre Godard}, 132 | keywords = {Wind farm layout optimization, Ensemble optimization (EnOpt and EPF-EnOpt), Constrained optimization, Levelized cost of energy (LCOE), Floating offshore wind}, 133 | } 134 | ``` 135 | -------------------------------------------------------------------------------- /docs/dev_guide.md: -------------------------------------------------------------------------------- 1 | # Developer guide 2 | 3 | ## Writing documentation 4 | 5 | The documentation is built with `mkdocs`. 6 | 7 | - It should be written in [the syntax of markdown](https://www.markdownguide.org/cheat-sheet/). 8 | - The syntax is further augmented by [several pymdown plugins](https://squidfunk.github.io/mkdocs-material/reference/). 9 | - **Docstrings** are processed as above, but should also 10 | declare parameters and return values in the [style of numpy](https://mkdocstrings.github.io/griffe/reference/docstrings/#numpydoc-style), 11 | and `>>>` markers must follow the "Examples" section. 12 | 13 | !!! note 14 | You can preview the rendered html docs by running 15 | ```sh 16 | mkdocs serve 17 | ``` 18 | 19 | - Temporarily disable `mkdocs-jupyter` in `mkdocs.yml` to speed up build reloads. 20 | - Set `validation: unrecognized_links: warn` to get warnings about linking issues. 21 | 22 | A summary of how to add cross-reference links is given below. 23 | 24 | ### Linking to pages 25 | 26 | You should use relative page links, including the `.md` extension. 27 | For example, `[link label](sibling-page.md)`. 28 | 29 | The following works, but does not get validated! `[link label](../sibling-page)` 30 | 31 | !!! hint "Why not absolute links?" 32 | 33 | The downside of relative links is that if you move/rename source **or** destination, 34 | then they will need to be changed, whereas only the destination needs be watched 35 | when using absolute links. 36 | 37 | Previously, absolute links were not officially supported by MkDocs, meaning "not modified at all". 38 | Thus, if made like so `[label](/PET/references)`, 39 | i.e. without `.md` and including `/PET`, 40 | then they would **work** (locally with `mkdocs serve` and with GitHub hosting). 41 | Since [#3485](https://github.com/mkdocs/mkdocs/pull/3485) you can instead use `[label](/references)` 42 | i.e. omitting `PET` (or whatever domain sub-dir is applied in `site_url`) 43 | by setting `mkdocs.yml: validation: absolute_links: relative_to_docs`. 44 | A different workaround is the [`mkdocs-site-url` plugin](https://github.com/OctoPrint/mkdocs-site-urls). 45 | 46 | !!! tip "Either way" 47 | It will not be link that your editor can follow to the relevant markdown file 48 | (unless you create a symlink in your file system root?) 49 | nor will GitHub's internal markdown rendering manage to make sense of it, 50 | so my advise is not to use absolute links. 51 | 52 | ### Linking to headers/anchors 53 | 54 | Thanks to the `autorefs` plugin, 55 | links to **headings** (including page titles) don't even require specifying the page path! 56 | Syntax: `[visible label][link]` i.e. double pairs of _brackets_. Shorthand: `[link][]`. 57 | !!! info 58 | - Clearly, non-unique headings risk being confused with others in this way. 59 | - The link (anchor) must be lowercase! 60 | 61 | This facilitates linking to 62 | 63 | - **API (code reference)** items. 64 | For example, ``[`da_methods.ensemble`][]``, 65 | where the backticks are optional (makes the link _look_ like a code reference). 66 | - **References**. For example ``[`bocquet2016`][]``, 67 | 68 | ### Docstring injection 69 | 70 | Use the following syntax to inject the docstring of a code object. 71 | 72 | ```markdown 73 | ::: da_methods.ensemble 74 | ``` 75 | 76 | But we generally don't do so manually. 77 | Instead it's taken care of by the reference generation via `docs/gen_ref_pages.py`. 78 | 79 | ### Including other files 80 | 81 | The `pymdown` extension ["snippets"](https://facelessuser.github.io/pymdown-extensions/extensions/snippets/#snippets-notation) 82 | enables the following syntax to include text from other files. 83 | 84 | `--8<-- "/path/from/project/root/filename.ext"` 85 | 86 | ### Adding to the examples 87 | 88 | Example scripts are very useful, and contributions are very desirable. As well 89 | as showcasing some feature, new examples should make sure to reproduce some 90 | published literature results. After making the example, consider converting 91 | the script to the Jupyter notebook format (or vice versa) so that the example 92 | can be run on Colab without users needing to install anything (see 93 | `docs/examples/README.md`). This should be done using the `jupytext` plug-in (with 94 | the `lightscript` format), so that the paired files can be kept in synch. 95 | 96 | ### Bibliography 97 | 98 | In order to add new references, 99 | insert their bibtex into `docs/bib/refs.bib`, 100 | then run `docs/bib/bib2md.py` 101 | which will format and add entries to `docs/references.md` 102 | that can be cited with regular cross-reference syntax, e.g. `[bocquet2010a][]`. 103 | 104 | ### Hosting 105 | 106 | The above command is run by a GitHub Actions workflow whenever 107 | the `master` branch gets updated. 108 | The `gh-pages` branch is no longer being used. 109 | Instead [actions/deploy-pages](https://github.com/actions/deploy-pages) 110 | creates an artefact that is deployed to Github Pages. 111 | 112 | ## Tests 113 | 114 | The test suite is orchestrated using `pytest`. Both in **CI** and locally. 115 | I.e. you can run the tests simply by the command 116 | 117 | ```sh 118 | pytest 119 | ``` 120 | 121 | It will discover all [appropriately named tests](https://docs.pytest.org) 122 | in the source (see the `tests` dir). 123 | 124 | Use (for example) `pytest --doctest-modules some_file.py` to 125 | *also* run any example code **within** docstrings. 126 | 127 | We should also soon make use of a config file (for example `pyproject.toml`) for `pytest`. 128 | -------------------------------------------------------------------------------- /misc/grid/unstruct.py: -------------------------------------------------------------------------------- 1 | """ 2 | Convert cornerpoint grids to unstructured grids. 3 | 4 | Examples 5 | -------- 6 | >>> import pyresito.grid.unstruct as us 7 | >>> import pyresito.io.grdecl as grdecl 8 | >>> g = grdecl.read('~/proj/cmgtools/bld/overlap.grdecl') 9 | """ 10 | # pylint: disable=too-few-public-methods, multiple-statements 11 | import numpy as np 12 | 13 | 14 | class Ridge (object): 15 | """A ridge consists of two points, anchored in each their pillar. We only 16 | need to store the z-values, because the x- and y- values are determined by 17 | the pillar themselves. 18 | """ 19 | __slots__ = ['left', 'right'] 20 | 21 | def __init__(self, left, right): 22 | self.left = left 23 | self.right = right 24 | 25 | def is_not_below(self, other): 26 | """ 27 | Weak ordering of ridges based on vertical placement. 28 | 29 | Parameters 30 | ---------- 31 | other : Ridge 32 | Ridge to be compared to this object. 33 | 34 | Returns 35 | ------- 36 | bool or None 37 | True if no point on self is below any on the other, 38 | None if the ridges cross, and False if there is a point 39 | on the other ridge that is above any on self. 40 | """ 41 | # test each side separately. if self is at the same level as other, 42 | # this should count positively towards the test (i.e. it is regarded 43 | # as "at least as high"), so use less-or-equal. 44 | left_above = other.left <= self.left 45 | right_above = other.right <= self.right 46 | 47 | # if both sides are at least as high, then self is regarded as at 48 | # least as high as other, and vice versa. if one side is lower and 49 | # one side is higher, then the ridges cross. 50 | if left_above: 51 | return True if right_above else None 52 | else: 53 | return None if right_above else False 54 | 55 | 56 | class Face (object): 57 | """A (vertical) face consists of two ridges, because all of the faces in a 58 | hexahedron can be seen as (possibly degenerate) quadrilaterals. 59 | """ 60 | __slots__ = ['top', 'btm'] 61 | 62 | def __init__(self, top, btm): 63 | self.top = top 64 | self.btm = btm 65 | 66 | def is_above(self, other): 67 | """ 68 | Weak ordering of faces based on vertical placement. 69 | 70 | Parameters 71 | ---------- 72 | other : Face 73 | Face to be compared to this object. 74 | 75 | Returns 76 | ------- 77 | bool 78 | True if all points in face self are above all points in face other, False otherwise. 79 | """ 80 | # if the bottom of self is aligned with the top of other, then the 81 | # face itself is considered to be above. since the ridge test also 82 | # has an indeterminate result, we test explicitly like this 83 | return True if self.btm.is_not_below(other.top) else False 84 | 85 | 86 | def conv(grid): 87 | """ 88 | Convert a cornerpoint grid to an unstructured grid. 89 | 90 | Parameters 91 | ---------- 92 | grid : dict 93 | Cornerpoint grid to be converted. Should contain 'COORD', 'ZCORN', 'ACTNUM'. 94 | 95 | Returns 96 | ------- 97 | dict 98 | Unstructured grid. 99 | """ 100 | # extract the properties of interest from the cornerpoint grid 101 | zcorn = grid['ZCORN'] 102 | actnum = grid['ACTNUM'] 103 | ni, nj, nk = grid['DIMENS'] 104 | 105 | # zcorn has now dimensionality (k, b, j, f, i, r) and actnum is (k, j, i); 106 | # permute the cubes to get the (b, k)-dimensions varying quickest, to avoid 107 | # page faults when we move along pillars/columns 108 | zcorn = np.transpose(zcorn, axes=[2, 3, 4, 5, 1, 0]) 109 | actnum = np.transpose(actnum, axes=[1, 2, 0]) 110 | 111 | # memory allocation: number of unique cornerpoints along each pillar, and 112 | # the index of each cornerpoint into the global list of vertices 113 | num_cp = np.empty((nj + 1, ni + 1), dtype=np.int32) 114 | ndx_cp = np.empty((nk, 2, nj, 2, ni, 2), dtype=np.int32) 115 | 116 | # each pillar is connected to at most 2*2 columns (front/back, right/left), 117 | # and each column has at most 2*nk (one top and one bottom) corners 118 | corn_z = np.empty((2, 2, 2, nk), dtype=np.float32) 119 | corn_i = np.empty((2, 2, 2, nk), dtype=np.int32) 120 | corn_j = np.empty((2, 2, 2, nk), dtype=np.int32) 121 | corn_a = np.empty((2, 2, 2, nk), dtype=np.bool) 122 | 123 | # get all unique points that are hinged to a certain pillar (p, q) 124 | for q, p in np.ndindex((nj + 1, ni + 1)): 125 | # reset number of corners found for this column 126 | num_corn_z = 0 127 | 128 | for f, r in np.ndindex((2, 2)): # front/back, right/left 129 | # calculate the cell index of the column at this position 130 | j = q - f 131 | i = p - r 132 | 133 | for b in range(2): # bottom/top 134 | 135 | # copy depth values for this corner; notice that we have 136 | # pivoted the zcorn matrix so that values going upwards are in 137 | # last dim. 138 | corn_z[f, r, b, :] = zcorn[j, f, i, r, b, :] 139 | 140 | # same for active numbers, but this is reused for top/bottom; 141 | # if the cell is inactive, then both top and bottom point are. 142 | corn_a[f, r, b, :] = actnum[j, i, :] 143 | 144 | # save the original indices into these auxiliary arrays so that 145 | # we can figure out where each point came from after they are 146 | # sorted 147 | corn_i[f, r, b] = i 148 | corn_j[f, r, b] = j 149 | -------------------------------------------------------------------------------- /docs/tutorials/popt/3WELL.mako: -------------------------------------------------------------------------------- 1 | <%! 2 | import numpy as np 3 | import datetime as dt 4 | %> 5 | -- *------------------------------------------* 6 | -- * * 7 | -- * base grid model with input parameters * 8 | -- * * 9 | -- *------------------------------------------* 10 | RUNSPEC 11 | 12 | TITLE 13 | 3 WELL MODEL 14 | 15 | DIMENS 16 | -- NDIVIX NDIVIY NDIVIZ 17 | 100 100 1 / 18 | 19 | -- Gradient option 20 | -- AJGRADNT 21 | 22 | -- Gradients readeable 23 | -- UNCODHMD 24 | 25 | --BLACKOIL 26 | OIL 27 | WATER 28 | 29 | METRIC 30 | 31 | TABDIMS 32 | -- NTSFUN NTPVT NSSFUN NPPVT NTFIP NRPVT NTENDP 33 | 1 1 35 30 5 30 1 / 34 | 35 | EQLDIMS 36 | -- NTEQUL NDRXVD NDPRVD 37 | 1 5 100 / 38 | 39 | WELLDIMS 40 | -- NWMAXZ NCWMAX NGMAXZ MWGMAX 41 | 10 1 2 20 / 42 | 43 | VFPPDIMS 44 | -- MXMFLO MXMTHP MXMWFR MXMGFR MXMALQ NMMVFT 45 | 10 10 10 10 1 1 / 46 | 47 | VFPIDIMS 48 | -- MXSFLO MXSTHP NMSVFT 49 | 10 10 1 / 50 | 51 | AQUDIMS 52 | -- MXNAQN MXNAQC NIFTBL NRIFTB NANAQU NCAMAX 53 | 0 0 1 36 2 200/ 54 | 55 | START 56 | 09 FEB 1994 / 57 | 58 | NSTACK 59 | 25 / 60 | 61 | NOECHO 62 | 63 | GRID 64 | INIT 65 | 66 | INCLUDE 67 | '../TRUEPERMX.INC' 68 | / 69 | 70 | 71 | COPY 72 | 'PERMX' 'PERMY' / 73 | 'PERMX' 'PERMZ' / 74 | / 75 | 76 | DX 77 | 10000*10 / 78 | DY 79 | 10000*10 / 80 | DZ 81 | 10000*10 / 82 | 83 | TOPS 84 | 10000*2355 / 85 | 86 | PORO 87 | 10000*0.18 / 88 | 89 | 90 | PROPS =============================================================== 91 | 92 | -- Two-phase (water-oil) rel perm curves 93 | -- Sw Krw Kro Pcow 94 | SWOF 95 | 0.1500 0.0 1.0000 0.0 96 | 0.2000 0.0059 0.8521 0.0 97 | 0.2500 0.0237 0.7160 0.0 98 | 0.3000 0.0533 0.5917 0.0 99 | 0.3500 0.0947 0.4793 0.0 100 | 0.4000 0.1479 0.3787 0.0 101 | 0.4500 0.2130 0.2899 0.0 102 | 0.5000 0.2899 0.2130 0.0 103 | 0.5500 0.3787 0.1479 0.0 104 | 0.6000 0.4793 0.0947 0.0 105 | 0.6500 0.5917 0.0533 0.0 106 | 0.7000 0.7160 0.0237 0.0 107 | 0.7500 0.8521 0.0059 0.0 108 | 0.8000 1.0000 0.0 0.0 109 | / 110 | 111 | --PVCDO 112 | -- REF.PRES. FVF COMPRESSIBILITY REF.VISC. VISCOSIBILITY 113 | -- 234 1.065 6.65e-5 5.0 1.9e-3 / 114 | 115 | -- In a e300 run we must use PVDO 116 | PVDO 117 | 220 1.065 5.0 118 | 240 1.06499 5.0 / 119 | 120 | DENSITY 121 | 912.0 1000.0 0.8266 122 | / 123 | 124 | PVTW 125 | 234.46 1.0042 5.43E-05 0.5 1.11E-04 / 126 | 127 | 128 | -- ROCK COMPRESSIBILITY 129 | -- 130 | -- REF. PRES COMPRESSIBILITY 131 | ROCK 132 | 235 0.00045 / 133 | 134 | 135 | 136 | REGIONS =============================================================== 137 | 138 | ENDBOX 139 | 140 | SOLUTION =============================================================== 141 | 142 | 143 | -- DATUM DATUM OWC OWC GOC GOC RSVD RVVD SOLN 144 | -- DEPTH PRESS DEPTH PCOW DEPTH PCOG TABLE TABLE METH 145 | EQUIL 146 | 2355.00 200.46 3000 0.00 2355.0 0.000 0 0 / 147 | 148 | 149 | RPTSOL 150 | 'PRES' 'SWAT' / 151 | 152 | RPTRST 153 | BASIC=2 / 154 | 155 | 156 | 157 | SUMMARY ================================================================ 158 | 159 | RUNSUM 160 | 161 | EXCEL 162 | 163 | --RPTONLY 164 | FOPT 165 | FGPT 166 | FWPT 167 | FWIT 168 | 169 | WWIR 170 | 'INJ-1' 171 | / 172 | 173 | WOPR 174 | 'PRO-1' 175 | / 176 | 177 | WWPR 178 | 'PRO-1' 179 | / 180 | 181 | SCHEDULE ============================================================= 182 | 183 | 184 | RPTSCHED 185 | 'NEWTON=2' / 186 | 187 | RPTRST 188 | BASIC=2 / 189 | 190 | -- AJGWELLS 191 | -- 'INJ-1' 'WWIR' / 192 | -- 'PRO-1' 'WLPR' / 193 | --/ 194 | 195 | -- AJGPARAM 196 | -- 'PERMX' 'PORO' / 197 | 198 | ------------------- WELL SPECIFICATION DATA -------------------------- 199 | WELSPECS 200 | 'INJ-1' 'G' 1 1 2357 WATER 1* 'STD' 3* / 201 | 'INJ-2' 'G' 100 1 2357 WATER 1* 'STD' 3* / 202 | 'PRO-1' 'G' 100 100 2357 OIL 1* 'STD' 3* / 203 | / 204 | COMPDAT 205 | -- RADIUS SKIN 206 | 'INJ-1' 1 1 1 1 'OPEN' 2* 0.15 1* 5.0 / 207 | 'INJ-2' 100 1 1 1 'OPEN' 2* 0.15 1* 5.0 / 208 | 'PRO-1' 100 100 1 1 'OPEN' 2* 0.15 1* 5.0 / 209 | / 210 | 211 | WCONINJE 212 | --'INJ-1' WATER 'OPEN' BHP 2* 300 / 213 | --'INJ-1' WATER 'OPEN' BHP 2* 250 / 214 | 'INJ-1' WATER 'OPEN' BHP 2* ${injbhp[0]} / 215 | 'INJ-2' WATER 'OPEN' BHP 2* ${injbhp[1]} / 216 | / 217 | 218 | WCONPROD 219 | --'PRO-1' 'OPEN' BHP 5* 100 / 220 | 'PRO-1' 'OPEN' BHP 5* ${prodbhp[0]} / 221 | / 222 | 223 | 224 | --------------------- PRODUCTION SCHEDULE ---------------------------- 225 | 226 | 227 | 228 | DATES 229 | 1 JAN 1995 / 230 | / 231 | 232 | DATES -- Generated : Petrel 233 | 1 JAN 1996 / 234 | / 235 | 236 | DATES -- Generated : Petrel 237 | 1 JAN 1997 / 238 | / 239 | 240 | DATES -- Generated : Petrel 241 | 1 JAN 1998 / 242 | / 243 | 244 | DATES -- Generated : Petrel 245 | 1 JAN 1999 / 246 | / 247 | 248 | 249 | 250 | -------------------------------------------------------------------------------- /popt/cost_functions/ren_npv_co2.py: -------------------------------------------------------------------------------- 1 | ''' Net Present Value with Renewable Power and co2 emissions ''' 2 | 3 | import pandas as pd 4 | import numpy as np 5 | import os 6 | import yaml 7 | 8 | from pathlib import Path 9 | from pqdm.processes import pqdm 10 | 11 | __all__ = ['ren_npv_co2'] 12 | 13 | HERE = Path().cwd() # fallback for ipynb's 14 | HERE = HERE.resolve() 15 | 16 | def ren_npv_co2(pred_data, keys_opt, report, save_emissions=False): 17 | ''' 18 | Net Present Value with Renewable Power and co2 emissions (with eCalc) 19 | 20 | Parameters 21 | ---------- 22 | pred_data : array_like 23 | Ensemble of predicted data. 24 | 25 | keys_opt : list 26 | Keys with economic data. 27 | 28 | report : list 29 | Report dates. 30 | 31 | Returns 32 | ------- 33 | objective_values : array_like 34 | Objective function values (NPV) for all ensemble members. 35 | ''' 36 | 37 | # some globals, for pqdm 38 | global const 39 | global kwargs 40 | global report_dates 41 | global sim_data 42 | 43 | # define a data getter 44 | get_data = lambda i, key: pred_data[i+1][key].squeeze() - pred_data[i][key].squeeze() 45 | 46 | # ensemble size (ne), number of report-dates (nt) 47 | nt = len(pred_data) 48 | try: 49 | ne = len(get_data(1,'fopt')) 50 | except: 51 | ne = 1 52 | 53 | np.save('co2_emissions', np.zeros((ne, nt-1))) 54 | 55 | # Economic and other constatns 56 | const = dict(keys_opt['npv_const']) 57 | kwargs = dict(keys_opt['npv_kwargs']) 58 | report_dates = report[1] 59 | 60 | # Load energy arrays. These arrays contain the excess windpower used for gas compression, 61 | # and the energy from gas which is used in the water intection. 62 | power_arrays = np.load(kwargs['power']+'.npz') 63 | 64 | sim_data = {'fopt': np.zeros((ne, nt-1)), 65 | 'fgpt': np.zeros((ne, nt-1)), 66 | 'fwpt': np.zeros((ne, nt-1)), 67 | 'fwit': np.zeros((ne, nt-1)), 68 | 'thp' : np.zeros((ne, nt-1)), 69 | 'days': np.zeros(nt-1), 70 | 'wind': power_arrays['wind'][:,:-1]} 71 | 72 | # loop over pred_data 73 | for t in range(nt-1): 74 | 75 | for datatype in ['fopt', 'fgpt', 'fwpt', 'fwit']: 76 | sim_data[datatype][:,t] = get_data(t, datatype) 77 | 78 | # days in time-step 79 | sim_data['days'][t] = (report_dates[t+1] - report_dates[t]).days 80 | 81 | # get maximum well head pressure (for each ensemble member) 82 | thp_keys = [k for k in keys_opt['datatype'] if 'wthp' in k] # assume only injection wells 83 | thp_vals = [] 84 | for key in thp_keys: 85 | thp_vals.append(pred_data[t][key].squeeze()) 86 | 87 | sim_data['thp'][:,t] = np.max(np.array(thp_vals), axis=0) 88 | 89 | # calculate NPV values 90 | npv_values = pqdm(array=range(ne), function=npv, n_jobs=keys_opt['parallel'], disable=True) 91 | 92 | if not save_emissions: 93 | os.remove('co2_emissions.npy') 94 | 95 | # clear energy arrays 96 | np.savez(kwargs['power']+'.npz', wind=np.zeros((ne, nt)), ren=np.zeros((ne,nt)), gas=np.zeros((ne,nt))) 97 | 98 | scaling = 1.0 99 | if 'obj_scaling' in const: 100 | scaling = const['obj_scaling'] 101 | 102 | return np.asarray(npv_values)/scaling 103 | 104 | 105 | def emissions(yaml_file="ecalc_config.yaml"): 106 | 107 | from libecalc.application.energy_calculator import EnergyCalculator 108 | from libecalc.common.time_utils import Frequency 109 | from libecalc.presentation.yaml.model import YamlModel 110 | 111 | # Config 112 | model_path = HERE / yaml_file 113 | yaml_model = YamlModel(path=model_path, output_frequency=Frequency.NONE) 114 | 115 | # Compute energy, emissions 116 | model = EnergyCalculator(graph=yaml_model.graph) 117 | consumer_results = model.evaluate_energy_usage(yaml_model.variables) 118 | emission_results = model.evaluate_emissions(yaml_model.variables, consumer_results) 119 | 120 | # print power from pump 121 | co2 = [] 122 | for identity, component in yaml_model.graph.nodes.items(): 123 | if identity in emission_results: 124 | co2.append(emission_results[identity]['co2_fuel_gas'].rate.values) 125 | 126 | co2 = np.sum(np.asarray(co2), axis=0) 127 | return co2 128 | 129 | 130 | def npv(n): 131 | 132 | days = sim_data['days'] 133 | 134 | # config eCalc 135 | pd.DataFrame( {'dd-mm-yyyy' : report_dates[1:], 136 | 'OIL_PROD' : sim_data['fopt'][n]/days, 137 | 'GAS_PROD' : sim_data['fgpt'][n]/days, 138 | 'WATER_INJ' : sim_data['fwit'][n]/days, 139 | 'THP_MAX' : sim_data['thp'][n], 140 | 'WIND_POWER' : sim_data['wind'][n]*(-1) 141 | } ).to_csv(f'ecalc_input_{n}.csv', index=False) 142 | 143 | ecalc_yaml_file = kwargs['yamlfile']+'.yaml' 144 | new_yaml = duplicate_yaml_file(filename=ecalc_yaml_file, member=n) 145 | 146 | #calc emissions 147 | co2 = emissions(new_yaml)*days 148 | 149 | # save emissions 150 | try: 151 | en_co2 = np.load('co2_emissions.npy') 152 | en_co2[n] = co2 153 | np.save('co2_emissions', en_co2) 154 | except: 155 | import time 156 | time.sleep(1) 157 | 158 | #calc npv 159 | gain = const['wop']*sim_data['fopt'][n] + const['wgp']*sim_data['fgpt'][n] 160 | loss = const['wwp']*sim_data['fwpt'][n] + const['wwi']*sim_data['fwit'][n] + const['wem']*co2 161 | disc = (1+const['disc'])**(days/365) 162 | 163 | npv_value = np.sum( (gain-loss)/disc ) 164 | 165 | # delete dummy files 166 | os.remove(new_yaml) 167 | os.remove(f'ecalc_input_{n}.csv') 168 | 169 | return npv_value 170 | 171 | 172 | def duplicate_yaml_file(filename, member): 173 | 174 | try: 175 | # Load the YAML file 176 | with open(filename, 'r') as yaml_file: 177 | data = yaml.safe_load(yaml_file) 178 | 179 | input_name = data['TIME_SERIES'][0]['FILE'] 180 | data['TIME_SERIES'][0]['FILE'] = input_name.replace('.csv', f'_{member}.csv') 181 | 182 | # Write the updated content to a new file 183 | new_filename = filename.replace(".yaml", f"_{member}.yaml") 184 | with open(new_filename, 'w') as new_yaml_file: 185 | yaml.dump(data, new_yaml_file, default_flow_style=False) 186 | 187 | except FileNotFoundError: 188 | print(f"File '{filename}' not found.") 189 | 190 | return new_filename 191 | 192 | 193 | 194 | 195 | 196 | 197 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: PET 2 | site_author: Patrick N. Raanes 3 | site_url: https://python-ensemble-toolbox.github.io/PET 4 | repo_url: https://github.com/Python-Ensemble-Toolbox/PET 5 | edit_uri: edit/main/docs/ 6 | 7 | docs_dir: docs 8 | watch: [.] 9 | 10 | nav: 11 | - Home: index.md 12 | - Reference: reference/ 13 | - Tutorials: tutorials/ 14 | - Bibliography: references.md 15 | - dev_guide.md 16 | 17 | validation: 18 | absolute_links: relative_to_docs 19 | # Disabled coz it complains about relative links that work just fine 20 | # (and enable interlinking READMEs even on GitHub). 21 | # Set to "warn" to help find non-working links. 22 | unrecognized_links: ignore 23 | 24 | theme: 25 | name: material 26 | logo: https://github.com/Python-Ensemble-Toolbox/.github/blob/main/profile/pictures/logo.png?raw=true 27 | favicon: https://github.com/Python-Ensemble-Toolbox/.github/blob/main/profile/pictures/logo.png?raw=true 28 | palette: 29 | primary: custom 30 | icon: 31 | repo: fontawesome/brands/github 32 | edit: material/pencil 33 | view: material/eye 34 | features: 35 | - content.code.copy # copy button 36 | # - content.code.select # [insiders only] 37 | - content.action.edit 38 | - content.code.annotate # clickable toggler for code comments 39 | - navigation.indexes # enable subdir/index.md 40 | - toc.follow # ? 41 | # - toc.integrate # render ToC as part of left sidebar [not compat with navigation.indexes] 42 | - navigation.tabs # top-level sections rendered in header tabs (for wide screens) instead of left sidebar 43 | # - navigation.tabs.sticky # don't hide tabs when scrolled 44 | # - navigation.expand # expand sections 45 | # - navigation.sections # group (rather than collapse) top-level sections 46 | - navigation.path # breadcrumbs 47 | # - navigation.prune # for large projects 48 | - navigation.tracking # update URL to current section 49 | # - navigation.footer # "Next/Prev" 50 | - navigation.instant # SSA 51 | - navigation.instant.progress # Progbar on top 52 | - navigation.top # "back to top when scrolling up" 53 | - header.autohide 54 | - search.highlight # highlight matches on page 55 | - search.share # deep link to search 56 | - search.suggest # search suggestions 57 | 58 | custom_dir: docs/overrides 59 | 60 | plugins: 61 | - glightbox # zoom functionality 62 | # - roamlinks 63 | # - blog 64 | - search 65 | - autorefs # enable anchor/heading links without specifying its page 66 | # - meta # [insiders only] enable front-matter defaults as in Jekyll 67 | - mkdocs-jupyter: 68 | include: ["*.ipynb"] # Default: ["*.py", "*.ipynb"] 69 | ignore: 70 | - ".ipynb_checkpoints/*" 71 | # show_input: False 72 | # no_input: True 73 | execute: false 74 | # - bibtex: 75 | # # NB: does not work with mkdocstrings (https://github.com/shyamd/mkdocs-bibtex/issues/246) 76 | # # PS: requires pandoc installed on system and markdown_extensions: footnotes 77 | # bib_file: docs/bib/refs.bib 78 | # bib_by_default: true 79 | # csl_file: docs/bib/data-science-journal.csl 80 | 81 | # Autodoc from docstrings: 82 | - gen-files: # Genrate .md reflecting .py hierarchy: 83 | scripts: 84 | - docs/gen_ref_pages.py 85 | - literate-nav: # Auto-generate nav 86 | nav_file: SUMMARY.md 87 | - section-index 88 | - mkdocstrings: # Parse docstrings 89 | handlers: 90 | python: 91 | # load_external_modules: true 92 | paths: [.] 93 | # NB: The following does not work coz pipt and popt contain submodules with the same name. 94 | # paths: [pipt, popt, simulator, ensemble, misc, input_output, tests] 95 | import: 96 | - https://docs.python.org/3/objects.inv 97 | - https://mkdocstrings.github.io/objects.inv 98 | options: 99 | # Overview at https://mkdocstrings.github.io/python/usage/?h=show_source#globallocal-options 100 | # See sidebar for details. 101 | docstring_style: numpy 102 | # TODO: activate `show_source` when these [insiders!?] are done 103 | # - https://github.com/mkdocstrings/mkdocstrings/issues/525 104 | # - https://github.com/mkdocstrings/mkdocstrings/issues/249 105 | show_source: false 106 | show_symbol_type_heading: true 107 | show_symbol_type_toc: true 108 | show_root_heading: false 109 | summary: 110 | modules: false 111 | classes: false 112 | attributes: false 113 | functions: false # class methods 114 | show_labels: false 115 | 116 | # Mostly from 117 | # https://squidfunk.github.io/mkdocs-material/setup/extensions/#recommended-configuration 118 | markdown_extensions: 119 | # Python Markdown official extensions 120 | - abbr 121 | - admonition 122 | - def_list 123 | - footnotes 124 | - toc: 125 | title: On this page 126 | permalink: ⚓︎ 127 | toc_depth: 3 128 | # Adds the ability to align images, add captions to images (rendering them as figures), and mark large images for lazy-loading: 129 | - attr_list 130 | - md_in_html 131 | 132 | # Extensions part of Pymdown 133 | - pymdownx.arithmatex: 134 | generic: true 135 | - pymdownx.betterem: 136 | smart_enable: all 137 | - pymdownx.caret 138 | - pymdownx.details 139 | - pymdownx.emoji: 140 | emoji_index: !!python/name:material.extensions.emoji.twemoji 141 | emoji_generator: !!python/name:material.extensions.emoji.to_svg 142 | - pymdownx.highlight: 143 | anchor_linenums: true 144 | line_spans: __span 145 | pygments_lang_class: true 146 | use_pygments: true 147 | - pymdownx.inlinehilite 148 | - pymdownx.snippets: 149 | base_path: 150 | - !relative # dir of current Markdown file 151 | # NB: does not work with the virtual files generated by mkdocs-gen-files, 152 | # ref https://github.com/oprypin/mkdocs-gen-files/issues/25 153 | - !relative $docs_dir # dir of docs 154 | - !relative $config_dir # dir of mkdocs.yml 155 | - pymdownx.superfences 156 | - pymdownx.keys 157 | - pymdownx.mark 158 | - pymdownx.smartsymbols 159 | - pymdownx.tabbed: 160 | alternate_style: true 161 | - pymdownx.tasklist: 162 | custom_checkbox: true 163 | - pymdownx.tilde 164 | 165 | # hooks: 166 | # - list_logos.py 167 | 168 | extra_javascript: 169 | - javascripts/extra.js 170 | # For MathJax 171 | - javascripts/mathjax.js 172 | - https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js 173 | 174 | extra_css: 175 | - stylesheets/code_select.css 176 | - stylesheets/jupyter.css 177 | - stylesheets/extra.css 178 | 179 | 180 | extra: 181 | generator: false # "Made with Material for MkDocs" 182 | -------------------------------------------------------------------------------- /popt/cost_functions/ecalc_npv.py: -------------------------------------------------------------------------------- 1 | """Net present value.""" 2 | import numpy as np 3 | import csv 4 | from pathlib import Path 5 | import pandas as pd 6 | import sys 7 | 8 | HERE = Path().cwd() # fallback for ipynb's 9 | HERE = HERE.resolve() 10 | 11 | 12 | def ecalc_npv(pred_data, **kwargs): 13 | """ 14 | Net present value cost function using eCalc to calculate emmisions 15 | 16 | Parameters 17 | ---------- 18 | pred_data : array_like 19 | Ensemble of predicted data. 20 | 21 | **kwargs : dict 22 | Other arguments sent to the npv function 23 | 24 | keys_opt : list 25 | Keys with economic data. 26 | 27 | report : list 28 | Report dates. 29 | 30 | Returns 31 | ------- 32 | objective_values : array_like 33 | Objective function values (NPV) for all ensemble members. 34 | """ 35 | 36 | from libecalc.application.energy_calculator import EnergyCalculator 37 | from libecalc.common.time_utils import Frequency 38 | from ecalc_cli.infrastructure.file_resource_service import FileResourceService 39 | from libecalc.presentation.yaml.file_configuration_service import FileConfigurationService 40 | from libecalc.presentation.yaml.model import YamlModel 41 | 42 | # Get the necessary input 43 | keys_opt = kwargs.get('input_dict', {}) 44 | report = kwargs.get('true_order', []) 45 | 46 | # Economic values 47 | npv_const = dict(keys_opt['npv_const']) 48 | 49 | # Collect production data 50 | Qop = [] 51 | Qgp = [] 52 | Qwp = [] 53 | Qwi = [] 54 | Dd = [] 55 | T = [] 56 | objective = [] 57 | L = None 58 | for i in np.arange(1, len(pred_data)): 59 | 60 | if not isinstance(pred_data[i],list): 61 | pred_data[i] = [pred_data[i]] 62 | if i == 1: 63 | pred_data[i-1] = [pred_data[i-1]] 64 | L = len(pred_data[i]) 65 | for l in range(L): 66 | Qop.append([]) 67 | Qgp.append([]) 68 | Qwp.append([]) 69 | Qwi.append([]) 70 | Qop[l].append(np.squeeze(pred_data[i][l]['fopt']) - np.squeeze(pred_data[i - 1][l]['fopt'])) 71 | Qgp[l].append(np.squeeze(pred_data[i][l]['fgpt']) - np.squeeze(pred_data[i - 1][l]['fgpt'])) 72 | Qwp[l].append(np.squeeze(pred_data[i][l]['fwpt']) - np.squeeze(pred_data[i - 1][l]['fwpt'])) 73 | Qwi[l].append(np.squeeze(pred_data[i][l]['fwit']) - np.squeeze(pred_data[i - 1][l]['fwit'])) 74 | Dd.append((report[1][i] - report[1][i - 1]).days) 75 | T.append((report[1][i] - report[1][0]).days) 76 | 77 | Dd = np.array(Dd) 78 | T = np.array(T) 79 | for l in range(L): 80 | 81 | objective.append([]) 82 | 83 | # Write production data to .csv file for eCalc input, for each ensemble member 84 | Qop[l] = np.array(Qop[l]).T 85 | Qwp[l] = np.array(Qwp[l]).T 86 | Qgp[l] = np.array(Qgp[l]).T 87 | Qwi[l] = np.array(Qwi[l]).T 88 | 89 | if len(Qop[l].shape) == 1: 90 | Qop[l] = np.expand_dims(Qop[l],0) 91 | Qwp[l] = np.expand_dims(Qwp[l], 0) 92 | Qgp[l] = np.expand_dims(Qgp[l], 0) 93 | Qwi [l]= np.expand_dims(Qwi[l], 0) 94 | 95 | N = Qop[l].shape[0] 96 | NT = Qop[l].shape[1] 97 | values = [] 98 | em_values = [] 99 | for n in range(N): 100 | with open('ecalc_input.csv', 'w') as csvfile: 101 | writer = csv.writer(csvfile, delimiter=',') 102 | writer.writerow(['dd/mm/yyyy', 'GAS_PROD', 'OIL_PROD', 'WATER_INJ']) 103 | for t in range(NT): 104 | D = report[1][t] 105 | writer.writerow([D.strftime("%d/%m/%Y"), Qgp[l][n, t]/Dd[t], Qop[l][n, t]/Dd[t], Qwi[l][n, t]/Dd[t]]) 106 | 107 | # Config 108 | model_path = HERE / "ecalc_config.yaml" # "drogn.yaml" 109 | configuration_service = FileConfigurationService(configuration_path=model_path) 110 | resource_service = FileResourceService(working_directory=model_path.parent) 111 | yaml_model = YamlModel( 112 | configuration_service=configuration_service, 113 | resource_service=resource_service, 114 | output_frequency=Frequency.NONE, 115 | ) 116 | # comps = {c.name: id_hash for (id_hash, c) in yaml_model.graph.components.items()} 117 | 118 | # Compute energy, emissions 119 | #model = EnergyCalculator(energy_model=yaml_model, expression_evaluator=yaml_model.variables) 120 | #consumer_results = model.evaluate_energy_usage() 121 | #emission_results = model.evaluate_emissions() 122 | model = EnergyCalculator(graph=yaml_model.get_graph()) 123 | consumer_results = model.evaluate_energy_usage(yaml_model.variables) 124 | emission_results = model.evaluate_emissions(yaml_model.variables, consumer_results) 125 | 126 | # Extract 127 | # energy = results_as_df(yaml_model, consumer_results, lambda r: r.component_result.energy_usage) 128 | emissions = results_as_df(yaml_model, emission_results, lambda r: r['co2_fuel_gas'].rate) 129 | emissions_total = emissions.sum(1).rename("emissions_total") 130 | emissions_total.to_csv(HERE / "emissions.csv") 131 | Qem = emissions_total.values * Dd # total number of tons 132 | em_values.append(Qem) 133 | 134 | value = (Qop[l][n, :] * npv_const['wop'] + Qgp[l][n, :] * npv_const['wgp'] - Qwp[l][n, :] * npv_const['wwp'] - 135 | Qwi[l][n, :] * npv_const['wwi'] - Qem * npv_const['wem']) / ( 136 | (1 + npv_const['disc']) ** (T / 365)) 137 | objective[l].append(np.sum(value)) 138 | 139 | # Save emissions for later inspection 140 | np.savez(f'em_values_level{l}.npz', em_values=np.array([em_values])) 141 | 142 | objective[l] = np.array(objective[l]) / npv_const.get('obj_scaling', 1) 143 | 144 | return objective 145 | 146 | 147 | def results_as_df(yaml_model, results, getter) -> pd.DataFrame: 148 | """Extract relevant values, as well as some meta (`attrs`).""" 149 | df = {} 150 | attrs = {} 151 | res = None 152 | for id_hash in results: 153 | res = results[id_hash] 154 | res = getter(res) 155 | component = yaml_model.get_graph().get_node(id_hash) 156 | df[component.name] = res.values 157 | attrs[component.name] = {'id_hash': id_hash, 158 | 'kind': type(component).__name__, 159 | 'unit': res.unit} 160 | if res is None: 161 | sys.exit('No emission results from eCalc!') 162 | df = pd.DataFrame(df, index=res.timesteps) 163 | df.index.name = "dates" 164 | df.attrs = attrs 165 | return df 166 | -------------------------------------------------------------------------------- /misc/read_input_csv.py: -------------------------------------------------------------------------------- 1 | """ 2 | File for reading CSV files and returning a 2D list 3 | """ 4 | import pandas as pd 5 | import numpy as np 6 | 7 | 8 | def read_data_csv(filename, datatype, truedataindex): 9 | """ 10 | Parameters 11 | ---------- 12 | filename: 13 | Name of csv-file 14 | datatype: 15 | List of data types as strings 16 | truedataindex: 17 | List of where the "TRUEDATA" has been extracted (e.g., at which time, etc) 18 | 19 | Returns 20 | ------- 21 | some-type: 22 | List of observed data 23 | """ 24 | 25 | df = pd.read_csv(filename) # Read the file 26 | 27 | imported_data = [] # Initialize the 2D list of csv data 28 | tlength = len(truedataindex) 29 | dnumber = len(datatype) 30 | 31 | if df.columns[0] == 'header_both': # csv file has column and row headers 32 | pos = [None] * dnumber 33 | for col in range(dnumber): 34 | # find index of data type in csv file header 35 | pos[col] = df.columns.get_loc(datatype[col]) 36 | for t in truedataindex: 37 | row = df[df['header_both'] == t] # pick row corresponding to truedataindex 38 | row = row.values[0] # select the values of the dataframe row 39 | csv_data = [None] * dnumber 40 | for col in range(dnumber): 41 | if (not type(row[pos[col]]) == str) and (np.isnan(row[pos[col]])): # do not check strings 42 | csv_data[col] = 'n/a' 43 | else: 44 | try: # Making a float 45 | csv_data[col] = float(row[pos[col]]) 46 | except: # It is a string 47 | csv_data[col] = row[pos[col]] 48 | imported_data.append(csv_data) 49 | else: # No row headers (the rows in the csv file must correspond to the order in truedataindex) 50 | if tlength == df.shape[0]: # File has column headers 51 | pos = [None] * dnumber 52 | for col in range(dnumber): 53 | # Find index of the header in datatype 54 | pos[col] = df.columns.get_loc(datatype[col]) 55 | # File has no column headers (columns must correspond to the order in datatype) 56 | elif tlength == df.shape[0]+1: 57 | # First row has been misinterpreted as header, so we read first row again: 58 | temp = pd.read_csv(filename, header=None, nrows=1).values[0] 59 | pos = list(range(df.shape[1])) # Assume the data is in the correct order 60 | csv_data = [None] * len(temp) 61 | for col in range(len(temp)): 62 | if (not type(temp[col]) == str) and (np.isnan(temp[col])): # do not check strings 63 | csv_data[col] = 'n/a' 64 | else: 65 | try: # Making a float 66 | csv_data[col] = float(temp[col]) 67 | except: # It is a string 68 | csv_data[col] = temp[col] 69 | imported_data.append(csv_data) 70 | 71 | for rows in df.values: 72 | csv_data = [None] * dnumber 73 | for col in range(dnumber): 74 | if (not type(rows[pos[col]]) == str) and (np.isnan(rows[pos[col]])): # do not check strings 75 | csv_data[col] = 'n/a' 76 | else: 77 | try: # Making a float 78 | csv_data[col] = float(rows[pos[col]]) 79 | except: # It is a string 80 | csv_data[col] = rows[pos[col]] 81 | imported_data.append(csv_data) 82 | 83 | return imported_data 84 | 85 | 86 | def read_var_csv(filename, datatype, truedataindex): 87 | """ 88 | Parameters 89 | ---------- 90 | filename : str 91 | Name of the CSV file. 92 | 93 | datatype : list 94 | List of data types as strings. 95 | 96 | truedataindex : list 97 | List of indices where the "TRUEDATA" has been extracted. 98 | 99 | Returns 100 | ------- 101 | imported_var : list 102 | List of variances. 103 | """ 104 | 105 | df = pd.read_csv(filename) # Read the file 106 | 107 | imported_var = [] # Initialize the 2D list of csv data 108 | tlength = len(truedataindex) 109 | dnumber = len(datatype) 110 | 111 | if df.columns[0] == 'header_both': # csv file has column and row headers 112 | pos = [None] * dnumber 113 | for col in range(dnumber): 114 | # find index of data type in csv file header 115 | pos[col] = df.columns.get_loc(datatype[col]) 116 | for t in truedataindex: 117 | row = df[df['header_both'] == t] # pick row 118 | row = row.values[0] # select the values of the dataframe 119 | csv_data = [None] * 2 * dnumber 120 | for col in range(dnumber): 121 | csv_data[2*col] = row[pos[col]] 122 | try: # Making a float 123 | csv_data[2*col+1] = float(row[pos[col]]+1) 124 | except: # It is a string 125 | csv_data[2*col+1] = row[pos[col]+1] 126 | # Make sure the string input is lowercase 127 | csv_data[0::2] = [x.lower() for x in csv_data[0::2]] 128 | imported_var.append(csv_data) 129 | else: # No row headers (the rows in the csv file must correspond to the order in truedataindex) 130 | if tlength == df.shape[0]: # File has column headers 131 | pos = [None] * dnumber 132 | for col in range(dnumber): 133 | # Find index of datatype in csv file header 134 | pos[col] = df.columns.get_loc(datatype[col]) 135 | # File has no column headers (columns must correspond to the order in datatype) 136 | elif tlength == df.shape[0]+1: 137 | # First row has been misinterpreted as header, so we read first row again: 138 | temp = pd.read_csv(filename, header=None, nrows=1).values[0] 139 | # Make sure the string input is lowercase 140 | temp[0::2] = [x.lower() for x in temp[0::2]] 141 | # Assume the data is in the correct order 142 | pos = list(range(0, df.shape[1], 2)) 143 | csv_data = [None] * len(temp) 144 | for col in range(dnumber): 145 | csv_data[2 * col] = temp[2 * col] 146 | try: # Making a float 147 | csv_data[2*col+1] = float(temp[2*col+1]) 148 | except: # It is a string 149 | csv_data[2*col+1] = temp[2*col+1] 150 | imported_var.append(csv_data) 151 | 152 | for rows in df.values: 153 | csv_data = [None] * 2 * dnumber 154 | for col in range(dnumber): 155 | csv_data[2*col] = rows[2*col] 156 | try: # Making a float 157 | csv_data[2*col+1] = float(rows[pos[col]+1]) 158 | except: # It is a string 159 | csv_data[2*col+1] = rows[pos[col]+1] 160 | # Make sure the string input is lowercase 161 | csv_data[0::2] = [x.lower() for x in csv_data[0::2]] 162 | imported_var.append(csv_data) 163 | 164 | return imported_var 165 | -------------------------------------------------------------------------------- /popt/cost_functions/ecalc_pareto_npv.py: -------------------------------------------------------------------------------- 1 | """Net present value.""" 2 | import numpy 3 | import numpy as np 4 | import csv 5 | from pathlib import Path 6 | import pandas as pd 7 | import sys 8 | import os 9 | 10 | HERE = Path().cwd() # fallback for ipynb's 11 | HERE = HERE.resolve() 12 | 13 | 14 | def ecalc_pareto_npv(pred_data, kwargs): 15 | """ 16 | Net present value cost function using eCalc to calculate emmisions 17 | 18 | Parameters 19 | ---------- 20 | pred_data : array_like 21 | Ensemble of predicted data. 22 | 23 | **kwargs : dict 24 | Other arguments sent to the npv function 25 | 26 | keys_opt : list 27 | Keys with economic data. 28 | 29 | report : list 30 | Report dates. 31 | 32 | Returns 33 | ------- 34 | objective_values : array_like 35 | Objective function values (NPV) for all ensemble members. 36 | """ 37 | 38 | from libecalc.application.energy_calculator import EnergyCalculator 39 | from libecalc.common.time_utils import Frequency 40 | from ecalc_cli.infrastructure.file_resource_service import FileResourceService 41 | from libecalc.presentation.yaml.file_configuration_service import FileConfigurationService 42 | from libecalc.presentation.yaml.model import YamlModel 43 | 44 | # Get the necessary input 45 | keys_opt = kwargs.get('input_dict', {}) 46 | report = kwargs.get('true_order', []) 47 | 48 | # Economic values 49 | npv_const = dict(keys_opt['npv_const']) 50 | 51 | # Collect production data 52 | Qop = [] 53 | Qgp = [] 54 | Qwp = [] 55 | Qwi = [] 56 | Dd = [] 57 | T = [] 58 | for i in np.arange(1, len(pred_data)): 59 | 60 | Qop.append(np.squeeze(pred_data[i]['fopt']) - np.squeeze(pred_data[i - 1]['fopt'])) 61 | Qgp.append(np.squeeze(pred_data[i]['fgpt']) - np.squeeze(pred_data[i - 1]['fgpt'])) 62 | Qwp.append(np.squeeze(pred_data[i]['fwpt']) - np.squeeze(pred_data[i - 1]['fwpt'])) 63 | Qwi.append(np.squeeze(pred_data[i]['fwit']) - np.squeeze(pred_data[i - 1]['fwit'])) 64 | Dd.append((report[1][i] - report[1][i - 1]).days) 65 | T.append((report[1][i] - report[1][0]).days) 66 | 67 | # Write production data to .csv file for eCalc input, for each ensemble member 68 | Qop = np.array(Qop).T 69 | Qwp = np.array(Qwp).T 70 | Qgp = np.array(Qgp).T 71 | Qwi = np.array(Qwi).T 72 | Dd = np.array(Dd) 73 | T = np.array(T) 74 | if len(Qop.shape) == 1: 75 | Qop = np.expand_dims(Qop,0) 76 | Qwp = np.expand_dims(Qwp, 0) 77 | Qgp = np.expand_dims(Qgp, 0) 78 | Qwi = np.expand_dims(Qwi, 0) 79 | 80 | N = Qop.shape[0] 81 | NT = Qop.shape[1] 82 | values = [] 83 | pareto_values = [] 84 | for n in range(N): 85 | with open('ecalc_input.csv', 'w') as csvfile: 86 | writer = csv.writer(csvfile, delimiter=',') 87 | writer.writerow(['dd/mm/yyyy', 'GAS_PROD', 'OIL_PROD', 'WATER_INJ']) 88 | for t in range(NT): 89 | D = report[1][t] 90 | writer.writerow([D.strftime("%d/%m/%Y"), Qgp[n, t]/Dd[t], Qop[n, t]/Dd[t], Qwi[n, t]/Dd[t]]) 91 | 92 | # Config 93 | model_path = HERE / "ecalc_config.yaml" # "drogn.yaml" 94 | configuration_service = FileConfigurationService(configuration_path=model_path) 95 | resource_service = FileResourceService(working_directory=model_path.parent) 96 | yaml_model = YamlModel( 97 | configuration_service=configuration_service, 98 | resource_service=resource_service, 99 | output_frequency=Frequency.NONE, 100 | ) 101 | #yaml_model = YamlModel(path=model_path, output_frequency=Frequency.NONE) 102 | # comps = {c.name: id_hash for (id_hash, c) in yaml_model.graph.components.items()} 103 | 104 | # Compute energy, emissions 105 | # model = EnergyCalculator(energy_model=yaml_model, expression_evaluator=yaml_model.variables) 106 | # consumer_results = model.evaluate_energy_usage() 107 | # emission_results = model.evaluate_emissions() 108 | model = EnergyCalculator(graph=yaml_model.get_graph()) 109 | consumer_results = model.evaluate_energy_usage(yaml_model.variables) 110 | emission_results = model.evaluate_emissions(yaml_model.variables, consumer_results) 111 | 112 | # Extract 113 | # energy = results_as_df(yaml_model, consumer_results, lambda r: r.component_result.energy_usage) 114 | emissions = results_as_df(yaml_model, emission_results, lambda r: r['co2_fuel_gas'].rate) 115 | emissions_total = emissions.sum(1).rename("emissions_total") 116 | emissions_total.to_csv(HERE / "emissions.csv") 117 | Qem = emissions_total.values * Dd # total number of tons 118 | 119 | value1 = (Qop[n, :] * npv_const['wop'] + Qgp[n, :] * npv_const['wgp'] - Qwp[n, :] * npv_const['wwp'] - 120 | Qwi[n, :] * npv_const['wwi'] - Qem * npv_const['wem']) / ( 121 | (1 + npv_const['disc']) ** (T / 365)) 122 | value1 = np.sum(value1) 123 | if 'obj_scaling' in npv_const: 124 | value1 /= npv_const['obj_scaling'] 125 | 126 | value2 = np.array([]) 127 | if 'wemc' in npv_const: # multi-opjective with co2 cost correction 128 | value2 = - Qem * npv_const['wemc'] / ((1 + npv_const['disc']) ** (T / 365)) 129 | value2 = np.sum(value2) 130 | if 'obj_scaling' in npv_const: 131 | value2 /= npv_const['obj_scaling'] 132 | elif 'w' in npv_const: # multi-objective with emission intensity 133 | rho_o = 840.0 # oil density 134 | rho_g = 1 # gas density 135 | conv = 1000 # convert from kilo to tonne 136 | value2 = np.sum(Qem*conv) / (np.sum(Qop[n, :]*rho_o + Qgp[n, :]*rho_g)/conv) # kg/toe 137 | 138 | pareto_values.append([np.sum(Qem), value1, value2]) 139 | if 'w' in npv_const: 140 | values.append((1-npv_const['w'])*value1 + npv_const['w']*value2) 141 | else: 142 | values.append(value1 + value2) # total objective function 143 | 144 | # Save emissions and both objective functions for later analysis 145 | pareto_file = 'pareto_values.npz' 146 | if os.path.exists(pareto_file): 147 | pareto_iterations = np.load(pareto_file,allow_pickle=True)['pareto_iterations'][()] 148 | else: 149 | pareto_iterations = {} 150 | num_eval = len(pareto_iterations) 151 | pareto_iterations[str(num_eval)] = pareto_values 152 | np.savez('pareto_values.npz', pareto_iterations=pareto_iterations) 153 | 154 | return np.array(values) 155 | 156 | def results_as_df(yaml_model, results, getter) -> pd.DataFrame: 157 | """Extract relevant values, as well as some meta (`attrs`).""" 158 | df = {} 159 | attrs = {} 160 | res = None 161 | for id_hash in results: 162 | res = results[id_hash] 163 | res = getter(res) 164 | component = yaml_model.get_graph().get_node(id_hash) 165 | df[component.name] = res.values 166 | attrs[component.name] = {'id_hash': id_hash, 167 | 'kind': type(component).__name__, 168 | 'unit': res.unit} 169 | if res is None: 170 | sys.exit('No emission results from eCalc!') 171 | df = pd.DataFrame(df, index=res.timesteps) 172 | df.index.name = "dates" 173 | df.attrs = attrs 174 | return df 175 | -------------------------------------------------------------------------------- /input_output/organize.py: -------------------------------------------------------------------------------- 1 | """Descriptive description.""" 2 | 3 | from copy import deepcopy 4 | import csv 5 | import datetime as dt 6 | import pandas as pd 7 | 8 | 9 | class Organize_input(): 10 | def __init__(self, keys_pr, keys_fwd, keys_en=None): 11 | self.keys_pr = keys_pr 12 | self.keys_fwd = keys_fwd 13 | self.keys_en = keys_en 14 | 15 | def organize(self): 16 | # Organize the data types given by DATATYPE keyword 17 | self._org_datatype() 18 | # Organize the observed data given by TRUEDATA keyword and initialize predicted data variable 19 | self._org_report() 20 | 21 | def get_keys_pr(self): 22 | return deepcopy(self.keys_pr) 23 | 24 | def get_keys_fwd(self): 25 | return deepcopy(self.keys_fwd) 26 | 27 | def get_keys_en(self): 28 | return deepcopy(self.keys_en) 29 | 30 | def _org_datatype(self): 31 | """ Check if datatype is given as a csv file. If so, we read and make a list.""" 32 | if isinstance(self.keys_fwd['datatype'], str) and self.keys_fwd['datatype'].endswith('.csv'): 33 | with open(self.keys_fwd['datatype']) as csvfile: 34 | reader = csv.reader(csvfile) # get a reader object 35 | datatype = [] # Initialize the list of csv data 36 | for rows in reader: # Rows is a list of values in the csv file 37 | csv_data = [None] * len(rows) 38 | for col in range(len(rows)): 39 | csv_data[col] = str(rows[col]) 40 | datatype.extend(csv_data) 41 | self.keys_fwd['datatype'] = datatype 42 | 43 | if not isinstance(self.keys_fwd['datatype'], list): 44 | self.keys_fwd['datatype'] = [self.keys_fwd['datatype']] 45 | # make copy for problem keywords 46 | self.keys_pr['datatype'] = self.keys_fwd['datatype'] 47 | 48 | def _org_report(self): 49 | """ 50 | Organize the input true observed data. The obs_data will be a list of length equal length of "TRUEDATAINDEX", 51 | and each entery in the list will be a dictionary with keys equal to the "DATATYPE". 52 | Also, the pred_data variable (predicted data or forward simulation) will be initialized here with the same 53 | structure as the obs_data variable. 54 | 55 | !!! warning 56 | An "N/A" entry in "TRUEDATA" is treated as a None-entry; that is, there is NOT an observed data at this 57 | assimilation step.' 58 | 59 | !!! warning 60 | The array associated with the first string inputted in "TRUEDATAINDEX" is assumed to be the "main" 61 | index, that is, the length of this array will determine the length of the obs_data list! There arrays 62 | associated with the subsequent strings in "TRUEDATAINDEX" are then assumed to be a subset of the first 63 | string. 64 | An example: the first string is SOURCE (e.g., sources in CSEM), where the array will be a list of numbering 65 | for the sources; and the second string is FREQ, where the array associated will be a list of frequencies. 66 | 67 | !!! info 68 | It is assumed that the number of data associated with a subset is the same for each index in the subset. 69 | For example: If two frequencies are inputted in FREQ, then the number of data for one SOURCE index and one 70 | frequency is 1/2 of the total no. of data for that SOURCE index. If three frequencies are inputted, the number 71 | of data for one SOURCE index and one frequencies is 1/3 of the total no of data for that SOURCE index, 72 | and so on. 73 | """ 74 | 75 | # Extract primary indices from "TRUEDATAINDEX" 76 | if 'truedataindex' in self.keys_pr: 77 | 78 | if isinstance(self.keys_pr['truedataindex'], list): # List of prim. ind 79 | true_prim = self.keys_pr['truedataindex'] 80 | else: # Float 81 | true_prim = [self.keys_pr['truedataindex']] 82 | 83 | # Check if a csv file has been included as "TRUEDATAINDEX". If so, we read it and make a list, 84 | if isinstance(self.keys_pr['truedataindex'], str) and self.keys_pr['truedataindex'].endswith('.csv'): 85 | with open(self.keys_pr['truedataindex']) as csvfile: 86 | reader = csv.reader(csvfile) # get a reader object 87 | true_prim = [] # Initialize the list of csv data 88 | for rows in reader: # Rows is a list of values in the csv file 89 | csv_data = [None] * len(rows) 90 | for ind, col in enumerate(rows): 91 | csv_data[ind] = int(col) 92 | true_prim.extend(csv_data) 93 | self.keys_pr['truedataindex'] = true_prim 94 | 95 | # Check if a csv file has been included as "REPORTPOINT". If so, we read it and make a list, 96 | if 'reportpoint' in self.keys_fwd: 97 | if isinstance(self.keys_fwd['reportpoint'], str) and self.keys_fwd['reportpoint'].endswith('.csv'): 98 | with open(self.keys_fwd['reportpoint']) as csvfile: 99 | reader = csv.reader(csvfile) # get a reader object 100 | pred_prim = [] # Initialize the list of csv data 101 | for rows in reader: # Rows is a list of values in the csv file 102 | csv_data = [None] * len(rows) 103 | for ind, col in enumerate(rows): 104 | try: 105 | csv_data[ind] = int(col) 106 | except ValueError: 107 | csv_data[ind] = dt.datetime.strptime( 108 | col, '%Y-%m-%d %H:%M:%S') 109 | 110 | pred_prim.extend(csv_data) 111 | self.keys_fwd['reportpoint'] = pred_prim 112 | 113 | elif isinstance(self.keys_fwd['reportpoint'], dict): 114 | self.keys_fwd['reportpoint'] = pd.date_range(**self.keys_fwd['reportpoint']).to_pydatetime().tolist() 115 | 116 | else: 117 | pass 118 | 119 | 120 | # Check if assimindex is given as a csv file. If so, we read and make a potential 2D list (if sequential). 121 | if 'assimindex' in self.keys_pr: 122 | if isinstance(self.keys_pr['assimindex'], str) and self.keys_pr['assimindex'].endswith('.csv'): 123 | with open(self.keys_pr['assimindex']) as csvfile: 124 | reader = csv.reader(csvfile) # get a reader object 125 | assimindx = [] # Initialize the 2D list of csv data 126 | for rows in reader: # Rows is a list of values in the csv file 127 | csv_data = [None] * len(rows) 128 | for col in range(len(rows)): 129 | csv_data[col] = int(rows[col]) 130 | assimindx.append(csv_data) 131 | self.keys_pr['assimindex'] = assimindx 132 | 133 | # check that they are lists 134 | if not isinstance(self.keys_pr['truedataindex'], list): 135 | self.keys_pr['truedataindex'] = [self.keys_pr['truedataindex']] 136 | if not isinstance(self.keys_fwd['reportpoint'], list): 137 | self.keys_fwd['reportpoint'] = [self.keys_fwd['reportpoint']] 138 | if not isinstance(self.keys_pr['assimindex'], list): 139 | self.keys_pr['assimindex'] = [self.keys_pr['assimindex']] 140 | -------------------------------------------------------------------------------- /popt/loop/ensemble_base.py: -------------------------------------------------------------------------------- 1 | # External imports 2 | import numpy as np 3 | import sys 4 | import warnings 5 | 6 | from copy import deepcopy 7 | 8 | # Internal imports 9 | from popt.misc_tools import optim_tools as ot 10 | from pipt.misc_tools import analysis_tools as at 11 | from ensemble.ensemble import Ensemble as SupEnsemble 12 | from simulator.simple_models import noSimulation 13 | 14 | __all__ = ['EnsembleOptimizationBaseClass'] 15 | 16 | class EnsembleOptimizationBaseClass(SupEnsemble): 17 | ''' 18 | Base class for the popt ensemble 19 | ''' 20 | def __init__(self, options, simulator, objective): 21 | ''' 22 | Parameters 23 | ---------- 24 | options : dict 25 | Options for the ensemble class 26 | 27 | simulator : callable 28 | The forward simulator (e.g. flow). If None, no simulation is performed. 29 | 30 | objective : callable 31 | The objective function (e.g. npv) 32 | ''' 33 | if simulator is None: 34 | sim = noSimulation() 35 | else: 36 | sim = simulator 37 | 38 | # Initialize the PET Ensemble 39 | super().__init__(options, sim) 40 | 41 | # Unpack some options 42 | self.save_prediction = options.get('save_prediction', None) 43 | self.num_models = options.get('num_models', 1) 44 | self.transform = options.get('transform', False) 45 | self.num_samples = self.ne 46 | 47 | # Set objective function (callable) 48 | self.obj_func = objective 49 | self.state_func_values = None 50 | self.ens_func_values = None 51 | 52 | # Initialize prior 53 | self._initialize_state_info() # Initialize cov, bounds, and state 54 | self._scale_state() # Scale self.state to [0, 1] if transform is True 55 | 56 | def _initialize_state_info(self): 57 | ''' 58 | Initialize covariance and bounds based on prior information. 59 | ''' 60 | self.cov = np.array([]) 61 | self.lb = [] 62 | self.ub = [] 63 | self.bounds = [] 64 | 65 | for key in self.prior_info.keys(): 66 | variable = self.prior_info[key] 67 | 68 | # mean 69 | self.state[key] = np.asarray(variable['mean']) 70 | 71 | # Covariance 72 | dim = self.state[key].size 73 | var = variable['variance']*np.ones(dim) 74 | 75 | if 'limits' in variable.keys(): 76 | lb, ub = variable['limits'] 77 | self.lb.append(lb) 78 | self.ub.append(ub) 79 | 80 | # transform var to [0, 1] if transform is True 81 | if self.transform: 82 | var = var/(ub - lb)**2 83 | var = np.clip(var, 0, 1, out=var) 84 | self.bounds += dim*[(0, 1)] 85 | else: 86 | self.bounds += dim*[(lb, ub)] 87 | else: 88 | self.bounds += dim*[(None, None)] 89 | 90 | # Add to covariance 91 | self.cov = np.append(self.cov, var) 92 | self.dim = self.cov.shape[0] 93 | 94 | # Make cov full covariance matrix 95 | self.cov = np.diag(self.cov) 96 | 97 | def get_state(self): 98 | """ 99 | Returns 100 | ------- 101 | x : numpy.ndarray 102 | Control vector as ndarray, shape (number of controls, number of perturbations) 103 | """ 104 | return ot.aug_optim_state(self.state, list(self.state.keys())) 105 | 106 | def get_cov(self): 107 | """ 108 | Returns 109 | ------- 110 | cov : numpy.ndarray 111 | Covariance matrix, shape (number of controls, number of controls) 112 | """ 113 | return self.cov 114 | 115 | def vec_to_state(self, x): 116 | """ 117 | Converts a control vector to the internal state representation. 118 | """ 119 | return ot.update_optim_state(x, self.state, list(self.state.keys())) 120 | 121 | def get_bounds(self): 122 | """ 123 | Returns 124 | ------- 125 | bounds : list 126 | (min, max) pairs for each element in x. None is used to specify no bound. 127 | """ 128 | 129 | return self.bounds 130 | 131 | def function(self, x, *args, **kwargs): 132 | """ 133 | This is the main function called during optimization. 134 | 135 | Parameters 136 | ---------- 137 | x : ndarray 138 | Control vector, shape (number of controls, number of perturbations) 139 | 140 | Returns 141 | ------- 142 | obj_func_values : numpy.ndarray 143 | Objective function values, shape (number of perturbations, ) 144 | """ 145 | self._aux_input() 146 | 147 | # check for ensmble 148 | if len(x.shape) == 1: self.ne = self.num_models 149 | else: self.ne = x.shape[1] 150 | 151 | # convert x (nparray) to state (dict) 152 | self.state = self.vec_to_state(x) 153 | 154 | # run the simulation 155 | self._invert_scale_state() # ensure that state is in [lb,ub] 156 | self._set_multilevel_state(self.state, x) # set multilevel state if applicable 157 | run_success = self.calc_prediction(save_prediction=self.save_prediction) # calculate flow data 158 | self._set_multilevel_state(self.state, x) # toggle back after calc_prediction 159 | 160 | # Evaluate the objective function 161 | if run_success: 162 | func_values = self.obj_func( 163 | self.pred_data, 164 | input_dict=self.sim.input_dict, 165 | true_order=self.sim.true_order, 166 | state=self.state, # pass state for possible use in objective function 167 | **kwargs 168 | ) 169 | else: 170 | func_values = np.inf # the simulations have crashed 171 | 172 | self._scale_state() # scale back to [0, 1] 173 | if len(x.shape) == 1: self.state_func_values = func_values 174 | else: self.ens_func_values = func_values 175 | 176 | return func_values 177 | 178 | def _set_multilevel_state(self, state, x): 179 | if 'multilevel' in self.keys_en.keys() and len(x.shape) > 1: 180 | en_size = ot.get_list_element(self.keys_en['multilevel'], 'en_size') 181 | self.state = ot.toggle_ml_state(self.state, en_size) 182 | 183 | 184 | def _aux_input(self): 185 | """ 186 | Set the auxiliary input used for multiple geological realizations 187 | """ 188 | 189 | nr = 1 # nr is the ratio of samples over models 190 | if self.num_models > 1: 191 | if np.remainder(self.num_samples, self.num_models) == 0: 192 | nr = int(self.num_samples / self.num_models) 193 | self.aux_input = list(np.repeat(np.arange(self.num_models), nr)) 194 | else: 195 | print('num_samples must be a multiplum of num_models!') 196 | sys.exit(0) 197 | return nr 198 | 199 | def _scale_state(self): 200 | """ 201 | Transform the internal state from [lb, ub] to [0, 1] 202 | """ 203 | if self.transform and (self.lb and self.ub): 204 | for i, key in enumerate(self.state): 205 | self.state[key] = (self.state[key] - self.lb[i])/(self.ub[i] - self.lb[i]) 206 | np.clip(self.state[key], 0, 1, out=self.state[key]) 207 | 208 | def _invert_scale_state(self): 209 | """ 210 | Transform the internal state from [0, 1] to [lb, ub] 211 | """ 212 | if self.transform and (self.lb and self.ub): 213 | for i, key in enumerate(self.state): 214 | if self.transform: 215 | self.state[key] = self.lb[i] + self.state[key]*(self.ub[i] - self.lb[i]) 216 | np.clip(self.state[key], self.lb[i], self.ub[i], out=self.state[key]) -------------------------------------------------------------------------------- /popt/update_schemes/smcopt.py: -------------------------------------------------------------------------------- 1 | """Stochastic Monte-Carlo optimisation.""" 2 | # External imports 3 | import numpy as np 4 | import time 5 | import pprint 6 | 7 | # Internal imports 8 | from popt.loop.optimize import Optimize 9 | import popt.update_schemes.subroutines.optimizers as opt 10 | from popt.misc_tools import optim_tools as ot 11 | 12 | 13 | class SmcOpt(Optimize): 14 | """ 15 | TODO: Write docstring ala EnOpt 16 | """ 17 | 18 | def __init__(self, fun, x, args, sens, bounds=None, **options): 19 | """ 20 | Parameters 21 | ---------- 22 | fun : callable 23 | objective function 24 | 25 | x : ndarray 26 | Initial state 27 | 28 | sens : callable 29 | Ensemble sensitivity 30 | 31 | bounds : list, optional 32 | (min, max) pairs for each element in x. None is used to specify no bound. 33 | 34 | options : dict 35 | Optimization options 36 | 37 | - maxiter: maximum number of iterations (default 10) 38 | - restart: restart optimization from a restart file (default false) 39 | - restartsave: save a restart file after each successful iteration (defalut false) 40 | - tol: convergence tolerance for the objective function (default 1e-6) 41 | - alpha: weight between previous and new step (default 0.1) 42 | - alpha_maxiter: maximum number of backtracing trials (default 5) 43 | - resample: number indicating how many times resampling is tried if no improvement is found 44 | - cov_factor: factor used to shrink the covariance for each resampling trial (defalut 0.5) 45 | - inflation_factor: term used to weight down prior influence (defalult 1) 46 | - survival_factor: fraction of surviving samples 47 | - savedata: specify which class variables to save to the result files (state, objective function 48 | value, iteration number, number of function evaluations, and number of gradient 49 | evaluations, are always saved) 50 | """ 51 | 52 | # init PETEnsemble 53 | super(SmcOpt, self).__init__(**options) 54 | 55 | def __set__variable(var_name=None, defalut=None): 56 | if var_name in options: 57 | return options[var_name] 58 | else: 59 | return defalut 60 | 61 | # Set input as class variables 62 | self.options = options # options 63 | self.function = fun # objective function 64 | self.sens = sens # gradient function 65 | self.bounds = bounds # parameter bounds 66 | self.mean_state = x # initial mean state 67 | self.best_state = None # best ensemble member 68 | self.cov = args[0] # covariance matrix for sampling 69 | 70 | # Set other optimization parameters 71 | self.obj_func_tol = __set__variable('tol', 1e-6) 72 | self.alpha = __set__variable('alpha', 0.1) 73 | self.alpha_iter_max = __set__variable('alpha_maxiter', 5) 74 | self.max_resample = __set__variable('resample', 0) 75 | self.cov_factor = __set__variable('cov_factor', 0.5) 76 | self.inflation_factor = __set__variable('inflation_factor', 1.0) 77 | self.survival_factor = __set__variable('survival_factor', 1.0) 78 | self.survival_factor = np.clip(self.survival_factor,0.1, 1.0) 79 | 80 | # Calculate objective function of startpoint 81 | if not self.restart: 82 | self.start_time = time.perf_counter() 83 | self.obj_func_values = self.function(self.mean_state) 84 | self.best_func = np.mean(self.obj_func_values) 85 | self.nfev += 1 86 | self.optimize_result = ot.get_optimize_result(self) 87 | ot.save_optimize_results(self.optimize_result) 88 | if self.logger is not None: 89 | self.logger.info(' ====== Running optimization - SmcOpt ======') 90 | self.logger.info('\n' + pprint.pformat(self.options)) 91 | info_str = ' {:<10} {:<10} {:<15} {:<15} '.format('iter', 'alpha_iter', 92 | 'obj_func', 'step-size') 93 | self.logger.info(info_str) 94 | self.logger.info(' {:<21} {:<15.4e}'.format(self.iteration, np.mean(self.obj_func_values))) 95 | 96 | self.optimizer = opt.GradientDescent(self.alpha, 0) 97 | 98 | # The SmcOpt class self-ignites 99 | self.run_loop() # run_loop resides in the Optimization class (super) 100 | 101 | def fun(self, x, *args, **kwargs): 102 | return self.function(x, *args, **kwargs) 103 | 104 | @property 105 | def xk(self): 106 | return self._xk 107 | 108 | @property 109 | def fk(self): 110 | return self.obj_func_values 111 | 112 | @property 113 | def ftol(self): 114 | return self.obj_func_tol 115 | 116 | @ftol.setter 117 | def ftol(self, value): 118 | self.obj_func_tol = value 119 | 120 | def calc_update(self,): 121 | """ 122 | Update using sequential monte carlo method 123 | """ 124 | 125 | improvement = False 126 | success = False 127 | resampling_iter = 0 128 | inflate = 2 * (self.inflation_factor + self.iteration) 129 | self.optimizer.restore_parameters() 130 | 131 | while improvement is False: # resampling loop 132 | 133 | # Shrink covariance and step size each time we try resampling 134 | shrink = self.cov_factor ** resampling_iter 135 | self.optimizer.apply_backtracking(np.sqrt(self.cov_factor) ** resampling_iter) 136 | 137 | # Calc sensitivity 138 | (sens_matrix, self.best_state, best_func_tmp) = self.sens(self.mean_state, inflate, 139 | shrink*self.cov, self.survival_factor) 140 | self.njev += 1 141 | 142 | # Initialize for this step 143 | alpha_iter = 0 144 | 145 | while improvement is False: # backtracking loop 146 | 147 | search_direction = sens_matrix 148 | new_state = self.optimizer.apply_smc_update(self.mean_state, search_direction, iter=self.iteration) 149 | new_state = ot.clip_state(new_state, self.bounds) 150 | 151 | # Calculate new objective function 152 | new_func_values = self.function(new_state) 153 | self.nfev += 1 154 | 155 | if np.mean(self.obj_func_values) - np.mean(new_func_values) > self.obj_func_tol or \ 156 | (self.best_func - best_func_tmp) > self.obj_func_tol: 157 | 158 | # Update objective function values and step 159 | self.obj_func_values = new_func_values 160 | self.mean_state = new_state 161 | if (self.best_func - best_func_tmp) > self.obj_func_tol: 162 | self.best_func = best_func_tmp 163 | 164 | # Write logging info 165 | if self.logger is not None: 166 | info_str_iter = ' {:<10} {:<10} {:<15.4e} {:<15.2e}'. \ 167 | format(self.iteration, alpha_iter, self.best_func, 168 | self.alpha) 169 | self.logger.info(info_str_iter) 170 | 171 | # Iteration was a success 172 | improvement = True 173 | success = True 174 | self.optimizer.restore_parameters() 175 | 176 | # Save variables defined in savedata keyword. 177 | self.optimize_result = ot.get_optimize_result(self) 178 | ot.save_optimize_results(self.optimize_result) 179 | 180 | # Update iteration counter if iteration was successful and save current state 181 | self.iteration += 1 182 | 183 | else: 184 | 185 | # If we do not have a reduction in the objective function, we reduce the step limiter 186 | if alpha_iter < self.alpha_iter_max: 187 | self.optimizer.apply_backtracking() # decrease alpha 188 | alpha_iter += 1 189 | elif (resampling_iter < self.max_resample and 190 | np.mean(new_func_values) - np.mean(self.obj_func_values) > 0): # update gradient 191 | resampling_iter += 1 192 | self.optimizer.restore_parameters() 193 | break 194 | else: 195 | success = False 196 | return success 197 | 198 | return success 199 | -------------------------------------------------------------------------------- /pipt/update_schemes/enkf.py: -------------------------------------------------------------------------------- 1 | """ 2 | EnKF type schemes 3 | """ 4 | # External imports 5 | import numpy as np 6 | from scipy.linalg import solve 7 | from copy import deepcopy 8 | from geostat.decomp import Cholesky # Making realizations 9 | 10 | # Internal imports 11 | from pipt.loop.ensemble import Ensemble 12 | # Misc. tools used in analysis schemes 13 | from pipt.misc_tools import analysis_tools as at 14 | 15 | from pipt.update_schemes.update_methods_ns.approx_update import approx_update 16 | from pipt.update_schemes.update_methods_ns.full_update import full_update 17 | from pipt.update_schemes.update_methods_ns.subspace_update import subspace_update 18 | 19 | 20 | class enkfMixIn(Ensemble): 21 | """ 22 | Straightforward EnKF analysis scheme implementation. The sequential updating can be done with general grouping and 23 | ordering of data. If only one-step EnKF is to be done, use `es` instead. 24 | """ 25 | 26 | def __init__(self, keys_da, keys_en, sim): 27 | """ 28 | The class is initialized by passing the PIPT init. file upwards in the hierarchy to be read and parsed in 29 | `pipt.input_output.pipt_init.ReadInitFile`. 30 | """ 31 | # Pass the init_file upwards in the hierarchy 32 | super().__init__(keys_da, keys_en, sim) 33 | 34 | self.prev_data_misfit = None 35 | 36 | if self.restart is False: 37 | self.prior_state = deepcopy(self.state) 38 | self.list_states = list(self.state.keys()) 39 | # At the moment, the iterative loop is threated as an iterative smoother an thus we check if assim. indices 40 | # are given as in the Simultaneous loop. 41 | self.check_assimindex_sequential() 42 | 43 | # Extract no. assimilation steps from MDA keyword in DATAASSIM part of init. file and set this equal to 44 | # the number of iterations pluss one. Need one additional because the iter=0 is the prior run. 45 | self.max_iter = len(self.keys_da['assimindex'])+1 46 | self.iteration = 0 47 | self.lam = 0 # set LM lamda to zero as we are doing one full update. 48 | if 'energy' in self.keys_da: 49 | # initial energy (Remember to extract this) 50 | self.trunc_energy = self.keys_da['energy'] 51 | if self.trunc_energy > 1: # ensure that it is given as percentage 52 | self.trunc_energy /= 100. 53 | else: 54 | self.trunc_energy = 0.98 55 | self.current_state = deepcopy(self.state) 56 | 57 | self.state_scaling = at.calc_scaling( 58 | self.prior_state, self.list_states, self.prior_info) 59 | 60 | def calc_analysis(self): 61 | """ 62 | Calculate the analysis step of the EnKF procedure. The updating is done using the Kalman filter equations, using 63 | svd for numerical stability. Localization is available. 64 | """ 65 | # If this is initial analysis we calculate the objective function for all data. In the final convergence check 66 | # we calculate the posterior objective function for all data 67 | if not hasattr(self, 'prior_data_misfit'): 68 | assim_index = [self.keys_da['obsname'], list( 69 | np.concatenate(self.keys_da['assimindex']))] 70 | list_datatypes, list_active_dataypes = at.get_list_data_types( 71 | self.obs_data, assim_index) 72 | if not hasattr(self, 'cov_data'): 73 | self.full_cov_data = at.gen_covdata( 74 | self.datavar, assim_index, list_datatypes) 75 | else: 76 | self.full_cov_data = self.cov_data 77 | obs_data_vector, pred_data = at.aug_obs_pred_data( 78 | self.obs_data, self.pred_data, assim_index, list_datatypes) 79 | # Generate realizations of the observed data 80 | init_en = Cholesky() # Initialize GeoStat class for generating realizations 81 | self.full_real_obs_data = init_en.gen_real( 82 | obs_data_vector, self.full_cov_data, self.ne) 83 | 84 | # Calc. misfit for the initial iteration 85 | data_misfit = at.calc_objectivefun( 86 | self.full_real_obs_data, pred_data, self.full_cov_data) 87 | 88 | # Store the (mean) data misfit (also for conv. check) 89 | self.data_misfit = np.mean(data_misfit) 90 | self.prior_data_misfit = np.mean(data_misfit) 91 | self.data_misfit_std = np.std(data_misfit) 92 | 93 | self.logger.info( 94 | f'Prior run complete with data misfit: {self.prior_data_misfit:0.1f}.') 95 | 96 | # Get assimilation order as a list 97 | # must subtract one to be inline 98 | self.assim_index = [self.keys_da['obsname'], 99 | self.keys_da['assimindex'][self.iteration-1]] 100 | 101 | # Get list of data types to be assimilated and of the free states. Do this once, because listing keys from a 102 | # Python dictionary just when needed (in different places) may not yield the same list! 103 | self.list_datatypes, list_active_dataypes = at.get_list_data_types( 104 | self.obs_data, self.assim_index) 105 | 106 | # Augment observed and predicted data 107 | self.obs_data_vector, self.aug_pred_data = at.aug_obs_pred_data(self.obs_data, self.pred_data, self.assim_index, 108 | self.list_datatypes) 109 | self.cov_data = at.gen_covdata( 110 | self.datavar, self.assim_index, self.list_datatypes) 111 | 112 | init_en = Cholesky() # Initialize GeoStat class for generating realizations 113 | self.data_random_state = deepcopy(np.random.get_state()) 114 | self.real_obs_data, self.scale_data = init_en.gen_real(self.obs_data_vector, self.cov_data, self.ne, 115 | return_chol=True) 116 | 117 | self.E = np.dot(self.real_obs_data, self.proj) 118 | 119 | if 'localanalysis' in self.keys_da: 120 | self.local_analysis_update() 121 | else: 122 | # Mean pred_data and perturbation matrix with scaling 123 | if len(self.scale_data.shape) == 1: 124 | self.pert_preddata = np.dot(np.expand_dims(self.scale_data ** (-1), axis=1), 125 | np.ones((1, self.ne))) * np.dot(self.aug_pred_data, self.proj) 126 | else: 127 | self.pert_preddata = solve( 128 | self.scale_data, np.dot(self.aug_pred_data, self.proj)) 129 | 130 | aug_state = at.aug_state(self.current_state, self.list_states) 131 | self.update() 132 | if hasattr(self, 'step'): 133 | aug_state_upd = aug_state + self.step 134 | if hasattr(self, 'w_step'): 135 | self.W = self.current_W + self.w_step 136 | aug_prior_state = at.aug_state(self.prior_state, self.list_states) 137 | aug_state_upd = np.dot(aug_prior_state, (np.eye( 138 | self.ne) + self.W / np.sqrt(self.ne - 1))) 139 | # Extract updated state variables from aug_update 140 | self.state = at.update_state(aug_state_upd, self.state, self.list_states) 141 | self.state = at.limits(self.state, self.prior_info) 142 | 143 | def check_convergence(self): 144 | """ 145 | Calculate the "convergence" of the method. Important to 146 | """ 147 | self.prev_data_misfit = self.prior_data_misfit 148 | # only calulate for the final (posterior) estimate 149 | if self.iteration == len(self.keys_da['assimindex']): 150 | assim_index = [self.keys_da['obsname'], list( 151 | np.concatenate(self.keys_da['assimindex']))] 152 | list_datatypes = self.list_datatypes 153 | obs_data_vector, pred_data = at.aug_obs_pred_data(self.obs_data, self.pred_data, assim_index, 154 | list_datatypes) 155 | 156 | data_misfit = at.calc_objectivefun( 157 | self.full_real_obs_data, pred_data, self.full_cov_data) 158 | self.data_misfit = np.mean(data_misfit) 159 | self.data_misfit_std = np.std(data_misfit) 160 | 161 | else: # sequential updates not finished. Misfit is not relevant 162 | self.data_misfit = self.prior_data_misfit 163 | 164 | # Logical variables for conv. criteria 165 | why_stop = {'rel_data_misfit': 1 - (self.data_misfit / self.prev_data_misfit), 166 | 'data_misfit': self.data_misfit, 167 | 'prev_data_misfit': self.prev_data_misfit} 168 | 169 | self.current_state = deepcopy(self.state) 170 | if self.data_misfit == self.prev_data_misfit: 171 | self.logger.info( 172 | f'EnKF update {self.iteration} complete!') 173 | else: 174 | if self.data_misfit < self.prior_data_misfit: 175 | self.logger.info( 176 | f'EnKF update complete! Objective function decreased from {self.prior_data_misfit:0.1f} to {self.data_misfit:0.1f}.') 177 | else: 178 | self.logger.info( 179 | f'EnKF update complete! Objective function increased from {self.prior_data_misfit:0.1f} to {self.data_misfit:0.1f}.') 180 | # Return conv = False, why_stop var. 181 | return False, True, why_stop 182 | 183 | 184 | class enkf_approx(enkfMixIn, approx_update): 185 | """ 186 | MixIn the main EnKF update class with the standard analysis scheme. 187 | """ 188 | pass 189 | 190 | 191 | class enkf_full(enkfMixIn, approx_update): 192 | """ 193 | MixIn the main EnKF update class with the standard analysis scheme. Note that this class is only included for 194 | completness. The EnKF does not iterate, and the standard scheme is therefor always applied. 195 | """ 196 | pass 197 | 198 | 199 | class enkf_subspace(enkfMixIn, subspace_update): 200 | """ 201 | MixIn the main EnKF update class with the subspace analysis scheme. 202 | """ 203 | pass 204 | -------------------------------------------------------------------------------- /popt/loop/optimize.py: -------------------------------------------------------------------------------- 1 | # External imports 2 | import os 3 | import numpy as np 4 | import logging 5 | import time 6 | import pickle 7 | from abc import ABC, abstractmethod 8 | 9 | # Internal imports 10 | import popt.misc_tools.optim_tools as ot 11 | 12 | # Gets or creates a logger 13 | logger = logging.getLogger(__name__) 14 | logger.setLevel(logging.DEBUG) # set log level 15 | file_handler = logging.FileHandler('popt.log') # define file handler and set formatter 16 | formatter = logging.Formatter('%(asctime)s : %(levelname)s : %(name)s : %(message)s') 17 | file_handler.setFormatter(formatter) 18 | logger.addHandler(file_handler) # add file handler to logger 19 | console_handler = logging.StreamHandler() 20 | console_handler.setFormatter(formatter) 21 | logger.addHandler(console_handler) 22 | 23 | 24 | class Optimize(ABC): 25 | """ 26 | Class for ensemble optimization algorithms. These are classified by calculating the sensitivity or gradient using 27 | ensemble instead of classical derivatives. The loop is else as a classic optimization loop: a state (or control 28 | variable) will be iterated upon using an algorithm defined in the update_scheme package. 29 | 30 | Attributes 31 | ---------- 32 | logger : Logger 33 | Print output to screen and log-file 34 | 35 | pickle_restart_file : str 36 | Save name for pickle dump/load 37 | 38 | optimize_result : OptimizeResult 39 | Dictionary with results for the current iteration 40 | 41 | iteration : int 42 | Iteration index 43 | 44 | max_iter : int 45 | Max number of iterations 46 | 47 | restart : bool 48 | Restart flag 49 | 50 | restartsave : bool 51 | Save restart information flag 52 | 53 | Methods 54 | ------- 55 | run_loop() 56 | The main optimization loop 57 | 58 | save() 59 | Save restart file 60 | 61 | load() 62 | Load restart file 63 | 64 | calc_update() 65 | Empty dummy function, actual functionality must be defined by the subclasses 66 | 67 | """ 68 | 69 | def __init__(self, **options): 70 | """ 71 | Parameters 72 | ---------- 73 | options : dict 74 | Optimization options 75 | """ 76 | # Set the logger 77 | self.logger = logger 78 | 79 | # Save name for (potential) pickle dump/load 80 | self.pickle_restart_file = 'popt_restart_dump' 81 | 82 | # Dictionary with results for the current iteration 83 | self.optimize_result = None 84 | 85 | # Initial iteration index 86 | self.iteration = 0 87 | 88 | # Time counter and random generator 89 | self.start_time = None 90 | self.rnd = None 91 | 92 | # Max number of iterations 93 | self.max_iter = options.get('maxiter', 20) 94 | 95 | # Restart flag 96 | self.restart = options.get('restart', False) 97 | 98 | # Save restart information flag 99 | self.restartsave = options.get('restartsave', False) 100 | 101 | # Optimze with external penalty function for constraints, provide r_0 as input 102 | self.epf = options.get('epf', {}) 103 | self.epf_iteration = 0 104 | 105 | # Initialize variables (set in subclasses) 106 | self.options = None 107 | self.obj_func_values = None 108 | 109 | # Initialize number of function and jacobi evaluations 110 | self.nfev = 0 111 | self.njev = 0 112 | 113 | self.msg = 'Convergence was met :)' 114 | 115 | # Abstract function that subclasses are forced to define 116 | @abstractmethod 117 | def fun(self, x, *args, **kwargs): # objective function 118 | pass 119 | 120 | # Abstract properties that subclasses are forced to define 121 | @property 122 | @abstractmethod 123 | def xk(self): # current state 124 | pass 125 | 126 | @property 127 | @abstractmethod 128 | def ftol(self): # function tolerance 129 | pass 130 | 131 | @ftol.setter 132 | @abstractmethod 133 | def ftol(self, value): # setter for function tolerance 134 | pass 135 | 136 | def run_loop(self): 137 | """ 138 | This is the main optimization loop. 139 | """ 140 | 141 | # If it is a restart run, we load the self info that exists in the pickle save file. 142 | if self.restart: 143 | try: 144 | self.load() 145 | except (FileNotFoundError, pickle.UnpicklingError) as e: 146 | raise RuntimeError(f"Failed to load restart file '{self.pickle_restart_file}': {e}") 147 | # Set the random generator to be the saved value 148 | np.random.set_state(self.rnd) 149 | else: 150 | # delete potential restart files to avoid any problems 151 | if self.pickle_restart_file in [f for f in os.listdir('.') if os.path.isfile(f)]: 152 | os.remove(self.pickle_restart_file) 153 | self.iteration += 1 154 | 155 | # Check if external penalty function (epf) for handling constraints should be used 156 | epf_not_converged = True 157 | previous_state = None 158 | if self.epf: 159 | previous_state = self.xk 160 | logger.info(f' -----> EPF-EnOpt: {self.epf_iteration}, {self.epf["r"]} (outer iteration, penalty factor)') # print epf info 161 | 162 | while epf_not_converged: # outer loop using epf 163 | 164 | # Run a while loop until max iterations or convergence is reached 165 | is_successful = True 166 | while self.iteration <= self.max_iter and is_successful: 167 | 168 | # Update control variable 169 | is_successful = self.calc_update() 170 | 171 | # Save restart file (if requested) 172 | if self.restartsave: 173 | self.rnd = np.random.get_state() # get the current random state 174 | self.save() 175 | 176 | # Check if max iterations was reached 177 | if self.iteration >= self.max_iter: 178 | self.optimize_result['message'] = 'Iterations stopped due to max iterations reached!' 179 | else: 180 | if not isinstance(self.msg, str): self.msg = '' 181 | self.optimize_result['message'] = self.msg 182 | 183 | # Logging some info to screen 184 | logger.info(' Optimization converged in %d iterations ', self.iteration-1) 185 | logger.info(' Optimization converged with final obj_func = %.4f', 186 | np.mean(self.optimize_result['fun'])) 187 | logger.info(' Total number of function evaluations = %d', self.optimize_result['nfev']) 188 | logger.info(' Total number of jacobi evaluations = %d', self.optimize_result['njev']) 189 | if self.start_time is not None: 190 | logger.info(' Total elapsed time = %.2f minutes', (time.perf_counter()-self.start_time)/60) 191 | logger.info(' ============================================') 192 | 193 | # Test for convergence of outer epf loop 194 | epf_not_converged = False 195 | if self.epf: 196 | if self.epf_iteration > self.epf['max_epf_iter']: # max epf_iterations set to 10 197 | logger.info(f' -----> EPF-EnOpt: maximum epf iterations reached') # print epf info 198 | break 199 | p = np.abs(previous_state-self.xk) / (np.abs(previous_state) + 1.0e-9) 200 | conv_crit = self.epf['conv_crit'] 201 | if np.any(p > conv_crit): 202 | epf_not_converged = True 203 | previous_state = self.xk 204 | self.epf['r'] *= self.epf['r_factor'] # increase penalty factor 205 | self.ftol *= self.epf['tol_factor'] # decrease tolerance 206 | self.obj_func_values = self.fun(self.xk, epf = self.epf) 207 | self.iteration = 0 208 | info_str = ' {:<10} {:<10} {:<15} {:<15} {:<15} '.format('iter', 'alpha_iter', 209 | 'obj_func', 'step-size', 'cov[0,0]') 210 | self.logger.info(info_str) 211 | self.logger.info(' {:<21} {:<15.4e}'.format(self.iteration, np.mean(self.obj_func_values))) 212 | self.epf_iteration += 1 213 | optimize_result = ot.get_optimize_result(self) 214 | ot.save_optimize_results(optimize_result) 215 | self.nfev += 1 216 | self.iteration = +1 217 | r = self.epf['r'] 218 | logger.info(f' -----> EPF-EnOpt: {self.epf_iteration}, {r} (outer iteration, penalty factor)') # print epf info 219 | else: 220 | logger.info(f' -----> EPF-EnOpt: converged, no variables changed more than {conv_crit*100} %') # print epf info 221 | final_obj_no_penalty = str(round(float(np.mean(self.fun(self.xk))),4)) 222 | logger.info(f' -----> EPF-EnOpt: objective value without penalty = {final_obj_no_penalty}') # print epf info 223 | 224 | 225 | def save(self): 226 | """ 227 | We use pickle to dump all the information we have in 'self'. Can be used, e.g., if some error has occurred. 228 | """ 229 | # Open save file and dump all info. in self 230 | with open(self.pickle_restart_file, 'wb') as f: 231 | pickle.dump(self.__dict__, f) 232 | 233 | def load(self): 234 | """ 235 | Load a pickled file and save all info. in self. 236 | """ 237 | # Open file and read with pickle 238 | with open(self.pickle_restart_file, 'rb') as f: 239 | tmp_load = pickle.load(f) 240 | 241 | # Save in 'self' 242 | self.__dict__.update(tmp_load) 243 | 244 | @abstractmethod 245 | def calc_update(self): 246 | """ 247 | This is an empty dummy function. Actual functionality must be defined by the subclasses. 248 | """ 249 | pass 250 | -------------------------------------------------------------------------------- /misc/grid/sector.py: -------------------------------------------------------------------------------- 1 | """\ 2 | Extract a sector from an existing cornerpoint grid. 3 | """ 4 | import argparse 5 | import collections 6 | import logging 7 | import numpy 8 | import re 9 | import sys 10 | 11 | import misc.grid as pyr 12 | import misc.grdecl as grdecl 13 | 14 | 15 | # add a valid log in case we are not run through the main program which 16 | # sets up one for us 17 | log = logging.getLogger(__name__) # pylint: disable=invalid-name 18 | log.addHandler(logging.NullHandler()) 19 | 20 | # three consecutive integers, separated by comma, and perhaps some spaces 21 | # thrown in for readability, optionally enclosed in parenthesis 22 | tuple_format = re.compile(r'\(?([0-9]+)\ *\,\ *([0-9]+)\ *\,\ *([0-9]+)\)?') 23 | 24 | 25 | def parse_tuple(corner): 26 | """ 27 | Parse a coordinate specification string into a tuple of zero-based coordinates. 28 | 29 | Parameters 30 | ---------- 31 | corner : str 32 | Coordinate specification in the format "(i1,j1,k1)". 33 | 34 | Returns 35 | ------- 36 | tuple of int 37 | The parsed tuple, converted into zero-based coordinates and in Python-matrix order: (k, j, i). 38 | """ 39 | # let the regular expression engine parse the string 40 | match = re.match(tuple_format, corner.strip()) 41 | 42 | # if the string matched, then we know that each of the sub-groups can be 43 | # parsed into strings successfully. group 0 is the entire string, so we 44 | # get natural numbering into the parenthesized expressions. subtract one 45 | # to get zero-based coordinates 46 | if match: 47 | i = int(match.group(1)) - 1 48 | j = int(match.group(2)) - 1 49 | k = int(match.group(3)) - 1 50 | return (k, j, i) 51 | # if we didn't got any valid string, then return a bottom value 52 | else: 53 | return None 54 | 55 | 56 | def sort_tuples(corner, opposite): 57 | """ 58 | Parameters 59 | ---------- 60 | corner : tuple of int 61 | Coordinates of one corner. 62 | opposite : tuple of int 63 | Coordinates of the opposite corner. 64 | 65 | Returns 66 | ------- 67 | tuple of tuple of int 68 | The two tuples, but with coordinates interchanged so that one corner is always in the lower, left, back and the other is in the upper, right, front. 69 | """ 70 | # pick out the most extreme variant in either direction, into each its own 71 | # variable; this may be the same as the input or not, but at least we know 72 | # for sure when we return from this method 73 | least = (min(corner[0], opposite[0]), 74 | min(corner[1], opposite[1]), 75 | min(corner[2], opposite[2])) 76 | most = (max(corner[0], opposite[0]), 77 | max(corner[1], opposite[1]), 78 | max(corner[2], opposite[2])) 79 | return (least, most) 80 | 81 | 82 | def extract_dimens(least, most): 83 | """ 84 | Build a new dimension tuple for a submodel. 85 | 86 | Parameters 87 | ---------- 88 | least : tuple of int 89 | Lower, left-most, back corner of submodel, (k1, j1, i1). 90 | most : tuple of int 91 | Upper, right-most, front corner of submodel, (k2, j2, i2). 92 | 93 | Returns 94 | ------- 95 | numpy.ndarray 96 | Dimensions of the submodel. 97 | """ 98 | # split the corners into constituents 99 | k1, j1, i1 = least 100 | k2, j2, i2 = most 101 | 102 | # make an array out of the cartesian distance of the two corners 103 | sector_dimens = numpy.array([i2-i1+1, j2-j1+1, k2-k1+1], dtype=numpy.int32) 104 | return sector_dimens 105 | 106 | 107 | def extract_coord(coord, least, most): 108 | """ 109 | Extract the coordinate pillars for a submodel. 110 | 111 | Parameters 112 | ---------- 113 | coord : numpy.ndarray 114 | Coordinate pillars for the entire grid with shape (nj+1, ni+1, 2, 3). 115 | least : tuple of int 116 | Lower, left-most, back corner of submodel, (k1, j1, i1). 117 | most : tuple of int 118 | Upper, right-most, front corner of submodel, (k2, j2, i2). 119 | 120 | Returns 121 | ------- 122 | numpy.ndarray 123 | Coordinate pillars for the submodel with shape (j2-j1+2, i2-i1+2, 2, 3). 124 | """ 125 | # split the corners into constituents 126 | k1, j1, i1 = least 127 | k2, j2, i2 = most 128 | 129 | # add one to get the pillar on the other side of the element, so that 130 | # we include the last element, add one more since Python indexing is 131 | # end-exclusive 132 | sector_coord = coord[j1:(j2+2), i1:(i2+2), :, :] 133 | return sector_coord 134 | 135 | 136 | def extract_zcorn(zcorn, least, most): 137 | """ 138 | Extract hinge depth values for a submodel from the entire grid. 139 | 140 | Parameters 141 | ---------- 142 | zcorn : numpy.ndarray 143 | Hinge depth values for the entire grid with shape (nk, 2, nj, 2, ni, 2). 144 | least : tuple of int 145 | Lower, left-most, back corner of submodel, (k1, j1, i1). 146 | most : tuple of int 147 | Upper, right-most, front corner of submodel, (k2, j2, i2). 148 | 149 | Returns 150 | ------- 151 | numpy.ndarray 152 | Hinge depth values for the submodel with shape (k2-k1+1, 2, j2-j1+1, 2, i2-i1+1). 153 | """ 154 | # split the corners into constituents 155 | k1, j1, i1 = least 156 | k2, j2, i2 = most 157 | 158 | # add one since Python-indexing is end-exclusive, and we want to include 159 | # the element in the opposite corner. all eight hinges are returned for 160 | # each element (we only select in every other dimension) 161 | sector_zcorn = zcorn[k1:(k2+1), :, j1:(j2+1), :, i1:(i2+1), :] 162 | return sector_zcorn 163 | 164 | 165 | def extract_cell_prop(prop, least, most): 166 | """ 167 | Extract the property values for a submodel. 168 | 169 | Parameters 170 | ---------- 171 | prop : numpy.ndarray 172 | Property values for each cell in the entire grid with shape (nk, nj, ni). 173 | least : tuple of int 174 | Lower, left-most, back corner of submodel, (k1, j1, i1). 175 | most : tuple of int 176 | Upper, right-most, front corner of submodel, (k2, j2, i2). 177 | 178 | Returns 179 | ------- 180 | numpy.ndarray 181 | Property values for each cell in the submodel with shape (k2-k1+1, j2-j1+1, i2-i1+1). 182 | """ 183 | # split the corners into constituents 184 | k1, j1, i1 = least 185 | k2, j2, i2 = most 186 | 187 | # add one since Python-indexing is end-exclusive, and we want to include 188 | # the element in the opposite corner. 189 | sector_prop = prop[k1:(k2+1), j1:(j2+1), i1:(i2+1)] 190 | return sector_prop 191 | 192 | 193 | def extract_grid(grid, least, most): 194 | """ 195 | Extract a submodel from a full grid. 196 | 197 | Parameters 198 | ---------- 199 | grid : dict 200 | Attributes of the full grid, such as COORD, ZCORN, ACTNUM. 201 | least : tuple of int 202 | Lower, left-most, back corner of the submodel, (k1, j1, i1). 203 | most : tuple of int 204 | Upper, right-most, front corner of the submodel, (k2, j2, i2). 205 | 206 | Returns 207 | ------- 208 | dict 209 | Attributes of the sector model. 210 | """ 211 | # create a new grid and fill extract standard properties 212 | sector = collections.OrderedDict() 213 | sector['DIMENS'] = extract_dimens(least, most) 214 | sector['COORD'] = extract_coord(grid['COORD'], least, most) 215 | sector['ZCORN'] = extract_zcorn(grid['ZCORN'], least, most) 216 | sector['ACTNUM'] = extract_cell_prop(grid['ACTNUM'], least, most) 217 | 218 | # then extract all extra cell properties, such as PORO, PERMX that 219 | # may have been added 220 | for prop_name in grid: 221 | # ignore the standard properties; they have already been converted 222 | # by specialized methods above 223 | prop_upper = prop_name.upper() 224 | std_prop = ((prop_upper == 'DIMENS') or (prop_upper == 'COORD') or 225 | (prop_upper == 'ZCORN') or (prop_upper == 'ACTNUM')) 226 | # use the generic method to convert this property 227 | if not std_prop: 228 | sector[prop_name] = extract_cell_prop(grid[prop_name], least, most) 229 | 230 | return sector 231 | 232 | 233 | def main(*args): 234 | """Read a data file to see if it parses OK.""" 235 | # setup simple logging where we prefix each line with a letter code 236 | logging.basicConfig(level=logging.INFO, 237 | format="%(levelname).1s: %(message).76s") 238 | 239 | # parse command-line arguments 240 | parser = argparse.ArgumentParser() 241 | parser.add_argument("infile", metavar="infile.grdecl", help="Input model") 242 | parser.add_argument("outfile", metavar="outfile", 243 | help="Output sector model") 244 | parser.add_argument("first", metavar="i1,j1,k1", type=parse_tuple, 245 | help="A corner of the sector model (one-based)") 246 | parser.add_argument("last", metavar="i2,j2,k2", type=parse_tuple, 247 | help="The opposite corner of the sector (one-based)") 248 | parser.add_argument("--dialect", choices=['ecl', 'cmg'], default='ecl') 249 | parser.add_argument("--verbose", action='store_true') 250 | parser.add_argument("--quiet", action='store_true') 251 | cmd_args = parser.parse_args(*args) 252 | 253 | # adjust the verbosity of the program 254 | if cmd_args.verbose: 255 | logging.getLogger(__name__).setLevel(logging.DEBUG) 256 | if cmd_args.quiet: 257 | logging.getLogger(__name__).setLevel(logging.NOTSET) 258 | 259 | # read the input file 260 | log.info('Reading full grid') 261 | grid = pyr.read_grid(cmd_args.infile) 262 | 263 | # get the two opposite corners that defines the submodel 264 | log.info('Determining scope of sector model') 265 | least, most = sort_tuples(cmd_args.first, cmd_args.last) 266 | log.info('Sector model is (%d, %d, %d)-(%d, %d, %d)', 267 | least[2]+1, least[1]+1, least[0]+1, 268 | most[2]+1, most[1]+1, most[0]+1) 269 | 270 | # extract the data for the submodel into a new grid 271 | log.info('Extracting sector model from full grid') 272 | submodel = extract_grid(grid, least, most) 273 | 274 | # write this grid to output 275 | log.info('Writing sector model to disk') 276 | grdecl.write(cmd_args.outfile, submodel, cmd_args.dialect, 277 | multi_file=True) 278 | 279 | 280 | # if this module is called as a standalone program, then pass all the 281 | # arguments, except the program name 282 | if __name__ == "__main__": 283 | main(sys.argv[1:]) 284 | -------------------------------------------------------------------------------- /popt/update_schemes/genopt.py: -------------------------------------------------------------------------------- 1 | """Non-Gaussian generalisation of EnOpt.""" 2 | # External imports 3 | import numpy as np 4 | from numpy import linalg as la 5 | import time 6 | 7 | # Internal imports 8 | from popt.misc_tools import optim_tools as ot 9 | from popt.loop.optimize import Optimize 10 | import popt.update_schemes.subroutines.optimizers as opt 11 | from popt.update_schemes.subroutines.cma import CMA 12 | 13 | 14 | class GenOpt(Optimize): 15 | 16 | def __init__(self, fun, x, args, jac, jac_mut, corr_adapt=None, bounds=None, **options): 17 | 18 | """ 19 | Parameters 20 | ---------- 21 | fun : callable 22 | objective function 23 | 24 | x : ndarray 25 | Initial state 26 | 27 | args : tuple 28 | Initial covariance 29 | 30 | jac : callable 31 | Gradient function 32 | 33 | jac_mut : callable 34 | Mutation gradient function 35 | 36 | corr_adapt : callable 37 | Function for correalation matrix adaption 38 | 39 | bounds : list, optional 40 | (min, max) pairs for each element in x. None is used to specify no bound. 41 | 42 | options : dict 43 | Optimization options 44 | """ 45 | 46 | # init PETEnsemble 47 | super(GenOpt, self).__init__(**options) 48 | 49 | def __set__variable(var_name=None, defalut=None): 50 | if var_name in options: 51 | return options[var_name] 52 | else: 53 | return defalut 54 | 55 | # Set input as class variables 56 | self.options = options # options 57 | self.function = fun # objective function 58 | self.jac = jac # gradient function 59 | self.jac_mut = jac_mut # mutation function 60 | self.corr_adapt = corr_adapt # correlation adaption function 61 | self.bounds = bounds # parameter bounds 62 | self.mean_state = x # initial mean state 63 | self.theta = args[0] # initial theta and correlation 64 | self.corr = args[1] # inital correlation 65 | 66 | # Set other optimization parameters 67 | self.obj_func_tol = __set__variable('obj_func_tol', 1e-6) 68 | self.alpha = __set__variable('alpha', 0.1) 69 | self.alpha_theta = __set__variable('alpha_theta', 0.1) 70 | self.alpha_corr = __set__variable('alpha_theta', 0.1) 71 | self.beta = __set__variable('beta', 0.0) # this is stored in the optimizer class 72 | self.nesterov = __set__variable('nesterov', False) # use Nesterov acceleration if value is true 73 | self.alpha_iter_max = __set__variable('alpha_maxiter', 5) 74 | self.max_resample = __set__variable('resample', 0) 75 | self.normalize = __set__variable('normalize', True) 76 | self.cov_factor = __set__variable('cov_factor', 0.5) 77 | 78 | # Initialize other variables 79 | self.state_step = 0 # state step 80 | self.theta_step = 0 # covariance step 81 | 82 | # Calculate objective function of startpoint 83 | if not self.restart: 84 | self.start_time = time.perf_counter() 85 | self.obj_func_values = self.function(self.mean_state) 86 | self.nfev += 1 87 | self.optimize_result = ot.get_optimize_result(self) 88 | ot.save_optimize_results(self.optimize_result) 89 | if self.logger is not None: 90 | self.logger.info(' Running optimization...') 91 | info_str = ' {:<10} {:<10} {:<15} {:<15} {:<10} {:<10} {:<10} {:<10} '.format('iter', 92 | 'alpha_iter', 93 | 'obj_func', 94 | 'step-size', 95 | 'alpha0', 96 | 'beta0', 97 | 'max corr', 98 | 'min_corr') 99 | self.logger.info(info_str) 100 | self.logger.info(' {:<21} {:<15.4e}'.format(self.iteration, 101 | round(np.mean(self.obj_func_values),4))) 102 | 103 | # Initialize optimizer 104 | optimizer = __set__variable('optimizer', 'GD') 105 | if optimizer == 'GD': 106 | self.optimizer = opt.GradientDescent(self.alpha, self.beta) 107 | elif optimizer == 'Adam': 108 | self.optimizer = opt.Adam(self.alpha, self.beta) 109 | 110 | # The GenOpt class self-ignites, and it is possible to send the EnOpt class as a callale method to scipy.minimize 111 | self.run_loop() # run_loop resides in the Optimization class (super) 112 | 113 | def fun(self, x, *args, **kwargs): 114 | return self.function(x, *args, **kwargs) 115 | 116 | @property 117 | def xk(self): 118 | return self.mean_state 119 | 120 | @property 121 | def fk(self): 122 | return self.obj_func_values 123 | 124 | @property 125 | def ftol(self): 126 | return self.obj_func_tol 127 | 128 | @ftol.setter 129 | def ftol(self, value): 130 | self.obj_func_tol = value 131 | 132 | def calc_update(self): 133 | """ 134 | Update using steepest descent method with ensemble gradients 135 | """ 136 | 137 | # Initialize variables for this step 138 | improvement = False 139 | success = False 140 | resampling_iter = 0 141 | self.optimizer.restore_parameters() 142 | 143 | while improvement is False: # resampling loop 144 | 145 | # Shrink covariance each time we try resampling 146 | shrink = self.cov_factor ** resampling_iter 147 | 148 | # Calculate gradient 149 | if self.nesterov: 150 | gradient = self.jac(self.mean_state + self.beta*self.state_step, 151 | self.theta + self.beta*self.theta_step, self.corr) 152 | else: 153 | gradient = self.jac(self.mean_state, self.theta, self.corr) 154 | self.njev += 1 155 | 156 | # Compute the mutation gradient 157 | gradient_theta, en_matrices = self.jac_mut(return_ensembles=True) 158 | if self.normalize: 159 | gradient /= np.maximum(la.norm(gradient, np.inf), 1e-12) # scale the gradient with inf-norm 160 | gradient_theta /= np.maximum(la.norm(gradient_theta, np.inf), 1e-12) # scale the mutation with inf-norm 161 | 162 | # Initialize for this step 163 | alpha_iter = 0 164 | 165 | while improvement is False: # backtracking loop 166 | 167 | new_state, new_step = self.optimizer.apply_update(self.mean_state, gradient, iter=self.iteration) 168 | new_state = ot.clip_state(new_state, self.bounds) 169 | 170 | # Calculate new objective function 171 | new_func_values = self.function(new_state) 172 | self.nfev += 1 173 | 174 | if np.mean(self.obj_func_values) - np.mean(new_func_values) > self.obj_func_tol: 175 | 176 | # Update objective function values and state 177 | self.obj_func_values = new_func_values 178 | self.mean_state = new_state 179 | self.state_step = new_step 180 | self.alpha = self.optimizer.get_step_size() 181 | 182 | # Update theta (currently we don't apply backtracking for theta) 183 | self.theta_step = self.beta*self.theta_step - self.alpha_theta*gradient_theta 184 | self.theta = self.theta + self.theta_step 185 | 186 | # update correlation matrix 187 | if isinstance(self.corr_adapt, CMA): 188 | enZ = en_matrices['gaussian'] 189 | enJ = en_matrices['objective'] 190 | self.corr = self.corr_adapt(cov = self.corr, 191 | step = new_step/self.alpha, 192 | X = enZ, 193 | J = enJ) 194 | 195 | elif callable(self.corr_adapt): 196 | self.corr = self.corr - self.alpha_corr*self.corr_adapt() 197 | 198 | # Write logging info 199 | if self.logger is not None: 200 | corr_max = round(np.max(self.corr-np.eye(self.corr.shape[0])), 3) 201 | corr_min = round(np.min(self.corr), 3) 202 | info_str_iter = ' {:<10} {:<10} {:<15.4e} {:<15} {:<10} {:<10} {:<10} {:<10}'.\ 203 | format(self.iteration, 204 | alpha_iter, 205 | round(np.mean(self.obj_func_values),4), 206 | self.alpha, 207 | round(self.theta[0, 0],2), 208 | round(self.theta[0, 1],2), 209 | corr_max, 210 | corr_min) 211 | 212 | self.logger.info(info_str_iter) 213 | 214 | # Update step size in the one-dimensional case 215 | if new_state.size == 1: 216 | self.optimizer.step_size /= 2 217 | 218 | # Iteration was a success 219 | improvement = True 220 | success = True 221 | self.optimizer.restore_parameters() 222 | 223 | # Save variables defined in savedata keyword. 224 | self.optimize_result = ot.get_optimize_result(self) 225 | ot.save_optimize_results(self.optimize_result) 226 | 227 | # Update iteration counter if iteration was successful and save current state 228 | self.iteration += 1 229 | 230 | else: 231 | 232 | # If we do not have a reduction in the objective function, we reduce the step limiter 233 | if alpha_iter < self.alpha_iter_max: 234 | self.optimizer.apply_backtracking() # decrease alpha 235 | alpha_iter += 1 236 | elif (resampling_iter < self.max_resample and 237 | np.mean(new_func_values) - np.mean(self.obj_func_values) > 0): # update gradient 238 | resampling_iter += 1 239 | self.optimizer.restore_parameters() 240 | break 241 | else: 242 | success = False 243 | return success 244 | 245 | return success 246 | -------------------------------------------------------------------------------- /popt/loop/extensions.py: -------------------------------------------------------------------------------- 1 | # External imports 2 | import numpy as np 3 | from scipy import stats 4 | from scipy.special import polygamma, digamma 5 | 6 | # Internal imports 7 | from popt.misc_tools import optim_tools as ot 8 | 9 | # NB! THIS FILE IS NOT USED ANYMORE 10 | 11 | __all__ = ['GenOptExtension'] 12 | 13 | class GenOptExtension: 14 | ''' 15 | Class that contains all the operations on the mutation distribution of GenOpt 16 | ''' 17 | def __init__(self, x, cov, theta0=[20.0, 20.0], func=None, ne=None): 18 | ''' 19 | Parameters 20 | ---------- 21 | x : array_like, shape (d,) 22 | Initial control vector. Used initally to get the dimensionality of the problem. 23 | 24 | cov : array_like, shape (d,d) 25 | Initial covaraince matrix. Used to construct the correlation matrix and 26 | epsilon parameter of GenOpt 27 | 28 | theta0 : list, of length 2 ([alpha, beta]) 29 | Initial alpha and beta parameter of the marginal Beta distributions. 30 | 31 | func : callable (optional) 32 | An objective function that can be used later for the gradeint. 33 | Can also be passed directly to the gradeint fucntion. 34 | 35 | ne : int 36 | ''' 37 | self.dim = x.size # dimension of state 38 | self.corr = ot.cov2corr(cov) # initial correlation 39 | self.var = np.diag(cov) # initial varaince 40 | self.theta = np.tile(theta0, (self.dim,1)) # initial theta parameters, shape (dim, 2) 41 | self.eps = var2eps(self.var, self.theta) # epsilon parameter(s). SET BY VARIANCE (NOT MANUALLY BU USER ANYMORE)! 42 | self.func = func # an objective function (optional) 43 | self.size = ne # ensemble size 44 | 45 | def update_distribution(self, theta, corr): 46 | ''' 47 | Updates the parameters (theta and corr) of the distirbution. 48 | 49 | Parameters 50 | ---------- 51 | theta : array_like, shape (d,2) 52 | Contains the alpha (first column) and beta (second column) 53 | of the marginal distirbutions. 54 | 55 | corr : array_like, shape (d,d) 56 | Correlation matrix 57 | ''' 58 | self.theta = theta 59 | self.corr = corr 60 | return 61 | 62 | def get_theta(self): 63 | return self.theta 64 | 65 | def get_corr(self): 66 | return self.corr 67 | 68 | def get_cov(self): 69 | 70 | std = np.zeros(self.dim) 71 | for d in range(self.dim): 72 | std[d] = stats.beta(*self.theta[d]).std() * 2 * self.eps[d] 73 | 74 | return ot.corr2cov(self.corr, std=std) 75 | 76 | def sample(self, size): 77 | ''' 78 | Samples the mutation distribution as described in the GenOpt paper (NOT PUBLISHED YET!) 79 | 80 | Parameters 81 | ---------- 82 | size : int 83 | Ensemble size (ne). Size of the sample to be drawn. 84 | 85 | Returns 86 | ------- 87 | out : tuple, (enZ, enX) 88 | 89 | enZ : array_like, shape (ne,d) 90 | Zero-mean Gaussain ensemble, drawn with the correlation matrix, corr 91 | 92 | enX : array_like, shape (ne,d) 93 | The drawn ensemble matrix, X ~ p(x|θ,R) (GenOpt pdf) 94 | ''' 95 | # Sample normal distribution with correlation 96 | enZ = np.random.multivariate_normal(mean=np.zeros(self.dim), 97 | cov=self.corr, 98 | size=size) 99 | 100 | # Transform Z to a uniform variable U 101 | enU = stats.norm.cdf(enZ) 102 | 103 | # Initialize enX 104 | enX = np.zeros_like(enZ) 105 | 106 | # Loop over dim 107 | for d in range(self.dim): 108 | marginal = stats.beta(*self.theta[d]) # Make marginal dist. 109 | enX[:,d] = marginal.ppf(enU[:,d]) # Transform U to marginal variables X 110 | 111 | return enZ, enX 112 | 113 | def eps_trafo(self, x, enX): 114 | ''' 115 | Performs the epsilon transformation, 116 | X ∈ [0, 1] ---> Y ∈ [x-ε, x+ε] 117 | 118 | Parameters 119 | ---------- 120 | x : array_like, shape (d,) 121 | Current state vector. 122 | 123 | enX : array_like, shape (ne,d) 124 | Ensemble matrix X sampled from GenOpt distribution 125 | 126 | Returns 127 | ------- 128 | out : array_like, shape (ne,d) 129 | Epsilon transfromed ensemble matrix, Y 130 | ''' 131 | enY = np.zeros_like(enX) # tranfomred ensemble 132 | 133 | # loop over dimenstion 134 | for d, xd in enumerate(x): 135 | eps = self.eps[d] 136 | 137 | a = (xd-eps) - ( (xd-eps)*(xd-eps < 0) ) \ 138 | - ( (xd+eps-1)*(xd+eps > 1) ) \ 139 | + (xd+eps-1)*(xd-eps < 0)*(xd+eps > 1) #Lower bound of ensemble 140 | 141 | b = (xd+eps) - ( (xd-eps)*(xd-eps < 0) ) \ 142 | - ( (xd+eps-1)*(xd+eps > 1) ) \ 143 | + (xd-eps)*(xd-eps < 0)*(xd+eps > 1) #Upper bound of ensemble 144 | 145 | enY[:,d] = a + enX[:, d]*(b-a) #Component-wise trafo. 146 | 147 | return enY 148 | 149 | def gradient(self, x, *args, **kwargs): 150 | ''' 151 | Calcualtes the average gradient of func using Stein's Lemma. 152 | Described in GenOpt paper. 153 | 154 | Parameters 155 | ---------- 156 | x : array_like, shape (d,) 157 | Current state vector. 158 | 159 | args : (theta, corr) 160 | theta (parameters of distribution), shape (d,2) 161 | corr (correlation matrix), shape (d,d) 162 | 163 | kwargs : 164 | func : callable objectvie function 165 | ne : ensemble size 166 | 167 | Returns 168 | ------- 169 | out : array_like, shape (d,) 170 | The average gradient. 171 | ''' 172 | # check for objective fucntion 173 | func = kwargs.get('func') 174 | if (func is None) and (self.func is not None): 175 | func = self.func 176 | else: 177 | raise ValueError('No objectvie fucntion given. Please pass keyword argument: func=') 178 | 179 | # check for ensemble size 180 | if 'ne' in kwargs: 181 | ne = kwargs.get('ne') 182 | elif self.size is None: 183 | ne = max(int(0.25*self.dim), 5) 184 | else: 185 | ne = self.size 186 | 187 | # update dist 188 | if args: 189 | self.update_distribution(*args) 190 | 191 | # sample 192 | self.enZ, self.enX = self.sample(size=ne) 193 | 194 | # create ensembles 195 | self.enY = self.eps_trafo(x, self.enX) 196 | self.enJ = func(self.enY.T) 197 | meanJ = self.enJ.mean() 198 | 199 | # parameters 200 | a = self.theta[:,0] # shape (d,) 201 | b = self.theta[:,1] # shape (d,) 202 | 203 | # copula term 204 | matH = np.linalg.inv(self.corr) - np.identity(self.dim) 205 | 206 | # empty gradients 207 | gx = np.zeros(self.dim) 208 | gt = np.zeros_like(self.theta) 209 | 210 | for d in range(self.dim): 211 | for n in range(ne): 212 | 213 | j = self.enJ[n] 214 | x = self.enX[n] 215 | z = self.enZ[n] 216 | 217 | # gradient componets 218 | g_marg = (a[d]-1)/x[d] - (b[d]-1)/(1-x[d]) 219 | g_dist = np.inner(matH[d], z)*stats.beta.pdf(x[d], a[d], b[d])/stats.norm.pdf(z[d]) 220 | gx[d] += (j-meanJ)*(g_marg - g_dist) 221 | 222 | # mutation gradient 223 | log_term = [np.log(x[d]), np.log(1-x[d])] 224 | psi_term = [delA(a[d], b[d]), delA(b[d], a[d])] 225 | gt[d] += (j-meanJ)*(np.array(log_term)-np.array(psi_term)) 226 | 227 | # fisher matrix 228 | f_inv = np.linalg.inv(self.fisher_matrix(a[d], b[d])) 229 | gt[d] = np.matmul(f_inv, gt[d]) 230 | 231 | 232 | gx = -np.matmul(self.get_cov(), gx)/(2*self.eps*(ne-1)) 233 | self.grad_theta = gt/(ne-1) 234 | 235 | return gx 236 | 237 | def mutation_gradient(self, x=None, *args, **kwargs): 238 | ''' 239 | Returns the mutation gradient of theta. It is actually calulated in 240 | self.ensemble_gradient. 241 | 242 | Parameters 243 | ---------- 244 | kwargs: 245 | return_ensemble : bool 246 | If True, all the ensemble matrices are also returned in a dictionary. 247 | 248 | Returns 249 | ------- 250 | out : array_like, shape (d,2) 251 | Mutation gradeint of theta 252 | 253 | NB! If return_ensembles=True, the ensmebles are also returned! 254 | ''' 255 | if 'return_ensembles' in kwargs: 256 | ensembles = {'gaussian' : self.enZ, 257 | 'vanilla' : self.enX, 258 | 'transformed': self.enY, 259 | 'objective' : self.enJ} 260 | return self.grad_theta, ensembles 261 | else: 262 | return self.grad_theta 263 | 264 | def corr_gradient(self): 265 | ''' 266 | Returns the mutation gradeint of the correlation matrix 267 | ''' 268 | enZ = self.enZ 269 | enJ = self.enJ 270 | ne = np.squeeze(enJ).size 271 | grad_corr = np.zeros_like(self.corr) 272 | 273 | for n in range(ne): 274 | grad_corr += enJ[n]*(np.outer(enZ[:,n], enZ[:,n]) - self.corr) 275 | 276 | np.fill_diagonal(grad_corr, 0) 277 | corr_gradient = grad_corr/(ne-1) 278 | 279 | return corr_gradient 280 | 281 | def fisher_matrix(self, alpha, beta): 282 | ''' 283 | Calculates the Fisher matrix of a Beta distribution. 284 | 285 | Parameters 286 | ---------------------------------------------- 287 | alpha : float 288 | alpha parameter in Beta distribution 289 | 290 | beta : float 291 | beta parameter in Beta distribution 292 | 293 | Returns 294 | ---------------------------------------------- 295 | out : array_like, of shape (2, 2) 296 | Fisher matrix 297 | ''' 298 | a = alpha 299 | b = beta 300 | 301 | upper_row = [polygamma(1, a) - polygamma(1, a+b), -polygamma(1, a + b)] 302 | lower_row = [-polygamma(1, a + b), polygamma(1, b) - polygamma(1, a+b)] 303 | fisher_matrix = np.array([upper_row, 304 | lower_row]) 305 | return fisher_matrix 306 | 307 | 308 | # Some helping functions 309 | def var2eps(var, theta): 310 | alphas = theta[:,0] 311 | betas = theta[:,1] 312 | frac = alphas*betas / ( (alphas+betas)**2 * (alphas+betas+1) ) 313 | epsilon = np.sqrt(0.25*var/frac) 314 | return epsilon 315 | 316 | def delA(a, b): 317 | ''' 318 | Calculates the expression psi(a) - psi(a+b), 319 | where psi() is the digamma function. 320 | 321 | Parameters 322 | -------------------------------------------- 323 | a : float 324 | b : float 325 | 326 | Returns 327 | -------------------------------------------- 328 | out : float 329 | ''' 330 | return digamma(a)-digamma(a+b) 331 | --------------------------------------------------------------------------------