├── .github ├── check_run_scripts.sh └── workflows │ ├── cloc.yml │ ├── filechk.yml │ └── pylinter.yml ├── .gitignore ├── LICENSE ├── README.md ├── Visualization.ipynb ├── data ├── covid_conformal_scaled.pkl ├── drone_dataset_smallnoise.npy └── nridata │ ├── loc_test_springs2_noise_0.01.npy │ ├── loc_test_springs2_noise_0.05.npy │ ├── loc_train_springs2_noise_0.01.npy │ ├── loc_train_springs2_noise_0.05.npy │ ├── loc_valid_springs2_noise_0.01.npy │ └── loc_valid_springs2_noise_0.05.npy ├── fig3.png ├── requirements.txt ├── run_experiment.sh └── src ├── experiments ├── __init__.py ├── covid_exp.py ├── drone_exp.py └── particle_exp.py └── models ├── __init__.py ├── bjrnn.py ├── cfrnn.py ├── copulaCPTS.py ├── dplstm.py ├── influence ├── __init__.py ├── influence_computation.py └── influence_utils.py ├── lstm.py ├── rnn.py ├── rnn_rnn.py ├── transformer.py ├── utils.py └── vanila_copula.py /.github/check_run_scripts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | rm filechk_results.txt 4 | 5 | declare -a FilesArray=("requirements.txt" "run_experiment.sh") 6 | 7 | for val in ${FilesArray[@]}; 8 | do 9 | echo "Checking presence of $val" 10 | if test -f "$val"; 11 | then 12 | echo "$val exists" >> filechk_results.txt 13 | else 14 | echo "$val does not exist" >> filechk_results.txt 15 | fi 16 | done 17 | -------------------------------------------------------------------------------- /.github/workflows/cloc.yml: -------------------------------------------------------------------------------- 1 | name: Count Lines of Code 2 | 3 | # Controls when the action will run. Triggers the workflow on push or pull request 4 | # events but only for the main branch 5 | on: [pull_request] 6 | 7 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 8 | jobs: 9 | 10 | # This workflow contains a single job called "build" 11 | cloc: 12 | 13 | # The type of runner that the job will run on 14 | runs-on: ubuntu-latest 15 | 16 | # Steps represent a sequence of tasks that will be executed as part of the job 17 | steps: 18 | 19 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 20 | - name: Checkout repo 21 | uses: actions/checkout@v3 22 | 23 | # Runs djdefi/cloc-action 24 | - name: Install and run cloc 25 | run: | 26 | sudo apt-get install cloc 27 | cloc src --csv --quiet --report-file=cloc_report.csv 28 | 29 | - name: Read CSV 30 | id: csv 31 | uses: juliangruber/read-file-action@v1 32 | with: 33 | path: ./cloc_report.csv 34 | 35 | - name: Create MD 36 | uses: petems/csv-to-md-table-action@master 37 | id: csv-table-output 38 | with: 39 | csvinput: ${{ steps.csv.outputs.content }} 40 | 41 | - name: Write file 42 | uses: "DamianReeves/write-file-action@master" 43 | with: 44 | path: ./cloc_report.md 45 | write-mode: overwrite 46 | contents: | 47 | ${{steps.csv-table-output.outputs.markdown-table}} 48 | 49 | - name: PR comment with file 50 | uses: thollander/actions-comment-pull-request@v2 51 | with: 52 | filePath: ./cloc_report.md 53 | -------------------------------------------------------------------------------- /.github/workflows/filechk.yml: -------------------------------------------------------------------------------- 1 | name: Checks if files are present 2 | 3 | on: [pull_request] 4 | 5 | jobs: 6 | 7 | # This workflow contains a single job called "test" 8 | check_files: 9 | 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - name: Checkout repo 14 | uses: actions/checkout@v3 15 | 16 | - name: Check files script 17 | run: | 18 | chmod +x ./.github/check_run_scripts.sh 19 | ./.github/check_run_scripts.sh 20 | 21 | - name: PR comment with file 22 | uses: thollander/actions-comment-pull-request@v2 23 | with: 24 | filePath: ./filechk_results.txt 25 | -------------------------------------------------------------------------------- /.github/workflows/pylinter.yml: -------------------------------------------------------------------------------- 1 | name: Checks python code for linting 2 | 3 | on: [pull_request] 4 | 5 | jobs: 6 | 7 | # This workflow contains a single job called "test" 8 | check_pylint: 9 | 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - name: Checkout repo 14 | uses: actions/checkout@v4 15 | - name: Install flake8 16 | run: pip install flake8 17 | - name: Python Linter 18 | uses: py-actions/flake8@v2 19 | with: 20 | ignore: "E402,E731,F541,W291,E122,E127,F401,E266,E241,C901,E741,W293,F811,W503,E203,F403,F405,B007" 21 | max-line-length: "150" 22 | path: "src" 23 | plugins: "flake8-bugbear==22.1.11" 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #project specific 2 | *.DS_Store 3 | trained/* 4 | .vscode/* 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | share/python-wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | MANIFEST 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .nox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *.cover 54 | *.py,cover 55 | .hypothesis/ 56 | .pytest_cache/ 57 | cover/ 58 | 59 | # Translations 60 | *.mo 61 | *.pot 62 | 63 | # Django stuff: 64 | *.log 65 | local_settings.py 66 | db.sqlite3 67 | db.sqlite3-journal 68 | 69 | # Flask stuff: 70 | instance/ 71 | .webassets-cache 72 | 73 | # Scrapy stuff: 74 | .scrapy 75 | 76 | # Sphinx documentation 77 | docs/_build/ 78 | 79 | # PyBuilder 80 | .pybuilder/ 81 | target/ 82 | 83 | # Jupyter Notebook 84 | .ipynb_checkpoints 85 | 86 | # IPython 87 | profile_default/ 88 | ipython_config.py 89 | 90 | # pyenv 91 | # For a library or package, you might want to ignore these files since the code is 92 | # intended to run in multiple environments; otherwise, check them in: 93 | # .python-version 94 | 95 | # pipenv 96 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 97 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 98 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 99 | # install all needed dependencies. 100 | #Pipfile.lock 101 | 102 | # poetry 103 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 104 | # This is especially recommended for binary packages to ensure reproducibility, and is more 105 | # commonly ignored for libraries. 106 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 107 | #poetry.lock 108 | 109 | # pdm 110 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 111 | #pdm.lock 112 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 113 | # in version control. 114 | # https://pdm.fming.dev/#use-with-ide 115 | .pdm.toml 116 | 117 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 118 | __pypackages__/ 119 | 120 | # Celery stuff 121 | celerybeat-schedule 122 | celerybeat.pid 123 | 124 | # SageMath parsed files 125 | *.sage.py 126 | 127 | # Environments 128 | .env 129 | .venv 130 | env/ 131 | venv/ 132 | ENV/ 133 | env.bak/ 134 | venv.bak/ 135 | 136 | # Spyder project settings 137 | .spyderproject 138 | .spyproject 139 | 140 | # Rope project settings 141 | .ropeproject 142 | 143 | # mkdocs documentation 144 | /site 145 | 146 | # mypy 147 | .mypy_cache/ 148 | .dmypy.json 149 | dmypy.json 150 | 151 | # Pyre type checker 152 | .pyre/ 153 | 154 | # pytype static type analyzer 155 | .pytype/ 156 | 157 | # Cython debug symbols 158 | cython_debug/ 159 | 160 | # PyCharm 161 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 162 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 163 | # and can be added to the global gitignore or merged into this file. For a more nuclear 164 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 165 | #.idea/ 166 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Spatiotemporal Machine Learning 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Copula Conformal Prediction for Multi-step Time Series Forecasting [[Paper](https://arxiv.org/abs/2212.03281)] 2 | 3 | 4 | 5 | ## | Introduction 6 | 7 | **Copula** **C**onformal **P**rediction algorithm for multivariate, multi-step **T**ime **S**eries (CopulaCPTS) is a conformal prediction algorithm with full-horizon validity guarantee. 8 | 9 | ## | Citation 10 | 11 | [[2212.03281] Copula Conformal Prediction for Multi-step Time Series Forecasting](https://arxiv.org/abs/2212.03281) 12 | 13 | ``` 14 | @inproceedings{sun2023copula, 15 | title={Copula Conformal prediction for multi-step time series prediction}, 16 | author={Sun, Sophia Huiwen and Yu, Rose}, 17 | booktitle={The Twelfth International Conference on Learning Representations}, 18 | year={2023} 19 | } 20 | ``` 21 | 22 | ## | Installation 23 | 24 | 25 | ```bash 26 | pip install -r requirements.txt 27 | ``` 28 | 29 | ## | Datasets 30 | 31 | Please see below for links and refer to Section 5.1 and Appendix C.1 in the paper for processing details. 32 | 33 | [Particles](https://github.com/mitmul/chainer-nri) | [Drone](https://github.com/AtsushiSakai/PythonRobotics)| [Epidemiology](https://coronavirus.data.gov.uk/details/download) | [Argoverse 1](https://www.argoverse.org/av1.html) 34 | 35 | The processed files for Particles, Drone, and Epidemiology datasets are located in the `./data` directory. If you want to reporduce the visualizations, you might need to refer to the original sources for metadata. 36 | 37 | 38 | ## | Training and Testing 39 | 40 | To illustrate the usage of our code, we have included pre-generated NRI Particles data in this repository. To replicate the experiment, simply run: 41 | 42 | ```bash 43 | ./run_experiment.sh 44 | ``` 45 | 46 | ## | Recreate plots in the paper 47 | 48 | Please see ```Visualization.ipynb``` for example code for creating Figure 3 in the paper. 49 | ![fig](fig3.png) 50 | -------------------------------------------------------------------------------- /data/covid_conformal_scaled.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/CopulaCPTS/56f29640edfe5e19e37efa0eb5eb372d5c3a9c87/data/covid_conformal_scaled.pkl -------------------------------------------------------------------------------- /data/drone_dataset_smallnoise.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/CopulaCPTS/56f29640edfe5e19e37efa0eb5eb372d5c3a9c87/data/drone_dataset_smallnoise.npy -------------------------------------------------------------------------------- /data/nridata/loc_test_springs2_noise_0.01.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/CopulaCPTS/56f29640edfe5e19e37efa0eb5eb372d5c3a9c87/data/nridata/loc_test_springs2_noise_0.01.npy -------------------------------------------------------------------------------- /data/nridata/loc_test_springs2_noise_0.05.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/CopulaCPTS/56f29640edfe5e19e37efa0eb5eb372d5c3a9c87/data/nridata/loc_test_springs2_noise_0.05.npy -------------------------------------------------------------------------------- /data/nridata/loc_train_springs2_noise_0.01.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/CopulaCPTS/56f29640edfe5e19e37efa0eb5eb372d5c3a9c87/data/nridata/loc_train_springs2_noise_0.01.npy -------------------------------------------------------------------------------- /data/nridata/loc_train_springs2_noise_0.05.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/CopulaCPTS/56f29640edfe5e19e37efa0eb5eb372d5c3a9c87/data/nridata/loc_train_springs2_noise_0.05.npy -------------------------------------------------------------------------------- /data/nridata/loc_valid_springs2_noise_0.01.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/CopulaCPTS/56f29640edfe5e19e37efa0eb5eb372d5c3a9c87/data/nridata/loc_valid_springs2_noise_0.01.npy -------------------------------------------------------------------------------- /data/nridata/loc_valid_springs2_noise_0.05.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/CopulaCPTS/56f29640edfe5e19e37efa0eb5eb372d5c3a9c87/data/nridata/loc_valid_springs2_noise_0.05.npy -------------------------------------------------------------------------------- /fig3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/CopulaCPTS/56f29640edfe5e19e37efa0eb5eb372d5c3a9c87/fig3.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | copulae==0.7.7 2 | keras==2.13.1 3 | matplotlib==3.5.0 4 | numpy==1.23.5 5 | pandas==1.3.4 6 | scikit_learn==1.0.1 7 | scipy==1.12.0 8 | statsmodels==0.13.2 9 | torch==2.0.0 10 | tqdm==4.66.2 11 | -------------------------------------------------------------------------------- /run_experiment.sh: -------------------------------------------------------------------------------- 1 | python -m src.experiments.particle_exp -------------------------------------------------------------------------------- /src/experiments/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/CopulaCPTS/56f29640edfe5e19e37efa0eb5eb372d5c3a9c87/src/experiments/__init__.py -------------------------------------------------------------------------------- /src/experiments/covid_exp.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import numpy as np 3 | import pandas as pd 4 | import torch 5 | from sklearn.preprocessing import StandardScaler 6 | import matplotlib.pyplot as plt 7 | from src.models import rnn, lstm, bjrnn, cfrnn, copulaCPTS, dplstm, vanila_copula 8 | 9 | 10 | class COVIDDataset(torch.utils.data.Dataset): 11 | def __init__(self, X, Y, sequence_lengths): 12 | super(COVIDDataset, self).__init__() 13 | self.X = X 14 | self.Y = Y 15 | self.sequence_lengths = sequence_lengths 16 | 17 | def __len__(self): 18 | return len(self.X) 19 | 20 | def __getitem__(self, idx): 21 | return self.X[idx], self.Y[idx], self.sequence_lengths[idx] 22 | 23 | 24 | def experiment(train, valid, test, target_len=50, name="exp"): 25 | rnn_model = rnn.rnn( 26 | input_size=1, embedding_size=128, output_size=1, horizon=target_len 27 | ) 28 | encdec_model = lstm.lstm_seq2seq( 29 | input_size=1, output_size=1, embedding_size=128, target_len=target_len 30 | ) 31 | models = [rnn_model, encdec_model] 32 | 33 | x_train = train.X 34 | y_train = train.Y 35 | 36 | for m in models: 37 | m.train_model(x_train, y_train, n_epochs=200, batch_size=50) 38 | 39 | with open("./trained/models_%s.pkl" % name, "wb") as f: 40 | pickle.dump(models, f) 41 | 42 | x_cali = valid.X 43 | y_cali = valid.Y 44 | 45 | UQ = {} 46 | 47 | dprnn = dplstm.DPRNN( 48 | epochs=150, input_size=1, output_size=1, n_steps=target_len, dropout_prob=0.2 49 | ) 50 | dprnn.fit(x_train, y_train) 51 | UQ["dprnn"] = dprnn 52 | 53 | bj_class = bjrnn.bj_rnn(models[0], recursion_depth=15) 54 | bj_class.LOBO_residuals = np.expand_dims(bj_class.LOBO_residuals, axis=-1) 55 | UQ["bjrnn"] = bj_class 56 | 57 | cf = cfrnn.CFRNN(models[0], x_cali, y_cali) 58 | cf.calibrate() 59 | UQ["cfrnn"] = cf 60 | 61 | cf = cfrnn.CFRNN(models[1], x_cali, y_cali) 62 | cf.calibrate() 63 | UQ["cf-EncDec"] = cf 64 | 65 | copula = copulaCPTS.copulaCPTS(models[0], x_cali, y_cali) 66 | copula.calibrate() 67 | UQ["copula-rnn"] = copula 68 | 69 | copula = copulaCPTS.copulaCPTS(models[1], x_cali, y_cali) 70 | copula.calibrate() 71 | UQ["copula-EncDec"] = copula 72 | 73 | vanilla = vanila_copula.vanila_copula(models[0], x_cali, y_cali) 74 | vanilla.calibrate() 75 | UQ["vanila-copula"] = vanilla 76 | 77 | x_test = test.X 78 | y_test = test.Y 79 | 80 | areas = {} 81 | coverages = {} 82 | 83 | epsilon_ls = np.linspace(0.05, 0.50, 10) 84 | 85 | for k, uqmethod in UQ.items(): 86 | print(k) 87 | area = [] 88 | coverage = [] 89 | for eps in epsilon_ls: 90 | pred, box = uqmethod.predict(x_test, epsilon=eps) 91 | area.append(uqmethod.calc_area_1d(box)) 92 | pred = torch.tensor(pred) 93 | coverage.append(uqmethod.calc_coverage_1d(box, pred, y_test)) 94 | areas[k] = area 95 | coverages[k] = coverage 96 | 97 | with open("./trained/uq_%s.pkl" % name, "wb") as f: 98 | pickle.dump(UQ, f) 99 | with open("./trained/results_%s.pkl" % name, "wb") as f: 100 | pickle.dump((areas, coverages), f) 101 | 102 | return areas, coverages, (models, UQ) 103 | 104 | 105 | def main(): 106 | 107 | with open("data/covid_conformal_scaled.pkl", "rb") as f: 108 | train_dataset, calibration_dataset, test_dataset = pickle.load(f) 109 | 110 | for i in range(3): 111 | res = experiment( 112 | train_dataset, 113 | calibration_dataset, 114 | test_dataset, 115 | target_len=50, 116 | name="covid_daily_vanilla" + str(i), 117 | ) # target len 6 118 | print("finished run " + str(i)) 119 | 120 | del res 121 | 122 | 123 | if __name__ == "__main__": 124 | main() 125 | -------------------------------------------------------------------------------- /src/experiments/drone_exp.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | import pandas as pd 4 | import torch 5 | import pickle 6 | from src.models import rnn, lstm, bjrnn, cfrnn, copulaCPTS, dplstm, vanila_copula 7 | 8 | 9 | datapath = "data/drone_dataset_smallnoise.npy" 10 | 11 | data = np.load(datapath, allow_pickle=True) 12 | x_downsample = data[:, ::3, :].astype(np.float32) 13 | 14 | train_len = 60 15 | pred_len = 10 16 | 17 | x_train = torch.tensor(x_downsample[:600, :train_len, :], dtype=torch.float) 18 | y_train = torch.tensor( 19 | x_downsample[:600, train_len : train_len + pred_len, :3], dtype=torch.float 20 | ) 21 | 22 | x_cali = torch.tensor(x_downsample[600:800, :train_len, :], dtype=torch.float) 23 | y_cali = torch.tensor( 24 | x_downsample[600:800, train_len : train_len + pred_len, :3], dtype=torch.float 25 | ) 26 | 27 | x_test = torch.tensor(x_downsample[800:, :train_len, :], dtype=torch.float) 28 | y_test = torch.tensor( 29 | x_downsample[800:, train_len : train_len + pred_len, :3], dtype=torch.float 30 | ) 31 | 32 | 33 | def experiment( 34 | x_downsample, 35 | train_len=60, 36 | pred_len=10, 37 | name="exp", 38 | x_train=x_train, 39 | y_train=y_train, 40 | x_cali=x_cali, 41 | y_cali=y_cali, 42 | x_test=x_test, 43 | y_test=y_test, 44 | ): 45 | 46 | rnn_model = rnn.rnn( 47 | embedding_size=128, input_size=6, output_size=3, horizon=pred_len 48 | ) 49 | encdec_model = lstm.lstm_seq2seq( 50 | input_size=6, output_size=3, embedding_size=128, target_len=pred_len 51 | ) 52 | models = [rnn_model, encdec_model] 53 | 54 | models[0].train_model(x_train, y_train, n_epochs=500, batch_size=150) 55 | # models[1].train_model(x_train,y_train, n_epochs=500, batch_size=150) 56 | 57 | with open("./trained/models_%s.pkl" % name, "wb") as f: 58 | pickle.dump(models, f) 59 | 60 | UQ = {} 61 | 62 | bj_class = bjrnn.bj_rnn(models[0], recursion_depth=10) 63 | UQ["bjrnn"] = bj_class 64 | 65 | dprnn = dplstm.DPRNN( 66 | epochs=150, 67 | input_size=6, 68 | output_size=3, 69 | n_steps=pred_len, 70 | batch_size=64, 71 | dropout_prob=0.1, 72 | ) 73 | dprnn.fit(x_train, y_train) 74 | UQ["dprnn"] = dprnn 75 | 76 | cf = cfrnn.CFRNN(models[0], x_cali, y_cali) 77 | cf.calibrate() 78 | UQ["cfrnn"] = cf 79 | 80 | cf = cfrnn.CFRNN(models[1], x_cali, y_cali) 81 | cf.calibrate() 82 | UQ["cf-EncDec"] = cf 83 | 84 | copula = copulaCPTS.copulaCPTS(models[0], x_cali, y_cali) 85 | copula.calibrate() 86 | UQ["copula-rnn"] = copula 87 | 88 | copula = copulaCPTS.copulaCPTS(models[1], x_cali, y_cali) 89 | copula.calibrate() 90 | UQ["copula-EncDec"] = copula 91 | 92 | areas = {} 93 | coverages = {} 94 | 95 | epsilon_ls = np.linspace(0.05, 0.50, 10) 96 | 97 | for k, uqmethod in UQ.items(): 98 | print(k) 99 | 100 | area = [] 101 | coverage = [] 102 | for eps in epsilon_ls: 103 | pred, box = uqmethod.predict(x_test, epsilon=eps) 104 | area.append(uqmethod.calc_area_3d(box)) 105 | pred = torch.tensor(pred) 106 | coverage.append(uqmethod.calc_coverage_3d(box, pred, y_test)) 107 | areas[k] = area 108 | coverages[k] = coverage 109 | 110 | with open("./trained/uq_%s.pkl" % name, "wb") as f: 111 | pickle.dump(UQ, f) 112 | with open("./trained/results_%s.pkl" % name, "wb") as f: 113 | pickle.dump((areas, coverages), f) 114 | 115 | return areas, coverages, (models, UQ) 116 | 117 | 118 | def main(): 119 | lens = [1, 5, 10, 15] 120 | for pred_len in lens: 121 | x_train = torch.tensor(x_downsample[:600, :train_len, :], dtype=torch.float) 122 | y_train = torch.tensor( 123 | x_downsample[:600, train_len : train_len + pred_len, :3], dtype=torch.float 124 | ) 125 | 126 | x_cali = torch.tensor(x_downsample[600:800, :train_len, :], dtype=torch.float) 127 | y_cali = torch.tensor( 128 | x_downsample[600:800, train_len : train_len + pred_len, :3], 129 | dtype=torch.float, 130 | ) 131 | 132 | x_test = torch.tensor(x_downsample[800:, :train_len, :], dtype=torch.float) 133 | y_test = torch.tensor( 134 | x_downsample[800:, train_len : train_len + pred_len, :3], dtype=torch.float 135 | ) 136 | 137 | for i in range(3): 138 | res = experiment( 139 | x_downsample, 140 | pred_len=pred_len, 141 | name="drone_split_len_" + str(pred_len) + "_run_" + str(i), 142 | x_train=x_train, 143 | y_train=y_train, 144 | x_cali=x_cali, 145 | y_cali=y_cali, 146 | x_test=x_test, 147 | y_test=y_test, 148 | ) 149 | print("finished run " + str(pred_len)) 150 | del res 151 | 152 | for i in range(3): 153 | res = experiment(x_downsample, name="drone_vanila_" + str(i)) 154 | print("finished run " + str(i)) 155 | del res 156 | 157 | 158 | if __name__ == "__main__": 159 | 160 | main() 161 | -------------------------------------------------------------------------------- /src/experiments/particle_exp.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import torch 4 | import matplotlib.pyplot as plt 5 | from matplotlib.collections import PatchCollection 6 | from matplotlib.patches import Rectangle 7 | import pickle 8 | import os 9 | 10 | 11 | from src.models import rnn, lstm, bjrnn, cfrnn, copulaCPTS, dplstm 12 | 13 | 14 | def experiment(train, valid, test, name="exp"): 15 | 16 | torch._C._set_mkldnn_enabled(False) # this is to avoid a bug in pytorch that causes the code to crash 17 | rnn_model = rnn.rnn(embedding_size=24, input_size=2, output_size=2, horizon=24) 18 | encdec_model = lstm.lstm_seq2seq(input_size=2, embedding_size=24, target_len=24) 19 | models = [rnn_model, encdec_model] 20 | 21 | a = np.load(train) 22 | x_train = torch.tensor(a[:, :35, 0, :], dtype=torch.float) 23 | y_train = torch.tensor(a[:, 35:, 0, :], dtype=torch.float) 24 | 25 | for m in models: 26 | m.train_model(x_train, y_train, n_epochs=150, batch_size=150) 27 | 28 | with open("./trained/models_%s.pkl" % name, "wb") as f: 29 | pickle.dump(models, f) 30 | 31 | b = np.load(valid) 32 | x_cali = torch.tensor(b[:, :35, 0, :], dtype=torch.float) 33 | y_cali = torch.tensor(b[:, 35:, 0, :], dtype=torch.float) 34 | 35 | UQ = {} 36 | 37 | print("dprnn") 38 | dprnn = dplstm.DPRNN( 39 | epochs=90, input_size=2, output_size=2, n_steps=24, dropout_prob=0.1 40 | ) 41 | dprnn.fit(x_train, y_train) 42 | UQ["dprnn"] = dprnn 43 | 44 | bj_class = bjrnn.bj_rnn(models[0], recursion_depth=20) 45 | UQ["bjrnn"] = bj_class 46 | 47 | cf = cfrnn.CFRNN(models[0], x_cali, y_cali) 48 | cf.calibrate() 49 | UQ["cfrnn"] = cf 50 | 51 | copula = copulaCPTS.copulaCPTS(models[0], x_cali, y_cali) 52 | copula.calibrate() 53 | UQ["copula-rnn"] = copula 54 | 55 | cf = cfrnn.CFRNN(models[1], x_cali, y_cali) 56 | cf.calibrate() 57 | UQ["cf-EncDec"] = cf 58 | 59 | copula = copulaCPTS.copulaCPTS(models[1], x_cali, y_cali) 60 | copula.calibrate() 61 | UQ["copula-EncDec"] = copula 62 | 63 | c = np.load(test) 64 | x_test = torch.tensor(c[:, :35, 0, :], dtype=torch.float) 65 | y_test = torch.tensor(c[:, 35:, 0, :], dtype=torch.float) 66 | 67 | areas = {} 68 | coverages = {} 69 | 70 | epsilon_ls = np.linspace(0.05, 0.50, 10) 71 | 72 | for k, uqmethod in UQ.items(): 73 | print(k) 74 | area = [] 75 | coverage = [] 76 | for eps in epsilon_ls: 77 | pred, box = uqmethod.predict(x_test, epsilon=eps) 78 | area.append(uqmethod.calc_area(box)) 79 | pred = torch.tensor(pred) 80 | coverage.append(uqmethod.calc_coverage(box, pred, y_test)) 81 | areas[k] = area 82 | coverages[k] = coverage 83 | 84 | with open("./trained/uq_%s.pkl" % name, "wb") as f: 85 | pickle.dump(UQ, f) 86 | with open("./trained/results_%s.pkl" % name, "wb") as f: 87 | pickle.dump((areas, coverages), f) 88 | 89 | return areas, coverages, (models, UQ) 90 | 91 | 92 | def main(): 93 | 94 | os.makedirs("trained", exist_ok=True) 95 | 96 | train = "data/nridata/loc_train_springs2_noise_0.05.npy" 97 | valid = "data/nridata/loc_valid_springs2_noise_0.05.npy" 98 | test = "data/nridata/loc_test_springs2_noise_0.05.npy" 99 | 100 | for i in range(3): 101 | res = experiment(train, valid, test, name="particle5_run" + str(i)) 102 | print("run " + str(i) + "done") 103 | del res 104 | 105 | train = "data/nridata/loc_train_springs2_noise_0.01.npy" 106 | valid = "data/nridata/loc_valid_springs2_noise_0.01.npy" 107 | test = "data/nridata/loc_test_springs2_noise_0.01.npy" 108 | 109 | for i in range(3): 110 | res = experiment(train, valid, test, name="particle1_run" + str(i)) 111 | print("run " + str(i) + "done") 112 | del res 113 | 114 | 115 | if __name__ == "__main__": 116 | main() 117 | -------------------------------------------------------------------------------- /src/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/CopulaCPTS/56f29640edfe5e19e37efa0eb5eb372d5c3a9c87/src/models/__init__.py -------------------------------------------------------------------------------- /src/models/bjrnn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, Kamilė Stankevičiūtė 2 | # Adapted from Ahmed M. Alaa github.com/ahmedmalaa/rnn-blockwise-jackknife 3 | # Licensed under the BSD 3-clause license 4 | 5 | import numpy as np 6 | import torch 7 | 8 | from .influence import * 9 | 10 | # influence_function, perturb_model_ 11 | 12 | 13 | class bj_rnn: 14 | def __init__( 15 | self, model, mode="stochastic", damp=1e-4, rnn_mode="RNN", recursion_depth=20 16 | ): 17 | 18 | self.model = model 19 | self.rnn_mode = rnn_mode 20 | self.IF = influence_function( 21 | model, 22 | train_index=list(range(model.X.shape[0])[:200]), 23 | mode=mode, 24 | damp=damp, 25 | recursion_depth=recursion_depth, 26 | ) 27 | 28 | # X_ = [model.X[k][: int(torch.sum(model.masks[k, :]).detach().numpy())] for k in range(model.X.shape[0])] 29 | X_ = model.X 30 | self.LOBO_residuals = [] 31 | self.dim = model.output_size 32 | self.variable_preds = None 33 | 34 | for k in range(len(self.IF)): 35 | perturbed_models = perturb_model_(self.model, self.IF[k]) 36 | 37 | self.LOBO_residuals.append( 38 | np.abs( 39 | np.array(self.model.y[k].unsqueeze(0)) 40 | - np.array(perturbed_models.predict(X_[k].unsqueeze(0)).detach()) 41 | ) 42 | ) 43 | 44 | del perturbed_models 45 | 46 | self.LOBO_residuals = np.squeeze(np.array(self.LOBO_residuals)) 47 | 48 | self.results_dict = {} 49 | 50 | def variable_predict(self, X_test): 51 | variable_preds = [] 52 | 53 | for k in range(len(self.IF)): 54 | perturbed_models = perturb_model_(self.model, self.IF[k]) 55 | variable_preds.append(perturbed_models.predict(X_test).detach().numpy()) 56 | 57 | del perturbed_models 58 | 59 | variable_preds = np.array(variable_preds) 60 | self.variable_preds = variable_preds 61 | 62 | def predict(self, X_test, epsilon=0.1): 63 | 64 | if self.variable_preds is None: 65 | self.variable_predict(X_test) 66 | 67 | variable_preds = self.variable_preds 68 | 69 | num_sequences = X_test.shape[0] 70 | y_l_approx = np.zeros(variable_preds.shape[1:]) 71 | y_u_approx = np.zeros(variable_preds.shape[1:]) 72 | 73 | if len(self.LOBO_residuals.shape) == 2: 74 | self.LOBO_residuals = np.expand_dims(self.LOBO_residuals, axis=1) 75 | 76 | for i in range(self.dim): 77 | y_u_approx[:, :, i] = np.quantile( 78 | variable_preds[..., i] 79 | + np.repeat( 80 | np.expand_dims(self.LOBO_residuals[..., i], axis=1), 81 | num_sequences, 82 | axis=1, 83 | ) 84 | * 5, 85 | 1 - epsilon / 2, 86 | axis=0, 87 | keepdims=False, 88 | ) 89 | y_l_approx[:, :, i] = np.quantile( 90 | variable_preds[..., i] 91 | - np.repeat( 92 | np.expand_dims(self.LOBO_residuals[..., i], axis=1), 93 | num_sequences, 94 | axis=1, 95 | ) 96 | * 5, 97 | 1 - epsilon / 2, 98 | axis=0, 99 | keepdims=False, 100 | ) 101 | y_pred = self.model.predict(X_test) 102 | 103 | self.results_dict[epsilon] = { 104 | "y_pred": y_pred, 105 | "y_l_approx": y_l_approx, 106 | "y_u_approx": y_u_approx, 107 | } 108 | 109 | return y_pred, (y_l_approx, y_u_approx) 110 | 111 | def calc_area(self, box): 112 | 113 | y_l_approx, y_u_approx = box 114 | delta = y_u_approx - y_l_approx 115 | all_area = np.multiply(delta[:, :, 0], delta[:, :, 1]) # (num_samples, len) 116 | area = all_area.sum(axis=1).mean() 117 | return area 118 | 119 | def calc_area_1d(self, box): 120 | 121 | y_l_approx, y_u_approx = box 122 | delta = y_u_approx - y_l_approx 123 | area = delta.sum(axis=1).mean() 124 | return area 125 | 126 | def calc_area_3d(self, box): 127 | 128 | y_l_approx, y_u_approx = box 129 | delta = y_u_approx - y_l_approx 130 | all_area = np.multiply( 131 | delta[:, :, 0], delta[:, :, 1], delta[:, :, 2] 132 | ) # (num_samples, len) 133 | area = all_area.sum(axis=1).mean() 134 | return area 135 | 136 | def calc_coverage(self, box, y_pred, y_test): 137 | 138 | y_l_approx, y_u_approx = box 139 | lower = y_test.detach().numpy() > y_l_approx 140 | upper = y_test.detach().numpy() < y_u_approx 141 | an = np.logical_and(lower, upper) 142 | cov = np.all(np.all(an, axis=2), axis=1).mean() 143 | return cov 144 | 145 | def calc_coverage_3d(self, box, y_pred, y_test): 146 | 147 | return self.calc_coverage(box, y_pred, y_test) 148 | 149 | def calc_coverage_1d(self, box, y_pred, y_test): 150 | 151 | return self.calc_coverage(box, y_pred, y_test) 152 | -------------------------------------------------------------------------------- /src/models/cfrnn.py: -------------------------------------------------------------------------------- 1 | # adapted from Kamilė Stankevičiūtė 2 | # https://github.com/kamilest/conformal-rnn/tree/5f6dc9e3118bcea631745391f4efb246733a11c7 3 | 4 | """ CFRNN model. """ 5 | 6 | 7 | import numpy as np 8 | import torch 9 | 10 | 11 | class CFRNN: 12 | """ 13 | The auxiliary RNN issuing point predictions. 14 | Point predictions are used as baseline to which the (normalised) 15 | uncertainty intervals are added in the main CFRNN network. 16 | """ 17 | 18 | def __init__(self, model, cali_x, cali_y): 19 | """ 20 | Initialises the auxiliary forecaster. 21 | Args: 22 | embedding_size: hyperparameter indicating the size of the latent 23 | RNN embeddings. 24 | input_size: dimensionality of the input time-series 25 | output_size: dimensionality of the forecast 26 | horizon: forecasting horizon 27 | rnn_mode: type of the underlying RNN network 28 | path: optional path where to save the auxiliary model to be used 29 | in the main CFRNN network 30 | """ 31 | self.model = model 32 | 33 | self.cali_x = cali_x 34 | self.cali_y = cali_y 35 | self.nonconformity = None 36 | self.results_dict = {} 37 | 38 | def calibrate(self): 39 | dim = self.cali_y.shape[-1] 40 | pred_y = self.model.predict(self.cali_x) 41 | nonconformity = ( 42 | torch.norm((pred_y[..., :dim] - self.cali_y), p=2, dim=-1).detach().numpy() 43 | ) 44 | self.nonconformity = nonconformity 45 | 46 | def calibrate_l1(self): 47 | dim = self.cali_y.shape[-1] 48 | pred_y = self.model.predict(self.cali_x) 49 | nonconformity = ( 50 | torch.norm((pred_y[..., :dim] - self.cali_y), p=1, dim=-1).detach().numpy() 51 | ) 52 | self.nonconformity = nonconformity 53 | 54 | def predict(self, X_test, epsilon=0.1): 55 | 56 | nonconformity = self.nonconformity 57 | n_calibration = nonconformity.shape[0] 58 | new_quantile = min( 59 | (n_calibration + 1.0) 60 | * (1 - (epsilon / self.cali_y.shape[-2])) 61 | / n_calibration, 62 | 1, 63 | ) 64 | 65 | radius = [ 66 | np.quantile(nonconformity[:, r], new_quantile) 67 | for r in range(nonconformity.shape[1]) 68 | ] 69 | y_pred = self.model.predict(X_test) 70 | 71 | self.results_dict[epsilon] = {"y_pred": y_pred, "radius": radius} 72 | 73 | return y_pred, radius 74 | 75 | def calc_area(self, radius): 76 | 77 | area = sum([np.pi * r**2 for r in radius]) 78 | 79 | return area 80 | 81 | def calc_area_l1(self, radius): 82 | 83 | area = sum([2 * r**2 for r in radius]) 84 | 85 | return area 86 | 87 | def calc_area_3d(self, radius): 88 | 89 | area = sum([4 / 3.0 * np.pi * r**3 for r in radius]) 90 | 91 | return area 92 | 93 | def calc_area_1d(self, radius): 94 | 95 | area = sum(radius) 96 | 97 | return area 98 | 99 | def calc_coverage(self, radius, y_pred, y_test): 100 | dim = y_test.shape[-1] 101 | testnonconformity = ( 102 | torch.norm((y_pred[..., :dim] - y_test), p=2, dim=-1).detach().numpy() 103 | ) 104 | 105 | circle_covs = [] 106 | for j in range(y_test.shape[-2]): 107 | circle_covs.append(testnonconformity[:, j] < radius[j]) 108 | 109 | circle_covs = np.array(circle_covs) 110 | coverage = np.mean(np.all(circle_covs, axis=0)) 111 | return coverage 112 | 113 | def calc_coverage_l1(self, radius, y_pred, y_test): 114 | dim = y_test.shape[-1] 115 | testnonconformity = ( 116 | torch.norm((y_pred[..., :dim] - y_test), p=1, dim=-1).detach().numpy() 117 | ) 118 | 119 | circle_covs = [] 120 | for j in range(y_test.shape[-2]): 121 | circle_covs.append(testnonconformity[:, j] < radius[j]) 122 | 123 | circle_covs = np.array(circle_covs) 124 | coverage = np.mean(np.all(circle_covs, axis=0)) 125 | return coverage 126 | 127 | def calc_coverage_3d(self, radius, y_pred, y_test): 128 | 129 | return self.calc_coverage(radius, y_pred[:3], y_test[:3]) 130 | 131 | def calc_coverage_1d(self, radius, y_pred, y_test): 132 | 133 | return self.calc_coverage(radius, y_pred, y_test) 134 | -------------------------------------------------------------------------------- /src/models/copulaCPTS.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import torch 4 | from copulae.core import pseudo_obs 5 | from .utils import search_alpha 6 | 7 | 8 | def empirical_copula_loss(x, data, epsilon): 9 | pseudo_data = pseudo_obs(data) 10 | return np.fabs( 11 | np.mean( 12 | np.all( 13 | np.less_equal(pseudo_data, np.array([x] * pseudo_data.shape[1])), axis=1 14 | ) 15 | ) 16 | - 1 17 | + epsilon 18 | ) 19 | 20 | 21 | class copulaCPTS: 22 | def __init__(self, model, cali_x, cali_y): 23 | """ 24 | Copula conformal prediction with two-step calibration. 25 | """ 26 | self.model = model 27 | 28 | self.cali_x = None 29 | self.cali_y = None 30 | self.copula_x = None 31 | self.copula_y = None 32 | self.split_cali(cali_x, cali_y) 33 | 34 | self.nonconformity = None 35 | self.results_dict = {} 36 | 37 | def split_cali(self, cali_x, cali_y, split=0.6): 38 | if self.copula_x: 39 | print("already split") 40 | return 41 | size = cali_x.shape[0] 42 | halfsize = int(split * size) 43 | 44 | idx = np.random.choice(range(size), halfsize, replace=False) 45 | 46 | self.cali_x = cali_x[idx] 47 | self.copula_x = cali_x[list(set(range(size)) - set(idx))] 48 | self.cali_y = cali_y[idx] 49 | self.copula_y = cali_y[list(set(range(size)) - set(idx))] 50 | 51 | def calibrate(self): 52 | 53 | pred_y = self.model.predict(self.cali_x) 54 | nonconformity = torch.norm((pred_y - self.cali_y), p=2, dim=-1).detach().numpy() 55 | self.nonconformity = nonconformity 56 | 57 | def calibrate_l1(self): 58 | 59 | pred_y = self.model.predict(self.cali_x) 60 | nonconformity = torch.norm((pred_y - self.cali_y), p=1, dim=-1).detach().numpy() 61 | self.nonconformity = nonconformity 62 | 63 | def predict(self, X_test, epsilon=0.1): 64 | 65 | # alphas = self.nonconformity 66 | pred_y = self.model.predict(self.copula_x) 67 | scores = torch.norm((pred_y - self.copula_y), p=2, dim=-1).detach().numpy() 68 | alphas = [] 69 | for i in range(scores.shape[0]): 70 | a = (scores[i] > self.nonconformity).mean(axis=0) 71 | alphas.append(a) 72 | alphas = np.array(alphas) 73 | 74 | threshold = search_alpha(alphas, epsilon, epochs=800) 75 | 76 | mapping_shape = self.nonconformity.shape[0] 77 | mapping = { 78 | i: sorted(self.nonconformity[:, i].tolist()) for i in range(alphas.shape[1]) 79 | } 80 | 81 | quantile = [] 82 | mapping_shape = self.nonconformity.shape[0] 83 | 84 | for i in range(alphas.shape[1]): 85 | idx = int(threshold[i] * mapping_shape) + 1 86 | if idx >= mapping_shape: 87 | idx = mapping_shape - 1 88 | quantile.append(mapping[i][idx]) 89 | 90 | radius = np.array(quantile) 91 | 92 | y_pred = self.model.predict(X_test) 93 | 94 | self.results_dict[epsilon] = {"y_pred": y_pred, "radius": radius} 95 | 96 | return y_pred, radius 97 | 98 | def calc_area(self, radius): 99 | 100 | area = sum([np.pi * r**2 for r in radius]) 101 | 102 | return area 103 | 104 | def calc_area_l1(self, radius): 105 | 106 | area = sum([2 * r**2 for r in radius]) 107 | 108 | return area 109 | 110 | def calc_area_3d(self, radius): 111 | 112 | area = sum([4 / 3.0 * np.pi * r**3 for r in radius]) 113 | 114 | return area 115 | 116 | def calc_area_1d(self, radius): 117 | 118 | area = sum(radius) 119 | 120 | return area 121 | 122 | def calc_coverage(self, radius, y_pred, y_test): 123 | 124 | testnonconformity = torch.norm((y_pred - y_test), p=2, dim=-1).detach().numpy() 125 | 126 | circle_covs = [] 127 | for j in range(y_test.shape[-2]): 128 | circle_covs.append(testnonconformity[:, j] < radius[j]) 129 | 130 | circle_covs = np.array(circle_covs) 131 | coverage = np.mean(np.all(circle_covs, axis=0)) 132 | return coverage 133 | 134 | def calc_coverage_l1(self, radius, y_pred, y_test): 135 | testnonconformity = ( 136 | torch.norm((y_pred - y_test), p=1, dim=-1).detach().numpy() 137 | ) # change back to p=2 138 | circle_covs = [] 139 | for j in range(y_test.shape[-2]): 140 | circle_covs.append(testnonconformity[:, j] < radius[j]) 141 | 142 | circle_covs = np.array(circle_covs) 143 | coverage = np.mean(np.all(circle_covs, axis=0)) 144 | return coverage 145 | 146 | def calc_coverage_3d(self, radius, y_pred, y_test): 147 | 148 | return self.calc_coverage(radius, y_pred, y_test) 149 | 150 | def calc_coverage_1d(self, radius, y_pred, y_test): 151 | 152 | return self.calc_coverage(radius, y_pred, y_test) 153 | -------------------------------------------------------------------------------- /src/models/dplstm.py: -------------------------------------------------------------------------------- 1 | # Adapted from Ahmed M. Alaa github.com/ahmedmalaa/rnn-blockwise-jackknife 2 | 3 | import numpy as np 4 | import torch 5 | from scipy import stats as st 6 | from torch import nn 7 | from torch.autograd import Variable 8 | 9 | 10 | torch.manual_seed(1) 11 | 12 | 13 | class DPRNN(nn.Module): 14 | def __init__( 15 | self, 16 | epochs=5, 17 | batch_size=150, 18 | max_steps=50, 19 | input_size=2, 20 | lr=0.01, 21 | output_size=2, 22 | embedding_size=20, 23 | n_layers=1, 24 | n_steps=50, 25 | dropout_prob=0.2, 26 | **kwargs 27 | ): 28 | 29 | super(DPRNN, self).__init__() 30 | 31 | self.EPOCH = epochs 32 | self.BATCH_SIZE = batch_size 33 | self.MAX_STEPS = max_steps 34 | self.INPUT_SIZE = input_size 35 | self.LR = lr 36 | self.OUTPUT_SIZE = output_size 37 | self.HIDDEN_UNITS = embedding_size 38 | self.NUM_LAYERS = n_layers 39 | self.N_STEPS = n_steps 40 | 41 | self.dropout_prob = dropout_prob 42 | 43 | self.encoder = nn.LSTM( 44 | input_size=self.INPUT_SIZE, 45 | hidden_size=self.HIDDEN_UNITS, 46 | num_layers=self.NUM_LAYERS, 47 | batch_first=True, 48 | dropout=self.dropout_prob, 49 | ) 50 | 51 | self.decoder = nn.LSTM( 52 | input_size=self.OUTPUT_SIZE, 53 | hidden_size=self.HIDDEN_UNITS, 54 | num_layers=self.NUM_LAYERS, 55 | batch_first=True, 56 | dropout=self.dropout_prob, 57 | ) 58 | 59 | self.dropout = nn.Dropout(p=dropout_prob) 60 | 61 | self.out = nn.Linear(self.HIDDEN_UNITS, self.OUTPUT_SIZE) 62 | 63 | def forward(self, x): 64 | # x shape (batch, time_step, input_size) 65 | # r_out shape (batch, time_step, output_size) 66 | # h_n shape (n_layers, batch, hidden_size) 67 | # h_c shape (n_layers, batch, hidden_size) 68 | 69 | # r_out, (h_n, h_c) = self.rnn(x, None) # None represents zero 70 | # out = self.out(self.dropout(h_n)) 71 | 72 | # here stharts me 73 | encoder_output, encoder_hidden = self.encoder(x) 74 | 75 | # initialize tensor for predictions 76 | outputs = [] 77 | 78 | # decode input_tensor 79 | decoder_input = x[:, -1, : self.OUTPUT_SIZE].unsqueeze( 80 | 1 81 | ) # (batch_size, 1, input_dim) 82 | decoder_hidden = encoder_hidden 83 | 84 | for t in range(self.N_STEPS): 85 | decoderout, decoder_hidden = self.decoder(decoder_input, decoder_hidden) 86 | 87 | dropout = self.dropout(decoder_hidden[0]) # (1, batch_size, input_dim) 88 | decoder_output = self.out(dropout).squeeze(0).unsqueeze(1) 89 | 90 | outputs.append(decoder_output) 91 | decoder_input = decoder_output 92 | 93 | # np_outputs = outputs.detach().numpy() 94 | return torch.cat(outputs, dim=1) 95 | 96 | def fit(self, X, Y): 97 | 98 | optimizer = torch.optim.Adam( 99 | self.parameters(), lr=self.LR 100 | ) # optimize all rnn parameters 101 | self.loss_func = nn.MSELoss() 102 | 103 | # training and testing 104 | for epoch in range(self.EPOCH): 105 | for step in range(50): 106 | batch_indexes = np.random.choice( 107 | list(range(X.shape[0])), size=self.BATCH_SIZE, replace=True, p=None 108 | ) 109 | output = self( 110 | X[batch_indexes] 111 | ) # .reshape(-1, self.OUTPUT_SIZE) # rnn output 112 | 113 | loss = self.loss_func(output, Y[batch_indexes]) # MSE loss 114 | 115 | optimizer.zero_grad() # clear gradients for this training step 116 | loss.backward() # backpropagation, compute gradients 117 | optimizer.step() # apply gradients 118 | 119 | if epoch % 10 == 0: 120 | print("Epoch: ", epoch, "| train loss: %.4f" % loss.data) 121 | 122 | def predict_dprnn(self, X, num_samples=100, alpha=0.05): 123 | z_critical = st.norm.ppf(1 - alpha / 2) 124 | 125 | predictions = [] 126 | 127 | for idx in range(num_samples): 128 | predicts_ = self(X) 129 | predictions.append(predicts_.detach()) 130 | 131 | pred_mean = np.mean(np.stack(predictions, axis=1), axis=1) 132 | pred_std = z_critical * np.std(np.stack(predictions, axis=1), axis=1) 133 | 134 | return pred_mean, pred_std 135 | 136 | def predict(self, x_test, epsilon): 137 | # epsilon = 1- (1-epsilon) ** (1/self.N_STEPS) 138 | 139 | mean, std = self.predict_dprnn(x_test, alpha=epsilon) 140 | 141 | return mean, std 142 | 143 | def calc_coverage(self, std, y_pred, y_test): 144 | 145 | rectangle_covs = [] 146 | test_residuals = (y_pred - y_test).detach().numpy() 147 | 148 | rectangle_covs = test_residuals < std 149 | coverage = np.mean(np.all(np.all(rectangle_covs, axis=-1), axis=1)) 150 | return coverage 151 | 152 | def calc_coverage_3d(self, std, y_pred, y_test): 153 | 154 | rectangle_covs = [] 155 | test_residuals = (y_pred - y_test).detach().numpy() 156 | rectangle_covs = test_residuals < std 157 | coverage = np.mean(np.all(np.all(rectangle_covs[..., :3], axis=-1), axis=1)) 158 | return coverage 159 | 160 | def calc_coverage_1d(self, std, y_pred, y_test): 161 | 162 | rectangle_covs = [] 163 | test_residuals = (y_pred - y_test).detach().numpy() 164 | rectangle_covs = test_residuals < std 165 | coverage = np.mean(np.all(rectangle_covs, axis=1)) 166 | return coverage 167 | 168 | def calc_area(self, std): 169 | area = np.mean(np.sum(std[..., 0] * std[..., 1] * 4, axis=1)) 170 | return area 171 | 172 | def calc_area_3d(self, std): 173 | area = np.mean(np.sum(std[..., 0] * std[..., 1] * std[..., 2] * 8, axis=1)) 174 | return area 175 | 176 | def calc_area_1d(self, std): 177 | area = np.mean(np.sum(std[..., 0], axis=1)) 178 | return area 179 | -------------------------------------------------------------------------------- /src/models/influence/__init__.py: -------------------------------------------------------------------------------- 1 | from .influence_utils import * 2 | from .influence_computation import * 3 | -------------------------------------------------------------------------------- /src/models/influence/influence_computation.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, Ahmed M. Alaa 2 | # Licensed under the BSD 3-clause license (see LICENSE.txt) 3 | 4 | # --------------------------------------------------------- 5 | # Code for influence functions computation in Pytorch 6 | # --------------------------------------------------------- 7 | 8 | import warnings 9 | 10 | import numpy as np 11 | import torch 12 | 13 | from .influence_utils import ( 14 | exact_hessian, 15 | exact_hessian_ij, 16 | hessian_vector_product, 17 | stack_torch_tensors, 18 | ) 19 | 20 | warnings.simplefilter("ignore") 21 | 22 | 23 | def influence_function( 24 | model, train_index, W=None, mode="stochastic", batch_size=100, damp=1e-3, scale=1000, order=1, recursion_depth=100 25 | ): 26 | """ 27 | Computes the influence function defined as H^-1 dLoss/d theta. This is the impact that each 28 | training data point has on the learned model parameters. 29 | """ 30 | 31 | if mode == "stochastic": 32 | 33 | IF = influence_stochastic_estimation(model, train_index, batch_size, damp, scale, recursion_depth) 34 | 35 | if mode == "exact": 36 | 37 | IF = exact_influence(model, train_index, damp, W, order) 38 | 39 | return IF 40 | 41 | 42 | def influence_stochastic_estimation(model, train_index, batch_size=100, damp=1e-3, scale=1000, recursion_depth=1000): 43 | """ 44 | This function applies the stochastic estimation approach to evaluating influence function based on the power-series 45 | approximation of matrix inversion. Recall that the exact inverse Hessian H^-1 can be computed as follows: 46 | 47 | H^-1 = \\sum^\\infty_{i=0} (I - H) ^ j 48 | 49 | This series converges if all the eigen values of H are less than 1. 50 | 51 | Arguments: 52 | loss: scalar/tensor, for example the output of the loss function 53 | rnn: the model for which the Hessian of the loss is evaluated 54 | v: list of torch tensors, rnn.parameters(), 55 | will be multiplied with the Hessian 56 | 57 | Returns: 58 | return_grads: list of torch tensors, contains product of Hessian and v. 59 | """ 60 | 61 | NUM_SAMPLES = model.X.shape[0] 62 | SUBSAMPLES = batch_size 63 | 64 | loss = [ 65 | model.loss_fn(model.y[train_index[_]].unsqueeze(0), model.predict(model.X[train_index[_]].unsqueeze(0))) 66 | for _ in range(len(train_index)) 67 | ] 68 | 69 | grads = [ 70 | stack_torch_tensors(torch.autograd.grad(loss[_], model.parameters(), create_graph=True)) 71 | for _ in range(len(train_index)) 72 | ] 73 | 74 | IHVP_ = [grads[_].clone().detach() for _ in range(len(train_index))] 75 | 76 | for j in range(recursion_depth): 77 | sampled_indx = np.random.choice(list(range(NUM_SAMPLES)), SUBSAMPLES, replace=False) 78 | 79 | sampled_loss = model.loss_fn(model.y[sampled_indx], model.predict(model.X[sampled_indx, :])) 80 | 81 | IHVP_prev = [IHVP_[_].clone().detach() for _ in range(len(train_index))] 82 | 83 | hvps_ = [ 84 | stack_torch_tensors(hessian_vector_product(sampled_loss, model, [IHVP_prev[_]])) 85 | for _ in range(len(train_index)) 86 | ] 87 | 88 | IHVP_ = [g_ + (1 - damp) * ihvp_ - hvp_ / scale for (g_, ihvp_, hvp_) in zip(grads, IHVP_prev, hvps_)] 89 | 90 | return [-1 * IHVP_[_] / (scale * NUM_SAMPLES) for _ in range(len(train_index))] 91 | 92 | 93 | def exact_influence(model, train_index, damp=0, W=None, order=1): 94 | 95 | params_ = [] 96 | 97 | for param in model.parameters(): 98 | 99 | params_.append(param) 100 | 101 | num_par = stack_torch_tensors(params_).shape[0] 102 | Hinv = torch.inverse(exact_hessian(model) + damp * torch.eye(num_par)) 103 | 104 | if order == 2: 105 | 106 | H_ij = [ 107 | exact_hessian_ij(model, model.loss_fn(model.predict(model.X[_index], numpy_output=False), model.y[_index])) 108 | for _index in range(model.X.shape[0]) 109 | ] 110 | 111 | if W is None: 112 | 113 | y_preds = [model.predict(model.X[k, :]) for k in train_index] 114 | 115 | if hasattr(model, "masks"): 116 | 117 | losses = [torch.sum(model.sequence_loss()[train_index[k], :]) for k in range(len(train_index))] 118 | n_factor = torch.sum(model.masks) 119 | 120 | else: 121 | 122 | losses = [model.loss_fn(y_preds[k], model.y[train_index[k]]) for k in range(len(train_index))] 123 | n_factor = model.X.shape[0] 124 | 125 | grads = [ 126 | stack_torch_tensors(torch.autograd.grad(losses[k], model.parameters(), create_graph=True)) 127 | for k in range(len(losses)) 128 | ] 129 | 130 | if order == 1: 131 | 132 | IFs_ = [-1 * torch.mm(Hinv, grads[k].reshape((grads[k].shape[0], 1))) / n_factor for k in range(len(grads))] 133 | 134 | elif order == 2: 135 | 136 | IF_ = [-1 * torch.mm(Hinv, grads[k].reshape((grads[k].shape[0], 1))) / n_factor for k in range(len(grads))] 137 | IF2_ = [torch.mm(Hinv, torch.mm(H_ij[k], IF_[k])) * (2 / n_factor) for k in range(len(grads))] 138 | IFs_ = [IF_[k] + 0.5 * IF2_[k] for k in range(len(grads))] 139 | 140 | else: 141 | 142 | y_preds = model.predict(model.X, numpy_output=False) 143 | losses = [model.loss_fn(y_preds[k], model.y[k]) * W[k] for k in range(len(y_preds))] 144 | grads = stack_torch_tensors( 145 | torch.autograd.grad(torch.sum(stack_torch_tensors(losses)), model.parameters(), create_graph=True) 146 | ) 147 | IFs_ = [-1 * torch.mm(Hinv, grads.reshape((-1, 1))) / model.X.shape[0]] 148 | 149 | return IFs_ 150 | -------------------------------------------------------------------------------- /src/models/influence/influence_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, Ahmed M. Alaa 2 | # Licensed under the BSD 3-clause license (see LICENSE.txt) 3 | 4 | # --------------------------------------------------------- 5 | # Helper functions for influence functions' 6 | # computation in Pytorch 7 | # --------------------------------------------------------- 8 | 9 | import collections 10 | import copy 11 | 12 | import numpy as np 13 | import torch 14 | 15 | 16 | def stack_torch_tensors(input_tensors): 17 | """ 18 | Takes a list of tensors and stacks them into one tensor 19 | """ 20 | 21 | unrolled = [input_tensors[k].view(-1, 1) for k in range(len(input_tensors))] 22 | 23 | return torch.cat(unrolled) 24 | 25 | 26 | def get_numpy_parameters(model): 27 | """Recovers the parameters of a pytorch model in numpy format.""" 28 | 29 | params = [] 30 | 31 | for param in model.parameters(): 32 | params.append(param) 33 | 34 | return stack_torch_tensors(params).detach().numpy() 35 | 36 | 37 | def exact_hessian(model): 38 | grad_params = torch.autograd.grad( 39 | model.loss, model.parameters(), retain_graph=True, create_graph=True 40 | ) 41 | grad_params = stack_torch_tensors(grad_params) 42 | temp = [] 43 | 44 | for u in range(len(grad_params)): 45 | second_grad = torch.autograd.grad( 46 | grad_params[u], model.parameters(), retain_graph=True 47 | ) 48 | temp.append(stack_torch_tensors(second_grad)) 49 | 50 | Hessian = torch.cat(temp, dim=1) 51 | return Hessian 52 | 53 | 54 | def exact_hessian_ij(model, loss_ij): 55 | grad_params = torch.autograd.grad( 56 | loss_ij, model.parameters(), retain_graph=True, create_graph=True 57 | ) 58 | grad_params = stack_torch_tensors(grad_params) 59 | temp = [] 60 | 61 | for u in range(len(grad_params)): 62 | second_grad = torch.autograd.grad( 63 | grad_params[u], model.parameters(), retain_graph=True 64 | ) 65 | temp.append(stack_torch_tensors(second_grad)) 66 | Hessian = torch.cat(temp, dim=1) 67 | 68 | return Hessian 69 | 70 | 71 | def hessian_vector_product(loss, model, v): 72 | """ 73 | Multiplies the Hessians of the loss of a model with respect to its parameters by a vector v. 74 | Adapted from: https://github.com/kohpangwei/influence-release 75 | 76 | This function uses a backproplike approach to compute the product between the Hessian 77 | and another vector efficiently, which even works for large Hessians with O(p) complexity for p parameters. 78 | 79 | Arguments: 80 | loss: scalar/tensor, for example the output of the loss function 81 | rnn: the model for which the Hessian of the loss is evaluated 82 | v: list of torch tensors, rnn.parameters(), 83 | will be multiplied with the Hessian 84 | 85 | Returns: 86 | return_grads: list of torch tensors, contains product of Hessian and v. 87 | """ 88 | 89 | # First backprop 90 | first_grads = stack_torch_tensors( 91 | torch.autograd.grad( 92 | loss, model.parameters(), retain_graph=True, create_graph=True 93 | ) 94 | ) 95 | 96 | """ 97 | # Elementwise products 98 | elemwise_products = 0 99 | 100 | for grad_elem, v_elem in zip(first_grads, v): 101 | elemwise_products += torch.sum(grad_elem * v_elem) 102 | """ 103 | 104 | elemwise_products = torch.mm( 105 | first_grads.view(-1, first_grads.shape[0]).float(), 106 | v[0].view(first_grads.shape[0], -1).float(), 107 | ) 108 | 109 | # Second backprop 110 | HVP_ = torch.autograd.grad(elemwise_products, model.parameters(), create_graph=True) 111 | 112 | return HVP_ 113 | 114 | 115 | def perturb_model_(model, perturb): 116 | """ 117 | Perturbs the parameters of a model by a given vector of influences 118 | 119 | Arguments: 120 | model: a pytorch model with p parameters 121 | perturb: a tensors with size p designating the desired parameter-wise perturbation 122 | 123 | Returns: 124 | perturbed_model : a copy of the original model with perturbed parameters 125 | """ 126 | 127 | params = [] 128 | 129 | for param in model.parameters(): 130 | params.append(param.clone()) 131 | 132 | param_ = stack_torch_tensors(params) 133 | new_param_ = param_ - perturb 134 | 135 | # copy all model attributes 136 | perturbed_model = type(model)() 137 | 138 | new_model_dict = dict.fromkeys(model.__dict__.keys()) 139 | new_model_state = collections.OrderedDict.fromkeys(model.state_dict().keys()) 140 | 141 | for key in new_model_dict.keys(): 142 | if type(model.__dict__[key]) is torch.Tensor: 143 | new_model_dict[key] = model.__dict__[key].clone() 144 | else: 145 | new_model_dict[key] = copy.deepcopy(model.__dict__[key]) 146 | 147 | for key in new_model_state.keys(): 148 | if type(model.state_dict()[key]) == torch.Tensor: 149 | new_model_state[key] = model.state_dict()[key].clone() 150 | else: 151 | new_model_state[key] = copy.deepcopy(model.state_dict()[key]) 152 | 153 | perturbed_model.__dict__.update(new_model_dict) 154 | perturbed_model.load_state_dict(new_model_state) 155 | 156 | index = 0 157 | 158 | for param in perturbed_model.parameters(): 159 | if len(param.data.shape) > 1: 160 | new_size = np.max((1, param.data.shape[0])) * np.max( 161 | (1, param.data.shape[1]) 162 | ) 163 | param.data = new_param_[index : index + new_size].view( 164 | param.data.shape[0], param.data.shape[1] 165 | ) 166 | else: 167 | new_size = param.data.shape[0] 168 | param.data = np.squeeze(new_param_[index : index + new_size]) 169 | 170 | index += new_size 171 | 172 | return perturbed_model 173 | -------------------------------------------------------------------------------- /src/models/lstm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import random 3 | import errno 4 | import sys 5 | from tqdm import trange 6 | 7 | import torch 8 | import torch.nn as nn 9 | from torch import optim 10 | import torch.nn.functional as F 11 | 12 | 13 | class lstm_encoder(nn.Module): 14 | """Encodes time-series sequence""" 15 | 16 | def __init__(self, input_size, hidden_size, num_layers=1): 17 | """ 18 | : param input_size: the number of features in the input X 19 | : param hidden_size: the number of features in the hidden state h 20 | : param num_layers: number of recurrent layers (i.e., 2 means there are 21 | : 2 stacked LSTMs) 22 | """ 23 | 24 | super(lstm_encoder, self).__init__() 25 | self.input_size = input_size 26 | self.hidden_size = hidden_size 27 | self.num_layers = num_layers 28 | 29 | # define LSTM layer 30 | self.lstm = nn.LSTM( 31 | input_size=input_size, 32 | hidden_size=hidden_size, 33 | num_layers=num_layers, 34 | batch_first=True, 35 | ) 36 | 37 | def forward(self, x_input): 38 | """ 39 | : param x_input: input of shape (seq_len, # in batch, input_size) 40 | : return lstm_out, hidden: lstm_out gives all the hidden states in the sequence; 41 | : hidden gives the hidden state and cell state for the last 42 | : element in the sequence 43 | """ 44 | 45 | lstm_out, self.hidden = self.lstm( 46 | x_input.view(x_input.shape[0], x_input.shape[1], self.input_size) 47 | ) 48 | 49 | return lstm_out, self.hidden 50 | 51 | def init_hidden(self, batch_size): 52 | """ 53 | initialize hidden state 54 | : param batch_size: x_input.shape[1] 55 | : return: zeroed hidden state and cell state 56 | """ 57 | 58 | return ( 59 | torch.zeros(self.num_layers, batch_size, self.hidden_size), 60 | torch.zeros(self.num_layers, batch_size, self.hidden_size), 61 | ) 62 | 63 | 64 | class lstm_decoder(nn.Module): 65 | """Decodes hidden state output by encoder""" 66 | 67 | def __init__(self, input_size, hidden_size, num_layers=1): 68 | """ 69 | : param input_size: the number of features in the input X 70 | : param hidden_size: the number of features in the hidden state h 71 | : param num_layers: number of recurrent layers (i.e., 2 means there are 72 | : 2 stacked LSTMs) 73 | """ 74 | 75 | super(lstm_decoder, self).__init__() 76 | self.input_size = input_size 77 | self.hidden_size = hidden_size 78 | self.num_layers = num_layers 79 | 80 | self.lstm = nn.LSTM( 81 | input_size=input_size, 82 | hidden_size=hidden_size, 83 | num_layers=num_layers, 84 | batch_first=True, 85 | ) 86 | self.linear = nn.Linear(hidden_size, input_size) 87 | 88 | def forward(self, x_input, encoder_hidden_states): 89 | """ 90 | : param x_input: should be 2D (batch_size, input_size) 91 | : param encoder_hidden_states: hidden states 92 | : return output, hidden: output gives all the hidden states in the sequence; 93 | : hidden gives the hidden state and cell state for the last 94 | : element in the sequence 95 | """ 96 | lstm_out, self.hidden = self.lstm(x_input.unsqueeze(1), encoder_hidden_states) 97 | output = self.linear(lstm_out.squeeze(1)) 98 | 99 | return output, self.hidden 100 | 101 | 102 | class lstm_seq2seq(nn.Module): 103 | """train LSTM encoder-decoder and make predictions""" 104 | 105 | def __init__( 106 | self, 107 | input_size=2, 108 | output_size=2, 109 | embedding_size=24, 110 | num_layers=1, 111 | target_len=24, 112 | ): 113 | """ 114 | : param input_size: the number of expected features in the input X 115 | : param hidden_size: the number of features in the hidden state h 116 | """ 117 | 118 | super(lstm_seq2seq, self).__init__() 119 | 120 | self.input_size = input_size 121 | self.output_size = output_size 122 | self.hidden_size = embedding_size 123 | self.target_len = target_len 124 | self.encoder = lstm_encoder( 125 | input_size=input_size, hidden_size=embedding_size, num_layers=num_layers 126 | ) 127 | self.decoder = lstm_decoder( 128 | input_size=output_size, hidden_size=embedding_size, num_layers=num_layers 129 | ) 130 | self.X = None 131 | self.Y = None 132 | self.loss_fn = None 133 | self.loss = None 134 | 135 | def train_model( 136 | self, 137 | input_tensor, 138 | target_tensor, 139 | n_epochs=100, 140 | batch_size=150, 141 | training_prediction="recursive", 142 | teacher_forcing_ratio=0.5, 143 | learning_rate=0.01, 144 | dynamic_tf=False, 145 | ): 146 | """ 147 | train lstm encoder-decoder 148 | 149 | : param input_tensor: input data with shape (seq_len, # in batch, number features); PyTorch tensor 150 | : param target_tensor: target data with shape (seq_len, # in batch, number features); PyTorch tensor 151 | : param n_epochs: number of epochs 152 | : param target_len: number of values to predict 153 | : param batch_size: number of samples per gradient update 154 | : param training_prediction: type of prediction to make during training ('recursive', 'teacher_forcing', or 155 | : 'mixed_teacher_forcing'); default is 'recursive' 156 | : param teacher_forcing_ratio: float [0, 1) indicating how much teacher forcing to use when 157 | : training_prediction = 'teacher_forcing.' For each batch in training, we generate a random 158 | : number. If the random number is less than teacher_forcing_ratio, we use teacher forcing. 159 | : Otherwise, we predict recursively. If teacher_forcing_ratio = 1, we train only using 160 | : teacher forcing. 161 | : param learning_rate: float >= 0; learning rate 162 | : param dynamic_tf: use dynamic teacher forcing (True/False); dynamic teacher forcing 163 | : reduces the amount of teacher forcing for each epoch 164 | : return losses: array of loss function for each epoch 165 | """ 166 | 167 | # initialize array of losses 168 | losses = np.full(n_epochs, np.nan) 169 | 170 | self.X = input_tensor 171 | self.Y = target_tensor 172 | 173 | optimizer = optim.Adam(self.parameters(), lr=learning_rate) 174 | criterion = nn.MSELoss() 175 | self.loss_fn = criterion 176 | 177 | # calculate number of batch iterations 178 | n_batches = int(input_tensor.shape[0] / batch_size) 179 | 180 | with trange(n_epochs) as tr: 181 | for it in tr: 182 | 183 | batch_loss = 0.0 184 | 185 | for b in range(n_batches): 186 | # select data 187 | input_batch = input_tensor[ 188 | b * batch_size : (b + 1) * batch_size, :, : 189 | ] 190 | # print('input_batch', input_batch.shape) 191 | target_batch = target_tensor[ 192 | b * batch_size : (b + 1) * batch_size, :, : 193 | ] 194 | 195 | # outputs tensor 196 | # outputs = torch.zeros(batch_size, self.target_len, self.output_size) 197 | 198 | # zero the gradient 199 | optimizer.zero_grad() 200 | 201 | outputs = self.predict(input_batch) 202 | 203 | # compute the loss 204 | loss = criterion(outputs, target_batch) 205 | batch_loss += loss.item() 206 | 207 | # backpropagation 208 | self.loss = loss 209 | loss.backward() 210 | optimizer.step() 211 | 212 | # loss for epoch 213 | batch_loss /= n_batches 214 | losses[it] = batch_loss 215 | 216 | # dynamic teacher forcing 217 | if dynamic_tf and teacher_forcing_ratio > 0: 218 | teacher_forcing_ratio = teacher_forcing_ratio - 0.02 219 | 220 | # progress bar 221 | tr.set_postfix(loss="{0:.3f}".format(batch_loss)) 222 | 223 | return losses 224 | 225 | def predict(self, input_tensor): 226 | """ 227 | : param input_tensor: input data (seq_len, input_size); PyTorch tensor 228 | : param target_len: number of target values to predict 229 | : return np_outputs: tensor containing predicted values; prediction done recursively 230 | """ 231 | 232 | # encode input_tensor 233 | # input_tensor = input_tensor.unsqueeze(1) # add in batch size of 1 234 | 235 | # initialize hidden state 236 | encoder_hidden = self.encoder.init_hidden(input_tensor.shape[0]) 237 | 238 | encoder_output, encoder_hidden = self.encoder(input_tensor) 239 | 240 | # initialize tensor for predictions 241 | outputs = [] 242 | 243 | # decode input_tensor 244 | decoder_input = input_tensor[:, -1, : self.output_size] 245 | decoder_hidden = encoder_hidden 246 | 247 | for t in range(self.target_len): 248 | decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden) 249 | outputs.append(decoder_output.unsqueeze(1)) 250 | decoder_input = decoder_output 251 | 252 | return torch.cat(outputs, dim=1) 253 | 254 | 255 | # encoder outputs 256 | """ 257 | encoder_output, encoder_hidden = self.encoder(input_batch) 258 | 259 | # decoder with teacher forcing 260 | decoder_input = input_batch[:, -1, :self.output_size] # shape: (batch_size, input_size) 261 | decoder_hidden = encoder_hidden 262 | 263 | 264 | 265 | if training_prediction == 'recursive': 266 | # predict recursively 267 | for t in range(self.target_len): 268 | 269 | decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden) 270 | outputs[:,t,:] = decoder_output 271 | decoder_input = decoder_output 272 | 273 | if training_prediction == 'teacher_forcing': 274 | # use teacher forcing 275 | if random.random() < teacher_forcing_ratio: 276 | for t in range(self.target_len): 277 | decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden) 278 | outputs[:, t] = decoder_output 279 | decoder_input = target_batch[t, :, :] 280 | 281 | # predict recursively 282 | else: 283 | for t in range(self.target_len): 284 | decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden) 285 | outputs[:, t] = decoder_output 286 | decoder_input = decoder_output 287 | 288 | if training_prediction == 'mixed_teacher_forcing': 289 | # predict using mixed teacher forcing 290 | for t in range(self.target_len): 291 | decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden) 292 | outputs[:, t] = decoder_output 293 | 294 | # predict with teacher forcing 295 | if random.random() < teacher_forcing_ratio: 296 | decoder_input = target_batch[:, t, :] 297 | 298 | # predict recursively 299 | else: 300 | decoder_input = decoder_output 301 | """ 302 | -------------------------------------------------------------------------------- /src/models/rnn.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torch 3 | from torch.utils.data import TensorDataset 4 | import numpy as np 5 | 6 | 7 | class rnn(torch.nn.Module): 8 | """ 9 | The auxiliary RNN issuing point predictions. 10 | Point predictions are used as baseline to which the (normalised) 11 | uncertainty intervals are added in the main CFRNN network. 12 | """ 13 | 14 | def __init__( 15 | self, 16 | embedding_size=24, 17 | input_size=2, 18 | output_size=1, 19 | horizon=1, 20 | num_layers=1, 21 | path=None, 22 | ): 23 | """ 24 | Initialises the auxiliary forecaster. 25 | Args: 26 | embedding_size: hyperparameter indicating the size of the latent 27 | RNN embeddings. 28 | input_size: dimensionality of the input time-series 29 | output_size: dimensionality of the forecast 30 | horizon: forecasting horizon 31 | rnn_mode: type of the underlying RNN network 32 | path: optional path where to save the auxiliary model to be used 33 | in the main CFRNN network 34 | """ 35 | super(rnn, self).__init__() 36 | # input_size indicates the number of features in the time series 37 | # input_size=1 for univariate series. 38 | self.input_size = input_size 39 | self.embedding_size = embedding_size 40 | self.horizon = horizon 41 | self.output_size = output_size 42 | self.path = path 43 | 44 | # self.forecaster_rnn = torch.nn.LSTM(input_size=input_size, hidden_size=embedding_size, batch_first=True) 45 | self.forecaster_rnn = torch.nn.LSTM( 46 | input_size=input_size, 47 | hidden_size=embedding_size, 48 | num_layers=num_layers, 49 | batch_first=True, 50 | ) 51 | self.forecaster_out = torch.nn.Linear(embedding_size, horizon * output_size) 52 | self.X = None 53 | self.y = None 54 | 55 | self.loss_fn = None 56 | self.loss = None 57 | 58 | def forward(self, x, state=None): 59 | 60 | # [batch, horizon, output_size] 61 | _, (h_n, c_n) = self.forecaster_rnn(x, state) 62 | 63 | out = self.forecaster_out(h_n[-1]).reshape(-1, self.horizon, self.output_size) 64 | 65 | return out, (h_n, c_n) 66 | 67 | def train_model(self, x_train, y_train, n_epochs=100, batch_size=150, lr=0.01): 68 | """ 69 | Trains the auxiliary forecaster to the training dataset. 70 | Args: 71 | x_train, y_train: tensor input 72 | batch_size: batch size 73 | epochs: number of training epochs 74 | lr: learning rate 75 | """ 76 | 77 | self.X = x_train 78 | self.y = y_train # [:, :self.horizon, :] 79 | 80 | train_dataset = TensorDataset(self.X, self.y) 81 | 82 | train_loader = torch.utils.data.DataLoader( 83 | train_dataset, batch_size=batch_size, shuffle=True 84 | ) 85 | 86 | optimizer = torch.optim.Adam(self.parameters(), lr=lr) 87 | self.loss_fn = torch.nn.MSELoss() 88 | 89 | self.train() 90 | for epoch in range(n_epochs): 91 | for sequences, targets in train_loader: 92 | 93 | out, _ = self(sequences.detach()) 94 | self.loss = self.loss_fn(out, targets.detach()) 95 | 96 | if epoch == n_epochs - 1: # for bjrnn 97 | break 98 | optimizer.zero_grad() 99 | self.loss.backward( 100 | retain_graph=True 101 | ) # backpropagation, compute gradients 102 | optimizer.step() # apply gradients 103 | 104 | if epoch % 10 == 0: 105 | print("Epoch: ", epoch, "| train loss: %.4f" % self.loss.data) 106 | 107 | if self.path is not None: 108 | torch.save(self, self.path) 109 | 110 | def predict(self, x): 111 | """ 112 | x: tensor input 113 | """ 114 | 115 | pred_y, _ = self(x) 116 | return pred_y 117 | -------------------------------------------------------------------------------- /src/models/rnn_rnn.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torch 3 | from torch.utils.data import TensorDataset 4 | import numpy as np 5 | 6 | 7 | class rnn(torch.nn.Module): 8 | """ 9 | The auxiliary RNN issuing point predictions. 10 | Point predictions are used as baseline to which the (normalised) 11 | uncertainty intervals are added in the main CFRNN network. 12 | """ 13 | 14 | def __init__( 15 | self, embedding_size=24, input_size=2, output_size=1, horizon=1, path=None 16 | ): 17 | """ 18 | Initialises the auxiliary forecaster. 19 | Args: 20 | embedding_size: hyperparameter indicating the size of the latent 21 | RNN embeddings. 22 | input_size: dimensionality of the input time-series 23 | output_size: dimensionality of the forecast 24 | horizon: forecasting horizon 25 | rnn_mode: type of the underlying RNN network 26 | path: optional path where to save the auxiliary model to be used 27 | in the main CFRNN network 28 | """ 29 | super(rnn, self).__init__() 30 | # input_size indicates the number of features in the time series 31 | # input_size=1 for univariate series. 32 | self.input_size = input_size 33 | self.embedding_size = embedding_size 34 | self.horizon = horizon 35 | self.output_size = output_size 36 | self.path = path 37 | 38 | # self.forecaster_rnn = torch.nn.LSTM(input_size=input_size, hidden_size=embedding_size, batch_first=True) 39 | self.forecaster_rnn = torch.nn.RNN( 40 | input_size=input_size, hidden_size=embedding_size, batch_first=True 41 | ) 42 | self.forecaster_out = torch.nn.Linear(embedding_size, horizon * output_size) 43 | self.X = None 44 | self.y = None 45 | 46 | self.loss_fn = None 47 | self.loss = None 48 | 49 | def forward(self, x, state=None): 50 | 51 | # [batch, horizon, output_size] 52 | _, h_n = self.forecaster_rnn(x, state) 53 | 54 | out = self.forecaster_out(h_n).reshape(-1, self.horizon, self.output_size) 55 | 56 | return out, h_n 57 | 58 | def train_model(self, x_train, y_train, n_epochs=100, batch_size=150, lr=0.01): 59 | """ 60 | Trains the auxiliary forecaster to the training dataset. 61 | Args: 62 | x_train, y_train: tensor input 63 | batch_size: batch size 64 | epochs: number of training epochs 65 | lr: learning rate 66 | """ 67 | 68 | self.X = x_train 69 | self.y = y_train 70 | print("yshape", y_train.shape) 71 | 72 | train_dataset = TensorDataset(x_train, y_train) 73 | 74 | train_loader = torch.utils.data.DataLoader( 75 | train_dataset, batch_size=batch_size, shuffle=True 76 | ) 77 | 78 | optimizer = torch.optim.Adam(self.parameters(), lr=lr) 79 | self.loss_fn = torch.nn.MSELoss() 80 | 81 | self.train() 82 | for epoch in range(n_epochs): 83 | for sequences, targets in train_loader: 84 | 85 | out, _ = self(sequences.detach()) 86 | self.loss = self.loss_fn(out, targets.detach()) 87 | 88 | optimizer.zero_grad() 89 | self.loss.backward( 90 | retain_graph=True 91 | ) # backpropagation, compute gradients 92 | optimizer.step() # apply gradients 93 | 94 | # if epoch % 10 == 0: 95 | # print("Epoch: ", epoch, "| train loss: %.4f" % self.loss.data) 96 | 97 | if self.path is not None: 98 | torch.save(self, self.path) 99 | 100 | def predict(self, x): 101 | """ 102 | x: tensor input 103 | """ 104 | 105 | pred_y, _ = self(x) 106 | return pred_y 107 | -------------------------------------------------------------------------------- /src/models/transformer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import numpy as np 4 | import time 5 | import math 6 | from matplotlib import pyplot 7 | 8 | torch.manual_seed(0) 9 | np.random.seed(0) 10 | 11 | # This concept is also called teacher forceing. 12 | # The flag decides if the loss will be calculted over all 13 | # or just the predicted values. 14 | calculate_loss_over_all_values = False 15 | 16 | # S is the source sequence length 17 | # T is the target sequence length 18 | # N is the batch size 19 | # E is the feature number 20 | 21 | # src = torch.rand((10, 32, 512)) # (S,N,E) 22 | # tgt = torch.rand((20, 32, 512)) # (T,N,E) 23 | # out = transformer_model(src, tgt) 24 | # 25 | # print(out) 26 | 27 | input_window = 100 28 | output_window = 5 29 | batch_size = 10 # batch size 30 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 31 | 32 | 33 | class PositionalEncoding(nn.Module): 34 | 35 | def __init__(self, d_model, max_len=5000): 36 | super(PositionalEncoding, self).__init__() 37 | pe = torch.zeros(max_len, d_model) 38 | position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) 39 | div_term = torch.exp( 40 | torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model) 41 | ) 42 | pe[:, 0::2] = torch.sin(position * div_term) 43 | pe[:, 1::2] = torch.cos(position * div_term) 44 | pe = pe.unsqueeze(0).transpose(0, 1) 45 | # pe.requires_grad = False 46 | self.register_buffer("pe", pe) 47 | 48 | def forward(self, x): 49 | return x + self.pe[: x.size(0), :] 50 | 51 | 52 | class TransAm(nn.Module): 53 | def __init__(self, feature_size=250, num_layers=1, dropout=0.1): 54 | super(TransAm, self).__init__() 55 | self.model_type = "Transformer" 56 | 57 | self.src_mask = None 58 | self.pos_encoder = PositionalEncoding(feature_size) 59 | self.encoder_layer = nn.TransformerEncoderLayer( 60 | d_model=feature_size, nhead=10, dropout=dropout 61 | ) 62 | self.transformer_encoder = nn.TransformerEncoder( 63 | self.encoder_layer, num_layers=num_layers 64 | ) 65 | self.decoder = nn.Linear(feature_size, 1) 66 | self.init_weights() 67 | 68 | def init_weights(self): 69 | initrange = 0.1 70 | self.decoder.bias.data.zero_() 71 | self.decoder.weight.data.uniform_(-initrange, initrange) 72 | 73 | def forward(self, src): 74 | if self.src_mask is None or self.src_mask.size(0) != len(src): 75 | device = src.device 76 | mask = self._generate_square_subsequent_mask(len(src)).to(device) 77 | self.src_mask = mask 78 | 79 | src = self.pos_encoder(src) 80 | output = self.transformer_encoder(src, self.src_mask) # , self.src_mask) 81 | output = self.decoder(output) 82 | return output 83 | 84 | def _generate_square_subsequent_mask(self, sz): 85 | mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1) 86 | mask = ( 87 | mask.float() 88 | .masked_fill(mask == 0, float("-inf")) 89 | .masked_fill(mask == 1, float(0.0)) 90 | ) 91 | return mask 92 | 93 | 94 | # if window is 100 and prediction step is 1 95 | # in -> [0..99] 96 | # target -> [1..100] 97 | def create_inout_sequences(input_data, tw): 98 | inout_seq = [] 99 | L = len(input_data) 100 | for i in range(L - tw): 101 | train_seq = np.append( 102 | input_data[i : i + tw][:-output_window], output_window * [0] 103 | ) 104 | train_label = input_data[i : i + tw] 105 | # train_label = input_data[i+output_window:i+tw+output_window] 106 | inout_seq.append((train_seq, train_label)) 107 | return torch.FloatTensor(inout_seq) 108 | 109 | 110 | def get_data(): 111 | time = np.arange(0, 400, 0.1) 112 | amplitude = ( 113 | np.sin(time) 114 | + np.sin(time * 0.05) 115 | + np.sin(time * 0.12) * np.random.normal(-0.2, 0.2, len(time)) 116 | ) 117 | 118 | # from pandas import read_csv 119 | # series = read_csv('daily-min-temperatures.csv', header=0, index_col=0, parse_dates=True, squeeze=True) 120 | 121 | from sklearn.preprocessing import MinMaxScaler 122 | 123 | scaler = MinMaxScaler(feature_range=(-1, 1)) 124 | # amplitude = scaler.fit_transform(series.to_numpy().reshape(-1, 1)).reshape(-1) 125 | amplitude = scaler.fit_transform(amplitude.reshape(-1, 1)).reshape(-1) 126 | 127 | sampels = 2800 128 | train_data = amplitude[:sampels] 129 | test_data = amplitude[sampels:] 130 | 131 | # convert our train data into a pytorch train tensor 132 | # train_tensor = torch.FloatTensor(train_data).view(-1) 133 | # todo: add comment.. 134 | train_sequence = create_inout_sequences(train_data, input_window) 135 | train_sequence = train_sequence[:-output_window] # todo: fix hack? 136 | 137 | # test_data = torch.FloatTensor(test_data).view(-1) 138 | test_data = create_inout_sequences(test_data, input_window) 139 | test_data = test_data[:-output_window] # todo: fix hack? 140 | 141 | return train_sequence.to(device), test_data.to(device) 142 | 143 | 144 | def get_batch(source, i, batch_size): 145 | seq_len = min(batch_size, len(source) - 1 - i) 146 | data = source[i : i + seq_len] 147 | input = torch.stack( 148 | torch.stack([item[0] for item in data]).chunk(input_window, 1) 149 | ) # 1 is feature size 150 | target = torch.stack(torch.stack([item[1] for item in data]).chunk(input_window, 1)) 151 | return input, target 152 | 153 | 154 | def train(train_data): 155 | model.train() # Turn on the train mode 156 | total_loss = 0.0 157 | start_time = time.time() 158 | 159 | for batch, i in enumerate(range(0, len(train_data) - 1, batch_size)): 160 | data, targets = get_batch(train_data, i, batch_size) 161 | optimizer.zero_grad() 162 | output = model(data) 163 | 164 | if calculate_loss_over_all_values: 165 | loss = criterion(output, targets) 166 | else: 167 | loss = criterion(output[-output_window:], targets[-output_window:]) 168 | 169 | loss.backward() 170 | torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5) 171 | optimizer.step() 172 | 173 | total_loss += loss.item() 174 | log_interval = int(len(train_data) / batch_size / 5) 175 | if batch % log_interval == 0 and batch > 0: 176 | cur_loss = total_loss / log_interval 177 | elapsed = time.time() - start_time 178 | print( 179 | "| epoch {:3d} | {:5d}/{:5d} batches | " 180 | "lr {:02.6f} | {:5.2f} ms | " 181 | "loss {:5.5f} | ppl {:8.2f}".format( 182 | epoch, 183 | batch, 184 | len(train_data) // batch_size, 185 | scheduler.get_lr()[0], 186 | elapsed * 1000 / log_interval, 187 | cur_loss, 188 | math.exp(cur_loss), 189 | ) 190 | ) 191 | total_loss = 0 192 | start_time = time.time() 193 | 194 | 195 | def plot_and_loss(eval_model, data_source, epoch): 196 | eval_model.eval() 197 | total_loss = 0.0 198 | test_result = torch.Tensor(0) 199 | truth = torch.Tensor(0) 200 | with torch.no_grad(): 201 | for i in range(0, len(data_source) - 1): 202 | data, target = get_batch(data_source, i, 1) 203 | # look like the model returns static values for the output window 204 | output = eval_model(data) 205 | if calculate_loss_over_all_values: 206 | total_loss += criterion(output, target).item() 207 | else: 208 | total_loss += criterion( 209 | output[-output_window:], target[-output_window:] 210 | ).item() 211 | 212 | test_result = torch.cat( 213 | (test_result, output[-1].view(-1).cpu()), 0 214 | ) # todo: check this. -> looks good to me 215 | truth = torch.cat((truth, target[-1].view(-1).cpu()), 0) 216 | 217 | # test_result = test_result.cpu().numpy() 218 | len(test_result) 219 | 220 | pyplot.plot(test_result, color="red") 221 | pyplot.plot(truth[:500], color="blue") 222 | pyplot.plot(test_result - truth, color="green") 223 | pyplot.grid(True, which="both") 224 | pyplot.axhline(y=0, color="k") 225 | pyplot.savefig("graph/transformer-epoch%d.png" % epoch) 226 | pyplot.close() 227 | 228 | return total_loss / i 229 | 230 | 231 | def predict_future(eval_model, data_source, steps): 232 | eval_model.eval() 233 | _, data = get_batch(data_source, 0, 1) 234 | with torch.no_grad(): 235 | for i in range(0, steps, 1): 236 | input = torch.clone(data[-input_window:]) 237 | input[-output_window:] = 0 238 | output = eval_model(data[-input_window:]) 239 | data = torch.cat((data, output[-1:])) 240 | 241 | data = data.cpu().view(-1) 242 | 243 | pyplot.plot(data, color="red") 244 | pyplot.plot(data[:input_window], color="blue") 245 | pyplot.grid(True, which="both") 246 | pyplot.axhline(y=0, color="k") 247 | pyplot.savefig("graph/transformer-future%d.png" % steps) 248 | pyplot.close() 249 | 250 | 251 | # entweder ist hier ein fehler im loss oder in der train methode, aber die ergebnisse sind unterschiedlich 252 | # auch zu denen der predict_future 253 | def evaluate(eval_model, data_source): 254 | eval_model.eval() # Turn on the evaluation mode 255 | total_loss = 0.0 256 | eval_batch_size = 1000 257 | with torch.no_grad(): 258 | for i in range(0, len(data_source) - 1, eval_batch_size): 259 | data, targets = get_batch(data_source, i, eval_batch_size) 260 | output = eval_model(data) 261 | if calculate_loss_over_all_values: 262 | total_loss += len(data[0]) * criterion(output, targets).cpu().item() 263 | else: 264 | total_loss += ( 265 | len(data[0]) 266 | * criterion(output[-output_window:], targets[-output_window:]) 267 | .cpu() 268 | .item() 269 | ) 270 | return total_loss / len(data_source) 271 | 272 | 273 | train_data, val_data = get_data() 274 | model = TransAm().to(device) 275 | 276 | criterion = nn.MSELoss() 277 | lr = 0.005 278 | # optimizer = torch.optim.SGD(model.parameters(), lr=lr) 279 | optimizer = torch.optim.AdamW(model.parameters(), lr=lr) 280 | scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.98) 281 | 282 | best_val_loss = float("inf") 283 | epochs = 100 # The number of epochs 284 | best_model = None 285 | 286 | for epoch in range(1, epochs + 1): 287 | epoch_start_time = time.time() 288 | train(train_data) 289 | 290 | if epoch % 10 == 0: 291 | val_loss = plot_and_loss(model, val_data, epoch) 292 | predict_future(model, val_data, 200) 293 | else: 294 | val_loss = evaluate(model, val_data) 295 | 296 | print("-" * 89) 297 | print( 298 | "| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.5f} | valid ppl {:8.2f}".format( 299 | epoch, (time.time() - epoch_start_time), val_loss, math.exp(val_loss) 300 | ) 301 | ) 302 | print("-" * 89) 303 | 304 | # if val_loss < best_val_loss: 305 | # best_val_loss = val_loss 306 | # best_model = model 307 | 308 | scheduler.step() 309 | 310 | # src = torch.rand(input_window, batch_size, 1) # (source sequence length,batch size,feature number) 311 | # out = model(src) 312 | # 313 | # print(out) 314 | -------------------------------------------------------------------------------- /src/models/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | from tqdm import trange 5 | from copulae import GumbelCopula 6 | from copulae.core import pseudo_obs 7 | from copy import deepcopy 8 | 9 | 10 | def gumbel_copula_loss(x, cop, data, epsilon): 11 | return np.fabs(cop.cdf([x] * data.shape[1]) - 1 + epsilon) 12 | 13 | 14 | def empirical_copula_loss(x, data, epsilon): 15 | pseudo_data = pseudo_obs(data) 16 | return np.fabs( 17 | np.mean( 18 | np.all( 19 | np.less_equal(pseudo_data, np.array([x] * pseudo_data.shape[1])), axis=1 20 | ) 21 | ) 22 | - 1 23 | + epsilon 24 | ) 25 | 26 | 27 | def empirical_copula_loss_new(x, data, epsilon): 28 | pseudo_data = pseudo_obs(data) 29 | return ( 30 | np.mean( 31 | np.all( 32 | np.less_equal(pseudo_data, np.array([x] * pseudo_data.shape[1])), axis=1 33 | ) 34 | ) 35 | - 1 36 | + epsilon 37 | ) 38 | 39 | 40 | # def mace(cov): 41 | # x_axis = [i/10.0 for i in range(1,10)] 42 | # return np.mean([abs(x_axis[8-i] - cov[i]) for i in range(9)]) 43 | 44 | 45 | class CP(nn.Module): 46 | def __init__(self, dimension, epsilon): 47 | super(CP, self).__init__() 48 | self.alphas = nn.Parameter(torch.ones(dimension)) 49 | self.epsilon = epsilon 50 | self.relu = torch.nn.ReLU() 51 | 52 | def forward(self, pseudo_data): 53 | coverage = torch.mean( 54 | torch.relu( 55 | torch.prod(torch.sigmoid((self.alphas - pseudo_data) * 1000), dim=1) 56 | ) 57 | ) 58 | return torch.abs(coverage - 1 + self.epsilon) 59 | 60 | 61 | def search_alpha(alpha_input, epsilon, epochs=500): 62 | # pseudo_data = torch.tensor(pseudo_obs(alpha_input)) 63 | pseudo_data = torch.tensor(alpha_input) 64 | dim = alpha_input.shape[-1] 65 | cp = CP(dim, epsilon) 66 | optimizer = torch.optim.Adam(cp.parameters(), weight_decay=1e-4) 67 | 68 | with trange(epochs, desc="training", unit="epochs") as pbar: 69 | for i in pbar: 70 | optimizer.zero_grad() 71 | loss = cp(pseudo_data) 72 | 73 | loss.backward() 74 | optimizer.step() 75 | pbar.set_postfix(loss=loss.detach().numpy()) 76 | 77 | return cp.alphas.detach().numpy() 78 | -------------------------------------------------------------------------------- /src/models/vanila_copula.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import torch 4 | from copulae import GumbelCopula 5 | from copulae.core import pseudo_obs 6 | from .utils import search_alpha 7 | 8 | 9 | def gumbel_copula_loss(x, cop, data, epsilon): 10 | return np.fabs(cop.cdf([x] * data.shape[1]) - 1 + epsilon) 11 | 12 | 13 | def empirical_copula_loss(x, data, epsilon): 14 | pseudo_data = pseudo_obs(data) 15 | return np.fabs( 16 | np.mean( 17 | np.all( 18 | np.less_equal(pseudo_data, np.array([x] * pseudo_data.shape[1])), axis=1 19 | ) 20 | ) 21 | - 1 22 | + epsilon 23 | ) 24 | 25 | 26 | class vanila_copula: 27 | """ 28 | copula as implemented by Messoudi et al. 29 | """ 30 | 31 | def __init__(self, model, cali_x, cali_y): 32 | 33 | self.model = model 34 | 35 | self.cali_x = cali_x 36 | self.cali_y = cali_y 37 | 38 | self.nonconformity = None 39 | self.results_dict = {} 40 | 41 | def calibrate(self): 42 | 43 | pred_y = self.model.predict(self.cali_x) 44 | nonconformity = torch.norm((pred_y - self.cali_y), p=2, dim=-1).detach().numpy() 45 | self.nonconformity = nonconformity 46 | 47 | def predict(self, X_test, epsilon=0.1): 48 | 49 | # alphas = self.nonconformity 50 | pred_y = self.model.predict(self.cali_x) 51 | scores = torch.norm((pred_y - self.cali_y), p=2, dim=-1).detach().numpy() 52 | alphas = [] 53 | for i in range(scores.shape[0]): 54 | a = (scores[i] > self.nonconformity).mean(axis=0) 55 | alphas.append(a) 56 | alphas = np.array(alphas) 57 | 58 | x_candidates = np.linspace(0.0001, 0.999, num=300) 59 | x_fun = [empirical_copula_loss(x, alphas, epsilon) for x in x_candidates] 60 | x_sorted = sorted(list(zip(x_fun, x_candidates))) 61 | 62 | mapping = { 63 | i: sorted(self.nonconformity[:, i].tolist()) for i in range(alphas.shape[1]) 64 | } 65 | quantile = int(x_sorted[0][1] * alphas.shape[0]) 66 | 67 | radius = np.array([mapping[i][quantile] for i in range(alphas.shape[1])]) 68 | 69 | y_pred = self.model.predict(X_test) 70 | 71 | self.results_dict[epsilon] = {"y_pred": y_pred, "radius": radius} 72 | 73 | return y_pred, radius 74 | 75 | def calc_area(self, radius): 76 | 77 | area = sum([np.pi * r**2 for r in radius]) 78 | 79 | return area 80 | 81 | def calc_area_3d(self, radius): 82 | 83 | area = sum([4 / 3.0 * np.pi * r**3 for r in radius]) 84 | 85 | return area 86 | 87 | def calc_area_1d(self, radius): 88 | 89 | area = sum(radius) 90 | 91 | return area 92 | 93 | def calc_coverage(self, radius, y_pred, y_test): 94 | 95 | testnonconformity = torch.norm((y_pred - y_test), p=2, dim=-1).detach().numpy() 96 | 97 | circle_covs = [] 98 | for j in range(y_test.shape[-2]): 99 | circle_covs.append(testnonconformity[:, j] < radius[j]) 100 | 101 | circle_covs = np.array(circle_covs) 102 | coverage = np.mean(np.all(circle_covs, axis=0)) 103 | return coverage 104 | 105 | def calc_coverage_3d(self, radius, y_pred, y_test): 106 | 107 | return self.calc_coverage(radius, y_pred, y_test) 108 | 109 | def calc_coverage_1d(self, radius, y_pred, y_test): 110 | 111 | return self.calc_coverage(radius, y_pred, y_test) 112 | --------------------------------------------------------------------------------