├── .gitignore ├── LICENSE ├── README.md ├── __init__.py ├── conf └── example.json ├── model ├── DenseAE.py ├── FDJAE.py ├── InceptionAE.py ├── RNNAE.py ├── ResidualAE.py ├── __init__.py ├── activation.py ├── builder.py ├── commons.py ├── initialization.py ├── loss.py └── normalization.py ├── run.py └── util ├── __init__.py ├── conf.py ├── data.py ├── experiment.py ├── flops_counter.py ├── pwlf.py ├── summarization.py ├── torchsummary.py └── train.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SEAnet 2 | 3 | SEAnet is a novel architecture especially designed for data series representation learning (DEA). 4 | 5 | Codes were developed and tested under Linux environment. 6 | 7 | ## Train SEAnet 8 | 9 | 1. _**Compile Coconut Sampling**_ 10 | 11 | ```bash 12 | cd lib/ 13 | make 14 | ``` 15 | 16 | 2. **Add a configuration file** 17 | 18 | An example configuration for SEAnet is given in *conf/example.json*. 19 | Two fields with *TO_BE_CHANGED* are required to get changed. 20 | 21 | > **database_path**: indicates the dataset to be indexed \ 22 | > **query_path**: indicates the query set 23 | 24 | Other fields could be left by default. 25 | Please refer to *util/conf.py* for all possible configurations. 26 | 27 | 3. **Train SEAnet** 28 | 29 | ```bash 30 | python run.py -C conf/example.json 31 | ``` 32 | 33 | ## Approximate Similarity Search 34 | 35 | The indexing and query answering of DEA is in https://github.com/qtwang/isax-modularized 36 | 37 | ## Cite this work 38 | 39 | ```latex 40 | @inproceedings{kdd21-Wang-SEAnet, 41 | author = {Wang, Qitong and 42 | Palpanas, Themis}, 43 | title = {Deep Learning Embeddings for Data Series Similarity Search}, 44 | booktitle = {{KDD} '21: The 27th {ACM} {SIGKDD} Conference on Knowledge Discovery 45 | and Data Mining, Virtual Event, Singapore, August 14-18, 2021}, 46 | publisher = {{ACM}}, 47 | year = {2021}, 48 | url = {https://doi.org/10.1145/3447548.3467317}, 49 | doi = {10.1145/3447548.3467317}, 50 | timestamp = {Thu, 05 Aug 2021 09:46:47 +0800} 51 | } 52 | ``` 53 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qtwang/SEAnet/0cd38ff770bae8c4fe4f8d5f227a645b8c4c0ec9/__init__.py -------------------------------------------------------------------------------- /conf/example.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset_name": "rw", 3 | "database_path": "TO_BE_CHANGED", 4 | "query_path": "TO_BE_CHANGED", 5 | "train_path": "default", 6 | "log_filepath": "default", 7 | "val_path": "default", 8 | "sampling_name": "coconut", 9 | "coconut_libpath": "./lib/coconut_sampling.so", 10 | "encoder": "residual", 11 | "decoder": "residual", 12 | "dilation_type": "exponential", 13 | "dim_series": 256, 14 | "dim_embedding": 16, 15 | "dim_en_latent": 256, 16 | "dim_de_latent": 256, 17 | "size_db": 10000000, 18 | "size_query": 1000, 19 | "size_train": 200000, 20 | "size_val": 10000, 21 | "size_batch": 256, 22 | "size_kernel": 3, 23 | "num_en_resblock": 3, 24 | "num_de_resblock": 2, 25 | "num_en_channels": 256, 26 | "num_de_channels": 256, 27 | "num_epoch": 100, 28 | "relu_slope": 1e-2, 29 | "optim_type": "sgd", 30 | "momentum": 0.9, 31 | "lr_mode": "linear", 32 | "lr_max": 1e-2, 33 | "lr_min": 1e-5, 34 | "wd_mode": "linear", 35 | "wd_max": 1e-4, 36 | "wd_min": 1e-8, 37 | "device": "cuda", 38 | "activation_conv": "relu", 39 | "activation_linear": "lecuntanh", 40 | "resblock_pre_activation": true, 41 | "layernorm_type": "layernorm", 42 | "layernorm_elementwise_affine": true, 43 | "train_type": "linearlycombine", 44 | "recons_weight": 0.25, 45 | "train_detach_query": true, 46 | "model_init": "lsuv", 47 | "lsuv_size": 2000, 48 | "lsuv_ortho": true, 49 | "if_record": false, 50 | "orth_regularizer": "srip", 51 | "srip_mode": "linear", 52 | "srip_max": 5e-4, 53 | "srip_min": 0, 54 | "checkpoint_mode": "last", 55 | "checkpoint_postfix": "pickle", 56 | "checkpoint_folder": "default", 57 | "to_embed": true, 58 | "db_embedding_path": "default", 59 | "query_embedding_path": "default", 60 | "embed_batch": 2000, 61 | "encoder_normalize_embedding": true, 62 | "decoder_normalize_reconstruction": true, 63 | "default_conf_filename": "conf.json", 64 | "reverse_de_dilation": false, 65 | "to_scale_lc": true, 66 | "to_scale_lr": true, 67 | "name": "example" 68 | } -------------------------------------------------------------------------------- /model/DenseAE.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | # modified from https://github.com/pytorch/vision/blob/c558be6b3b6ed5270ed2db0c5edc872c0d089c52/torchvision/models/densenet.py 3 | 4 | from typing import List 5 | from collections import OrderedDict 6 | 7 | import torch 8 | from torch import nn, Tensor 9 | 10 | from util.conf import Configuration 11 | from model.commons import Squeeze, Reshape 12 | 13 | 14 | 15 | class _DenseLayer(nn.Module): 16 | def __init__(self, conf: Configuration, in_channels: int, dilation: int): 17 | super(_DenseLayer, self).__init__() 18 | 19 | dim_series = conf.getHP('dim_series') 20 | kernel_size = conf.getHP('size_kernel') 21 | padding = int(kernel_size / 2) * dilation 22 | activation_name = conf.getHP('activation_conv') 23 | bias = conf.getHP('layernorm_type') == 'none' or not conf.getHP('layernorm_elementwise_affine') 24 | 25 | growth_rate = conf.getHP('dense_growth_rate') 26 | bottleneck_multiplier = conf.getHP('dense_bottleneck_multiplier') 27 | bottleneck_channels = int(growth_rate * bottleneck_multiplier) 28 | 29 | self.__bottleneck = nn.Sequential(conf.getLayerNorm(dim_series), 30 | conf.getActivation(activation_name), 31 | conf.getWeightNorm(nn.Conv1d(in_channels, bottleneck_channels, 1, bias=bias))) 32 | 33 | self.__convolution = nn.Sequential(conf.getLayerNorm(dim_series), 34 | conf.getActivation(activation_name), 35 | conf.getWeightNorm(nn.Conv1d(bottleneck_channels, growth_rate, kernel_size, padding=padding, dilation=dilation, bias=bias))) 36 | 37 | 38 | def forward(self, input) -> Tensor: 39 | if isinstance(input, List): 40 | input = torch.cat(input, 1) 41 | 42 | bottleneck = self.__bottleneck(input) 43 | 44 | return self.__convolution(bottleneck) 45 | 46 | 47 | 48 | class _DenseBlock(nn.ModuleDict): 49 | def __init__(self, conf: Configuration, in_channels: int, dilation: int, num_layers: int): 50 | super(_DenseBlock, self).__init__() 51 | 52 | growth_rate = conf.getHP('dense_growth_rate') 53 | 54 | for i in range(num_layers): 55 | self.add_module('denselayer{:d}'.format(i + 1), _DenseLayer(conf, in_channels + i * growth_rate, dilation)) 56 | 57 | 58 | def forward(self, input: Tensor) -> Tensor: 59 | latent = [input] 60 | 61 | for _, layer in self.items(): 62 | latent.append(layer(latent)) 63 | 64 | return torch.cat(latent, 1) 65 | 66 | 67 | 68 | class _Transition(nn.Sequential): 69 | def __init__(self, conf: Configuration, in_channels: int): 70 | super(_Transition, self).__init__() 71 | 72 | bias = conf.getHP('layernorm_type') == 'none' or not conf.getHP('layernorm_elementwise_affine') 73 | 74 | self.add_module('normalize', conf.getLayerNorm(conf.getHP('dim_series'))) 75 | self.add_module('activate', conf.getActivation(conf.getHP('activation_conv'))) 76 | self.add_module('convolute', conf.getWeightNorm(nn.Conv1d(in_channels, conf.getHP('dense_transition_channels'), 1, bias=bias))) 77 | 78 | 79 | 80 | class _DenseNet(nn.Module): 81 | def __init__(self, conf: Configuration, to_encode: bool): 82 | super(_DenseNet, self).__init__() 83 | 84 | kernel_size = conf.getHP('size_kernel') 85 | bias = conf.getHP('layernorm_type') == 'none' or not conf.getHP('layernorm_elementwise_affine') 86 | 87 | num_init_channels = conf.getHP('dense_init_channels') 88 | 89 | # DenseNet is by default pre-activation 90 | self.__model = nn.Sequential(OrderedDict([ 91 | ('conv0', conf.getWeightNorm(nn.Conv1d(1, num_init_channels, kernel_size, padding=int(kernel_size / 2), dilation=1, bias=bias))) 92 | ])) 93 | 94 | num_blocks = conf.getHP('num_en_denseblocks') if to_encode else conf.getHP('num_de_denseblocks') 95 | num_layers = conf.getHP('num_en_denselayers') if to_encode else conf.getHP('num_de_denselayers') 96 | 97 | growth_rate = conf.getHP('dense_growth_rate') 98 | num_transition_channels = conf.getHP('dense_transition_channels') 99 | 100 | if conf.getHP('dilation_type') == 'exponential': 101 | assert num_blocks > 1 and 2 ** (num_blocks + 1) <= conf.getHP('dim_series') + 1 102 | 103 | in_channels = num_init_channels 104 | for depth in range(1, num_blocks + 1): 105 | denseblock = _DenseBlock(conf, in_channels, conf.getDilatoin(depth, to_encode), num_layers) 106 | self.__model.add_module('denseblock{:d}'.format(depth), denseblock) 107 | in_channels = in_channels + num_layers * growth_rate 108 | 109 | # different from original DenseNet, output_chennels is controlled as in the corresponding ResNet 110 | transblock = _Transition(conf, in_channels) 111 | self.__model.add_module('transition{:d}'.format(depth), transblock) 112 | in_channels = num_transition_channels 113 | 114 | # finalization of pre-activation mode 115 | self.__model.add_module('normalize', conf.getLayerNorm(conf.getHP('dim_series'))) 116 | self.__model.add_module('activate', conf.getActivation(conf.getHP('activation_conv'))) 117 | 118 | 119 | def forward(self, input: Tensor) -> Tensor: 120 | return self.__model(input) 121 | 122 | 123 | 124 | class DenseEncoder(nn.Module): 125 | def __init__(self, conf: Configuration): 126 | super(DenseEncoder, self).__init__() 127 | 128 | dim_embedding = conf.getHP('dim_embedding') 129 | num_channels = conf.getHP('num_en_channels') 130 | dim_latent = conf.getHP('dim_en_latent') 131 | 132 | self.__model = nn.Sequential(_DenseNet(conf, to_encode=True), 133 | nn.AdaptiveMaxPool1d(1), 134 | Squeeze(), 135 | 136 | nn.Linear(num_channels, dim_latent), 137 | conf.getActivation(conf.getHP('activation_linear')), 138 | 139 | nn.Linear(dim_latent, dim_embedding, bias=False), 140 | nn.LayerNorm(dim_embedding, elementwise_affine=False) if conf.getHP('encoder_normalize_embedding') else nn.Identity()) 141 | 142 | self.__model.to(conf.getHP('device')) 143 | 144 | 145 | def forward(self, input: Tensor) -> Tensor: 146 | return self.__model(input) 147 | 148 | 149 | 150 | class DenseDecoder(nn.Module): 151 | def __init__(self, conf: Configuration): 152 | super(DenseDecoder, self).__init__() 153 | 154 | dim_series = conf.getHP('dim_series') 155 | dim_embedding = conf.getHP('dim_embedding') 156 | num_channels = conf.getHP('num_de_channels') 157 | dim_latent = conf.getHP('dim_de_latent') 158 | 159 | self.__model = nn.Sequential(Reshape([-1, 1, dim_embedding]), 160 | nn.Linear(dim_embedding, dim_series), 161 | conf.getActivation(conf.getHP('activation_linear')), 162 | 163 | _DenseNet(conf, to_encode=False), 164 | nn.AdaptiveMaxPool1d(1), 165 | Reshape([-1, 1, num_channels]), 166 | 167 | nn.Linear(num_channels, dim_latent), 168 | conf.getActivation(conf.getHP('activation_linear')), 169 | 170 | nn.Linear(dim_latent, dim_series, bias=False), 171 | nn.LayerNorm(dim_series, elementwise_affine=False) if conf.getHP('decoder_normalize_reconstruction') else nn.Identity()) 172 | 173 | self.__model.to(conf.getHP('device')) 174 | 175 | 176 | def forward(self, input: Tensor) -> Tensor: 177 | return self.__model(input) 178 | -------------------------------------------------------------------------------- /model/FDJAE.py: -------------------------------------------------------------------------------- 1 | # cite: https://github.com/White-Link/UnsupervisedScalableRepresentationLearningTimeSeries/blob/4baa655c3a761001f100c4a9955e57dc93589957/networks/causal_cnn.py 2 | # which cites: https://github.com/locuslab/TCN/blob/master/TCN/tcn.py 3 | # coding = utf-8 4 | 5 | from torch import nn, Tensor 6 | 7 | from util.conf import Configuration 8 | from model.commons import Squeeze, Reshape 9 | 10 | 11 | class Chomp1d(nn.Module): 12 | def __init__(self, chomp_size): 13 | super(Chomp1d, self).__init__() 14 | self.chomp_size = chomp_size 15 | 16 | def forward(self, input: Tensor) -> Tensor: 17 | return input[:, :, : -self.chomp_size] 18 | 19 | 20 | class _OriginalResBlock(nn.Module): 21 | def __init__(self, conf: Configuration, in_channels, out_channels, dilation, final: bool = False): 22 | super(_OriginalResBlock, self).__init__() 23 | 24 | kernel_size = conf.getHP('size_kernel') 25 | padding = (kernel_size - 1) * dilation 26 | activation_name = conf.getHP('activation_conv') 27 | bias = conf.getHP('layernorm_type') == 'none' 28 | 29 | self.__residual_link = nn.Sequential(conf.getWeightNorm(nn.Conv1d(in_channels, out_channels, kernel_size, padding=padding, dilation=dilation, bias=bias)), 30 | Chomp1d(padding), 31 | conf.getActivation(activation_name), 32 | 33 | conf.getWeightNorm(nn.Conv1d(out_channels, out_channels, kernel_size, padding=padding, dilation=dilation, bias=bias)), 34 | Chomp1d(padding), 35 | conf.getActivation(activation_name)) 36 | 37 | if in_channels != out_channels: 38 | self.__identity_link = nn.Conv1d(in_channels, out_channels, 1, bias=bias) 39 | else: 40 | self.__identity_link = nn.Identity() 41 | 42 | self.__after_addition = conf.getActivation(activation_name) if final else nn.Identity() 43 | 44 | 45 | def forward(self, input: Tensor) -> Tensor: 46 | residual = self.__residual_link(input) 47 | identity = self.__identity_link(input) 48 | 49 | return self.__after_addition(identity + residual) 50 | 51 | 52 | 53 | # class _PreActivatedResBlock(nn.Module): 54 | # def __init__(self, conf: Configuration, in_channels, out_channels, dilation, first = False, last = False): 55 | # super(_PreActivatedResBlock, self).__init__() 56 | 57 | # dim_series = conf.getHP('dim_series') 58 | # kernel_size = conf.getHP('size_kernel') 59 | # padding = int(kernel_size / 2) * dilation 60 | # activation_name = conf.getHP('activation_conv') 61 | # bias = conf.getHP('layernorm_type') == 'none' or not conf.getHP('layernorm_elementwise_affine') 62 | 63 | # if first: 64 | # self.__first_block = conf.getWeightNorm(nn.Conv1d(in_channels, out_channels, kernel_size, padding=padding, dilation=dilation, bias=bias)) 65 | # in_channels = out_channels 66 | # else: 67 | # self.__first_block = nn.Identity() 68 | 69 | # self.__residual_link = nn.Sequential(conf.getLayerNorm(dim_series), 70 | # conf.getActivation(activation_name), 71 | # conf.getWeightNorm(nn.Conv1d(in_channels, out_channels, kernel_size, padding=padding, dilation=dilation, bias=bias)), 72 | 73 | # conf.getLayerNorm(dim_series), 74 | # conf.getActivation(activation_name), 75 | # conf.getWeightNorm(nn.Conv1d(out_channels, out_channels, kernel_size, padding=padding, dilation=dilation, bias=bias))) 76 | 77 | # if in_channels != out_channels: 78 | # self.__identity_link = conf.getWeightNorm(nn.Conv1d(in_channels, out_channels, 1, bias=bias)) 79 | # else: 80 | # self.__identity_link = nn.Identity() 81 | 82 | # if last: 83 | # self.__after_addition = nn.Sequential(conf.getLayerNorm(dim_series), 84 | # conf.getActivation(activation_name)) 85 | # else: 86 | # self.__after_addition = nn.Identity() 87 | 88 | 89 | # def forward(self, input: Tensor) -> Tensor: 90 | # input = self.__first_block(input) 91 | 92 | # residual = self.__residual_link(input) 93 | # identity = self.__identity_link(input) 94 | 95 | # return self.__after_addition(identity + residual) 96 | 97 | 98 | 99 | class _ResNet(nn.Module): 100 | def __init__(self, conf: Configuration, to_encode: bool): 101 | super(_ResNet, self).__init__() 102 | 103 | num_resblock = conf.getHP('num_en_resblock') if to_encode else conf.getHP('num_de_resblock') 104 | 105 | if conf.getHP('dilation_type') == 'exponential': 106 | assert num_resblock > 1 and 2 ** (num_resblock + 1) <= conf.getHP('dim_series') + 1 107 | 108 | inner_channels = conf.getHP('num_en_channels') if to_encode else conf.getHP('num_de_channels') 109 | out_channels = conf.getHP('dim_en_latent') if to_encode else conf.getHP('dim_de_latent') 110 | 111 | if conf.getHP('resblock_pre_activation'): 112 | raise ValueError('pre-activated FDJ is not yet supported') 113 | # layers = [_PreActivatedResBlock(conf, 1, inner_channels, conf.getDilatoin(1, to_encode), first=True)] 114 | # layers += [_PreActivatedResBlock(conf, inner_channels, inner_channels, conf.getDilatoin(depth, to_encode)) for depth in range(2, num_resblock)] 115 | # layers += [_PreActivatedResBlock(conf, inner_channels, out_channels, conf.getDilatoin(num_resblock, to_encode), last=True)] 116 | else: 117 | layers = [_OriginalResBlock(conf, 1, inner_channels, conf.getDilatoin(1, to_encode))] 118 | layers += [_OriginalResBlock(conf, inner_channels, inner_channels, conf.getDilatoin(depth, to_encode)) for depth in range(2, num_resblock)] 119 | layers += [_OriginalResBlock(conf, inner_channels, out_channels, conf.getDilatoin(num_resblock, to_encode), final=True)] 120 | 121 | self.__model = nn.Sequential(*layers) 122 | 123 | 124 | def forward(self, input: Tensor) -> Tensor: 125 | return self.__model(input) 126 | 127 | 128 | 129 | class FDJEncoder(nn.Module): 130 | def __init__(self, conf: Configuration): 131 | super(FDJEncoder, self).__init__() 132 | 133 | dim_embedding = conf.getHP('dim_embedding') 134 | num_channels = conf.getHP('num_en_channels') 135 | dim_latent = conf.getHP('dim_en_latent') 136 | 137 | self.__model = nn.Sequential(_ResNet(conf, to_encode=True), 138 | nn.AdaptiveMaxPool1d(1), 139 | Squeeze(), 140 | 141 | nn.Linear(num_channels, dim_latent), 142 | conf.getActivation(conf.getHP('activation_linear')), 143 | 144 | nn.Linear(dim_latent, dim_embedding, bias=False), 145 | nn.LayerNorm(dim_embedding, elementwise_affine=False) if conf.getHP('encoder_normalize_embedding') else nn.Identity()) 146 | 147 | self.__model.to(conf.getHP('device')) 148 | 149 | 150 | def forward(self, input: Tensor) -> Tensor: 151 | return self.__model(input) 152 | 153 | 154 | 155 | class FDJDecoder(nn.Module): 156 | def __init__(self, conf: Configuration): 157 | super(FDJDecoder, self).__init__() 158 | 159 | dim_series = conf.getHP('dim_series') 160 | dim_embedding = conf.getHP('dim_embedding') 161 | num_channels = conf.getHP('num_de_channels') 162 | dim_latent = conf.getHP('dim_de_latent') 163 | 164 | self.__model = nn.Sequential(Reshape([-1, 1, dim_embedding]), 165 | nn.Linear(dim_embedding, dim_series), 166 | conf.getActivation(conf.getHP('activation_linear')), 167 | 168 | _ResNet(conf, to_encode=False), 169 | nn.AdaptiveMaxPool1d(1), 170 | Reshape([-1, 1, num_channels]), 171 | 172 | nn.Linear(num_channels, dim_latent), 173 | conf.getActivation(conf.getHP('activation_linear')), 174 | 175 | nn.Linear(dim_latent, dim_series, bias=False), 176 | nn.LayerNorm(dim_series, elementwise_affine=False) if conf.getHP('decoder_normalize_reconstruction') else nn.Identity()) 177 | 178 | self.__model.to(conf.getHP('device')) 179 | 180 | 181 | def forward(self, input: Tensor) -> Tensor: 182 | return self.__model(input) 183 | -------------------------------------------------------------------------------- /model/InceptionAE.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | # modified from https://github.com/hfawaz/InceptionTime/blob/458e5caf2093762fe2ac38f701304a600a08e6d7/classifiers/inception.py 3 | 4 | from typing import List 5 | from collections import OrderedDict 6 | 7 | import torch 8 | from torch import nn, Tensor 9 | import numpy as np 10 | 11 | from util.conf import Configuration 12 | from model.commons import Squeeze, Reshape 13 | 14 | 15 | 16 | class _InceptionLayer(nn.Module): 17 | def __init__(self, kernel_size: int, in_channels: int, out_channels: int, dilation: int = 1): 18 | super(_InceptionLayer, self).__init__() 19 | 20 | padding = int(kernel_size / 2) * dilation 21 | 22 | # TODO bottleneck is more commonly inside layer 23 | # bottleneck_channels = conf.getHP('inception_bottleneck_channels') 24 | # self.__bottleneck = nn.Conv1d(in_channels, bottleneck_channels, 1, bias=False) 25 | # self.__convolution = nn.Conv1d(bottleneck_channels, out_channels, kernel_size, padding=padding, dilation=dilation) 26 | 27 | self.__convolution = nn.Conv1d(in_channels, out_channels, kernel_size, padding=padding, dilation=dilation) 28 | 29 | 30 | def forward(self, input) -> Tensor: 31 | # bottleneck = self.__bottleneck(input) 32 | # return self.__convolution(bottleneck) 33 | 34 | return self.__convolution(input) 35 | 36 | 37 | class _InceptionBlock(nn.ModuleDict): 38 | def __init__(self, conf: Configuration, in_channels: int, out_channels: int, to_encode: bool): 39 | super(_InceptionBlock, self).__init__() 40 | 41 | # TODO bottleneck is more commonly inside layer 42 | bottleneck_channels = conf.getHP('inception_bottleneck_channels') 43 | self.__bottleneck = nn.Conv1d(in_channels, bottleneck_channels, 1, bias=False) 44 | 45 | inception_kernel_sizes = np.sort(np.array(conf.getHP('inception_kernel_sizes'))) 46 | num_layers = len(inception_kernel_sizes) 47 | 48 | layer_chennels = np.array([out_channels // num_layers] * num_layers) 49 | 50 | for i in range(out_channels % num_layers): 51 | layer_chennels[i] += 1 52 | 53 | assert np.sum(layer_chennels) == out_channels 54 | assert inception_kernel_sizes[0] == 1 55 | 56 | self.__identity_link = nn.Sequential(nn.MaxPool1d(3, stride=1, padding=1), 57 | nn.Conv1d(in_channels, layer_chennels[0], 1, bias=False)) 58 | 59 | for i, (kernel_size, layer_out_channels) in enumerate(zip(inception_kernel_sizes[1: ], layer_chennels[1: ])): 60 | self.add_module('InceptionLayer{:d}'.format(i), _InceptionLayer(kernel_size, bottleneck_channels, layer_out_channels)) 61 | 62 | self.__finalize = nn.Sequential(nn.BatchNorm1d(out_channels), 63 | conf.getActivation(conf.getHP('activation_conv'))) 64 | 65 | 66 | def forward(self, input: Tensor) -> Tensor: 67 | latent = [self.__identity_link(input)] 68 | 69 | bottleneck = self.__bottleneck(input) 70 | 71 | for label, layer in self.items(): 72 | if label.startswith('InceptionLayer'): 73 | latent.append(layer(bottleneck)) 74 | 75 | latent = torch.cat(latent, 1) 76 | 77 | return self.__finalize(latent) 78 | 79 | 80 | 81 | class _InceptionNet(nn.Module): 82 | def __init__(self, conf: Configuration, to_encode: bool): 83 | super(_InceptionNet, self).__init__() 84 | 85 | num_chennels = conf.getHP('num_en_channels') if to_encode else conf.getHP('num_de_channels') 86 | num_blocks = conf.getHP('num_inceptionen_blocks') if to_encode else conf.getHP('num_inceptionde_blocks') 87 | assert num_blocks >= 1 88 | 89 | self.__model = nn.Sequential(OrderedDict([ 90 | ('InceptionBlock0', _InceptionBlock(conf, 1, num_chennels, to_encode)) 91 | ])) 92 | 93 | for depth in range(1, num_blocks): 94 | block = _InceptionBlock(conf, num_chennels, num_chennels, to_encode) 95 | self.__model.add_module('InceptionBlock{:d}'.format(depth), block) 96 | 97 | 98 | def forward(self, input: Tensor) -> Tensor: 99 | return self.__model(input) 100 | 101 | 102 | 103 | class InceptionEncoder(nn.Module): 104 | def __init__(self, conf: Configuration): 105 | super(InceptionEncoder, self).__init__() 106 | 107 | dim_embedding = conf.getHP('dim_embedding') 108 | num_channels = conf.getHP('num_en_channels') 109 | dim_latent = conf.getHP('dim_en_latent') 110 | 111 | self.__model = nn.Sequential(_InceptionNet(conf, to_encode=True), 112 | nn.AdaptiveMaxPool1d(1), 113 | Squeeze(), 114 | 115 | nn.Linear(num_channels, dim_latent), 116 | conf.getActivation(conf.getHP('activation_linear')), 117 | 118 | nn.Linear(dim_latent, dim_embedding, bias=False), 119 | nn.LayerNorm(dim_embedding, elementwise_affine=False) if conf.getHP('encoder_normalize_embedding') else nn.Identity()) 120 | 121 | self.__model.to(conf.getHP('device')) 122 | 123 | 124 | def forward(self, input: Tensor) -> Tensor: 125 | return self.__model(input) 126 | 127 | 128 | 129 | class InceptionDecoder(nn.Module): 130 | def __init__(self, conf: Configuration): 131 | super(InceptionDecoder, self).__init__() 132 | 133 | dim_series = conf.getHP('dim_series') 134 | dim_embedding = conf.getHP('dim_embedding') 135 | num_channels = conf.getHP('num_de_channels') 136 | dim_latent = conf.getHP('dim_de_latent') 137 | 138 | self.__model = nn.Sequential(Reshape([-1, 1, dim_embedding]), 139 | nn.Linear(dim_embedding, dim_series), 140 | conf.getActivation(conf.getHP('activation_linear')), 141 | 142 | _InceptionNet(conf, to_encode=False), 143 | nn.AdaptiveMaxPool1d(1), 144 | Reshape([-1, 1, num_channels]), 145 | 146 | nn.Linear(num_channels, dim_latent), 147 | conf.getActivation(conf.getHP('activation_linear')), 148 | 149 | nn.Linear(dim_latent, dim_series, bias=False), 150 | nn.LayerNorm(dim_series, elementwise_affine=False) if conf.getHP('decoder_normalize_reconstruction') else nn.Identity()) 151 | 152 | self.__model.to(conf.getHP('device')) 153 | 154 | 155 | def forward(self, input: Tensor) -> Tensor: 156 | return self.__model(input) 157 | -------------------------------------------------------------------------------- /model/RNNAE.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | from torch import nn, Tensor 4 | 5 | from util.conf import Configuration 6 | from model.commons import Squeeze, Reshape 7 | 8 | 9 | class RNNEncoder(nn.Module): 10 | def __init__(self, conf: Configuration): 11 | super(RNNEncoder, self).__init__() 12 | 13 | model_type: str = conf.getHP('encoder') 14 | 15 | if model_type == 'gru': 16 | model_class = nn.GRU 17 | elif model_type == 'lstm': 18 | model_class = nn.LSTM 19 | else: 20 | raise ValueError('encoder {:s} is not supported'.format(model_type)) 21 | 22 | dim_embedding = conf.getHP('dim_embedding') 23 | dim_latent = conf.getHP('dim_rnnen_latent') 24 | bidirectional = conf.getHP('if_rnnen_bidirectional') 25 | dropout = conf.getHP('rnnen_dropout') 26 | device = conf.getHP('device') 27 | 28 | self.__transform = model_class( 29 | input_size = 1, 30 | hidden_size = dim_latent, 31 | num_layers = conf.getHP('num_rnnen_layers'), 32 | batch_first = True, 33 | bidirectional = bidirectional, 34 | dropout = dropout 35 | ) 36 | 37 | self.__map = model_class( 38 | input_size = dim_latent * (2 if bidirectional else 1), 39 | hidden_size = dim_embedding, 40 | num_layers = 1, 41 | batch_first = True 42 | ) 43 | 44 | self.__normalize = nn.LayerNorm(dim_embedding, elementwise_affine=False) if conf.getHP('encoder_normalize_embedding') else nn.Identity() 45 | 46 | self.__transform.to(device) 47 | self.__map.to(device) 48 | self.__normalize.to(device) 49 | 50 | 51 | def forward(self, input: Tensor) -> Tensor: 52 | output, _ = self.__transform(input) 53 | output, _ = self.__map(output) 54 | 55 | return self.__normalize(output[:, -1]) 56 | 57 | 58 | 59 | class RNNDecoder(nn.Module): 60 | def __init__(self, conf: Configuration): 61 | super(RNNDecoder, self).__init__() 62 | 63 | model_type: str = conf.getHP('decoder') 64 | 65 | if model_type == 'gru': 66 | model_class = nn.GRU 67 | elif model_type == 'lstm': 68 | model_class = nn.LSTM 69 | else: 70 | raise ValueError('decoder {:s} is not supported'.format(model_type)) 71 | 72 | dim_embedding = conf.getHP('dim_embedding') 73 | dim_latent = conf.getHP('dim_rnnde_latent') 74 | bidirectional = conf.getHP('if_rnnde_bidirectional') 75 | dropout = conf.getHP('rnnde_dropout') 76 | device = conf.getHP('device') 77 | 78 | self.dim_series = conf.getHP('dim_series') 79 | 80 | self.__transform = model_class( 81 | input_size = dim_embedding, 82 | hidden_size = dim_latent, 83 | num_layers = conf.getHP('num_rnnde_layers'), 84 | batch_first = True, 85 | bidirectional = bidirectional, 86 | dropout = dropout 87 | ) 88 | 89 | self.__map = nn.Sequential(nn.Linear(dim_latent * (2 if bidirectional else 1), dim_latent), 90 | conf.getActivation(conf.getHP('activation_linear')), 91 | 92 | nn.Linear(dim_latent, 1, bias=False)) 93 | 94 | self.__normalize = nn.LayerNorm([self.dim_series, 1], elementwise_affine=False) if conf.getHP('decoder_normalize_reconstruction') else nn.Identity() 95 | 96 | self.__transform.to(device) 97 | self.__map.to(device) 98 | self.__normalize.to(device) 99 | 100 | 101 | def forward(self, input: Tensor) -> Tensor: 102 | input = input.unsqueeze(1).repeat((1, self.dim_series, 1)) 103 | 104 | output, _ = self.__transform(input) 105 | output = self.__map(output) 106 | 107 | return self.__normalize(output.flip(1)) 108 | -------------------------------------------------------------------------------- /model/ResidualAE.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | from torch import nn, Tensor 4 | 5 | from util.conf import Configuration 6 | from model.commons import Squeeze, Reshape 7 | 8 | 9 | class _OriginalResBlock(nn.Module): 10 | def __init__(self, conf: Configuration, in_channels, out_channels, dilation): 11 | super(_OriginalResBlock, self).__init__() 12 | 13 | dim_series = conf.getHP('dim_series') 14 | kernel_size = conf.getHP('size_kernel') 15 | padding = int(kernel_size / 2) * dilation 16 | activation_name = conf.getHP('activation_conv') 17 | bias = conf.getHP('layernorm_type') == 'none' 18 | 19 | self.__residual_link = nn.Sequential(conf.getWeightNorm(nn.Conv1d(in_channels, out_channels, kernel_size, padding=padding, dilation=dilation, bias=bias)), 20 | conf.getLayerNorm(dim_series), 21 | conf.getActivation(activation_name), 22 | 23 | conf.getWeightNorm(nn.Conv1d(out_channels, out_channels, kernel_size, padding=padding, dilation=dilation, bias=bias)), 24 | conf.getLayerNorm(dim_series)) 25 | 26 | if in_channels != out_channels: 27 | self.__identity_link = conf.getWeightNorm(nn.Conv1d(in_channels, out_channels, 1, bias=bias)) 28 | else: 29 | self.__identity_link = nn.Identity() 30 | 31 | self.__after_addition = conf.getActivation(activation_name) 32 | 33 | 34 | def forward(self, input: Tensor) -> Tensor: 35 | residual = self.__residual_link(input) 36 | identity = self.__identity_link(input) 37 | 38 | return self.__after_addition(identity + residual) 39 | 40 | 41 | 42 | class _PreActivatedResBlock(nn.Module): 43 | def __init__(self, conf: Configuration, in_channels, out_channels, dilation, first = False, last = False): 44 | super(_PreActivatedResBlock, self).__init__() 45 | 46 | dim_series = conf.getHP('dim_series') 47 | kernel_size = conf.getHP('size_kernel') 48 | padding = int(kernel_size / 2) * dilation 49 | activation_name = conf.getHP('activation_conv') 50 | bias = conf.getHP('layernorm_type') == 'none' or not conf.getHP('layernorm_elementwise_affine') 51 | 52 | if first: 53 | self.__first_block = conf.getWeightNorm(nn.Conv1d(in_channels, out_channels, kernel_size, padding=padding, dilation=dilation, bias=bias)) 54 | in_channels = out_channels 55 | else: 56 | self.__first_block = nn.Identity() 57 | 58 | self.__residual_link = nn.Sequential(conf.getLayerNorm(dim_series), 59 | conf.getActivation(activation_name), 60 | conf.getWeightNorm(nn.Conv1d(in_channels, out_channels, kernel_size, padding=padding, dilation=dilation, bias=bias)), 61 | 62 | conf.getLayerNorm(dim_series), 63 | conf.getActivation(activation_name), 64 | conf.getWeightNorm(nn.Conv1d(out_channels, out_channels, kernel_size, padding=padding, dilation=dilation, bias=bias))) 65 | 66 | if in_channels != out_channels: 67 | self.__identity_link = conf.getWeightNorm(nn.Conv1d(in_channels, out_channels, 1, bias=bias)) 68 | else: 69 | self.__identity_link = nn.Identity() 70 | 71 | if last: 72 | self.__after_addition = nn.Sequential(conf.getLayerNorm(dim_series), 73 | conf.getActivation(activation_name)) 74 | else: 75 | self.__after_addition = nn.Identity() 76 | 77 | 78 | def forward(self, input: Tensor) -> Tensor: 79 | input = self.__first_block(input) 80 | 81 | residual = self.__residual_link(input) 82 | identity = self.__identity_link(input) 83 | 84 | return self.__after_addition(identity + residual) 85 | 86 | 87 | 88 | class _ResNet(nn.Module): 89 | def __init__(self, conf: Configuration, to_encode: bool): 90 | super(_ResNet, self).__init__() 91 | 92 | num_resblock = conf.getHP('num_en_resblock') if to_encode else conf.getHP('num_de_resblock') 93 | 94 | if conf.getHP('dilation_type') == 'exponential': 95 | assert num_resblock > 1 and 2 ** (num_resblock + 1) <= conf.getHP('dim_series') + 1 96 | 97 | inner_channels = conf.getHP('num_en_channels') if to_encode else conf.getHP('num_de_channels') 98 | out_channels = conf.getHP('dim_en_latent') if to_encode else conf.getHP('dim_de_latent') 99 | 100 | if conf.getHP('resblock_pre_activation'): 101 | layers = [_PreActivatedResBlock(conf, 1, inner_channels, conf.getDilatoin(1, to_encode), first=True)] 102 | layers += [_PreActivatedResBlock(conf, inner_channels, inner_channels, conf.getDilatoin(depth, to_encode)) for depth in range(2, num_resblock)] 103 | layers += [_PreActivatedResBlock(conf, inner_channels, out_channels, conf.getDilatoin(num_resblock, to_encode), last=True)] 104 | else: 105 | layers = [_OriginalResBlock(conf, 1, inner_channels, conf.getDilatoin(1, to_encode))] 106 | layers += [_OriginalResBlock(conf, inner_channels, inner_channels, conf.getDilatoin(depth, to_encode)) for depth in range(2, num_resblock)] 107 | layers += [_OriginalResBlock(conf, inner_channels, out_channels, conf.getDilatoin(num_resblock, to_encode))] 108 | 109 | self.__model = nn.Sequential(*layers) 110 | 111 | 112 | def forward(self, input: Tensor) -> Tensor: 113 | return self.__model(input) 114 | 115 | 116 | 117 | class ResidualEncoder(nn.Module): 118 | def __init__(self, conf: Configuration): 119 | super(ResidualEncoder, self).__init__() 120 | 121 | dim_embedding = conf.getHP('dim_embedding') 122 | num_channels = conf.getHP('num_en_channels') 123 | dim_latent = conf.getHP('dim_en_latent') 124 | 125 | self.__model = nn.Sequential(_ResNet(conf, to_encode=True), 126 | nn.AdaptiveMaxPool1d(1), 127 | Squeeze(), 128 | 129 | nn.Linear(num_channels, dim_latent), 130 | conf.getActivation(conf.getHP('activation_linear')), 131 | 132 | nn.Linear(dim_latent, dim_embedding, bias=False), 133 | nn.LayerNorm(dim_embedding, elementwise_affine=False) if conf.getHP('encoder_normalize_embedding') else nn.Identity()) 134 | 135 | self.__model.to(conf.getHP('device')) 136 | 137 | 138 | def forward(self, input: Tensor) -> Tensor: 139 | return self.__model(input) 140 | 141 | 142 | 143 | class ResidualDecoder(nn.Module): 144 | def __init__(self, conf: Configuration): 145 | super(ResidualDecoder, self).__init__() 146 | 147 | dim_series = conf.getHP('dim_series') 148 | dim_embedding = conf.getHP('dim_embedding') 149 | num_channels = conf.getHP('num_de_channels') 150 | dim_latent = conf.getHP('dim_de_latent') 151 | 152 | self.__model = nn.Sequential(Reshape([-1, 1, dim_embedding]), 153 | nn.Linear(dim_embedding, dim_series), 154 | conf.getActivation(conf.getHP('activation_linear')), 155 | 156 | _ResNet(conf, to_encode=False), 157 | nn.AdaptiveMaxPool1d(1), 158 | Reshape([-1, 1, num_channels]), 159 | 160 | nn.Linear(num_channels, dim_latent), 161 | conf.getActivation(conf.getHP('activation_linear')), 162 | 163 | nn.Linear(dim_latent, dim_series, bias=False), 164 | nn.LayerNorm(dim_series, elementwise_affine=False) if conf.getHP('decoder_normalize_reconstruction') else nn.Identity()) 165 | 166 | self.__model.to(conf.getHP('device')) 167 | 168 | 169 | def forward(self, input: Tensor) -> Tensor: 170 | return self.__model(input) 171 | 172 | 173 | 174 | class SingleResidualDecoder(nn.Module): 175 | def __init__(self, conf: Configuration): 176 | super(SingleResidualDecoder, self).__init__() 177 | 178 | dim_series = conf.getHP('dim_series') 179 | assert dim_series == 256 180 | 181 | dim_embedding = conf.getHP('dim_embedding') 182 | assert dim_embedding == 16 183 | 184 | in_channels = 1 185 | assert in_channels == 1 186 | 187 | inner_channels = conf.getHP('num_de_channels') 188 | intermediate_channels = int(inner_channels / 2) 189 | kernel_size = conf.getHP('size_kernel') 190 | padding = int(kernel_size / 2) 191 | 192 | conv_activation_name = conf.getHP('activation_conv') 193 | linear_activation_name = conf.getHP('activation_linear') 194 | 195 | bias = conf.getHP('layernorm_type') == 'none' 196 | 197 | self.__reshape = Reshape([-1, in_channels, dim_embedding]) 198 | 199 | if conf.getHP('resblock_pre_activation'): 200 | self.__residual_link = nn.Sequential(conf.getLayerNorm(16), 201 | conf.getActivation(conv_activation_name), 202 | conf.getWeightNorm(nn.ConvTranspose1d(in_channels, intermediate_channels, kernel_size=kernel_size, stride=2, padding=padding, output_padding=1, bias=bias)), 203 | 204 | conf.getLayerNorm(32), 205 | conf.getActivation(conv_activation_name), 206 | conf.getWeightNorm(nn.ConvTranspose1d(intermediate_channels, inner_channels, kernel_size=kernel_size, stride=2, padding=padding, output_padding=1, bias=bias)), 207 | 208 | conf.getLayerNorm(64), 209 | conf.getActivation(conv_activation_name), 210 | conf.getWeightNorm(nn.ConvTranspose1d(inner_channels, intermediate_channels, kernel_size=kernel_size, stride=2, padding=padding, output_padding=1, bias=bias)), 211 | 212 | conf.getLayerNorm(128), 213 | conf.getActivation(conv_activation_name), 214 | conf.getWeightNorm(nn.ConvTranspose1d(intermediate_channels, in_channels, kernel_size=kernel_size, stride=2, padding=padding, output_padding=1, bias=bias))) 215 | 216 | self.__after_addition = nn.Sequential(conf.getLayerNorm(256), 217 | conf.getActivation(conv_activation_name)) 218 | else: 219 | self.__residual_link = nn.Sequential(conf.getWeightNorm(nn.ConvTranspose1d(in_channels, intermediate_channels, kernel_size=kernel_size, stride=2, padding=padding, output_padding=1, bias=bias)), 220 | conf.getLayerNorm(32), 221 | conf.getActivation(conv_activation_name), 222 | 223 | conf.getWeightNorm(nn.ConvTranspose1d(intermediate_channels, inner_channels, kernel_size=kernel_size, stride=2, padding=padding, output_padding=1, bias=bias)), 224 | conf.getLayerNorm(64), 225 | conf.getActivation(conv_activation_name), 226 | 227 | conf.getWeightNorm(nn.ConvTranspose1d(inner_channels, intermediate_channels, kernel_size=kernel_size, stride=2, padding=padding, output_padding=1, bias=bias)), 228 | conf.getLayerNorm(128), 229 | conf.getActivation(conv_activation_name), 230 | 231 | conf.getWeightNorm(nn.ConvTranspose1d(intermediate_channels, in_channels, kernel_size=kernel_size, stride=2, padding=padding, output_padding=1, bias=bias)), 232 | conf.getLayerNorm(256)) 233 | 234 | self.__after_addition = conf.getActivation(conv_activation_name) 235 | 236 | self.__identity_link = nn.Linear(dim_embedding, dim_series) 237 | 238 | self.__linear = nn.Sequential(nn.Linear(dim_series, dim_series), 239 | conf.getActivation(linear_activation_name), 240 | 241 | nn.Linear(dim_series, dim_series, bias=False)) 242 | 243 | device = conf.getHP('device') 244 | 245 | self.__reshape.to(device) 246 | self.__residual_link.to(device) 247 | self.__identity_link.to(device) 248 | self.__after_addition.to(device) 249 | self.__linear.to(device) 250 | 251 | 252 | def forward(self, input: Tensor) -> Tensor: 253 | input = self.__reshape(input) 254 | 255 | residual = self.__residual_link(input) 256 | identity = self.__identity_link(input) 257 | output = self.__after_addition(identity + residual) 258 | 259 | return self.__linear(output) 260 | -------------------------------------------------------------------------------- /model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qtwang/SEAnet/0cd38ff770bae8c4fe4f8d5f227a645b8c4c0ec9/model/__init__.py -------------------------------------------------------------------------------- /model/activation.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | import numpy as np 4 | from torch import nn, tanh, Tensor 5 | 6 | EPSILON = 1e-7 7 | 8 | 9 | def EQUAL(a:float, b:float): 10 | return (a - b <= EPSILON) and (b - a <= EPSILON) 11 | 12 | 13 | class TanhAdjusted(nn.Module): 14 | def __init__(self, outer:float=1., inner:float=1., specified=False): 15 | super(TanhAdjusted, self).__init__() 16 | 17 | self.a = outer 18 | self.b = inner 19 | 20 | if not specified: 21 | if EQUAL(self.a, 1.) and not EQUAL(self.b, 1.): 22 | self.a = 1. / np.tanh(self.b) 23 | elif not EQUAL(self.a, 1.) and EQUAL(self.b, 1.): 24 | self.b = np.log((self.a + 1.) / (self.a - 1.)) / 2. 25 | 26 | 27 | def forward(self, input: Tensor) -> Tensor: 28 | return self.a * tanh(self.b * input) 29 | 30 | 31 | 32 | class LeCunTanh(nn.Module): 33 | def __init__(self): 34 | super(LeCunTanh, self).__init__() 35 | 36 | self.adjustedTanh = TanhAdjusted(outer=1.7159, inner=2/3, specified=True) 37 | 38 | 39 | def forward(self, input: Tensor) -> Tensor: 40 | return self.adjustedTanh(input) 41 | 42 | 43 | 44 | if __name__ == "__main__": 45 | print('Welcome to where the activations got defined!') 46 | -------------------------------------------------------------------------------- /model/builder.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | from torch import nn, Tensor 4 | 5 | from util.conf import Configuration 6 | from model.ResidualAE import ResidualEncoder, ResidualDecoder, SingleResidualDecoder 7 | from model.DenseAE import DenseEncoder, DenseDecoder 8 | from model.RNNAE import RNNEncoder, RNNDecoder 9 | from model.FDJAE import FDJEncoder, FDJDecoder 10 | from model.InceptionAE import InceptionEncoder, InceptionDecoder 11 | 12 | 13 | class AEBuilder(nn.Module): 14 | def __init__(self, conf: Configuration): 15 | super(AEBuilder, self).__init__() 16 | 17 | encoder_name = conf.getHP('encoder') 18 | 19 | if encoder_name == 'residual': 20 | self.__encoder = ResidualEncoder(conf) 21 | elif encoder_name == 'dense': 22 | self.__encoder = DenseEncoder(conf) 23 | elif encoder_name == 'fdj': 24 | self.__encoder = FDJEncoder(conf) 25 | elif encoder_name == 'inception': 26 | self.__encoder = InceptionEncoder(conf) 27 | elif encoder_name == 'gru' or encoder_name == 'lstm': 28 | self.__encoder = RNNEncoder(conf) 29 | else: 30 | raise ValueError('encoder {:s} isn\'t supported yet'.format(encoder_name)) 31 | 32 | decoder_name = conf.getHP('decoder') 33 | 34 | if decoder_name == 'residual': 35 | self.__decoder = ResidualDecoder(conf) 36 | elif decoder_name == 'singleresidual': 37 | self.__decoder = SingleResidualDecoder(conf) 38 | elif decoder_name == 'dense': 39 | self.__decoder = DenseDecoder(conf) 40 | elif decoder_name == 'fdj': 41 | self.__decoder = FDJDecoder(conf) 42 | elif decoder_name == 'inception': 43 | self.__decoder = InceptionDecoder(conf) 44 | elif decoder_name == 'gru' or decoder_name == 'lstm': 45 | self.__decoder = RNNDecoder(conf) 46 | elif decoder_name == 'none': 47 | self.__decoder = None 48 | else: 49 | raise ValueError('decoder {:s} isn\'t supported yet'.format(decoder_name)) 50 | 51 | 52 | def encode(self, input: Tensor) -> Tensor: 53 | return self.__encoder(input) 54 | 55 | 56 | def decode(self, input: Tensor) -> Tensor: 57 | if self.__decoder is None: 58 | raise ValueError('No decoder') 59 | 60 | return self.__decoder(input) 61 | 62 | 63 | # explicit model.encode/decode is preferred as decoder might not exist 64 | # forward is mostly for examining no. parameters 65 | def forward(self, input: Tensor) -> Tensor: 66 | embedding = self.encode(input) 67 | 68 | if self.__decoder is None: 69 | return embedding 70 | 71 | return self.decode(embedding) 72 | -------------------------------------------------------------------------------- /model/commons.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | from torch import nn, Tensor 4 | 5 | 6 | class Reshape(nn.Module): 7 | def __init__(self, shape): 8 | super(Reshape, self).__init__() 9 | self.shape = shape 10 | 11 | def forward(self, input: Tensor) -> Tensor: 12 | return input.view(self.shape) 13 | 14 | 15 | 16 | class Squeeze(nn.Module): 17 | def __init__(self, dim = None): 18 | super(Squeeze, self).__init__() 19 | self.dim = dim 20 | 21 | def forward(self, input: Tensor) -> Tensor: 22 | if self.dim is None: 23 | return input.squeeze() 24 | else: 25 | return input.squeeze(dim=self.dim) 26 | 27 | -------------------------------------------------------------------------------- /model/initialization.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | from __future__ import print_function 4 | 5 | import numpy as np 6 | import torch 7 | import torch.nn.init 8 | import torch.nn as nn 9 | 10 | 11 | gg = {} 12 | gg['hook_position'] = 0 13 | gg['total_fc_conv_layers'] = 0 14 | gg['done_counter'] = -1 15 | gg['hook'] = None 16 | gg['act_dict'] = {} 17 | gg['counter_to_apply_correction'] = 0 18 | gg['correction_needed'] = False 19 | gg['current_coef'] = 1.0 20 | 21 | 22 | # Orthonorm init code is taked from Lasagne 23 | # https://github.com/Lasagne/Lasagne/blob/master/lasagne/init.py 24 | def svd_orthonormal(w): 25 | shape = w.shape 26 | if len(shape) < 2: 27 | raise RuntimeError("Only shapes of length 2 or more are supported.") 28 | flat_shape = (shape[0], np.prod(shape[1:])) 29 | a = np.random.normal(0.0, 1.0, flat_shape)#w; 30 | u, _, v = np.linalg.svd(a, full_matrices=False) 31 | q = u if u.shape == flat_shape else v 32 | print (shape, flat_shape) 33 | q = q.reshape(shape) 34 | return q.astype(np.float32) 35 | 36 | def store_activations(self, input, output): 37 | gg['act_dict'] = output.data.cpu().numpy(); 38 | #print('act shape = ', gg['act_dict'].shape) 39 | return 40 | 41 | 42 | def add_current_hook(m): 43 | if gg['hook'] is not None: 44 | return 45 | if (isinstance(m, nn.Conv2d)) or (isinstance(m, nn.Linear)): 46 | #print 'trying to hook to', m, gg['hook_position'], gg['done_counter'] 47 | if gg['hook_position'] > gg['done_counter']: 48 | gg['hook'] = m.register_forward_hook(store_activations) 49 | #print ' hooking layer = ', gg['hook_position'], m 50 | else: 51 | #print m, 'already done, skipping' 52 | gg['hook_position'] += 1 53 | return 54 | 55 | def count_conv_fc_layers(m): 56 | if (isinstance(m, nn.Conv2d)) or (isinstance(m, nn.Linear)): 57 | gg['total_fc_conv_layers'] +=1 58 | return 59 | 60 | def remove_hooks(hooks): 61 | for h in hooks: 62 | h.remove() 63 | return 64 | def orthogonal_weights_init(m): 65 | if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear): 66 | if hasattr(m, 'weight'): 67 | w_ortho = svd_orthonormal(m.weight.data.cpu().numpy()) 68 | m.weight.data = torch.from_numpy(w_ortho) 69 | try: 70 | nn.init.constant(m.bias, 0) 71 | except: 72 | pass 73 | else: 74 | #nn.init.orthogonal(m.weight) 75 | w_ortho = svd_orthonormal(m.weight.data.cpu().numpy()) 76 | #print w_ortho 77 | #m.weight.data.copy_(torch.from_numpy(w_ortho)) 78 | m.weight.data = torch.from_numpy(w_ortho) 79 | try: 80 | nn.init.constant(m.bias, 0) 81 | except: 82 | pass 83 | return 84 | 85 | def apply_weights_correction(m): 86 | if gg['hook'] is None: 87 | return 88 | if not gg['correction_needed']: 89 | return 90 | if (isinstance(m, nn.Conv2d)) or (isinstance(m, nn.Linear)): 91 | if gg['counter_to_apply_correction'] < gg['hook_position']: 92 | gg['counter_to_apply_correction'] += 1 93 | else: 94 | if hasattr(m, 'weight'): 95 | m.weight.data *= float(gg['current_coef']) 96 | gg['correction_needed'] = False 97 | if hasattr(m, 'bias'): 98 | if m.bias is not None: 99 | m.bias.data += float(gg['current_bias']) 100 | return 101 | return 102 | 103 | 104 | # cite: ICLR16 All you need is a good init 105 | # cite: https://github.com/ducha-aiki/LSUV-pytorch/tree/0c45eb2a9bd8978f13572c328f2f0d5d11939c99 106 | def LSUVinit(model,data, needed_std = 1.0, std_tol = 0.1, max_attempts = 10, do_orthonorm = True,needed_mean = 0., cuda = False, verbose = True): 107 | cuda = data.is_cuda 108 | gg['total_fc_conv_layers']=0 109 | gg['done_counter']= 0 110 | gg['hook_position'] = 0 111 | gg['hook'] = None 112 | model.eval(); 113 | if cuda: 114 | model = model.cuda() 115 | data = data.cuda() 116 | else: 117 | model = model.cpu() 118 | data = data.cpu() 119 | if verbose: print( 'Starting LSUV') 120 | model.apply(count_conv_fc_layers) 121 | if verbose: print ('Total layers to process:', gg['total_fc_conv_layers']) 122 | with torch.no_grad(): 123 | if do_orthonorm: 124 | model.apply(orthogonal_weights_init) 125 | if verbose: print ('Orthonorm done') 126 | if cuda: 127 | model = model.cuda() 128 | for layer_idx in range(gg['total_fc_conv_layers']): 129 | if verbose: print (layer_idx) 130 | model.apply(add_current_hook) 131 | out = model(data) 132 | current_std = gg['act_dict'].std() 133 | current_mean = gg['act_dict'].mean() 134 | if verbose: print ('std at layer ',layer_idx, ' = ', current_std) 135 | #print gg['act_dict'].shape 136 | attempts = 0 137 | while (np.abs(current_std - needed_std) > std_tol): 138 | gg['current_coef'] = needed_std / (current_std + 1e-8); 139 | gg['current_bias'] = needed_mean - current_mean * gg['current_coef']; 140 | gg['correction_needed'] = True 141 | model.apply(apply_weights_correction) 142 | if cuda: 143 | model = model.cuda() 144 | out = model(data) 145 | current_std = gg['act_dict'].std() 146 | current_mean = gg['act_dict'].mean() 147 | if verbose: print ('std at layer ',layer_idx, ' = ', current_std, 'mean = ', current_mean) 148 | attempts+=1 149 | if attempts > max_attempts: 150 | if verbose: print ('Cannot converge in ', max_attempts, 'iterations') 151 | break 152 | if gg['hook'] is not None: 153 | gg['hook'].remove() 154 | gg['done_counter']+=1 155 | gg['counter_to_apply_correction'] = 0 156 | gg['hook_position'] = 0 157 | gg['hook'] = None 158 | if verbose: print ('finish at layer',layer_idx ) 159 | if verbose: print ('LSUV init done!') 160 | if not cuda: 161 | model = model.cpu() 162 | return model 163 | -------------------------------------------------------------------------------- /model/loss.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | from numpy import sqrt 4 | from torch import mean, squeeze 5 | from torch.nn import Module, PairwiseDistance 6 | 7 | 8 | # TODO squeeze is not time-comusing. While it's still good to remove it 9 | class ScaledL2Trans(Module): 10 | def __init__(self, original_dimension:int = 256, embedding_dimension: int = 16, to_scale: bool = False): 11 | super(ScaledL2Trans, self).__init__() 12 | 13 | self.__l2 = PairwiseDistance(p=2).cuda() 14 | self.__l1 = PairwiseDistance(p=1).cuda() 15 | 16 | if to_scale: 17 | self.__scale_factor_original = sqrt(original_dimension) 18 | self.__scale_factor_embedding = sqrt(embedding_dimension) 19 | else: 20 | self.__scale_factor_original = 1 21 | self.__scale_factor_embedding = 1 22 | 23 | 24 | 25 | def forward(self, database, query, db_embedding, query_embedding): 26 | original_l2 = self.__l2(squeeze(database), squeeze(query)) / self.__scale_factor_original 27 | embedding_l2 = self.__l2(squeeze(db_embedding), squeeze(query_embedding)) / self.__scale_factor_embedding 28 | 29 | return self.__l1(original_l2.view([1, -1]), embedding_l2.view([1, -1]))[0] / database.shape[0] 30 | 31 | 32 | 33 | class ScaledL2Recons(Module): 34 | def __init__(self, original_dimension: int = 256, to_scale: bool = False): 35 | super(ScaledL2Recons, self).__init__() 36 | 37 | self.__l2 = PairwiseDistance(p=2).cuda() 38 | 39 | if to_scale: 40 | self.__scale_factor = sqrt(original_dimension) 41 | else: 42 | self.__scale_factor = 1 43 | 44 | def forward(self, database, reconstructed): 45 | return mean(self.__l2(squeeze(database), squeeze(reconstructed))) / self.__scale_factor 46 | -------------------------------------------------------------------------------- /model/normalization.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | import numbers 4 | from typing import Union, List 5 | 6 | import torch 7 | from torch import Tensor, Size 8 | from torch.nn import Module, init 9 | from torch.nn.parameter import Parameter 10 | from torch.autograd import Variable 11 | from torch.nn.functional import normalize 12 | 13 | 14 | # cite: https://github.com/lancopku/AdaNorms 15 | # cite: NeurIPS19 Understanding and Improving Layer Normalization 16 | class AdaNorm(Module): 17 | __constants__ = ['normalized_shape', 'eps', 'elementwise_affine', 'k', 'scale'] 18 | 19 | normalized_shape: Union[int, List[int], torch.Size] 20 | eps: float 21 | elementwise_affine: bool 22 | k: float 23 | scale: float 24 | 25 | def __init__(self, normalized_shape: Union[int, List[int], torch.Size], k: float = 1 / 10, scale: float = 2., eps: float = 1e-5, elementwise_affine: bool = True) -> None: 26 | super(AdaNorm, self).__init__() 27 | 28 | self.k = k 29 | self.scale = scale 30 | self.eps = eps 31 | self.elementwise_affine = elementwise_affine 32 | 33 | if isinstance(normalized_shape, numbers.Integral): 34 | normalized_shape = (normalized_shape,) 35 | else: 36 | raise ValueError('Only last layer for AdaNorm currently') 37 | self.normalized_shape = tuple(normalized_shape) 38 | 39 | if self.elementwise_affine: 40 | self.weight = Parameter(torch.Tensor(*normalized_shape)) 41 | self.bias = Parameter(torch.Tensor(*normalized_shape)) 42 | else: 43 | self.register_parameter('weight', None) 44 | self.register_parameter('bias', None) 45 | self.reset_parameters() 46 | 47 | 48 | def reset_parameters(self) -> None: 49 | if self.elementwise_affine: 50 | init.ones_(self.weight) 51 | init.zeros_(self.bias) 52 | 53 | 54 | def forward(self, input: Tensor) -> Tensor: 55 | mean = input.mean(-1, keepdim=True) 56 | std = input.std(-1, keepdim=True) 57 | 58 | input = input - mean 59 | mean = input.mean(-1, keepdim=True) 60 | 61 | graNorm = (self.k * (input - mean) / (std + self.eps)).detach() 62 | input_norm = (input - input * graNorm) / (std + self.eps) 63 | 64 | return self.scale * input_norm 65 | 66 | 67 | def extra_repr(self) -> Tensor: 68 | return '{normalized_shape}, eps={eps}, elementwise_affine={elementwise_affine}, k={k}, scale={scale}'.format(**self.__dict__) 69 | 70 | 71 | 72 | # regulize (preserve) orthogonality among output features/channels 73 | # under Spectral Restricted Isometry Property (of orthogonal matrix) 74 | # extra hyper-parameters is added and should be searched: coefficient (weight) of SRIPTerm 75 | # recommended by the authors: 1e-1(epoch 0) --> 1e-3(20) --> 1e-4(50) --> 1e-6(70) --> 0(120) of 200 epochs totally 76 | # while at the same time changing coefficient of weight decay: 1e-8(0) --> 1e-4(20) 77 | # cite: https://github.com/VITA-Group/Orthogonality-in-CNNs/blob/master/Imagenet/resnet/train_n.py 78 | # cite: NeurIPS18 Can We Gain More from Orthogonality Regularizations in Training Deep CNNs? 79 | def getSRIPTerm(model: Module, device='cpu'): 80 | term = None 81 | 82 | for W in model.parameters(): 83 | if W.ndimension() < 2: 84 | continue 85 | else: 86 | # for convolutional: 87 | # W.shape = [OUTPUT_CHANNELS, INPUT_CHANNELS, KERNEL_SIZE] 88 | # rows = OUTPUT_CHANNELS, cols = INPUT_CHANNELS * KERNEL_SIZE 89 | 90 | # for linner: 91 | # W.shape = [OUTPUT_FEATURES, INTPUT_FEATURES] 92 | # rows = OUTPUT_FEATURES, cols = INTPUT_FEATURES 93 | 94 | cols = W[0].numel() 95 | rows = W.shape[0] 96 | 97 | w1 = W.view(-1, cols) 98 | wt = torch.transpose(w1, 0, 1) 99 | m = torch.matmul(wt, w1) 100 | 101 | ident = Variable(torch.eye(cols,cols)) 102 | ident = ident.to(device) 103 | 104 | w_tmp = (m - ident) 105 | height = w_tmp.size(0) 106 | 107 | # iterative computing approximate sigma 108 | u = normalize(w_tmp.new_empty(height).normal_(0, 1), dim=0, eps=1e-12) 109 | v = normalize(torch.matmul(w_tmp.t(), u), dim=0, eps=1e-12) 110 | u = normalize(torch.matmul(w_tmp, v), dim=0, eps=1e-12) 111 | 112 | sigma = torch.dot(u, torch.matmul(w_tmp, v)) 113 | 114 | if term is None: 115 | term = (sigma) ** 2 116 | else: 117 | term = term + (sigma) ** 2 118 | 119 | return term 120 | 121 | -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | import sys 4 | import argparse 5 | 6 | from util.experiment import Experiment 7 | from util.conf import Configuration 8 | 9 | 10 | def main(argv): 11 | parser = argparse.ArgumentParser(description='Command-line parameters for Indexing Embedding experiments') 12 | 13 | parser.add_argument('-C', '--conf', type=str, required=True, dest='confpath', help='path of conf file') 14 | parser.add_argument('-E', '--embed', default=False, dest='to_embed', action='store_true', help='whether to embed database/query') 15 | 16 | args = parser.parse_args(argv[1: ]) 17 | 18 | conf = Configuration(args.confpath, dump=True) 19 | 20 | if args.to_embed: 21 | conf.setHP('to_embed', True) 22 | 23 | experiment = Experiment(conf) 24 | experiment.run() 25 | 26 | 27 | if __name__ == "__main__": 28 | main(sys.argv) 29 | -------------------------------------------------------------------------------- /util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qtwang/SEAnet/0cd38ff770bae8c4fe4f8d5f227a645b8c4c0ec9/util/__init__.py -------------------------------------------------------------------------------- /util/conf.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | import os 4 | from os.path import basename 5 | import json 6 | import platform 7 | from datetime import date 8 | from typing import Union, List 9 | from pathlib import Path 10 | 11 | import torch 12 | from torch import Tensor, nn, optim 13 | 14 | from model.activation import LeCunTanh 15 | from model.normalization import AdaNorm 16 | 17 | 18 | class Configuration: 19 | def __init__(self, path: str = '', dump: bool = False, existing: bool = False): 20 | self.defaults = { 21 | 'dataset_name': 'rw', 22 | 'database_path': 'default', 23 | 'query_path': 'default', 24 | 'train_path': 'default', 25 | 'train_indices_path': 'default', 26 | 'val_path': 'default', 27 | 'val_indices_path': 'default', 28 | 'sampling_name': 'coconut', 29 | 'coconut_libpath': 'default', 30 | 'coconut_cardinality': 8, 31 | 'encoder': 'residual', 32 | 'decoder': 'residual', 33 | 'dilation_type': 'exponential', 34 | 'dilation_cons': 1, 35 | 'dilation_base': 1, 36 | 'dilation_slope': 1, 37 | 'dim_series': 256, 38 | 'dim_embedding': 16, 39 | 'dim_en_latent': 256, 40 | 'dim_de_latent': 256, 41 | 'dim_coconut': 16, 42 | 'size_db': 10000000, 43 | 'size_query': 1000, 44 | 'size_train': 200000, 45 | 'size_val': 10000, 46 | 'size_batch': 256, 47 | 'size_kernel': 3, 48 | 'num_en_resblock': 3, 49 | 'num_de_resblock': 2, 50 | 'num_en_channels': 256, 51 | 'num_de_channels': 256, 52 | 'num_epoch': 100, 53 | 'relu_slope': 1e-2, 54 | 'optim_type': 'sgd', 55 | 'momentum': 0.9, 56 | 'lr_mode': 'linear', 57 | 'lr_cons': 1e-3, 58 | 'lr_max': 1e-3, 59 | 'lr_min': 1e-5, 60 | 'lr_everyk': 2, 61 | 'lr_ebase': 0.9, 62 | 'wd_mode': 'fix', 63 | 'wd_cons': 1e-4, 64 | 'wd_max': 1e-4, 65 | 'wd_min': 1e-8, 66 | 'device': 'cuda', 67 | 'activation_conv': 'relu', 68 | 'activation_linear': 'lecuntanh', 69 | 'resblock_pre_activation': True, 70 | 'layernorm_type': 'layernorm', 71 | 'layernorm_elementwise_affine': True, 72 | 'adanorm_k': 1 / 10, 73 | 'adanorm_scale': 2., 74 | 'weightnorm_type': 'none', 75 | 'weightnorm_dim': 0, 76 | 'epsilon': 1e-5, 77 | 'train_type': 'linearlycombine', 78 | 'recons_weight': 1/4, 79 | 'train_detach_query': True, 80 | 'model_init': 'lsuv', 81 | 'lsuv_size': 2000, 82 | 'lsuv_mean': 0, 83 | 'lsuv_std': 1., 84 | 'lsuv_std_tol': 0.1, 85 | 'lsuv_maxiter': 10, 86 | 'lsuv_ortho': True, 87 | 'torch_rdseed': 1997, 88 | 'cuda_rdseed': 1229, 89 | 'log_folder': 'default', 90 | 'log_filename': 'default', 91 | 'log_filepath': 'default', 92 | 'if_record': True, 93 | 'num_record': 20, 94 | 'record_folder': 'default', 95 | 'orth_regularizer': 'none', 96 | 'srip_mode': 'linear', 97 | 'srip_cons': 0.1, 98 | 'srip_max': 5e-4, 99 | 'srip_min': 0, 100 | 'checkpoint_mode': 'last', 101 | 'checkpoint_k': 1, 102 | 'checkpoint_postfix': 'pickle', 103 | 'checkpoint_folder': 'default', 104 | 'to_embed': True, 105 | 'db_embedding_path': 'default', 106 | 'query_embedding_path': 'default', 107 | 'embed_batch': 2000, 108 | 'encoder_normalize_embedding': True, 109 | 'decoder_normalize_reconstruction': True, 110 | 'dense_growth_rate': 32, 111 | 'dense_bottleneck_multiplier': 4, 112 | 'dense_transition_channels': 256, 113 | 'dense_init_channels': 64, 114 | 'num_en_denselayers': 8, 115 | 'num_en_denseblocks': 7, 116 | 'num_de_denselayers': 8, 117 | 'num_de_denseblocks': 1, 118 | 'default_conf_filename': 'conf.json', 119 | 'reverse_de_dilation': False, 120 | 'num_rnnen_layers': 7, 121 | 'num_rnnde_layers': 7, 122 | 'dim_rnnen_latent': 256, 123 | 'dim_rnnde_latent': 256, 124 | 'if_rnnen_bidirectional': False, 125 | 'if_rnnde_bidirectional': False, 126 | 'rnnen_dropout': 0.4, 127 | 'rnnde_dropout': 0.4, 128 | 'inception_bottleneck_channels': 64, 129 | 'inception_kernel_sizes': [1, 3, 5, 9, 17], 130 | 'num_inceptionen_blocks': 5, 131 | 'num_inceptionde_blocks': 5, 132 | 'to_scale_lc': False, 133 | 'to_scale_lr': False, 134 | 'name': 'default' 135 | } 136 | 137 | self.legals = { 138 | 'device': {'cpu', 'cuda'}, 139 | 'encoder': {'residual', 'dense', 'gru', 'lstm', 'fdj', 'inception'}, 140 | 'decoder': {'residual', 'dense', 'singleresidual', 'none', 'gru', 'lstm', 'fdj', 'inception'}, 141 | 'activation_conv': {'relu', 'leakyrelu', 'tanh', 'lecuntanh'}, 142 | 'activation_linear': {'relu', 'leakyrelu', 'tanh', 'lecuntanh'}, 143 | 'layernorm_type': {'layernorm', 'adanorm', 'none'}, 144 | 'weightnorm_type': {'weightnorm', 'none'}, 145 | 'dilation_type': {'exponential', 'linear', 'fixed'}, 146 | 'train_type': {'interleaving', 'linearlycombine'}, 147 | # 'dataset_name': {'rw', 'f5', 'f10', 'f20', 'f30', 'ucr18', 'seismic', 'astro', 'deep1b', 'sald'}, 148 | 'sampling_name': {'coconut', 'uniform'}, 149 | 'lr_mode': {'linear', 'fix', 'exponentially', 'exponentiallyhalve', 'plateauhalve'}, 150 | 'wd_mode': {'linear', 'fix'}, 151 | 'srip_mode': {'linear', 'fix'}, 152 | 'model_init': {'lsuv', 'default'}, 153 | 'orth_regularizer': {'srip', 'none'}, 154 | 'checkpoint_mode': {'last', 'everyk', 'none'} 155 | } 156 | 157 | self.settings = {} 158 | 159 | if path != '': 160 | if os.path.isdir(path): 161 | existing = True 162 | path = os.path.join(path, self.getHP('default_conf_filename')) 163 | 164 | self.loadConf(path, existing) 165 | 166 | # if dump and not existing: 167 | if dump: 168 | self.dumpConf() 169 | 170 | 171 | def getHP(self, name: str): 172 | if name in self.settings: 173 | return self.settings[name] 174 | 175 | if name in self.defaults: 176 | return self.defaults[name] 177 | 178 | raise ValueError('hyperparmeter {} doesn\'t exist'.format(name)) 179 | 180 | 181 | def setHP(self, key: str, value): 182 | self.settings[key] = value 183 | 184 | 185 | def __validate(self) -> None: 186 | for key, value in self.settings.items(): 187 | if key in self.legals and value not in self.legals[key]: 188 | raise ValueError('illegal setting {} for {} ({})'.format(value, key, ', '.join([str(item) for item in self.legals[key]]))) 189 | 190 | assert self.getHP('lr_mode') != 'exponentially' or (0 < self.getHP('lr_ebase') < 1) 191 | assert (self.getHP('encoder') == 'gru' or self.getHP('encoder') == 'lstm') == (self.getHP('decoder') == 'gru' or self.getHP('decoder') == 'lstm') 192 | 193 | if self.getHP('encoder') == 'fdj': 194 | assert self.getHP('decoder') == 'fdj' or self.getHP('decoder') == 'none' 195 | assert self.getHP('resblock_pre_activation') == False 196 | elif self.getHP('decoder') == 'fdj': 197 | raise ValueError('decoder {:s} shoud have encoder {:s}, while got {:s}'.format(self.getHP('decoder'), 'fdj', self.getHP('encoder'))) 198 | 199 | if self.getHP('encoder') == 'inception' or self.getHP('decoder') == 'inception': 200 | inception_kernel_sizes = self.getHP('inception_kernel_sizes') 201 | assert type(inception_kernel_sizes) == list and len(inception_kernel_sizes) != 0 and 1 in inception_kernel_sizes 202 | 203 | assert self.getHP('database_path') != 'default' 204 | assert self.getHP('query_path') != 'default' 205 | assert self.getHP('coconut_libpath') != 'default' 206 | 207 | 208 | def __setup(self, existing: bool = False) -> None: 209 | dataset_name = self.getHP('dataset_name') 210 | dim_series = self.getHP('dim_series') 211 | 212 | db_size = self.getHP('size_db') 213 | assert db_size % 1000000 == 0 214 | 215 | train_size = self.getHP('size_train') 216 | assert train_size % 1000 == 0 217 | 218 | encoder_code = self.getHP('encoder') 219 | decoder_code = self.getHP('decoder') 220 | 221 | sampling_code = self.getHP('sampling_name') 222 | if sampling_code == 'coconut': 223 | sampling_code += ('-' + str(self.getHP('dim_coconut'))) 224 | 225 | if existing: 226 | result_root = str(Path(self.getHP('conf_path')).parent) 227 | else: 228 | result_root = os.path.join(os.getcwd(), self.getHP('name')) 229 | os.makedirs(result_root, exist_ok=True) 230 | 231 | sample_root = os.path.join(result_root, 'samples') 232 | 233 | assert self.getHP('database_path') != 'default' and os.path.isfile(self.getHP('database_path')) 234 | assert self.getHP('query_path') != 'default' and os.path.isfile(self.getHP('query_path')) 235 | assert self.getHP('coconut_libpath') != 'default' and os.path.isfile(self.getHP('coconut_libpath')) 236 | 237 | if self.getHP('train_path') == 'default': 238 | filename = '-'.join([sampling_code, dataset_name, str(dim_series), str(int(db_size / 1000000)) + 'm', str(int(train_size / 1000)) + 'k']) + '.bin' 239 | self.setHP('train_path', os.path.join(sample_root, filename)) 240 | 241 | if self.getHP('train_indices_path') == 'default' or (existing and sample_root not in self.getHP('train_indices_path')): 242 | filename = '-'.join([sampling_code, dataset_name, str(dim_series), str(int(db_size / 1000000)) + 'm-indices', str(int(train_size / 1000)) + 'k']) + '.bin' 243 | self.setHP('train_indices_path', os.path.join(sample_root, filename)) 244 | 245 | if self.getHP('val_path') == 'default': 246 | val_size = self.getHP('size_val') 247 | # assert val_size % 1000 == 0 248 | 249 | filename = '-'.join([sampling_code, dataset_name, str(dim_series), str(int(db_size / 1000000)) + 'm', str(int(val_size / 1000)) + 'k']) + '.bin' 250 | self.setHP('val_path', os.path.join(sample_root, filename)) 251 | 252 | if self.getHP('val_indices_path') == 'default' or (existing and sample_root not in self.getHP('val_indices_path')): 253 | val_size = self.getHP('size_val') 254 | # assert val_size % 1000 == 0 255 | 256 | filename = '-'.join([sampling_code, dataset_name, str(dim_series), str(int(db_size / 1000000)) + 'm-indices', str(int(val_size / 1000)) + 'k']) + '.bin' 257 | self.setHP('val_indices_path', os.path.join(sample_root, filename)) 258 | 259 | embedding_prefix = '-'.join([dataset_name, str(dim_series), str(int(db_size / 1000000)) + 'm', 260 | encoder_code, decoder_code, str(self.getHP('dim_embedding')), 261 | sampling_code, str(int(train_size / 1000)) + 'k']) 262 | 263 | if self.getHP('db_embedding_path') == 'default': 264 | self.setHP('db_embedding_path', os.path.join(result_root, embedding_prefix + '.bin')) 265 | 266 | if self.getHP('query_embedding_path') == 'default': 267 | # assert self.getHP('size_query') % 1000 == 0 268 | self.setHP('query_embedding_path', os.path.join(result_root, embedding_prefix + '-1k.bin')) 269 | 270 | if self.getHP('log_filepath') == 'default': 271 | log_folder = self.getHP('log_folder') 272 | 273 | if log_folder == 'default': 274 | log_folder = result_root 275 | 276 | log_filename = self.getHP('log_filename') 277 | 278 | if log_filename == 'default': 279 | log_filename = 'fit.log' 280 | 281 | log_filepath = os.path.join(log_folder, log_filename) 282 | 283 | self.setHP('log_filepath', log_filepath) 284 | 285 | if self.getHP('record_folder') == 'default': 286 | self.setHP('record_folder', result_root) 287 | 288 | if self.getHP('checkpoint_folder') == 'default': 289 | self.setHP('checkpoint_folder', result_root) 290 | 291 | self.default_confpath = os.path.join(result_root, self.getHP('default_conf_filename')) 292 | 293 | 294 | def loadConf(self, path: str, existing: bool = False) -> None: 295 | with open(path, 'r') as fin: 296 | loaded = json.load(fin) 297 | 298 | if 'settings' in loaded: 299 | self.settings = loaded['settings'] 300 | 301 | local_defaults = self.defaults 302 | self.defaults = loaded['defaults'] 303 | 304 | for name, value in local_defaults.items(): 305 | if name not in self.defaults: 306 | self.defaults[name] = value 307 | else: 308 | self.settings = loaded 309 | 310 | self.settings['conf_path'] = path 311 | 312 | self.__validate() 313 | self.__setup(existing) 314 | 315 | 316 | def dumpConf(self, path: str = None) -> None: 317 | with open(self.default_confpath if path is None else path, 'w') as fout: 318 | json.dump({'settings': self.settings, 'defaults': self.defaults}, fout, sort_keys=True, indent=4) 319 | 320 | 321 | # TODO this design (of getActivation in conf) is a little tricky 322 | def getActivation(self, name: str) -> nn.Module: 323 | if name == 'tanh': 324 | return nn.Tanh() 325 | elif name == 'lecuntanh': 326 | return LeCunTanh() 327 | elif name == 'relu': 328 | return nn.ReLU() 329 | elif name == 'leakyrelu': 330 | return nn.LeakyReLU(self.getHP('relu_slope')) 331 | 332 | return nn.Identity() 333 | 334 | 335 | def getWeightNorm(self, model: nn.Module) -> nn.Module: 336 | weightnorm_type = self.getHP('weightnorm_type') 337 | 338 | if weightnorm_type == 'weightnorm' or self.getHP('encoder') == 'fdj': 339 | return nn.utils.weight_norm(model, dim=self.getHP('weightnorm_dim')) 340 | 341 | return model 342 | 343 | 344 | def getLayerNorm(self, shape: Union[int, List[int], torch.Size]) -> nn.Module: 345 | if self.getHP('encoder') != 'fdj': 346 | layernorm_type = self.getHP('layernorm_type') 347 | if layernorm_type == 'layernorm': 348 | return nn.LayerNorm(shape, elementwise_affine=self.getHP('layernorm_elementwise_affine')) 349 | elif layernorm_type == 'adanorm': 350 | return AdaNorm(shape, self.getHP('adanorm_k'), self.getHP('adanorm_scale'), self.getHP('eps'), self.getHP('layernorm_elementwise_affine')) 351 | 352 | return nn.Identity() 353 | 354 | 355 | # depth starts from 1 356 | def getDilatoin(self, depth: int, to_encode: bool = True) -> int: 357 | dilation_type = self.getHP('dilation_type') 358 | 359 | if not to_encode: 360 | decoder_name = self.getHP('decoder') 361 | 362 | if decoder_name == 'residual' or decoder_name == 'fdj': 363 | depth = self.getHP('num_de_resblock') + 1 - depth 364 | elif decoder_name == 'dense': 365 | depth = self.getHP('num_de_denseblocks') + 1 - depth 366 | elif not dilation_type == 'fixed': 367 | raise ValueError('illegal decoder {:s} to change dilation (residual/dense only)'.format(self.getHP('decoder'))) 368 | 369 | if dilation_type == 'exponential': 370 | return int(2 ** (depth - 1)) 371 | elif dilation_type == 'linear': 372 | return self.getHP('dilation_base') + self.getHP('dilation_slope') * (depth - 1) 373 | 374 | return self.getHP('dilation_constant') 375 | 376 | -------------------------------------------------------------------------------- /util/data.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | import os 4 | import struct 5 | import platform 6 | import subprocess 7 | from os.path import isfile 8 | from pathlib import Path 9 | from ctypes import CDLL, c_char_p, c_long 10 | from _ctypes import dlclose 11 | 12 | import torch 13 | import numpy as np 14 | from torch.utils.data import Dataset 15 | 16 | from util.conf import Configuration 17 | 18 | 19 | class TSDataset(Dataset): 20 | def __init__(self, data): 21 | self.data = data 22 | 23 | def __len__(self): 24 | return self.data.shape[0] 25 | 26 | def __getitem__(self, indices): 27 | return self.data[indices] 28 | 29 | 30 | 31 | def getSamples(conf: Configuration): 32 | dim_series = conf.getHP('dim_series') 33 | size_train = conf.getHP('size_train') 34 | size_val = conf.getHP('size_val') 35 | device = conf.getHP('device') 36 | 37 | train_path = conf.getHP('train_path') 38 | val_path = conf.getHP('val_path') 39 | 40 | if os.path.exists(train_path) and os.path.exists(val_path): 41 | train_samples = torch.from_numpy(np.fromfile(train_path, dtype=np.float32, count=dim_series * size_train)) 42 | val_samples = torch.from_numpy(np.fromfile(val_path, dtype=np.float32, count=dim_series * size_val)) 43 | else: 44 | if conf.getHP('sampling_name') == 'coconut' or conf.getHP('sampling_name') == 'uniform': 45 | train_samples, val_samples = sample(conf) 46 | else: 47 | raise ValueError('sampling {:s} is not supported'.format(conf.getHP('sampling_name'))) 48 | 49 | if conf.getHP('encoder') == 'gru' or conf.getHP('encoder') == 'lstm': 50 | train_samples = train_samples.view([-1, dim_series, 1]) 51 | val_samples = val_samples.view([-1, dim_series, 1]) 52 | else: 53 | train_samples = train_samples.view([-1, 1, dim_series]) 54 | val_samples = val_samples.view([-1, 1, dim_series]) 55 | 56 | train_samples = train_samples.to(device) 57 | val_samples = val_samples.to(device) 58 | 59 | return train_samples, val_samples 60 | 61 | 62 | # def loadTrainValCocunut(dataset_name, dataset_path, dataset_size, train_size, val_size, series_length=256, sax_length=16, sax_cardinality=8): 63 | def sample(conf: Configuration): 64 | dataset_path = conf.getHP('database_path') 65 | 66 | train_path = conf.getHP('train_path') 67 | val_path = conf.getHP('val_path') 68 | train_indices_path = conf.getHP('train_indices_path') 69 | val_indices_path = conf.getHP('val_indices_path') 70 | 71 | os.makedirs(Path(train_path).parent, exist_ok=True) 72 | os.makedirs(Path(val_path).parent, exist_ok=True) 73 | os.makedirs(Path(train_indices_path).parent, exist_ok=True) 74 | os.makedirs(Path(val_indices_path).parent, exist_ok=True) 75 | 76 | dim_coconut = conf.getHP('dim_coconut') 77 | dim_series = conf.getHP('dim_series') 78 | size_train = conf.getHP('size_train') 79 | size_val = conf.getHP('size_val') 80 | size_db = conf.getHP('size_db') 81 | 82 | sampling_method = conf.getHP('sampling_name') 83 | 84 | if sampling_method == 'coconut': 85 | if not (os.path.exists(train_indices_path) and isfile(train_indices_path)) or not (os.path.exists(val_indices_path) and isfile(val_indices_path)): 86 | c_functions = CDLL(conf.getHP('coconut_libpath')) 87 | 88 | return_code = c_functions.sample_coconut(c_char_p(dataset_path.encode('ASCII')), 89 | c_long(size_db), 90 | c_char_p(train_indices_path.encode('ASCII')), 91 | size_train, 92 | c_char_p(val_indices_path.encode('ASCII')), 93 | size_val, 94 | dim_series, 95 | conf.getHP('coconut_cardinality'), 96 | dim_coconut) 97 | dlclose(c_functions._handle) 98 | 99 | if return_code != 0: 100 | print(return_code) 101 | elif sampling_method == 'uniform': 102 | if not (os.path.exists(train_indices_path) and isfile(train_indices_path)) or not (os.path.exists(val_indices_path) and isfile(val_indices_path)): 103 | train_sample_indices = np.random.randint(0, size_db, size=size_train, dtype=np.int64) 104 | val_samples_indices = np.random.randint(0, size_db, size=size_val, dtype=np.int64) 105 | 106 | train_sample_indices.tofile(train_indices_path) 107 | val_samples_indices.tofile(val_indices_path) 108 | else: 109 | raise ValueError('sampling {:s} is not supported'.format(sampling_method)) 110 | 111 | train_sample_indices = np.fromfile(train_indices_path, dtype=np.int64) 112 | assert len(train_sample_indices) == size_train 113 | 114 | loaded = [] 115 | for index in train_sample_indices: 116 | sequence = np.fromfile(dataset_path, dtype=np.float32, count=dim_series, offset=4 * dim_series * index) 117 | 118 | if not np.isnan(np.sum(sequence)): 119 | loaded.append(sequence) 120 | 121 | train_samples = np.asarray(loaded, dtype=np.float32) 122 | train_samples.tofile(train_path) 123 | train_samples = torch.from_numpy(train_samples) 124 | 125 | val_samples_indices = np.fromfile(val_indices_path, dtype=np.int64) 126 | assert len(val_samples_indices) == size_val 127 | 128 | loaded = [] 129 | for index in val_samples_indices: 130 | sequence = np.fromfile(dataset_path, dtype=np.float32, count=dim_series, offset=4 * dim_series * index) 131 | 132 | if not np.isnan(np.sum(sequence)): 133 | loaded.append(sequence) 134 | 135 | val_samples = np.asarray(loaded, dtype=np.float32) 136 | val_samples.tofile(val_path) 137 | val_samples = torch.from_numpy(val_samples) 138 | 139 | return train_samples, val_samples 140 | 141 | 142 | 143 | class FileContainer(object): 144 | def __init__(self, filename, binary=True): 145 | self.filename = filename 146 | self.binary = binary 147 | if self.binary: 148 | self.f = open(filename, "wb") 149 | else: 150 | self.f = open(filename, "w") 151 | 152 | def write(self, ts): 153 | if self.binary: 154 | s = struct.pack('f' * len(ts), *ts) 155 | self.f.write(s) 156 | else: 157 | self.f.write(" ".join(map(str, ts)) + "\n") 158 | 159 | def close(self): 160 | self.f.close() 161 | 162 | 163 | 164 | def embedData(model, data_filepath, embedding_filepath, data_size, batch_size = 2000, original_dim = 256, 165 | embedded_dim = 16, device = 'cuda', is_rnn = False, encoder = ''): 166 | if encoder == 'gru' or encoder == 'lstm': 167 | is_rnn = True 168 | 169 | num_segments = int(data_size / batch_size) 170 | 171 | if data_size < batch_size: 172 | num_segments = 1 173 | batch_size = data_size 174 | else: 175 | assert data_size % batch_size == 0 176 | 177 | nan_replacement_original = np.array([0.] * original_dim).reshape([original_dim, 1] if is_rnn else [1, original_dim]) 178 | nan_replacement_embedding = [0.] * embedded_dim 179 | 180 | writer = FileContainer(embedding_filepath) 181 | 182 | try: 183 | with torch.no_grad(): 184 | total_nans = 0 185 | 186 | for segment in range(num_segments): 187 | batch = np.fromfile(data_filepath, dtype=np.float32, count=original_dim * batch_size, offset=4 * original_dim * batch_size * segment) 188 | 189 | if is_rnn: 190 | batch = batch.reshape([-1, original_dim, 1]) 191 | else: 192 | batch = batch.reshape([-1, 1, original_dim]) 193 | 194 | nan_indices = set() 195 | for i, sequence in zip(range(batch.shape[0]), batch): 196 | if np.isnan(np.sum(sequence)): 197 | nan_indices.add(i) 198 | batch[i] = nan_replacement_original 199 | 200 | embedding = model.encode(torch.from_numpy(batch).to(device)).detach().cpu().numpy() 201 | 202 | for i in nan_indices: 203 | embedding[i] = nan_replacement_embedding 204 | 205 | writer.write(embedding.flatten()) 206 | 207 | total_nans += len(nan_indices) 208 | 209 | print('nans = {:d}'.format(total_nans)) 210 | finally: 211 | writer.close() 212 | -------------------------------------------------------------------------------- /util/experiment.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | import os 4 | import io 5 | import logging 6 | from datetime import date 7 | from timeit import default_timer as timer 8 | 9 | import torch 10 | import numpy as np 11 | from torch import nn, optim 12 | from torch.utils.data import DataLoader 13 | import matplotlib.pyplot as plt 14 | 15 | from model.initialization import LSUVinit 16 | from model.normalization import getSRIPTerm 17 | from model.loss import ScaledL2Recons, ScaledL2Trans 18 | from model.builder import AEBuilder 19 | from util.data import TSDataset, getSamples 20 | from util.conf import Configuration 21 | from util.data import embedData 22 | 23 | 24 | class Experiment: 25 | def __init__(self, conf: Configuration): 26 | self.__conf = conf 27 | self.has_setup = False 28 | self.epoch = 0 29 | 30 | self.device = conf.getHP('device') 31 | 32 | self.max_epoch = self.__conf.getHP('num_epoch') 33 | 34 | self.checkpoint_folder = conf.getHP('checkpoint_folder') 35 | self.checkpoint_postfix = conf.getHP('checkpoint_postfix') 36 | 37 | 38 | def loadModel(self, epoch: int = -1) -> nn.Module: 39 | assert epoch < self.max_epoch 40 | checkpoint_path = os.path.join(self.checkpoint_folder, str(epoch) + '.' + self.checkpoint_postfix) 41 | 42 | if epoch == -1: 43 | epoch = self.max_epoch 44 | checkpoint_path = os.path.join(self.checkpoint_folder, str(epoch) + '.' + self.checkpoint_postfix) 45 | 46 | while not os.path.exists(checkpoint_path) and epoch >= 0: 47 | epoch -= 1 48 | checkpoint_path = os.path.join(self.checkpoint_folder, str(epoch) + '.' + self.checkpoint_postfix) 49 | 50 | if epoch != -1: 51 | print('loading checkpoint at epoch {:d}'.format(epoch)) 52 | else: 53 | assert os.path.exists(checkpoint_path) 54 | 55 | if epoch == -1: 56 | return None 57 | 58 | self.epoch = epoch 59 | 60 | model = AEBuilder(self.__conf) 61 | with open(checkpoint_path, 'rb') as fin: 62 | model.load_state_dict(torch.load(fin)) 63 | 64 | return model 65 | 66 | 67 | def getSample(self, size: int = -1) -> torch.Tensor: 68 | _, val_samples = getSamples(self.__conf) 69 | 70 | if size == -1: 71 | return val_samples 72 | elif size <= self.__conf.getHP('size_val'): 73 | indices = torch.randperm(val_samples.shape[0]) 74 | return val_samples[indices][: size].to(self.device) 75 | else: 76 | raise ValueError('cannot provide {:d} samples ({:d} valset)'.format(size, self.__conf.getHP('size_val'))) 77 | 78 | 79 | def setup(self) -> None: 80 | self.has_setup = True 81 | 82 | logging.basicConfig(filename=self.__conf.getHP('log_filepath'), 83 | filemode='a+', 84 | format='%(asctime)s,%(msecs)d %(levelname).3s [%(filename)s:%(lineno)d] %(message)s', 85 | level=logging.DEBUG, 86 | datefmt='%m/%d/%Y:%I:%M:%S') 87 | 88 | self.logger = logging.getLogger(self.__class__.__name__) 89 | 90 | torch.manual_seed(self.__conf.getHP('torch_rdseed')) 91 | if self.device == 'cuda': 92 | if torch.cuda.is_available(): 93 | torch.cuda.empty_cache() 94 | torch.cuda.manual_seed_all(self.__conf.getHP('cuda_rdseed')) 95 | else: 96 | raise ValueError('cuda is not available') 97 | 98 | batch_size = self.__conf.getHP('size_batch') 99 | 100 | train_samples, val_samples = getSamples(self.__conf) 101 | 102 | self.train_db_loader = DataLoader(TSDataset(train_samples), batch_size=batch_size, shuffle=True) 103 | self.train_query_loader = DataLoader(TSDataset(train_samples), batch_size=batch_size, shuffle=True) 104 | 105 | self.val_db_loader = DataLoader(TSDataset(val_samples), batch_size=batch_size, shuffle=True) 106 | self.val_query_loader = DataLoader(TSDataset(val_samples), batch_size=batch_size, shuffle=True) 107 | 108 | dim_series = self.__conf.getHP('dim_series') 109 | dim_embedding = self.__conf.getHP('dim_embedding') 110 | 111 | if self.__conf.getHP('to_scale_lc'): 112 | self.trans_loss = ScaledL2Trans(dim_series, dim_embedding, to_scale=True).to(self.device) 113 | else: 114 | self.trans_loss = ScaledL2Trans().to(self.device) 115 | 116 | if self.__conf.getHP('to_scale_lr'): 117 | self.recons_reg = ScaledL2Recons(dim_series, to_scale=True).to(self.device) 118 | else: 119 | self.recons_reg = ScaledL2Recons().to(self.device) 120 | 121 | self.model = self.loadModel() 122 | if self.model is None: 123 | self.model = AEBuilder(self.__conf) 124 | self.model = self.__init_model(self.model, val_samples) 125 | 126 | self.optimizer = self.__getOptimizer() 127 | 128 | self.checkpoint_mode = self.__conf.getHP('checkpoint_mode') 129 | if self.checkpoint_mode != 'none': 130 | if self.checkpoint_mode == 'everyk': 131 | self.checkpoint_k = self.__conf.getHP('checkpoint_k') 132 | 133 | if self.__conf.getHP('if_record'): 134 | indices = torch.randperm(val_samples.shape[0]) 135 | 136 | self.samples2plot = val_samples[indices][: self.__conf.getHP('num_record')].to(self.device) 137 | self.record_folder = self.__conf.getHP('record_folder') 138 | 139 | self.detch_query = self.__conf.getHP('train_detach_query') 140 | 141 | self.encoder_only = self.__conf.getHP('decoder') == 'none' 142 | if not self.encoder_only: 143 | self.recons_weight = self.__conf.getHP('recons_weight') 144 | 145 | self.orth_regularizer = self.__conf.getHP('orth_regularizer') 146 | if self.orth_regularizer == 'srip': 147 | if self.__conf.getHP('srip_mode') == 'fix': 148 | self.srip_weight = self.__conf.getHP('srip_cons') 149 | elif self.__conf.getHP('srip_mode') == 'linear': 150 | self.srip_weight = self.__conf.getHP('srip_max') 151 | 152 | if torch.cuda.is_available(): 153 | torch.cuda.empty_cache() 154 | 155 | 156 | def run(self) -> None: 157 | if not self.has_setup: 158 | self.setup() 159 | 160 | self.__checkpoint(persist_model=False) 161 | 162 | while self.epoch < self.max_epoch: 163 | start = timer() 164 | 165 | self.__adjust_lr() 166 | self.__adjust_wd() 167 | if self.orth_regularizer == 'srip': 168 | self.__adjust_srip() 169 | 170 | self.epoch += 1 171 | 172 | self.__train() 173 | self.__validate() 174 | 175 | self.logger.info('e{:d} time = {:.3f}s'.format(self.epoch, timer() - start)) 176 | 177 | self.__checkpoint() 178 | 179 | if self.__conf.getHP('to_embed'): 180 | embedData(self.model, self.__conf.getHP('database_path'), self.__conf.getHP('db_embedding_path'), 181 | self.__conf.getHP('size_db'), batch_size=self.__conf.getHP('embed_batch'), original_dim=self.__conf.getHP('dim_series'), 182 | embedded_dim=self.__conf.getHP('dim_embedding'), device=self.device, encoder=self.__conf.getHP('encoder')) 183 | embedData(self.model, self.__conf.getHP('query_path'), self.__conf.getHP('query_embedding_path'), 184 | self.__conf.getHP('size_query'), batch_size=self.__conf.getHP('embed_batch'), original_dim=self.__conf.getHP('dim_series'), 185 | embedded_dim=self.__conf.getHP('dim_embedding'), device=self.device, encoder=self.__conf.getHP('encoder')) 186 | 187 | 188 | def __train(self) -> None: 189 | recons_errors = [] 190 | orth_terms = [] 191 | trans_errors = [] 192 | 193 | if self.__conf.getHP('train_type') == 'interleaving': 194 | if not self.encoder_only: 195 | for batch in self.train_db_loader: 196 | self.optimizer.zero_grad() 197 | 198 | embedding = self.model.encode(batch) 199 | reconstructed = self.model.decode(embedding) 200 | 201 | recons_term = self.recons_weight * self.recons_reg(batch, reconstructed) 202 | orth_term = self.__orth_reg() 203 | regularization = recons_term + orth_term 204 | 205 | regularization.backward() 206 | self.optimizer.step() 207 | 208 | recons_errors.append(recons_term.detach().item()) 209 | orth_terms.append(orth_term.detach().item()) 210 | 211 | self.logger.info('t{:d} recons = {:.4f}'.format(self.epoch, np.mean(recons_errors))) 212 | self.logger.info('t{:d} orth = {:.4f}'.format(self.epoch, np.mean(orth_terms))) 213 | 214 | for db_batch, query_batch in zip(self.train_db_loader, self.train_query_loader): 215 | self.optimizer.zero_grad() 216 | 217 | if self.detch_query: 218 | with torch.no_grad(): 219 | query_embedding = self.model.encode(query_batch).detach() 220 | query_batch = query_batch.detach() 221 | else: 222 | query_embedding = self.model.encode(query_batch) 223 | 224 | db_embedding = self.model.encode(db_batch) 225 | trans_error = self.trans_loss(db_batch, query_batch, db_embedding, query_embedding) 226 | 227 | trans_error.backward() 228 | self.optimizer.step() 229 | 230 | trans_errors.append(trans_error.detach().item()) 231 | 232 | logging.info('t{:d} trans = {:.4f}'.format(self.epoch, np.mean(trans_errors))) 233 | 234 | elif self.__conf.getHP('train_type') == 'linearlycombine': 235 | for db_batch, query_batch in zip(self.train_db_loader, self.train_query_loader): 236 | self.optimizer.zero_grad() 237 | 238 | if self.detch_query: 239 | with torch.no_grad(): 240 | query_embedding = self.model.encode(query_batch).detach() 241 | query_batch = query_batch.detach() 242 | else: 243 | query_embedding = self.model.encode(query_batch) 244 | 245 | db_embedding = self.model.encode(db_batch) 246 | 247 | if not self.encoder_only: 248 | db_reconstructed = self.model.decode(db_embedding) 249 | recons_term = self.recons_weight * self.recons_reg(db_batch, db_reconstructed) 250 | else: 251 | recons_term = torch.zeros(1).to(self.device) 252 | 253 | trans_error = self.trans_loss(db_batch, query_batch, db_embedding, query_embedding) 254 | orth_term = self.__orth_reg() 255 | 256 | loss = trans_error + recons_term + orth_term 257 | 258 | loss.backward() 259 | self.optimizer.step() 260 | 261 | recons_errors.append(recons_term.detach().item()) 262 | orth_terms.append(orth_term.detach().item()) 263 | trans_errors.append(trans_error.detach().item()) 264 | 265 | self.logger.info('t{:d} recons = {:.4f}'.format(self.epoch, np.mean(recons_errors))) 266 | self.logger.info('t{:d} orth = {:.4f}'.format(self.epoch, np.mean(orth_terms))) 267 | self.logger.info('t{:d} trans = {:.4f}'.format(self.epoch, np.mean(trans_errors))) 268 | 269 | else: 270 | raise ValueError('cannot train') 271 | 272 | 273 | def __validate(self) -> None: 274 | trans_errors = [] 275 | 276 | with torch.no_grad(): 277 | for db_batch, query_batch in zip(self.val_db_loader, self.val_query_loader): 278 | db_embedding = self.model.encode(db_batch) 279 | query_embedding = self.model.encode(query_batch) 280 | 281 | trans_error = self.trans_loss(db_batch, query_batch, db_embedding, query_embedding) 282 | trans_errors.append(trans_error.detach().item()) 283 | 284 | self.logger.info('v{:d} trans = {:.4f}'.format(self.epoch, np.mean(trans_errors))) 285 | 286 | 287 | def __checkpoint(self, persist_model: bool = True) -> None: 288 | if self.__conf.getHP('if_record'): 289 | if self.encoder_only: 290 | fig, ax = plt.subplots(2, 1, figsize=(12, 6)) 291 | else: 292 | fig, ax = plt.subplots(3, 1, figsize=(12, 9)) 293 | 294 | with torch.no_grad(): 295 | for series in torch.squeeze(self.samples2plot).detach().cpu(): 296 | ax[0].plot(series) 297 | 298 | embedding = self.model.encode(self.samples2plot) 299 | for series in embedding.detach().cpu(): 300 | ax[1].plot(series) 301 | 302 | if not self.encoder_only: 303 | reconstructed = self.model.decode(embedding) 304 | for series in torch.squeeze(reconstructed).detach().cpu(): 305 | ax[2].plot(series) 306 | 307 | fig.tight_layout() 308 | plt.savefig(self.record_folder + str(self.epoch) + '.eps', dpi=456) 309 | 310 | if persist_model and self.checkpoint_mode != 'none' and (self.epoch == self.max_epoch or (self.checkpoint_mode == 'everyk' and self.epoch % self.checkpoint_k == 0)): 311 | torch.save(self.model.state_dict(), os.path.join(self.checkpoint_folder, str(self.epoch) + '.' + self.checkpoint_postfix)) 312 | 313 | 314 | def __orth_reg(self) -> torch.Tensor: 315 | if self.orth_regularizer == 'srip': 316 | return self.srip_weight * getSRIPTerm(self.model, self.device) 317 | 318 | return torch.zeros(1).to(self.device) 319 | 320 | 321 | def __adjust_lr(self) -> None: 322 | # should be based on self.epoch and hyperparameters ONLY for easily resumming 323 | 324 | for param_group in self.optimizer.param_groups: 325 | current_lr = param_group['lr'] 326 | break 327 | 328 | new_lr = current_lr 329 | 330 | if self.__conf.getHP('lr_mode') == 'linear': 331 | lr_max = self.__conf.getHP('lr_max') 332 | lr_min = self.__conf.getHP('lr_min') 333 | 334 | new_lr = lr_max - self.epoch * (lr_max - lr_min) / self.max_epoch 335 | elif self.__conf.getHP('lr_mode') == 'exponentiallyhalve': 336 | lr_max = self.__conf.getHP('lr_max') 337 | lr_min = self.__conf.getHP('lr_min') 338 | 339 | for i in range(1, 11): 340 | if (self.max_epoch - self.epoch) * (2 ** i) == self.max_epoch: 341 | new_lr = lr_max / (10 ** i) 342 | break 343 | 344 | if new_lr < lr_min: 345 | new_lr = lr_min 346 | elif self.__conf.getHP('lr_mode') == 'exponentially': 347 | lr_max = self.__conf.getHP('lr_max') 348 | lr_min = self.__conf.getHP('lr_min') 349 | lr_k = self.__conf.getHP('lr_everyk') 350 | lr_ebase = self.__conf.getHP('lr_ebase') 351 | 352 | lr_e = int(np.floor(self.epoch / lr_k)) 353 | new_lr = lr_max * (lr_ebase ** lr_e) 354 | 355 | if new_lr < lr_min: 356 | new_lr = lr_min 357 | elif self.__conf.getHP('lr_mode') == 'plateauhalve': 358 | raise ValueError('plateauhalve is not yet supported') 359 | 360 | for param_group in self.optimizer.param_groups: 361 | param_group['lr'] = new_lr 362 | 363 | 364 | def __adjust_wd(self): 365 | # should be based on self.epoch and hyperparameters ONLY for easily resumming 366 | 367 | for param_group in self.optimizer.param_groups: 368 | current_wd = param_group['weight_decay'] 369 | break 370 | 371 | new_wd = current_wd 372 | 373 | if self.__conf.getHP('wd_mode') == 'linear': 374 | wd_max = self.__conf.getHP('wd_max') 375 | wd_min = self.__conf.getHP('wd_min') 376 | 377 | new_wd = wd_min + self.epoch * (wd_max - wd_min) / self.max_epoch 378 | 379 | for param_group in self.optimizer.param_groups: 380 | param_group['weight_decay'] = new_wd 381 | 382 | 383 | def __adjust_srip(self): 384 | # should be based on self.epoch and hyperparameters ONLY for easily resumming 385 | 386 | if self.__conf.getHP('srip_mode') == 'linear': 387 | srip_max = self.__conf.getHP('srip_max') 388 | srip_min = self.__conf.getHP('srip_min') 389 | 390 | self.srip_weight = srip_max - self.epoch * (srip_max - srip_min) / self.max_epoch 391 | 392 | 393 | 394 | def __getOptimizer(self) -> optim.Optimizer: 395 | if self.__conf.getHP('optim_type') == 'sgd': 396 | if self.__conf.getHP('lr_mode') == 'fix': 397 | initial_lr = self.__conf.getHP('lr_cons') 398 | else: 399 | initial_lr = self.__conf.getHP('lr_max') 400 | 401 | if self.__conf.getHP('wd_mode') == 'fix': 402 | initial_wd = self.__conf.getHP('wd_cons') 403 | else: 404 | initial_wd = self.__conf.getHP('wd_min') 405 | 406 | momentum = self.__conf.getHP('momentum') 407 | 408 | return optim.SGD(self.model.parameters(), lr=initial_lr, momentum=momentum, weight_decay=initial_wd) 409 | 410 | raise ValueError('cannot obtain optimizer') 411 | 412 | 413 | def __init_model(self, model: nn.Module, samples: torch.Tensor = None) -> nn.Module: 414 | if self.__conf.getHP('model_init') == 'lsuv': 415 | assert samples is not None 416 | 417 | return LSUVinit(model, samples[torch.randperm(samples.shape[0])][: self.__conf.getHP('lsuv_size')], 418 | needed_mean=self.__conf.getHP('lsuv_mean'), needed_std=self.__conf.getHP('lsuv_std'), 419 | std_tol=self.__conf.getHP('lsuv_std_tol'), max_attempts=self.__conf.getHP('lsuv_maxiter'), 420 | do_orthonorm=self.__conf.getHP('lsuv_ortho')) 421 | 422 | return model 423 | -------------------------------------------------------------------------------- /util/flops_counter.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | # https://github.com/sovrasov/flops-counter.pytorch 4 | # version 20200606 5 | 6 | ''' 7 | Copyright (C) 2019 Sovrasov V. - All Rights Reserved 8 | * You may use, distribute and modify this code under the 9 | * terms of the MIT license. 10 | * You should have received a copy of the MIT license with 11 | * this file. If not visit https://opensource.org/licenses/MIT 12 | ''' 13 | 14 | import sys 15 | from functools import partial 16 | 17 | import torch 18 | import torch.nn as nn 19 | import numpy as np 20 | 21 | 22 | def get_model_complexity_info(model, input_res, 23 | print_per_layer_stat=True, 24 | as_strings=True, 25 | input_constructor=None, ost=sys.stdout, 26 | verbose=False, ignore_modules=[], 27 | custom_modules_hooks={}): 28 | assert type(input_res) is tuple 29 | assert len(input_res) >= 1 30 | assert isinstance(model, nn.Module) 31 | global CUSTOM_MODULES_MAPPING 32 | CUSTOM_MODULES_MAPPING = custom_modules_hooks 33 | flops_model = add_flops_counting_methods(model) 34 | flops_model.eval() 35 | flops_model.start_flops_count(ost=ost, verbose=verbose, ignore_list=ignore_modules) 36 | if input_constructor: 37 | input = input_constructor(input_res) 38 | _ = flops_model(**input) 39 | else: 40 | try: 41 | batch = torch.ones(()).new_empty((1, *input_res), 42 | dtype=next(flops_model.parameters()).dtype, 43 | device=next(flops_model.parameters()).device) 44 | except StopIteration: 45 | batch = torch.ones(()).new_empty((1, *input_res)) 46 | 47 | _ = flops_model(batch) 48 | 49 | flops_count, params_count = flops_model.compute_average_flops_cost() 50 | if print_per_layer_stat: 51 | print_model_with_flops(flops_model, flops_count, params_count, ost=ost) 52 | flops_model.stop_flops_count() 53 | CUSTOM_MODULES_MAPPING = {} 54 | 55 | if as_strings: 56 | return flops_to_string(flops_count), params_to_string(params_count) 57 | 58 | return flops_count, params_count 59 | 60 | 61 | def flops_to_string(flops, units='GMac', precision=2): 62 | if units is None: 63 | if flops // 10**9 > 0: 64 | return str(round(flops / 10.**9, precision)) + ' GMac' 65 | elif flops // 10**6 > 0: 66 | return str(round(flops / 10.**6, precision)) + ' MMac' 67 | elif flops // 10**3 > 0: 68 | return str(round(flops / 10.**3, precision)) + ' KMac' 69 | else: 70 | return str(flops) + ' Mac' 71 | else: 72 | if units == 'GMac': 73 | return str(round(flops / 10.**9, precision)) + ' ' + units 74 | elif units == 'MMac': 75 | return str(round(flops / 10.**6, precision)) + ' ' + units 76 | elif units == 'KMac': 77 | return str(round(flops / 10.**3, precision)) + ' ' + units 78 | else: 79 | return str(flops) + ' Mac' 80 | 81 | 82 | def params_to_string(params_num, units=None, precision=2): 83 | if units is None: 84 | if params_num // 10 ** 6 > 0: 85 | return str(round(params_num / 10 ** 6, 2)) + ' M' 86 | elif params_num // 10 ** 3: 87 | return str(round(params_num / 10 ** 3, 2)) + ' k' 88 | else: 89 | return str(params_num) 90 | else: 91 | if units == 'M': 92 | return str(round(params_num / 10.**6, precision)) + ' ' + units 93 | elif units == 'K': 94 | return str(round(params_num / 10.**3, precision)) + ' ' + units 95 | else: 96 | return str(params_num) 97 | 98 | 99 | def print_model_with_flops(model, total_flops, total_params, units='GMac', 100 | precision=3, ost=sys.stdout): 101 | 102 | def accumulate_params(self): 103 | if is_supported_instance(self): 104 | return self.__params__ 105 | else: 106 | sum = 0 107 | for m in self.children(): 108 | sum += m.accumulate_params() 109 | return sum 110 | 111 | def accumulate_flops(self): 112 | if is_supported_instance(self): 113 | return self.__flops__ / model.__batch_counter__ 114 | else: 115 | sum = 0 116 | for m in self.children(): 117 | sum += m.accumulate_flops() 118 | return sum 119 | 120 | def flops_repr(self): 121 | accumulated_params_num = self.accumulate_params() 122 | accumulated_flops_cost = self.accumulate_flops() 123 | return ', '.join([params_to_string(accumulated_params_num, units='M', precision=precision), 124 | '{:.3%} Params'.format(accumulated_params_num / total_params), 125 | flops_to_string(accumulated_flops_cost, units=units, precision=precision), 126 | '{:.3%} MACs'.format(accumulated_flops_cost / total_flops), 127 | self.original_extra_repr()]) 128 | 129 | def add_extra_repr(m): 130 | m.accumulate_flops = accumulate_flops.__get__(m) 131 | m.accumulate_params = accumulate_params.__get__(m) 132 | flops_extra_repr = flops_repr.__get__(m) 133 | if m.extra_repr != flops_extra_repr: 134 | m.original_extra_repr = m.extra_repr 135 | m.extra_repr = flops_extra_repr 136 | assert m.extra_repr != m.original_extra_repr 137 | 138 | def del_extra_repr(m): 139 | if hasattr(m, 'original_extra_repr'): 140 | m.extra_repr = m.original_extra_repr 141 | del m.original_extra_repr 142 | if hasattr(m, 'accumulate_flops'): 143 | del m.accumulate_flops 144 | 145 | model.apply(add_extra_repr) 146 | print(model, file=ost) 147 | model.apply(del_extra_repr) 148 | 149 | 150 | def get_model_parameters_number(model): 151 | params_num = sum(p.numel() for p in model.parameters() if p.requires_grad) 152 | return params_num 153 | 154 | 155 | def add_flops_counting_methods(net_main_module): 156 | # adding additional methods to the existing module object, 157 | # this is done this way so that each function has access to self object 158 | net_main_module.start_flops_count = start_flops_count.__get__(net_main_module) 159 | net_main_module.stop_flops_count = stop_flops_count.__get__(net_main_module) 160 | net_main_module.reset_flops_count = reset_flops_count.__get__(net_main_module) 161 | net_main_module.compute_average_flops_cost = compute_average_flops_cost.__get__(net_main_module) 162 | 163 | net_main_module.reset_flops_count() 164 | 165 | return net_main_module 166 | 167 | 168 | def compute_average_flops_cost(self): 169 | """ 170 | A method that will be available after add_flops_counting_methods() is called 171 | on a desired net object. 172 | 173 | Returns current mean flops consumption per image. 174 | 175 | """ 176 | 177 | batches_count = self.__batch_counter__ 178 | flops_sum = 0 179 | params_sum = 0 180 | for module in self.modules(): 181 | if is_supported_instance(module): 182 | flops_sum += module.__flops__ 183 | params_sum = get_model_parameters_number(self) 184 | return flops_sum / batches_count, params_sum 185 | 186 | 187 | def start_flops_count(self, **kwargs): 188 | """ 189 | A method that will be available after add_flops_counting_methods() is called 190 | on a desired net object. 191 | 192 | Activates the computation of mean flops consumption per image. 193 | Call it before you run the network. 194 | 195 | """ 196 | add_batch_counter_hook_function(self) 197 | 198 | seen_types = set() 199 | def add_flops_counter_hook_function(module, ost, verbose, ignore_list): 200 | if type(module) in ignore_list: 201 | seen_types.add(type(module)) 202 | if is_supported_instance(module): 203 | module.__params__ = 0 204 | elif is_supported_instance(module): 205 | if hasattr(module, '__flops_handle__'): 206 | return 207 | if type(module) in CUSTOM_MODULES_MAPPING: 208 | handle = module.register_forward_hook(CUSTOM_MODULES_MAPPING[type(module)]) 209 | else: 210 | handle = module.register_forward_hook(MODULES_MAPPING[type(module)]) 211 | module.__flops_handle__ = handle 212 | seen_types.add(type(module)) 213 | else: 214 | if verbose and not type(module) in (nn.Sequential, nn.ModuleList) and not type(module) in seen_types: 215 | print('Warning: module ' + type(module).__name__ + ' is treated as a zero-op.', file=ost) 216 | seen_types.add(type(module)) 217 | 218 | self.apply(partial(add_flops_counter_hook_function, **kwargs)) 219 | 220 | 221 | def stop_flops_count(self): 222 | """ 223 | A method that will be available after add_flops_counting_methods() is called 224 | on a desired net object. 225 | 226 | Stops computing the mean flops consumption per image. 227 | Call whenever you want to pause the computation. 228 | 229 | """ 230 | remove_batch_counter_hook_function(self) 231 | self.apply(remove_flops_counter_hook_function) 232 | 233 | 234 | def reset_flops_count(self): 235 | """ 236 | A method that will be available after add_flops_counting_methods() is called 237 | on a desired net object. 238 | 239 | Resets statistics computed so far. 240 | 241 | """ 242 | add_batch_counter_variables_or_reset(self) 243 | self.apply(add_flops_counter_variable_or_reset) 244 | 245 | 246 | # ---- Internal functions 247 | def empty_flops_counter_hook(module, input, output): 248 | module.__flops__ += 0 249 | 250 | 251 | def upsample_flops_counter_hook(module, input, output): 252 | output_size = output[0] 253 | batch_size = output_size.shape[0] 254 | output_elements_count = batch_size 255 | for val in output_size.shape[1:]: 256 | output_elements_count *= val 257 | module.__flops__ += int(output_elements_count) 258 | 259 | 260 | def relu_flops_counter_hook(module, input, output): 261 | active_elements_count = output.numel() 262 | module.__flops__ += int(active_elements_count) 263 | 264 | 265 | def linear_flops_counter_hook(module, input, output): 266 | input = input[0] 267 | output_last_dim = output.shape[-1] # pytorch checks dimensions, so here we don't care much 268 | module.__flops__ += int(np.prod(input.shape) * output_last_dim) 269 | 270 | 271 | def pool_flops_counter_hook(module, input, output): 272 | input = input[0] 273 | module.__flops__ += int(np.prod(input.shape)) 274 | 275 | 276 | def bn_flops_counter_hook(module, input, output): 277 | module.affine 278 | input = input[0] 279 | 280 | batch_flops = np.prod(input.shape) 281 | if module.affine: 282 | batch_flops *= 2 283 | module.__flops__ += int(batch_flops) 284 | 285 | 286 | def deconv_flops_counter_hook(conv_module, input, output): 287 | # Can have multiple inputs, getting the first one 288 | input = input[0] 289 | 290 | batch_size = input.shape[0] 291 | input_height, input_width = input.shape[2:] 292 | 293 | kernel_height, kernel_width = conv_module.kernel_size 294 | in_channels = conv_module.in_channels 295 | out_channels = conv_module.out_channels 296 | groups = conv_module.groups 297 | 298 | filters_per_channel = out_channels // groups 299 | conv_per_position_flops = kernel_height * kernel_width * in_channels * filters_per_channel 300 | 301 | active_elements_count = batch_size * input_height * input_width 302 | overall_conv_flops = conv_per_position_flops * active_elements_count 303 | bias_flops = 0 304 | if conv_module.bias is not None: 305 | output_height, output_width = output.shape[2:] 306 | bias_flops = out_channels * batch_size * output_height * output_height 307 | overall_flops = overall_conv_flops + bias_flops 308 | 309 | conv_module.__flops__ += int(overall_flops) 310 | 311 | 312 | def conv_flops_counter_hook(conv_module, input, output): 313 | # Can have multiple inputs, getting the first one 314 | input = input[0] 315 | 316 | batch_size = input.shape[0] 317 | output_dims = list(output.shape[2:]) 318 | 319 | kernel_dims = list(conv_module.kernel_size) 320 | in_channels = conv_module.in_channels 321 | out_channels = conv_module.out_channels 322 | groups = conv_module.groups 323 | 324 | filters_per_channel = out_channels // groups 325 | conv_per_position_flops = int(np.prod(kernel_dims)) * in_channels * filters_per_channel 326 | 327 | active_elements_count = batch_size * int(np.prod(output_dims)) 328 | 329 | overall_conv_flops = conv_per_position_flops * active_elements_count 330 | 331 | bias_flops = 0 332 | 333 | if conv_module.bias is not None: 334 | 335 | bias_flops = out_channels * active_elements_count 336 | 337 | overall_flops = overall_conv_flops + bias_flops 338 | 339 | conv_module.__flops__ += int(overall_flops) 340 | 341 | 342 | def batch_counter_hook(module, input, output): 343 | batch_size = 1 344 | if len(input) > 0: 345 | # Can have multiple inputs, getting the first one 346 | input = input[0] 347 | batch_size = len(input) 348 | else: 349 | pass 350 | print('Warning! No positional inputs found for a module, assuming batch size is 1.') 351 | module.__batch_counter__ += batch_size 352 | 353 | 354 | def rnn_flops(flops, rnn_module, w_ih, w_hh, input_size): 355 | # matrix matrix mult ih state and internal state 356 | flops += w_ih.shape[0]*w_ih.shape[1] 357 | # matrix matrix mult hh state and internal state 358 | flops += w_hh.shape[0]*w_hh.shape[1] 359 | if isinstance(rnn_module, (nn.RNN, nn.RNNCell)): 360 | # add both operations 361 | flops += rnn_module.hidden_size 362 | elif isinstance(rnn_module, (nn.GRU, nn.GRUCell)): 363 | # hadamard of r 364 | flops += rnn_module.hidden_size 365 | # adding operations from both states 366 | flops += rnn_module.hidden_size*3 367 | # last two hadamard product and add 368 | flops += rnn_module.hidden_size*3 369 | elif isinstance(rnn_module, (nn.LSTM, nn.LSTMCell)): 370 | # adding operations from both states 371 | flops += rnn_module.hidden_size*4 372 | # two hadamard product and add for C state 373 | flops += rnn_module.hidden_size + rnn_module.hidden_size + rnn_module.hidden_size 374 | # final hadamard 375 | flops += rnn_module.hidden_size + rnn_module.hidden_size + rnn_module.hidden_size 376 | return flops 377 | 378 | 379 | def rnn_flops_counter_hook(rnn_module, input, output): 380 | """ 381 | Takes into account batch goes at first position, contrary 382 | to pytorch common rule (but actually it doesn't matter). 383 | IF sigmoid and tanh are made hard, only a comparison FLOPS should be accurate 384 | """ 385 | flops = 0 386 | inp = input[0] # input is a tuble containing a sequence to process and (optionally) hidden state 387 | batch_size = inp.shape[0] 388 | seq_length = inp.shape[1] 389 | num_layers = rnn_module.num_layers 390 | 391 | for i in range(num_layers): 392 | w_ih = rnn_module.__getattr__('weight_ih_l' + str(i)) 393 | w_hh = rnn_module.__getattr__('weight_hh_l' + str(i)) 394 | if i == 0: 395 | input_size = rnn_module.input_size 396 | else: 397 | input_size = rnn_module.hidden_size 398 | flops = rnn_flops(flops, rnn_module, w_ih, w_hh, input_size) 399 | if rnn_module.bias: 400 | b_ih = rnn_module.__getattr__('bias_ih_l' + str(i)) 401 | b_hh = rnn_module.__getattr__('bias_hh_l' + str(i)) 402 | flops += b_ih.shape[0] + b_hh.shape[0] 403 | 404 | flops *= batch_size 405 | flops *= seq_length 406 | if rnn_module.bidirectional: 407 | flops *= 2 408 | rnn_module.__flops__ += int(flops) 409 | 410 | 411 | def rnn_cell_flops_counter_hook(rnn_cell_module, input, output): 412 | flops = 0 413 | inp = input[0] 414 | batch_size = inp.shape[0] 415 | w_ih = rnn_cell_module.__getattr__('weight_ih') 416 | w_hh = rnn_cell_module.__getattr__('weight_hh') 417 | input_size = inp.shape[1] 418 | flops = rnn_flops(flops, rnn_cell_module, w_ih, w_hh, input_size) 419 | if rnn_cell_module.bias: 420 | b_ih = rnn_cell_module.__getattr__('bias_ih') 421 | b_hh = rnn_cell_module.__getattr__('bias_hh') 422 | flops += b_ih.shape[0] + b_hh.shape[0] 423 | 424 | flops *= batch_size 425 | rnn_cell_module.__flops__ += int(flops) 426 | 427 | 428 | def add_batch_counter_variables_or_reset(module): 429 | 430 | module.__batch_counter__ = 0 431 | 432 | 433 | def add_batch_counter_hook_function(module): 434 | if hasattr(module, '__batch_counter_handle__'): 435 | return 436 | 437 | handle = module.register_forward_hook(batch_counter_hook) 438 | module.__batch_counter_handle__ = handle 439 | 440 | 441 | def remove_batch_counter_hook_function(module): 442 | if hasattr(module, '__batch_counter_handle__'): 443 | module.__batch_counter_handle__.remove() 444 | del module.__batch_counter_handle__ 445 | 446 | 447 | def add_flops_counter_variable_or_reset(module): 448 | if is_supported_instance(module): 449 | if hasattr(module, '__flops__') or hasattr(module, '__params__'): 450 | print('Warning: variables __flops__ or __params__ are already ' 451 | 'defined for the module' + type(module).__name__ + 452 | ' ptflops can affect your code!') 453 | module.__flops__ = 0 454 | module.__params__ = get_model_parameters_number(module) 455 | 456 | CUSTOM_MODULES_MAPPING = {} 457 | 458 | MODULES_MAPPING = { 459 | # convolutions 460 | nn.Conv1d: conv_flops_counter_hook, 461 | nn.Conv2d: conv_flops_counter_hook, 462 | nn.Conv3d: conv_flops_counter_hook, 463 | # activations 464 | nn.ReLU: relu_flops_counter_hook, 465 | nn.PReLU: relu_flops_counter_hook, 466 | nn.ELU: relu_flops_counter_hook, 467 | nn.LeakyReLU: relu_flops_counter_hook, 468 | nn.ReLU6: relu_flops_counter_hook, 469 | # poolings 470 | nn.MaxPool1d: pool_flops_counter_hook, 471 | nn.AvgPool1d: pool_flops_counter_hook, 472 | nn.AvgPool2d: pool_flops_counter_hook, 473 | nn.MaxPool2d: pool_flops_counter_hook, 474 | nn.MaxPool3d: pool_flops_counter_hook, 475 | nn.AvgPool3d: pool_flops_counter_hook, 476 | nn.AdaptiveMaxPool1d: pool_flops_counter_hook, 477 | nn.AdaptiveAvgPool1d: pool_flops_counter_hook, 478 | nn.AdaptiveMaxPool2d: pool_flops_counter_hook, 479 | nn.AdaptiveAvgPool2d: pool_flops_counter_hook, 480 | nn.AdaptiveMaxPool3d: pool_flops_counter_hook, 481 | nn.AdaptiveAvgPool3d: pool_flops_counter_hook, 482 | # BNs 483 | nn.BatchNorm1d: bn_flops_counter_hook, 484 | nn.BatchNorm2d: bn_flops_counter_hook, 485 | nn.BatchNorm3d: bn_flops_counter_hook, 486 | # FC 487 | nn.Linear: linear_flops_counter_hook, 488 | # Upscale 489 | nn.Upsample: upsample_flops_counter_hook, 490 | # Deconvolution 491 | nn.ConvTranspose2d: deconv_flops_counter_hook, 492 | # RNN 493 | nn.RNN: rnn_flops_counter_hook, 494 | nn.GRU: rnn_flops_counter_hook, 495 | nn.LSTM: rnn_flops_counter_hook, 496 | nn.RNNCell: rnn_cell_flops_counter_hook, 497 | nn.LSTMCell: rnn_cell_flops_counter_hook, 498 | nn.GRUCell: rnn_cell_flops_counter_hook 499 | } 500 | 501 | 502 | def is_supported_instance(module): 503 | if type(module) in MODULES_MAPPING or type(module) in CUSTOM_MODULES_MAPPING: 504 | return True 505 | return False 506 | 507 | 508 | def remove_flops_counter_hook_function(module): 509 | if is_supported_instance(module): 510 | if hasattr(module, '__flops_handle__'): 511 | module.__flops_handle__.remove() 512 | del module.__flops_handle__ 513 | -------------------------------------------------------------------------------- /util/pwlf.py: -------------------------------------------------------------------------------- 1 | # cite: https://github.com/cjekel/piecewise_linear_fit_py/blob/78834466ea42f8d30a1bcbf1d81410011ad43583/pwlf/pwlf.py 2 | 3 | # -- coding: utf-8 -- 4 | # MIT License 5 | # 6 | # Copyright (c) 2017-2019 Charles Jekel 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining a copy 9 | # of this software and associated documentation files (the "Software"), to deal 10 | # in the Software without restriction, including without limitation the rights 11 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | # copies of the Software, and to permit persons to whom the Software is 13 | # furnished to do so, subject to the following conditions: 14 | # 15 | # The above copyright notice and this permission notice shall be included in 16 | # all copies or substantial portions of the Software. 17 | # 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | # SOFTWARE. 25 | 26 | # import libraries 27 | import numpy as np 28 | from scipy.optimize import differential_evolution 29 | from scipy.optimize import fmin_l_bfgs_b 30 | from scipy import linalg 31 | from scipy import stats 32 | from pyDOE import lhs 33 | 34 | # piecewise linear fit library 35 | 36 | 37 | class PiecewiseLinFit(object): 38 | 39 | def __init__(self, x, y, disp_res=False, lapack_driver='gelsd', degree=1, 40 | weights=None): 41 | r""" 42 | An object to fit a continuous piecewise linear function 43 | to data. 44 | 45 | Initiate the library with the supplied x and y data. 46 | Supply the x and y data of which you'll be fitting 47 | a continuous piecewise linear model to where y(x). 48 | by default pwlf won't print the optimization results.; 49 | 50 | Parameters 51 | ---------- 52 | x : array_like 53 | The x or independent data point locations as list or 1 dimensional 54 | numpy array. 55 | y : array_like 56 | The y or dependent data point locations as list or 1 dimensional 57 | numpy array. 58 | disp_res : bool, optional 59 | Whether the optimization results should be printed. Default is 60 | False. 61 | lapack_driver : str, optional 62 | Which LAPACK driver is used to solve the least-squares problem. 63 | Default lapack_driver='gelsd'. Options are 'gelsd', 'gelsy', 64 | 'gelss'. For more see 65 | https://docs.scipy.org/doc/scipy/reference/generated/scipy.linalg.lstsq.html 66 | http://www.netlib.org/lapack/lug/node27.html 67 | degree : int, optional 68 | The degree of polynomial to use. The default is degree=1 for 69 | linear models. Use degree=0 for constant models. 70 | weights : None, or array_like 71 | The individual weights are typically the reciprocal of the 72 | standard deviation for each data point, where weights[i] 73 | corresponds to one over the standard deviation of the ith data 74 | point. Default weights=None. 75 | 76 | Attributes 77 | ---------- 78 | beta : ndarray (1-D) 79 | The model parameters for the continuous piecewise linear fit. 80 | break_0 : float 81 | The smallest x value. 82 | break_n : float 83 | The largest x value. 84 | c_n : int 85 | The number of constraint points. This is the same as len(x_c). 86 | degree: int 87 | The degree of polynomial to use. The default is degree=1 for 88 | linear models. Use degree=0 for constant models. 89 | fit_breaks : ndarray (1-D) 90 | breakpoint locations stored as a 1-D numpy array. 91 | intercepts : ndarray (1-D) 92 | The y-intercept of each line segment as a 1-D numpy array. 93 | lapack_driver : str 94 | Which LAPACK driver is used to solve the least-squares problem. 95 | print : bool 96 | Whether the optimization results should be printed. Default is 97 | False. 98 | n_data : int 99 | The number of data points. 100 | n_parameters : int 101 | The number of model parameters. This is equivalent to the 102 | len(beta). 103 | n_segments : int 104 | The number of line segments. 105 | nVar : int 106 | The number of variables in the global optimization problem. 107 | se : ndarray (1-D) 108 | Standard errors associated with each beta parameter. Specifically 109 | se[0] correspounds to the standard error for beta[0], and so forth. 110 | slopes : ndarray (1-D) 111 | The slope of each ling segment as a 1-D numpy array. This assumes 112 | that x[0] <= x[1] <= ... <= x[n]. Thus, slopes[0] is the slope 113 | of the first line segment. 114 | ssr : float 115 | Optimal sum of square error. 116 | x_c : ndarray (1-D) 117 | The x locations of the data points that the piecewise linear 118 | function will be forced to go through. 119 | y_c : ndarray (1-D) 120 | The x locations of the data points that the piecewise linear 121 | function will be forced to go through. 122 | x_data : ndarray (1-D) 123 | The inputted parameter x from the 1-D data set. 124 | y_data : ndarray (1-D) 125 | The inputted parameter y from the 1-D data set. 126 | y_w : ndarray (1-D) 127 | The weighted y data vector. 128 | zeta : ndarray (1-D) 129 | The model parameters associated with the constraint function. 130 | 131 | Methods 132 | ------- 133 | fit(n_segments, x_c=None, y_c=None, **kwargs) 134 | Fit a continuous piecewise linear function for a specified number 135 | of line segments. 136 | fitfast(n_segments, pop=2, **kwargs) 137 | Fit a continuous piecewise linear function for a specified number 138 | of line segments using a specialized optimization routine that 139 | should be faster than fit() for large problems. The tradeoff may 140 | be that fitfast() results in a lower quality model. 141 | fit_with_breaks(breaks) 142 | Fit a continuous piecewise linear function where the breakpoint 143 | locations are known. 144 | fit_with_breaks_force_points(breaks, x_c, y_c) 145 | Fit a continuous piecewise linear function where the breakpoint 146 | locations are known, and force the fit to go through points at x_c 147 | and y_c. 148 | predict(x, beta=None, breaks=None) 149 | Evaluate the continuous piecewise linear function at new untested 150 | points. 151 | fit_with_breaks_opt(var) 152 | The objective function to perform a continuous piecewise linear 153 | fit for a specified number of breakpoints. This is to be used 154 | with a custom optimization routine, and after use_custom_opt has 155 | been called. 156 | fit_force_points_opt(var)' 157 | Same as fit_with_breaks_opt(var), except this allows for points to 158 | be forced through x_c and y_c. 159 | use_custom_opt(n_segments, x_c=None, y_c=None) 160 | Function to initialize the attributes necessary to use a custom 161 | optimization routine. Must be used prior to calling 162 | fit_with_breaks_opt() or fit_force_points_opt(). 163 | calc_slopes() 164 | Calculate the slopes of the lines after a piecewise linear 165 | function has been fitted. 166 | standard_errors() 167 | Calculate the standard error of each model parameter in the fitted 168 | piecewise linear function. Note, this assumes no uncertainty in 169 | breakpoint locations. 170 | prediction_variance(x) 171 | Calculate the prediction variance at x locations for the fitted 172 | piecewise linear function. Note, assumes no uncertainty in break 173 | point locations. 174 | r_squared() 175 | Calculate the coefficient of determination, or 'R-squared' value 176 | for a fitted piecewise linear function. 177 | 178 | Examples 179 | -------- 180 | Initialize for x, y data 181 | 182 | >>> import pwlf 183 | >>> my_pwlf = pwlf.PiecewiseLinFit(x, y) 184 | 185 | Initialize for x,y data and print optimization results 186 | 187 | >>> my_pwlf = pwlf.PiecewiseLinFit(x, y, disp_res=True) 188 | 189 | """ 190 | 191 | self.print = disp_res 192 | self.lapack_driver = lapack_driver 193 | x, y = self._switch_to_np_array(x), self._switch_to_np_array(y) 194 | 195 | self.x_data, self.y_data = x, y 196 | 197 | # calculate the number of data points 198 | self.n_data = x.size 199 | 200 | # set the first and last break x values to be the min and max of x 201 | self.break_0, self.break_n = np.min(self.x_data), np.max(self.x_data) 202 | 203 | if 12 > degree >= 0: 204 | # I actually don't know what the upper degree limit should 205 | self.degree = int(degree) 206 | else: 207 | raise ValueError(f"degree = {degree} is not supported.") 208 | 209 | self.y_w = None 210 | self.weights = None 211 | # self.weights2 = None # the squared weights vector 212 | if weights is not None: 213 | self.weights = self._switch_to_np_array(weights) 214 | # self.weights2 = weights*weights 215 | self.y_w = np.dot(self.y_data, np.eye(self.n_data) * self.weights) 216 | 217 | # initialize all empty attributes as None 218 | self.fit_breaks = None 219 | self.n_segments = None 220 | self.n_parameters = None 221 | self.beta = None 222 | self.ssr = None 223 | self.x_c = None 224 | self.y_c = None 225 | self.c_n = None 226 | self.zeta = None 227 | self.nVar = None 228 | self.slopes = None 229 | self.intercepts = None 230 | self.se = None 231 | 232 | @staticmethod 233 | def _switch_to_np_array(input_): 234 | r""" 235 | Check the input, if it's not a Numpy array transform it to one. 236 | 237 | Parameters 238 | ---------- 239 | input_ : array_like 240 | The object that requires a check. 241 | 242 | Returns 243 | ------- 244 | input_ : ndarray 245 | The input data that's been transformed when required. 246 | """ 247 | if isinstance(input_, np.ndarray) is False: 248 | input_ = np.array(input_) 249 | return input_ 250 | 251 | def _get_breaks(self, input_): 252 | r""" 253 | Use input to form a ndarray containing breakpoints 254 | 255 | Parameters 256 | ---------- 257 | input_ : array_like 258 | The object containing some of the breakpoints 259 | 260 | Returns 261 | ------- 262 | b_ : ndarray 263 | All the line breaks 264 | """ 265 | v = np.sort(input_) 266 | b_ = np.zeros(len(v) + 2) 267 | b_[0], b_[1:-1], b_[-1] = self.break_0, v.copy(), self.break_n 268 | return b_ 269 | 270 | def assemble_regression_matrix(self, breaks, x): 271 | r""" 272 | Assemble the linear regression matrix A 273 | 274 | Parameters 275 | ---------- 276 | breaks : array_like 277 | The x locations where each line segment terminates. These are 278 | referred to as breakpoints for each line segment. This should be 279 | structured as a 1-D numpy array. 280 | x : ndarray (1-D) 281 | The x locations which the linear regression matrix is assembled on. 282 | This must be a numpy array! 283 | 284 | Returns 285 | ------- 286 | A : ndarray (2-D) 287 | The assembled linear regression matrix. 288 | 289 | Examples 290 | -------- 291 | Assemble the linear regression matrix on the x data for some set of 292 | breakpoints. 293 | 294 | >>> import pwlf 295 | >>> my_pwlf = pwlf.PiecewiseLinFit(x, y) 296 | >>> breaks = [0.0, 0.5, 1.0] 297 | >>> A = assemble_regression_matrix(breaks, self.x_data) 298 | 299 | """ 300 | 301 | breaks = self._switch_to_np_array(breaks) 302 | 303 | # Sort the breaks, then store them 304 | breaks_order = np.argsort(breaks) 305 | self.fit_breaks = breaks[breaks_order] 306 | # store the number of parameters and line segments 307 | self.n_segments = len(breaks) - 1 308 | 309 | # Assemble the regression matrix 310 | A_list = [np.ones_like(x)] 311 | if self.degree >= 1: 312 | A_list.append(x - self.fit_breaks[0]) 313 | for i in range(self.n_segments - 1): 314 | A_list.append(np.where(x > self.fit_breaks[i+1], 315 | x - self.fit_breaks[i+1], 316 | 0.0)) 317 | if self.degree >= 2: 318 | for k in range(2, self.degree + 1): 319 | A_list.append((x - self.fit_breaks[0])**k) 320 | for i in range(self.n_segments - 1): 321 | A_list.append(np.where(x > self.fit_breaks[i+1], 322 | (x - self.fit_breaks[i+1])**k, 323 | 0.0)) 324 | else: 325 | A_list = [np.ones_like(x)] 326 | for i in range(self.n_segments - 1): 327 | A_list.append(np.where(x > self.fit_breaks[i+1], 1.0, 0.0)) 328 | A = np.vstack(A_list).T 329 | self.n_parameters = A.shape[1] 330 | return A 331 | 332 | def fit_with_breaks(self, breaks): 333 | r""" 334 | A function which fits a continuous piecewise linear function 335 | for specified breakpoint locations. 336 | 337 | The function minimizes the sum of the square of the residuals for the 338 | x y data. 339 | 340 | If you want to understand the math behind this read 341 | https://jekel.me/2018/Continous-piecewise-linear-regression/ 342 | 343 | Other useful resources: 344 | http://golovchenko.org/docs/ContinuousPiecewiseLinearFit.pdf 345 | https://www.mathworks.com/matlabcentral/fileexchange/40913-piecewise-linear-least-square-fittic 346 | http://www.regressionist.com/2018/02/07/continuous-piecewise-linear-fitting/ 347 | 348 | Parameters 349 | ---------- 350 | breaks : array_like 351 | The x locations where each line segment terminates. These are 352 | referred to as breakpoints for each line segment. This should be 353 | structured as a 1-D numpy array. 354 | 355 | Returns 356 | ------- 357 | ssr : float 358 | Returns the sum of squares of the residuals. 359 | 360 | Raises 361 | ------ 362 | LinAlgError 363 | This typically means your regression problem is ill-conditioned. 364 | 365 | Examples 366 | -------- 367 | If your x data exists from 0 <= x <= 1 and you want three 368 | piecewise linear lines where the lines terminate at x = 0.0, 0.3, 0.6, 369 | and 1.0. This assumes that x is linearly spaced from [0, 1), and y is 370 | random. 371 | 372 | >>> import pwlf 373 | >>> x = np.linspace(0.0, 1.0, 10) 374 | >>> y = np.random.random(10) 375 | >>> my_pwlf = pwlf.PiecewiseLinFit(x, y) 376 | >>> breaks = [0.0, 0.3, 0.6, 1.0] 377 | >>> ssr = my_pwlf.fit_with_breaks(breaks) 378 | 379 | """ 380 | 381 | breaks = self._switch_to_np_array(breaks) 382 | 383 | A = self.assemble_regression_matrix(breaks, self.x_data) 384 | 385 | # try to solve the regression problem 386 | try: 387 | ssr = self.lstsq(A) 388 | 389 | except linalg.LinAlgError: 390 | # the computation could not converge! 391 | # on an error, return ssr = np.print_function 392 | # You might have a singular Matrix!!! 393 | ssr = np.inf 394 | if ssr is None: 395 | ssr = np.inf 396 | # something went wrong... 397 | self.ssr = ssr 398 | return ssr 399 | 400 | def fit_with_breaks_force_points(self, breaks, x_c, y_c): 401 | r""" 402 | A function which fits a continuous piecewise linear function 403 | for specified breakpoint locations, where you force the 404 | fit to go through the data points at x_c and y_c. 405 | 406 | The function minimizes the sum of the square of the residuals for the 407 | pair of x, y data points. If you want to understand the math behind 408 | this read https://jekel.me/2018/Force-piecwise-linear-fit-through-data/ 409 | 410 | Parameters 411 | ---------- 412 | breaks : array_like 413 | The x locations where each line segment terminates. These are 414 | referred to as breakpoints for each line segment. This should be 415 | structured as a 1-D numpy array. 416 | x_c : array_like 417 | The x locations of the data points that the piecewise linear 418 | function will be forced to go through. 419 | y_c : array_like 420 | The x locations of the data points that the piecewise linear 421 | function will be forced to go through. 422 | 423 | Returns 424 | ------- 425 | L : float 426 | Returns the Lagrangian function value. This is the sum of squares 427 | of the residuals plus the constraint penalty. 428 | 429 | Raises 430 | ------ 431 | LinAlgError 432 | This typically means your regression problem is ill-conditioned. 433 | ValueError 434 | You can't specify weights with x_c and y_c. 435 | 436 | Examples 437 | -------- 438 | If your x data exists from 0 <= x <= 1 and you want three 439 | piecewise linear lines where the lines terminate at x = 0.0, 0.3, 0.6, 440 | and 1.0. This assumes that x is linearly spaced from [0, 1), and y is 441 | random. Additionally you desired that the piecewise linear function go 442 | through the point (0.0, 0.0) 443 | 444 | >>> import pwlf 445 | >>> x = np.linspace(0.0, 1.0, 10) 446 | >>> y = np.random.random(10) 447 | >>> x_c = [0.0] 448 | >>> y_c = [0.0] 449 | >>> my_pwlf = pwlf.PiecewiseLinFit(x, y) 450 | >>> breaks = [0.0, 0.3, 0.6, 1.0] 451 | >>> L = my_pwlf.fit_with_breaks_force_points(breaks, x_c, y_c) 452 | 453 | """ 454 | 455 | x_c, y_c = self._switch_to_np_array(x_c), self._switch_to_np_array(y_c) 456 | # sort the x_c and y_c data points, then store them 457 | x_c_order = np.argsort(x_c) 458 | self.x_c, self.y_c = x_c[x_c_order], y_c[x_c_order] 459 | # store the number of constraints 460 | self.c_n = len(self.x_c) 461 | 462 | if self.weights is not None: 463 | raise ValueError('Constrained least squares with weights are' 464 | ' not supported since these have a tendency ' 465 | 'of being numerically instable.') 466 | 467 | breaks = self._switch_to_np_array(breaks) 468 | 469 | A = self.assemble_regression_matrix(breaks, self.x_data) 470 | L = self.conlstsq(A) 471 | return L 472 | 473 | def predict(self, x, beta=None, breaks=None): 474 | r""" 475 | Evaluate the fitted continuous piecewise linear function at untested 476 | points. 477 | 478 | You can manfully specify the breakpoints and calculated 479 | values for beta if you want to quickly predict from different models 480 | and the same data set. 481 | 482 | Parameters 483 | ---------- 484 | x : array_like 485 | The x locations where you want to predict the output of the fitted 486 | continuous piecewise linear function. 487 | beta : none or ndarray (1-D), optional 488 | The model parameters for the continuous piecewise linear fit. 489 | Default is None. 490 | breaks : none or array_like, optional 491 | The x locations where each line segment terminates. These are 492 | referred to as breakpoints for each line segment. This should be 493 | structured as a 1-D numpy array. Default is None. 494 | 495 | Returns 496 | ------- 497 | y_hat : ndarray (1-D) 498 | The predicted values at x. 499 | 500 | Examples 501 | -------- 502 | Fits a simple model, then predict at x_new locations which are 503 | linearly spaced. 504 | 505 | >>> import pwlf 506 | >>> x = np.linspace(0.0, 1.0, 10) 507 | >>> y = np.random.random(10) 508 | >>> my_pwlf = pwlf.PiecewiseLinFit(x, y) 509 | >>> breaks = [0.0, 0.3, 0.6, 1.0] 510 | >>> ssr = my_pwlf.fit_with_breaks(breaks) 511 | >>> x_new = np.linspace(0.0, 1.0, 100) 512 | >>> yhat = my_pwlf.predict(x_new) 513 | 514 | """ 515 | if beta is not None and breaks is not None: 516 | self.beta = beta 517 | # Sort the breaks, then store them 518 | breaks_order = np.argsort(breaks) 519 | self.fit_breaks = breaks[breaks_order] 520 | self.n_parameters = len(self.fit_breaks) 521 | self.n_segments = self.n_parameters - 1 522 | 523 | x = self._switch_to_np_array(x) 524 | 525 | A = self.assemble_regression_matrix(self.fit_breaks, x) 526 | 527 | # solve the regression problem 528 | y_hat = np.dot(A, self.beta) 529 | return y_hat 530 | 531 | def fit_with_breaks_opt(self, var): 532 | r""" 533 | The objective function to perform a continuous piecewise linear 534 | fit for a specified number of breakpoints. This is to be used 535 | with a custom optimization routine, and after use_custom_opt has 536 | been called. 537 | 538 | This was intended for advanced users only. 539 | 540 | See the following example 541 | https://github.com/cjekel/piecewise_linear_fit_py/blob/master/examples/useCustomOptimizationRoutine.py 542 | 543 | Parameters 544 | ---------- 545 | var : array_like 546 | The breakpoint locations, or variable, in a custom 547 | optimization routine. 548 | 549 | Returns 550 | ------- 551 | ssr : float 552 | The sum of square of the residuals. 553 | 554 | Raises 555 | ------ 556 | LinAlgError 557 | This typically means your regression problem is ill-conditioned. 558 | 559 | Notes 560 | ----- 561 | You should run use_custom_opt to initialize necessary object 562 | attributes first. 563 | 564 | Unlike fit_with_breaks, fit_with_breaks_opt automatically 565 | assumes that the first and last breakpoints occur at the min and max 566 | values of x. 567 | """ 568 | 569 | breaks = self._get_breaks(input_=var) 570 | A = self.assemble_regression_matrix(breaks, self.x_data) 571 | 572 | # try to solve the regression problem 573 | try: 574 | # least squares solver 575 | ssr = self.lstsq(A) 576 | 577 | except linalg.LinAlgError: 578 | # the computation could not converge! 579 | # on an error, return ssr = np.inf 580 | # You might have a singular Matrix!!! 581 | ssr = np.inf 582 | if ssr is None: 583 | ssr = np.inf 584 | # something went wrong... 585 | return ssr 586 | 587 | def fit_force_points_opt(self, var): 588 | r""" 589 | The objective function to perform a continuous piecewise linear 590 | fit for a specified number of breakpoints. This is to be used 591 | with a custom optimization routine, and after use_custom_opt has 592 | been called. 593 | 594 | Use this function if you intend to be force the model through 595 | x_c and y_c, while performing a custom optimization. 596 | 597 | This was intended for advanced users only. 598 | See the following example 599 | https://github.com/cjekel/piecewise_linear_fit_py/blob/master/examples/useCustomOptimizationRoutine.py 600 | 601 | Parameters 602 | ---------- 603 | var : array_like 604 | The breakpoint locations, or variable, in a custom 605 | optimization routine. 606 | 607 | Returns 608 | ------- 609 | ssr : float 610 | The sum of square of the residuals. 611 | 612 | Raises 613 | ------ 614 | LinAlgError 615 | This typically means your regression problem is ill-conditioned. 616 | 617 | Notes 618 | ----- 619 | You should run use_custom_opt to initialize necessary object 620 | attributes first. 621 | 622 | Unlike fit_with_breaks_force_points, fit_force_points_opt 623 | automatically assumes that the first and last breakpoints occur 624 | at the min and max values of x. 625 | """ 626 | 627 | breaks = self._get_breaks(input_=var) 628 | # Sort the breaks, then store them 629 | breaks_order = np.argsort(breaks) 630 | breaks = breaks[breaks_order] 631 | 632 | A = self.assemble_regression_matrix(breaks, self.x_data) 633 | L = self.conlstsq(A) 634 | return L 635 | 636 | def fit(self, n_segments, x_c=None, y_c=None, bounds=None, **kwargs): 637 | r""" 638 | Fit a continuous piecewise linear function for a specified number 639 | of line segments. Uses differential evolution to finds the optimum 640 | location of breakpoints for a given number of line segments by 641 | minimizing the sum of the square error. 642 | 643 | Parameters 644 | ---------- 645 | n_segments : int 646 | The desired number of line segments. 647 | x_c : array_like, optional 648 | The x locations of the data points that the piecewise linear 649 | function will be forced to go through. 650 | y_c : array_like, optional 651 | The x locations of the data points that the piecewise linear 652 | function will be forced to go through. 653 | bounds : array_like, optional 654 | Bounds for each breakpoint location within the optimization. This 655 | should have the shape of (n_segments, 2). 656 | **kwargs : optional 657 | Directly passed into scipy.optimize.differential_evolution(). This 658 | will override any pwlf defaults when provided. See Note for more 659 | information. 660 | 661 | Returns 662 | ------- 663 | fit_breaks : float 664 | breakpoint locations stored as a 1-D numpy array. 665 | 666 | Raises 667 | ------ 668 | ValueError 669 | You probably provided x_c without y_c (or vice versa). 670 | You must provide both x_c and y_c if you plan to force 671 | the model through data point(s). 672 | ValueError 673 | You can't specify weights with x_c and y_c. 674 | 675 | Notes 676 | ----- 677 | All **kwargs are passed into sicpy.optimize.differential_evolution. 678 | If any **kwargs is used, it will override my differential_evolution, 679 | defaults. This allows advanced users to tweak their own optimization. 680 | For me information see: 681 | https://github.com/cjekel/piecewise_linear_fit_py/issues/15#issuecomment-434717232 682 | 683 | Examples 684 | -------- 685 | This example shows you how to fit three continuous piecewise lines to 686 | a dataset. This assumes that x is linearly spaced from [0, 1), and y is 687 | random. 688 | 689 | >>> import pwlf 690 | >>> x = np.linspace(0.0, 1.0, 10) 691 | >>> y = np.random.random(10) 692 | >>> my_pwlf = pwlf.PiecewiseLinFit(x, y) 693 | >>> breaks = my_pwlf.fit(3) 694 | 695 | Additionally you desired that the piecewise linear function go 696 | through the point (0.0, 0.0). 697 | 698 | >>> x_c = [0.0] 699 | >>> y_c = [0.0] 700 | >>> breaks = my_pwlf.fit(3, x_c=x_c, y_c=y_c) 701 | 702 | Additionally you desired that the piecewise linear function go 703 | through the points (0.0, 0.0) and (1.0, 1.0). 704 | 705 | >>> x_c = [0.0, 1.0] 706 | >>> y_c = [0.0, 1.0] 707 | >>> breaks = my_pwlf.fit(3, x_c=x_c, y_c=y_c) 708 | 709 | """ 710 | 711 | # check to see if you've provided just x_c or y_c 712 | logic1 = x_c is not None and y_c is None 713 | logic2 = y_c is not None and x_c is None 714 | if logic1 or logic2: 715 | raise ValueError('You must provide both x_c and y_c!') 716 | 717 | # set the function to minimize 718 | min_function = self.fit_with_breaks_opt 719 | 720 | # if you've provided both x_c and y_c 721 | if x_c is not None and y_c is not None: 722 | x_c = self._switch_to_np_array(x_c) 723 | y_c = self._switch_to_np_array(y_c) 724 | # sort the x_c and y_c data points, then store them 725 | x_c_order = np.argsort(x_c) 726 | self.x_c, self.y_c = x_c[x_c_order], y_c[x_c_order] 727 | # store the number of constraints 728 | self.c_n = len(self.x_c) 729 | # Use a different function to minimize 730 | min_function = self.fit_force_points_opt 731 | if self.weights is not None: 732 | raise ValueError('Constrained least squares with weights are' 733 | ' not supported since these have a tendency ' 734 | 'of being numerically instable.') 735 | 736 | # store the number of line segments and number of parameters 737 | self.n_segments = int(n_segments) 738 | self.n_parameters = self.n_segments + 1 739 | 740 | # calculate the number of variables I have to solve for 741 | self.nVar = self.n_segments - 1 742 | 743 | # initiate the bounds of the optimization 744 | if bounds is None: 745 | bounds = np.zeros([self.nVar, 2]) 746 | bounds[:, 0] = self.break_0 747 | bounds[:, 1] = self.break_n 748 | 749 | # run the optimization 750 | if len(kwargs) == 0: 751 | res = differential_evolution(min_function, bounds, 752 | strategy='best1bin', maxiter=1000, 753 | popsize=50, tol=1e-3, 754 | mutation=(0.5, 1), recombination=0.7, 755 | seed=None, callback=None, disp=False, 756 | polish=True, init='latinhypercube', 757 | atol=1e-4) 758 | else: 759 | res = differential_evolution(min_function, 760 | bounds, **kwargs) 761 | if self.print is True: 762 | print(res) 763 | 764 | self.ssr = res.fun 765 | 766 | breaks = self._get_breaks(input_=res.x) 767 | 768 | # assign values 769 | if x_c is None and y_c is None: 770 | self.fit_with_breaks(breaks) 771 | else: 772 | self.fit_with_breaks_force_points(breaks, self.x_c, self.y_c) 773 | 774 | return self.fit_breaks 775 | 776 | def fitfast(self, n_segments, pop=2, bounds=None, **kwargs): 777 | r""" 778 | Uses multi start LBFGSB optimization to find the location of 779 | breakpoints for a given number of line segments by minimizing the sum 780 | of the square of the errors. 781 | 782 | The idea is that we generate n random latin hypercube samples 783 | and run LBFGSB optimization on each one. This isn't guaranteed to 784 | find the global optimum. It's suppose to be a reasonable compromise 785 | between speed and quality of fit. Let me know how it works. 786 | 787 | Since this is based on random sampling, you might want to run it 788 | multiple times and save the best version... The best version will 789 | have the lowest self.ssr (sum of square of residuals). 790 | 791 | There is no guarantee that this will be faster than fit(), however 792 | you may find it much faster sometimes. 793 | 794 | Parameters 795 | ---------- 796 | n_segments : int 797 | The desired number of line segments. 798 | pop : int, optional 799 | The number of latin hypercube samples to generate. Default pop=2. 800 | bounds : array_like, optional 801 | Bounds for each breakpoint location within the optimization. This 802 | should have the shape of (n_segments, 2). 803 | **kwargs : optional 804 | Directly passed into scipy.optimize.fmin_l_bfgs_b(). This 805 | will override any pwlf defaults when provided. See Note for more 806 | information. 807 | 808 | Returns 809 | ------- 810 | fit_breaks : float 811 | breakpoint locations stored as a 1-D numpy array. 812 | 813 | Notes 814 | ----- 815 | The default number of multi start optimizations is 2. 816 | - Decreasing this number will result in a faster run time. 817 | - Increasing this number will improve the likelihood of finding 818 | good results 819 | - You can specify the number of starts using the following call 820 | - Minimum value of pop is 2 821 | 822 | All **kwargs are passed into sicpy.optimize.fmin_l_bfgs_b. If any 823 | **kwargs is used, it will override my defaults. This allows 824 | advanced users to tweak their own optimization. For me information see: 825 | https://github.com/cjekel/piecewise_linear_fit_py/issues/15#issuecomment-434717232 826 | 827 | Examples 828 | -------- 829 | This example shows you how to fit three continuous piecewise lines to 830 | a dataset. This assumes that x is linearly spaced from [0, 1), and y is 831 | random. 832 | 833 | >>> import pwlf 834 | >>> x = np.linspace(0.0, 1.0, 10) 835 | >>> y = np.random.random(10) 836 | >>> my_pwlf = pwlf.PiecewiseLinFit(x, y) 837 | >>> breaks = my_pwlf.fitfast(3) 838 | 839 | You can change the number of latin hypercube samples (or starting 840 | point, locations) to use with pop. The following example will use 50 841 | samples. 842 | 843 | >>> breaks = my_pwlf.fitfast(3, pop=50) 844 | 845 | """ 846 | pop = int(pop) # ensure that the population is integer 847 | 848 | self.n_segments = int(n_segments) 849 | self.n_parameters = self.n_segments + 1 850 | 851 | # calculate the number of variables I have to solve for 852 | self.nVar = self.n_segments - 1 853 | 854 | # initiate the bounds of the optimization 855 | if bounds is None: 856 | bounds = np.zeros([self.nVar, 2]) 857 | bounds[:, 0] = self.break_0 858 | bounds[:, 1] = self.break_n 859 | 860 | # perform latin hypercube sampling 861 | mypop = lhs(self.nVar, samples=pop, criterion='maximin') 862 | # scale the sampling to my variable range 863 | mypop = mypop * (self.break_n - self.break_0) + self.break_0 864 | 865 | x = np.zeros((pop, self.nVar)) 866 | f = np.zeros(pop) 867 | d = [] 868 | 869 | for i, x0 in enumerate(mypop): 870 | if len(kwargs) == 0: 871 | resx, resf, resd = fmin_l_bfgs_b(self.fit_with_breaks_opt, x0, 872 | fprime=None, args=(), 873 | approx_grad=True, 874 | bounds=bounds, m=10, 875 | factr=1e2, pgtol=1e-05, 876 | epsilon=1e-08, iprint=-1, 877 | maxfun=15000, maxiter=15000, 878 | disp=None, callback=None) 879 | else: 880 | resx, resf, resd = fmin_l_bfgs_b(self.fit_with_breaks_opt, x0, 881 | fprime=None, approx_grad=True, 882 | bounds=bounds, **kwargs) 883 | x[i, :] = resx 884 | f[i] = resf 885 | d.append(resd) 886 | if self.print is True: 887 | print(f"{i + 1} of {pop} complete") 888 | 889 | # find the best result 890 | best_ind = np.nanargmin(f) 891 | best_val = f[best_ind] 892 | best_break = x[best_ind] 893 | res = (x[best_ind], f[best_ind], d[best_ind]) 894 | if self.print is True: 895 | print(res) 896 | 897 | self.ssr = best_val 898 | 899 | breaks = self._get_breaks(input_=best_break) 900 | 901 | # assign parameters 902 | self.fit_with_breaks(breaks) 903 | 904 | return self.fit_breaks 905 | 906 | def fit_guess(self, guess_breakpoints, bounds=None, **kwargs): 907 | r""" 908 | Uses L-BFGS-B optimization to find the location of breakpoints 909 | from a guess of where breakpoint locations should be. 910 | 911 | In some cases you may have a good idea where the breakpoint locations 912 | occur. It generally won't be necessary to run a full global 913 | optimization to search the entire domain for the breakpoints when you 914 | have a good idea where the breakpoints occur. Here a local optimization 915 | is run from a guess of the breakpoint locations. 916 | 917 | Parameters 918 | ---------- 919 | guess_breakpoints : array_like 920 | Guess where the breakpoints occur. This should be a list or numpy 921 | array containing the locations where it appears breakpoints occur. 922 | bounds : array_like, optional 923 | Bounds for each breakpoint location within the optimization. This 924 | should have the shape of (n_segments, 2). 925 | **kwargs : optional 926 | Directly passed into scipy.optimize.fmin_l_bfgs_b(). This 927 | will override any pwlf defaults when provided. See Note for more 928 | information. 929 | 930 | Returns 931 | ------- 932 | fit_breaks : float 933 | breakpoint locations stored as a 1-D numpy array. 934 | 935 | Notes 936 | ----- 937 | All **kwargs are passed into sicpy.optimize.fmin_l_bfgs_b. If any 938 | **kwargs is used, it will override my defaults. This allows 939 | advanced users to tweak their own optimization. For me information see: 940 | https://github.com/cjekel/piecewise_linear_fit_py/issues/15#issuecomment-434717232 941 | 942 | You do not need to specify the x.min() or x.max() in geuss_breakpoints! 943 | 944 | Examples 945 | -------- 946 | In this example we see two distinct linear regions, and we believe a 947 | breakpoint occurs at 6.0. We'll use the fit_guess() function to find 948 | the best breakpoint location starting with this guess. 949 | 950 | >>> import pwlf 951 | >>> x = np.array([4., 5., 6., 7., 8.]) 952 | >>> y = np.array([11., 13., 16., 28.92, 42.81]) 953 | >>> my_pwlf = pwlf.PiecewiseLinFit(x, y) 954 | >>> breaks = my_pwlf.fit_guess([6.0]) 955 | 956 | Note specifying one breakpoint will result in two line segments. 957 | If we wanted three line segments, we'll have to specify two 958 | breakpoints. 959 | 960 | >>> breaks = my_pwlf.fit_guess([5.5, 6.0]) 961 | 962 | """ 963 | # calculate the number of variables I have to solve for 964 | self.nVar = len(guess_breakpoints) 965 | self.n_segments = self.nVar + 1 966 | self.n_parameters = self.n_segments + 1 967 | 968 | # initiate the bounds of the optimization 969 | if bounds is None: 970 | bounds = np.zeros([self.nVar, 2]) 971 | bounds[:, 0] = self.break_0 972 | bounds[:, 1] = self.break_n 973 | 974 | if len(kwargs) == 0: 975 | resx, resf, _ = fmin_l_bfgs_b(self.fit_with_breaks_opt, 976 | guess_breakpoints, 977 | fprime=None, args=(), 978 | approx_grad=True, 979 | bounds=bounds, m=10, 980 | factr=1e2, pgtol=1e-05, 981 | epsilon=1e-08, iprint=-1, 982 | maxfun=15000, maxiter=15000, 983 | disp=None, callback=None) 984 | else: 985 | resx, resf, _ = fmin_l_bfgs_b(self.fit_with_breaks_opt, 986 | guess_breakpoints, 987 | fprime=None, approx_grad=True, 988 | bounds=bounds, **kwargs) 989 | 990 | self.ssr = resf 991 | 992 | breaks = self._get_breaks(input_=resx) 993 | 994 | # assign values 995 | self.fit_with_breaks(breaks) 996 | 997 | return self.fit_breaks 998 | 999 | def use_custom_opt(self, n_segments, x_c=None, y_c=None): 1000 | r""" 1001 | Provide the number of line segments you want to use with your 1002 | custom optimization routine. 1003 | 1004 | Run this function first to initialize necessary attributes!!! 1005 | 1006 | This was intended for advanced users only. 1007 | 1008 | See the following example 1009 | https://github.com/cjekel/piecewise_linear_fit_py/blob/master/examples/useCustomOptimizationRoutine.py 1010 | 1011 | Parameters 1012 | ---------- 1013 | n_segments : int 1014 | The x locations where each line segment terminates. These are 1015 | referred to as breakpoints for each line segment. This should be 1016 | structured as a 1-D numpy array. 1017 | x_c : none or array_like, optional 1018 | The x locations of the data points that the piecewise linear 1019 | function will be forced to go through. 1020 | y_c : none or array_like, optional 1021 | The x locations of the data points that the piecewise linear 1022 | function will be forced to go through. 1023 | 1024 | Notes 1025 | ----- 1026 | Optimize fit_with_breaks_opt(var) where var is a 1D array 1027 | containing the x locations of your variables 1028 | var has length n_segments - 1, because the two breakpoints 1029 | are always defined (1. the min of x, 2. the max of x). 1030 | 1031 | fit_with_breaks_opt(var) will return the sum of the square of the 1032 | residuals which you'll want to minimize with your optimization 1033 | routine. 1034 | 1035 | Raises 1036 | ------ 1037 | ValueError 1038 | You can't specify weights with x_c and y_c. 1039 | 1040 | """ 1041 | 1042 | self.n_segments = int(n_segments) 1043 | self.n_parameters = self.n_segments + 1 1044 | 1045 | # calculate the number of variables I have to solve for 1046 | self.nVar = self.n_segments - 1 1047 | if x_c is not None or y_c is not None: 1048 | x_c = self._switch_to_np_array(x_c) 1049 | y_c = self._switch_to_np_array(y_c) 1050 | # sort the x_c and y_c data points, then store them 1051 | x_c_order = np.argsort(x_c) 1052 | self.x_c, self.y_c = x_c[x_c_order], y_c[x_c_order] 1053 | # store the number of constraints 1054 | self.c_n = len(self.x_c) 1055 | if self.weights is not None: 1056 | raise ValueError('Constrained least squares with weights are' 1057 | ' not supported since these have a tendency ' 1058 | 'of being numerically instable.') 1059 | 1060 | def calc_slopes(self): 1061 | r""" 1062 | Calculate the slopes of the lines after a piecewise linear 1063 | function has been fitted. 1064 | 1065 | This will also calculate the y-intercept from each line in the form 1066 | y = mx + b. The intercepts are stored at self.intercepts. 1067 | 1068 | Returns 1069 | ------- 1070 | slopes : ndarray(1-D) 1071 | The slope of each ling segment as a 1-D numpy array. This assumes 1072 | that x[0] <= x[1] <= ... <= x[n]. Thus, slopes[0] is the slope 1073 | of the first line segment. 1074 | 1075 | Examples 1076 | -------- 1077 | Calculate the slopes after performing a simple fit 1078 | 1079 | >>> import pwlf 1080 | >>> x = np.linspace(0.0, 1.0, 10) 1081 | >>> y = np.random.random(10) 1082 | >>> my_pwlf = pwlf.PiecewiseLinFit(x, y) 1083 | >>> breaks = my_pwlf.fit(3) 1084 | >>> slopes = my_pwlf.calc_slopes() 1085 | 1086 | """ 1087 | y_hat = self.predict(self.fit_breaks) 1088 | self.slopes = np.divide( 1089 | (y_hat[1:self.n_segments + 1] - 1090 | y_hat[:self.n_segments]), 1091 | (self.fit_breaks[1:self.n_segments + 1] - 1092 | self.fit_breaks[:self.n_segments])) 1093 | self.intercepts = y_hat[0:-1] - self.slopes * self.fit_breaks[0:-1] 1094 | return self.slopes 1095 | 1096 | def standard_errors(self, method='linear', step_size=1e-4): 1097 | r""" 1098 | Calculate the standard errors for each beta parameter determined 1099 | from the piecewise linear fit. Typically +- 1.96*se will yield the 1100 | center of a 95% confidence region around your parameters. This 1101 | assumes the parmaters follow a normal distribution. For more 1102 | information see: 1103 | https://en.wikipedia.org/wiki/Standard_error 1104 | 1105 | This calculation follows the derivation provided in [1]_. 1106 | 1107 | Parameters 1108 | ---------- 1109 | method : string, optional 1110 | Calculate the standard errors for a linear or non-linear 1111 | regression problem. The default is method='linear'. A taylor- 1112 | series expansion is performed when method='non-linear' (which is 1113 | commonly referred to as the Delta method). 1114 | step_size : float, optional 1115 | The step size to perform forward differences for the taylor- 1116 | series expansion when method='non-linear'. Default is 1117 | step_size=1e-4. 1118 | 1119 | Returns 1120 | ------- 1121 | se : ndarray (1-D) 1122 | Standard errors associated with each beta parameter. Specifically 1123 | se[0] correspounds to the standard error for beta[0], and so forth. 1124 | 1125 | Raises 1126 | ------ 1127 | AttributeError 1128 | You have probably not performed a fit yet. 1129 | ValueError 1130 | You supplied an unsupported method. 1131 | LinAlgError 1132 | This typically means your regression problem is ill-conditioned. 1133 | 1134 | Notes 1135 | ----- 1136 | The linear regression problem is when you know the breakpoint 1137 | locations (e.g. when using the fit_with_breaks function). 1138 | 1139 | The non-linear regression problem is when you don't know the 1140 | breakpoint locations (e.g. when using the fit, fitfast, and fit_guess 1141 | functions). 1142 | 1143 | References 1144 | ---------- 1145 | .. [1] Coppe, A., Haftka, R. T., and Kim, N. H., “Uncertainty 1146 | Identification of Damage Growth Parameters Using Nonlinear 1147 | Regression,” AIAA Journal, Vol. 49, No. 12, dec 2011, pp. 1148 | 2818–2821. 1149 | 1150 | Examples 1151 | -------- 1152 | Calculate the standard errors after performing a simple fit. 1153 | 1154 | >>> import pwlf 1155 | >>> x = np.linspace(0.0, 1.0, 10) 1156 | >>> y = np.random.random(10) 1157 | >>> my_pwlf = pwlf.PiecewiseLinFit(x, y) 1158 | >>> breaks = my_pwlf.fitfast(3) 1159 | >>> se = my_pwlf.standard_errors() 1160 | 1161 | """ 1162 | try: 1163 | nb = self.beta.size 1164 | except AttributeError: 1165 | errmsg = 'You do not have any beta parameters. You must perform' \ 1166 | ' a fit before using standard_errors().' 1167 | raise AttributeError(errmsg) 1168 | ny = self.n_data 1169 | if method == 'linear': 1170 | A = self.assemble_regression_matrix(self.fit_breaks, self.x_data) 1171 | y_hat = np.dot(A, self.beta) 1172 | e = y_hat - self.y_data 1173 | 1174 | elif method == 'non-linear': 1175 | nb = self.beta.size + self.fit_breaks.size - 2 1176 | f0 = self.predict(self.x_data) 1177 | A = np.zeros((ny, nb)) 1178 | orig_beta = self.beta.copy() 1179 | orig_breaks = self.fit_breaks.copy() 1180 | # calculate differentials due to betas 1181 | for i in range(self.beta.size): 1182 | temp_beta = orig_beta.copy() 1183 | temp_beta[i] += step_size 1184 | # vary beta and keep breaks constant 1185 | f = self.predict(self.x_data, beta=temp_beta, 1186 | breaks=orig_breaks) 1187 | A[:, i] = (f - f0) / step_size 1188 | # append differentials due to break points 1189 | for i in range(self.beta.size, nb): 1190 | # 'ind' ignores first and last entry in self.fit_breaks since 1191 | # these are simply the min/max of self.x_data. 1192 | ind = i - self.beta.size + 1 1193 | temp_breaks = orig_breaks.copy() 1194 | temp_breaks[ind] += step_size 1195 | # vary break and keep betas constant 1196 | f = self.predict(self.x_data, beta=orig_beta, 1197 | breaks=temp_breaks) 1198 | A[:, i] = (f - f0) / step_size 1199 | e = f0 - self.y_data 1200 | # reset beta and breaks back to original values 1201 | self.beta = orig_beta 1202 | self.fit_breaks = orig_breaks 1203 | 1204 | else: 1205 | errmsg = f"Error: method='{method}' is not supported!" 1206 | raise ValueError(errmsg) 1207 | # try to solve for the standard errors 1208 | try: 1209 | variance = np.dot(e, e) / (ny - nb) 1210 | if self.weights is None: 1211 | # solve for the unbiased estimate of variance 1212 | A2inv = np.abs(linalg.inv(np.dot(A.T, A)).diagonal()) 1213 | self.se = np.sqrt(variance * A2inv) 1214 | else: 1215 | A = (A.T*self.weights).T 1216 | A2inv = np.abs(linalg.inv(np.dot(A.T, A)).diagonal()) 1217 | self.se = np.sqrt(variance * A2inv) 1218 | return self.se 1219 | 1220 | except linalg.LinAlgError: 1221 | raise linalg.LinAlgError('Singular matrix') 1222 | 1223 | def prediction_variance(self, x): 1224 | r""" 1225 | Calculate the prediction variance for each specified x location. The 1226 | prediction variance is the uncertainty of the model due to the lack of 1227 | data. This can be used to find a 95% confidence interval of possible 1228 | piecewise linear models based on the current data. This would be 1229 | done typically as y_hat +- 1.96*np.sqrt(pre_var). The 1230 | prediction_variance needs to be calculated at various x locations. 1231 | For more information see: 1232 | www2.mae.ufl.edu/haftka/vvuq/lectures/Regression-accuracy.pptx 1233 | 1234 | Parameters 1235 | ---------- 1236 | x : array_like 1237 | The x locations where you want the prediction variance from the 1238 | fitted continuous piecewise linear function. 1239 | 1240 | Returns 1241 | ------- 1242 | pre_var : ndarray (1-D) 1243 | Numpy array (floats) of prediction variance at each x location. 1244 | 1245 | Raises 1246 | ------ 1247 | AttributeError 1248 | You have probably not performed a fit yet. 1249 | LinAlgError 1250 | This typically means your regression problem is ill-conditioned. 1251 | 1252 | Notes 1253 | ----- 1254 | This assumes that your breakpoint locations are exact! and does 1255 | not consider the uncertainty with your breakpoint locations. 1256 | 1257 | Examples 1258 | -------- 1259 | Calculate the prediction variance at x_new after performing a simple 1260 | fit. 1261 | 1262 | >>> import pwlf 1263 | >>> x = np.linspace(0.0, 1.0, 10) 1264 | >>> y = np.random.random(10) 1265 | >>> my_pwlf = pwlf.PiecewiseLinFit(x, y) 1266 | >>> breaks = my_pwlf.fitfast(3) 1267 | >>> x_new = np.linspace(0.0, 1.0, 100) 1268 | >>> pre_var = my_pwlf.prediction_variance(x_new) 1269 | 1270 | see also examples/prediction_variance.py 1271 | 1272 | """ 1273 | try: 1274 | nb = self.beta.size 1275 | except AttributeError: 1276 | errmsg = 'You do not have any beta parameters. You must perform' \ 1277 | ' a fit before using standard_errors().' 1278 | raise AttributeError(errmsg) 1279 | 1280 | ny = self.n_data 1281 | x = self._switch_to_np_array(x) 1282 | 1283 | # Regression matrix on training data 1284 | Ad = self.assemble_regression_matrix(self.fit_breaks, self.x_data) 1285 | 1286 | # try to solve for the unbiased variance estimation 1287 | try: 1288 | y_hat = np.dot(Ad, self.beta) 1289 | e = y_hat - self.y_data 1290 | # solve for the unbiased estimate of variance 1291 | variance = np.dot(e, e) / (ny - nb) 1292 | 1293 | except linalg.LinAlgError: 1294 | raise linalg.LinAlgError('Singular matrix') 1295 | 1296 | # Regression matrix on prediction data 1297 | A = self.assemble_regression_matrix(self.fit_breaks, x) 1298 | 1299 | # try to solve for the prediction variance at the x locations 1300 | try: 1301 | pre_var = variance * \ 1302 | np.dot(np.dot(A, linalg.inv(np.dot(Ad.T, Ad))), A.T) 1303 | return pre_var.diagonal() 1304 | 1305 | except linalg.LinAlgError: 1306 | raise linalg.LinAlgError('Singular matrix') 1307 | 1308 | def r_squared(self): 1309 | r""" 1310 | Calculate the coefficient of determination ("R squared", R^2) value 1311 | after a fit has been performed. 1312 | For more information see: 1313 | https://en.wikipedia.org/wiki/Coefficient_of_determination 1314 | 1315 | Returns 1316 | ------- 1317 | rsq : float 1318 | Coefficient of determination, or 'R squared' value. 1319 | 1320 | Raises 1321 | ------ 1322 | AttributeError 1323 | You have probably not performed a fit yet. 1324 | LinAlgError 1325 | This typically means your regression problem is ill-conditioned. 1326 | 1327 | Examples 1328 | -------- 1329 | Calculate the R squared value after performing a simple fit. 1330 | 1331 | >>> import pwlf 1332 | >>> x = np.linspace(0.0, 1.0, 10) 1333 | >>> y = np.random.random(10) 1334 | >>> my_pwlf = pwlf.PiecewiseLinFit(x, y) 1335 | >>> breaks = my_pwlf.fitfast(3) 1336 | >>> rsq = my_pwlf.r_squared() 1337 | 1338 | """ 1339 | if self.fit_breaks is None: 1340 | errmsg = 'You do not have any beta parameters. You must perform' \ 1341 | ' a fit before using standard_errors().' 1342 | raise AttributeError(errmsg) 1343 | ssr = self.fit_with_breaks(self.fit_breaks) 1344 | ybar = np.ones(self.n_data) * np.mean(self.y_data) 1345 | ydiff = self.y_data - ybar 1346 | try: 1347 | sst = np.dot(ydiff, ydiff) 1348 | rsq = 1.0 - (ssr/sst) 1349 | return rsq 1350 | except linalg.LinAlgError: 1351 | raise linalg.LinAlgError('Singular matrix') 1352 | 1353 | def p_values(self, method='linear', step_size=1e-4): 1354 | r""" 1355 | Calculate the p-values for each beta parameter. 1356 | 1357 | This calculates the p-values for the beta parameters under the 1358 | assumption that your breakpoint locations are known. Section 2.4.2 of 1359 | [2]_ defines how to calculate the p-value of individual parameters. 1360 | This is really a marginal test since each parameter is dependent upon 1361 | the other parameters. 1362 | 1363 | These values are typically compared to some confidence level alpha for 1364 | significance. A 95% confidence level would have alpha = 0.05. 1365 | 1366 | Parameters 1367 | ---------- 1368 | method : string, optional 1369 | Calculate the standard errors for a linear or non-linear 1370 | regression problem. The default is method='linear'. A taylor- 1371 | series expansion is performed when method='non-linear' (which is 1372 | commonly referred to as the Delta method). 1373 | step_size : float, optional 1374 | The step size to perform forward differences for the taylor- 1375 | series expansion when method='non-linear'. Default is 1376 | step_size=1e-4. 1377 | 1378 | Returns 1379 | ------- 1380 | p : ndarray (1-D) 1381 | p-values for each beta parameter where p-value[0] corresponds to 1382 | beta[0] and so forth 1383 | 1384 | Raises 1385 | ------ 1386 | AttributeError 1387 | You have probably not performed a fit yet. 1388 | ValueError 1389 | You supplied an unsupported method. 1390 | 1391 | Notes 1392 | ----- 1393 | The linear regression problem is when you know the breakpoint 1394 | locations (e.g. when using the fit_with_breaks function). 1395 | 1396 | The non-linear regression problem is when you don't know the 1397 | breakpoint locations (e.g. when using the fit, fitfast, and fit_guess 1398 | functions). 1399 | 1400 | See https://github.com/cjekel/piecewise_linear_fit_py/issues/14 1401 | 1402 | References 1403 | ---------- 1404 | .. [2] Myers RH, Montgomery DC, Anderson-Cook CM. Response surface 1405 | methodology . Hoboken. New Jersey: John Wiley & Sons, Inc. 1406 | 2009;20:38-44. 1407 | 1408 | Examples 1409 | -------- 1410 | After performing a fit, one can calculate the p-value for each beta 1411 | parameter 1412 | 1413 | >>> import pwlf 1414 | >>> x = np.linspace(0.0, 1.0, 10) 1415 | >>> y = np.random.random(10) 1416 | >>> my_pwlf = pwlf.PiecewiseLinFit(x, y) 1417 | >>> breaks = my_pwlf.fitfast(3) 1418 | >>> x_new = np.linspace(0.0, 1.0, 100) 1419 | >>> p = my_pwlf.p_values(x_new) 1420 | 1421 | see also examples/standard_errrors_and_p-values.py 1422 | 1423 | """ 1424 | n = self.n_data 1425 | # degrees of freedom for t-distribution 1426 | if self.beta is None: 1427 | errmsg = 'You do not have any beta parameters. You must perform' \ 1428 | ' a fit before using standard_errors().' 1429 | raise AttributeError(errmsg) 1430 | k = self.beta.size - 1 1431 | 1432 | if method == 'linear': 1433 | self.standard_errors() 1434 | # calculate my t-value 1435 | t = self.beta / self.se 1436 | 1437 | elif method == 'non-linear': 1438 | nb = self.beta.size + self.fit_breaks.size - 2 1439 | k = nb - 1 1440 | self.standard_errors(method=method, step_size=step_size) 1441 | # the parameters for a non-linear model include interior breaks 1442 | parameters = np.concatenate((self.beta, self.fit_breaks[1:-1])) 1443 | # calculate my t-value 1444 | t = parameters / self.se 1445 | else: 1446 | errmsg = f"Error: method='{method}' is not supported!" 1447 | raise ValueError(errmsg) 1448 | # calculate the p-values 1449 | p = 2.0 * stats.t.sf(np.abs(t), df=n - k - 1) 1450 | return p 1451 | 1452 | def lstsq(self, A): 1453 | r""" 1454 | Perform the least squares fit for A matrix. 1455 | """ 1456 | if self.weights is None: 1457 | beta, ssr, _, _ = linalg.lstsq(A, self.y_data, 1458 | lapack_driver=self.lapack_driver) 1459 | # ssr is only calculated if self.n_data > self.n_parameters 1460 | # in this case I'll need to calculate ssr manually 1461 | # where ssr = sum of square of residuals 1462 | if self.n_data <= self.n_parameters: 1463 | y_hat = np.dot(A, beta) 1464 | e = y_hat - self.y_data 1465 | ssr = np.dot(e, e) 1466 | else: 1467 | beta, _, _, _ = linalg.lstsq((A.T*self.weights).T, self.y_w, 1468 | lapack_driver=self.lapack_driver) 1469 | # calculate the weighted sum of square of residuals 1470 | y_hat = np.dot(A, beta) 1471 | e = y_hat - self.y_data 1472 | r = e * self.weights 1473 | ssr = np.dot(r, r) 1474 | if isinstance(ssr, list): 1475 | ssr = ssr[0] 1476 | elif isinstance(ssr, np.ndarray): 1477 | if ssr.size == 0: 1478 | y_hat = np.dot(A, beta) 1479 | e = y_hat - self.y_data 1480 | ssr = np.dot(e, e) 1481 | else: 1482 | ssr = ssr[0] 1483 | # save the beta parameters 1484 | self.beta = beta 1485 | 1486 | # save the slopes 1487 | self.calc_slopes() 1488 | return ssr 1489 | 1490 | def conlstsq(self, A): 1491 | r""" 1492 | Perform a constrained least squares fit for A matrix. 1493 | """ 1494 | # Assemble the constraint matrix 1495 | C_list = [np.ones_like(self.x_c)] 1496 | if self.degree >= 1: 1497 | C_list.append(self.x_c - self.fit_breaks[0]) 1498 | for i in range(self.n_segments - 1): 1499 | C_list.append(np.where(self.x_c > self.fit_breaks[i+1], 1500 | self.x_c - self.fit_breaks[i+1], 1501 | 0.0)) 1502 | if self.degree >= 2: 1503 | for k in range(2, self.degree + 1): 1504 | C_list.append((self.x_c - self.fit_breaks[0])**k) 1505 | for i in range(self.n_segments - 1): 1506 | C_list.append(np.where(self.x_c > self.fit_breaks[i+1], 1507 | (self.x_c 1508 | - self.fit_breaks[i+1])**k, 1509 | 0.0)) 1510 | else: 1511 | for i in range(self.n_segments - 1): 1512 | C_list.append(np.where(self.x_c > self.fit_breaks[i+1], 1513 | 1.0, 1514 | 0.0)) 1515 | C = np.vstack(C_list).T 1516 | 1517 | _, m = A.shape 1518 | o, _ = C.shape 1519 | 1520 | K = np.zeros((m + o, m + o)) 1521 | 1522 | K[:m, :m] = 2.0 * np.dot(A.T, A) 1523 | K[:m, m:] = C.T 1524 | K[m:, :m] = C 1525 | # Assemble right hand side vector 1526 | yt = np.dot(2.0*A.T, self.y_data) 1527 | 1528 | z = np.zeros(self.n_parameters + self.c_n) 1529 | z[:self.n_parameters] = yt 1530 | z[self.n_parameters:] = self.y_c 1531 | 1532 | # try to solve the regression problem 1533 | try: 1534 | # Solve the least squares problem 1535 | beta_prime = linalg.solve(K, z) 1536 | 1537 | # save the beta parameters 1538 | self.beta = beta_prime[0:self.n_parameters] 1539 | # save the zeta parameters 1540 | self.zeta = beta_prime[self.n_parameters:] 1541 | 1542 | # save the slopes 1543 | self.calc_slopes() 1544 | 1545 | # Calculate ssr 1546 | # where ssr = sum of square of residuals 1547 | y_hat = np.dot(A, self.beta) 1548 | e = y_hat - self.y_data 1549 | ssr = np.dot(e, e) 1550 | self.ssr = ssr 1551 | 1552 | # Calculate the Lagrangian function 1553 | # c_x_y = np.dot(C, self.x_c.T) - self.y_c 1554 | p = np.dot(C.T, self.zeta) 1555 | L = np.sum(np.abs(p)) + ssr 1556 | 1557 | except linalg.LinAlgError: 1558 | # the computation could not converge! 1559 | # on an error, return L = np.inf 1560 | # You might have a singular Matrix!!! 1561 | L = np.inf 1562 | if L is None: 1563 | L = np.inf 1564 | # something went wrong... 1565 | return L 1566 | -------------------------------------------------------------------------------- /util/summarization.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | from abc import ABC, abstractmethod 4 | 5 | import numpy as np 6 | from numpy import linalg as la 7 | from tslearn.piecewise import PiecewiseAggregateApproximation 8 | import scipy.fft 9 | from pywt import wavedecn, waverecn 10 | 11 | from util.pwlf import PiecewiseLinFit 12 | 13 | 14 | class Summarization(ABC): 15 | def __init__(self, original: np.ndarray, summarized: np.ndarray = None, reconstructed: np.ndarray = None): 16 | assert original is not None 17 | self.original = np.array(original) 18 | 19 | self.summarized = summarized 20 | if self.summarized is not None: 21 | self.summarized = np.array(self.summarized) 22 | 23 | self.reconstructed = reconstructed 24 | if self.reconstructed is not None: 25 | self.reconstructed = np.array(self.reconstructed) 26 | 27 | 28 | @abstractmethod 29 | def summarize(self): 30 | raise NotImplementedError 31 | 32 | 33 | @abstractmethod 34 | def reconstruct(self): 35 | raise NotImplementedError 36 | 37 | 38 | def diff(self, dist: str = 'Euclidean', aggregate: bool = True): 39 | # original and reconstructed should be np.ndarray 40 | 41 | assert dist in {'Euclidean', 'RMS'} 42 | 43 | if self.reconstructed is None: 44 | if self.summarized is None: 45 | self.summarize() 46 | self.reconstruct() 47 | 48 | if dist == 'Euclidean': 49 | distances = la.norm(self.original - self.reconstructed, axis=1) 50 | elif dist == 'RMS': 51 | distances = np.sqrt(np.mean(np.square(self.original - self.reconstructed), axis=-1)) 52 | 53 | if aggregate: 54 | return np.mean(distances) 55 | else: 56 | return distances 57 | 58 | 59 | 60 | class DEA(Summarization): 61 | def __init__(self, original: np.ndarray, reconstructed: np.ndarray): 62 | super(DEA, self).__init__(original, reconstructed=reconstructed) 63 | 64 | 65 | def summarize(self): 66 | raise NotImplementedError 67 | 68 | 69 | def reconstruct(self): 70 | raise NotImplementedError 71 | 72 | 73 | class DFT(Summarization): 74 | def __init__(self, original: np.ndarray, summarized_length: int, mode: str = 'best', num_components: int = 7): 75 | super(DFT, self).__init__(original) 76 | # / 2 for positive & negative frequency 77 | # / 2 for complex 78 | assert summarized_length % 2 == 0 79 | assert mode in {'best', 'first'} 80 | 81 | self.mode = mode 82 | 83 | # TODO only first several frequencies are perserved 84 | self.length_frequencies2perserve = int(summarized_length / 2) 85 | 86 | if mode == 'best': 87 | assert self.length_frequencies2perserve > num_components 88 | self.length_frequencies2perserve = num_components 89 | 90 | 91 | def summarize(self): 92 | self.summarized = np.zeros(self.original.shape, dtype=complex) 93 | 94 | if self.mode == 'first': 95 | for i, spectrum in enumerate(scipy.fft.fft(self.original, axis=-1)): 96 | # assuming spectrum[0] = 0 as z-normalization 97 | self.summarized[i][1: 1 + self.length_frequencies2perserve] = spectrum[1: 1 + self.length_frequencies2perserve] 98 | # self.summarized[i][-self.length_frequencies2perserve: ] = spectrum[-self.length_frequencies2perserve: ] 99 | # self.summarized[i][-self.length_frequencies2perserve: ] = np.flip(spectrum[1: 1 + self.length_frequencies2perserve]) 100 | elif self.mode == 'best': 101 | for i, spectrum in enumerate(scipy.fft.fft(self.original, axis=-1)): 102 | for j in spectrum.argsort()[-self.length_frequencies2perserve: ]: 103 | self.summarized[i][j] = spectrum[j] 104 | else: 105 | raise ValueError('not support {:} mode'.format(self.mode)) 106 | 107 | 108 | def reconstruct(self): 109 | self.reconstructed = scipy.fft.ifft(self.summarized).real 110 | 111 | 112 | 113 | class PAA(Summarization): 114 | def __init__(self, original: np.ndarray, summarized_length: int): 115 | super(PAA, self).__init__(original) 116 | 117 | self.original = original 118 | self.paa = PiecewiseAggregateApproximation(n_segments=summarized_length) 119 | 120 | 121 | def summarize(self): 122 | self.summarized = np.squeeze(self.paa.fit_transform(self.original)) 123 | 124 | 125 | def reconstruct(self): 126 | self.reconstructed = np.squeeze(self.paa.inverse_transform(self.summarized)) 127 | 128 | 129 | 130 | class DCT(Summarization): 131 | def __init__(self, original: np.ndarray, summarized_length: int, dct_type: int = 2): 132 | super(DCT, self).__init__(original) 133 | assert 0 < dct_type < 5 134 | 135 | self.type = dct_type 136 | self.summarized_length = summarized_length 137 | 138 | 139 | def summarize(self): 140 | self.summarized = np.zeros(self.original.shape) 141 | 142 | for i, spectrum in enumerate(scipy.fft.dct(self.original, self.type, axis=-1)): 143 | # TODO spectrum[0] = 0 as z-normalization? 144 | self.summarized[i][: self.summarized_length] = spectrum[: self.summarized_length] 145 | 146 | 147 | def reconstruct(self): 148 | self.reconstructed = scipy.fft.idct(self.summarized, self.type, axis=-1) 149 | 150 | 151 | 152 | class DWT(Summarization): 153 | def __init__(self, original: np.ndarray, summarized_length: int, wavelet: str): 154 | super(DWT, self).__init__(original) 155 | assert wavelet in {'db2', 'haar'} 156 | 157 | self.wavelet = wavelet 158 | self.level = int(np.ceil(np.log2(self.original.shape[-1] / summarized_length))) 159 | print('{:d} --> {:d} via level {:d}'.format(self.original.shape[-1], summarized_length, self.level)) 160 | 161 | 162 | def summarize(self): 163 | self.coeffs = wavedecn(self.original, self.wavelet, mode='smooth', level=self.level, axes=-1) 164 | self.summarized = self.coeffs[0] 165 | 166 | print('exact summarized_length = {:d}'.format(self.summarized.shape[-1])) 167 | 168 | 169 | def reconstruct(self): 170 | for i in range(1, self.level + 1): 171 | self.coeffs[-i] = {k: np.zeros_like(v) for k, v in self.coeffs[-i].items()} 172 | 173 | self.reconstructed = waverecn(self.coeffs, self.wavelet, mode='smooth', axes=-1) 174 | 175 | 176 | 177 | 178 | class PXA(Summarization): 179 | def __init__(self, original: np.ndarray, degree: int, num_segments: int): 180 | super(PXA, self).__init__(original) 181 | assert 0 <= degree <= 2 182 | 183 | self.x = np.fromiter(range(1, self.original.shape[-1] + 1), np.int) 184 | self.degree = degree 185 | self.num_segments = num_segments 186 | 187 | 188 | def summarize(self): 189 | self.summarized = [] 190 | 191 | for series in self.original: 192 | self.summarized.append(PiecewiseLinFit(self.x, series, degree=self.degree)) 193 | self.summarized[-1].fit(self.num_segments) 194 | 195 | 196 | def reconstruct(self): 197 | self.reconstructed = np.zeros(self.original.shape) 198 | 199 | for i, local_pwlf in enumerate(self.summarized): 200 | self.reconstructed[i] = local_pwlf.predict(self.x) 201 | -------------------------------------------------------------------------------- /util/torchsummary.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | # cite: https://github.com/sksq96/pytorch-summary/blob/f9c58b4e07e3932c790055ac460b757e12939477/torchsummary/torchsummary.py 4 | 5 | import torch 6 | import torch.nn as nn 7 | from torch.autograd import Variable 8 | 9 | from collections import OrderedDict 10 | import numpy as np 11 | 12 | 13 | def summary(model, input_size, batch_size=-1, device=torch.device('cuda:0'), dtypes=None): 14 | result, params_info = summary_string( 15 | model, input_size, batch_size, device, dtypes) 16 | 17 | return result, params_info 18 | 19 | 20 | def summary_string(model, input_size, batch_size=-1, device=torch.device('cuda:0'), dtypes=None): 21 | if dtypes == None: 22 | dtypes = [torch.FloatTensor]*len(input_size) 23 | 24 | summary_str = '' 25 | 26 | def register_hook(module): 27 | def hook(module, input, output): 28 | class_name = str(module.__class__).split(".")[-1].split("'")[0] 29 | module_idx = len(summary) 30 | 31 | m_key = "%s-%i" % (class_name, module_idx + 1) 32 | summary[m_key] = OrderedDict() 33 | summary[m_key]["input_shape"] = list(input[0].size()) 34 | summary[m_key]["input_shape"][0] = batch_size 35 | if isinstance(output, (list, tuple)): 36 | summary[m_key]["output_shape"] = [ 37 | [-1] + list(o.size())[1:] for o in output 38 | ] 39 | else: 40 | summary[m_key]["output_shape"] = list(output.size()) 41 | summary[m_key]["output_shape"][0] = batch_size 42 | 43 | params = 0 44 | if hasattr(module, "weight") and hasattr(module.weight, "size"): 45 | params += torch.prod(torch.LongTensor(list(module.weight.size()))) 46 | summary[m_key]["trainable"] = module.weight.requires_grad 47 | if hasattr(module, "bias") and hasattr(module.bias, "size"): 48 | params += torch.prod(torch.LongTensor(list(module.bias.size()))) 49 | summary[m_key]["nb_params"] = params 50 | 51 | if ( 52 | not isinstance(module, nn.Sequential) 53 | and not isinstance(module, nn.ModuleList) 54 | ): 55 | hooks.append(module.register_forward_hook(hook)) 56 | 57 | # multiple inputs to the network 58 | if isinstance(input_size, tuple): 59 | input_size = [input_size] 60 | 61 | # batch_size of 2 for batchnorm 62 | x = [torch.rand(2, *in_size).type(dtype).to(device=device) 63 | for in_size, dtype in zip(input_size, dtypes)] 64 | 65 | # create properties 66 | summary = OrderedDict() 67 | hooks = [] 68 | 69 | # register hook 70 | model.apply(register_hook) 71 | 72 | # make a forward pass 73 | # print(x.shape) 74 | model(*x) 75 | 76 | # remove these hooks 77 | for h in hooks: 78 | h.remove() 79 | 80 | summary_str += "----------------------------------------------------------------" + "\n" 81 | line_new = "{:>20} {:>25} {:>15}".format( 82 | "Layer (type)", "Output Shape", "Param #") 83 | summary_str += line_new + "\n" 84 | summary_str += "================================================================" + "\n" 85 | total_params = 0 86 | total_output = 0 87 | trainable_params = 0 88 | for layer in summary: 89 | # input_shape, output_shape, trainable, nb_params 90 | line_new = "{:>20} {:>25} {:>15}".format( 91 | layer, 92 | str(summary[layer]["output_shape"]), 93 | "{0:,}".format(summary[layer]["nb_params"]), 94 | ) 95 | total_params += summary[layer]["nb_params"] 96 | 97 | total_output += np.prod(summary[layer]["output_shape"]) 98 | if "trainable" in summary[layer]: 99 | if summary[layer]["trainable"] == True: 100 | trainable_params += summary[layer]["nb_params"] 101 | summary_str += line_new + "\n" 102 | 103 | # assume 4 bytes/number (float on cuda). 104 | total_input_size = abs(np.prod(sum(input_size, ())) 105 | * batch_size * 4. / (1024 ** 2.)) 106 | total_output_size = abs(2. * total_output * 4. / 107 | (1024 ** 2.)) # x2 for gradients 108 | total_params_size = abs(total_params * 4. / (1024 ** 2.)) 109 | total_size = total_params_size + total_output_size + total_input_size 110 | 111 | summary_str += "================================================================" + "\n" 112 | summary_str += "Total params: {0:,}".format(total_params) + "\n" 113 | summary_str += "Trainable params: {0:,}".format(trainable_params) + "\n" 114 | summary_str += "Non-trainable params: {0:,}".format(total_params - 115 | trainable_params) + "\n" 116 | summary_str += "----------------------------------------------------------------" + "\n" 117 | summary_str += "Input size (MB): %0.2f" % total_input_size + "\n" 118 | summary_str += "Forward/backward pass size (MB): %0.2f" % total_output_size + "\n" 119 | summary_str += "Params size (MB): %0.2f" % total_params_size + "\n" 120 | summary_str += "Estimated Total Size (MB): %0.2f" % total_size + "\n" 121 | summary_str += "----------------------------------------------------------------" + "\n" 122 | # return summary 123 | return summary_str, (total_params, trainable_params) 124 | -------------------------------------------------------------------------------- /util/train.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | 3 | import os 4 | import numpy as np 5 | import torch 6 | from torch import optim 7 | from torch.utils.data import DataLoader 8 | from ray import tune 9 | from apex import amp 10 | 11 | from model.loss import L2ScaledTransformation, L2Rreconstruction 12 | from util.data import TSDataset 13 | 14 | 15 | class AlternativeTrainingAutoencoder(tune.Trainable): 16 | # def setup(self, config): 17 | def _setup(self, config): 18 | torch.manual_seed(97) 19 | if torch.cuda.is_available(): 20 | torch.cuda.manual_seed_all(29) 21 | 22 | self.epoch = 0 23 | self.ALTERNATIVE_EPOCHES = int(config["epoch"] / 5 * 4) 24 | self.ADJUSTION_EPOCHES = int(config["epoch"] / 4) 25 | 26 | self.train_dataset = TSDataset(config["train_samples"].cuda()) 27 | self.train_db_loader = DataLoader(self.train_dataset, batch_size=config.get('batch_size', 64), shuffle=True) 28 | self.train_query_loader = DataLoader(self.train_dataset, batch_size=config.get('batch_size', 64), shuffle=True) 29 | 30 | self.val_dataset = TSDataset(config["val_samples"].cuda()) 31 | self.val_db_loader = DataLoader(self.val_dataset, batch_size=config.get('batch_size', 64), shuffle=True) 32 | self.val_query_loader = DataLoader(self.val_dataset, batch_size=config.get('batch_size', 64), shuffle=True) 33 | 34 | self.transformation_loss = L2ScaledTransformation().cuda() 35 | self.reconstruction_loss = L2Rreconstruction().cuda() 36 | 37 | if 'negative_slope' in config: 38 | self.model = config["model"](dim_embedding=config.get('dim_embedding', 16), 39 | dim_sequence=config.get('dim_sequence', 256), 40 | negative_slope=config.get('negative_slope', 1e-2)).cuda() 41 | else: 42 | self.model = config["model"](dim_embedding=config.get('dim_embedding', 16), 43 | dim_sequence=config.get('dim_sequence', 256)).cuda() 44 | 45 | self.optimizer = optim.SGD(self.model.parameters(), 46 | lr=config.get('lr', 1e-3), 47 | momentum=config.get('momentum', 0.9), 48 | weight_decay=config.get('weight_decay', 1e-5)) 49 | 50 | self.model, self.optimizer = amp.initialize(self.model, self.optimizer, opt_level=config.get('opt_level', 'O0')) 51 | 52 | 53 | # def step(self): 54 | def _train(self): 55 | self.__adjust_learning_rate() 56 | self.epoch += 1 57 | 58 | train_reconstruction_batch = [] 59 | if self.epoch < self.ALTERNATIVE_EPOCHES: 60 | for batch in self.train_db_loader: 61 | self.optimizer.zero_grad() 62 | 63 | reconstruction_error = self.reconstruction_loss(batch, self.model(batch)) 64 | 65 | # reconstruction_error.backward() 66 | with amp.scale_loss(reconstruction_error, self.optimizer) as scaled_loss: 67 | scaled_loss.backward() 68 | 69 | self.optimizer.step() 70 | 71 | train_reconstruction_batch.append(reconstruction_error.detach().item()) 72 | else: 73 | with torch.no_grad(): 74 | for batch in self.train_db_loader: 75 | train_reconstruction_batch.append(self.reconstruction_loss(batch, self.model(batch)).detach().item()) 76 | 77 | train_diffences_batch = [] 78 | for db_batch, query_batch in zip(self.train_db_loader, self.train_query_loader): 79 | self.optimizer.zero_grad() 80 | 81 | transformation_error = self.transformation_loss(db_batch, query_batch, self.model.encode(db_batch), self.model.encode(query_batch)) 82 | 83 | # transformation_error.backward() 84 | with amp.scale_loss(transformation_error, self.optimizer) as scaled_loss: 85 | scaled_loss.backward() 86 | 87 | self.optimizer.step() 88 | 89 | train_diffences_batch.append(transformation_error.detach().item()) 90 | 91 | val_diffences_batch = [] 92 | with torch.no_grad(): 93 | for db_batch, query_batch in zip(self.val_db_loader, self.val_query_loader): 94 | val_diffences_batch.append(self.transformation_loss(db_batch, query_batch, self.model.encode(db_batch), self.model.encode(query_batch)).detach().item()) 95 | 96 | return {'val_diff': np.mean(val_diffences_batch), 'train_diff': np.mean(train_diffences_batch), 'rec_error': np.mean(train_reconstruction_batch)} 97 | 98 | 99 | def __adjust_learning_rate(self): 100 | if self.epoch % self.ADJUSTION_EPOCHES == 0: 101 | for param_group in self.optimizer.param_groups: 102 | param_group['lr'] *= 0.1 103 | 104 | 105 | # def save_checkpoint(self, checkpoint_dir): 106 | def _save(self, checkpoint_dir): 107 | checkpoint = { 108 | 'model': self.model.state_dict(), 109 | 'optimizer': self.optimizer.state_dict(), 110 | 'amp': amp.state_dict() 111 | } 112 | 113 | # checkpoint_path = os.path.join(checkpoint_dir, 'model.pth') 114 | checkpoint_path = os.path.join(checkpoint_dir, 'amp_checkpoint.pt') 115 | torch.save(checkpoint, checkpoint_path) 116 | return checkpoint_path 117 | 118 | 119 | # def load_checkpoint(self, checkpoint_path): 120 | def _restore(self, checkpoint_path): 121 | # self.model.load_state_dict(torch.load(checkpoint_path)) 122 | checkpoint = torch.load(checkpoint_path) 123 | 124 | self.model.load_state_dict(checkpoint['model']) 125 | self.optimizer.load_state_dict(checkpoint['optimizer']) 126 | amp.load_state_dict(checkpoint['amp']) 127 | 128 | 129 | # def cleanup(self): 130 | def _stop(self): 131 | self.train_dataset = None 132 | self.train_db_loader = None 133 | self.train_query_loader = None 134 | 135 | self.val_dataset = None 136 | self.val_db_loader = None 137 | self.val_query_loader = None 138 | 139 | self.model = None 140 | self.transformation_loss = None 141 | self.reconstruction_loss = None 142 | self.optimizer = None 143 | 144 | torch.cuda.empty_cache() 145 | 146 | 147 | if __name__ == "__main__": 148 | print('Welcome to where the training methods got defined!') 149 | --------------------------------------------------------------------------------