├── LICENSE ├── README.md ├── alternate_clustering.py ├── archives └── UCRArchive_2018 │ └── CBF │ ├── dtw_matrix_test.npy │ ├── dtw_matrix_train.npy │ ├── x_test.npy │ ├── x_train.npy │ ├── y_test.npy │ └── y_train.npy ├── clustering_stats.py ├── default_hyperparameters.json ├── extract_stats.py ├── load_graph.py ├── losses ├── __init__.py └── losses.py ├── merge_trainers.py ├── networks ├── ClusterGAN.py ├── DEPICT.py ├── DTCR.py ├── IDEC.py ├── SDCN.py ├── VADE.py ├── __init__.py ├── __pycache__ │ ├── ClusterGAN.cpython-39.pyc │ ├── DEPICT.cpython-39.pyc │ ├── DTCR.cpython-39.pyc │ ├── IDEC.cpython-39.pyc │ ├── SDCN.cpython-39.pyc │ ├── VADE.cpython-39.pyc │ ├── __init__.cpython-39.pyc │ ├── attention_rnn.cpython-39.pyc │ ├── bi_dilated_RNN.cpython-39.pyc │ ├── bi_dilated_RNN_variable_length.cpython-39.pyc │ ├── bilstm_ae.cpython-39.pyc │ ├── birnn_ae.cpython-39.pyc │ ├── dilated_causal_cnn.cpython-39.pyc │ ├── encoders.cpython-39.pyc │ ├── fcnn_ae.cpython-39.pyc │ ├── mlp_ae.cpython-39.pyc │ ├── resnet.cpython-39.pyc │ └── trainer.cpython-39.pyc ├── attention_rnn.py ├── bi_dilated_RNN.py ├── bi_dilated_RNN_variable_length.py ├── bilstm_ae.py ├── birnn_ae.py ├── dilated_causal_cnn.py ├── encoders.py ├── fcnn_ae.py ├── mlp_ae.py ├── resnet.py └── trainer.py ├── paper_results ├── Multivariate_Archive_all_results_NMI.csv ├── Multivariate_Archive_all_results_with_UMAP_NMI.csv ├── Multivariate_Archive_detailed_results.csv ├── Univariate_Archive_all_results_NMI.csv ├── Univariate_Archive_all_results_with_UMAP_NMI.csv ├── Univariate_Archive_detailed_results.csv └── igarss2022_results.csv ├── requirements.txt ├── supervised_training.py ├── ucr_baseline.py ├── ucr_baseline_umap.py ├── ucr_experiments.py └── utils.py /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # End-to-end deep representation learning for time series clustering 2 | 3 | This is the code corresponding to the experiments conducted for the work 4 | "End-to-end deep representation learning for time series clustering: a comparative study" 5 | (Baptiste Lafabregue, Jonathan Weber, Pierre Gançarki & Germain Forestier), 6 | in submission 7 | 8 | The results obtained for this paper on both archives are reported in the [paper_results/ folder](https://github.com/blafabregue/TimeSeriesDeepClustering/tree/main/paper_results) 9 | 10 | ## Datasets 11 | 12 | The dataset used for the paper are available at : http://www.timeseriesclassification.com/ 13 | Both the univariate and the multivariate archive can be used. 14 | 15 | ## Usage 16 | 17 | ### Install packages 18 | You can use your favorite package manager (conda is recommended), create a new environment of python 3.8 or greater 19 | and use the packages listed in [requirements.txt](requirements.txt) 20 | ```sh 21 | pip install -r requirements.txt 22 | ``` 23 | 24 | ### For training on GPU 25 | If you wish to train the networks on GPU you must install the tensorflow-gpu package. For example with pip: 26 | ```sh 27 | pip install tensorflow-gpu 28 | ``` 29 | or 30 | ```sh 31 | conda install tensorflow-gpu 32 | ``` 33 | ### Extract dataset to numpy format 34 | 35 | First, to train the networks you need to convert them into .npy files. 36 | The CBF dataset is provided as an example. 37 | To do so you can use the `utils.py` script but you need first to change the two last line 38 | of the script [here](https://github.com/blafabregue/TimeSeriesDeepClustering/blob/1503c70053bbc8ec5ff34032c69e45099012c4ea/utils.py#L552). 39 | Note that this script is suited to extract data from .sktime files 40 | 41 | ### Train networks 42 | 43 | To train networks and obtain the results you can use the following command: 44 | ```sh 45 | python ucr_experiments.py --dataset --itr --architecture --encoder_loss --clustering_loss IDEC --archives --nbneg --hyper default_hyperparameters.json 46 | ``` 47 | Here is an example of the DRNN-rec-DEC combination on the CBF dataset 48 | ```sh 49 | python ucr_experiments.py --dataset CBF --itr 1 --architecture dilated_cnn --encoder_loss reconstruction --clustering_loss DEC --archives UCRArchive_2018 --hyper default_hyperparameters.json 50 | ``` 51 | more informations are provided if you type directly `python ucr_experiments.py` in your prompt 52 | 53 | ### Results 54 | 55 | The results are stored in two folders: 56 | 57 | * ae_weights/// 58 | will contain the logs (performance/loss evolution) and the saved weights of both 59 | the pretrained model (without clustering loss) and the final model, 60 | for the final model it will also save the representation learned of the train and test test 61 | ,the clustering map and the centroids 62 | * stats/ will contain the final statistics used to store the summarized 63 | stats used to evaluate methods. It will contain a .error file if an error occurred during the process 64 | 65 | ### Other computations 66 | Baseline can be computed with following script: 67 | ```sh 68 | python ucr_baseline.py --dataset --itr --archives 69 | ``` 70 | To combine already computed representations (i.e. for triplet compbined) it can be done with the following script (example with DCNN architecture): 71 | ```sh 72 | python merge_trainers.py --dataset --itr --archives --prefix "dilated_cnn_tripletK" --suffix "_None_0.0" 73 | ``` 74 | To use a reduction method on top of an already computed representation you can use: 75 | ```sh 76 | python alternate_clustering.py --dataset --itr --architecture --encoder_loss --clustering_loss IDEC --archives --nbneg 77 | ``` 78 | For this last option the argument should be the semae as the one used to first compute the model. 79 | Note that the arguments are used to find load saved .npy files in ae_weights folder, 80 | so they should not have been used or renamed. -------------------------------------------------------------------------------- /alternate_clustering.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script to launch experiments with a dimension reduction 3 | See parse_arguments() function for details on arguments 4 | 5 | Based on implementation in https://github.com/rymc/n2d 6 | and article: 7 | McConville, R., Santos-Rodriguez, R., Piechocki, R. J., & Craddock, I. (2019). 8 | N2d:(not too) deep clustering via clustering the local manifold of an autoencoded embedding 9 | 10 | Author: 11 | Baptiste Lafabregue 2019.25.04 12 | """ 13 | import numpy as np 14 | import argparse 15 | import traceback 16 | 17 | import tensorflow as tf 18 | from sklearn.manifold import TSNE 19 | from sklearn.cluster import KMeans, SpectralClustering 20 | from sklearn.manifold import Isomap 21 | from sklearn.manifold import LocallyLinearEmbedding 22 | from sklearn import metrics 23 | from sklearn import mixture 24 | 25 | import umap 26 | 27 | import utils 28 | from networks.SDCN import SDCN 29 | from networks.IDEC import IDEC 30 | from networks.DTCR import DTCR 31 | from networks.DEPICT import DEPICT 32 | from networks.ClusterGAN import ClusterGAN 33 | from networks.VADE import VADE 34 | from networks.trainer import get_logs 35 | 36 | UMAP_MIN_DIST = 0 37 | UMAP_NEIGHBORS = 10 38 | 39 | 40 | def regrister_logs(y, y_pred): 41 | logs = {'ari': np.round(metrics.adjusted_rand_score(y, y_pred), 5), 42 | 'nmi': np.round(metrics.normalized_mutual_info_score(y, y_pred), 5), 43 | 'acc': np.round(utils.cluster_acc(y, y_pred), 5), 44 | 'max_ari': -1.0, 'max_nmi': -1.0, 'max_acc': -1.0} 45 | return logs 46 | 47 | 48 | def eval_all_methods(x, y, n_clusters): 49 | meta_logs = {} 50 | for manifold_learner in ['UMAP', 'TSNE', 'Isomap', 'LLE', '']: 51 | hle = x 52 | if manifold_learner == 'UMAP': 53 | md = float(UMAP_MIN_DIST) 54 | hle = umap.UMAP( 55 | random_state=0, 56 | metric='euclidean', 57 | n_components=n_clusters, 58 | n_neighbors=UMAP_NEIGHBORS, 59 | min_dist=md).fit_transform(x) 60 | elif manifold_learner == 'LLE': 61 | hle = LocallyLinearEmbedding( 62 | n_components=n_clusters, 63 | n_neighbors=UMAP_NEIGHBORS).fit_transform(x) 64 | elif manifold_learner == 'TSNE': 65 | hle = TSNE( 66 | n_components=3, 67 | n_jobs=2, 68 | random_state=0, 69 | verbose=0).fit_transform(x) 70 | elif manifold_learner == 'Isomap': 71 | hle = Isomap( 72 | n_components=n_clusters, 73 | n_neighbors=5, 74 | ).fit_transform(x) 75 | 76 | for method in ['Gmm', 'Kmeans', 'Spectral']: 77 | y_pred = None 78 | if method == 'Gmm': 79 | gmm = mixture.GaussianMixture( 80 | covariance_type='full', 81 | n_components=n_clusters, 82 | random_state=0) 83 | gmm.fit(hle) 84 | y_pred_prob = gmm.predict_proba(hle) 85 | y_pred = y_pred_prob.argmax(1) 86 | elif method == 'Kmeans': 87 | y_pred = KMeans( 88 | n_clusters=n_clusters, 89 | random_state=0).fit_predict(hle) 90 | elif method == 'Spectral': 91 | sc = SpectralClustering( 92 | n_clusters=n_clusters, 93 | random_state=0, 94 | affinity='nearest_neighbors') 95 | y_pred = sc.fit_predict(hle) 96 | meta_logs[method + manifold_learner] = regrister_logs(y, y_pred) 97 | return meta_logs 98 | 99 | 100 | def launch_clustering(output_directory, trainer, seeds_itr, root_dir, itr, framework_name, 101 | dataset_name, y_test, y_train, x_train, x_test): 102 | features_test = np.load(output_directory + trainer + '/x_test_encoded_' + str(seeds_itr) + '.npy') 103 | features_train = np.load(output_directory + trainer + '/x_encoded_' + str(seeds_itr) + '.npy') 104 | nb_cluster = len(np.unique(y_train)) 105 | 106 | train_logs = eval_all_methods(features_train, y_train, nb_cluster) 107 | test_logs = eval_all_methods(features_test, y_test, nb_cluster) 108 | concat_logs = eval_all_methods(np.concatenate((features_train, features_test), axis=0), 109 | np.concatenate((y_train, y_test), axis=0), nb_cluster) 110 | 111 | for key in train_logs.keys(): 112 | stats_dir = root_dir + '/stats/'+key+'/' + str(itr) + '/' + str(seeds_itr) + '/' 113 | f = open(stats_dir + framework_name + "_" + dataset_name, "w") 114 | f.write(get_logs(train_logs[key]) + '\n') 115 | f.write(get_logs(test_logs[key]) + '\n') 116 | f.write(get_logs(concat_logs[key]) + '\n') 117 | f.close() 118 | 119 | 120 | def parse_arguments(): 121 | parser = argparse.ArgumentParser( 122 | description='Alternative clustering methods on embeddings' 123 | ) 124 | parser.add_argument('--dataset', type=str, metavar='d', required=True, 125 | help='dataset name') 126 | parser.add_argument('--archives', type=str, metavar='DIR', required=True, 127 | help='archive name') 128 | parser.add_argument('--architecture', type=str, metavar='xxx', required=True, 129 | choices=['dilated_cnn', 'mlp', 'fcnn', 'res_cnn', 'bi_lstm', 'dilated_rnn', 'bi_rnn', 'bi_gru'], 130 | help='Type of encoder architecture to use among : ' 131 | '[dilated_cnn, mlp, fcnn, res_cnn, bi_lstm, dilated_rnn, bi_rnn, bi_gru]') 132 | parser.add_argument('--encoder_loss', type=str, metavar='xxx', required=True, 133 | choices=['joint', 'reconstruction', 'triplet', 'vae', 'combined'], 134 | help='Type of loss to pretrain the encoder to use among : ' 135 | '[joint, reconstruction, triplet, vae, combined]') 136 | parser.add_argument('--clustering_loss', type=str, metavar='xxx', required=True, 137 | choices=['DEPICT', 'DTCR', 'SDCN', 'DEC', 'IDEC', 'GAN', 'VADE', 'None', 'All'], 138 | help='Type of clustering framework to use among : [DEPICT, DTCR, SDCN, DEC, IDEC, GAN, VADE, ' 139 | 'None, All]') 140 | parser.add_argument('--itr', type=str, metavar='X', default='0', 141 | help='iteration index') 142 | parser.add_argument('--seeds_itr', type=int, metavar='X', default=None, 143 | help='seeds index, do not specify or -1 if none') 144 | parser.add_argument('--dropout', type=float, default=0.0, metavar="X.X", 145 | help='Rate of dropout to use in the encoder architecture (not supported in RNN architecture)') 146 | parser.add_argument('--noise', type=str, metavar='xxx', default=None, 147 | choices=['uniform', 'gaussian', 'laplace', 'drop', 'none'], 148 | help='Type of noise to use, no noise is applied if not specified : ' 149 | '[uniform, gaussian, laplace, drop, none]') 150 | parser.add_argument('--nbneg', type=str, metavar='X', default=None, 151 | help='number of negative sample, only for triplet loss, 1, 2, 5 or 10 are recommended') 152 | parser.add_argument('--root_dir', type=str, metavar='PATH', default='.', 153 | help='path of the root dir where archives and results are stored') 154 | 155 | return parser.parse_args() 156 | 157 | 158 | def main(root_dir, dataset_name, archive_name, architecture, encoder_loss, 159 | clustering_loss, dropout_rate, itr, seeds_itr): 160 | if encoder_loss == "triplet": 161 | encoder_loss = encoder_loss + 'K' + str(args.nbneg) 162 | 163 | print('Launch ' + clustering_loss + ' with ' + architecture + 'encoder on : ' + dataset_name) 164 | 165 | train_dict = utils.read_dataset(root_dir, archive_name, dataset_name, True) 166 | x_train = train_dict[dataset_name][0] 167 | y_train = train_dict[dataset_name][1] 168 | 169 | test_dict = utils.read_dataset(root_dir, archive_name, dataset_name, False) 170 | x_test = test_dict[dataset_name][0] 171 | y_test = test_dict[dataset_name][1] 172 | 173 | # we init the directory here because of different triplet loss versions 174 | enc_name = architecture + '_' + encoder_loss + '_' + str(noise) + '_' + str(dropout_rate) 175 | framework_name = enc_name 176 | # if clustering_loss != 'None': 177 | # framework_name += '_' + clustering_loss 178 | # output_directory = utils.create_output_path(root_dir, itr, framework_name, dataset_name) 179 | # if clustering_loss is not 'All': 180 | # output_directory = utils.create_directory(output_directory) 181 | # create akk 182 | for manifold_learner in ['UMAP', 'TSNE', 'Isomap', 'LLE', '']: 183 | for method in ['Gmm', 'Kmeans', 'Spectral']: 184 | stats_dir = root_dir + '/stats/' + method + manifold_learner + '/' + str(itr) + '/' + str(seeds_itr) + '/' 185 | utils.create_directory(stats_dir) 186 | 187 | if clustering_loss == "All": 188 | for clust_loss in ['DEPICT', 'SDCN', 'DTCR', 'DEC', 'IDEC', 'GAN', 'VADE', 'None']: 189 | framework_name = enc_name 190 | if clust_loss != 'None': 191 | framework_name += '_' + clust_loss 192 | # save_path = stats_dir+framework_name+"_"+dataset_name 193 | output_directory = utils.create_output_path(root_dir, itr, framework_name, dataset_name) 194 | 195 | trainer = None 196 | if clust_loss == 'DTCR': 197 | trainer = DTCR(dataset_name, framework_name, None).get_trainer_name() 198 | elif clust_loss == 'IDEC': 199 | trainer = IDEC(dataset_name, framework_name, None).get_trainer_name() 200 | elif clust_loss == 'DEC': 201 | trainer = IDEC(dataset_name, framework_name, None).get_trainer_name() 202 | elif clust_loss == 'DEPICT': 203 | trainer = DEPICT(dataset_name, framework_name, None).get_trainer_name() 204 | elif clust_loss == 'SDCN': 205 | trainer = SDCN(dataset_name, framework_name, None).get_trainer_name() 206 | elif clust_loss == "GAN": 207 | if encoder_loss != "reconstruction": 208 | continue 209 | trainer = ClusterGAN(dataset_name, framework_name, None, None, None).get_trainer_name() 210 | elif clust_loss == "VADE": 211 | if encoder_loss != "vae": 212 | continue 213 | trainer = VADE(dataset_name, framework_name, None).get_trainer_name() 214 | elif clust_loss == "None": 215 | trainer = 'pre_train_encoder' 216 | try: 217 | launch_clustering(output_directory, trainer, seeds_itr, root_dir, itr, framework_name, 218 | dataset_name, y_test, y_train, x_train, x_test) 219 | except: 220 | print('***********************************************************') 221 | print('ERROR ' + clust_loss) 222 | traceback.print_exc() 223 | print('***********************************************************') 224 | else: 225 | trainer = None 226 | if clustering_loss == 'DTCR': 227 | trainer = DTCR(dataset_name, framework_name, None).get_trainer_name() 228 | elif clustering_loss == 'IDEC': 229 | trainer = IDEC(dataset_name, framework_name, None).get_trainer_name() 230 | elif clustering_loss == 'DEC': 231 | trainer = IDEC(dataset_name, framework_name, None).get_trainer_name() 232 | elif clustering_loss == 'DEPICT': 233 | trainer = DEPICT(dataset_name, framework_name, None).get_trainer_name() 234 | elif clustering_loss == 'SDCN': 235 | trainer = SDCN(dataset_name, framework_name, None).get_trainer_name() 236 | elif clustering_loss == "GAN": 237 | if encoder_loss is not "reconstruction": 238 | print('GAN only works with reconstruction loss') 239 | return 240 | trainer = ClusterGAN(dataset_name, framework_name, None, None, None).get_trainer_name() 241 | elif clustering_loss == "VADE": 242 | if encoder_loss != "vae": 243 | print('VADE only works with vae loss') 244 | return 245 | trainer = VADE(dataset_name, framework_name, None).get_trainer_name() 246 | elif clustering_loss == "None": 247 | trainer = 'pre_train_encoder' 248 | output_directory = utils.create_output_path(root_dir, itr, framework_name, dataset_name) 249 | 250 | launch_clustering(output_directory, trainer, seeds_itr, root_dir, itr, framework_name, 251 | dataset_name, y_test, y_train, x_train, x_test) 252 | 253 | 254 | if __name__ == '__main__': 255 | args = parse_arguments() 256 | 257 | tf.keras.backend.set_floatx('float64') 258 | 259 | root_dir = args.root_dir 260 | dataset_name = args.dataset 261 | archive_name = args.archives 262 | architecture = args.architecture 263 | encoder_loss = args.encoder_loss 264 | clustering_loss = args.clustering_loss 265 | dropout_rate = args.dropout 266 | noise = args.noise 267 | if noise == 'none': 268 | noise = None 269 | itr = args.itr 270 | seeds_itr = args.seeds_itr 271 | main(root_dir, dataset_name, archive_name, architecture, encoder_loss, 272 | clustering_loss, dropout_rate, itr, seeds_itr) 273 | -------------------------------------------------------------------------------- /archives/UCRArchive_2018/CBF/dtw_matrix_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/archives/UCRArchive_2018/CBF/dtw_matrix_test.npy -------------------------------------------------------------------------------- /archives/UCRArchive_2018/CBF/dtw_matrix_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/archives/UCRArchive_2018/CBF/dtw_matrix_train.npy -------------------------------------------------------------------------------- /archives/UCRArchive_2018/CBF/x_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/archives/UCRArchive_2018/CBF/x_test.npy -------------------------------------------------------------------------------- /archives/UCRArchive_2018/CBF/x_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/archives/UCRArchive_2018/CBF/x_train.npy -------------------------------------------------------------------------------- /archives/UCRArchive_2018/CBF/y_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/archives/UCRArchive_2018/CBF/y_test.npy -------------------------------------------------------------------------------- /archives/UCRArchive_2018/CBF/y_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/archives/UCRArchive_2018/CBF/y_train.npy -------------------------------------------------------------------------------- /clustering_stats.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | 4 | import matplotlib 5 | import matplotlib.pyplot as plt 6 | import matplotlib.patheffects as PathEffects 7 | import seaborn as sns 8 | 9 | # from sklearn.manifold import TSNE 10 | import umap 11 | from sklearn.cluster import KMeans 12 | 13 | import utils 14 | 15 | UMAP_MIN_DIST = 0 16 | UMAP_NEIGHBORS = 10 17 | 18 | def fashion_scatter(x, colors, text): 19 | # choose a color palette with seaborn. 20 | colors +=1 21 | unique = np.unique(colors) 22 | num_classes = len(unique) 23 | col_min = np.min(colors) 24 | col_max = np.max(colors) 25 | norm = matplotlib.colors.Normalize(vmin=col_min, vmax=col_max) 26 | # format color if out of bound 27 | if np.average(colors) < (col_max-col_min)/3: 28 | print("use log scale") 29 | norm = matplotlib.colors.LogNorm(vmin=col_min, vmax=col_max) 30 | palette = 'viridis' 31 | 32 | while not np.isin(np.array([0]), unique): 33 | colors = colors -1 34 | unique = np.unique(colors) 35 | 36 | # create a scatter plot. 37 | f = plt.figure(figsize=(8, 8)) 38 | ax = plt.subplot(aspect='equal') 39 | 40 | # no clustering ------------------------- 41 | # sc = ax.scatter(x[:, 0], x[:, 1], lw=0, s=200, c=colors, 42 | # cmap=palette, norm=norm) 43 | # for clustering <<<<<<<<<<<<<<<<<<<<< 44 | palette = np.array(sns.color_palette("hls", num_classes)) 45 | sc = ax.scatter(x[:, 0], x[:, 1], lw=0, s=200, c=palette[colors.astype(np.int)]) 46 | # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> 47 | plt.xlim(-25, 25) 48 | plt.ylim(-25, 25) 49 | ax.axis('off') 50 | ax.axis('tight') 51 | 52 | # txts = [] 53 | # for xtext, ytext, text in text: 54 | # txt = ax.text(xtext, ytext, text, fontsize=24) 55 | # txt.set_path_effects([ 56 | # PathEffects.Stroke(linewidth=5, foreground="w"), 57 | # PathEffects.Normal()]) 58 | # txts.append(txt) 59 | 60 | # add the labels for each digit corresponding to the label 61 | txts = [] 62 | 63 | if num_classes < 20: 64 | for i in range(num_classes): 65 | 66 | # Position of each label at median of data points. 67 | xtext, ytext = np.median(x[colors == i, :], axis=0) 68 | txt = ax.text(xtext, ytext, str(i), fontsize=34) 69 | txt.set_path_effects([ 70 | PathEffects.Stroke(linewidth=5, foreground="w"), 71 | PathEffects.Normal()]) 72 | txts.append(txt) 73 | 74 | return f, ax, sc, txts 75 | 76 | 77 | if __name__ == '__main__': 78 | root = '' 79 | base_path = root+'stats_extract/20000/' 80 | stat_path = base_path+'0to4test_per_dataset_nmi.csv' 81 | info_path = root+'archives/UCRArchive_2018/dataset_info.csv' 82 | dataset_key = 'dataset_name' 83 | utils.create_directory(base_path+'clustering/') 84 | 85 | df_perf = pd.read_csv(stat_path, index_col=False) 86 | df_perf = df_perf.fillna(0) 87 | df_info = pd.read_csv(info_path) 88 | dataset_names = df_perf[dataset_key].values 89 | join = df_perf.join(df_info.set_index(dataset_key), on=dataset_key) 90 | 91 | cols = ['dilated_cnn_reconstruction_None', 'dilated_rnn_reconstruction_None', 92 | 'bi_gru_reconstruction_None', 'bi_lstm_reconstruction_None', 'bi_rnn_reconstruction_None', 93 | 'res_cnn_reconstruction_None', 'fcnn_reconstruction_None', 'mlp_reconstruction_None'] 94 | # cols = ['dilated_cnn_reconstruction_None', 'bi_rnn_reconstruction_None', '' 95 | # 'res_cnn_reconstruction_None'] 96 | df_perf = df_perf[cols] 97 | values = df_perf.values 98 | color = 'clustering' 99 | kmeans = KMeans(n_clusters=10, n_init=20) 100 | pred = kmeans.fit_predict(values) 101 | 102 | centroids = pd.DataFrame(kmeans.cluster_centers_) 103 | centroids.columns = df_perf.columns 104 | centroids.to_csv(base_path+'clustering/'+color+'centroids_datasets.csv') 105 | 106 | cluster_map = np.concatenate([[dataset_names], [pred]], axis=0) 107 | cluster_map = pd.DataFrame(cluster_map) 108 | cluster_map.to_csv(base_path+'clustering/'+color+'cluster_map.csv', index=False) 109 | # np.savetxt(root+'clustering/cluster_map.csv', cluster_map, delimiter=",") 110 | 111 | # color = 'Type' 112 | # # color = 'Length' 113 | # pred = join[color].values 114 | # pred = np.unique(pred, return_inverse=True)[1].tolist() 115 | # pred = np.array(pred) 116 | 117 | md = float(UMAP_MIN_DIST) 118 | hle = umap.UMAP( 119 | random_state=0, 120 | metric='euclidean', 121 | n_components=2, 122 | n_neighbors=UMAP_NEIGHBORS, 123 | min_dist=md).fit_transform(values) 124 | 125 | txt = [] 126 | # i = 101 127 | # txt.append((hle[i][0], hle[i][1], 'CNN >> others')) 128 | # i = 111 129 | # txt.append((hle[i][0], hle[i][1], 'All high')) 130 | # i = 112 131 | # txt.append((hle[i][0], hle[i][1], 'All low')) 132 | # i = 112 133 | # txt.append((hle[i][0], hle[i][1], 'All medium')) 134 | fashion_scatter(hle, pred, txt) 135 | 136 | plt.savefig(base_path+'clustering/umap_on_perf_'+color+'.pdf') 137 | -------------------------------------------------------------------------------- /default_hyperparameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "batch_size": 10, 3 | "nb_filters": 40, 4 | "compared_length": null, 5 | "depth": 10, 6 | "nb_steps": 10, 7 | "nb_steps_pretrain": 10, 8 | "kernel_size": 3, 9 | "penalty": null, 10 | "lr": 0.001, 11 | "nb_random_samples": 1, 12 | "negative_penalty": 1, 13 | "latent_dim": 320, 14 | "reduced_size": 160 15 | } 16 | -------------------------------------------------------------------------------- /extract_stats.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script to extract stats into csv files 3 | 4 | Author: 5 | Baptiste Lafabregue 2019.25.04 6 | """ 7 | 8 | import os 9 | import pathlib 10 | import argparse 11 | 12 | import pandas as pd 13 | import numpy as np 14 | 15 | import utils 16 | 17 | ELEMENT_COUNT = 6 18 | 19 | 20 | def get_elements(file_name): 21 | split = file_name.split('_') 22 | arch = split[0] 23 | limit = ELEMENT_COUNT 24 | shift = 0 25 | if arch == "dilated" or arch == "bi" or arch == 'res': 26 | arch += "_" + split[1] 27 | enc_loss = split[2] 28 | limit += 1 29 | else: 30 | enc_loss = split[1] 31 | if len(split) < limit: 32 | clust_loss = "None" 33 | shift = 1 34 | else: 35 | clust_loss = split[-2] 36 | dropout = split[(shift-4)] 37 | noise = split[(shift-3)] 38 | dataset_name = split[-1] 39 | 40 | return arch, enc_loss, clust_loss, dataset_name, dropout, noise 41 | 42 | 43 | def main(itr, seed_itrs, type, stat_path, root_dir): 44 | dfs = [] 45 | for seed in seed_itrs: 46 | stats = [] 47 | 48 | stat_path = root_dir + '/stats/' + str(itr) + '/' + str(seed) 49 | 50 | stats_files = [name for name in os.listdir(stat_path) 51 | if os.path.isfile(os.path.join(stat_path, name)) and 52 | pathlib.Path(name).suffix != '.error'] 53 | stats_files.sort() 54 | 55 | error_files = [name for name in os.listdir(stat_path) 56 | if os.path.isfile(os.path.join(stat_path, name)) and 57 | pathlib.Path(name).suffix == '.error'] 58 | error_files.sort() 59 | 60 | # each element contains in the following order : 61 | # encoder architecture, encoder loss, clustering loss, dataset name, nmi, max nmi 62 | for file_name in stats_files: 63 | arch, enc_loss, clust_loss, dataset_name, dropout, noise = get_elements(file_name) 64 | try: 65 | with open(os.path.join(stat_path, file_name)) as f: 66 | row = f.readline() 67 | # the test's stats are in the second row 68 | if type == 'test': 69 | row = f.readline() 70 | split_stats = row.split(',') 71 | acc = float(split_stats[0]) 72 | max_acc = float(split_stats[1]) 73 | nmi = float(split_stats[2]) 74 | max_nmi = float(split_stats[3]) 75 | ari = float(split_stats[4]) 76 | max_ari = float(split_stats[5]) 77 | gap = max_nmi - nmi 78 | except: 79 | print('error with '+os.path.join(stat_path, file_name)) 80 | nmi, max_nmi, gap, ari, acc = -1, -1, 0, -1, -1 81 | stats.append([arch, enc_loss, clust_loss, dataset_name, nmi, max_nmi, gap, ari, acc]) 82 | 83 | for file_name in error_files: 84 | arch, enc_loss, clust_loss, dataset_name, dropout, noise = get_elements(file_name.split('.')[0]) 85 | stats.append([arch, enc_loss, clust_loss, dataset_name, np.nan, np.nan, 0.0, np.nan, np.nan]) 86 | 87 | stats = np.array(stats) 88 | dfs.append(pd.DataFrame({"encoder_architecture": stats[:, 0], 89 | "encoder_loss": stats[:, 1], 90 | "clustering_loss": stats[:, 2], 91 | "dataset_name": stats[:, 3], 92 | "nmi": stats[:, 4], 93 | "max_nmi": stats[:, 5], 94 | "gap": stats[:, 6], 95 | "ari": stats[:, 7], 96 | "acc": stats[:, 8]})) 97 | 98 | concat = pd.concat(dfs) 99 | arch_list = concat.encoder_architecture.unique() 100 | encoder_loss = concat.encoder_loss.unique() 101 | clustering_loss = concat.clustering_loss.unique() 102 | dataset_name = concat.dataset_name.unique() 103 | out_path = root_dir + '/stats_extract/' + str(itr) + '/' + str(seed) 104 | if len(seed_itrs) > 1: 105 | out_path = root_dir + '/stats_extract/' + str(itr) + '/' + str(seed_itrs[0]) + 'to' + str(seed_itrs[-1]) + type 106 | utils.create_directory(root_dir + '/stats_extract/' + str(itr) + '/') 107 | 108 | concat["nmi"] = concat["nmi"].astype(float) 109 | concat["max_nmi"] = concat["max_nmi"].astype(float) 110 | concat["gap"] = concat["gap"].astype(float) 111 | concat["acc"] = concat["acc"].astype(float) 112 | concat["ari"] = concat["ari"].astype(float) 113 | concat.to_csv(out_path + '_raw_data.csv', index=False) 114 | concat["model"] = concat["encoder_architecture"] + "_" + concat["encoder_loss"] + "_" + concat["clustering_loss"] 115 | # concat_cleaned = concat.drop(["encoder_architecture", "encoder_loss", "clustering_loss"], axis=1) 116 | # concat_cleaned = concat_cleaned.set_index(["model"]) 117 | # df_nmi = df2.drop(["max_nmi"]) 118 | df_nmi = pd.pivot_table(concat, values="nmi", index="dataset_name", columns="model", aggfunc=np.nanmean) 119 | df_nmi.to_csv(out_path + '_per_dataset_nmi.csv', index=True) 120 | df_max_nmi = pd.pivot_table(concat, values="max_nmi", index="dataset_name", columns="model", aggfunc=np.nanmean) 121 | df_max_nmi.to_csv(out_path + '_per_dataset_max_nmi.csv', index=True) 122 | df_gap = pd.pivot_table(concat, values="gap", index="dataset_name", columns="model", aggfunc=np.nanmean) 123 | df_gap.to_csv(out_path + '_per_dataset_gap.csv', index=True) 124 | df_gap = pd.pivot_table(concat, values="ari", index="dataset_name", columns="model", aggfunc=np.nanmean) 125 | df_gap.to_csv(out_path + '_per_dataset_ari.csv', index=True) 126 | df_gap = pd.pivot_table(concat, values="acc", index="dataset_name", columns="model", aggfunc=np.nanmean) 127 | df_gap.to_csv(out_path + '_per_dataset_acc.csv', index=True) 128 | 129 | 130 | if __name__ == '__main__': 131 | parser = argparse.ArgumentParser( 132 | description='Extract stats from logs' 133 | ) 134 | parser.add_argument('--itr', type=str, metavar='X', default='50320', 135 | help='iteration index') 136 | parser.add_argument('--seeds_itr', type=str, metavar='X', default='0,1,2,3,4', 137 | help='seeds index, can be either an integer or a comma separated list, ' 138 | 'example : --seeds_itr 0,1,2,3,4') 139 | parser.add_argument('--root_dir', type=str, metavar='PATH', default='.', 140 | help='path of the root dir where archives and results are stored') 141 | parser.add_argument('--type', type=str, metavar='xxx', default='test', 142 | choices=['train', 'test'], 143 | help='run on either train or test results clustering') 144 | 145 | args = parser.parse_args() 146 | itr = args.itr 147 | seed_itrs = args.seeds_itr.split(',') 148 | type = args.type 149 | dfs = [] 150 | stat_path = '' 151 | root_dir = args.root_dir 152 | main(itr, seed_itrs, type, stat_path, root_dir) 153 | for manifold_learner in ['UMAP', 'TSNE', 'Isomap', 'LLE', '']: 154 | for method in ['Gmm', 'Kmeans', 'Spectral']: 155 | sub_itr = method+manifold_learner+'/'+itr 156 | print('start '+sub_itr) 157 | main(sub_itr, seed_itrs, type, stat_path, root_dir) 158 | -------------------------------------------------------------------------------- /load_graph.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script to compute KNN graphs for SDCN method 3 | 4 | Author: 5 | Baptiste Lafabregue 2019.25.04 6 | """ 7 | import utils 8 | import argparse 9 | 10 | if __name__ == '__main__': 11 | parser = argparse.ArgumentParser( 12 | description='create KNN graphs for UCR or MTS repository data sets' 13 | ) 14 | parser.add_argument('--dataset', type=str, metavar='D', required=True, 15 | help='dataset name') 16 | parser.add_argument('--archives', type=str, metavar='D', required=True, 17 | help='archive name') 18 | args = parser.parse_args() 19 | root_dir = '.' 20 | dataset_name = args.dataset 21 | archive_name = args.archives 22 | 23 | train_dict = utils.read_dataset(root_dir, archive_name, dataset_name, True) 24 | x_train = train_dict[dataset_name][0] 25 | y_train = train_dict[dataset_name][1] 26 | graph_path = root_dir + '/graphs/' 27 | utils.create_directory(graph_path) 28 | utils.construct_knn_graph(x_train, y_train, graph_path + dataset_name + '.npy', n_jobs=32) 29 | -------------------------------------------------------------------------------- /losses/__init__.py: -------------------------------------------------------------------------------- 1 | import pkgutil 2 | 3 | __all__ = [] 4 | for loader, module_name, is_pkg in pkgutil.walk_packages(__path__): 5 | __all__.append(module_name) 6 | module = loader.find_module(module_name).load_module(module_name) 7 | exec('%s = module' % module_name) 8 | -------------------------------------------------------------------------------- /losses/losses.py: -------------------------------------------------------------------------------- 1 | """ 2 | Define loss used in the experiment 3 | The triplet loss is based on torch implementation https://github.com/White-Link/UnsupervisedScalableRepresentationLearningTimeSeries 4 | and article: 5 | Franceschi, J. Y., Dieuleveut, A., & Jaggi, M. (2019). 6 | Unsupervised scalable representation learning for multivariate time series 7 | 8 | Author: 9 | Baptiste Lafabregue 2019.25.04 10 | """ 11 | import numpy as np 12 | 13 | from tensorflow.keras import backend as K 14 | import tensorflow as tf 15 | 16 | import utils 17 | 18 | 19 | class TripletLoss(object): 20 | def __init__(self, encoder, train_set, compared_length, 21 | nb_random_samples, negative_penalty, fixed_time_dim=False): 22 | self.encoder = encoder 23 | self.train_set = train_set 24 | self.compared_length = compared_length 25 | self.nb_random_samples = nb_random_samples 26 | self.negative_penalty = negative_penalty 27 | self.fixed_time_dim = fixed_time_dim 28 | 29 | def compute_loss(self, batch, noisy_batch=None, training=True): 30 | batch_size = batch.shape[0] 31 | train_size = self.train_set.shape[0] 32 | length = min(self.compared_length, self.train_set.shape[1]) 33 | fixed_length = self.train_set.shape[1] 34 | 35 | # For each batch element, we pick nb_random_samples possible random 36 | # time series in the training set (choice of batches from where the 37 | # negative examples will be sampled) 38 | samples = np.random.choice( 39 | train_size, size=(self.nb_random_samples, batch_size) 40 | ) 41 | 42 | # Choice of length of positive and negative samples 43 | length_pos_neg = np.random.randint(1, high=length + 1) 44 | 45 | # We choose for each batch example a random interval in the time 46 | # series, which is the 'anchor' 47 | random_length = np.random.randint( 48 | length_pos_neg, high=length + 1 49 | ) # Length of anchors 50 | beginning_batches = np.random.randint( 51 | 0, high=length - random_length + 1, size=batch_size 52 | ) # Start of anchors 53 | 54 | # The positive samples are chosen at random in the chosen anchors 55 | beginning_samples_pos = np.random.randint( 56 | 0, high=random_length - length_pos_neg + 1, size=batch_size 57 | ) # Start of positive samples in the anchors 58 | # Start of positive samples in the batch examples 59 | beginning_positive = beginning_batches + beginning_samples_pos 60 | # End of positive samples in the batch examples 61 | end_positive = beginning_positive + length_pos_neg 62 | 63 | # We randomly choose nb_random_samples potential negative samples for 64 | # each batch example 65 | beginning_samples_neg = np.random.randint( 66 | 0, high=length - length_pos_neg + 1, 67 | size=(self.nb_random_samples, batch_size) 68 | ) 69 | anchor = K.concatenate([batch[j: j + 1, 70 | beginning_batches[j]: beginning_batches[j] + random_length, 71 | :, 72 | ] for j in range(batch_size)], axis=0) 73 | if self.fixed_time_dim: 74 | anchor = tf.pad(anchor, tf.constant([[0, 0], [0, fixed_length-random_length], [0, 0]])) 75 | 76 | representation = self.encoder( 77 | anchor, 78 | training=training 79 | ) # Anchors representations 80 | 81 | positive = K.concatenate([batch[j: j + 1, 82 | end_positive[j] - length_pos_neg: end_positive[j], 83 | :, 84 | ] for j in range(batch_size)], axis=0) 85 | if self.fixed_time_dim: 86 | positive = tf.pad(positive, tf.constant([[0, 0], [0, fixed_length-length_pos_neg], [0, 0]])) 87 | positive_representation = self.encoder( 88 | positive, 89 | training=training 90 | ) # Positive samples representations 91 | 92 | size_representation = K.int_shape(representation)[1] 93 | # Positive loss: -logsigmoid of dot product between anchor and positive 94 | # representations 95 | loss = -K.mean(K.log(K.sigmoid(K.batch_dot( 96 | K.reshape(representation, (batch_size, 1, size_representation)), 97 | K.reshape(positive_representation, (batch_size, size_representation, 1))) 98 | ))) 99 | 100 | multiplicative_ratio = self.negative_penalty / self.nb_random_samples 101 | for i in range(self.nb_random_samples): 102 | # Negative loss: -logsigmoid of minus the dot product between 103 | # anchor and negative representations 104 | negative = K.concatenate([self.train_set[samples[i, j]: samples[i, j] + 1] 105 | [:, 106 | beginning_samples_neg[i, j]:beginning_samples_neg[i, j] + length_pos_neg, 107 | :] for j in range(batch_size)], axis=0) 108 | if self.fixed_time_dim: 109 | negative = tf.pad(negative, tf.constant([[0, 0], [0, fixed_length-length_pos_neg], [0, 0]])) 110 | negative_representation = self.encoder( 111 | negative, 112 | training=training 113 | ) 114 | loss += multiplicative_ratio * -K.mean( 115 | K.log(K.sigmoid(-K.batch_dot( 116 | K.reshape(representation, (batch_size, 1, size_representation)), 117 | K.reshape(negative_representation, (batch_size, size_representation, 1)) 118 | ))) 119 | ) 120 | 121 | return loss 122 | 123 | 124 | class MSELoss(object): 125 | def __init__(self, autoencoder): 126 | self.autoencoder = autoencoder 127 | self.loss = tf.keras.losses.MeanSquaredError() 128 | 129 | def compute_loss(self, batch, noisy_batch=None, training=True): 130 | if noisy_batch is None: 131 | noisy_batch = batch 132 | decoding = self.autoencoder(noisy_batch, training=training) 133 | # y_pred = ops.convert_to_tensor(decoding) 134 | # # y_true = math_ops.cast(batch, y_pred.dtype) 135 | # # return K.mean(math_ops.squared_difference(y_pred, y_true)) 136 | return self.loss(batch, decoding) 137 | 138 | 139 | class JointLearningLoss(object): 140 | def __init__(self, layers_generator, hlayer_loss_param=0.1): 141 | if not layers_generator.support_joint_training: 142 | raise utils.CompatibilityException('architecture incompatible with Joint Learning loss') 143 | self.encoder = layers_generator.get_all_layers_encoder() 144 | self.decoder = layers_generator.get_all_layers_decoder() 145 | self.loss = tf.keras.losses.MeanSquaredError() 146 | self.hlayer_loss_param = hlayer_loss_param 147 | 148 | def compute_loss(self, batch, noisy_batch=None, training=True): 149 | if noisy_batch is None: 150 | noisy_batch = batch 151 | encoding = self.encoder(noisy_batch, training=training) 152 | decoding = self.decoder(encoding[-1], training=training) 153 | 154 | loss = 0 155 | for i in range(len(encoding) - 1): 156 | loss += self.hlayer_loss_param*self.loss(encoding[i], decoding[-2 - i]) 157 | 158 | loss += self.loss(batch, decoding[-1]) 159 | return loss 160 | 161 | 162 | class VAELoss(object): 163 | def __init__(self, encoder, decoder): 164 | self.encoder = encoder 165 | self.decoder = decoder 166 | if K.floatx() == 'float64': 167 | dtype = tf.dtypes.float64 168 | else: 169 | dtype = tf.dtypes.float32 170 | self.dtype = dtype 171 | 172 | def log_normal_pdf(self, sample, mean, logvar, raxis=1): 173 | log2pi = tf.math.log(2. * np.pi) 174 | log2pi = tf.cast(log2pi, self.dtype) 175 | return tf.reduce_sum(-.5 * ((sample - mean) ** 2. * tf.cast(tf.exp(-logvar), self.dtype) + logvar + log2pi), 176 | axis=raxis) 177 | 178 | def compute_loss(self, batch, noisy_batch=None, training=True): 179 | mean, logvar = tf.split(self.encoder(batch), num_or_size_splits=2, axis=1) 180 | epsilon = K.random_normal(shape=mean.shape) 181 | z = mean + K.exp(logvar / 2) * epsilon 182 | x_logit = self.decoder(z, training=training) 183 | 184 | # we use the classic mse because values are z-normalized (so not necessarily between 0 and 1) 185 | recon = K.sum(tf.keras.losses.mean_squared_error(batch, x_logit), axis=1) 186 | 187 | kl = 0.5 * K.sum(K.exp(logvar) + K.square(mean) - 1. - logvar, axis=1) 188 | 189 | return recon + kl 190 | 191 | 192 | class SiameseTSLoss(object): 193 | def __init__(self, autoencoder1, autoencoder2, filter1, filter2): 194 | self.autoencoder1 = autoencoder1 195 | self.autoencoder2 = autoencoder2 196 | self.filter1 = filter1 197 | self.filter2 = filter2 198 | self.loss = tf.keras.losses.MeanSquaredError() 199 | 200 | def compute_loss(self, batch, noisy_batch=None, training=True): 201 | if noisy_batch is None: 202 | noisy_batch = batch 203 | decoding1 = self.autoencoder1(noisy_batch, training=training) 204 | loss1 = self.loss(self.filter1(batch), decoding1) 205 | decoding2 = self.autoencoder2(noisy_batch, training=training) 206 | loss2 = self.loss(self.filter2(batch), decoding2) 207 | return loss1 + loss2 208 | 209 | 210 | class CombinedLoss(object): 211 | def __init__(self, losses, weights=None): 212 | self.losses = losses 213 | self.weights = weights 214 | if self.weights is None: 215 | self.weights = np.ones_like(self.losses) 216 | 217 | def compute_loss(self, batch, noisy_batch=None, training=True): 218 | total_loss = 0 219 | for loss, weight in zip(self.losses, self.weights): 220 | total_loss += weight * loss.compute_loss(batch, noisy_batch=noisy_batch, training=training) 221 | return total_loss 222 | -------------------------------------------------------------------------------- /merge_trainers.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script to merge representation (used for TripletKcombine loss) 3 | 4 | Author: 5 | Baptiste Lafabregue 2019.25.04 6 | """ 7 | import os 8 | import numpy as np 9 | import argparse 10 | 11 | from sklearn import metrics 12 | from sklearn.cluster import KMeans 13 | 14 | import utils 15 | from networks.trainer import get_logs 16 | 17 | TRANSLATE_DICT = {'GAN': 'ClusterGAN', 'DEC': 'IDEC'} 18 | 19 | 20 | def gen_triplet_list(prefix='dilated_cnn_tripletK', suffix='_None_0.0', itrs=['1', '2', '5', '10']): 21 | list = [] 22 | for k in itrs: 23 | list.append(prefix + k + suffix) 24 | return list 25 | 26 | 27 | def merge(methods_to_merge, itr, seed_itrs, output_method, stats_dir, dataset_names=None, 28 | archive_name='UCRArchive_2018'): 29 | # extract representation of each method 30 | if dataset_names is None: 31 | dataset_names = [name for name in os.listdir('./ae_weights/' + itr + '/' + methods_to_merge[0])] 32 | dataset_names.sort() 33 | 34 | types = ['_test', 'train'] 35 | if type == 'test': 36 | types = ['_test'] 37 | elif type == 'train': 38 | types = '' 39 | 40 | for seed_itr in seed_itrs: 41 | for dataset in dataset_names: 42 | print(dataset) 43 | train_dict = utils.read_dataset('.', archive_name, dataset, True) 44 | y_train = train_dict[dataset][1] 45 | nb_classes = np.shape(np.unique(y_train, return_counts=True)[1])[0] 46 | 47 | test_dict = utils.read_dataset('.', archive_name, dataset, False) 48 | y_test = test_dict[dataset][1] 49 | 50 | representations_train = [] 51 | representations_test = [] 52 | last_logs_train = {'ari': 0.0, 'nmi': 0.0, 'acc': 0.0, 53 | 'max_ari': -1.0, 'max_nmi': -1.0, 'max_acc': -1.0} 54 | last_logs_test = {'ari': 0.0, 'nmi': 0.0, 'acc': 0.0, 55 | 'max_ari': -1.0, 'max_nmi': -1.0, 'max_acc': -1.0} 56 | try: 57 | for method in methods_to_merge: 58 | enc_type = method.split('_')[-1] 59 | # special case for DEC that use the IDEC object 60 | if enc_type in TRANSLATE_DICT: 61 | enc_type = TRANSLATE_DICT[enc_type] 62 | if not os.path.exists('./ae_weights/' + itr + '/' + method + '/' + dataset + '/' + enc_type): 63 | enc_type = 'pre_train_encoder' 64 | file = './ae_weights/' + itr + '/' + method + '/' + dataset + '/' + enc_type + '/x_encoded_'\ 65 | + seed_itr + '.npy' 66 | representations_train.append(np.load(file)) 67 | file = './ae_weights/' + itr + '/' + method + '/' + dataset + '/' + enc_type + '/x_test_encoded_'\ 68 | + seed_itr + '.npy' 69 | representations_test.append(np.load(file)) 70 | 71 | x_train = np.concatenate(representations_train, axis=1) 72 | kmeans_train = KMeans(n_clusters=nb_classes, n_init=20) 73 | y_pred_train = kmeans_train.fit_predict(x_train) 74 | last_logs_train['acc'] = np.round(utils.cluster_acc(y_train, y_pred_train), 5) 75 | last_logs_train['nmi'] = np.round(metrics.normalized_mutual_info_score(y_train, y_pred_train), 5) 76 | last_logs_train['ari'] = np.round(metrics.adjusted_rand_score(y_train, y_pred_train), 5) 77 | 78 | x_test = np.concatenate(representations_test, axis=1) 79 | kmeans_test = KMeans(n_clusters=nb_classes, n_init=20) 80 | y_pred_test = kmeans_test.fit_predict(x_test) 81 | last_logs_test['acc'] = np.round(utils.cluster_acc(y_test, y_pred_test), 5) 82 | last_logs_test['nmi'] = np.round(metrics.normalized_mutual_info_score(y_test, y_pred_test), 5) 83 | last_logs_test['ari'] = np.round(metrics.adjusted_rand_score(y_test, y_pred_test), 5) 84 | 85 | save_dir = './ae_weights/' + itr + '/' + output_method + '/' + dataset 86 | utils.create_directory(save_dir + '/pre_train_encoder') 87 | np.save(save_dir + '/pre_train_encoder/kmeans_affect_' + str(seed_itr) + '.npy', y_pred_test) 88 | np.save(save_dir + '/pre_train_encoder/kmeans_clusters_' + str(seed_itr) + '.npy', 89 | kmeans_test.cluster_centers_) 90 | np.save(save_dir + '/pre_train_encoder/x_encoded_' + str(seed_itr) + '.npy', x_train) 91 | np.save(save_dir + '/pre_train_encoder/x_test_encoded_' + str(seed_itr) + '.npy', x_test) 92 | 93 | itr_stats_dir = stats_dir + itr + '/' + seed_itr + '/' + output_method 94 | f = open(itr_stats_dir + "_" + dataset, "w") 95 | f.write(get_logs(last_logs_train) + '\n') 96 | f.write(get_logs(last_logs_test) + '\n') 97 | f.close() 98 | except: 99 | print("error on dataset " + dataset + " itr " + seed_itr) 100 | 101 | 102 | if __name__ == '__main__': 103 | parser = argparse.ArgumentParser( 104 | description='Compute combined kmeans' 105 | ) 106 | parser.add_argument('--dataset', type=str, metavar='d', default=None, 107 | help='dataset name') 108 | parser.add_argument('--archives', type=str, metavar='DIR', required=True, 109 | help='archive name') 110 | parser.add_argument('--itr', type=str, metavar='d', default='20000', 111 | help='iteration number') 112 | parser.add_argument('--prefix', type=str, metavar='d', default='dilated_cnn_tripletK', 113 | help='prefix to use to select methods') 114 | parser.add_argument('--suffix', type=str, metavar='d', default='_None_0.0', 115 | help='suffix to use to select methods') 116 | args = parser.parse_args() 117 | itr = args.itr 118 | prefix = args.prefix 119 | suffix = args.suffix 120 | dataset_name = args.dataset 121 | archive_name = args.archives 122 | if dataset_name is not None: 123 | dataset_name = [dataset_name] 124 | 125 | output_method = prefix + 'combined' + suffix 126 | methods_to_merge = gen_triplet_list(prefix=prefix, suffix=suffix) 127 | seed_itr = ['0', '1', '2', '3', '4'] 128 | merge(methods_to_merge, itr, seed_itr, output_method, './stats/', dataset_names=dataset_name, 129 | archive_name=archive_name) 130 | -------------------------------------------------------------------------------- /networks/ClusterGAN.py: -------------------------------------------------------------------------------- 1 | """ 2 | Based on tensorflow implementation https://github.com/sudiptodip15/ClusterGAN 3 | and article : 4 | Sudipto Mukherjee, Himanshu Asnani, Eugene Lin and Sreeram Kannan, 5 | ClusterGAN : Latent Space Clustering in Generative Adversarial Networks 6 | 7 | Author: 8 | Baptiste Lafabregue 2019.25.04 9 | """ 10 | 11 | import numpy as np 12 | import os 13 | import csv 14 | import copy 15 | 16 | import tensorflow as tf 17 | import tensorflow.keras.backend as K 18 | from tensorflow.keras import Model, Input 19 | from tensorflow.keras.layers import Dense, Activation 20 | 21 | from networks.trainer import Trainer 22 | import utils 23 | 24 | 25 | def sample_z(batch, z_dim, sampler='one_hot', num_class=10, n_cat=1, label_index=None): 26 | if sampler == 'mul_cat': 27 | if label_index is None: 28 | label_index = np.random.randint(low=0, high=num_class, size=batch) 29 | return np.hstack((0.10 * np.random.randn(batch, z_dim - num_class * n_cat), 30 | np.tile(np.eye(num_class)[label_index], (1, n_cat)))) 31 | elif sampler == 'one_hot': 32 | if label_index is None: 33 | label_index = np.random.randint(low=0, high=num_class, size=batch) 34 | return np.hstack((0.10 * np.random.randn(batch, z_dim - num_class), np.eye(num_class)[label_index])) 35 | elif sampler == 'uniform': 36 | return np.random.uniform(-1., 1., size=[batch, z_dim]) 37 | elif sampler == 'normal': 38 | return 0.15 * np.random.randn(batch, z_dim) 39 | elif sampler == 'mix_gauss': 40 | if label_index is None: 41 | label_index = np.random.randint(low=0, high=num_class, size=batch) 42 | return 0.1 * np.random.randn(batch, z_dim) + np.eye(num_class, z_dim)[label_index] 43 | 44 | 45 | class ClusterGAN(Trainer): 46 | def __init__(self, 47 | dataset_name, 48 | classifier_name, 49 | encoder, 50 | generator, 51 | discriminator, 52 | n_clusters=10, 53 | batch_size=10, 54 | beta_cycle_gen=2, 55 | beta_cycle_label=2, 56 | optimizer=None): 57 | super(ClusterGAN, self).__init__(dataset_name, classifier_name, None, batch_size, n_clusters, optimizer) 58 | 59 | self.encoder = encoder 60 | self.discriminator = discriminator 61 | self.generator = generator 62 | self.beta_cycle_gen = beta_cycle_gen 63 | self.beta_cycle_label = beta_cycle_label 64 | self.allow_pretrain = False 65 | if optimizer is None: 66 | optimizer = tf.keras.optimizers.legacy.Adam(lr=1e-4) 67 | self.optimizer_discriminator = copy.deepcopy(optimizer) 68 | self.optimizer_generator = copy.deepcopy(optimizer) 69 | self.dim_gen = 0 70 | self.zdim = 0 71 | self.scale = 10 72 | 73 | def initialize_model(self, x, y, ae_weights=None): 74 | """ 75 | Initialize the model for training 76 | :param ae_weights: arguments to let the encoder load its weights, None to pre-train the encoder 77 | """ 78 | self.pretrain_model = False 79 | input_shape = K.int_shape(self.encoder.input)[1:] 80 | enc_input = Input(shape=input_shape) 81 | enc_output = self.encoder(enc_input) 82 | enc_shape = K.int_shape(self.encoder.output) 83 | self.dim_gen = enc_shape[1] - self.n_clusters 84 | self.zdim = enc_shape[1] 85 | self.xdim = x.shape[1] * x.shape[2] 86 | 87 | disc_input = Input(shape=input_shape) 88 | disc_output = self.discriminator(disc_input) 89 | disc_output = Dense(1)(disc_output) 90 | disc_output = Activation('sigmoid')(disc_output) 91 | self.discriminator = Model(inputs=disc_input, outputs=disc_output) 92 | 93 | logits = enc_output[:, self.dim_gen:] 94 | y = tf.nn.softmax(logits) 95 | self.encoder = Model(inputs=enc_input, outputs=[enc_output[:, 0:self.dim_gen], y, logits]) 96 | 97 | def load_weights(self, weights_path): 98 | """ 99 | Load weights of IDEC model 100 | :param weights_path: path to load weights from 101 | """ 102 | 103 | def save_weights(self, weights_path): 104 | """ 105 | Save weights of IDEC model 106 | :param weights_path: path to save weights to 107 | """ 108 | 109 | def extract_features(self, x): 110 | """ 111 | Extract features from the encoder (before the clustering layer) 112 | :param x: the data to extract features from 113 | :return: the encoded features 114 | """ 115 | return self.encoder.predict(x)[0] 116 | 117 | def reconstruct_features(self, x): 118 | """ 119 | Reconstruct features from the autoencoder (encode and decode) 120 | :param x: the data to reconstruct features from 121 | :return: the reconstructed features, None if not supported 122 | """ 123 | return None 124 | 125 | def _run_training(self, x, y, x_test, y_test, nb_steps, 126 | seeds, verbose, log_writer, dist_matrix=None): 127 | discriminator_steps = 4 128 | 129 | i = 0 # Number of performed optimization steps 130 | epoch = 0 # Number of performed epochs 131 | 132 | # define the train function 133 | discriminator_loss = tf.keras.metrics.Mean(name='discriminator train_loss') 134 | generator_loss = tf.keras.metrics.Mean(name='generator train_loss') 135 | 136 | @tf.function 137 | def train_discriminator(x_batch, y_real, y_fake): 138 | with tf.GradientTape() as tape: 139 | x_batch_size = K.int_shape(x_batch)[0] 140 | z = sample_z(x_batch_size, self.zdim, num_class=self.n_clusters) 141 | x_gen = self.generator(z) 142 | 143 | disc_pred = self.discriminator(x_batch) 144 | disc_pred_gen = self.discriminator(x_gen) 145 | real_loss = tf.keras.losses.binary_crossentropy(y_real, disc_pred) 146 | fake_loss = tf.keras.losses.binary_crossentropy(y_fake, disc_pred_gen) 147 | loss = real_loss + fake_loss 148 | 149 | gradients = tape.gradient(loss, self.discriminator.trainable_variables) 150 | self.optimizer_discriminator.apply_gradients(zip(gradients, self.discriminator.trainable_variables)) 151 | 152 | discriminator_loss(loss) 153 | 154 | @tf.function 155 | def train_generator(): 156 | with tf.GradientTape() as tape: 157 | z = sample_z(self.batch_size, self.zdim, num_class=self.n_clusters) 158 | z_gen = z[:, 0:self.dim_gen] 159 | z_hot = z[:, self.dim_gen:] 160 | x_gen = self.generator(z) 161 | 162 | z_enc_gen, z_enc_label, z_enc_logits = self.encoder(x_gen) 163 | disc_pred_gen = self.discriminator(x_gen) 164 | 165 | # v_loss = tf.reduce_mean(disc_pred_gen) 166 | dtype = tf.float32 167 | if K.floatx() == 'float64': 168 | dtype = tf.float64 169 | v_loss = tf.keras.losses.binary_crossentropy(disc_pred_gen, tf.ones((self.batch_size, 1), dtype=dtype)) 170 | a = self.beta_cycle_gen * tf.reduce_mean(tf.square(z_gen - z_enc_gen)) 171 | b = self.beta_cycle_label * tf.reduce_mean( 172 | tf.nn.softmax_cross_entropy_with_logits(logits=z_enc_logits, labels=z_hot)) 173 | loss = v_loss + a + b 174 | 175 | gradients = tape.gradient(loss, self.encoder.trainable_variables + self.generator.trainable_variables) 176 | self.optimizer_generator.apply_gradients( 177 | zip(gradients, self.encoder.trainable_variables + self.generator.trainable_variables)) 178 | 179 | generator_loss(loss) 180 | 181 | if verbose: 182 | print('start training') 183 | 184 | while i < nb_steps: 185 | discriminator_loss.reset_states() 186 | generator_loss.reset_states() 187 | 188 | train_ds = tf.data.Dataset.from_tensor_slices(x) \ 189 | .shuffle(x.shape[0], reshuffle_each_iteration=True) \ 190 | .batch(self.batch_size).as_numpy_iterator() 191 | sub_i = 0 192 | # Train discriminator 193 | for x_batch in train_ds: 194 | x_batch_size = K.int_shape(x_batch)[0] 195 | z = sample_z(x_batch_size, self.zdim, num_class=self.n_clusters) 196 | x_gen = self.generator(z) 197 | 198 | # disc_pred = self.discriminator(x_batch) 199 | # disc_pred_gen = self.discriminator(x_gen) 200 | train_discriminator(x_batch, np.ones(shape=(x_batch.shape[0], 1)), 201 | np.zeros(shape=(x_batch.shape[0], 1))) 202 | sub_i += 1 203 | if sub_i >= discriminator_steps: 204 | break 205 | 206 | # Train generator 207 | train_generator() 208 | z = sample_z(self.batch_size, self.zdim, num_class=self.n_clusters) 209 | z_gen = z[:, 0:self.dim_gen] 210 | z_hot = z[:, self.dim_gen:] 211 | x_gen = self.generator(z) 212 | 213 | z_enc_gen, z_enc_label, z_enc_logits = self.encoder(x_gen) 214 | disc_pred_gen = self.discriminator(x_gen) 215 | 216 | a = tf.reduce_mean(disc_pred_gen) + self.beta_cycle_gen * tf.reduce_mean(tf.square(z_gen - z_enc_gen)) 217 | b = self.beta_cycle_label * tf.reduce_mean( 218 | tf.nn.softmax_cross_entropy_with_logits(logits=z_enc_logits, labels=z_hot)) 219 | z_enc_gen = z_enc_gen.numpy() 220 | z_enc_logits = z_enc_logits.numpy() 221 | z_enc_label = z_enc_label.numpy() 222 | x_gen = np.reshape(x_gen.numpy(), (self.batch_size, -1)) 223 | loss = a + b 224 | epoch += 1 225 | 226 | i += 1 227 | if i >= nb_steps: 228 | break 229 | 230 | if verbose: 231 | template = 'Epoch {}, Loss discriminator : {}, Loss generator : {}' 232 | print(template.format(epoch + 1, discriminator_loss.result(), generator_loss.result())) 233 | 234 | y_pred = self.log_stats(x, y, 235 | x_test, y_test, 236 | [discriminator_loss.result() + generator_loss.result(), 237 | discriminator_loss.result(), generator_loss.result()], 238 | epoch, 239 | log_writer, 240 | 'train') 241 | 242 | return epoch 243 | -------------------------------------------------------------------------------- /networks/DEPICT.py: -------------------------------------------------------------------------------- 1 | """ 2 | Based on Theano implementation https://github.com/herandy/DEPICT 3 | and article : 4 | Dizaji, K. G., Herandi, A., & Huang, H. (2017). 5 | Deep clustering via joint convolutional autoencoder embedding and relative entropy minimization 6 | 7 | Author: 8 | Baptiste Lafabregue 2019.25.04 9 | """ 10 | import numpy as np 11 | from sklearn import metrics 12 | from sklearn.cluster import KMeans 13 | 14 | import tensorflow as tf 15 | from tensorflow.keras.models import Model 16 | 17 | from networks.trainer import Trainer 18 | 19 | 20 | class DEPICT(Trainer): 21 | def __init__(self, 22 | dataset_name, 23 | classifier_name, 24 | encoder_model, 25 | keep_both_losses=True, 26 | gamma=0.1, 27 | n_clusters=10, 28 | alpha=1.0, 29 | batch_size=10, 30 | tol=1e-3, 31 | update_interval=5, 32 | pred_normalizition_flag=True, 33 | optimizer=None): 34 | 35 | super(DEPICT, self).__init__(dataset_name, classifier_name, encoder_model, batch_size, n_clusters, optimizer) 36 | 37 | self.keep_both_losses = keep_both_losses 38 | self.gamma = gamma 39 | self.alpha = alpha 40 | self.tol = tol 41 | self.update_interval = update_interval 42 | self.pred_normalizition_flag = pred_normalizition_flag 43 | self.clust_model = None 44 | self.clust_loss = None 45 | 46 | def initialize_model(self, x, y, ae_weights=None): 47 | """ 48 | Initialize the model for training 49 | :param ae_weights: arguments to let the encoder load its weights, None to pre-train the encoder 50 | """ 51 | if ae_weights is not None: 52 | self.encoder_model.load_weights(ae_weights) 53 | print('Pretrained AE weights are loaded successfully.') 54 | self.pretrain_model = False 55 | else: 56 | self.pretrain_model = True 57 | 58 | if self.optimizer is None: 59 | self.optimizer = tf.keras.optimizers.legacy.Adam() 60 | 61 | clustering_layer = tf.keras.layers.Dense(self.n_clusters, name='clustering')(self.encoder.output) 62 | clustering_layer = tf.keras.layers.Softmax()(clustering_layer) 63 | self.clust_model = Model(inputs=self.encoder.input, outputs=clustering_layer) 64 | self.clust_loss = tf.keras.losses.CategoricalCrossentropy() 65 | 66 | def load_weights(self, weights_path): 67 | """ 68 | Load weights of IDEC model 69 | :param weights_path: path to load weights from 70 | """ 71 | self.clust_model.load_weights(weights_path + '.tf') 72 | 73 | def save_weights(self, weights_path): 74 | """ 75 | Save weights of IDEC model 76 | :param weights_path: path to save weights to 77 | """ 78 | self.clust_model.save_weights(weights_path + '.tf') 79 | 80 | def predict_clusters(self, x, seeds=None): 81 | """ 82 | Predict cluster labels using the output of clustering layer 83 | :param x: the data to evaluate 84 | :param seeds: seeds to initialize the K-Means if needed 85 | :return: the predicted cluster labels 86 | """ 87 | q = self.clust_model.predict(x, verbose=0) 88 | y_pred = q.argmax(1) 89 | 90 | return y_pred, np.transpose(self.clust_model.get_layer(name='clustering').get_weights()[0]) 91 | 92 | def _run_training(self, x, y, x_test, y_test, nb_steps, 93 | seeds, verbose, log_writer, dist_matrix=None): 94 | if seeds is not None: 95 | seeds_enc = self.extract_features(seeds) 96 | kmeans = KMeans(n_clusters=self.n_clusters, n_init=20, init=seeds_enc) 97 | else: 98 | kmeans = KMeans(n_clusters=self.n_clusters, n_init=20) 99 | x_pred = self.extract_features(x) 100 | y_pred = kmeans.fit_predict(x_pred) 101 | 102 | centroids = kmeans.cluster_centers_.T 103 | centroids = centroids / np.sqrt(np.diag(np.matmul(centroids.T, centroids))) 104 | self.clust_model.get_layer(name='clustering').set_weights([centroids, np.zeros((self.n_clusters,))]) 105 | 106 | if y is not None: 107 | ari = np.round(metrics.adjusted_rand_score(y, y_pred), 5) 108 | if verbose: 109 | print('ari kmeans: ', str(ari)) 110 | self.log_stats(x, y, x_test, y_test, [0, 0, 0], 0, log_writer, 'init') 111 | 112 | i = 0 # Number of performed optimization steps 113 | epoch = 0 # Number of performed epochs 114 | 115 | # evaluate the clustering performance 116 | target_pred = self.clust_model.predict(x, verbose=0) 117 | if self.pred_normalizition_flag: 118 | cluster_frequency = np.sum(target_pred, axis=0) # avoid unbalanced assignment 119 | target_pred = target_pred ** 2 / cluster_frequency 120 | # y_prob = y_prob / np.sqrt(cluster_frequency) 121 | target_pred = np.transpose(target_pred.T / np.sum(target_pred, axis=1)) 122 | target_pred_last = target_pred 123 | 124 | # define the train function 125 | train_enc_loss = tf.keras.metrics.Mean(name='encoder train_loss') 126 | clust_enc_loss = tf.keras.metrics.Mean(name='clustering train_loss') 127 | depict_enc_loss = tf.keras.metrics.Mean(name='DEPICT train_loss') 128 | 129 | @tf.function 130 | def train_step(x_batch, target_batch): 131 | with tf.GradientTape() as tape: 132 | encoder_loss = self.encoder_model.loss.compute_loss(x_batch, training=True) 133 | encoding_x = self.clust_model(x_batch, training=True) 134 | depict_loss = self.clust_loss(target_batch, encoding_x) 135 | loss = (1 - self.gamma) * encoder_loss + self.gamma * depict_loss 136 | gradients = tape.gradient(loss, 137 | self.encoder_model.get_trainable_variables() + self.clust_model.trainable_variables) 138 | self.optimizer.apply_gradients( 139 | zip(gradients, self.encoder_model.get_trainable_variables() + self.clust_model.trainable_variables)) 140 | 141 | train_enc_loss(encoder_loss) 142 | clust_enc_loss(loss) 143 | depict_enc_loss(loss) 144 | 145 | if verbose: 146 | print('start training') 147 | # idec training 148 | while i < nb_steps: 149 | train_enc_loss.reset_states() 150 | clust_enc_loss.reset_states() 151 | depict_enc_loss.reset_states() 152 | # shuffle the train set 153 | 154 | # computes P each update_interval epoch 155 | if epoch % self.update_interval == 0: 156 | # evaluate the clustering performance 157 | target_pred = self.clust_model.predict(x, verbose=0) 158 | if self.pred_normalizition_flag: 159 | cluster_frequency = np.sum(target_pred, axis=0) # avoid unbalanced assignment 160 | target_pred = target_pred ** 2 / cluster_frequency 161 | # y_prob = y_prob / np.sqrt(cluster_frequency) 162 | target_pred = np.transpose(target_pred.T / np.sum(target_pred, axis=1)) 163 | delta_label = np.sum(target_pred != target_pred_last).astype(np.float32) / target_pred.shape[0] 164 | target_pred_last = target_pred 165 | 166 | # check stop criterion 167 | if epoch > 0 and delta_label < self.tol: 168 | if verbose: 169 | print('delta_label ', delta_label, '< tol ', self.tol) 170 | print('Reached tolerance threshold. Stopping training.') 171 | self.log_stats(x, y, x_test, y_test, [0, 0, 0], 172 | epoch, log_writer, 'reached_stop_criterion') 173 | break 174 | 175 | train_ds = tf.data.Dataset.from_tensor_slices((x, target_pred)) \ 176 | .shuffle(x.shape[0], reshuffle_each_iteration=True) \ 177 | .batch(self.batch_size).as_numpy_iterator() 178 | 179 | for x_batch, target_batch in train_ds: 180 | train_step(x_batch, target_batch) 181 | i += 1 182 | if i >= nb_steps: 183 | break 184 | 185 | if verbose: 186 | template = 'Epoch {}, Loss: {}' 187 | print(template.format(epoch + 1, depict_enc_loss.result())) 188 | epoch += 1 189 | 190 | y_pred = self.log_stats(x, y, 191 | x_test, y_test, 192 | [depict_enc_loss.result(), clust_enc_loss.result(), train_enc_loss.result()], 193 | epoch, 194 | log_writer, 195 | 'train') 196 | 197 | return epoch 198 | -------------------------------------------------------------------------------- /networks/DTCR.py: -------------------------------------------------------------------------------- 1 | """ 2 | Based on tensorflow implementation https://github.com/qianlima-lab/DTCR 3 | and article : 4 | Ma, Q., Zheng, J., Li, S., & Cottrell, G. W. (2019), 5 | Learning Representations for Time Series Clustering 6 | 7 | Author: 8 | Baptiste Lafabregue 2019.25.04 9 | """ 10 | 11 | import numpy as np 12 | import tensorflow as tf 13 | import tensorflow.keras.backend as K 14 | from sklearn import metrics 15 | from sklearn.cluster import KMeans 16 | 17 | import utils 18 | from networks.trainer import Trainer 19 | 20 | 21 | class DTCR(Trainer): 22 | def __init__(self, 23 | dataset_name, 24 | classifier_name, 25 | encoder_model, 26 | keep_both_losses=True, 27 | lamda=1, 28 | n_clusters=10, 29 | batch_size=10, 30 | classification_dim=128, 31 | update_interval=1, 32 | optimizer=None): 33 | 34 | super(DTCR, self).__init__(dataset_name, classifier_name, encoder_model, batch_size, n_clusters, optimizer) 35 | 36 | self.keep_both_losses = keep_both_losses 37 | self.lamda = lamda 38 | self.update_interval = update_interval 39 | self.pretrain_model = True 40 | self.classification_dim = classification_dim 41 | self.classification_model = None 42 | self.classification_loss = tf.keras.losses.CategoricalCrossentropy( 43 | reduction=tf.keras.losses.Reduction.SUM_OVER_BATCH_SIZE) 44 | 45 | def initialize_model(self, x, y, ae_weights=None): 46 | """ 47 | Initialize the model for training 48 | :param ae_weights: arguments to let the encoder load its weights, None to pre-train the encoder 49 | """ 50 | if ae_weights is not None: 51 | self.encoder_model.load_weights(ae_weights) 52 | print('Pretrained AE weights are loaded successfully.') 53 | self.pretrain_model = False 54 | else: 55 | self.pretrain_model = True 56 | 57 | if self.optimizer is None: 58 | self.optimizer = tf.keras.optimizers.legacy.Adam(learning_rate=5e-3) 59 | 60 | # fake/real classification task 61 | inputs = self.encoder.inputs 62 | h = self.encoder(inputs) 63 | h = tf.keras.layers.Dense(self.classification_dim)(h) 64 | h = tf.keras.layers.Dense(2, activation='softmax')(h) 65 | self.classification_model = tf.keras.Model(inputs=inputs, outputs=h) 66 | 67 | def load_weights(self, weights_path): 68 | """ 69 | Load weights of IDEC model 70 | :param weights_path: path to load weights from 71 | """ 72 | self.encoder_model.load_weights(weights_path) 73 | 74 | def save_weights(self, weights_path): 75 | """ 76 | Save weights of IDEC model 77 | :param weights_path: path to save weights to 78 | """ 79 | self.encoder_model.save_weights(weights_path) 80 | 81 | def _construct_F(self, H, k): 82 | # u, s, vh = np.linalg.svd(H, full_matrices=True) 83 | # F = np.transpose(vh[:k]) 84 | kmeans = KMeans(n_clusters=self.n_clusters, n_init=20) 85 | 86 | y_pred = kmeans.fit_predict(H) 87 | F = np.zeros(shape=(self.n_clusters, len(y_pred))) 88 | for i in range(len(y_pred)): 89 | F[y_pred[i]][i] = 1.0 90 | F = [sub * 1 / np.sqrt(np.sum(sub)) for sub in F] 91 | F = np.transpose(F) 92 | return F 93 | 94 | def _compute_loss(self, H, F): 95 | H_transpose = K.transpose(H) 96 | F_transpose = K.transpose(F) 97 | 98 | # Tr(HT H) − Tr(FT HT H F) 99 | # loss = tf.linalg.trace(K.dot(H_transpose, H)) - \ 100 | # tf.linalg.trace(K.dot(K.dot(F_transpose, H_transpose), 101 | # K.dot(H, F))) 102 | HTH = K.dot(H_transpose, H) 103 | FTHTHF= K.dot(K.dot(F_transpose, HTH), F) 104 | loss = tf.linalg.trace(HTH) - tf.linalg.trace(FTHTHF) 105 | 106 | return loss 107 | 108 | def _run_training(self, x, y, x_test, y_test, nb_steps, 109 | seeds, verbose, log_writer, dist_matrix=None): 110 | 111 | if seeds is not None: 112 | seeds_enc = self.extract_features(seeds) 113 | kmeans = KMeans(n_clusters=self.n_clusters, n_init=20, init=seeds_enc) 114 | else: 115 | kmeans = KMeans(n_clusters=self.n_clusters, n_init=20) 116 | x_pred = self.extract_features(x) 117 | y_pred = kmeans.fit_predict(x_pred) 118 | 119 | if y is not None: 120 | ari = np.round(metrics.adjusted_rand_score(y, y_pred), 5) 121 | if verbose: 122 | print('ari kmeans: ', str(ari)) 123 | self.log_stats_encoder(x, y, x_test, y_test, [0, 0, 0], 0, log_writer, 'init') 124 | 125 | # initialize cluster centers using k-means 126 | if verbose: 127 | print('Initializing F (k-means loss).') 128 | 129 | x_pred = self.extract_features(x) 130 | H = np.transpose(x_pred) 131 | # first on is initialized with k-means 132 | F = self._construct_F(H, self.n_clusters) 133 | if seeds is not None: 134 | seeds_enc = self.extract_features(seeds) 135 | kmeans = KMeans(n_clusters=self.n_clusters, n_init=20, init=seeds_enc) 136 | else: 137 | kmeans = KMeans(n_clusters=self.n_clusters, n_init=20) 138 | y_pred = kmeans.fit_predict(x_pred) 139 | F = np.zeros(shape=(self.n_clusters, len(y_pred))) 140 | for i in range(len(y_pred)): 141 | F[y_pred[i]][i] = 1.0 142 | F = [sub * 1 / np.sqrt(np.sum(sub)) for sub in F] 143 | F = np.transpose(F) 144 | 145 | i = 0 # Number of performed optimization steps 146 | epoch = 0 # Number of performed epochs 147 | 148 | # define the train function 149 | train_enc_loss = tf.keras.metrics.Mean(name='encoder train_loss') 150 | dec_enc_loss = tf.keras.metrics.Mean(name='dec train_loss') 151 | idec_enc_loss = tf.keras.metrics.Mean(name='idec train_loss') 152 | 153 | @tf.function 154 | def train_step(x_batch, f_batch, fake_batch, classif_true): 155 | f_batch = tf.convert_to_tensor(f_batch) 156 | with tf.GradientTape() as tape: 157 | # reconstruction loss 158 | encoder_loss = self.encoder_model.loss.compute_loss(x_batch, training=True) 159 | # k-means loss 160 | encoding_x = K.transpose(self.encoder(x_batch)) 161 | kmeans_loss = self._compute_loss(encoding_x, f_batch) 162 | # fake/real classificationloss 163 | classif_pred = self.classification_model(fake_batch) 164 | classification_loss = self.classification_loss(classif_true, classif_pred) 165 | loss = encoder_loss + classification_loss + self.lamda/2 * kmeans_loss 166 | gradients = tape.gradient(loss, 167 | self.encoder_model.get_trainable_variables() + 168 | self.classification_model.trainable_variables) 169 | self.optimizer.apply_gradients(zip(gradients, self.encoder_model.get_trainable_variables() + 170 | self.classification_model.trainable_variables)) 171 | 172 | dec_enc_loss(loss) 173 | idec_enc_loss(loss) 174 | 175 | return encoder_loss, kmeans_loss, classification_loss 176 | 177 | if verbose: 178 | print('start training') 179 | while i < nb_steps: 180 | train_enc_loss.reset_states() 181 | dec_enc_loss.reset_states() 182 | idec_enc_loss.reset_states() 183 | fake_x = utils.generate_fake_samples(x) 184 | # shuffle the train set 185 | 186 | # computes P each update_interval epoch 187 | if epoch % self.update_interval == 0 and epoch != 0: 188 | print('update F') 189 | # H = np.transpose(self.encoder.predict(x)) 190 | F = self._construct_F(self.encoder.predict(x), self.n_clusters) 191 | 192 | # # evaluate the clustering performance 193 | # delta_label = np.sum(y_pred != y_pred_last).astype(np.float32) / y_pred.shape[0] 194 | # F_last = F 195 | 196 | # check stop criterion 197 | # if epoch > 0 and delta_label < tol: 198 | # if verbose: 199 | # print('delta_label ', delta_label, '< tol ', tol) 200 | # print('Reached tolerance threshold. Stopping training.') 201 | # self.log_stats_encoder(x, y, x_test, y_test, [0, 0, 0], 202 | # epoch, log_writer, 'reached_stop_criterion') 203 | # break 204 | 205 | train_ds = tf.data.Dataset.from_tensor_slices((x, F, fake_x)) \ 206 | .shuffle(x.shape[0], reshuffle_each_iteration=True) \ 207 | .batch(self.batch_size).as_numpy_iterator() 208 | 209 | for x_batch, f_batch, fake_batch in train_ds: 210 | # this is the labels for real/fake loss, it is always in the same order 211 | classif_true = np.concatenate([np.zeros(len(x_batch)), 212 | np.ones(len(x_batch))]) 213 | classif_true = classif_true.astype(int) 214 | n_values = np.max(classif_true) + 1 215 | classif_true = np.eye(n_values)[classif_true] 216 | 217 | # train_step(x_batch, f_batch, np.concatenate([x_batch, fake_batch])) 218 | # encoder_loss, kmeans_loss = train_step(x_batch, f_batch, np.concatenate([x_batch, fake_batch])) 219 | encoder_loss, kmeans_loss, classification_loss = train_step(x_batch, 220 | # encoder_loss, classification_loss = train_step(x_batch, 221 | f_batch, 222 | np.concatenate([x_batch, fake_batch]), 223 | classif_true) 224 | # x_pred = self.encoder.predict(x_batch) 225 | # fake_pred = self.encoder.predict(fake_batch) 226 | 227 | # x_decod = self.encoder_model.autoencoder.decoder_predict(inputs=x_batch) 228 | # x_decod = np.transpose(np.reshape(x_decod, (x_decod.shape[0], x_decod.shape[1]))) 229 | # x_batch_ = np.transpose(np.reshape(x_batch, (x_batch.shape[0], x.shape[1]))) 230 | 231 | i += 1 232 | if i >= nb_steps: 233 | break 234 | 235 | if verbose: 236 | template = 'Epoch {}, Loss: {}' 237 | print(template.format(epoch + 1, idec_enc_loss.result())) 238 | epoch += 1 239 | 240 | y_pred = self.log_stats_encoder(x, y, 241 | x_test, y_test, 242 | [idec_enc_loss.result(), dec_enc_loss.result(), train_enc_loss.result()], 243 | epoch, 244 | log_writer, 245 | 'pretrain') 246 | 247 | return epoch 248 | -------------------------------------------------------------------------------- /networks/IDEC.py: -------------------------------------------------------------------------------- 1 | """ 2 | Based on Keras implementation https://github.com/XifengGuo/IDEC: 3 | and article : 4 | Xifeng Guo, Long Gao, Xinwang Liu, Jianping Yin. 5 | Improved Deep Embedded Clustering with Local Structure Preservation. IJCAI 2017. 6 | 7 | Original Author: 8 | Xifeng Guo. 2017.1.30 9 | Author: 10 | Baptiste Lafabregue 2019.25.04 11 | """ 12 | 13 | import numpy as np 14 | import tensorflow as tf 15 | import tensorflow.keras.backend as K 16 | from sklearn import metrics 17 | from tensorflow.keras.layers import Layer, InputSpec 18 | from tensorflow.keras.models import Model 19 | from sklearn.cluster import KMeans 20 | 21 | from networks.trainer import Trainer 22 | 23 | 24 | class ClusteringLayer(Layer): 25 | """ 26 | Clustering layer converts input sample (feature) to soft label, i.e. a vector that represents the probability of the 27 | sample belonging to each cluster. The probability is calculated with student's t-distribution. 28 | 29 | # Example 30 | ``` 31 | model.add(ClusteringLayer(n_clusters=10)) 32 | ``` 33 | # Arguments 34 | n_clusters: number of clusters. 35 | weights: list of Numpy array with shape `(n_clusters, n_features)` witch represents the initial cluster centers. 36 | alpha: parameter in Student's t-distribution. Default to 1.0. 37 | # Input shape 38 | 2D tensor with shape: `(n_samples, n_features)`. 39 | # Output shape 40 | 2D tensor with shape: `(n_samples, n_clusters)`. 41 | """ 42 | 43 | def __init__(self, n_clusters, weights=None, alpha=1.0, **kwargs): 44 | if 'input_shape' not in kwargs and 'input_dim' in kwargs: 45 | kwargs['input_shape'] = (kwargs.pop('input_dim'),) 46 | super(ClusteringLayer, self).__init__(**kwargs) 47 | self.n_clusters = n_clusters 48 | self.alpha = alpha 49 | self.initial_weights = weights 50 | self.input_spec = InputSpec(ndim=2) 51 | 52 | def build(self, input_shape): 53 | assert len(input_shape) == 2 54 | input_dim = input_shape[1] 55 | self.input_spec = InputSpec(dtype=K.floatx(), shape=(None, input_dim)) 56 | self.clusters = self.add_weight(shape=(self.n_clusters, input_dim), 57 | initializer='glorot_uniform', name='clustering') 58 | if self.initial_weights is not None: 59 | self.set_weights(self.initial_weights) 60 | del self.initial_weights 61 | self.built = True 62 | 63 | def call(self, inputs, **kwargs): 64 | """ student t-distribution, as same as used in t-SNE algorithm. 65 | q_ij = 1/(1+dist(x_i, u_j)^2), then normalize it. 66 | Arguments: 67 | inputs: the variable containing data, shape=(n_samples, n_features) 68 | Return: 69 | q: student's t-distribution, or soft labels for each sample. shape=(n_samples, n_clusters) 70 | """ 71 | q = 1.0 / (1.0 + (K.sum(K.square(K.expand_dims(inputs, axis=1) - self.clusters), axis=2) / self.alpha)) 72 | q **= (self.alpha + 1.0) / 2.0 73 | q = K.transpose(K.transpose(q) / K.sum(q, axis=1)) 74 | return q 75 | 76 | def compute_output_shape(self, input_shape): 77 | assert input_shape and len(input_shape) == 2 78 | return input_shape[0], self.n_clusters 79 | 80 | def get_config(self): 81 | config = {'n_clusters': self.n_clusters} 82 | base_config = super(ClusteringLayer, self).get_config() 83 | return dict(list(base_config.items()) + list(config.items())) 84 | 85 | 86 | class IDEC(Trainer): 87 | def __init__(self, 88 | dataset_name, 89 | classifier_name, 90 | encoder_model, 91 | keep_both_losses=True, 92 | gamma=0.1, 93 | n_clusters=10, 94 | alpha=1.0, 95 | batch_size=10, 96 | tol=1e-3, 97 | update_interval=5, 98 | optimizer=None): 99 | 100 | super(IDEC, self).__init__(dataset_name, classifier_name, encoder_model, batch_size, n_clusters, optimizer) 101 | 102 | self.keep_both_losses = keep_both_losses 103 | self.gamma = gamma 104 | self.alpha = alpha 105 | self.tol = tol 106 | self.update_interval = update_interval 107 | self.dec_model = None 108 | self.dec_loss = None 109 | 110 | def initialize_model(self, x, y, ae_weights=None): 111 | """ 112 | Initialize the model for training 113 | :param ae_weights: arguments to let the encoder load its weights, None to pre-train the encoder 114 | """ 115 | if ae_weights is not None: 116 | self.encoder_model.load_weights(ae_weights) 117 | print('Pretrained AE weights are loaded successfully.') 118 | self.pretrain_model = False 119 | else: 120 | self.pretrain_model = True 121 | 122 | if self.optimizer is None: 123 | self.optimizer = tf.keras.optimizers.legacy.Adam() 124 | 125 | clustering_layer = ClusteringLayer(self.n_clusters, name='clustering')(self.encoder.output) 126 | self.dec_model = Model(inputs=self.encoder.input, outputs=clustering_layer) 127 | self.dec_loss = tf.keras.losses.KLDivergence() 128 | 129 | def load_weights(self, weights_path): 130 | """ 131 | Load weights of IDEC model 132 | :param weights_path: path to load weights from 133 | """ 134 | self.dec_model.load_weights(weights_path + '.tf') 135 | 136 | def save_weights(self, weights_path): 137 | """ 138 | Save weights of IDEC model 139 | :param weights_path: path to save weights to 140 | """ 141 | self.dec_model.save_weights(weights_path + '.tf') 142 | 143 | def get_trainer_name(self): 144 | """ 145 | Return the name of the training method used 146 | :return: method name 147 | """ 148 | if self.gamma == 0.0: 149 | return 'DEC' 150 | return self.__class__.__name__ 151 | 152 | def predict_clusters(self, x, seeds=None): 153 | """ 154 | Predict cluster labels using the output of clustering layer 155 | :param x: the data to evaluate 156 | :param seeds: seeds to initialize the K-Means if needed 157 | :return: the predicted cluster labels 158 | """ 159 | q = self.dec_model.predict(x, verbose=0) 160 | return q.argmax(1), self.dec_model.get_layer(name='clustering').get_weights() 161 | 162 | @staticmethod 163 | def target_distribution(q): 164 | """ 165 | Target distribution P which enhances the discrimination of soft label Q 166 | :param q: the Q tensor 167 | :return: the P tensor 168 | """ 169 | weight = q ** 2 / q.sum(0) 170 | return (weight.T / weight.sum(1)).T 171 | 172 | def _run_training(self, x, y, x_test, y_test, nb_steps, 173 | seeds, verbose, log_writer, dist_matrix=None): 174 | if seeds is not None: 175 | seeds_enc = self.extract_features(seeds) 176 | kmeans = KMeans(n_clusters=self.n_clusters, n_init=20, init=seeds_enc) 177 | else: 178 | kmeans = KMeans(n_clusters=self.n_clusters, n_init=20) 179 | x_pred = self.extract_features(x) 180 | y_pred = kmeans.fit_predict(x_pred) 181 | 182 | self.dec_model.get_layer(name='clustering').set_weights([kmeans.cluster_centers_]) 183 | 184 | if y is not None: 185 | ari = np.round(metrics.adjusted_rand_score(y, y_pred), 5) 186 | if verbose: 187 | print('ari kmeans: ', str(ari)) 188 | self.log_stats(x, y, x_test, y_test, [0, 0, 0], 0, log_writer, 'init') 189 | 190 | q = self.dec_model.predict(x, verbose=0) 191 | p = self.target_distribution(q) # update the auxiliary target distribution p 192 | 193 | # evaluate the clustering performance 194 | y_pred = q.argmax(1) 195 | y_pred_last = y_pred 196 | 197 | i = 0 # Number of performed optimization steps 198 | epoch = 0 # Number of performed epochs 199 | 200 | # define the train function 201 | train_enc_loss = tf.keras.metrics.Mean(name='encoder train_loss') 202 | dec_enc_loss = tf.keras.metrics.Mean(name='dec train_loss') 203 | idec_enc_loss = tf.keras.metrics.Mean(name='idec train_loss') 204 | 205 | @tf.function 206 | def train_step(x_batch, p_batch): 207 | with tf.GradientTape() as tape: 208 | encoder_loss = self.encoder_model.loss.compute_loss(x_batch, training=True) 209 | encoding_x = self.dec_model(x_batch, training=True) 210 | dec_loss = tf.keras.losses.KLD(p_batch, encoding_x) 211 | loss = (1 - self.gamma) * encoder_loss + self.gamma * dec_loss 212 | gradients = tape.gradient(loss, 213 | self.encoder_model.get_trainable_variables() + self.dec_model.trainable_variables) 214 | self.optimizer.apply_gradients( 215 | zip(gradients, self.encoder_model.get_trainable_variables() + self.dec_model.trainable_variables)) 216 | 217 | train_enc_loss(encoder_loss) 218 | dec_enc_loss(loss) 219 | idec_enc_loss(loss) 220 | 221 | if verbose: 222 | print('start training') 223 | # idec training 224 | while i < nb_steps: 225 | train_enc_loss.reset_states() 226 | dec_enc_loss.reset_states() 227 | idec_enc_loss.reset_states() 228 | # shuffle the train set 229 | 230 | # computes P each update_interval epoch 231 | if epoch % self.update_interval == 0: 232 | q = self.dec_model.predict(x, verbose=0) 233 | p = self.target_distribution(q) # update the auxiliary target distribution p 234 | 235 | # evaluate the clustering performance 236 | y_pred = q.argmax(1) 237 | delta_label = np.sum(y_pred != y_pred_last).astype(np.float32) / y_pred.shape[0] 238 | y_pred_last = y_pred 239 | 240 | # check stop criterion 241 | if epoch > 0 and delta_label < self.tol: 242 | if verbose: 243 | print('delta_label ', delta_label, '< tol ', self.tol) 244 | print('Reached tolerance threshold. Stopping training.') 245 | self.log_stats(x, y, x_test, y_test, [0, 0, 0], 246 | epoch, log_writer, 'reached_stop_criterion') 247 | break 248 | 249 | train_ds = tf.data.Dataset.from_tensor_slices((x, p)) \ 250 | .shuffle(x.shape[0], reshuffle_each_iteration=True) \ 251 | .batch(self.batch_size).as_numpy_iterator() 252 | 253 | for x_batch, p_batch in train_ds: 254 | train_step(x_batch, p_batch) 255 | i += 1 256 | if i >= nb_steps: 257 | break 258 | 259 | if verbose: 260 | template = 'Epoch {}, Loss: {}' 261 | print(template.format(epoch + 1, idec_enc_loss.result())) 262 | epoch += 1 263 | 264 | y_pred = self.log_stats(x, y, 265 | x_test, y_test, 266 | [idec_enc_loss.result(), dec_enc_loss.result(), train_enc_loss.result()], 267 | epoch, 268 | log_writer, 269 | 'train') 270 | 271 | return epoch 272 | -------------------------------------------------------------------------------- /networks/SDCN.py: -------------------------------------------------------------------------------- 1 | """ 2 | Based on torch implementation https://github.com/bdy9527/SDCN 3 | and article : 4 | Deyu Bo, Xiao Wang, Chuan Shi, Meiqi Zhu, Emiao Lu, and Peng Cui, 5 | Structural Deep Clustering Network 6 | 7 | Author: 8 | Baptiste Lafabregue 2019.25.04 9 | """ 10 | 11 | import numpy as np 12 | import tensorflow as tf 13 | import tensorflow.keras.backend as K 14 | from sklearn import metrics 15 | from sklearn.cluster import KMeans 16 | from tensorflow.keras.layers import Layer, ReLU, Reshape, GlobalMaxPool1D 17 | from tensorflow.keras import Model, Input 18 | 19 | import utils 20 | from networks.IDEC import ClusteringLayer 21 | from networks.trainer import Trainer 22 | 23 | 24 | class GNNLayer(Layer): 25 | def __init__(self, 26 | in_features, 27 | out_features, 28 | adjency_matrix, 29 | final=False, 30 | **kwargs): 31 | super(GNNLayer, self).__init__() 32 | self.in_features = in_features 33 | self.out_features = out_features 34 | self.final = final 35 | w_init = tf.initializers.GlorotUniform() 36 | self.weight = tf.Variable(initial_value=w_init(shape=(in_features, out_features), dtype=K.floatx()), 37 | trainable=True) 38 | self.adj = adjency_matrix 39 | super(GNNLayer, self).__init__(**kwargs) 40 | 41 | def call(self, features): 42 | support = K.dot(features, self.weight) 43 | output = tf.sparse.sparse_dense_matmul(self.adj, support) 44 | if self.final: 45 | output = tf.keras.activations.softmax(output) 46 | else: 47 | output = ReLU()(output) 48 | return output 49 | 50 | 51 | class SDCN(Trainer): 52 | def __init__(self, 53 | dataset_name, 54 | classifier_name, 55 | encoder_model, 56 | keep_both_losses=True, 57 | gamma=0.4, 58 | n_clusters=10, 59 | alpha=1.0, 60 | tol=1e-3, 61 | update_interval=5, 62 | batch_size=10, 63 | optimizer=None, 64 | graph_path='.' 65 | ): 66 | 67 | super(SDCN, self).__init__(dataset_name, classifier_name, encoder_model, batch_size, n_clusters, optimizer) 68 | 69 | self.keep_both_losses = keep_both_losses 70 | self.gamma = gamma 71 | self.alpha = alpha 72 | self.tol = tol 73 | self.update_interval = update_interval 74 | self.graph_path = graph_path 75 | self.pretrain_model = True 76 | self.sdcn_model = None 77 | self.dec_loss = None 78 | self.gnn_layers = [] 79 | 80 | def initialize_model(self, x, y, ae_weights=None): 81 | """ 82 | Initialize the model for training 83 | :param ae_weights: arguments to let the encoder load its weights, None to pre-train the encoder 84 | """ 85 | if ae_weights is not None: 86 | self.encoder_model.load_weights(ae_weights) 87 | print('Pretrained AE weights are loaded successfully.') 88 | self.pretrain_model = False 89 | else: 90 | self.pretrain_model = True 91 | 92 | if self.optimizer is None: 93 | self.optimizer = tf.keras.optimizers.legacy.Adam() 94 | 95 | if not self.graph_path.endswith('.npy'): 96 | self.graph_path = self.graph_path + '/graph.npy' 97 | utils.construct_knn_graph(x, y, self.graph_path) 98 | adj = utils.load_graph(len(x), self.graph_path) 99 | 100 | inputs = Input((x.shape[1:])) 101 | # init gnn layers 102 | enc_output = self.encoder(inputs) 103 | # self._format_output(enc_output) 104 | h = Reshape((-1,))(inputs) 105 | ts_length = x.shape[1] 106 | channels = x.shape[2] 107 | output_shape_prev = K.int_shape(enc_output[0])[-1] 108 | h = GNNLayer(ts_length*channels, output_shape_prev, adj)(h) 109 | for i in range(0, len(enc_output) - 1): 110 | output_shape = K.int_shape(enc_output[i + 1])[-1] 111 | enc_layer = enc_output[i] 112 | if len(K.int_shape(enc_layer)) > 2: 113 | enc_layer = GlobalMaxPool1D()(enc_layer) 114 | h = GNNLayer(output_shape_prev, output_shape, adj)(tf.keras.layers.add([h, enc_layer])) 115 | output_shape_prev = output_shape 116 | enc_layer = enc_output[-1] 117 | gnn_output = GNNLayer(output_shape_prev, self.n_clusters, 118 | adj, final=True)(tf.keras.layers.add([h, enc_layer])) 119 | 120 | dnn_output = ClusteringLayer(self.n_clusters, name='clustering')(enc_layer) 121 | self.sdcn_model = Model(inputs=inputs, outputs=[dnn_output, gnn_output]) 122 | self.dec_loss = tf.keras.losses.KLDivergence() 123 | 124 | @staticmethod 125 | def _format_output(enc_output): 126 | for i in range(len(enc_output)): 127 | if len(K.int_shape(enc_output[i])) > 2: 128 | enc_output[i] = GlobalMaxPool1D()(enc_output[i]) 129 | 130 | def load_weights(self, weights_path): 131 | """ 132 | Load weights of IDEC model 133 | :param weights_path: path to load weights from 134 | """ 135 | self.sdcn_model.load_weights(weights_path) 136 | 137 | def save_weights(self, weights_path): 138 | """ 139 | Save weights of IDEC model 140 | :param weights_path: path to save weights to 141 | """ 142 | self.sdcn_model.save_weights(weights_path) 143 | 144 | def extract_features(self, x): 145 | """ 146 | Extract features from the encoder (before the clustering layer) 147 | :param x: the data to extract features from 148 | :return: the encoded features 149 | """ 150 | return self.encoder.predict(x)[-1] 151 | 152 | @staticmethod 153 | def target_distribution(q): 154 | """ 155 | Target distribution P which enhances the discrimination of soft label Q 156 | :param q: the Q tensor 157 | :return: the P tensor 158 | """ 159 | weight = q ** 2 / q.sum(0) 160 | return (weight.T / weight.sum(1)).T 161 | 162 | def _run_training(self, x, y, x_test, y_test, nb_steps, 163 | seeds, verbose, log_writer, dist_matrix=None): 164 | if seeds is not None: 165 | seeds_enc = self.extract_features(seeds) 166 | kmeans = KMeans(n_clusters=self.n_clusters, n_init=20, init=seeds_enc) 167 | else: 168 | kmeans = KMeans(n_clusters=self.n_clusters, n_init=20) 169 | x_pred = self.extract_features(x) 170 | y_pred = kmeans.fit_predict(x_pred) 171 | 172 | self.sdcn_model.get_layer(name='clustering').set_weights([kmeans.cluster_centers_]) 173 | 174 | if y is not None: 175 | ari = np.round(metrics.adjusted_rand_score(y, y_pred), 5) 176 | if verbose: 177 | print('ari kmeans: ', str(ari)) 178 | self.log_stats(x, y, x_test, y_test, [0, 0, 0], 0, log_writer, 'init') 179 | 180 | q, _ = self.sdcn_model.predict(x, verbose=0, batch_size=len(x)) 181 | p = self.target_distribution(q) # update the auxiliary target distribution p 182 | 183 | # evaluate the clustering performance 184 | y_pred = q.argmax(1) 185 | y_pred_last = y_pred 186 | 187 | i = 0 # Number of performed optimization steps 188 | epoch = 0 # Number of performed epochs 189 | 190 | # define the train function 191 | train_enc_loss = tf.keras.metrics.Mean(name='encoder train_loss') 192 | dec_enc_loss = tf.keras.metrics.Mean(name='dec train_loss') 193 | idec_enc_loss = tf.keras.metrics.Mean(name='idec train_loss') 194 | 195 | @tf.function 196 | def train_step(x_batch, p_batch): 197 | with tf.GradientTape() as tape: 198 | encoder_loss = self.encoder_model.loss.compute_loss(x_batch, training=True) 199 | q, q_gcn = self.sdcn_model(x_batch, training=True) 200 | dec_loss = tf.keras.losses.KLD(p_batch, q) 201 | gcn_loss = tf.keras.losses.KLD(p_batch, q_gcn) 202 | loss = (1 - self.gamma) * encoder_loss + self.gamma * (dec_loss + gcn_loss) 203 | gradients = tape.gradient(loss, self.encoder_model.get_trainable_variables() 204 | + self.sdcn_model.trainable_variables) 205 | self.optimizer.apply_gradients(zip(gradients, self.encoder_model.get_trainable_variables() 206 | + self.sdcn_model.trainable_variables)) 207 | 208 | train_enc_loss(encoder_loss) 209 | dec_enc_loss(loss) 210 | idec_enc_loss(loss) 211 | 212 | if verbose: 213 | print('start training') 214 | # idec training 215 | while i < nb_steps: 216 | train_enc_loss.reset_states() 217 | dec_enc_loss.reset_states() 218 | idec_enc_loss.reset_states() 219 | # shuffle the train set 220 | 221 | # computes P each update_interval epoch 222 | if epoch % self.update_interval == 0: 223 | q, _ = self.sdcn_model.predict(x, verbose=0, batch_size=len(x)) 224 | p = self.target_distribution(q) # update the auxiliary target distribution p 225 | 226 | # evaluate the clustering performance 227 | y_pred = q.argmax(1) 228 | delta_label = np.sum(y_pred != y_pred_last).astype(np.float32) / y_pred.shape[0] 229 | y_pred_last = y_pred 230 | 231 | # check stop criterion 232 | if epoch > 0 and delta_label < self.tol: 233 | if verbose: 234 | print('delta_label ', delta_label, '< tol ', self.tol) 235 | print('Reached tolerance threshold. Stopping training.') 236 | self.log_stats(x, y, x_test, y_test, [0, 0, 0], 237 | epoch, log_writer, 'reached_stop_criterion') 238 | break 239 | 240 | # for this method we cannot use mini batch 241 | # train_ds = tf.data.Dataset.from_tensor_slices((x, p)) \ 242 | # .shuffle(x.shape[0], reshuffle_each_iteration=True) \ 243 | # .batch(self.x.shape[0]).as_numpy_iterator() 244 | # 245 | # for x_batch, p_batch in train_ds: 246 | train_step(x, p) 247 | i += 1 248 | 249 | if verbose: 250 | template = 'Epoch {}, Loss: {}' 251 | print(template.format(epoch + 1, idec_enc_loss.result())) 252 | epoch += 1 253 | 254 | y_pred = self.log_stats(x, y, 255 | x_test, y_test, 256 | [idec_enc_loss.result(), dec_enc_loss.result(), train_enc_loss.result()], 257 | epoch, 258 | log_writer, 259 | 'train') 260 | 261 | return epoch 262 | -------------------------------------------------------------------------------- /networks/VADE.py: -------------------------------------------------------------------------------- 1 | """ 2 | Based on Keras implementation https://github.com/slim1017/VaDE 3 | and article : 4 | Jiang, Z., Zheng, Y., Tan, H., Tang, B., & Zhou, H. (2016). 5 | Variational deep embedding: A generative approach to clustering 6 | 7 | Author: 8 | Baptiste Lafabregue 2019.25.04 9 | """ 10 | 11 | import numpy as np 12 | import tensorflow as tf 13 | import tensorflow.keras.backend as K 14 | from sklearn import metrics 15 | from sklearn.mixture import GaussianMixture 16 | from tensorflow.keras.layers import Lambda 17 | from tensorflow.keras.models import Model 18 | 19 | from networks.trainer import Trainer 20 | import utils 21 | 22 | 23 | class VADE(Trainer): 24 | def __init__(self, 25 | dataset_name, 26 | classifier_name, 27 | encoder_model, 28 | decoder, 29 | keep_both_losses=True, 30 | n_clusters=10, 31 | alpha=1.0, 32 | batch_size=10, 33 | optimizer=None): 34 | 35 | super(VADE, self).__init__(dataset_name, classifier_name, encoder_model, batch_size, n_clusters, optimizer) 36 | 37 | self.decoder = decoder 38 | if decoder is None: 39 | raise utils.CompatibilityException('architecture incompatible with VADE') 40 | self.keep_both_losses = keep_both_losses 41 | self.alpha = alpha 42 | self.n_clusters = n_clusters 43 | enc_shape = np.array(K.int_shape(encoder_model.encoder.output)).tolist() 44 | self.latent_dim = enc_shape[1] // 2 45 | self.theta_p = K.variable(np.ones(n_clusters) / n_clusters, name="pi") 46 | self.u_p = K.variable(np.zeros((self.latent_dim, n_clusters)), name="mu") 47 | self.lambda_p = K.variable(np.ones((self.latent_dim, n_clusters)), name="lambda") 48 | self.p_c_z_output = None 49 | 50 | def initialize_model(self, x, y, ae_weights=None): 51 | """ 52 | Initialize the model for training 53 | :param ae_weights: arguments to let the encoder load its weights, None to pre-train the encoder 54 | """ 55 | if ae_weights is not None: 56 | self.encoder_model.load_weights(ae_weights) 57 | print('Pretrained AE weights are loaded successfully.') 58 | self.pretrain_model = False 59 | else: 60 | self.pretrain_model = True 61 | 62 | if self.optimizer is None: 63 | self.optimizer = tf.keras.optimizers.legacy.Adam() 64 | 65 | gamma_layer = Lambda(self.get_gamma, output_shape=(self.n_clusters,))(self.encoder.output) 66 | self.p_c_z_output = Model(inputs=self.encoder.input, outputs=gamma_layer) 67 | 68 | def predict_clusters(self, x, seeds=None): 69 | """ 70 | Predict cluster labels using the output of clustering layer 71 | :param x: the data to evaluate 72 | :param seeds: seeds to initialize the K-Means if needed 73 | :return: the predicted cluster labels 74 | """ 75 | center_shape = list(x.shape) 76 | center_shape[0] = self.n_clusters 77 | features = self.p_c_z_output.predict(x, batch_size=1) 78 | return np.argmax(features, axis=1), np.zeros(tuple(center_shape)) 79 | 80 | def get_gamma(self, z): 81 | mean, logvar = tf.split(z, num_or_size_splits=2, axis=1) 82 | mean_c = K.permute_dimensions(K.repeat(mean, self.n_clusters), (0, 2, 1)) 83 | size_z = 1 84 | 85 | # do the same for parameters but for batch size 86 | u_t = K.repeat_elements(K.expand_dims(self.u_p, axis=0), size_z, 0) 87 | lambda_t = K.repeat_elements(K.expand_dims(self.lambda_p, axis=0), size_z, 0) 88 | theta_t = K.expand_dims(K.expand_dims(self.theta_p, axis=0), axis=0) 89 | theta_t = tf.ones((size_z, self.latent_dim, self.n_clusters), dtype=K.floatx()) * theta_t 90 | 91 | temp_p_c_z = K.exp(K.sum((K.log(theta_t) - 0.5 * K.log(2 * np.pi * lambda_t) - 92 | K.square(mean_c - u_t) / (2 * lambda_t)), axis=1)) 93 | return temp_p_c_z / K.sum(temp_p_c_z, axis=-1, keepdims=True) 94 | 95 | def init_gmm_parameters(self, x): 96 | g = GaussianMixture(n_components=self.n_clusters, covariance_type='diag') 97 | sample = self.encoder.predict(x) 98 | sample = np.split(sample, 2, axis=1)[0] 99 | g.fit(sample) 100 | self.u_p.assign(g.means_.T) 101 | self.lambda_p.assign(g.covariances_.T) 102 | 103 | def _run_training(self, x, y, x_test, y_test, nb_steps, 104 | seeds, verbose, log_writer, dist_matrix=None): 105 | 106 | i = 0 # Number of performed optimization steps 107 | epoch = 0 # Number of performed epochs 108 | self.init_gmm_parameters(x) 109 | 110 | # define the train function 111 | vae_loss = tf.keras.metrics.Mean(name='encoder train_loss') 112 | 113 | # @tf.function 114 | def train_step(batch, batch_size): 115 | with tf.GradientTape() as tape: 116 | mean, logvar = tf.split(self.encoder(batch), num_or_size_splits=2, axis=1) 117 | epsilon = K.random_normal(shape=mean.shape) 118 | z = mean + K.exp(logvar / 2) * epsilon 119 | # repeat each element per # of clusters 120 | mean_c = K.permute_dimensions(K.repeat(mean, self.n_clusters), (0, 2, 1)) 121 | logvar_c = K.permute_dimensions(K.repeat(logvar, self.n_clusters), (0, 2, 1)) 122 | z_c = K.permute_dimensions(K.repeat(z, self.n_clusters), (0, 2, 1)) 123 | 124 | # do the same for parameters but for batch size 125 | u_t = K.repeat_elements(K.expand_dims(self.u_p, axis=0), self.batch_size, 0) 126 | lambda_t = K.repeat_elements(K.expand_dims(self.lambda_p, axis=0), self.batch_size, 0) 127 | theta_t = K.expand_dims(K.expand_dims(self.theta_p, axis=0), axis=0) 128 | theta_t = tf.ones((self.batch_size, self.latent_dim, self.n_clusters), dtype=K.floatx()) * theta_t 129 | 130 | x_logit = self.decoder(z, training=True) 131 | recon = self.alpha * K.sum(tf.keras.losses.mean_squared_error(batch, x_logit), axis=1) # * original_dim 132 | 133 | p_c_z = K.exp(K.sum((K.log(theta_t) - 0.5 * K.log(2 * np.pi * lambda_t) - 134 | K.square(z_c - u_t) / (2 * lambda_t)), axis=1)) + 1e-10 135 | gamma = p_c_z / K.sum(p_c_z, axis=-1, keepdims=True) 136 | 137 | gamma_t = K.repeat(gamma, self.latent_dim) 138 | 139 | # print(gamma_t) 140 | # print(theta_t) 141 | # print(lambda_t) 142 | # print(u_t) 143 | # print(gamma) 144 | # print(logvar_c) 145 | # print((mean_c)) 146 | a = K.sum(0.5 * gamma_t * (tf.cast(K.log(np.pi * 2), dtype=K.floatx()) + 147 | K.log(lambda_t) + K.exp(logvar_c) / lambda_t + 148 | K.square(mean_c - u_t) / lambda_t), axis=(1, 2)) 149 | b = 0.5 * K.sum(logvar + 1, axis=-1) 150 | c = K.sum(K.log(K.repeat_elements(K.expand_dims(self.theta_p, axis=0), batch_size, 0)) * gamma, axis=-1) 151 | d = K.sum(K.log(gamma) * gamma, axis=-1) 152 | loss = recon + a - b - c + d 153 | # print(a) 154 | # print(b) 155 | # print(c) 156 | # print(d) 157 | # print(recon) 158 | trainable_var = self.encoder_model.get_trainable_variables() + [self.lambda_p, self.u_p, self.theta_p] 159 | gradients = tape.gradient(loss, trainable_var) 160 | self.optimizer.apply_gradients(zip(gradients, trainable_var)) 161 | 162 | vae_loss(loss) 163 | 164 | if verbose: 165 | print('start training') 166 | 167 | while i < nb_steps: 168 | vae_loss.reset_states() 169 | # shuffle the train set 170 | train_ds = tf.data.Dataset.from_tensor_slices(x) \ 171 | .shuffle(x.shape[0], reshuffle_each_iteration=True) \ 172 | .batch(self.batch_size, drop_remainder=True).as_numpy_iterator() 173 | 174 | for x_batch in train_ds: 175 | train_step(x_batch, len(x_batch)) 176 | i += 1 177 | if i >= nb_steps: 178 | break 179 | 180 | if verbose: 181 | template = 'Epoch {}, Loss: {}' 182 | print(template.format(epoch + 1, vae_loss.result())) 183 | epoch += 1 184 | 185 | y_pred = self.log_stats(x, y, 186 | x_test, y_test, 187 | [vae_loss.result(), vae_loss.result(), vae_loss.result()], 188 | epoch, 189 | log_writer, 190 | 'train') 191 | 192 | return epoch 193 | -------------------------------------------------------------------------------- /networks/__init__.py: -------------------------------------------------------------------------------- 1 | import pkgutil 2 | 3 | __all__ = [] 4 | for loader, module_name, is_pkg in pkgutil.walk_packages(__path__): 5 | __all__.append(module_name) 6 | module = loader.find_module(module_name).load_module(module_name) 7 | exec('%s = module' % module_name) 8 | -------------------------------------------------------------------------------- /networks/__pycache__/ClusterGAN.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/ClusterGAN.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/DEPICT.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/DEPICT.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/DTCR.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/DTCR.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/IDEC.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/IDEC.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/SDCN.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/SDCN.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/VADE.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/VADE.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/attention_rnn.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/attention_rnn.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/bi_dilated_RNN.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/bi_dilated_RNN.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/bi_dilated_RNN_variable_length.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/bi_dilated_RNN_variable_length.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/bilstm_ae.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/bilstm_ae.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/birnn_ae.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/birnn_ae.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/dilated_causal_cnn.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/dilated_causal_cnn.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/encoders.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/encoders.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/fcnn_ae.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/fcnn_ae.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/mlp_ae.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/mlp_ae.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/resnet.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/resnet.cpython-39.pyc -------------------------------------------------------------------------------- /networks/__pycache__/trainer.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blafabregue/TimeSeriesDeepClustering/208fac0343d281f2c5997609916004875aae86fd/networks/__pycache__/trainer.cpython-39.pyc -------------------------------------------------------------------------------- /networks/attention_rnn.py: -------------------------------------------------------------------------------- 1 | """ 2 | Define a Bi-directional GRU autoencoder with attention mechanism. 3 | Based on implementation https://gitlab.irstea.fr/dino.ienco/detsec 4 | and article: 5 | Dino Ienco, Roberto Interdonato. 6 | Deep Multivariate Time Series Embedding Clustering via Attentive-Gated Autoencoder. PAKDD 2020: 318-329 7 | 8 | Author: 9 | Baptiste Lafabregue 2019.25.04 10 | """ 11 | import os 12 | import numpy as np 13 | 14 | import tensorflow as tf 15 | from tensorflow.keras import backend as K 16 | from tensorflow.keras import layers 17 | from tensorflow.keras import Model 18 | from tensorflow.keras.layers import Layer, InputSpec 19 | 20 | from networks.encoders import LayersGenerator 21 | 22 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 23 | 24 | 25 | class AttentionLayer(Layer): 26 | """ 27 | AttentionLayer apply a self-attention mechanism to abidirectionel RNN output 28 | 29 | # Arguments 30 | n_units : number of units returned by the previous (RNN) layer 31 | att_size : size of attention mechanism weights 32 | # Input shape 33 | (, , nunits) 34 | # Output shape 35 | (, att_size) 36 | """ 37 | 38 | def __init__(self, n_units, att_size, **kwargs): 39 | if 'input_shape' not in kwargs and 'input_dim' in kwargs: 40 | kwargs['input_shape'] = (kwargs.pop('input_dim'),) 41 | super(AttentionLayer, self).__init__(**kwargs) 42 | self.n_units = n_units 43 | self.att_size = att_size 44 | # self.input_spec = InputSpec(ndim=2) 45 | self.W_omega = None 46 | self.b_omega = None 47 | self.u_omega = None 48 | 49 | def build(self, input_shape): 50 | # input_dim = input_shape[1] 51 | initializer = tf.keras.initializers.RandomNormal(stddev=0.1) 52 | # self.input_spec = InputSpec(dtype=K.floatx(), shape=(None, input_dim)) 53 | self.W_omega = self.add_weight(shape=(self.n_units, self.att_size), 54 | initializer=initializer, name="W_omega") 55 | self.b_omega = self.add_weight(shape=(self.att_size,), 56 | initializer=initializer, name="b_omega") 57 | self.u_omega = self.add_weight(shape=(self.att_size,), 58 | initializer=initializer, name="u_omega") 59 | self.built = True 60 | 61 | def call(self, input, **kwargs): 62 | # outputs = tf.stack(input_list, axis=1) 63 | 64 | v = tf.tanh(tf.tensordot(input, self.W_omega, axes=1) + self.b_omega) 65 | # For each of the timestamps its vector of size A from `v` is reduced with `u` vector 66 | vu = tf.tensordot(v, self.u_omega, axes=1) # (B,T) shape 67 | alphas = tf.nn.softmax(vu) # (B,T) shape also 68 | 69 | output = tf.reduce_sum(input * tf.expand_dims(alphas, -1), 1) 70 | output = tf.reshape(output, [-1, self.n_units]) 71 | return output 72 | 73 | def compute_output_shape(self, input_shape): 74 | assert input_shape and len(input_shape) == 2 75 | return input_shape[0], self.n_units 76 | 77 | def get_config(self): 78 | config = {'n_units': self.n_units, 'att_size': self.att_size} 79 | base_config = super(AttentionLayer, self).get_config() 80 | return dict(list(base_config.items()) + list(config.items())) 81 | 82 | 83 | class AutoEncoder(LayersGenerator): 84 | """docstring for AutoEncoder""" 85 | 86 | def __init__(self, 87 | x, 88 | encoder_loss, 89 | n_clusters, 90 | latent_dim=None, 91 | ): 92 | self.n_clusters = n_clusters 93 | self.L = x.shape[1] 94 | self.C = x.shape[2] 95 | if latent_dim is None: 96 | # we fix the rnn size based on train set size 97 | self.n_filters_RNN = 64 98 | if x.shape[0] > 250: 99 | self.n_filters_RNN = 512 100 | else: 101 | self.n_filters_RNN = latent_dim 102 | 103 | self.input_ = layers.Input(shape=(None, self.C)) 104 | self.gate = layers.Dense(self.n_filters_RNN, activation='sigmoid') 105 | 106 | super().__init__(x, encoder_loss) 107 | 108 | def _create_encoder(self, x): 109 | 110 | with tf.name_scope('encoder'): 111 | h = self.input_ 112 | forward_layer = layers.GRU(self.n_filters_RNN, return_sequences=True)(h) 113 | backward_layer = layers.GRU(self.n_filters_RNN, activation='relu', return_sequences=True, 114 | go_backwards=True)(h) 115 | 116 | h_att_fw = AttentionLayer(self.n_filters_RNN, self.n_filters_RNN)(forward_layer) 117 | h_att_bw = AttentionLayer(self.n_filters_RNN, self.n_filters_RNN)(backward_layer) 118 | 119 | h = self.gate(h_att_fw) * h_att_fw + self.gate(h_att_bw) * h_att_bw 120 | return Model(inputs=self.input_, outputs=h), None 121 | 122 | def _create_decoder(self, x): 123 | 124 | with tf.name_scope('decoder'): 125 | decoder_input = layers.Input(shape=(self.enc_shape[1])) 126 | h = layers.RepeatVector(self.L)(decoder_input) 127 | h = layers.Bidirectional(layers.GRU(self.n_filters_RNN // 2, return_sequences=True))(h) 128 | 129 | decode = layers.TimeDistributed(layers.Dense(self.C))(h) 130 | 131 | return Model(inputs=decoder_input, outputs=decode), None 132 | -------------------------------------------------------------------------------- /networks/bi_dilated_RNN.py: -------------------------------------------------------------------------------- 1 | """ 2 | Based on tensorflow implementation https://github.com/qianlima-lab/DTCR 3 | and article : 4 | Ma, Q., Zheng, J., Li, S., & Cottrell, G. W. (2019), 5 | Learning Representations for Time Series Clustering 6 | 7 | Author: 8 | Baptiste Lafabregue 2019.25.04 9 | """ 10 | 11 | import numpy as np 12 | 13 | from tensorflow.keras import layers 14 | from tensorflow.keras import Model 15 | import tensorflow.keras.backend as K 16 | import tensorflow as tf 17 | 18 | from networks.encoders import RnnAutoencoderModel 19 | from networks.encoders import LayersGenerator 20 | 21 | 22 | class FormatLayer(layers.Layer): 23 | 24 | def __init__(self, n_steps, dilation_rate): 25 | super(FormatLayer, self).__init__() 26 | self.n_steps = n_steps 27 | self.dilation_rate = dilation_rate 28 | 29 | def call(self, inputs): 30 | # make the length of inputs divide 'rate', by using zero-padding 31 | EVEN = (self.n_steps % self.dilation_rate) == 0 32 | if not EVEN: 33 | # Create a tensor in shape (batch_size, input_dims), which all elements are zero. 34 | # This is used for zero padding 35 | zero_tensor = K.zeros_like(inputs[0]) 36 | zero_tensor = K.expand_dims(zero_tensor, axis=0) 37 | dilated_n_steps = self.n_steps // self.dilation_rate + 1 38 | # print("=====> %d time points need to be padded. " % (dilated_n_steps * self.dilation_rate - self.n_steps)) 39 | # print("=====> Input length for sub-RNN: %d" % dilated_n_steps) 40 | for i_pad in range(dilated_n_steps * self.dilation_rate - self.n_steps): 41 | inputs = K.concatenate((inputs, zero_tensor), axis=0) 42 | else: 43 | dilated_n_steps = self.n_steps // self.dilation_rate 44 | # print("=====> Input length for sub-RNN: %d" % dilated_n_steps) 45 | # now the length of 'inputs' divide rate 46 | # reshape it in the format of a list of tensors 47 | # the length of the list is 'dilated_n_steps' 48 | # the shape of each tensor is [batch_size * rate, input_dims] 49 | # by stacking tensors that "colored" the same 50 | 51 | # Example: 52 | # n_steps is 5, rate is 2, inputs = [x1, x2, x3, x4, x5] 53 | # zero-padding --> [x1, x2, x3, x4, x5, 0] 54 | # we want to have --> [[x1; x2], [x3; x4], [x_5; 0]] 55 | # which the length is the ceiling of n_steps/rate 56 | # print('format call inputs '+str(inputs)) 57 | dilated_inputs = [tf.concat(tf.unstack(inputs[i * self.dilation_rate:(i + 1) * self.dilation_rate]), 58 | axis=0) for i in range(dilated_n_steps)] 59 | 60 | # print('format call dilated '+str(dilated_inputs)) 61 | dilated_inputs = tf.stack(dilated_inputs) 62 | return dilated_inputs 63 | 64 | def compute_output_shape(self, input_shape): 65 | int_input_shape = input_shape.as_list() 66 | dilated_n_steps = int(np.ceil(self.n_steps / self.dilation_rate)) 67 | 68 | shape2 = None 69 | if int_input_shape[1] is not None: 70 | shape2 = self.dilation_rate * int_input_shape[1] 71 | 72 | output_shape = tf.TensorShape([dilated_n_steps, shape2, 73 | int_input_shape[2]]) 74 | return output_shape 75 | 76 | 77 | class ReformatLayer(layers.Layer): 78 | 79 | def __init__(self, n_steps, dilation_rate): 80 | super(ReformatLayer, self).__init__() 81 | self.n_steps = n_steps 82 | self.dilation_rate = dilation_rate 83 | 84 | def call(self, inputs): 85 | # reshape output back to the input format as a list of tensors with shape [batch_size, input_dims] 86 | # split each element of the outputs from size [batch_size*rate, input_dims] to 87 | # [[batch_size, input_dims], [batch_size, input_dims], ...] with length = rate 88 | # print("input 12 : "+str(inputs)) 89 | inputs = tf.unstack(inputs) 90 | splitted_outputs = [tf.split(output, self.dilation_rate, axis=0) 91 | for output in inputs] 92 | # print("splitted "+str(splitted_outputs)) 93 | unrolled_outputs = [output 94 | for sublist in splitted_outputs for output in sublist] 95 | # unrolled_outputs = tf.unstack(splitted_outputs) 96 | # remove padded zeros 97 | outputs = unrolled_outputs[:self.n_steps] 98 | # print("outputs "+str(outputs)) 99 | 100 | outputs = tf.stack(outputs) 101 | # print("stack "+str(outputs)) 102 | 103 | return outputs 104 | 105 | def compute_output_shape(self, input_shape): 106 | int_input_shape = input_shape.as_list() 107 | 108 | shape2 = None 109 | if int_input_shape[1] is not None: 110 | shape2 = int_input_shape[1] // self.dilation_rate 111 | 112 | output_shape = tf.TensorShape([self.n_steps, shape2, 113 | int_input_shape[2]]) 114 | return output_shape 115 | 116 | 117 | def _construct_dilated_rnn(input, cell, dilation_rate, n_steps, return_sequences=True, bidirectional=True): 118 | 119 | if dilation_rate < 0 or dilation_rate >= n_steps: 120 | raise ValueError('The \'dilation rate\' is lower than the number of time steps.') 121 | 122 | h = FormatLayer(n_steps, dilation_rate)(input) 123 | 124 | # building a dilated RNN with formated (dilated) inputs 125 | rnn = layers.RNN(cell, return_sequences=return_sequences, time_major=True, return_state=True) 126 | if bidirectional: 127 | rnn = layers.Bidirectional(rnn) 128 | outputs = rnn(h) 129 | h, state = outputs[0], outputs[1:] 130 | 131 | if return_sequences: 132 | h = ReformatLayer(n_steps, dilation_rate)(h) 133 | 134 | return h, state 135 | 136 | 137 | def reformat_representation_layer(inputs, dilation_rate): 138 | splitted_outputs = [tf.split(i, dilation_rate, axis=0) for i in inputs] 139 | return [tf.reduce_sum(s, axis=0) for s in splitted_outputs] 140 | 141 | 142 | class ReformatRepresentationLayer(layers.Layer): 143 | 144 | def __init__(self, dilation_rate, batch_size): 145 | super(ReformatRepresentationLayer, self).__init__() 146 | self.dilation_rate = dilation_rate 147 | self.batch_size = batch_size 148 | 149 | def call(self, inputs): 150 | # reshape output back to the input format as a list of tensors with shape [batch_size, input_dims] 151 | # split each element of the outputs from size [batch_size*rate, input_dims] to 152 | # [[batch_size, input_dims], [batch_size, input_dims], ...] with length = rate 153 | splitted_outputs = tf.split(inputs, self.dilation_rate, axis=0) 154 | # then we do a reduce sum to sum all tensors to one [batch_size, input_dims] tensor 155 | outputs = tf.reduce_sum(splitted_outputs, axis=0) 156 | return outputs 157 | 158 | def compute_output_shape(self, input_shape): 159 | int_input_shape = input_shape.as_list() 160 | output_shape = tf.TensorShape([self.batch_size, int_input_shape[1]]) 161 | return output_shape 162 | 163 | 164 | class TimeMajorLayer(layers.Layer): 165 | """ 166 | This Layer reformat input to the shape that standard RNN can take (Time Major). 167 | 168 | Inputs: 169 | x -- a tensor of shape (batch_size, n_steps, channel_dim). 170 | Outputs: 171 | x_reformat -- a list of 'n_steps' tensors, each has shape (batch_size, input_dims). 172 | """ 173 | 174 | def __init__(self, n_steps, channel_dim): 175 | super(TimeMajorLayer, self).__init__() 176 | self.n_steps = n_steps 177 | self.channel_dim = channel_dim 178 | 179 | def call(self, inputs): 180 | # permute batch_size and n_steps 181 | x_ = tf.transpose(inputs, [1, 0, 2]) 182 | # reshape to (n_steps*batch_size, input_dims) 183 | x_ = tf.reshape(x_, [-1, self.channel_dim]) 184 | # split to get a list of 'n_steps' tensors of shape (batch_size, input_dims) 185 | x_reformat = tf.split(x_, self.n_steps, 0) 186 | x_reformat = tf.stack(x_reformat) 187 | 188 | return x_reformat 189 | 190 | 191 | class BatchMajorLayer(layers.Layer): 192 | """ 193 | This Layer reformat input to standard shape (Batch Major). 194 | 195 | Inputs: 196 | x -- a list of 'n_steps' tensors, each has shape (batch_size, input_dims). 197 | Outputs: 198 | x_reformat -- a tensor of shape (batch_size, n_steps, channel_dim). 199 | """ 200 | 201 | def __init__(self, n_steps, channel_dim, batch_size): 202 | super(BatchMajorLayer, self).__init__() 203 | self.n_steps = n_steps 204 | self.channel_dim = channel_dim 205 | self.batch_size = batch_size 206 | 207 | def call(self, inputs): 208 | # permute batch_size and n_steps 209 | x_ = tf.transpose(inputs, [1, 0, 2]) 210 | # reshape to (n_steps*batch_size, input_dims) 211 | x_ = tf.reshape(x_, [-1, self.channel_dim]) 212 | # split to get a list of 'n_steps' tensors of shape (batch_size, input_dims) 213 | n_batch = self.batch_size 214 | if K.int_shape(x_)[0] is not None: 215 | n_batch = K.int_shape(x_)[0]//self.n_steps 216 | x_reformat = tf.split(x_, n_batch, 0) 217 | x_reformat = tf.stack(x_reformat) 218 | 219 | return x_reformat 220 | 221 | 222 | def _append_states(concat_states, new_state, bidirectional=True): 223 | if bidirectional: 224 | tmp_states = [] 225 | nb_states = int(len(new_state) / 2) 226 | for i in range(nb_states): 227 | tmp = tf.concat([new_state[i], new_state[nb_states + i]], axis=-1) 228 | tmp_states.append(tmp) 229 | new_state = tmp_states 230 | 231 | if len(concat_states) < len(new_state): 232 | for i in range(len(new_state)-len(concat_states)): concat_states.append([]) 233 | for i, si in zip(range(len(new_state)), new_state): 234 | concat_states[i].append(si) 235 | 236 | 237 | class AutoEncoder(LayersGenerator): 238 | """docstring for AutoEncoder""" 239 | 240 | def __init__(self, 241 | x, 242 | encoder_loss, 243 | batch_size=10, 244 | nb_RNN_units=[100, 50, 50], 245 | dilations=[1, 4, 16], 246 | cell_type='GRU', 247 | alpha=1.0, 248 | nb_steps=2000, 249 | bidirectional=True): 250 | self.input_shape = x.shape[1:] 251 | if len(self.input_shape) > 2: 252 | self.input_shape = self.input_shape[1:] 253 | # we only consider the time and channel dimensions 254 | self.L = self.input_shape[0] 255 | self.C = self.input_shape[1] 256 | self.batch_size = batch_size 257 | self.nb_RNN_units = nb_RNN_units 258 | self.cell_type = cell_type 259 | assert self.cell_type in ['GRU', 'LSTM', 'RNN'] 260 | self.dilations = dilations 261 | if self.dilations is None: 262 | self.dilations = [2**i for i in range(len(nb_RNN_units))] 263 | self.alpha = alpha 264 | self.bidirectional = bidirectional 265 | self.inputs_ = layers.Input(shape=(self.L, self.C)) 266 | self.encoder = None 267 | self._decoder = None 268 | # the decoder should be designed to fit general framework to ba added 269 | self.decoder = None 270 | self.nb_steps = nb_steps 271 | 272 | state = self._encoder_network() 273 | self._decoder_network(init_state=state) 274 | self.autoencoder = RnnAutoencoderModel(self.encoder, self._decoder) 275 | 276 | # self.input_batch_size = tf.placeholder(tf.int32, shape=[]) 277 | 278 | lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay( 279 | 0.1, 280 | decay_steps=nb_steps//4, 281 | decay_rate=0.1, 282 | staircase=True) 283 | self.optimizer = tf.keras.optimizers.SGD(learning_rate=lr_schedule, momentum=0.9) 284 | 285 | def _construct_encoder_cells(self): 286 | """ 287 | This function contructs a list of cells for the encoder. 288 | """ 289 | # define cells 290 | cells = [] 291 | for u in self.nb_RNN_units: 292 | if self.cell_type == "RNN": 293 | cell = layers.SimpleRNNCell(u) 294 | elif self.cell_type == "LSTM": 295 | cell = layers.LSTMCell(u) 296 | elif self.cell_type == "GRU": 297 | cell = layers.GRUCell(u) 298 | cells.append(cell) 299 | 300 | return cells 301 | 302 | def _encoder_network(self): 303 | 304 | cells = self._construct_encoder_cells() 305 | concat_states = [[]] 306 | 307 | with tf.name_scope('encoder'): 308 | h = TimeMajorLayer(self.L, self.C)(self.inputs_) 309 | 310 | for cell, dilation in zip(cells[:-1], self.dilations[:-1]): 311 | scope_name = 'dRNN_'+str(dilation) 312 | h, tmp_state = _construct_dilated_rnn(h, cell, dilation, self.L, 313 | bidirectional=self.bidirectional) 314 | _append_states(concat_states, reformat_representation_layer(tmp_state, dilation)) 315 | 316 | h, tmp_state = _construct_dilated_rnn(h, cells[-1], self.dilations[-1], self.L, 317 | return_sequences=False, bidirectional=self.bidirectional) 318 | _append_states(concat_states, reformat_representation_layer(tmp_state, self.dilations[-1])) 319 | 320 | h = ReformatRepresentationLayer(self.dilations[-1], self.batch_size)(h) 321 | 322 | # h = BatchMajorLayer(self.nb_steps, self.C, self.batch_size)(h) 323 | self.encoder = Model(inputs=self.inputs_, outputs=h) 324 | 325 | return [tf.concat(sub_states, axis=1) for sub_states in concat_states] 326 | 327 | def _construct_decoder_cell(self, bidirectional=True): 328 | """ 329 | This function contructs a cell for the decoder. 330 | """ 331 | nb_units = 0 332 | # compute nb units based on encoder units 333 | for u in self.nb_RNN_units: 334 | nb_units += u 335 | 336 | if bidirectional: 337 | nb_units *= 2 338 | 339 | if self.cell_type == "RNN": 340 | cell = layers.SimpleRNNCell(nb_units) 341 | elif self.cell_type == "LSTM": 342 | cell = layers.LSTMCell(nb_units) 343 | elif self.cell_type == "GRU": 344 | cell = layers.GRUCell(nb_units) 345 | 346 | return cell 347 | 348 | def _decoder_network(self, init_state=None): 349 | 350 | with tf.name_scope('decoder'): 351 | decoder_inputs = layers.Input(shape=(1, self.C)) 352 | tmp_inputs = decoder_inputs 353 | 354 | decode = layers.RNN(self._construct_decoder_cell(bidirectional=self.bidirectional), 355 | return_sequences=True, return_state=True) 356 | decode_dense = layers.Dense(self.C) 357 | 358 | all_outputs = [] 359 | for _ in range(self.L): 360 | # Run the decoder on one timestep 361 | outputs = decode(tmp_inputs, initial_state=init_state) 362 | output = outputs[0] 363 | output = decode_dense(output) 364 | # Store the current prediction (we will concatenate all predictions later) 365 | all_outputs.append(output) 366 | # Reinject the outputs as inputs for the next loop iteration 367 | # as well as update the states 368 | tmp_inputs = output 369 | init_state = outputs[1:] 370 | 371 | decoder_outputs = layers.Lambda(lambda x: K.concatenate(x, axis=1))(all_outputs) 372 | 373 | # decoder_input_data = np.zeros((num_samples, 1, num_decoder_tokens)) 374 | 375 | self._decoder = Model(inputs=[self.inputs_, decoder_inputs], outputs=decoder_outputs) 376 | -------------------------------------------------------------------------------- /networks/bi_dilated_RNN_variable_length.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | 3 | import numpy as np 4 | 5 | from tensorflow.keras import layers 6 | from tensorflow.keras import Model 7 | import tensorflow.keras.backend as K 8 | import tensorflow as tf 9 | 10 | from networks.encoders import RnnAutoencoderModel 11 | 12 | 13 | class FormatLayer(layers.Layer): 14 | 15 | def __init__(self, n_steps, dilation_rate): 16 | super(FormatLayer, self).__init__() 17 | self.n_steps = n_steps 18 | self.dilation_rate = dilation_rate 19 | 20 | def call(self, inputs): 21 | # make the length of inputs divide 'rate', by using zero-padding 22 | EVEN = (self.n_steps % self.dilation_rate) == 0 23 | if not EVEN: 24 | # Create a tensor in shape (batch_size, input_dims), which all elements are zero. 25 | # This is used for zero padding 26 | zero_tensor = K.zeros_like(inputs[0]) 27 | zero_tensor = K.expand_dims(zero_tensor, axis=0) 28 | dilated_n_steps = self.n_steps // self.dilation_rate + 1 29 | # print("=====> %d time points need to be padded. " % (dilated_n_steps * self.dilation_rate - self.n_steps)) 30 | # print("=====> Input length for sub-RNN: %d" % dilated_n_steps) 31 | for i_pad in range(dilated_n_steps * self.dilation_rate - self.n_steps): 32 | inputs = K.concatenate((inputs, zero_tensor), axis=0) 33 | else: 34 | dilated_n_steps = self.n_steps // self.dilation_rate 35 | # print("=====> Input length for sub-RNN: %d" % dilated_n_steps) 36 | # now the length of 'inputs' divide rate 37 | # reshape it in the format of a list of tensors 38 | # the length of the list is 'dilated_n_steps' 39 | # the shape of each tensor is [batch_size * rate, input_dims] 40 | # by stacking tensors that "colored" the same 41 | 42 | # Example: 43 | # n_steps is 5, rate is 2, inputs = [x1, x2, x3, x4, x5] 44 | # zero-padding --> [x1, x2, x3, x4, x5, 0] 45 | # we want to have --> [[x1; x2], [x3; x4], [x_5; 0]] 46 | # which the length is the ceiling of n_steps/rate 47 | # print('format call inputs '+str(inputs)) 48 | dilated_inputs = [tf.concat(tf.unstack(inputs[i * self.dilation_rate:(i + 1) * self.dilation_rate]), 49 | axis=0) for i in range(dilated_n_steps)] 50 | 51 | # print('format call dilated '+str(dilated_inputs)) 52 | dilated_inputs = tf.stack(dilated_inputs) 53 | return dilated_inputs 54 | 55 | def compute_output_shape(self, input_shape): 56 | int_input_shape = input_shape.as_list() 57 | dilated_n_steps = int(np.ceil(self.n_steps / self.dilation_rate)) 58 | 59 | shape2 = None 60 | if int_input_shape[1] is not None: 61 | shape2 = self.dilation_rate * int_input_shape[1] 62 | 63 | output_shape = tf.TensorShape([dilated_n_steps, shape2, 64 | int_input_shape[2]]) 65 | return output_shape 66 | 67 | 68 | class ReformatLayer(layers.Layer): 69 | 70 | def __init__(self, n_steps, dilation_rate): 71 | super(ReformatLayer, self).__init__() 72 | self.n_steps = n_steps 73 | self.dilation_rate = dilation_rate 74 | 75 | def call(self, inputs): 76 | # reshape output back to the input format as a list of tensors with shape [batch_size, input_dims] 77 | # split each element of the outputs from size [batch_size*rate, input_dims] to 78 | # [[batch_size, input_dims], [batch_size, input_dims], ...] with length = rate 79 | # print("input 12 : "+str(inputs)) 80 | inputs = tf.unstack(inputs) 81 | splitted_outputs = [tf.split(output, self.dilation_rate, axis=0) 82 | for output in inputs] 83 | # print("splitted "+str(splitted_outputs)) 84 | unrolled_outputs = [output 85 | for sublist in splitted_outputs for output in sublist] 86 | # unrolled_outputs = tf.unstack(splitted_outputs) 87 | # remove padded zeros 88 | outputs = unrolled_outputs[:self.n_steps] 89 | # print("outputs "+str(outputs)) 90 | 91 | outputs = tf.stack(outputs) 92 | # print("stack "+str(outputs)) 93 | 94 | return outputs 95 | 96 | def compute_output_shape(self, input_shape): 97 | int_input_shape = input_shape.as_list() 98 | 99 | shape2 = None 100 | if int_input_shape[1] is not None: 101 | shape2 = int_input_shape[1] // self.dilation_rate 102 | 103 | output_shape = tf.TensorShape([self.n_steps, shape2, 104 | int_input_shape[2]]) 105 | return output_shape 106 | 107 | 108 | def _construct_dilated_rnn(input, cell, dilation_rate, n_steps, return_sequences=True, bidirectional=True): 109 | 110 | if dilation_rate < 0 or dilation_rate >= n_steps: 111 | raise ValueError('The \'dilation rate\' is lower than the number of time steps.') 112 | 113 | h = FormatLayer(n_steps, dilation_rate)(input) 114 | 115 | # building a dilated RNN with formated (dilated) inputs 116 | rnn = layers.RNN(cell, return_sequences=return_sequences, time_major=True, return_state=True) 117 | if bidirectional: 118 | rnn = layers.Bidirectional(rnn) 119 | outputs = rnn(h) 120 | h, state = outputs[0], outputs[1:] 121 | 122 | if return_sequences: 123 | h = ReformatLayer(n_steps, dilation_rate)(h) 124 | 125 | return h, state 126 | 127 | 128 | def reformat_representation_layer(inputs, dilation_rate): 129 | splitted_outputs = [tf.split(i, dilation_rate, axis=0) for i in inputs] 130 | return [tf.reduce_sum(s, axis=0) for s in splitted_outputs] 131 | 132 | 133 | class ReformatRepresentationLayer(layers.Layer): 134 | 135 | def __init__(self, dilation_rate, batch_size): 136 | super(ReformatRepresentationLayer, self).__init__() 137 | self.dilation_rate = dilation_rate 138 | self.batch_size = batch_size 139 | 140 | def call(self, inputs): 141 | # reshape output back to the input format as a list of tensors with shape [batch_size, input_dims] 142 | # split each element of the outputs from size [batch_size*rate, input_dims] to 143 | # [[batch_size, input_dims], [batch_size, input_dims], ...] with length = rate 144 | splitted_outputs = tf.split(inputs, self.dilation_rate, axis=0) 145 | # then we do a reduce sum to sum all tensors to one [batch_size, input_dims] tensor 146 | outputs = tf.reduce_sum(splitted_outputs, axis=0) 147 | return outputs 148 | 149 | def compute_output_shape(self, input_shape): 150 | int_input_shape = input_shape.as_list() 151 | output_shape = tf.TensorShape([self.batch_size, int_input_shape[1]]) 152 | return output_shape 153 | 154 | 155 | class TimeMajorLayer(layers.Layer): 156 | """ 157 | This Layer reformat input to the shape that standard RNN can take (Time Major). 158 | 159 | Inputs: 160 | x -- a tensor of shape (batch_size, n_steps, channel_dim). 161 | Outputs: 162 | x_reformat -- a list of 'n_steps' tensors, each has shape (batch_size, input_dims). 163 | """ 164 | 165 | def __init__(self, n_steps, channel_dim): 166 | super(TimeMajorLayer, self).__init__() 167 | self.n_steps = n_steps 168 | self.channel_dim = channel_dim 169 | 170 | def call(self, inputs): 171 | # permute batch_size and n_steps 172 | x_ = tf.transpose(inputs, [1, 0, 2]) 173 | # reshape to (n_steps*batch_size, input_dims) 174 | x_ = tf.reshape(x_, [-1, self.channel_dim]) 175 | # split to get a list of 'n_steps' tensors of shape (batch_size, input_dims) 176 | x_reformat = tf.split(x_, self.n_steps, 0) 177 | x_reformat = tf.stack(x_reformat) 178 | 179 | return x_reformat 180 | 181 | 182 | class BatchMajorLayer(layers.Layer): 183 | """ 184 | This Layer reformat input to standard shape (Batch Major). 185 | 186 | Inputs: 187 | x -- a list of 'n_steps' tensors, each has shape (batch_size, input_dims). 188 | Outputs: 189 | x_reformat -- a tensor of shape (batch_size, n_steps, channel_dim). 190 | """ 191 | 192 | def __init__(self, n_steps, channel_dim, batch_size): 193 | super(BatchMajorLayer, self).__init__() 194 | self.n_steps = n_steps 195 | self.channel_dim = channel_dim 196 | self.batch_size = batch_size 197 | 198 | def call(self, inputs): 199 | # permute batch_size and n_steps 200 | x_ = tf.transpose(inputs, [1, 0, 2]) 201 | # reshape to (n_steps*batch_size, input_dims) 202 | x_ = tf.reshape(x_, [-1, self.channel_dim]) 203 | # split to get a list of 'n_steps' tensors of shape (batch_size, input_dims) 204 | n_batch = self.batch_size 205 | if K.int_shape(x_)[0] is not None: 206 | n_batch = K.int_shape(x_)[0]//self.n_steps 207 | x_reformat = tf.split(x_, n_batch, 0) 208 | x_reformat = tf.stack(x_reformat) 209 | 210 | return x_reformat 211 | 212 | 213 | def _append_states(concat_states, new_state, bidirectional=True): 214 | if bidirectional: 215 | tmp_states = [] 216 | nb_states = int(len(new_state) / 2) 217 | for i in range(nb_states): 218 | tmp = tf.concat([new_state[i], new_state[nb_states + i]], axis=-1) 219 | tmp_states.append(tmp) 220 | new_state = tmp_states 221 | 222 | if len(concat_states) < len(new_state): 223 | for i in range(len(new_state)-len(concat_states)): concat_states.append([]) 224 | for i, si in zip(range(len(new_state)), new_state): 225 | concat_states[i].append(si) 226 | 227 | 228 | class AutoEncoderGenerator(object): 229 | """docstring for AutoEncoder""" 230 | 231 | def __init__(self, 232 | input_shape, 233 | batch_size=10, 234 | nb_RNN_units=[100, 50, 50], 235 | dilations=[1, 4, 16], 236 | cell_type='GRU', 237 | n_filters_CNN=100, 238 | kernel_size=10, 239 | alpha=1.0, 240 | nb_steps=2000, 241 | bidirectional=True): 242 | self.input_shape = input_shape 243 | if len(self.input_shape) > 2: 244 | self.input_shape = self.input_shape[1:] 245 | # we only consider the time and channel dimensions 246 | self.L = self.input_shape[0] 247 | self.C = self.input_shape[1] 248 | self.batch_size = batch_size 249 | self.nb_RNN_units = nb_RNN_units 250 | self.cell_type = cell_type 251 | assert self.cell_type in ['GRU', 'LSTM', 'RNN'] 252 | self.n_filters_CNN = n_filters_CNN 253 | self.kernel_size = kernel_size 254 | self.dilations = dilations 255 | if self.dilations is None: 256 | self.dilations = [2**i for i in range(len(nb_RNN_units))] 257 | self.alpha = alpha 258 | self.bidirectional = bidirectional 259 | self.inputs_ = layers.Input(shape=(self.L, self.C)) 260 | self.encoder = None 261 | self._decoder = None 262 | self.nb_steps = nb_steps 263 | 264 | state = self._encoder_network() 265 | self._decoder_network(init_state=state) 266 | self.autoencoder = RnnAutoencoderModel(self.encoder, self._decoder) 267 | 268 | # self.input_batch_size = tf.placeholder(tf.int32, shape=[]) 269 | 270 | lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay( 271 | 0.1, 272 | decay_steps=nb_steps//4, 273 | decay_rate=0.1, 274 | staircase=True) 275 | self.optimizer = tf.keras.optimizers.SGD(learning_rate=lr_schedule, momentum=0.9) 276 | 277 | def _construct_encoder_cells(self): 278 | """ 279 | This function contructs a list of cells for the encoder. 280 | """ 281 | # define cells 282 | cells = [] 283 | for u in self.nb_RNN_units: 284 | if self.cell_type == "RNN": 285 | cell = layers.SimpleRNNCell(u) 286 | elif self.cell_type == "LSTM": 287 | cell = layers.LSTMCell(u) 288 | elif self.cell_type == "GRU": 289 | cell = layers.GRUCell(u) 290 | cells.append(cell) 291 | 292 | return cells 293 | 294 | def _encoder_network(self): 295 | 296 | cells = self._construct_encoder_cells() 297 | concat_states = [[]] 298 | 299 | with tf.name_scope('encoder'): 300 | h = TimeMajorLayer(self.L, self.C)(self.inputs_) 301 | 302 | for cell, dilation in zip(cells[:-1], self.dilations[:-1]): 303 | scope_name = 'dRNN_'+str(dilation) 304 | h, tmp_state = _construct_dilated_rnn(h, cell, dilation, self.L, 305 | bidirectional=self.bidirectional) 306 | _append_states(concat_states, reformat_representation_layer(tmp_state, dilation)) 307 | 308 | h, tmp_state = _construct_dilated_rnn(h, cells[-1], self.dilations[-1], self.L, 309 | return_sequences=False, bidirectional=self.bidirectional) 310 | _append_states(concat_states, reformat_representation_layer(tmp_state, self.dilations[-1])) 311 | 312 | h = ReformatRepresentationLayer(self.dilations[-1], self.batch_size)(h) 313 | 314 | # h = BatchMajorLayer(self.nb_steps, self.C, self.batch_size)(h) 315 | self.encoder = Model(inputs=self.inputs_, outputs=h) 316 | 317 | return [tf.concat(sub_states, axis=1) for sub_states in concat_states] 318 | 319 | def _construct_decoder_cell(self, bidirectional=True): 320 | """ 321 | This function contructs a cell for the decoder. 322 | """ 323 | nb_units = 0 324 | # compute nb units based on encoder units 325 | for u in self.nb_RNN_units: 326 | nb_units += u 327 | 328 | if bidirectional: 329 | nb_units *= 2 330 | 331 | if self.cell_type == "RNN": 332 | cell = layers.SimpleRNNCell(nb_units) 333 | elif self.cell_type == "LSTM": 334 | cell = layers.LSTMCell(nb_units) 335 | elif self.cell_type == "GRU": 336 | cell = layers.GRUCell(nb_units) 337 | 338 | return cell 339 | 340 | def _decoder_network(self, init_state=None): 341 | 342 | with tf.name_scope('decoder'): 343 | decoder_inputs = layers.Input(shape=(1, self.C)) 344 | tmp_inputs = decoder_inputs 345 | 346 | decode = layers.RNN(self._construct_decoder_cell(bidirectional=self.bidirectional), 347 | return_sequences=True, return_state=True) 348 | decode_dense = layers.Dense(self.C) 349 | 350 | all_outputs = [] 351 | for _ in range(self.L): 352 | # Run the decoder on one timestep 353 | outputs = decode(tmp_inputs, initial_state=init_state) 354 | output = outputs[0] 355 | output = decode_dense(output) 356 | # Store the current prediction (we will concatenate all predictions later) 357 | all_outputs.append(output) 358 | # Reinject the outputs as inputs for the next loop iteration 359 | # as well as update the states 360 | tmp_inputs = output 361 | init_state = outputs[1:] 362 | 363 | decoder_outputs = layers.Lambda(lambda x: K.concatenate(x, axis=1))(all_outputs) 364 | 365 | # decoder_input_data = np.zeros((num_samples, 1, num_decoder_tokens)) 366 | 367 | self._decoder = Model(inputs=[self.inputs_, decoder_inputs], outputs=decoder_outputs) 368 | -------------------------------------------------------------------------------- /networks/bilstm_ae.py: -------------------------------------------------------------------------------- 1 | """ 2 | Define the temporal Autoencoder proposed in: 3 | Madiraju, N. S., Sadat, S. M., Fisher, D., & Karimabadi, H. (2018). 4 | Deep temporal clustering: Fully unsupervised learning of time-domain features 5 | 6 | Author: 7 | Baptiste Lafabregue 2019.25.04 8 | """ 9 | 10 | import os 11 | 12 | import tensorflow as tf 13 | from tensorflow.keras import backend as K 14 | from tensorflow.keras import layers 15 | from tensorflow.keras import Model 16 | 17 | from networks.encoders import LayersGenerator 18 | 19 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 20 | 21 | 22 | class AutoEncoder(LayersGenerator): 23 | """docstring for AutoEncoder""" 24 | 25 | def __init__(self, 26 | x, 27 | encoder_loss, 28 | n_clusters, 29 | n_filters_CNN=100, 30 | kernel_size=10, 31 | P=10, 32 | n_filters_RNN_list=[50, 50], 33 | alpha=1.0, 34 | nb_steps=2000): 35 | self.n_clusters = n_clusters 36 | self.L = x.shape[1] 37 | self.C = x.shape[2] 38 | self.n_filters_CNN = n_filters_CNN 39 | self.kernel_size = kernel_size 40 | self.P = P 41 | self.n_filters_RNN_list = n_filters_RNN_list 42 | self.alpha = alpha 43 | self.input_ = layers.Input(shape=(None, self.C)) 44 | 45 | super().__init__(x, encoder_loss) 46 | 47 | lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay( 48 | 0.1, 49 | decay_steps=nb_steps//4, 50 | decay_rate=0.1, 51 | staircase=True) 52 | self.optimizer = tf.keras.optimizers.SGD(learning_rate=lr_schedule, momentum=0.9) 53 | 54 | def _create_encoder(self, x): 55 | 56 | with tf.name_scope('encoder'): 57 | h = layers.Conv1D(filters=self.n_filters_CNN, 58 | kernel_size=self.kernel_size, 59 | padding='same')(self.input_) 60 | # TODO: check alpha from leakyrelu 61 | h = layers.LeakyReLU()(h) 62 | h = layers.MaxPool1D(pool_size=self.kernel_size, 63 | strides=self.P)(h) 64 | for i in range(len(self.n_filters_RNN_list)): 65 | h = layers.Bidirectional(layers.LSTM(self.n_filters_RNN_list[i], return_sequences=True))(h) 66 | # h = layers.Bidirectional(layers.LSTM(self.n_filters_RNN_list[-1]))(h) 67 | h = layers.Flatten()(h) 68 | 69 | return Model(inputs=self.input_, outputs=h), None 70 | 71 | def _create_decoder(self, x): 72 | 73 | with tf.name_scope('decoder'): 74 | decoder_input = layers.Input(shape=(self.enc_shape[1])) 75 | upsampled = layers.Reshape((self.enc_shape[1], 1, 1))(decoder_input) 76 | 77 | upsampled = tf.image.resize(upsampled, [self.L + self.kernel_size - 1, 1]) 78 | upsampled_shape = K.int_shape(upsampled) 79 | upsampled = layers.Reshape((upsampled_shape[1], upsampled_shape[2]))(upsampled) 80 | 81 | decode = layers.Conv1D(filters=self.C, kernel_size=self.kernel_size)(upsampled) 82 | 83 | return Model(inputs=decoder_input, outputs=decode), None 84 | 85 | -------------------------------------------------------------------------------- /networks/birnn_ae.py: -------------------------------------------------------------------------------- 1 | """ 2 | Define the Bi-directional RNN autoencoder (GRU, LSTM and RNN gates are supported) 3 | 4 | Author: 5 | Baptiste Lafabregue 2019.25.04 6 | """ 7 | import os 8 | import numpy as np 9 | 10 | import tensorflow as tf 11 | from tensorflow.keras import backend as K 12 | from tensorflow.keras import layers 13 | from tensorflow.keras import Model 14 | 15 | from networks.encoders import LayersGenerator 16 | 17 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 18 | 19 | 20 | class AutoEncoder(LayersGenerator): 21 | """docstring for AutoEncoder""" 22 | 23 | def __init__(self, 24 | x, 25 | encoder_loss, 26 | n_clusters, 27 | n_filters_RNN_list=[50, 50], 28 | latent_dim=None, 29 | cell_type='LSTM' 30 | ): 31 | self.n_clusters = n_clusters 32 | self.L = x.shape[1] 33 | self.C = x.shape[2] 34 | self.cell_type = cell_type 35 | assert self.cell_type in ['GRU', 'LSTM', 'RNN'] 36 | self.n_filters_RNN_list = n_filters_RNN_list 37 | if latent_dim is not None: 38 | self.n_filters_RNN_list[-1] = latent_dim//2 39 | self.input_ = layers.Input(shape=(None, self.C)) 40 | 41 | super().__init__(x, encoder_loss) 42 | 43 | def _create_encoder(self, x): 44 | 45 | with tf.name_scope('encoder'): 46 | h = self.input_ 47 | for i in range(len(self.n_filters_RNN_list)-1): 48 | if self.cell_type == 'LSTM': 49 | h = layers.Bidirectional(layers.LSTM(self.n_filters_RNN_list[i], return_sequences=True))(h) 50 | elif self.cell_type == 'GRU': 51 | h = layers.Bidirectional(layers.GRU(self.n_filters_RNN_list[i], return_sequences=True))(h) 52 | elif self.cell_type == 'RNN': 53 | h = layers.Bidirectional(layers.RNN(self.n_filters_RNN_list[i], return_sequences=True))(h) 54 | 55 | if self.cell_type == 'LSTM': 56 | h = layers.Bidirectional(layers.LSTM(self.n_filters_RNN_list[-1]))(h) 57 | elif self.cell_type == 'GRU': 58 | h = layers.Bidirectional(layers.GRU(self.n_filters_RNN_list[-1]))(h) 59 | elif self.cell_type == 'RNN': 60 | h = layers.Bidirectional(layers.RNN(self.n_filters_RNN_list[-1]))(h) 61 | 62 | return Model(inputs=self.input_, outputs=h), None 63 | 64 | def _create_decoder(self, x): 65 | 66 | with tf.name_scope('decoder'): 67 | decoder_input = layers.Input(shape=(self.enc_shape[1])) 68 | h = layers.RepeatVector(self.L)(decoder_input) 69 | decoder_filters = np.flip(self.n_filters_RNN_list) 70 | for i in range(len(decoder_filters)): 71 | if self.cell_type == 'LSTM': 72 | h = layers.Bidirectional(layers.LSTM(decoder_filters[i], return_sequences=True))(h) 73 | elif self.cell_type == 'GRU': 74 | h = layers.Bidirectional(layers.GRU(decoder_filters[i], return_sequences=True))(h) 75 | elif self.cell_type == 'RNN': 76 | h = layers.Bidirectional(layers.RNN(decoder_filters[i], return_sequences=True))(h) 77 | 78 | decode = layers.TimeDistributed(layers.Dense(self.C))(h) 79 | 80 | return Model(inputs=decoder_input, outputs=decode), None 81 | 82 | -------------------------------------------------------------------------------- /networks/dilated_causal_cnn.py: -------------------------------------------------------------------------------- 1 | """ 2 | Based on torch implementation https://github.com/White-Link/UnsupervisedScalableRepresentationLearningTimeSeries 3 | and article: 4 | Franceschi, J. Y., Dieuleveut, A., & Jaggi, M. (2019). 5 | Unsupervised scalable representation learning for multivariate time series 6 | 7 | Author: 8 | Baptiste Lafabregue 2019.25.04 9 | """ 10 | import inspect 11 | import numpy as np 12 | 13 | from tensorflow.keras import Model 14 | from tensorflow.keras import backend as K 15 | from tensorflow.keras import layers 16 | from tensorflow.keras.layers import Conv1D, Dense, Input, Reshape, Dropout 17 | from tensorflow.keras.layers import GlobalMaxPool1D, Layer, LeakyReLU 18 | from tensorflow.keras import initializers 19 | 20 | import tensorflow_addons as tfa 21 | 22 | from networks.encoders import LayersGenerator 23 | 24 | 25 | def compute_adaptive_dilations(time_size): 26 | last_dilations=1 27 | dilations = [last_dilations] 28 | rate = 4 29 | if time_size < 50: 30 | rate = 2 31 | 32 | while True: 33 | if last_dilations > time_size/2: 34 | break 35 | last_dilations *= rate 36 | dilations.append(last_dilations) 37 | 38 | return dilations 39 | 40 | 41 | class CausalResidualBlock(Layer): 42 | 43 | def __init__(self, 44 | nb_filters, 45 | kernel_size, 46 | dilation_rate, 47 | final=False, 48 | kernel_initializer=None, 49 | **kwargs): 50 | self.dilation_rate = dilation_rate 51 | self.nb_filters = nb_filters 52 | self.kernel_size = kernel_size 53 | self.kernel_initializer = kernel_initializer 54 | if self.kernel_initializer is None: 55 | self.kernel_initializer = initializers.VarianceScaling((1.0/3), distribution="uniform") 56 | self.final = final 57 | self.layers_outputs = [] 58 | self.layers = [] 59 | self.shape_match_conv = None 60 | self.res_output_shape = None 61 | self.final_activation = None 62 | 63 | super(CausalResidualBlock, self).__init__(**kwargs) 64 | 65 | def build(self, input_shape): 66 | 67 | self.final = input_shape[-1] != self.nb_filters 68 | with K.name_scope(self.name): # name scope used to make sure weights get unique names 69 | self.layers = [] 70 | self.res_output_shape = input_shape 71 | 72 | for k in range(2): 73 | name = 'conv1D_{}'.format(k) 74 | with K.name_scope(name): 75 | self._add_and_activate_layer( 76 | tfa.layers.WeightNormalization( 77 | Conv1D(filters=self.nb_filters, 78 | kernel_size=self.kernel_size, 79 | dilation_rate=self.dilation_rate, 80 | kernel_initializer=self.kernel_initializer, 81 | bias_initializer=self.kernel_initializer, 82 | padding='causal', 83 | name=name) 84 | , data_init=False 85 | ) 86 | ) 87 | self._add_and_activate_layer(LeakyReLU(alpha=0.01)) 88 | 89 | if self.final: 90 | # 1x1 conv to match the shapes (channel dimension). 91 | name = 'conv1D_3' 92 | with K.name_scope(name): 93 | # make and build this layer separately because it directly uses input_shape 94 | self.shape_match_conv = Conv1D(filters=self.nb_filters, 95 | kernel_size=1, 96 | kernel_initializer=self.kernel_initializer, 97 | padding='same', 98 | name=name) 99 | # else: 100 | # self.shape_match_conv = Lambda(lambda x: x, name='identity') 101 | self.shape_match_conv.build(input_shape) 102 | self.res_output_shape = self.shape_match_conv.compute_output_shape(input_shape) 103 | 104 | # if self.final: 105 | # name = 'activ' 106 | # with K.name_scope(name): 107 | # self.final_activation = Activation('relu') 108 | # # self.final_activation = Activation(LeakyReLU(alpha=0.01)) 109 | # self.final_activation.build(self.res_output_shape) # probably isn't necessary 110 | # else: 111 | # self.final_activation = None 112 | 113 | # this is done to force Keras to add the layers in the list to self._layers 114 | for layer in self.layers: 115 | self.__setattr__(layer.name, layer) 116 | 117 | super(CausalResidualBlock, self).build(input_shape) # done to make sure self.built is set True 118 | 119 | def _add_and_activate_layer(self, layer): 120 | """Helper function for building layer 121 | 122 | Args: 123 | layer: Appends layer to internal layer list and builds it based on the current output 124 | shape of ResidualBlocK. Updates current output shape. 125 | 126 | """ 127 | self.layers.append(layer) 128 | self.layers[-1].build(self.res_output_shape) 129 | self.res_output_shape = self.layers[-1].compute_output_shape(self.res_output_shape) 130 | 131 | def call(self, inputs, training=None): 132 | """ 133 | Returns: A tuple where the first element is the residual model tensor, and the second 134 | is the skip connection tensor. 135 | """ 136 | x = inputs 137 | self.layers_outputs = [x] 138 | for layer in self.layers: 139 | training_flag = 'training' in dict(inspect.signature(layer.call).parameters) 140 | x = layer(x, training=training) if training_flag else layer(x) 141 | self.layers_outputs.append(x) 142 | 143 | if self.final: 144 | x2 = self.shape_match_conv(inputs) 145 | else: 146 | x2 = inputs 147 | self.layers_outputs.append(x2) 148 | res_x = layers.add([x2, x]) 149 | self.layers_outputs.append(res_x) 150 | 151 | if self.final_activation is not None: 152 | res_act_x = self.final_activation(res_x) 153 | self.layers_outputs.append(res_act_x) 154 | else: 155 | res_act_x = res_x 156 | return res_act_x 157 | 158 | def compute_output_shape(self, input_shape): 159 | return self.res_output_shape 160 | 161 | def get_config(self): 162 | config = super(CausalResidualBlock, self).get_config() 163 | config.update({'dilation_rate': self.dilation_rate}) 164 | config.update({'nb_filters': self.nb_filters}) 165 | config.update({'kernel_size': self.kernel_size}) 166 | config.update({'kernel_initializer': self.kernel_initializer}) 167 | config.update({'final': self.final}) 168 | return config 169 | 170 | 171 | class AutoEncoder(LayersGenerator): 172 | """docstring for AutoEncoder""" 173 | 174 | def __init__(self, 175 | x, 176 | encoder_loss, 177 | nb_filters=10, 178 | depth=1, 179 | reduced_size=10, 180 | latent_dim=10, 181 | kernel_size=4, 182 | dilations=None, 183 | dropout_rate=0 184 | ): 185 | self.nb_filters = nb_filters 186 | self.reduced_size = reduced_size 187 | self.latent_dim = latent_dim 188 | self.kernel_size = kernel_size 189 | self.dilations = dilations 190 | self.dilation_depth = depth 191 | self.dropout_rate = dropout_rate 192 | if dilations is None: 193 | self.dilations = [2**i for i in range(depth)] 194 | self.dilation_depth = len(self.dilations) 195 | 196 | super().__init__(x, encoder_loss, support_joint_training=True) 197 | 198 | def _create_encoder(self, x): 199 | 200 | i = Input(batch_shape=(None, None, x.shape[2])) 201 | dilation_size = self.dilations[0] 202 | layers_outputs = [] 203 | 204 | h = i 205 | if self.dropout_rate > 0: 206 | h = Dropout(self.dropout_rate)(h) 207 | for k in range(self.dilation_depth): 208 | h = CausalResidualBlock( 209 | nb_filters=self.nb_filters, 210 | kernel_size=self.kernel_size, 211 | dilation_rate=dilation_size, 212 | name='residual_block_{}'.format(k) 213 | )(h) 214 | layers_outputs.append(h) 215 | # build newest residual block 216 | dilation_size = self.dilations[k] 217 | 218 | # Last layer 219 | h = CausalResidualBlock( 220 | nb_filters=self.reduced_size, 221 | kernel_size=self.kernel_size, 222 | dilation_rate=dilation_size, 223 | name='residual_block_{}'.format(self.dilation_depth) 224 | )(h) 225 | 226 | h = GlobalMaxPool1D()(h) 227 | if self.dropout_rate > 0: 228 | h = Dropout(self.dropout_rate)(h) 229 | init = initializers.VarianceScaling((1.0 / 3), distribution="uniform") 230 | h = Dense(self.latent_dim, kernel_initializer=init, bias_initializer=init)(h) 231 | if x.shape[2]: 232 | h = layers.Activation('sigmoid')(h) 233 | layers_outputs.append(h) 234 | 235 | return Model(inputs=i, outputs=h), Model(inputs=i, outputs=layers_outputs) 236 | 237 | def _create_decoder(self, x): 238 | 239 | # Initial dilation size 240 | # we invert the order of dilations 241 | dilation_size = self.dilations[-1] 242 | dense_size = x.shape[1]*x.shape[2] 243 | layers_outputs = [] 244 | 245 | i = Input(shape=(self.enc_shape[1],)) 246 | h = Dense(dense_size)(i) 247 | h = Reshape((x.shape[1], x.shape[2]))(h) 248 | layers_outputs.append(h) 249 | for k in range(self.dilation_depth): 250 | h = CausalResidualBlock( 251 | nb_filters=self.nb_filters, 252 | kernel_size=self.kernel_size, 253 | dilation_rate=dilation_size, 254 | name='decoder_residual_block_{}'.format(k) 255 | )(h) 256 | layers_outputs.append(h) 257 | dilation_size = self.dilations[-(k+1)] 258 | 259 | h = CausalResidualBlock( 260 | nb_filters=x.shape[2], 261 | kernel_size=self.kernel_size, 262 | dilation_rate=dilation_size, 263 | name='residual_block_{}'.format(self.dilation_depth) 264 | )(h) 265 | layers_outputs.append(h) 266 | 267 | return Model(inputs=i, outputs=h), Model(inputs=i, outputs=layers_outputs) 268 | -------------------------------------------------------------------------------- /networks/encoders.py: -------------------------------------------------------------------------------- 1 | """ 2 | Define autoencoder abstract classes 3 | 4 | Author: 5 | Baptiste Lafabregue 2019.25.04 6 | """ 7 | 8 | import tensorflow.keras.backend as K 9 | from tensorflow.keras import Model 10 | import tensorflow as tf 11 | 12 | import numpy as np 13 | import os 14 | 15 | 16 | class EncoderWrapper(object): 17 | def __init__(self, encoder_clear, encoder_noisy): 18 | self.encoder_clear = encoder_clear 19 | self.encoder_noisy = encoder_noisy 20 | 21 | def __call__(self, inputs, clear=True, *args, **kwargs): 22 | if clear: 23 | return self.encoder_clear(inputs, *args, **kwargs) 24 | return self.encoder_noisy(inputs, *args, **kwargs) 25 | 26 | def predict(self, inputs, *args, **kwargs): 27 | return self.encoder_clear.predict(inputs, *args, **kwargs) 28 | 29 | 30 | class EncoderModel(object): 31 | def __init__(self, encoder, autoencoder, optimizer, loss, name, batch_size=10, nb_steps=100): 32 | self.encoder = encoder 33 | self.autoencoder = autoencoder 34 | self.optimize = optimizer 35 | self.loss = loss 36 | self.name = name 37 | self.batch_size = batch_size, 38 | self.nb_steps = nb_steps 39 | 40 | def load_weights(self, paths): 41 | if self.autoencoder is not None: 42 | self.autoencoder.load_weights(paths+self.name) 43 | else: 44 | self.encoder.load_weights(paths+self.name+'_encoder.h5') 45 | 46 | def save_weights(self, paths): 47 | if self.autoencoder is not None: 48 | self.autoencoder.save_weights(paths+self.name) 49 | else: 50 | self.encoder.save_weights(paths+self.name+'_encoder.h5') 51 | 52 | def exists(self, paths): 53 | if self.autoencoder is not None: 54 | return self.autoencoder.exists(paths+self.name) 55 | return os.path.exists(paths+self.name+'_encoder.h5') 56 | 57 | def get_trainable_variables(self): 58 | if self.autoencoder is not None: 59 | trainable_variables = self.autoencoder.get_trainable_variables() 60 | else: 61 | trainable_variables = self.encoder.trainable_variables 62 | return trainable_variables 63 | 64 | def get_name(self): 65 | return self.name 66 | 67 | def summary(self): 68 | if self.autoencoder is not None: 69 | self.autoencoder.summary() 70 | else: 71 | self.encoder.summary() 72 | 73 | 74 | class LayersGenerator(object): 75 | def __init__(self, x, encoder_loss, support_joint_training=False): 76 | self.encoder_loss = encoder_loss 77 | self.optimizer = None 78 | self.encoder, self.all_layers_encoder = self._create_encoder(x) 79 | self.discriminator, _ = self._create_encoder(x) 80 | self.enc_shape = np.array(K.int_shape(self.encoder.output)).tolist() 81 | if encoder_loss == 'vae': 82 | self.enc_shape[1] //= 2 83 | self.decoder, self.all_layers_decoder = self._create_decoder(x) 84 | self.autoencoder = self._create_autoencoder() 85 | self.support_joint_training = support_joint_training 86 | 87 | def get_encoder(self): 88 | return self.encoder 89 | 90 | def get_all_layers_encoder(self): 91 | return self.all_layers_encoder 92 | 93 | def get_all_layers_decoder(self): 94 | return self.all_layers_decoder 95 | 96 | def get_decoder(self): 97 | return self.decoder 98 | 99 | def get_discriminator(self): 100 | return self.discriminator 101 | 102 | def get_auto_encoder(self): 103 | return self.autoencoder 104 | 105 | def get_optimizer(self): 106 | return self.optimizer 107 | 108 | def support_joint_training(self): 109 | return self.support_joint_training 110 | 111 | def _create_encoder(self, x) -> (Model, Model): 112 | pass 113 | 114 | def _create_decoder(self, x) -> (Model, Model): 115 | pass 116 | 117 | def _create_autoencoder(self): 118 | return AutoencoderModel(self.encoder, self.decoder) 119 | 120 | 121 | class AutoencoderModel(object): 122 | def __init__(self, encoder, decoder): 123 | self.encoder = encoder 124 | self.decoder = decoder 125 | self.use_vae = False 126 | 127 | def __call__(self, inputs, *args, **kwargs): 128 | return self.decoder(self.encoder(inputs)) 129 | 130 | def decoder_predict(self, inputs=None, encoding=None): 131 | assert(inputs is not None or encoding is not None) 132 | if encoding is not None: 133 | return self.decoder.predict(encoding) 134 | encoded = self.encoder.predict(inputs) 135 | if self.use_vae: 136 | mean, logvar = tf.split(encoded, num_or_size_splits=2, axis=1) 137 | epsilon = K.random_normal(shape=mean.shape) 138 | encoded = mean + K.exp(logvar / 2) * epsilon 139 | return self.decoder.predict(encoded) 140 | 141 | def get_trainable_variables(self): 142 | trainable_variables = self.encoder.trainable_variables 143 | if self.decoder is not None: 144 | trainable_variables += self.decoder.trainable_variables 145 | return trainable_variables 146 | 147 | def save_weights(self, paths): 148 | self.encoder.save_weights(paths+'_encoder.h5') 149 | if self.decoder is not None: 150 | self.decoder.save_weights(paths+'_decoder.h5') 151 | 152 | def load_weights(self, paths): 153 | self.encoder.load_weights(paths+'_encoder.h5') 154 | if self.decoder is not None: 155 | self.decoder.load_weights(paths+'_decoder.h5') 156 | 157 | def exists(self, paths): 158 | decoder_exists = True 159 | if self.decoder is not None: 160 | decoder_exists = os.path.exists(paths+'_decoder.h5') 161 | encoder_exists = os.path.exists(paths+'_encoder.h5') 162 | return decoder_exists and encoder_exists 163 | 164 | def summary(self): 165 | self.encoder.summary() 166 | if self.decoder is not None: 167 | self.decoder.summary() 168 | 169 | def set_use_vae(self, use_vae): 170 | self.use_vae = use_vae 171 | 172 | 173 | class RnnAutoencoderModel(AutoencoderModel): 174 | def __init__(self, encoder, decoder): 175 | super(RnnAutoencoderModel, self).__init__(encoder, decoder) 176 | 177 | def __call__(self, inputs, *args, **kwargs): 178 | return self.decoder([inputs, np.zeros((inputs.shape[0], 1, inputs.shape[2]))], *args, **kwargs) 179 | 180 | def decoder_predict(self, inputs=None, encoding=None): 181 | return self.decoder.predict([inputs, np.zeros((inputs.shape[0], 1, inputs.shape[2]))]) 182 | 183 | def get_trainable_variables(self): 184 | return self.decoder.trainable_variables 185 | 186 | def save_weights(self, paths): 187 | self.decoder.save_weights(paths+'_autoencoder.tf') 188 | 189 | def load_weights(self, paths): 190 | self.decoder.load_weights(paths+'_autoencoder.tf') 191 | 192 | def exists(self, paths): 193 | return os.path.exists(paths+'_autoencoder.tf') 194 | 195 | def summary(self): 196 | self.decoder.summary() 197 | -------------------------------------------------------------------------------- /networks/fcnn_ae.py: -------------------------------------------------------------------------------- 1 | """ 2 | Define the fully convolutional autoencoder 3 | 4 | Author: 5 | Baptiste Lafabregue 2019.25.04 6 | """ 7 | 8 | import numpy as np 9 | from tensorflow.keras import Model 10 | from tensorflow.keras.layers import Dense, Input, Reshape, Activation, Dropout 11 | from tensorflow.keras.layers import Conv1D, BatchNormalization, GlobalAveragePooling1D 12 | 13 | from networks.encoders import LayersGenerator 14 | 15 | 16 | class AutoEncoder(LayersGenerator): 17 | """docstring for AutoEncoder""" 18 | 19 | def __init__(self, 20 | x, 21 | encoder_loss, 22 | filters=[64, 64, 64], 23 | kernels=[3, 5, 9], 24 | latent_dim=10, 25 | activation='relu', 26 | dropout_rate=0 27 | ): 28 | self.filters = np.array(filters) 29 | self.kernels = np.array(kernels) 30 | self.latent_dim = latent_dim 31 | self.activation = activation 32 | self.dropout_rate = dropout_rate 33 | 34 | super().__init__(x, encoder_loss, support_joint_training=True) 35 | 36 | def _create_encoder(self, x): 37 | i = Input(shape=x.shape[1:]) 38 | layers_outputs = [] 39 | h = i 40 | if self.dropout_rate > 0: 41 | h = Dropout(self.dropout_rate)(h) 42 | for filter, kernel in zip(self.filters, self.kernels): 43 | h = Conv1D(filters=filter, kernel_size=int(kernel), padding='same', strides=1)(h) 44 | if self.dropout_rate > 0: 45 | h = Dropout(self.dropout_rate)(h) 46 | h = BatchNormalization()(h) 47 | h = Activation(activation=self.activation)(h) 48 | layers_outputs.append(h) 49 | h = GlobalAveragePooling1D()(h) 50 | h = Dense(self.latent_dim)(h) 51 | layers_outputs.append(h) 52 | 53 | return Model(inputs=i, outputs=h), Model(inputs=i, outputs=layers_outputs) 54 | 55 | def _create_decoder(self, x): 56 | out_channels = 1 57 | layers_outputs = [] 58 | 59 | for s in x.shape[1:]: 60 | out_channels *= s 61 | i = Input(batch_shape=self.enc_shape) 62 | h = Dense(out_channels)(i) 63 | h = Reshape((x.shape[1:]))(h) 64 | layers_outputs.append(h) 65 | for filter, kernel in zip(np.flip(self.filters), np.flip(self.kernels)): 66 | h = Conv1D(filters=filter, kernel_size=int(kernel), padding='same', strides=1)(h) 67 | h = Activation(activation=self.activation)(h) 68 | layers_outputs.append(h) 69 | h = Conv1D(filters=x.shape[-1], kernel_size=int(self.kernels[-1]), padding='same', strides=1)(h) 70 | layers_outputs.append(h) 71 | 72 | return Model(inputs=i, outputs=h), Model(inputs=i, outputs=layers_outputs) 73 | -------------------------------------------------------------------------------- /networks/mlp_ae.py: -------------------------------------------------------------------------------- 1 | """ 2 | Define the multi layer perceptron autoencoder 3 | 4 | Author: 5 | Baptiste Lafabregue 2019.25.04 6 | """ 7 | 8 | import numpy as np 9 | from tensorflow.keras import Model 10 | from tensorflow.keras.layers import Dense, Input, Reshape, Flatten, Dropout 11 | 12 | from networks.encoders import LayersGenerator 13 | 14 | 15 | class AutoEncoder(LayersGenerator): 16 | """docstring for AutoEncoder""" 17 | 18 | def __init__(self, 19 | x, 20 | encoder_loss, 21 | layers=[500, 500, 2000], 22 | latent_dim=10, 23 | dropout_rate=0 24 | ): 25 | self.layers = np.array(layers) 26 | self.latent_dim = latent_dim 27 | self.dropout_rate = dropout_rate 28 | 29 | super().__init__(x, encoder_loss, support_joint_training=True) 30 | 31 | def _create_encoder(self, x): 32 | i = Input(shape=x.shape[1:]) 33 | layers_outputs = [] 34 | h = Flatten()(i) 35 | if self.dropout_rate > 0: 36 | h = Dropout(self.dropout_rate)(h) 37 | for layer in self.layers: 38 | h = Dense(layer)(h) 39 | if self.dropout_rate > 0: 40 | h = Dropout(self.dropout_rate)(h) 41 | layers_outputs.append(h) 42 | h = Dense(self.latent_dim)(h) 43 | layers_outputs.append(h) 44 | 45 | return Model(inputs=i, outputs=h), Model(inputs=i, outputs=layers_outputs) 46 | 47 | def _create_decoder(self, x): 48 | layers_outputs = [] 49 | i = Input(batch_shape=self.enc_shape) 50 | h = i 51 | for layer in np.flip(self.layers): 52 | h = Dense(layer)(h) 53 | layers_outputs.append(h) 54 | out_channels = 1 55 | for s in x.shape[1:]: 56 | out_channels *= s 57 | h = Dense(out_channels)(h) 58 | h = Reshape((x.shape[1:]))(h) 59 | layers_outputs.append(h) 60 | 61 | return Model(inputs=i, outputs=h), Model(inputs=i, outputs=layers_outputs) 62 | -------------------------------------------------------------------------------- /networks/resnet.py: -------------------------------------------------------------------------------- 1 | """ 2 | Define the ReseNet autoencoder. 3 | Based on keras Hassan Fawaz implementation https://github.com/hfawaz/dl-4-tsc 4 | and on article: 5 | Wang, Z., Yan, W., & Oates, T. (2016). 6 | Time series classification from scratch with deep neural networks: a strong baseline 7 | 8 | Author: 9 | Baptiste Lafabregue 2019.25.04 10 | """ 11 | import numpy as np 12 | from tensorflow.keras import Model 13 | from tensorflow.keras.layers import Dense, Input, Reshape, Activation, Dropout, add 14 | from tensorflow.keras.layers import Conv1D, BatchNormalization, GlobalAveragePooling1D 15 | 16 | from networks.encoders import LayersGenerator 17 | 18 | 19 | class AutoEncoder(LayersGenerator): 20 | """docstring for AutoEncoder""" 21 | 22 | def __init__(self, 23 | x, 24 | encoder_loss, 25 | filters=[64, 128, 128], 26 | kernels=[8, 5, 3], 27 | latent_dim=10, 28 | activation='relu', 29 | dropout_rate=0 30 | ): 31 | self.filters = np.array(filters) 32 | self.kernels = np.array(kernels) 33 | self.latent_dim = latent_dim 34 | self.activation = activation 35 | self.dropout_rate = dropout_rate 36 | 37 | super().__init__(x, encoder_loss, support_joint_training=True) 38 | 39 | def _create_res_block(self, i, n_features): 40 | conv_x = Conv1D(filters=n_features, kernel_size=int(self.kernels[0]), padding='same')(i) 41 | conv_x = BatchNormalization()(conv_x) 42 | conv_x = Activation('relu')(conv_x) 43 | 44 | conv_y = Conv1D(filters=n_features, kernel_size=int(self.kernels[1]), padding='same')(conv_x) 45 | conv_y = BatchNormalization()(conv_y) 46 | conv_y = Activation('relu')(conv_y) 47 | 48 | conv_z = Conv1D(filters=n_features, kernel_size=int(self.kernels[2]), padding='same')(conv_y) 49 | conv_z = BatchNormalization()(conv_z) 50 | 51 | # expand channels for the sum 52 | shortcut_y = Conv1D(filters=n_features, kernel_size=1, padding='same')(i) 53 | shortcut_y = BatchNormalization()(shortcut_y) 54 | 55 | output_block_1 = add([shortcut_y, conv_z]) 56 | output_block_1 = Activation('relu')(output_block_1) 57 | 58 | return output_block_1 59 | 60 | def _create_encoder(self, x): 61 | i = Input(shape=x.shape[1:]) 62 | layers_outputs = [] 63 | h = i 64 | 65 | for n_features in self.filters: 66 | h = self._create_res_block(h, n_features) 67 | if self.dropout_rate > 0: 68 | h = Dropout(self.dropout_rate)(h) 69 | layers_outputs.append(h) 70 | 71 | h = GlobalAveragePooling1D()(h) 72 | h = Dense(self.latent_dim)(h) 73 | layers_outputs.append(h) 74 | 75 | return Model(inputs=i, outputs=h), Model(inputs=i, outputs=layers_outputs) 76 | 77 | def _create_decoder(self, x): 78 | out_channels = 1 79 | layers_outputs = [] 80 | 81 | for s in x.shape[1:]: 82 | out_channels *= s 83 | i = Input(batch_shape=self.enc_shape) 84 | h = Dense(out_channels)(i) 85 | h = Reshape((x.shape[1:]))(h) 86 | layers_outputs.append(h) 87 | for n_features in np.flip(self.filters): 88 | h = self._create_res_block(h, n_features) 89 | layers_outputs.append(h) 90 | h = Conv1D(filters=x.shape[-1], kernel_size=int(self.kernels[-1]), padding='same', strides=1)(h) 91 | layers_outputs.append(h) 92 | 93 | return Model(inputs=i, outputs=h), Model(inputs=i, outputs=layers_outputs) 94 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib>=3.0.0 2 | numpy>=1.19.0 3 | scikit_learn>=0.24.1 4 | scipy>=1.4.0 5 | sktime>=0.5.2 6 | tensorflow>=2.3.0 7 | tensorflow-addons>=0.12.0 8 | tslearn>=0.5.0.5 9 | umap_learn>= 0.5.1 -------------------------------------------------------------------------------- /supervised_training.py: -------------------------------------------------------------------------------- 1 | """ 2 | Base on Keras Improved Deep Embedded Clustering (IDEC) algorithm implementation: 3 | 4 | Xifeng Guo, Long Gao, Xinwang Liu, Jianping Yin. Improved Deep Embedded Clustering with Local Structure Preservation. IJCAI 2017. 5 | 6 | Original Author: 7 | Xifeng Guo. 2017.1.30 8 | Autor: 9 | Baptiste Lafabregue 2019.25.04 10 | """ 11 | 12 | import csv 13 | import os 14 | 15 | import numpy as np 16 | import tensorflow as tf 17 | import tensorflow.keras.backend as K 18 | from sklearn import metrics 19 | from sklearn.cluster import KMeans 20 | from tensorflow.keras.layers import Layer, InputSpec 21 | from tensorflow.keras.models import Model 22 | 23 | import utils 24 | 25 | 26 | class SupervisedResearch(object): 27 | def __init__(self, 28 | dataset_name, 29 | classifier_name, 30 | encoder_model, 31 | gamma=0.1, 32 | n_clusters=10, 33 | batch_size=10, 34 | optimizer=None): 35 | 36 | super(SupervisedResearch, self).__init__() 37 | 38 | self.dataset_name = dataset_name 39 | self.classifier_name = classifier_name 40 | self.encoder_model = encoder_model 41 | self.encoder = encoder_model.encoder 42 | self.n_clusters = n_clusters 43 | self.batch_size = batch_size 44 | self.optimizer = optimizer 45 | 46 | if self.optimizer is None: 47 | self.optimizer = tf.keras.optimizers.legacy.Adam() 48 | 49 | def load_weights(self, weights_path): 50 | """ 51 | Load weights of IDEC model 52 | :param weights_path: path to load weights from 53 | """ 54 | self.dec_model.load_weights(weights_path) 55 | 56 | def save_weights(self, weights_path): 57 | """ 58 | Save weights of IDEC model 59 | :param weights_path: path to save weights to 60 | """ 61 | self.dec_model.save_weights(weights_path) 62 | 63 | def extract_feature(self, x): 64 | """ 65 | Extract features from the encoder (before the clustering layer) 66 | :param x: the data to extract features from 67 | :return: the encoded features 68 | """ 69 | return self.encoder.predict(x) 70 | 71 | def log_stats_encoder(self, x, y, x_test, y_test, loss, epoch, log_writer, comment): 72 | """ 73 | Log the intermediate result to a file 74 | :param x: train data 75 | :param y: train labels 76 | :param x_test: test data 77 | :param y_test: test labels 78 | :param loss: array of losses values 79 | :param epoch: current epoch 80 | :param logwriter: log file writer 81 | :param comment: comment to add to the log 82 | :return: 83 | """ 84 | acc = 0 85 | nmi = 0 86 | ari = 0 87 | acc_test = 0 88 | nmi_test = 0 89 | ari_test = 0 90 | 91 | loss = np.round(loss, 5) 92 | 93 | if y is not None: 94 | kmeans = KMeans(n_clusters=self.n_clusters, n_init=20) 95 | x_pred = self.encoder.predict(x) 96 | y_pred = kmeans.fit_predict(x_pred) 97 | 98 | acc = np.round(utils.cluster_acc(y, y_pred), 5) 99 | nmi = np.round(metrics.normalized_mutual_info_score(y, y_pred), 5) 100 | ari = np.round(metrics.adjusted_rand_score(y, y_pred), 5) 101 | # print('ari : '+str(ari)) 102 | # sq_error = utils.computes_dtw_regularized_square_error(x, y_pred) 103 | 104 | if x_test is not None and y_test is not None: 105 | kmeans = KMeans(n_clusters=self.n_clusters, n_init=20) 106 | x_pred = self.encoder.predict(x_test) 107 | y_pred_test = kmeans.fit_predict(x_pred) 108 | 109 | acc_test = np.round(utils.cluster_acc(y_test, y_pred_test), 5) 110 | nmi_test = np.round(metrics.normalized_mutual_info_score(y_test, y_pred_test), 5) 111 | ari_test = np.round(metrics.adjusted_rand_score(y_test, y_pred_test), 5) 112 | 113 | log_dict = dict(iter=epoch, acc=acc, nmi=nmi, ari=ari, L=loss[0], Lc=loss[1], Lr=loss[2], 114 | acc_test=acc_test, nmi_test=nmi_test, ari_test=ari_test, comment='encoder : '+comment) 115 | log_writer.writerow(log_dict) 116 | 117 | return nmi, y_pred 118 | 119 | def clustering(self, x, 120 | y=None, 121 | update_interval=1, 122 | nb_steps=50, 123 | save_dir='./results/idec', 124 | save_suffix='', 125 | x_test=None, 126 | y_test=None, 127 | verbose=True, 128 | ): 129 | 130 | if verbose: 131 | print('Update interval', update_interval) 132 | 133 | # logging file 134 | if not os.path.exists(save_dir): 135 | os.makedirs(save_dir) 136 | logfile = open(save_dir + '/idec_log.csv', 'w') 137 | log_writer = csv.DictWriter(logfile, fieldnames=['iter', 'acc', 'nmi', 'ari', 'L', 'Lc', 'Lr', 138 | 'acc_test', 'nmi_test', 'ari_test', 'comment']) 139 | log_writer.writeheader() 140 | max_ari = -np.inf 141 | min_sq_error = np.inf 142 | max_sc_error = -np.inf 143 | ari_sq_error = 0.0 144 | max_model = None 145 | loss = [0, 0, 0] 146 | 147 | if verbose: 148 | self.encoder_model.summary() 149 | 150 | i = 0 # Number of performed optimization steps 151 | epochs = 0 # Number of performed epochs 152 | 153 | train_loss = tf.keras.metrics.Mean(name='train_loss') 154 | 155 | dist_matrix = utils.cdist_dtw(x, n_jobs=3) 156 | 157 | # define the train function 158 | @tf.function 159 | def train_step(x_batch): 160 | with tf.GradientTape() as tape: 161 | loss = self.encoder_model.loss.compute_loss(x_batch) 162 | gradients = tape.gradient(loss, self.encoder_model.get_trainable_variables()) 163 | self.optimizer.apply_gradients(zip(gradients, self.encoder_model.get_trainable_variables())) 164 | 165 | train_loss(loss) 166 | if verbose: 167 | print('start pre-train') 168 | # Encoder training 169 | while i < nb_steps: 170 | train_loss.reset_states() 171 | train_ds = tf.data.Dataset.from_tensor_slices(x) 172 | train_ds = train_ds.shuffle(x.shape[0], reshuffle_each_iteration=True) 173 | train_ds = train_ds.batch(self.batch_size).as_numpy_iterator() 174 | for batch in train_ds: 175 | train_step(batch) 176 | 177 | i += 1 178 | if i >= nb_steps: 179 | break 180 | 181 | if verbose: 182 | template = 'Epoch {}, Loss: {}' 183 | print(template.format(epochs + 1, train_loss.result())) 184 | epochs += 1 185 | ari, y_pred = self.log_stats_encoder(x, y, x_test, y_test, [train_loss.result(), 0, train_loss.result()], 186 | epochs, log_writer, 'pretrain') 187 | sq_error = utils.computes_dtw_silhouette_score(dist_matrix, y_pred) 188 | if ari > max_ari: 189 | max_ari = ari 190 | self.encoder.save_weights(save_dir + '/DCC_model_max_sat_.h5') 191 | if sq_error > max_sc_error: 192 | ari_sq_error = ari 193 | max_sc_error = sq_error 194 | print("ari : "+str(ari)+", max ari"+str(max_ari)+" ## sq error : "+str(sq_error)+", min sq error"+str(max_sc_error)+" with ari of : "+str(ari_sq_error)) 195 | 196 | if verbose: 197 | print('end of pre-train') 198 | 199 | self.encoder.load_weights(save_dir + '/DCC_model_max_sat_.h5') 200 | # save idec model 201 | if verbose: 202 | print("max ari: "+str(max_ari)) 203 | print('model saved at: ', save_dir+'/DCC_model_max_sat_'+save_suffix+'.h5') 204 | self.log_stats_encoder(x, y, x_test, y_test, loss, epochs, log_writer, 'final') 205 | 206 | logfile.close() 207 | 208 | 209 | 210 | -------------------------------------------------------------------------------- /ucr_baseline.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script to launch experiments with non deep methods 3 | See parse_arguments() function for details on arguments 4 | 5 | Author: 6 | Baptiste Lafabregue 2019.25.04 7 | """ 8 | import os 9 | import json 10 | import numpy as np 11 | import argparse 12 | from tslearn.clustering import TimeSeriesKMeans 13 | from kshape.core import _kshape 14 | from sklearn import metrics 15 | 16 | import tensorflow as tf 17 | 18 | import utils 19 | 20 | 21 | def parse_arguments(): 22 | parser = argparse.ArgumentParser( 23 | description='Classification tests for UCR repository datasets' 24 | ) 25 | parser.add_argument('--dataset', type=str, metavar='d', required=True, 26 | help='dataset name') 27 | parser.add_argument('--archives', type=str, metavar='DIR', required=True, 28 | help='archive name') 29 | parser.add_argument('--itr', type=str, metavar='X', default='0', 30 | help='iteration index') 31 | parser.add_argument('--seeds_itr', type=int, metavar='X', default=None, 32 | help='seeds index, do not specify or -1 if none') 33 | parser.add_argument('--root_dir', type=str, metavar='PATH', default='.', 34 | help='path of the root dir where archives and results are stored') 35 | parser.add_argument('--not_use_previous', default=False, action="store_true", 36 | help='Flag to not use previous results and only computes one with errors, ' 37 | 'but computes everything again, only for All option') 38 | 39 | return parser.parse_args() 40 | 41 | 42 | if __name__ == '__main__': 43 | args = parse_arguments() 44 | 45 | tf.keras.backend.set_floatx('float64') 46 | 47 | root_dir = args.root_dir 48 | dataset_name = args.dataset 49 | archive_name = args.archives 50 | use_previous = not args.not_use_previous 51 | itr = args.itr 52 | seeds_itr = args.seeds_itr 53 | seeds = None 54 | if seeds_itr is not None and seeds_itr >= 0: 55 | seeds = utils.read_seeds(root_dir, archive_name, dataset_name, seeds_itr) 56 | 57 | train_dict = utils.read_dataset(root_dir, archive_name, dataset_name, True) 58 | x_train = train_dict[dataset_name][0] 59 | y_train = train_dict[dataset_name][1] 60 | input_shape = x_train.shape[1:] 61 | nb_classes = np.shape(np.unique(y_train, return_counts=True)[1])[0] 62 | 63 | train_dict = utils.read_dataset(root_dir, archive_name, dataset_name, False) 64 | x_test = train_dict[dataset_name][0] 65 | y_test = train_dict[dataset_name][1] 66 | 67 | stats_dir = root_dir + '/stats/' + str(itr) + '/' + str(seeds_itr) + '/' 68 | 69 | print('Launch euclidien kmeans : ' + dataset_name) 70 | km_euc = TimeSeriesKMeans(n_clusters=nb_classes, metric="euclidean", max_iter=200).fit(x_train) 71 | y_pred = km_euc.predict(x_train) 72 | km_euc = TimeSeriesKMeans(n_clusters=nb_classes, metric="euclidean", max_iter=200).fit(x_test) 73 | y_pred_test = km_euc.predict(x_test) 74 | 75 | acc = np.round(utils.cluster_acc(y_train, y_pred), 5) 76 | nmi = np.round(metrics.normalized_mutual_info_score(y_train, y_pred), 5) 77 | ari = np.round(metrics.adjusted_rand_score(y_train, y_pred), 5) 78 | summarized = str(acc)+","+str(0)+","+str(nmi)+","+str(0)+","+str(ari)+","+str(0) 79 | print('acc : ' + str(acc)) 80 | print('nmi : ' + str(nmi)) 81 | print('ari : ' + str(ari)) 82 | 83 | acc_test = np.round(utils.cluster_acc(y_test, y_pred_test), 5) 84 | nmi_test = np.round(metrics.normalized_mutual_info_score(y_test, y_pred_test), 5) 85 | ari_test = np.round(metrics.adjusted_rand_score(y_test, y_pred_test), 5) 86 | summarized_test = str(acc_test)+","+str(0)+","+str(nmi_test)+","+str(0)+","+str(ari_test)+","+str(0) 87 | 88 | utils.create_directory(stats_dir) 89 | with open(stats_dir + "kmeans_Eucl_None_0.0_" + dataset_name, "w") as f: 90 | f.write(summarized + '\n') 91 | f.write(summarized_test + '\n') 92 | f.write('0\n') 93 | 94 | print('Launch DTW/DBA kmeans : ' + dataset_name) 95 | km_dtw = TimeSeriesKMeans(n_clusters=nb_classes, metric="dtw", max_iter=200).fit(x_train) 96 | y_pred = km_dtw.predict(x_train) 97 | km_dtw = TimeSeriesKMeans(n_clusters=nb_classes, metric="dtw", max_iter=200).fit(x_test) 98 | y_pred_test = km_dtw.predict(x_test) 99 | 100 | acc = np.round(utils.cluster_acc(y_train, y_pred), 5) 101 | nmi = np.round(metrics.normalized_mutual_info_score(y_train, y_pred), 5) 102 | ari = np.round(metrics.adjusted_rand_score(y_train, y_pred), 5) 103 | summarized = str(acc)+","+str(0)+","+str(nmi)+","+str(0)+","+str(ari)+","+str(0) 104 | print('acc : ' + str(acc)) 105 | print('nmi : ' + str(nmi)) 106 | print('ari : ' + str(ari)) 107 | 108 | acc_test = np.round(utils.cluster_acc(y_test, y_pred_test), 5) 109 | nmi_test = np.round(metrics.normalized_mutual_info_score(y_test, y_pred_test), 5) 110 | ari_test = np.round(metrics.adjusted_rand_score(y_test, y_pred_test), 5) 111 | summarized_test = str(acc_test)+","+str(0)+","+str(nmi_test)+","+str(0)+","+str(ari_test)+","+str(0) 112 | 113 | with open(stats_dir + "kmeans_DTW_None_0.0_" + dataset_name, "w") as f: 114 | f.write(summarized + '\n') 115 | f.write(summarized_test + '\n') 116 | f.write('0\n') 117 | 118 | print('Launch Kshape : ' + dataset_name) 119 | x_train_kshape = np.reshape(x_train, (x_train.shape[0], -1)) 120 | x_test_kshape = np.reshape(x_test, (x_test.shape[0], -1)) 121 | y_pred, _ = _kshape(x_train_kshape, nb_classes) 122 | y_pred_test, _ = _kshape(x_test_kshape, nb_classes) 123 | 124 | acc = np.round(utils.cluster_acc(y_train, y_pred), 5) 125 | nmi = np.round(metrics.normalized_mutual_info_score(y_train, y_pred), 5) 126 | ari = np.round(metrics.adjusted_rand_score(y_train, y_pred), 5) 127 | summarized = str(acc) + "," + str(0) + "," + str(nmi) + "," + str(0) + "," + str(ari) + "," + str(0) 128 | print('acc : ' + str(acc)) 129 | print('nmi : ' + str(nmi)) 130 | print('ari : ' + str(ari)) 131 | 132 | acc_test = np.round(utils.cluster_acc(y_test, y_pred_test), 5) 133 | nmi_test = np.round(metrics.normalized_mutual_info_score(y_test, y_pred_test), 5) 134 | ari_test = np.round(metrics.adjusted_rand_score(y_test, y_pred_test), 5) 135 | summarized_test = str(acc_test) + "," + str(0) + "," + str(nmi_test) + "," + str(0) + "," + str( 136 | ari_test) + "," + str(0) 137 | 138 | utils.create_directory(stats_dir) 139 | with open(stats_dir + "kshape_None_None_0.0_" + dataset_name, "w") as f: 140 | f.write(summarized + '\n') 141 | f.write(summarized_test + '\n') 142 | f.write('0\n') 143 | 144 | -------------------------------------------------------------------------------- /ucr_baseline_umap.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script to launch non deep methods with UMAP reduction dimension algorithm 3 | See parse_arguments() function for details on arguments 4 | 5 | Author: 6 | Baptiste Lafabregue 2019.25.04 7 | """ 8 | import os 9 | import json 10 | import numpy as np 11 | import argparse 12 | from sklearn import metrics 13 | from sklearn.cluster import KMeans 14 | 15 | import tensorflow as tf 16 | import umap 17 | 18 | import utils 19 | 20 | 21 | def parse_arguments(): 22 | parser = argparse.ArgumentParser( 23 | description='Classification tests for UCR repository datasets' 24 | ) 25 | parser.add_argument('--dataset', type=str, metavar='d', required=True, 26 | help='dataset name') 27 | parser.add_argument('--archives', type=str, metavar='DIR', required=True, 28 | help='archive name') 29 | parser.add_argument('--itr', type=str, metavar='X', default='0', 30 | help='iteration index') 31 | parser.add_argument('--seeds_itr', type=int, metavar='X', default=None, 32 | help='seeds index, do not specify or -1 if none') 33 | parser.add_argument('--root_dir', type=str, metavar='PATH', default='.', 34 | help='path of the root dir where archives and results are stored') 35 | parser.add_argument('--not_use_previous', default=False, action="store_true", 36 | help='Flag to not use previous results and only computes one with errors, ' 37 | 'but computes everything again, only for All option') 38 | 39 | return parser.parse_args() 40 | 41 | 42 | if __name__ == '__main__': 43 | args = parse_arguments() 44 | 45 | tf.keras.backend.set_floatx('float64') 46 | 47 | root_dir = args.root_dir 48 | dataset_name = args.dataset 49 | archive_name = args.archives 50 | use_previous = not args.not_use_previous 51 | itr = args.itr 52 | seeds_itr = args.seeds_itr 53 | seeds = None 54 | if seeds_itr is not None and seeds_itr >= 0: 55 | seeds = utils.read_seeds(root_dir, archive_name, dataset_name, seeds_itr) 56 | 57 | train_dict = utils.read_dataset(root_dir, archive_name, dataset_name, True) 58 | x_train = train_dict[dataset_name][0] 59 | y_train = train_dict[dataset_name][1] 60 | input_shape = x_train.shape[1:] 61 | nb_classes = np.shape(np.unique(y_train, return_counts=True)[1])[0] 62 | 63 | train_dict = utils.read_dataset(root_dir, archive_name, dataset_name, False) 64 | x_test = train_dict[dataset_name][0] 65 | y_test = train_dict[dataset_name][1] 66 | 67 | x_train = np.reshape(x_train, (x_train.shape[0], -1)) 68 | x_test = np.reshape(x_test, (x_test.shape[0], -1)) 69 | 70 | UMAP_MIN_DIST = 0 71 | UMAP_NEIGHBORS = 10 72 | 73 | md = float(UMAP_MIN_DIST) 74 | x_train = umap.UMAP( 75 | random_state=0, 76 | metric='euclidean', 77 | n_components=nb_classes, 78 | n_neighbors=UMAP_NEIGHBORS, 79 | min_dist=md).fit_transform(x_train) 80 | x_test = umap.UMAP( 81 | random_state=0, 82 | metric='euclidean', 83 | n_components=nb_classes, 84 | n_neighbors=UMAP_NEIGHBORS, 85 | min_dist=md).fit_transform(x_test) 86 | 87 | stats_dir = root_dir + '/stats/' + str(itr) + '/' + str(seeds_itr) + '/' 88 | 89 | print('Launch euclidien kmeans on UMAP features : ' + dataset_name) 90 | y_pred = KMeans(n_clusters=nb_classes, random_state=0).fit_predict(x_train) 91 | y_pred_test = KMeans(n_clusters=nb_classes, random_state=0).fit_predict(x_test) 92 | 93 | acc = np.round(utils.cluster_acc(y_train, y_pred), 5) 94 | nmi = np.round(metrics.normalized_mutual_info_score(y_train, y_pred), 5) 95 | ari = np.round(metrics.adjusted_rand_score(y_train, y_pred), 5) 96 | summarized = str(acc)+","+str(0)+","+str(nmi)+","+str(0)+","+str(ari)+","+str(0) 97 | print('acc : ' + str(acc)) 98 | print('nmi : ' + str(nmi)) 99 | print('ari : ' + str(ari)) 100 | 101 | acc_test = np.round(utils.cluster_acc(y_test, y_pred_test), 5) 102 | nmi_test = np.round(metrics.normalized_mutual_info_score(y_test, y_pred_test), 5) 103 | ari_test = np.round(metrics.adjusted_rand_score(y_test, y_pred_test), 5) 104 | summarized_test = str(acc_test)+","+str(0)+","+str(nmi_test)+","+str(0)+","+str(ari_test)+","+str(0) 105 | 106 | utils.create_directory(stats_dir) 107 | with open(stats_dir + "kmeans_UMAP_None_0.0_" + dataset_name, "w") as f: 108 | f.write(summarized + '\n') 109 | f.write(summarized_test + '\n') 110 | f.write('0\n') 111 | --------------------------------------------------------------------------------