├── .flake8 ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── kutilities ├── __init__.py ├── callbacks.py ├── helpers │ ├── __init__.py │ ├── data_preparation.py │ ├── generic.py │ └── ui.py └── layers.py ├── local_install.sh ├── pypi_push.sh ├── requirements.txt ├── setup.cfg └── setup.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | 4 | exclude = 5 | .tox, 6 | __pycache__, 7 | build, 8 | dist 9 | 10 | ignore = 11 | # F401 imported but unused 12 | F401, 13 | # E501 line too long 14 | E501, 15 | # E303 too many blank lines 16 | E303, 17 | # E731 do not assign a lambda expression, use a def 18 | E731, 19 | # F812: list comprehension redefines ... 20 | F812, 21 | # E402 module level import not at top of file 22 | E402, 23 | # W292 no newline at end of file 24 | W292, 25 | # E999 SyntaxError: invalid syntax 26 | E999, 27 | # F821 undefined name 28 | F821, 29 | 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | dist/ 3 | build/ 4 | *.egg-info/ 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Christos Baziotis 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # MANIFEST.in 2 | exclude .gitignore 3 | exclude .coverage 4 | exclude .travis.yml 5 | include README.md 6 | include setup.cfg 7 | prune .cache 8 | prune .git 9 | prune build 10 | prune dist 11 | recursive-exclude *.egg-info * 12 | recursive-include tests * 13 | 14 | # data files 15 | #include stats/**/*.txt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # keras-utilities 2 | keras utilities 3 | 4 | --- 5 | documentation and examples coming soon... -------------------------------------------------------------------------------- /kutilities/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/keras-utilities/022856d86fd585a1c79af0772fbe8b57cab8f242/kutilities/__init__.py -------------------------------------------------------------------------------- /kutilities/callbacks.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | from collections import defaultdict 4 | import matplotlib 5 | 6 | matplotlib.use('TkAgg') 7 | 8 | import matplotlib.pyplot as plt 9 | import numpy 10 | import seaborn as sns 11 | from keras.callbacks import Callback 12 | import pandas as pd 13 | 14 | from kutilities.helpers.data_preparation import onehot_to_categories 15 | from kutilities.helpers.generic import get_model_desc 16 | from kutilities.helpers.ui import move_figure 17 | 18 | plt.rc('font', **{'family': 'serif', 'serif': ['Computer Modern Roman'], 19 | 'monospace': ['Computer Modern Typewriter']}) 20 | # plt.rc('text', usetex=True) 21 | plt.rc("figure", facecolor="white") 22 | 23 | 24 | class LossEarlyStopping(Callback): 25 | def __init__(self, metric, value, mode="less"): 26 | super().__init__() 27 | self.metric = metric 28 | self.value = value 29 | self.mode = mode 30 | 31 | def on_epoch_end(self, epoch, logs={}): 32 | if self.mode == "less": 33 | if logs[self.metric] < self.value: 34 | self.model.stop_training = True 35 | print('Early stopping - {} is {} than {}'.format(self.metric, 36 | self.mode, 37 | self.value)) 38 | if self.mode == "more": 39 | if logs[self.metric] > self.value: 40 | self.model.stop_training = True 41 | print('Early stopping - {} is {} than {}'.format(self.metric, 42 | self.mode, 43 | self.value)) 44 | 45 | 46 | class MetricsCallback(Callback): 47 | """ 48 | 49 | """ 50 | 51 | def __init__(self, metrics, datasets, regression=False, batch_size=512): 52 | super().__init__() 53 | self.datasets = datasets 54 | self.metrics = metrics 55 | self.regression = regression 56 | self.batch_size = batch_size 57 | 58 | @staticmethod 59 | def predic_classes(predictions): 60 | if predictions.shape[-1] > 1: 61 | return predictions.argmax(axis=-1) 62 | else: 63 | return (predictions > 0.5).astype('int32') 64 | 65 | def add_predictions(self, dataset, name="set", logs={}): 66 | X = dataset[0] 67 | y = dataset[1] 68 | 69 | if self.regression: 70 | y_pred = self.model.predict(X, batch_size=self.batch_size, 71 | verbose=0) 72 | y_pred = numpy.reshape(y_pred, y.shape) 73 | y_test = y 74 | 75 | # test if the labels are categorical or singular 76 | else: 77 | if len(y.shape) > 1: 78 | try: 79 | y_pred = self.model.predict_classes(X, 80 | batch_size=self.batch_size, 81 | verbose=0) 82 | except: 83 | y_pred = self.predic_classes( 84 | self.model.predict(X, batch_size=self.batch_size, 85 | verbose=0)) 86 | 87 | y_test = onehot_to_categories(y) 88 | 89 | else: 90 | y_pred = self.model.predict(X, batch_size=self.batch_size, 91 | verbose=0) 92 | y_pred = numpy.array([int(_y > 0.5) for _y in y_pred]) 93 | y_test = y 94 | 95 | for k, metric in self.metrics.items(): 96 | score = numpy.squeeze(metric(y_test, y_pred)) 97 | entry = ".".join([name, k]) 98 | self.params['metrics'].append(entry) 99 | logs[entry] = score 100 | 101 | def on_epoch_end(self, epoch, logs={}): 102 | for name, data in self.datasets.items(): 103 | self.add_predictions(data if len(data) > 1 else data[0], name=name, 104 | logs=logs) 105 | 106 | data = {} 107 | for dataset in sorted(self.datasets.keys()): 108 | data[dataset] = {metric: logs[".".join([dataset, metric])] 109 | for metric in sorted(self.metrics.keys())} 110 | print(pd.DataFrame.from_dict(data, orient="index")) 111 | print() 112 | 113 | 114 | class PlottingCallback(Callback): 115 | def __init__(self, benchmarks=None, grid_ranges=None, width=4, height=4, 116 | plot_name=None): 117 | super().__init__() 118 | self.height = height 119 | self.width = width 120 | self.benchmarks = benchmarks 121 | self.grid_ranges = grid_ranges 122 | self.model_loss = [] 123 | self.validation_loss = [] 124 | self.custom_metrics = defaultdict(list) 125 | self.fig = None 126 | 127 | res_path = os.path.join(os.getcwd(), 'experiments') 128 | if not os.path.exists(res_path): 129 | os.makedirs(res_path) 130 | 131 | models = len(glob.glob(os.path.join(res_path, "model*.png"))) 132 | 133 | if plot_name is None: 134 | self.plot_fname = os.path.join(res_path, 135 | 'model_{}.png'.format(models + 1)) 136 | else: 137 | self.plot_fname = os.path.join(res_path, 138 | '{}.png'.format(plot_name)) 139 | 140 | def on_train_begin(self, logs={}): 141 | sns.set_style("whitegrid") 142 | sns.set_style("whitegrid", {"grid.linewidth": 0.5, 143 | "lines.linewidth": 0.5, 144 | "axes.linewidth": 0.5}) 145 | flatui = ["#9b59b6", "#3498db", "#95a5a6", "#e74c3c", "#34495e", 146 | "#2ecc71"] 147 | sns.set_palette(sns.color_palette(flatui)) 148 | # flatui = ["#9b59b6", "#3498db", "#95a5a6", "#e74c3c", "#34495e", "#2ecc71"] 149 | # sns.set_palette(sns.color_palette("Set2", 10)) 150 | 151 | plt.ion() # set plot to animated 152 | width = self.width * (1 + len(self.get_metrics(logs))) 153 | height = self.height 154 | self.fig = plt.figure(figsize=(width, height)) 155 | 156 | # move it to the upper left corner 157 | move_figure(self.fig, 25, 25) 158 | 159 | def save_plot(self): 160 | self.fig.savefig(self.plot_fname, dpi=100) 161 | 162 | @staticmethod 163 | def get_metrics(logs): 164 | custom_metrics_keys = defaultdict(list) 165 | for entry in logs.keys(): 166 | metric = entry.split(".") 167 | if len(metric) > 1: # custom metric 168 | custom_metrics_keys[metric[0]].append(metric[1]) 169 | return custom_metrics_keys 170 | 171 | def on_epoch_end(self, epoch, logs={}): 172 | self.fig.clf() 173 | linewidth = 1.2 174 | self.fig.set_size_inches( 175 | self.width * (1 + len(self.get_metrics(logs))), 176 | self.height, forward=True) 177 | custom_metrics_keys = self.get_metrics(logs) 178 | 179 | total_plots = len(custom_metrics_keys) + 1 180 | ################################################## 181 | # First - Plot Models loss 182 | self.model_loss.append(logs['loss']) 183 | self.validation_loss.append(logs['val_loss']) 184 | 185 | ax = self.fig.add_subplot(1, total_plots, 1) 186 | ax.plot(self.model_loss, linewidth=linewidth) 187 | ax.plot(self.validation_loss, linewidth=linewidth) 188 | ax.set_title('model loss', fontsize=10) 189 | ax.set_ylabel('loss') 190 | ax.set_xlabel('epoch') 191 | ax.legend(['train', 'val'], loc='upper left', fancybox=True) 192 | ax.grid(True) 193 | ax.grid(b=True, which='major', color='gray', linewidth=.5) 194 | ax.grid(b=True, which='minor', color='gray', linewidth=0.5) 195 | ax.tick_params(labelsize=10) 196 | # leg = ax.gca().get_legend() 197 | 198 | ################################################## 199 | # Second - Plot Custom Metrics 200 | for i, (dataset_name, metrics) in enumerate( 201 | sorted(custom_metrics_keys.items(), reverse=False)): 202 | axs = self.fig.add_subplot(1, total_plots, i + 2) 203 | axs.set_title(dataset_name, fontsize=10) 204 | axs.set_ylabel('score') 205 | axs.set_xlabel('epoch') 206 | if self.grid_ranges: 207 | axs.set_ylim(self.grid_ranges) 208 | 209 | # append the values to the corresponding array 210 | for m in sorted(metrics): 211 | entry = ".".join([dataset_name, m]) 212 | self.custom_metrics[entry].append(logs[entry]) 213 | axs.plot(self.custom_metrics[entry], label=m, 214 | linewidth=linewidth) 215 | 216 | axs.tick_params(labelsize=10) 217 | labels = list(sorted(metrics)) 218 | if self.benchmarks: 219 | for (label, benchmark), color in zip(self.benchmarks.items(), 220 | ["y", "r"]): 221 | axs.axhline(y=benchmark, linewidth=linewidth, color=color) 222 | labels = labels + [label] 223 | axs.legend(labels, loc='upper left', fancybox=True) 224 | axs.grid(True) 225 | axs.grid(b=True, which='major', color='gray', linewidth=.5) 226 | axs.grid(b=True, which='minor', color='gray', linewidth=0.5) 227 | 228 | plt.rcParams.update({'font.size': 10}) 229 | 230 | desc = get_model_desc(self.model) 231 | self.fig.text(.02, .02, desc, verticalalignment='bottom', wrap=True, 232 | fontsize=8) 233 | self.fig.tight_layout() 234 | self.fig.subplots_adjust(bottom=.18) 235 | self.fig.canvas.draw() 236 | self.fig.canvas.flush_events() 237 | 238 | self.save_plot() 239 | 240 | def on_train_end(self, logs={}): 241 | plt.close(self.fig) 242 | # self.save_plot() 243 | 244 | 245 | class WeightsCallback(Callback): 246 | # todo: update to Keras 2.X 247 | def __init__(self, parameters=None, stats=None, merge_weights=True): 248 | super().__init__() 249 | self.layers_stats = defaultdict(dict) 250 | self.fig = None 251 | self.parameters = parameters 252 | self.stats = stats 253 | self.merge_weights = merge_weights 254 | if parameters is None: 255 | self.parameters = ["W"] 256 | if stats is None: 257 | self.stats = ["mean", "std"] 258 | 259 | def get_trainable_layers(self): 260 | layers = [] 261 | for layer in self.model.layers: 262 | if "merge" in layer.name: 263 | for l in layer.layers: 264 | if hasattr(l, 'trainable') and l.trainable and len( 265 | l.weights): 266 | if not any(x.name == l.name for x in layers): 267 | layers.append(l) 268 | else: 269 | if hasattr(layer, 'trainable') and layer.trainable and len( 270 | layer.weights): 271 | layers.append(layer) 272 | return layers 273 | 274 | def on_train_begin(self, logs={}): 275 | for layer in self.get_trainable_layers(): 276 | for param in self.parameters: 277 | if any(w for w in layer.weights if param in w.name.split("_")): 278 | name = layer.name + "_" + param 279 | self.layers_stats[name]["values"] = numpy.asarray( 280 | []).ravel() 281 | for s in self.stats: 282 | self.layers_stats[name][s] = [] 283 | 284 | # plt.style.use('ggplot') 285 | plt.ion() # set plot to animated 286 | width = 3 * (1 + len(self.stats)) 287 | height = 2 * len(self.layers_stats) 288 | self.fig = plt.figure(figsize=(width, height)) 289 | # sns.set_style("whitegrid") 290 | self.draw_plot() 291 | 292 | def draw_plot(self): 293 | self.fig.clf() 294 | 295 | layers = self.get_trainable_layers() 296 | height = len(self.layers_stats) 297 | width = len(self.stats) + 1 298 | 299 | plot_count = 1 300 | for layer in layers: 301 | for param in self.parameters: 302 | weights = [w for w in layer.weights if 303 | param in w.name.split("_")] 304 | 305 | if len(weights) == 0: 306 | continue 307 | 308 | val = numpy.column_stack((w.get_value() for w in weights)) 309 | name = layer.name + "_" + param 310 | 311 | self.layers_stats[name]["values"] = val.ravel() 312 | ax = self.fig.add_subplot(height, width, plot_count) 313 | ax.hist(self.layers_stats[name]["values"], bins=50) 314 | ax.set_title(name, fontsize=10) 315 | ax.grid(True) 316 | ax.tick_params(labelsize=8) 317 | plot_count += 1 318 | 319 | for s in self.stats: 320 | axs = self.fig.add_subplot(height, width, plot_count) 321 | 322 | if s == "raster": 323 | if len(val.shape) > 2: 324 | val = val.reshape((val.shape[0], -1), order='F') 325 | self.layers_stats[name][s] = val 326 | m = axs.imshow(self.layers_stats[name][s], 327 | cmap='coolwarm', 328 | interpolation='nearest', 329 | aspect='auto', ) # aspect='equal' 330 | cbar = self.fig.colorbar(mappable=m) 331 | cbar.ax.tick_params(labelsize=8) 332 | else: 333 | self.layers_stats[name][s].append( 334 | getattr(numpy, s)(val)) 335 | axs.plot(self.layers_stats[name][s]) 336 | axs.set_ylabel(s, fontsize="small") 337 | axs.set_xlabel('epoch', fontsize="small") 338 | axs.grid(True) 339 | 340 | axs.set_title(name + " - " + s, fontsize=10) 341 | axs.tick_params(labelsize=8) 342 | plot_count += 1 343 | 344 | # plt.figtext(.1, .1, get_model_desc(self.model), wrap=True, fontsize=8) 345 | desc = get_model_desc(self.model) 346 | self.fig.text(.02, .02, desc, verticalalignment='bottom', wrap=True, 347 | fontsize=8) 348 | self.fig.tight_layout() 349 | self.fig.subplots_adjust(bottom=.14) 350 | self.fig.canvas.draw() 351 | self.fig.canvas.flush_events() 352 | 353 | def update_plot(self): 354 | 355 | layers = self.get_trainable_layers() 356 | 357 | for layer in layers: 358 | for param in self.parameters: 359 | weights = [w for w in layer.weights if 360 | param in w.name.split("_")] 361 | 362 | if len(weights) == 0: 363 | continue 364 | 365 | val = numpy.column_stack((w.get_value() for w in weights)) 366 | name = layer.name + "_" + param 367 | self.layers_stats[name]["values"] = val.ravel() 368 | for s in self.stats: 369 | if s == "raster": 370 | if len(val.shape) > 2: 371 | val = val.reshape((val.shape[0], -1), order='F') 372 | self.layers_stats[name][s] = val 373 | # self.fig.colorbar() 374 | else: 375 | self.layers_stats[name][s].append( 376 | getattr(numpy, s)(val)) 377 | 378 | plt.figtext(.02, .02, get_model_desc(self.model), wrap=True, 379 | fontsize=8) 380 | self.fig.tight_layout() 381 | self.fig.subplots_adjust(bottom=.2) 382 | self.fig.canvas.draw() 383 | self.fig.canvas.flush_events() 384 | 385 | def on_epoch_end(self, epoch, logs={}): 386 | self.update_plot() 387 | -------------------------------------------------------------------------------- /kutilities/helpers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/keras-utilities/022856d86fd585a1c79af0772fbe8b57cab8f242/kutilities/helpers/__init__.py -------------------------------------------------------------------------------- /kutilities/helpers/data_preparation.py: -------------------------------------------------------------------------------- 1 | from collections import Counter 2 | 3 | import numpy 4 | from keras.utils import np_utils 5 | from sklearn.preprocessing import LabelEncoder 6 | from sklearn.utils import compute_class_weight 7 | 8 | 9 | def get_class_labels(y): 10 | """ 11 | Get the class labels 12 | :param y: list of labels, ex. ['positive', 'negative', 'positive', 'neutral', 'positive', ...] 13 | :return: sorted unique class labels 14 | """ 15 | return numpy.unique(y) 16 | 17 | 18 | def labels_to_categories(y): 19 | """ 20 | Labels to categories 21 | :param y: list of labels, ex. ['positive', 'negative', 'positive', 'neutral', 'positive', ...] 22 | :return: list of categories, ex. [0, 2, 1, 2, 0, ...] 23 | """ 24 | encoder = LabelEncoder() 25 | encoder.fit(y) 26 | y_num = encoder.transform(y) 27 | return y_num 28 | 29 | 30 | def get_labels_to_categories_map(y): 31 | """ 32 | Get the mapping of class labels to numerical categories 33 | :param y: list of labels, ex. ['positive', 'negative', 'positive', 'neutral', 'positive', ...] 34 | :return: dictionary with the mapping 35 | """ 36 | labels = get_class_labels(y) 37 | return {l: i for i, l in enumerate(labels)} 38 | 39 | 40 | def categories_to_onehot(y): 41 | """ 42 | Transform categorical labels to one-hot vectors 43 | :param y: list of categories, ex. [0, 2, 1, 2, 0, ...] 44 | :return: list of one-hot vectors, ex. [[0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1], ...] 45 | """ 46 | return np_utils.to_categorical(y) 47 | 48 | 49 | def onehot_to_categories(y): 50 | """ 51 | Transform categorical labels to one-hot vectors 52 | :param y: list of one-hot vectors, ex. [[0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1], ...] 53 | :return: list of categories, ex. [0, 2, 1, 2, 0, ...] 54 | """ 55 | return numpy.asarray(y).argmax(axis=-1) 56 | 57 | 58 | def get_class_weights(y): 59 | """ 60 | Returns the normalized weights for each class based on the frequencies of the samples 61 | :param y: list of true labels (the labels must be hashable) 62 | :return: dictionary with the weight for each class 63 | """ 64 | 65 | weights = compute_class_weight('balanced', numpy.unique(y), y) 66 | 67 | d = {c: w for c, w in zip(numpy.unique(y), weights)} 68 | 69 | return d 70 | 71 | 72 | def get_class_weights2(y, smooth_factor=0): 73 | """ 74 | Returns the normalized weights for each class based on the frequencies of the samples 75 | :param smooth_factor: factor that smooths extremely uneven weights 76 | :param y: list of true labels (the labels must be hashable) 77 | :return: dictionary with the weight for each class 78 | """ 79 | counter = Counter(y) 80 | 81 | if smooth_factor > 0: 82 | p = max(counter.values()) * smooth_factor 83 | for k in counter.keys(): 84 | counter[k] += p 85 | 86 | majority = max(counter.values()) 87 | 88 | return {cls: float(majority / count) for cls, count in counter.items()} 89 | 90 | 91 | def print_dataset_statistics(y): 92 | """ 93 | Returns the normalized weights for each class based on the frequencies of the samples 94 | :param y: list of true labels (the labels must be hashable) 95 | :return: dictionary with the weight for each class 96 | """ 97 | counter = Counter(y) 98 | print("Total:", len(y)) 99 | statistics = {c: str(counter[c]) + " (%.2f%%)" % (counter[c] / float(len(y)) * 100.0) 100 | for c in sorted(counter.keys())} 101 | print(statistics) 102 | 103 | 104 | def predic_classes(pred): 105 | if pred.shape[-1] > 1: 106 | return pred.argmax(axis=-1) 107 | else: 108 | return (pred > 0.5).astype('int32') 109 | -------------------------------------------------------------------------------- /kutilities/helpers/generic.py: -------------------------------------------------------------------------------- 1 | def get_model_desc(model): 2 | """ 3 | Generates a small description of a Keras model. Suitable for generating footer descriptions for charts. 4 | :param model: 5 | :return: 6 | """ 7 | desc = [] 8 | 9 | conf = model.get_config() 10 | 11 | for layer in (conf["layers"] if "layers" in conf else conf): 12 | if "layer" in layer["config"]: 13 | name = "_".join([layer['class_name'], layer["config"]['layer']['class_name']]) 14 | config = layer["config"]['layer']["config"] 15 | 16 | else: 17 | name = layer['class_name'] 18 | config = layer["config"] 19 | params = [] 20 | try: 21 | params.append(config["p"]) 22 | except: 23 | pass 24 | try: 25 | params.append(config["sigma"]) 26 | except: 27 | pass 28 | try: 29 | params.append(config["output_dim"]) 30 | except: 31 | pass 32 | try: 33 | params.append(config["activation"]) 34 | except: 35 | pass 36 | try: 37 | params.append(config['l2']) 38 | except: 39 | pass 40 | 41 | desc.append(name + "({})".format(",".join([str(p) for p in params]))) 42 | 43 | description = " -> ".join(desc) 44 | try: 45 | description += " : [optimizer= {}, clipnorm={} - batch_size={}]".format(model.optimizer.__class__.__name__, 46 | model.optimizer.clipnorm, 47 | model.model.history.params[ 48 | 'batch_size']) 49 | except: 50 | pass 51 | return description 52 | -------------------------------------------------------------------------------- /kutilities/helpers/ui.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | 3 | 4 | def move_figure(f, x, y): 5 | """Move figure's upper left corner to pixel (x, y)""" 6 | backend = matplotlib.get_backend() 7 | if backend == 'TkAgg': 8 | f.canvas.manager.window.wm_geometry("+%d+%d" % (x, y)) 9 | elif backend == 'WXAgg': 10 | f.canvas.manager.window.SetPosition((x, y)) 11 | else: 12 | # This works for QT and GTK 13 | # You can also use window.setGeometry 14 | f.canvas.manager.window.move(x, y) 15 | -------------------------------------------------------------------------------- /kutilities/layers.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K, regularizers, constraints, initializers 2 | from keras.engine.topology import Layer 3 | 4 | 5 | def dot_product(x, kernel): 6 | """ 7 | Wrapper for dot product operation, in order to be compatible with both 8 | Theano and Tensorflow 9 | Args: 10 | x (): input 11 | kernel (): weights 12 | Returns: 13 | """ 14 | if K.backend() == 'tensorflow': 15 | # todo: check that this is correct 16 | return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1) 17 | else: 18 | return K.dot(x, kernel) 19 | 20 | 21 | class MeanOverTime(Layer): 22 | """ 23 | Layer that computes the mean of timesteps returned from an RNN and supports masking 24 | Example: 25 | activations = LSTM(64, return_sequences=True)(words) 26 | mean = MeanOverTime()(activations) 27 | """ 28 | 29 | def __init__(self, **kwargs): 30 | self.supports_masking = True 31 | super(MeanOverTime, self).__init__(**kwargs) 32 | 33 | def call(self, x, mask=None): 34 | if mask is not None: 35 | mask = K.cast(mask, 'float32') 36 | return K.cast(K.sum(x, axis=1) / K.sum(mask, axis=1, keepdims=True), 37 | K.floatx()) 38 | else: 39 | return K.mean(x, axis=1) 40 | 41 | def compute_output_shape(self, input_shape): 42 | return input_shape[0], input_shape[-1] 43 | 44 | def compute_mask(self, input, input_mask=None): 45 | return None 46 | 47 | 48 | class Attention(Layer): 49 | def __init__(self, 50 | W_regularizer=None, b_regularizer=None, 51 | W_constraint=None, b_constraint=None, 52 | bias=True, 53 | return_attention=False, 54 | **kwargs): 55 | """ 56 | Keras Layer that implements an Attention mechanism for temporal data. 57 | Supports Masking. 58 | Follows the work of Raffel et al. [https://arxiv.org/abs/1512.08756] 59 | # Input shape 60 | 3D tensor with shape: `(samples, steps, features)`. 61 | # Output shape 62 | 2D tensor with shape: `(samples, features)`. 63 | :param kwargs: 64 | Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True. 65 | The dimensions are inferred based on the output shape of the RNN. 66 | Note: The layer has been tested with Keras 1.x 67 | Example: 68 | 69 | # 1 70 | model.add(LSTM(64, return_sequences=True)) 71 | model.add(Attention()) 72 | # next add a Dense layer (for classification/regression) or whatever... 73 | # 2 - Get the attention scores 74 | hidden = LSTM(64, return_sequences=True)(words) 75 | sentence, word_scores = Attention(return_attention=True)(hidden) 76 | """ 77 | self.supports_masking = True 78 | self.return_attention = return_attention 79 | self.init = initializers.get('glorot_uniform') 80 | 81 | self.W_regularizer = regularizers.get(W_regularizer) 82 | self.b_regularizer = regularizers.get(b_regularizer) 83 | 84 | self.W_constraint = constraints.get(W_constraint) 85 | self.b_constraint = constraints.get(b_constraint) 86 | 87 | self.bias = bias 88 | super(Attention, self).__init__(**kwargs) 89 | 90 | def build(self, input_shape): 91 | assert len(input_shape) == 3 92 | 93 | self.W = self.add_weight((input_shape[-1],), 94 | initializer=self.init, 95 | name='{}_W'.format(self.name), 96 | regularizer=self.W_regularizer, 97 | constraint=self.W_constraint) 98 | if self.bias: 99 | self.b = self.add_weight((input_shape[1],), 100 | initializer='zero', 101 | name='{}_b'.format(self.name), 102 | regularizer=self.b_regularizer, 103 | constraint=self.b_constraint) 104 | else: 105 | self.b = None 106 | 107 | self.built = True 108 | 109 | def compute_mask(self, input, input_mask=None): 110 | # do not pass the mask to the next layers 111 | return None 112 | 113 | def call(self, x, mask=None): 114 | eij = dot_product(x, self.W) 115 | 116 | if self.bias: 117 | eij += self.b 118 | 119 | eij = K.tanh(eij) 120 | 121 | a = K.exp(eij) 122 | 123 | # apply mask after the exp. will be re-normalized next 124 | if mask is not None: 125 | # Cast the mask to floatX to avoid float64 upcasting in theano 126 | a *= K.cast(mask, K.floatx()) 127 | 128 | # in some cases especially in the early stages of training the sum may be almost zero 129 | # and this results in NaN's. A workaround is to add a very small positive number ε to the sum. 130 | # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx()) 131 | a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx()) 132 | 133 | weighted_input = x * K.expand_dims(a) 134 | 135 | result = K.sum(weighted_input, axis=1) 136 | 137 | if self.return_attention: 138 | return [result, a] 139 | return result 140 | 141 | def compute_output_shape(self, input_shape): 142 | if self.return_attention: 143 | return [(input_shape[0], input_shape[-1]), 144 | (input_shape[0], input_shape[1])] 145 | else: 146 | return input_shape[0], input_shape[-1] 147 | 148 | 149 | class AttentionWithContext(Layer): 150 | """ 151 | Attention operation, with a context/query vector, for temporal data. 152 | Supports Masking. 153 | Follows the work of Yang et al. [https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf] 154 | "Hierarchical Attention Networks for Document Classification" 155 | by using a context vector to assist the attention 156 | # Input shape 157 | 3D tensor with shape: `(samples, steps, features)`. 158 | # Output shape 159 | 2D tensor with shape: `(samples, features)`. 160 | :param kwargs: 161 | Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True. 162 | The dimensions are inferred based on the output shape of the RNN. 163 | Example: 164 | model.add(LSTM(64, return_sequences=True)) 165 | model.add(AttentionWithContext()) 166 | """ 167 | 168 | def __init__(self, 169 | W_regularizer=None, u_regularizer=None, b_regularizer=None, 170 | W_constraint=None, u_constraint=None, b_constraint=None, 171 | bias=True, 172 | return_attention=False, **kwargs): 173 | 174 | self.supports_masking = True 175 | self.return_attention = return_attention 176 | self.init = initializers.get('glorot_uniform') 177 | 178 | self.W_regularizer = regularizers.get(W_regularizer) 179 | self.u_regularizer = regularizers.get(u_regularizer) 180 | self.b_regularizer = regularizers.get(b_regularizer) 181 | 182 | self.W_constraint = constraints.get(W_constraint) 183 | self.u_constraint = constraints.get(u_constraint) 184 | self.b_constraint = constraints.get(b_constraint) 185 | 186 | self.bias = bias 187 | super(AttentionWithContext, self).__init__(**kwargs) 188 | 189 | def build(self, input_shape): 190 | assert len(input_shape) == 3 191 | 192 | self.W = self.add_weight((input_shape[-1], input_shape[-1],), 193 | initializer=self.init, 194 | name='{}_W'.format(self.name), 195 | regularizer=self.W_regularizer, 196 | constraint=self.W_constraint) 197 | if self.bias: 198 | self.b = self.add_weight((input_shape[-1],), 199 | initializer='zero', 200 | name='{}_b'.format(self.name), 201 | regularizer=self.b_regularizer, 202 | constraint=self.b_constraint) 203 | 204 | self.u = self.add_weight((input_shape[-1],), 205 | initializer=self.init, 206 | name='{}_u'.format(self.name), 207 | regularizer=self.u_regularizer, 208 | constraint=self.u_constraint) 209 | 210 | super(AttentionWithContext, self).build(input_shape) 211 | 212 | def compute_mask(self, input, input_mask=None): 213 | # do not pass the mask to the next layers 214 | return None 215 | 216 | def call(self, x, mask=None): 217 | uit = dot_product(x, self.W) 218 | 219 | if self.bias: 220 | uit += self.b 221 | 222 | uit = K.tanh(uit) 223 | # ait = K.dot(uit, self.u) 224 | ait = dot_product(uit, self.u) 225 | 226 | a = K.exp(ait) 227 | 228 | # apply mask after the exp. will be re-normalized next 229 | if mask is not None: 230 | # Cast the mask to floatX to avoid float64 upcasting in theano 231 | a *= K.cast(mask, K.floatx()) 232 | 233 | # in some cases especially in the early stages of training the sum may be almost zero 234 | # and this results in NaN's. A workaround is to add a very small positive number ε to the sum. 235 | # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx()) 236 | a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx()) 237 | 238 | a = K.expand_dims(a) 239 | weighted_input = x * a 240 | result = K.sum(weighted_input, axis=1) 241 | 242 | if self.return_attention: 243 | return [result, a] 244 | return result 245 | 246 | def compute_output_shape(self, input_shape): 247 | if self.return_attention: 248 | return [(input_shape[0], input_shape[-1]), 249 | (input_shape[0], input_shape[1])] 250 | else: 251 | return input_shape[0], input_shape[-1] 252 | -------------------------------------------------------------------------------- /local_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | rm -rf build 4 | rm -rf keras_utilities.egg-info 5 | rm -rf dist 6 | 7 | python setup.py sdist bdist_wheel 8 | 9 | pip install keras_utilities --no-index --find-links=dist --force-reinstall --no-deps -U -------------------------------------------------------------------------------- /pypi_push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | rm -rf build 4 | rm -rf keras_utilities.egg-info 5 | rm -rf dist 6 | 7 | python setup.py sdist bdist_wheel 8 | #pip wheel -r requirements.txt 9 | 10 | twine register dist/*.tar.gz 11 | twine upload dist/* 12 | python setup.py sdist upload -r pypi 13 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.11.0 2 | setuptools==20.7.0 3 | matplotlib==1.5.1 4 | keras==2.2.0 5 | seaborn==0.8.1 6 | scikit_learn==0.19.1 7 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup(name='keras-utilities', 4 | version='0.5.0', 5 | description='Utilities for Keras.', 6 | url='https://github.com/cbaziotis/keras-utilities', 7 | author='Christos Baziotis', 8 | author_email='christos.baziotis@gmail.com', 9 | license='MIT', 10 | packages=find_packages(exclude=['docs', 'tests*']), 11 | include_package_data=True 12 | ) 13 | --------------------------------------------------------------------------------