├── nn_framework ├── __init__.py ├── error_fun.py ├── activation.py ├── layer.py └── framework.py ├── README.md ├── run_framework.py ├── LICENSE ├── data_loader_nordic_runes.py ├── .gitignore ├── data_loader_two_by_two.py ├── data_loader_three_by_three.py ├── elder_futhark.py └── autoencoder_viz.py /nn_framework/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /nn_framework/error_fun.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | # All of these expect two identically sized numpy arrays as inputs 4 | # and return the same size error output. 5 | 6 | 7 | class abs(object): 8 | @staticmethod 9 | def calc(x, y): 10 | return np.abs(y - x) 11 | 12 | @staticmethod 13 | def calc_d(x, y): 14 | return np.sign(y - x) 15 | 16 | 17 | class sqr(object): 18 | @staticmethod 19 | def calc(x, y): 20 | return (y - x)**2 21 | 22 | @staticmethod 23 | def calc_d(x, y): 24 | return 2 * (y - x) 25 | -------------------------------------------------------------------------------- /nn_framework/activation.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | # All of these need to be able to handle 2D numpy arrays as inputs. 4 | 5 | 6 | class tanh(object): 7 | @staticmethod 8 | def calc(v): 9 | return np.tanh(v) 10 | 11 | @staticmethod 12 | def calc_d(v): 13 | return 1 - np.tanh(v) ** 2 14 | 15 | 16 | def logistic(v): 17 | @staticmethod 18 | def calc(v): 19 | return 1 / (1 + np.exp(-v)) 20 | 21 | @staticmethod 22 | def calc_d(v): 23 | return calc(v) * (1 - calc(v)) 24 | 25 | 26 | def relu(v): 27 | @staticmethod 28 | def calc(v): 29 | return np.maximum(0, v) 30 | 31 | @staticmethod 32 | def calc_d(v): 33 | derivative = 0 34 | if v > 0: 35 | derivative = 1 36 | return derivative 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Write your own neural network framework 2 | 3 | This repository contains the code for a simple neural network framework. 4 | In [this course](https://end-to-end-machine-learning.teachable.com/p/write-a-neural-network-framework/) 5 | as part of the [End-to-end Machine Learning School](https://end-to-end-machine-learning.teachable.com/), 6 | we step through this code together, writing it part-by-part in a sequence of 7 | coding exercises. 8 | 9 | Come give it a try. 10 | 11 | 12 | To run this code: 13 | 14 | > python3 run_framework.py 15 | 16 | 17 | The autoencoder visualization code used here is adapted from another 18 | End-to-End Machine Learning course, [Neural Network Visualization](https://end-to-end-machine-learning.teachable.com/p/neural-network-visualization). 19 | To understand it down to it's roots, you can walk through that course too. 20 | -------------------------------------------------------------------------------- /run_framework.py: -------------------------------------------------------------------------------- 1 | import data_loader_nordic_runes as dat 2 | import nn_framework.activation as activation 3 | import nn_framework.framework as framework 4 | import nn_framework.error_fun as error_fun 5 | import nn_framework.layer as layer 6 | from autoencoder_viz import Printer 7 | 8 | N_NODES = [31, 22, 13] 9 | 10 | training_set, evaluation_set = dat.get_data_sets() 11 | 12 | sample = next(training_set()) 13 | input_value_range = (0, 1) 14 | n_pixels = sample.shape[0] * sample.shape[1] 15 | printer = Printer(input_shape=sample.shape) 16 | 17 | n_nodes = [n_pixels] + N_NODES + [n_pixels] 18 | model = [] 19 | for i_layer in range(len(n_nodes) - 1): 20 | model.append(layer.Dense( 21 | n_nodes[i_layer], 22 | n_nodes[i_layer + 1], 23 | activation.tanh 24 | )) 25 | 26 | autoencoder = framework.ANN( 27 | model=model, 28 | error_fun=error_fun.abs, 29 | printer=printer, 30 | expected_range=input_value_range, 31 | ) 32 | autoencoder.train(training_set) 33 | autoencoder.evaluate(evaluation_set) 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Brandon Rohrer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /data_loader_nordic_runes.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import elder_futhark as ef 3 | 4 | 5 | def get_data_sets(): 6 | """ 7 | This function creates two other functions that generate data. 8 | One generates a training data set and the other, an evaluation set. 9 | 10 | The examples have the format of a two-dimensional numpy array. 11 | They can be thought of as a very small (7-pixel by 7-pixel) image. 12 | 13 | The examples are drawn from the 24-rune alphabet of Elder Futhark. 14 | 15 | 16 | To use in a script: 17 | 18 | import data_loader_nordic_runes as dat 19 | 20 | training_generator, evaluation_grenerator = dat.get_data_sets() 21 | new_training_example = training_generator.next() 22 | new_evaluation_example = evaluation_generator.next() 23 | """ 24 | 25 | examples = list(ef.runes.values()) 26 | 27 | def training_set(): 28 | while True: 29 | index = np.random.choice(len(examples)) 30 | yield examples[index] 31 | 32 | def evaluation_set(): 33 | while True: 34 | index = np.random.choice(len(examples)) 35 | yield examples[index] 36 | 37 | return training_set, evaluation_set 38 | -------------------------------------------------------------------------------- /nn_framework/layer.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class Dense(object): 5 | def __init__( 6 | self, 7 | m_inputs, 8 | n_outputs, 9 | activate, 10 | ): 11 | self.m_inputs = int(m_inputs) 12 | self.n_outputs = int(n_outputs) 13 | self.activate = activate 14 | self.learning_rate = .001 15 | 16 | # Choose random weights. 17 | # Inputs match to rows. Outputs match to columns. 18 | self.weights = (np.random.sample( 19 | size=(self.m_inputs + 1, self.n_outputs)) * 2 - 1) 20 | self.x = np.zeros((1, self.m_inputs + 1)) 21 | self.y = np.zeros((1, self.n_outputs)) 22 | 23 | def forward_prop(self, inputs): 24 | """ 25 | Propagate the inputs forward through the network. 26 | 27 | inputs: 2D array 28 | One column array of input values. 29 | """ 30 | bias = np.ones((1, 1)) 31 | self.x = np.concatenate((inputs, bias), axis=1) 32 | v = self.x @ self.weights 33 | self.y = self.activate.calc(v) 34 | return self.y 35 | 36 | def back_prop(self, de_dy): 37 | """ 38 | Propagate the outputs back through the layer. 39 | """ 40 | dy_dv = self.activate.calc_d(self.y) 41 | # v = self.x @ self.weights 42 | # dv_dw = self.x 43 | # dv_dx = self.weights 44 | dy_dw = self.x.transpose() @ dy_dv 45 | de_dw = de_dy * dy_dw 46 | self.weights -= de_dw * self.learning_rate 47 | de_dx = (de_dy * dy_dv) @ self.weights.transpose() 48 | return de_dx[:, :-1] 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /data_loader_two_by_two.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def get_data_sets(): 5 | """ 6 | This function creates two other functions that generate data. 7 | One generates a training data set and the other, an evaluation set. 8 | 9 | The examples have the format of a two-dimensional numpy array. 10 | They can be thought of as a very small (two-pixel by two-pixel) image. 11 | 12 | 13 | To use in a script: 14 | 15 | import data_loader_two_by_two as dat 16 | 17 | training_generator, evaluation_grenerator = dat.get_data_sets() 18 | new_training_example = training_generator.next() 19 | new_evaluation_example = evaluation_generator.next() 20 | """ 21 | examples = [ 22 | np.array([ 23 | [0, 0], 24 | [1, 1] 25 | ]), 26 | np.array([ 27 | [1, 0], 28 | [1, 0] 29 | ]), 30 | np.array([ 31 | [1, 1], 32 | [0, 0] 33 | ]), 34 | np.array([ 35 | [0, 1], 36 | [0, 1] 37 | ]), 38 | np.array([ 39 | [1, 0], 40 | [0, 1] 41 | ]), 42 | np.array([ 43 | [0, 1], 44 | [1, 0] 45 | ]), 46 | np.array([ 47 | [0, 1], 48 | [1, 1] 49 | ]), 50 | np.array([ 51 | [1, 1], 52 | [1, 0] 53 | ]), 54 | np.array([ 55 | [1, 0], 56 | [1, 1] 57 | ]), 58 | np.array([ 59 | [1, 1], 60 | [0, 1] 61 | ]), 62 | np.array([ 63 | [1, 0], 64 | [0, 0] 65 | ]), 66 | np.array([ 67 | [0, 0], 68 | [0, 1] 69 | ]), 70 | np.array([ 71 | [0, 1], 72 | [0, 0] 73 | ]), 74 | np.array([ 75 | [0, 0], 76 | [1, 0] 77 | ]), 78 | ] 79 | 80 | def training_set(): 81 | while True: 82 | index = np.random.choice(len(examples)) 83 | yield examples[index] 84 | 85 | def evaluation_set(): 86 | while True: 87 | index = np.random.choice(len(examples)) 88 | yield examples[index] 89 | 90 | return training_set, evaluation_set 91 | -------------------------------------------------------------------------------- /data_loader_three_by_three.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def get_data_sets(): 5 | """ 6 | This function creates two other functions that generate data. 7 | One generates a training data set and the other, an evaluation set. 8 | 9 | The examples have the format of a two-dimensional numpy array. 10 | They can be thought of as a very small (three-pixel by three-pixel) image. 11 | 12 | 13 | To use in a script: 14 | 15 | import data_loader_three_by_three as dat 16 | 17 | training_generator, evaluation_grenerator = dat.get_data_sets() 18 | new_training_example = training_generator.next() 19 | new_evaluation_example = evaluation_generator.next() 20 | """ 21 | examples = [ 22 | np.array([ 23 | [1, 1, 1], 24 | [0, 0, 0], 25 | [0, 0, 0], 26 | ]), 27 | np.array([ 28 | [0, 0, 0], 29 | [1, 1, 1], 30 | [0, 0, 0], 31 | ]), 32 | np.array([ 33 | [0, 0, 0], 34 | [0, 0, 0], 35 | [1, 1, 1], 36 | ]), 37 | np.array([ 38 | [1, 0, 0], 39 | [1, 0, 0], 40 | [1, 0, 0], 41 | ]), 42 | np.array([ 43 | [0, 1, 0], 44 | [0, 1, 0], 45 | [0, 1, 0], 46 | ]), 47 | np.array([ 48 | [0, 0, 1], 49 | [0, 0, 1], 50 | [0, 0, 1], 51 | ]), 52 | np.array([ 53 | [0, 1, 0], 54 | [0, 0, 1], 55 | [1, 0, 0], 56 | ]), 57 | np.array([ 58 | [0, 0, 1], 59 | [1, 0, 0], 60 | [0, 1, 0], 61 | ]), 62 | np.array([ 63 | [1, 0, 0], 64 | [0, 1, 0], 65 | [0, 0, 1], 66 | ]), 67 | np.array([ 68 | [0, 0, 1], 69 | [0, 1, 0], 70 | [1, 0, 0], 71 | ]), 72 | np.array([ 73 | [0, 1, 0], 74 | [1, 0, 0], 75 | [0, 0, 1], 76 | ]), 77 | np.array([ 78 | [1, 0, 0], 79 | [0, 0, 1], 80 | [0, 1, 0], 81 | ]), 82 | np.array([ 83 | [1, 1, 1], 84 | [1, 0, 1], 85 | [1, 1, 1], 86 | ]), 87 | np.array([ 88 | [0, 1, 0], 89 | [1, 0, 1], 90 | [0, 1, 0], 91 | ]), 92 | np.array([ 93 | [1, 1, 1], 94 | [1, 0, 0], 95 | [1, 0, 0], 96 | ]), 97 | np.array([ 98 | [0, 0, 1], 99 | [0, 0, 1], 100 | [1, 1, 1], 101 | ]), 102 | np.array([ 103 | [1, 1, 1], 104 | [0, 0, 1], 105 | [0, 0, 1], 106 | ]), 107 | np.array([ 108 | [1, 0, 0], 109 | [1, 0, 0], 110 | [1, 1, 1], 111 | ]), 112 | ] 113 | 114 | def training_set(): 115 | while True: 116 | index = np.random.choice(len(examples)) 117 | yield examples[index] 118 | 119 | def evaluation_set(): 120 | while True: 121 | index = np.random.choice(len(examples)) 122 | yield examples[index] 123 | 124 | return training_set, evaluation_set 125 | -------------------------------------------------------------------------------- /nn_framework/framework.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | plt.switch_backend("agg") 5 | 6 | 7 | class ANN(object): 8 | def __init__( 9 | self, 10 | model=None, 11 | error_fun=None, 12 | printer=None, 13 | expected_range=(-1, 1), 14 | ): 15 | self.layers = model 16 | self.error_fun = error_fun 17 | self.error_history = [] 18 | self.n_iter_train = int(1e8) 19 | self.n_iter_evaluate = int(1e6) 20 | self.viz_interval = int(1e5) 21 | self.reporting_bin_size = int(1e3) 22 | self.report_min = -3 23 | self.report_max = 0 24 | self.printer = printer 25 | self.expected_range = expected_range 26 | 27 | self.reports_path = "reports" 28 | self.report_name = "performance_history.png" 29 | # Ensure that subdirectories exist. 30 | try: 31 | os.mkdir("reports") 32 | except Exception: 33 | pass 34 | 35 | def train(self, training_set): 36 | for i_iter in range(self.n_iter_train): 37 | x = self.normalize(next(training_set()).ravel()) 38 | y = self.forward_prop(x) 39 | error = self.error_fun.calc(x, y) 40 | error_d = self.error_fun.calc_d(x, y) 41 | self.error_history.append((np.mean(error**2))**.5) 42 | self.back_prop(error_d) 43 | 44 | if (i_iter + 1) % self.viz_interval == 0: 45 | self.report() 46 | self.printer.render(self, x, f"train_{i_iter + 1:08d}") 47 | 48 | def evaluate(self, evaluation_set): 49 | for i_iter in range(self.n_iter_evaluate): 50 | x = self.normalize(next(evaluation_set()).ravel()) 51 | y = self.forward_prop(x) 52 | error = self.error_fun.calc(x, y) 53 | self.error_history.append((np.mean(error**2))**.5) 54 | 55 | if (i_iter + 1) % self.viz_interval == 0: 56 | self.report() 57 | self.printer.render(self, x, f"eval_{i_iter + 1:08d}") 58 | 59 | def forward_prop(self, x): 60 | # Convert the inputs into a 2D array of the right shape. 61 | y = x.ravel()[np.newaxis, :] 62 | for layer in self.layers: 63 | y = layer.forward_prop(y) 64 | return y.ravel() 65 | 66 | def back_prop(self, de_dy): 67 | for i_layer, layer in enumerate(self.layers[::-1]): 68 | de_dx = layer.back_prop(de_dy) 69 | de_dy = de_dx 70 | 71 | def forward_prop_to_layer(self, x, i_layer): 72 | y = x.ravel()[np.newaxis, :] 73 | for layer in self.layers[:i_layer]: 74 | y = layer.forward_prop(y) 75 | return y.ravel() 76 | 77 | def forward_prop_from_layer(self, x, i_layer): 78 | y = x.ravel()[np.newaxis, :] 79 | for layer in self.layers[i_layer:]: 80 | y = layer.forward_prop(y) 81 | return y.ravel() 82 | 83 | def normalize(self, values): 84 | """ 85 | Transform the input/output values so that they tend to 86 | fall between -.5 and .5 87 | """ 88 | min_val = self.expected_range[0] 89 | max_val = self.expected_range[1] 90 | scale_factor = max_val - min_val 91 | offset_factor = min_val 92 | return (values - offset_factor) / scale_factor - .5 93 | 94 | def denormalize(self, transformed_values): 95 | min_val = self.expected_range[0] 96 | max_val = self.expected_range[1] 97 | scale_factor = max_val - min_val 98 | offset_factor = min_val 99 | return (transformed_values + .5) * scale_factor + offset_factor 100 | 101 | def report(self): 102 | n_bins = int(len(self.error_history) // self.reporting_bin_size) 103 | smoothed_history = [] 104 | for i_bin in range(n_bins): 105 | smoothed_history.append(np.mean(self.error_history[ 106 | i_bin * self.reporting_bin_size: 107 | (i_bin + 1) * self.reporting_bin_size 108 | ])) 109 | error_history = np.log10(np.array(smoothed_history) + 1e-10) 110 | ymin = np.minimum(self.report_min, np.min(error_history)) 111 | ymax = np.maximum(self.report_max, np.max(error_history)) 112 | fig = plt.figure() 113 | ax = plt.gca() 114 | ax.plot(error_history) 115 | ax.set_xlabel(f"x{self.reporting_bin_size} iterations") 116 | ax.set_ylabel("log error") 117 | ax.set_ylim(ymin, ymax) 118 | ax.grid() 119 | fig.savefig(os.path.join(self.reports_path, self.report_name)) 120 | plt.close() 121 | -------------------------------------------------------------------------------- /elder_futhark.py: -------------------------------------------------------------------------------- 1 | """ 2 | A dictionary of some real life cool-looking ancient runes, 3 | specifically Elder Futhark runes 4 | https://en.wikipedia.org/wiki/Runes#Elder_Futhark_(2nd_to_8th_centuries) 5 | 6 | Each rune is represented as a two-dimensional numpy array 7 | in a dictionary and can be rendered as a 7 x 7 image. 8 | The names of the runes (or a rough transliteration of them) 9 | are the keys of the dictionary. 10 | 11 | 12 | To see the runes as images, one by one: 13 | 14 | > python3 elder_futhark.py 15 | 16 | 17 | To use the runes in another script: 18 | 19 | import elder_futhark as ef 20 | 21 | my_rune_dict = ef.runes 22 | """ 23 | 24 | import numpy as np 25 | import matplotlib.pyplot as plt 26 | 27 | runes = { 28 | "frey": np.array([ 29 | [0, 1, 0, 1, 0, 0, 1], 30 | [0, 1, 1, 0, 0, 1, 0], 31 | [0, 1, 0, 0, 1, 0, 0], 32 | [0, 1, 0, 1, 0, 0, 0], 33 | [0, 1, 1, 0, 0, 0, 0], 34 | [0, 1, 0, 0, 0, 0, 0], 35 | [0, 1, 0, 0, 0, 0, 0], 36 | ]), 37 | "uruz": np.array([ 38 | [0, 1, 0, 0, 0, 0, 0], 39 | [0, 1, 1, 0, 0, 0, 0], 40 | [0, 1, 0, 1, 0, 0, 0], 41 | [0, 1, 0, 1, 0, 0, 0], 42 | [0, 1, 0, 0, 1, 0, 0], 43 | [0, 1, 0, 0, 0, 1, 0], 44 | [0, 1, 0, 0, 0, 0, 1], 45 | ]), 46 | "pursizas": np.array([ 47 | [0, 1, 0, 0, 0, 0, 0], 48 | [0, 1, 0, 0, 0, 0, 0], 49 | [0, 1, 1, 0, 0, 0, 0], 50 | [0, 1, 0, 1, 0, 0, 0], 51 | [0, 1, 1, 0, 0, 0, 0], 52 | [0, 1, 0, 0, 0, 0, 0], 53 | [0, 1, 0, 0, 0, 0, 0], 54 | ]), 55 | "ansuz": np.array([ 56 | [0, 1, 1, 0, 0, 0, 0], 57 | [0, 1, 0, 1, 0, 0, 0], 58 | [0, 1, 0, 0, 0, 0, 0], 59 | [0, 1, 1, 0, 0, 0, 0], 60 | [0, 1, 0, 1, 0, 0, 0], 61 | [0, 1, 0, 0, 0, 0, 0], 62 | [0, 1, 0, 0, 0, 0, 0], 63 | ]), 64 | "raido": np.array([ 65 | [0, 1, 1, 0, 0, 0, 0], 66 | [0, 1, 0, 1, 0, 0, 0], 67 | [0, 1, 0, 0, 1, 0, 0], 68 | [0, 1, 0, 1, 0, 0, 0], 69 | [0, 1, 0, 0, 1, 0, 0], 70 | [0, 1, 0, 0, 0, 1, 0], 71 | [0, 1, 0, 0, 0, 1, 0], 72 | ]), 73 | "kaunan": np.array([ 74 | [0, 0, 0, 0, 1, 0, 0], 75 | [0, 0, 0, 1, 0, 0, 0], 76 | [0, 0, 1, 0, 0, 0, 0], 77 | [0, 1, 0, 0, 0, 0, 0], 78 | [0, 0, 1, 0, 0, 0, 0], 79 | [0, 0, 0, 1, 0, 0, 0], 80 | [0, 0, 0, 0, 1, 0, 0], 81 | ]), 82 | "gebo": np.array([ 83 | [1, 0, 0, 0, 0, 0, 1], 84 | [0, 1, 0, 0, 0, 1, 0], 85 | [0, 0, 1, 0, 1, 0, 0], 86 | [0, 0, 0, 1, 0, 0, 0], 87 | [0, 0, 1, 0, 1, 0, 0], 88 | [0, 1, 0, 0, 0, 1, 0], 89 | [1, 0, 0, 0, 0, 0, 1], 90 | ]), 91 | "wunjo": np.array([ 92 | [0, 1, 1, 0, 0, 0, 0], 93 | [0, 1, 0, 1, 0, 0, 0], 94 | [0, 1, 0, 1, 0, 0, 0], 95 | [0, 1, 1, 0, 0, 0, 0], 96 | [0, 1, 0, 0, 0, 0, 0], 97 | [0, 1, 0, 0, 0, 0, 0], 98 | [0, 1, 0, 0, 0, 0, 0], 99 | ]), 100 | "hagalaz": np.array([ 101 | [0, 1, 0, 0, 0, 1, 0], 102 | [0, 1, 0, 0, 0, 1, 0], 103 | [0, 1, 1, 0, 0, 1, 0], 104 | [0, 1, 0, 1, 0, 1, 0], 105 | [0, 1, 0, 0, 1, 1, 0], 106 | [0, 1, 0, 0, 0, 1, 0], 107 | [0, 1, 0, 0, 0, 1, 0], 108 | ]), 109 | "naudiz": np.array([ 110 | [0, 0, 0, 1, 0, 0, 0], 111 | [0, 0, 0, 1, 0, 0, 0], 112 | [0, 0, 1, 1, 0, 0, 0], 113 | [0, 0, 0, 1, 1, 0, 0], 114 | [0, 0, 0, 1, 0, 0, 0], 115 | [0, 0, 0, 1, 0, 0, 0], 116 | [0, 0, 0, 1, 0, 0, 0], 117 | ]), 118 | "isaz": np.array([ 119 | [0, 0, 0, 1, 0, 0, 0], 120 | [0, 0, 0, 1, 0, 0, 0], 121 | [0, 0, 0, 1, 0, 0, 0], 122 | [0, 0, 0, 1, 0, 0, 0], 123 | [0, 0, 0, 1, 0, 0, 0], 124 | [0, 0, 0, 1, 0, 0, 0], 125 | [0, 0, 0, 1, 0, 0, 0], 126 | ]), 127 | "jera": np.array([ 128 | [0, 0, 1, 0, 0, 0, 0], 129 | [0, 1, 0, 0, 0, 0, 0], 130 | [1, 0, 0, 1, 0, 0, 0], 131 | [0, 1, 0, 0, 1, 0, 0], 132 | [0, 0, 1, 0, 0, 1, 0], 133 | [0, 0, 0, 0, 1, 0, 0], 134 | [0, 0, 0, 1, 0, 0, 0], 135 | ]), 136 | "iwaz": np.array([ 137 | [0, 0, 0, 1, 0, 0, 0], 138 | [0, 0, 0, 1, 1, 0, 0], 139 | [0, 0, 0, 1, 0, 1, 0], 140 | [0, 0, 0, 1, 0, 0, 0], 141 | [0, 1, 0, 1, 0, 0, 0], 142 | [0, 0, 1, 1, 0, 0, 0], 143 | [0, 0, 0, 1, 0, 0, 0], 144 | ]), 145 | "perp": np.array([ 146 | [0, 1, 0, 0, 0, 1, 0], 147 | [0, 1, 1, 0, 1, 0, 0], 148 | [0, 1, 0, 1, 0, 0, 0], 149 | [0, 1, 0, 0, 0, 0, 0], 150 | [0, 1, 0, 1, 0, 0, 0], 151 | [0, 1, 1, 0, 1, 0, 0], 152 | [0, 1, 0, 0, 0, 1, 0], 153 | ]), 154 | "algiz": np.array([ 155 | [1, 0, 0, 1, 0, 0, 1], 156 | [0, 1, 0, 1, 0, 1, 0], 157 | [0, 0, 1, 1, 1, 0, 0], 158 | [0, 0, 0, 1, 0, 0, 0], 159 | [0, 0, 0, 1, 0, 0, 0], 160 | [0, 0, 0, 1, 0, 0, 0], 161 | [0, 0, 0, 1, 0, 0, 0], 162 | ]), 163 | "sowilo": np.array([ 164 | [0, 0, 1, 0, 0, 0, 1], 165 | [0, 1, 0, 0, 0, 1, 0], 166 | [1, 0, 0, 0, 1, 0, 0], 167 | [0, 1, 0, 0, 0, 1, 0], 168 | [1, 0, 0, 0, 0, 0, 1], 169 | [0, 1, 0, 0, 0, 1, 0], 170 | [0, 0, 1, 0, 1, 0, 0], 171 | ]), 172 | "tiwaz": np.array([ 173 | [0, 0, 0, 1, 0, 0, 0], 174 | [0, 0, 1, 1, 1, 0, 0], 175 | [0, 1, 0, 1, 0, 1, 0], 176 | [0, 0, 0, 1, 0, 0, 0], 177 | [0, 0, 0, 1, 0, 0, 0], 178 | [0, 0, 0, 1, 0, 0, 0], 179 | [0, 0, 0, 1, 0, 0, 0], 180 | ]), 181 | "berkanan": np.array([ 182 | [0, 1, 1, 0, 0, 0, 0], 183 | [0, 1, 0, 1, 0, 0, 0], 184 | [0, 1, 0, 0, 1, 0, 0], 185 | [0, 1, 1, 1, 0, 0, 0], 186 | [0, 1, 0, 0, 1, 0, 0], 187 | [0, 1, 0, 1, 0, 0, 0], 188 | [0, 1, 1, 0, 0, 0, 0], 189 | ]), 190 | "ehwaz": np.array([ 191 | [0, 1, 0, 0, 0, 1, 0], 192 | [0, 1, 1, 0, 1, 1, 0], 193 | [0, 1, 0, 1, 0, 1, 0], 194 | [0, 1, 0, 0, 0, 1, 0], 195 | [0, 1, 0, 0, 0, 1, 0], 196 | [0, 1, 0, 0, 0, 1, 0], 197 | [0, 1, 0, 0, 0, 1, 0], 198 | ]), 199 | "mannaz": np.array([ 200 | [0, 1, 0, 0, 0, 1, 0], 201 | [0, 1, 1, 0, 1, 1, 0], 202 | [0, 1, 0, 1, 0, 1, 0], 203 | [0, 1, 1, 0, 1, 1, 0], 204 | [0, 1, 0, 0, 0, 1, 0], 205 | [0, 1, 0, 0, 0, 1, 0], 206 | [0, 1, 0, 0, 0, 1, 0], 207 | ]), 208 | "laguz": np.array([ 209 | [0, 0, 0, 1, 0, 0, 0], 210 | [0, 0, 0, 1, 1, 0, 0], 211 | [0, 0, 0, 1, 0, 1, 0], 212 | [0, 0, 0, 1, 0, 0, 0], 213 | [0, 0, 0, 1, 0, 0, 0], 214 | [0, 0, 0, 1, 0, 0, 0], 215 | [0, 0, 0, 1, 0, 0, 0], 216 | ]), 217 | "ingwaz": np.array([ 218 | [0, 0, 0, 0, 0, 0, 0], 219 | [0, 0, 0, 0, 1, 0, 1], 220 | [0, 1, 0, 0, 0, 1, 0], 221 | [1, 0, 1, 0, 1, 0, 1], 222 | [0, 1, 0, 0, 0, 1, 0], 223 | [0, 0, 0, 0, 1, 0, 1], 224 | [0, 0, 0, 0, 0, 0, 0], 225 | ]), 226 | "opila": np.array([ 227 | [0, 0, 0, 1, 0, 0, 0], 228 | [0, 0, 1, 0, 1, 0, 0], 229 | [0, 1, 0, 0, 0, 1, 0], 230 | [0, 0, 1, 0, 1, 0, 0], 231 | [0, 0, 0, 1, 0, 0, 0], 232 | [0, 0, 1, 0, 1, 0, 0], 233 | [0, 1, 0, 0, 0, 1, 0], 234 | ]), 235 | "dagaz": np.array([ 236 | [1, 0, 0, 0, 0, 0, 1], 237 | [1, 1, 0, 0, 0, 1, 1], 238 | [1, 0, 1, 0, 1, 0, 1], 239 | [1, 0, 0, 1, 0, 0, 1], 240 | [1, 0, 1, 0, 1, 0, 1], 241 | [1, 1, 0, 0, 0, 1, 1], 242 | [1, 0, 0, 0, 0, 0, 1], 243 | ]), 244 | } 245 | 246 | 247 | if __name__ == "__main__": 248 | for rune in runes.values(): 249 | fig = plt.figure() 250 | ax = fig.gca() 251 | ax.imshow(rune) 252 | plt.show() 253 | -------------------------------------------------------------------------------- /autoencoder_viz.py: -------------------------------------------------------------------------------- 1 | """ 2 | Generate an autoencoder neural network visualization 3 | """ 4 | import os 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | plt.switch_backend("agg") 8 | 9 | 10 | class Printer(object): 11 | def __init__(self, input_shape=None): 12 | 13 | # Choose a color palette 14 | self.blue = "#04253a" 15 | self.green = "#4c837a" 16 | self.tan = "#e1ddbf" 17 | self.cmap = "bone" 18 | self.error_cmap = "RdGy" 19 | self.im_vmax = .5 20 | self.im_vmin = -.5 21 | self.DPI = 300 22 | 23 | # Changing these adjusts the size and layout of the visualization 24 | self.figure_width = 16 25 | self.figure_height = 9 26 | self.right_border = 0.7 27 | self.left_border = 0.7 28 | self.top_border = 0.8 29 | self.bottom_border = 0.6 30 | 31 | self.n_image_rows = input_shape[0] 32 | self.n_image_cols = input_shape[1] 33 | 34 | self.input_image_bottom = 5 35 | self.input_image_height = 0.25 * self.figure_height 36 | self.error_image_scale = 0.7 37 | self.error_gap_scale = 0.3 38 | self.between_layer_scale = 0.8 39 | self.between_node_scale = 0.4 40 | 41 | self.savedir = "nn_images" 42 | try: 43 | os.mkdir(self.savedir) 44 | except Exception: 45 | pass 46 | try: 47 | for filename in os.listdir(self.savedir): 48 | if filename[-3:] == "png": 49 | os.remove(os.path.join(self.savedir, filename)) 50 | except Exception: 51 | pass 52 | 53 | def render(self, nn, inputs, name=""): 54 | """ 55 | Build a visualization of an image autoencoder neural network, 56 | piece by piece. 57 | """ 58 | fig, ax_boss = self.create_background() 59 | self.find_nn_size(nn) 60 | self.find_node_image_size() 61 | self.find_between_layer_gap() 62 | self.find_between_node_gap() 63 | self.find_error_image_position() 64 | 65 | image_axes = [] 66 | self.add_input_image(fig, image_axes, nn, inputs) 67 | for i_layer in range(self.n_layers): 68 | self.add_node_images(fig, i_layer, image_axes, nn, inputs) 69 | self.add_output_image(fig, image_axes, nn, inputs) 70 | self.add_error_image(fig, image_axes, nn, inputs) 71 | self.add_layer_connections(ax_boss, image_axes) 72 | self.save_nn_viz(fig, name) 73 | plt.close() 74 | 75 | def create_background(self): 76 | fig = plt.figure( 77 | edgecolor=self.tan, 78 | facecolor=self.green, 79 | figsize=(self.figure_width, self.figure_height), 80 | linewidth=4, 81 | ) 82 | ax_boss = fig.add_axes((0, 0, 1, 1), facecolor="none") 83 | ax_boss.set_xlim(0, 1) 84 | ax_boss.set_ylim(0, 1) 85 | return fig, ax_boss 86 | 87 | def find_nn_size(self, nn): 88 | """ 89 | Find all the parameters that describe the size and location 90 | of the elements of the visualization. 91 | """ 92 | # Enforce square pixels. 93 | # Each pixel will have the same height and width. 94 | self.aspect_ratio = self.n_image_cols / self.n_image_rows 95 | self.input_image_width = self.input_image_height * self.aspect_ratio 96 | 97 | # The network as a whole 98 | self.n_nodes = [] 99 | for layer in nn.layers: 100 | self.n_nodes.append(layer.m_inputs) 101 | self.n_nodes.append(layer.n_outputs) 102 | self.n_layers = len(self.n_nodes) 103 | self.max_nodes = np.max(self.n_nodes) 104 | 105 | def find_node_image_size(self): 106 | """ 107 | What should the height and width of each node image be? 108 | As big as possible, given the constraints. 109 | There are two possible constraints: 110 | 1. Fill the figure top-to-bottom. 111 | 2. Fill the figure side-to-side. 112 | To determine which of these limits the size of the node images, 113 | we'll calculate the image size assuming each constraint separately, 114 | then respect the one that results in the smaller node image. 115 | """ 116 | # First assume height is the limiting factor. 117 | total_space_to_fill = ( 118 | self.figure_height 119 | - self.bottom_border 120 | - self.top_border 121 | ) 122 | # Use the layer with the largest number of nodes (n_max). 123 | # Pack the images and the gaps as tight as possible. 124 | # In that case, if the image height is h, 125 | # the gaps will each be h * self.between_node_scale. 126 | # There will be n_max nodes and (n_max - 1) gaps. 127 | # After a wee bit of algebra: 128 | height_constrained_by_height = ( 129 | total_space_to_fill / ( 130 | self.max_nodes 131 | + (self.max_nodes - 1) 132 | * self.between_node_scale 133 | ) 134 | ) 135 | 136 | # Second assume width is the limiting factor. 137 | total_space_to_fill = ( 138 | self.figure_width 139 | - self.left_border 140 | - self.right_border 141 | - 2 * self.input_image_width 142 | ) 143 | # Again, pack the images as tightly as possible side-to-side. 144 | # In this case, if the image width is w, 145 | # the gaps will each be w * self.between_layer_scale. 146 | # There will be n_layer nodes and (n_layer + 1) gaps. 147 | # After another tidbit of algebra: 148 | width_constrained_by_width = ( 149 | total_space_to_fill / ( 150 | self.n_layers + (self.n_layers + 1) * self.between_layer_scale 151 | ) 152 | ) 153 | 154 | # Figure out what the height would be for this width. 155 | height_constrained_by_width = ( 156 | width_constrained_by_width 157 | / self.aspect_ratio 158 | ) 159 | 160 | # See which constraint is more restrictive, and go with that one. 161 | self.node_image_height = np.minimum( 162 | height_constrained_by_width, 163 | height_constrained_by_height) 164 | self.node_image_width = self.node_image_height * self.aspect_ratio 165 | 166 | def find_between_layer_gap(self): 167 | """ 168 | How big is the horizontal spacing between_layers? 169 | This is also the spacing between the input image and the first layer 170 | and between the last layer and the output image. 171 | """ 172 | horizontal_gap_total = ( 173 | self.figure_width 174 | - 2 * self.input_image_width 175 | - self.n_layers * self.node_image_width 176 | - self.left_border 177 | - self.right_border 178 | ) 179 | n_horizontal_gaps = self.n_layers + 1 180 | self.between_layer_gap = horizontal_gap_total / n_horizontal_gaps 181 | 182 | def find_between_node_gap(self): 183 | """ 184 | How big is the vertical gap between_node images? 185 | """ 186 | vertical_gap_total = ( 187 | self.figure_height 188 | - self.top_border 189 | - self.bottom_border 190 | - self.max_nodes 191 | * self.node_image_height 192 | ) 193 | n_vertical_gaps = self.max_nodes - 1 194 | self.between_node_gap = vertical_gap_total / n_vertical_gaps 195 | 196 | def find_error_image_position(self): 197 | """ 198 | Where exactly should the error image be positioned? 199 | """ 200 | self.error_image_width = ( 201 | self.input_image_width 202 | * self.error_image_scale 203 | ) 204 | self.error_image_height = ( 205 | self.input_image_height 206 | * self.error_image_scale 207 | ) 208 | self.error_image_bottom = ( 209 | self.input_image_bottom 210 | - self.input_image_height 211 | * self.error_gap_scale 212 | - self.error_image_height 213 | ) 214 | error_image_center = ( 215 | self.figure_width 216 | - self.right_border 217 | - self.input_image_width / 2 218 | ) 219 | self.error_image_left = ( 220 | error_image_center 221 | - self.error_image_width / 2 222 | ) 223 | 224 | def add_input_image(self, fig, image_axes, nn, inputs): 225 | """ 226 | All Axes to be added use the rectangle specification 227 | (left, bottom, width, height) 228 | """ 229 | input_image = inputs.reshape(self.n_image_rows, self.n_image_cols) 230 | absolute_pos = ( 231 | self.left_border, 232 | self.input_image_bottom, 233 | self.input_image_width, 234 | self.input_image_height) 235 | ax_input = self.add_image_axes(fig, image_axes, absolute_pos) 236 | ax_input.imshow( 237 | input_image, 238 | vmin=self.im_vmin, 239 | vmax=self.im_vmax, 240 | cmap=self.cmap, 241 | zorder=6, 242 | ) 243 | image_axes.append([ax_input]) 244 | 245 | def add_node_images(self, fig, i_layer, image_axes, nn, inputs): 246 | """ 247 | Add in all the node images for a single layer 248 | """ 249 | node_activities = nn.forward_prop_to_layer(inputs, i_layer) 250 | node_image_left = ( 251 | self.left_border 252 | + self.input_image_width 253 | + i_layer * self.node_image_width 254 | + (i_layer + 1) * self.between_layer_gap 255 | ) 256 | n_nodes = self.n_nodes[i_layer] 257 | total_layer_height = ( 258 | n_nodes * self.node_image_height 259 | + (n_nodes - 1) * self.between_node_gap 260 | ) 261 | layer_bottom = (self.figure_height - total_layer_height) / 2 262 | layer_axes = [] 263 | for i_node in range(n_nodes): 264 | node_signal = np.zeros(n_nodes) 265 | node_signal[i_node] = 1 266 | node_signature = nn.forward_prop_from_layer(node_signal, i_layer) 267 | node_image = node_signature.reshape( 268 | self.n_image_rows, self.n_image_cols) 269 | node_image *= node_activities[i_node] 270 | 271 | node_image_bottom = ( 272 | layer_bottom + i_node * ( 273 | self.node_image_height + self.between_node_gap)) 274 | 275 | absolute_pos = ( 276 | node_image_left, 277 | node_image_bottom, 278 | self.node_image_width, 279 | self.node_image_height) 280 | ax = self.add_image_axes(fig, image_axes, absolute_pos) 281 | ax.imshow( 282 | node_image, 283 | vmin=self.im_vmin, 284 | vmax=self.im_vmax, 285 | cmap=self.cmap, 286 | zorder=6, 287 | ) 288 | layer_axes.append(ax) 289 | image_axes.append(layer_axes) 290 | 291 | def add_output_image(self, fig, image_axes, nn, inputs): 292 | outputs = nn.forward_prop(inputs) 293 | output_image = outputs.reshape(self.n_image_rows, self.n_image_cols) 294 | output_image_left = ( 295 | self.figure_width 296 | - self.input_image_width 297 | - self.right_border 298 | ) 299 | absolute_pos = ( 300 | output_image_left, 301 | self.input_image_bottom, 302 | self.input_image_width, 303 | self.input_image_height) 304 | ax_output = self.add_image_axes(fig, image_axes, absolute_pos) 305 | ax_output.imshow( 306 | output_image, 307 | vmin=self.im_vmin, 308 | vmax=self.im_vmax, 309 | cmap=self.cmap, 310 | zorder=6, 311 | ) 312 | image_axes.append([ax_output]) 313 | 314 | def add_error_image(self, fig, image_axes, nn, inputs): 315 | outputs = nn.forward_prop(inputs) 316 | errors = inputs - outputs 317 | error_image = errors.reshape(self.n_image_rows, self.n_image_cols) 318 | absolute_pos = ( 319 | self.error_image_left, 320 | self.error_image_bottom, 321 | self.error_image_width, 322 | self.error_image_height) 323 | ax_error = self.add_image_axes(fig, image_axes, absolute_pos) 324 | ax_error.imshow( 325 | error_image, 326 | vmin=self.im_vmin, 327 | vmax=self.im_vmax, 328 | cmap=self.error_cmap, 329 | zorder=6, 330 | ) 331 | 332 | def add_image_axes(self, fig, image_axes, absolute_pos): 333 | """ 334 | Locate the Axes for the image corresponding to this node 335 | within the Figure. 336 | 337 | absolute_pos: Tuple of 338 | (left_position, bottom_position, width, height) 339 | in inches on the Figure. 340 | """ 341 | scaled_pos = ( 342 | absolute_pos[0] / self.figure_width, 343 | absolute_pos[1] / self.figure_height, 344 | absolute_pos[2] / self.figure_width, 345 | absolute_pos[3] / self.figure_height) 346 | ax = fig.add_axes(scaled_pos) 347 | ax.tick_params(bottom=False, top=False, left=False, right=False) 348 | ax.tick_params( 349 | labelbottom=False, 350 | labeltop=False, 351 | labelleft=False, 352 | labelright=False) 353 | ax.spines["top"].set_color(self.tan) 354 | ax.spines["bottom"].set_color(self.tan) 355 | ax.spines["left"].set_color(self.tan) 356 | ax.spines["right"].set_color(self.tan) 357 | return ax 358 | 359 | def add_layer_connections(self, ax_boss, image_axes): 360 | """ 361 | Add in the connectors between all the layers 362 | Treat the input image as the first layer and 363 | the output layer as the last. 364 | """ 365 | for i_start_layer in range(len(image_axes) - 1): 366 | n_start_nodes = len(image_axes[i_start_layer]) 367 | n_end_nodes = len(image_axes[i_start_layer + 1]) 368 | x_start = image_axes[i_start_layer][0].get_position().x1 369 | x_end = image_axes[i_start_layer + 1][0].get_position().x0 370 | 371 | for i_start_ax, ax_start in enumerate(image_axes[i_start_layer]): 372 | ax_start_pos = ax_start.get_position() 373 | y_start_min = ax_start_pos.y0 374 | y_start_max = ax_start_pos.y1 375 | start_spacing = (y_start_max - y_start_min) / (n_end_nodes + 1) 376 | 377 | for i_end_ax, ax_end in enumerate( 378 | image_axes[i_start_layer + 1] 379 | ): 380 | ax_end_pos = ax_end.get_position() 381 | y_end_min = ax_end_pos.y0 382 | y_end_max = ax_end_pos.y1 383 | end_spacing = (y_end_max - y_end_min) / (n_start_nodes + 1) 384 | 385 | # Spread out y_start and y_end a bit 386 | y_start = y_start_min + start_spacing * (i_end_ax + 1) 387 | y_end = y_end_min + end_spacing * (i_start_ax + 1) 388 | self.plot_connection( 389 | ax_boss, x_start, x_end, y_start, y_end) 390 | 391 | def plot_connection(self, ax_boss, x0, x1, y0, y1): 392 | """ 393 | Represent the weights connecting nodes in one layer 394 | to nodes in the next. 395 | """ 396 | weight = np.random.sample() * 2 - 1 397 | x = np.linspace(x0, x1, num=50) 398 | y = y0 + (y1 - y0) * ( 399 | -np.cos( 400 | np.pi * (x - x0) / (x1 - x0) 401 | ) + 1) / 2 402 | if weight > 0: 403 | conn_color = self.tan 404 | else: 405 | conn_color = self.blue 406 | ax_boss.plot(x, y, color=conn_color, linewidth=weight) 407 | 408 | def save_nn_viz(self, fig, postfix="0"): 409 | """ 410 | Generate a new filename for each step of the process. 411 | """ 412 | base_name = "nn_viz_" 413 | filename = base_name + postfix + ".png" 414 | filepath = os.path.join(self.savedir, filename) 415 | fig.savefig( 416 | filepath, 417 | edgecolor=fig.get_edgecolor(), 418 | facecolor=fig.get_facecolor(), 419 | dpi=self.DPI, 420 | ) 421 | --------------------------------------------------------------------------------