├── README.md └── network_optimisation.py /README.md: -------------------------------------------------------------------------------- 1 | # Network-Optimisation 2 | Neural Network architecture optimisation using genetic algorithms. 3 | -------------------------------------------------------------------------------- /network_optimisation.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import random 3 | from keras.models import Sequential 4 | from keras.layers import Dense, Activation 5 | from keras.datasets import mnist 6 | from keras.utils import to_categorical 7 | 8 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 9 | x_train = x_train.reshape(60000, 784).astype('float32')[:1000] / 255 10 | x_test = x_test.reshape(10000, 784).astype('float32')[:1000] / 255 11 | y_train = to_categorical(y_train, 10)[:1000] 12 | y_test = to_categorical(y_test, 10)[:1000] 13 | 14 | classes = 10 15 | batch_size = 64 16 | population = 20 17 | generations = 100 18 | threshold = 0.995 19 | 20 | def serve_model(epochs, units1, act1, units2, act2, classes, act3, loss, opt, xtrain, ytrain, summary=False): 21 | model = Sequential() 22 | model.add(Dense(units1, input_shape=[784,])) 23 | model.add(Activation(act1)) 24 | model.add(Dense(units2)) 25 | model.add(Activation(act2)) 26 | model.add(Dense(classes)) 27 | model.add(Activation(act3)) 28 | model.compile(loss=loss, optimizer=opt, metrics=['acc']) 29 | if summary: 30 | model.summary() 31 | 32 | model.fit(xtrain, ytrain, batch_size=batch_size, epochs=epochs, verbose=0) 33 | 34 | return model 35 | 36 | class Network(): 37 | def __init__(self): 38 | self._epochs = np.random.randint(1, 15) 39 | 40 | self._units1 = np.random.randint(1, 500) 41 | self._units2 = np.random.randint(1, 500) 42 | 43 | self._act1 = random.choice(['sigmoid', 'relu', 'softmax', 'tanh', 'elu', 'selu', 'linear']) 44 | self._act2 = random.choice(['sigmoid', 'relu', 'softmax', 'tanh', 'elu', 'selu', 'linear']) 45 | self._act3 = random.choice(['sigmoid', 'relu', 'softmax', 'tanh', 'elu', 'selu', 'linear']) 46 | 47 | self._loss = random.choice([ 48 | 'categorical_crossentropy', 49 | 'binary_crossentropy', 50 | 'mean_squared_error', 51 | 'mean_absolute_error', 52 | 'sparse_categorical_crossentropy' 53 | ]) 54 | self._opt = random.choice(['sgd', 'rmsprop', 'adagrad', 'adadelta', 'adam', 'adamax', 'nadam']) 55 | 56 | self._accuracy = 0 57 | 58 | def init_hyperparams(self): 59 | hyperparams = { 60 | 'epochs': self._epochs, 61 | 'units1': self._units1, 62 | 'act1': self._act1, 63 | 'units2': self._units2, 64 | 'act2': self._act2, 65 | 'act3': self._act3, 66 | 'loss': self._loss, 67 | 'optimizer': self._opt 68 | } 69 | return hyperparams 70 | 71 | def init_networks(population): 72 | return [Network() for _ in range(population)] 73 | 74 | def fitness(networks): 75 | for network in networks: 76 | hyperparams = network.init_hyperparams() 77 | epochs = hyperparams['epochs'] 78 | units1 = hyperparams['units1'] 79 | act1 = hyperparams['act1'] 80 | units2 = hyperparams['units2'] 81 | act2 = hyperparams['act2'] 82 | act3 = hyperparams['act3'] 83 | loss = hyperparams['loss'] 84 | opt = hyperparams['optimizer'] 85 | 86 | try: 87 | model = serve_model(epochs, units1, act1, units2, act2, classes, act3, loss, opt, x_train, y_train) 88 | accuracy = model.evaluate(x_test, y_test, verbose=0)[1] 89 | network._accuracy = accuracy 90 | print ('Accuracy: {}'.format(network._accuracy)) 91 | except: 92 | network._accuracy = 0 93 | print ('Build failed.') 94 | 95 | return networks 96 | 97 | def selection(networks): 98 | networks = sorted(networks, key=lambda network: network._accuracy, reverse=True) 99 | networks = networks[:int(0.2 * len(networks))] 100 | 101 | return networks 102 | 103 | def crossover(networks): 104 | offspring = [] 105 | for _ in range(int((population - len(networks)) / 2)): 106 | parent1 = random.choice(networks) 107 | parent2 = random.choice(networks) 108 | child1 = Network() 109 | child2 = Network() 110 | 111 | # Crossing over parent hyper-params 112 | child1._epochs = int(parent1._epochs/4) + int(parent2._epochs/2) 113 | child2._epochs = int(parent1._epochs/2) + int(parent2._epochs/4) 114 | 115 | child1._units1 = int(parent1._units1/4) + int(parent2._units1/2) 116 | child2._units1 = int(parent1._units1/2) + int(parent2._units1/4) 117 | 118 | child1._units2 = int(parent1._units2/4) + int(parent2._units2/2) 119 | child2._units2 = int(parent1._units2/2) + int(parent2._units2/4) 120 | 121 | child1._act1 = parent2._act2 122 | child2._act1 = parent1._act2 123 | 124 | child1._act2 = parent2._act1 125 | child2._act2 = parent1._act1 126 | 127 | child1._act3 = parent2._act2 128 | child2._act3 = parent1._act2 129 | 130 | offspring.append(child1) 131 | offspring.append(child2) 132 | 133 | networks.extend(offspring) 134 | 135 | return networks 136 | 137 | def mutate(networks): 138 | for network in networks: 139 | if np.random.uniform(0, 1) <= 0.1: 140 | network._epochs += np.random.randint(0,100) 141 | network._units1 += np.random.randint(0,100) 142 | network._units2 += np.random.randint(0,100) 143 | 144 | return networks 145 | 146 | def main(): 147 | networks = init_networks(population) 148 | 149 | for gen in range(generations): 150 | print ('Generation {}'.format(gen+1)) 151 | 152 | networks = fitness(networks) 153 | networks = selection(networks) 154 | networks = crossover(networks) 155 | networks = mutate(networks) 156 | 157 | for network in networks: 158 | if network._accuracy > threshold: 159 | print ('Threshold met') 160 | print (network.init_hyperparams()) 161 | print ('Best accuracy: {}'.format(network._accuracy)) 162 | exit(0) 163 | 164 | if __name__ == '__main__': 165 | main() --------------------------------------------------------------------------------