├── .gitignore ├── README.md └── Spark ├── Spark.py ├── __init__.py ├── activations ├── Activations.py └── __init__.py ├── layers ├── Activation.py ├── Dense.py ├── Layer.py └── __init__.py ├── losses ├── Losses.py └── __init__.py ├── optimizers ├── Optimizers.py └── __init__.py └── util ├── __init__.py ├── diff.py └── util.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | __pycache__ 3 | test.py 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Spark 2 | 3 | Flexible AI Library 4 | 5 | ### Usage 6 | 7 | ```python 8 | from Spark import Spark 9 | from Spark.layers import Dense 10 | ``` 11 | 12 | ### License 13 | 14 | Licensed under the [MIT License](https://kbrsh.github.io/license) by [Kabir Shah](https://kabir.ml) 15 | -------------------------------------------------------------------------------- /Spark/Spark.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from .losses import Losses 3 | from .optimizers import Optimizers 4 | 5 | class Spark(object): 6 | def __init__(self, inputs, outputs, learningRate=0.01, loss="Mean Squared", optimizer="Vanilla", layers=[]): 7 | # Inputs 8 | self.X = inputs 9 | 10 | # Outputs 11 | self.y = outputs 12 | 13 | # Loss 14 | self.loss, self.lossPrime = Losses(loss) 15 | 16 | # Layers 17 | self.layers = layers 18 | 19 | # Optimizers 20 | optimizer = Optimizers(optimizer) 21 | for layer in layers: 22 | layer.addOptimizer(learningRate, optimizer) 23 | 24 | def run(self, epochs=10): 25 | inputs = self.X 26 | outputs = self.y 27 | loss = self.loss 28 | lossPrime = self.lossPrime 29 | layers = self.layers 30 | 31 | for epoch in range(epochs): 32 | lastInput = inputs 33 | 34 | # Forward Propagate Layers 35 | for layer in layers: 36 | lastInput = layer.forward(lastInput) 37 | 38 | # Backward Propagate Layers 39 | gradient = lossPrime(lastInput, outputs) 40 | 41 | for layer in reversed(layers): 42 | gradient = layer.backward(gradient, loss, outputs) 43 | 44 | print("Epoch: " + str(epoch)) 45 | print("Loss: " + str(loss(lastInput, outputs))) 46 | print() 47 | 48 | def predict(self, y): 49 | for layer in self.layers: 50 | y = layer.forward(y) 51 | 52 | return y 53 | -------------------------------------------------------------------------------- /Spark/__init__.py: -------------------------------------------------------------------------------- 1 | from .Spark import Spark 2 | -------------------------------------------------------------------------------- /Spark/activations/Activations.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | # Linear Activations 4 | def Linear(x): 5 | return x 6 | 7 | def LinearPrime(x): 8 | return np.ones(x.shape) 9 | 10 | # Sigmoid Activations 11 | def Sigmoid(x): 12 | return 1 / (1 + np.exp(-x)) 13 | 14 | def SigmoidPrime(x): 15 | return np.exp(-x) / ((1 + np.exp(-x)) ** 2) 16 | 17 | # Tanh Activations 18 | def Tanh(x): 19 | return np.tanh(x) 20 | 21 | def TanhPrime(x): 22 | return 1 / (np.cosh(x)**2) 23 | 24 | # Softmax Activations 25 | def Softmax(x): 26 | exp = np.exp(x - np.max(x)) 27 | return exp / np.sum(exp, axis=1, keepdims=True) 28 | 29 | def SoftmaxPrime(x): 30 | return x / x.shape[0] 31 | 32 | # Global Activations Dictionary 33 | allActivations = globals() 34 | 35 | # Get Activation Function 36 | def Activations(name): 37 | name = "".join(name.title().split(" ")) 38 | return allActivations[name], allActivations[name + "Prime"] 39 | -------------------------------------------------------------------------------- /Spark/activations/__init__.py: -------------------------------------------------------------------------------- 1 | from .Activations import Activations 2 | -------------------------------------------------------------------------------- /Spark/layers/Activation.py: -------------------------------------------------------------------------------- 1 | from .Layer import Layer 2 | from ..activations import Activations 3 | 4 | import numpy as np 5 | 6 | class Activation(Layer): 7 | def __init__(self, name): 8 | # Setup Activation Function 9 | self.activation, self.activationPrime = Activations(name) 10 | 11 | def forward(self, X): 12 | self.o = self.activation(X) 13 | return self.o 14 | 15 | def backward(self, dO): 16 | return np.multiply(dO, self.activationPrime(self.o)) 17 | -------------------------------------------------------------------------------- /Spark/layers/Dense.py: -------------------------------------------------------------------------------- 1 | from .Layer import Layer 2 | from ..activations import Activations 3 | 4 | import numpy as np 5 | 6 | class Dense(Layer): 7 | def __init__(self, inputSize, outputSize, activation="Linear"): 8 | # Generate Weights 9 | self.W = np.random.randn(inputSize, outputSize) 10 | 11 | # Generate Biases 12 | self.b = np.zeros((1, outputSize)) 13 | 14 | # Setup Activation Function 15 | self.activation, self.activationPrime = Activations(activation) 16 | 17 | def addOptimizer(self, learningRate, optimizer): 18 | self.optimizer = optimizer(learningRate) 19 | 20 | def forward(self, X): 21 | self.X = X 22 | self.o = self.activation(np.dot(X, self.W) + self.b) 23 | return self.o 24 | 25 | def backward(self, gradient): 26 | X = self.X 27 | XShape = X.shape 28 | 29 | W = self.W 30 | WShape = W.shape 31 | 32 | b = self.b 33 | 34 | dO = np.multiply(gradient, self.activationPrime(self.o)) 35 | dW = np.add(np.dot(X, np.ones(WShape)), np.dot(np.zeros(XShape), W)) 36 | dB = np.add(dW, np.ones(b.shape)) 37 | 38 | optimizer = self.optimizer 39 | self.W = optimizer.optimize(W, np.multiply(dO, dW)) 40 | self.b = optimizer.optimize(b, np.multiply(dO, dB)) 41 | 42 | return np.add(np.dot(X, np.zeros(WShape)), np.dot(np.ones(XShape), W)) 43 | -------------------------------------------------------------------------------- /Spark/layers/Layer.py: -------------------------------------------------------------------------------- 1 | class Layer(object): 2 | def addOptimizer(self, learningRate, optimizer): 3 | pass 4 | 5 | def forward(self, X): 6 | pass 7 | 8 | def backward(self, gradient): 9 | pass 10 | -------------------------------------------------------------------------------- /Spark/layers/__init__.py: -------------------------------------------------------------------------------- 1 | from .Dense import Dense 2 | from .Activation import Activation 3 | -------------------------------------------------------------------------------- /Spark/losses/Losses.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | # Mean Squared 4 | def MeanSquared(o, y): 5 | return np.mean(np.square(o - y)) 6 | 7 | def MeanSquaredPrime(o, y): 8 | return o - y 9 | 10 | # Cross Entropy 11 | def CrossEntropy(o, y): 12 | return -np.log(o[range(o.shape[0]), np.argmax(y, axis=1)]) 13 | 14 | def CrossEntropyPrime(o, y): 15 | do = np.copy(o) 16 | do[range(o.shape[0]), np.argmax(y, axis=1)] -= 1 17 | return do 18 | 19 | # Global Loss Dictionary 20 | allLosses = globals() 21 | 22 | # Get Loss Function 23 | def Losses(name): 24 | name = "".join(name.title().split(" ")) 25 | return allLosses[name], allLosses[name + "Prime"] 26 | -------------------------------------------------------------------------------- /Spark/losses/__init__.py: -------------------------------------------------------------------------------- 1 | from .Losses import Losses 2 | -------------------------------------------------------------------------------- /Spark/optimizers/Optimizers.py: -------------------------------------------------------------------------------- 1 | class Optimizer(object): 2 | def optimize(self, weight, gradient): 3 | pass 4 | 5 | class VanillaOptimizer(Optimizer): 6 | def __init__(self, learningRate): 7 | # Learning Rate 8 | self.learningRate = learningRate 9 | 10 | def optimize(self, weight, gradient): 11 | return weight - (self.learningRate * gradient) 12 | 13 | allOptimizers = globals() 14 | 15 | def Optimizers(name): 16 | return allOptimizers["".join(name.title().split(" ")).title() + "Optimizer"] 17 | -------------------------------------------------------------------------------- /Spark/optimizers/__init__.py: -------------------------------------------------------------------------------- 1 | from .Optimizers import Optimizers 2 | -------------------------------------------------------------------------------- /Spark/util/__init__.py: -------------------------------------------------------------------------------- 1 | from .util import * 2 | -------------------------------------------------------------------------------- /Spark/util/diff.py: -------------------------------------------------------------------------------- 1 | import numbers 2 | import numpy as np 3 | 4 | outputVariableCount = 0 5 | def outputVariable(): 6 | global outputVariableCount 7 | outputVariableCount += 1 8 | return "o" + str(outputVariableCount) 9 | 10 | def isNumber(num): 11 | return isinstance(num, numbers.Real) 12 | 13 | def ADGradientConstant(constant): 14 | if isNumber(constant): 15 | return constant 16 | else: 17 | return "sp.array(" + np.array2string(arr, separator=", ") + ")" 18 | 19 | def ADGradientOperation(op, a, b): 20 | if len(a) > 9 and a[0:8] == "sp.zeros": 21 | if op == "add": 22 | return b 23 | elif op == "multiply": 24 | return a 25 | 26 | if len(a) > 8 and a[0:7] == "sp.ones": 27 | if op == "multiply": 28 | return b 29 | 30 | if len(b) > 9 and b[0:8] == "sp.zeros": 31 | if op == "add" or op == "subtract": 32 | return a 33 | elif op == "multiply": 34 | return b 35 | 36 | if len(b) > 8 and b[0:7] == "sp.ones": 37 | if op == "multiply": 38 | return a 39 | 40 | return "sp." + op + "(" + a + ", " + b + ")" 41 | 42 | class ADNode(object): 43 | def __init__(self, name, value): 44 | self.name = name 45 | self.value = value 46 | 47 | def __str__(self): 48 | return str(self.value) 49 | 50 | def compute(self): 51 | operation = self.operation 52 | inputs = operation.inputs 53 | 54 | for node in inputs: 55 | if type(node) is ADNode: 56 | node.compute() 57 | 58 | newValue = operation.compute() 59 | self.value = newValue 60 | return newValue 61 | 62 | def add(self, item): 63 | if type(item) is ADNode: 64 | node = ADNode(outputVariable(), 0) 65 | operation = AddNode(node, [self, item]) 66 | node.operation = operation 67 | node.value = operation.compute() 68 | 69 | return node 70 | else: 71 | node = ADNode(outputVariable(), 0) 72 | operation = AddConstant(node, [self, item]) 73 | node.operation = operation 74 | node.value = operation.compute() 75 | 76 | return node 77 | 78 | def subtract(self, item): 79 | if type(item) is ADNode: 80 | node = ADNode(outputVariable(), 0) 81 | operation = SubtractNode(node, [self, item]) 82 | node.operation = operation 83 | node.value = operation.compute() 84 | 85 | return node 86 | else: 87 | node = ADNode(outputVariable(), 0) 88 | operation = SubtractConstant(node, [self, item]) 89 | node.operation = operation 90 | node.value = operation.compute() 91 | 92 | return node 93 | 94 | def multiply(self, item): 95 | if type(item) is ADNode: 96 | node = ADNode(outputVariable(), 0) 97 | operation = MultiplyNode(node, [self, item]) 98 | node.operation = operation 99 | node.value = operation.compute() 100 | 101 | return node 102 | else: 103 | node = ADNode(outputVariable(), 0) 104 | operation = MultiplyConstant(node, [self, item]) 105 | node.operation = operation 106 | node.value = operation.compute() 107 | 108 | return node 109 | 110 | def dot(self, item): 111 | if type(item) is ADNode: 112 | node = ADNode(outputVariable(), 0) 113 | operation = DotNode(node, [self, item]) 114 | node.operation = operation 115 | node.value = operation.compute() 116 | 117 | return node 118 | else: 119 | node = ADNode(outputVariable(), 0) 120 | operation = DotConstant(node, [self, item]) 121 | node.operation = operation 122 | node.value = operation.compute() 123 | 124 | return node 125 | 126 | def graph(self, level=0): 127 | name = self.name 128 | nextLevel = level + 1 129 | indent = " |" * level + " " 130 | graph = "\x1b[34m" + self.operation.__class__.__name__ + "\x1b[0m \"" + name + "\"" 131 | 132 | for node in self.operation.inputs: 133 | if type(node) is ADNode: 134 | graph += "\n" + indent + "| " + node.graph(level=nextLevel) 135 | else: 136 | graph += "\n" + indent + "| \x1b[34mConstant\x1b[0m " + str(node) 137 | 138 | return graph 139 | 140 | class Operation(object): 141 | def compute(self): 142 | pass 143 | def gradient(self, respect): 144 | pass 145 | 146 | class Variable(Operation): 147 | def __init__(self, node): 148 | self.node = node 149 | self.inputs = [] 150 | 151 | def compute(self): 152 | return self.node.value 153 | 154 | def gradient(self, respect): 155 | node = self.node 156 | if respect == node: 157 | if isNumber(node.value): 158 | return "1" 159 | else: 160 | return "sp.ones(" + node.name + ".shape)" 161 | else: 162 | if isNumber(node.value): 163 | return "0" 164 | else: 165 | return "sp.zeros(" + node.name + ".shape)" 166 | 167 | class AddConstant(Operation): 168 | def __init__(self, node, inputs): 169 | self.node = node 170 | self.inputs = inputs 171 | 172 | def compute(self): 173 | inputs = self.inputs 174 | return np.add(inputs[0].value, inputs[1]) 175 | 176 | def gradient(self, respect): 177 | return self.inputs[0].operation.gradient(respect) 178 | 179 | class AddNode(Operation): 180 | def __init__(self, node, inputs): 181 | self.node = node 182 | self.inputs = inputs 183 | 184 | def compute(self): 185 | inputs = self.inputs 186 | return np.add(inputs[0].value, inputs[1].value) 187 | 188 | def gradient(self, respect): 189 | inputs = self.inputs 190 | return ADGradientOperation("add", inputs[0].operation.gradient(respect), inputs[1].operation.gradient(respect)) 191 | 192 | class SubtractConstant(Operation): 193 | def __init__(self, node, inputs): 194 | self.node = node 195 | self.inputs = inputs 196 | 197 | def compute(self): 198 | inputs = self.inputs 199 | return np.subtract(inputs[0].value, inputs[1]) 200 | 201 | def gradient(self, respect): 202 | return self.inputs[0].operation.gradient(respect) 203 | 204 | class SubtractNode(Operation): 205 | def __init__(self, node, inputs): 206 | self.node = node 207 | self.inputs = inputs 208 | 209 | def compute(self): 210 | inputs = self.inputs 211 | return np.subtract(inputs[0].value, inputs[1].value) 212 | 213 | def gradient(self, respect): 214 | inputs = self.inputs 215 | return ADGradientOperation("subtract", inputs[0].operation.gradient(respect), inputs[1].operation.gradient(respect)) 216 | 217 | class MultiplyConstant(Operation): 218 | def __init__(self, node, inputs): 219 | self.node = node 220 | self.inputs = inputs 221 | 222 | def compute(self): 223 | inputs = self.inputs 224 | return np.multiply(inputs[0].value, inputs[1]) 225 | 226 | def gradient(self, respect): 227 | inputs = self.inputs 228 | return ADGradientOperation("multiply", inputs[0].operation.gradient(respect), ADGradientConstant(inputs[1])) 229 | 230 | class MultiplyNode(Operation): 231 | def __init__(self, node, inputs): 232 | self.node = node 233 | self.inputs = inputs 234 | 235 | def compute(self): 236 | inputs = self.inputs 237 | return np.multiply(inputs[0].value, inputs[1].value) 238 | 239 | def gradient(self, respect): 240 | inputs = self.inputs 241 | return ADGradientOperation("add", 242 | ADGradientOperation("multiply", inputs[0].name, inputs[1].operation.gradient(respect)), 243 | ADGradientOperation("multiply", inputs[0].operation.gradient(respect), inputs[1].name) 244 | ) 245 | 246 | class DotConstant(Operation): 247 | def __init__(self, node, inputs): 248 | self.node = node 249 | self.inputs = inputs 250 | 251 | def compute(self): 252 | inputs = self.inputs 253 | return np.dot(inputs[0].value, inputs[1]) 254 | 255 | def gradient(self, respect): 256 | inputs = self.inputs 257 | return ADGradientOperation("dot", inputs[0].operation.gradient(respect), ADGradientConstant(inputs[1])) 258 | 259 | class DotNode(Operation): 260 | def __init__(self, node, inputs): 261 | self.node = node 262 | self.inputs = inputs 263 | 264 | def compute(self): 265 | inputs = self.inputs 266 | return np.dot(inputs[0].value, inputs[1].value) 267 | 268 | def gradient(self, respect): 269 | inputs = self.inputs 270 | return ADGradientOperation("add", 271 | ADGradientOperation("dot", inputs[0].name, inputs[1].operation.gradient(respect)), 272 | ADGradientOperation("dot", inputs[0].operation.gradient(respect), inputs[1].name) 273 | ) 274 | 275 | def variable(name, value): 276 | node = ADNode(name, value) 277 | node.operation = Variable(node) 278 | return node 279 | 280 | def gradient(output, respect): 281 | return output.operation.gradient(respect) 282 | -------------------------------------------------------------------------------- /Spark/util/util.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from .diff import variable, gradient 3 | 4 | def array(*args): 5 | return np.array(*args) 6 | 7 | def random(*args): 8 | return np.random.randn(*args) 9 | 10 | def zeros(*args): 11 | return np.zeros(*args) 12 | 13 | def ones(*args): 14 | return np.ones(*args) 15 | 16 | def add(arr, item): 17 | result = arr.add(item) 18 | return result 19 | 20 | def subtract(arr, item): 21 | result = arr.subtract(item) 22 | return result 23 | 24 | def multiply(arr, item): 25 | result = arr.multiply(item) 26 | return result 27 | 28 | def dot(arr, item): 29 | result = arr.dot(item) 30 | return result 31 | --------------------------------------------------------------------------------