├── .gitignore ├── .vscode └── settings.json ├── README.md ├── dl_numpy.py ├── test_library.py └── utilities.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/utilities.cpython-37.pyc 2 | __pycache__/dl_numpy.cpython-37.pyc 3 | classes.jpeg 4 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.linting.enabled": true 3 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dll_numpy 2 | Deep Learning framework implemented from scratch in python using Numpy package. 3 | -------------------------------------------------------------------------------- /dl_numpy.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | class Tensor(): 4 | def __init__(self,shape): 5 | self.data = np.ndarray(shape,np.float32) 6 | self.grad = np.ndarray(shape,np.float32) 7 | 8 | class Function(object): 9 | def forward(self): 10 | raise NotImplementedError 11 | 12 | def backward(self): 13 | raise NotImplementedError 14 | 15 | def getParams(self): 16 | return [] 17 | 18 | class Optimizer(object): 19 | def __init__(self,parameters): 20 | self.parameters = parameters 21 | 22 | def step(self): 23 | raise NotImplementedError 24 | 25 | def zeroGrad(self): 26 | for p in self.parameters: 27 | p.grad = 0. 28 | 29 | class Linear(Function): 30 | def __init__(self,in_nodes,out_nodes): 31 | self.weights = Tensor((in_nodes,out_nodes)) 32 | self.bias = Tensor((1,out_nodes)) 33 | self.type = 'linear' 34 | 35 | def forward(self,x): 36 | output = np.dot(x,self.weights.data)+self.bias.data 37 | self.input = x 38 | return output 39 | 40 | def backward(self,d_y): 41 | self.weights.grad += np.dot(self.input.T,d_y) 42 | self.bias.grad += np.sum(d_y,axis=0,keepdims=True) 43 | grad_input = np.dot(d_y,self.weights.data.T) 44 | return grad_input 45 | 46 | def getParams(self): 47 | return [self.weights,self.bias] 48 | 49 | class SoftmaxWithLoss(Function): 50 | def __init__(self): 51 | self.type = 'normalization' 52 | 53 | def forward(self,x,target): 54 | unnormalized_proba = np.exp(x-np.max(x,axis=1,keepdims=True)) 55 | self.proba = unnormalized_proba/np.sum(unnormalized_proba,axis=1,keepdims=True) 56 | self.target = target 57 | loss = -np.log(self.proba[range(len(target)),target]) 58 | return loss.mean() 59 | 60 | def backward(self): 61 | gradient = self.proba 62 | gradient[range(len(self.target)),self.target]-=1.0 63 | gradient/=len(self.target) 64 | return gradient 65 | 66 | class ReLU(Function): 67 | def __init__(self,inplace=True): 68 | self.type = 'activation' 69 | self.inplace = inplace 70 | 71 | def forward(self,x): 72 | if self.inplace: 73 | x[x<0] = 0. 74 | self.activated = x 75 | else: 76 | self.activated = x*(x>0) 77 | 78 | return self.activated 79 | 80 | def backward(self,d_y): 81 | return d_y*(self.activated>0) 82 | 83 | class SGD(Optimizer): 84 | def __init__(self,parameters,lr=.001,weight_decay=0.0,momentum = .9): 85 | super().__init__(parameters) 86 | self.lr = lr 87 | self.weight_decay = weight_decay 88 | self.momentum = momentum 89 | self.velocity = [] 90 | for p in parameters: 91 | self.velocity.append(np.zeros_like(p.grad)) 92 | 93 | def step(self): 94 | for p,v in zip(self.parameters,self.velocity): 95 | v = self.momentum*v+p.grad+self.weight_decay*p.data 96 | p.data=p.data-self.lr*v 97 | 98 | ##############################Library Implementation **END############################## -------------------------------------------------------------------------------- /test_library.py: -------------------------------------------------------------------------------- 1 | import dl_numpy as DL 2 | import utilities 3 | import numpy as np 4 | 5 | if __name__=="__main__": 6 | batch_size = 20 7 | num_epochs = 200 8 | samples_per_class = 100 9 | num_classes = 3 10 | hidden_units = 100 11 | data,target = utilities.genSpiralData(samples_per_class,num_classes) 12 | model = utilities.Model() 13 | model.add(DL.Linear(2,hidden_units)) 14 | model.add(DL.ReLU()) 15 | model.add(DL.Linear(hidden_units,num_classes)) 16 | optim = DL.SGD(model.parameters,lr=1.0,weight_decay=0.001,momentum=.9) 17 | loss_fn = DL.SoftmaxWithLoss() 18 | model.fit(data,target,batch_size,num_epochs,optim,loss_fn) 19 | predicted_labels = np.argmax(model.predict(data),axis=1) 20 | accuracy = np.sum(predicted_labels==target)/len(target) 21 | print("Model Accuracy = {}".format(accuracy)) 22 | utilities.plot2DDataWithDecisionBoundary(data,target,model) 23 | 24 | 25 | -------------------------------------------------------------------------------- /utilities.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | 5 | class Model(): 6 | def __init__(self): 7 | self.computation_graph = [] 8 | self.parameters = [] 9 | 10 | def add(self,layer): 11 | self.computation_graph.append(layer) 12 | self.parameters+=layer.getParams() 13 | 14 | def __innitializeNetwork(self): 15 | for f in self.computation_graph: 16 | if f.type=='linear': 17 | weights,bias = f.getParams() 18 | weights.data = .01*np.random.randn(weights.data.shape[0],weights.data.shape[1]) 19 | bias.data = 0. 20 | 21 | def fit(self,data,target,batch_size,num_epochs,optimizer,loss_fn): 22 | loss_history = [] 23 | self.__innitializeNetwork() 24 | data_gen = DataGenerator(data,target,batch_size) 25 | itr = 0 26 | for epoch in range(num_epochs): 27 | for X,Y in data_gen: 28 | optimizer.zeroGrad() 29 | for f in self.computation_graph: X=f.forward(X) 30 | loss = loss_fn.forward(X,Y) 31 | grad = loss_fn.backward() 32 | for f in self.computation_graph[::-1]: grad = f.backward(grad) 33 | loss_history+=[loss] 34 | print("Loss at epoch = {} and iteration = {}: {}".format(epoch,itr,loss_history[-1])) 35 | itr+=1 36 | optimizer.step() 37 | 38 | return loss_history 39 | 40 | def predict(self,data): 41 | X = data 42 | for f in self.computation_graph: X = f.forward(X) 43 | return X 44 | 45 | def genSpiralData(points_per_class,num_classes): 46 | data = np.ndarray((points_per_class*num_classes,2),np.float32) 47 | target = np.ndarray((points_per_class*num_classes,),np.uint8) 48 | r = np.linspace(0,1,points_per_class) 49 | radians_per_class = 2*np.pi/num_classes 50 | for i in range(num_classes): 51 | t = np.linspace(i*radians_per_class,(i+1.5)*radians_per_class,points_per_class)+0.1*np.random.randn(points_per_class) 52 | data[i*points_per_class:(i+1)*points_per_class] = np.c_[r*np.sin(t),r*np.cos(t)] 53 | target[i*points_per_class:(i+1)*points_per_class] = i 54 | 55 | return data,target 56 | 57 | 58 | class DataGenerator(): 59 | def __init__(self, data, target, batch_size, shuffle=True): 60 | self.shuffle = shuffle 61 | if shuffle: 62 | shuffled_indices = np.random.permutation(len(data)) 63 | else: 64 | shuffled_indices = range(len(data)) 65 | 66 | self.data = data[shuffled_indices] 67 | self.target = target[shuffled_indices] 68 | self.batch_size = batch_size 69 | self.num_batches = int(np.ceil(data.shape[0]/batch_size)) 70 | self.counter = 0 71 | 72 | 73 | def __iter__(self): 74 | return self 75 | 76 | def __next__(self): 77 | if self.counter