├── losses ├── hinge │ └── README.md ├── aam-softmax │ └── README.md ├── am-softmax │ └── README.md ├── softmax │ └── README.md ├── cross-entropy │ └── README.md ├── kullback-leibler │ └── README.md ├── prototypical │ └── README.md ├── angular-prototypical │ └── README.md ├── l1 │ ├── formulae.png │ ├── README.md │ └── L1.py ├── l2 │ ├── formulae.png │ ├── README.md │ └── L2.py └── README.md ├── optimizers ├── adagrad │ └── README.md ├── adam │ └── README.md ├── rmsprop │ └── README.md ├── sgd │ └── README.md └── README.md ├── architectures ├── VGG-19 │ └── README.md ├── alexnet │ └── README.md ├── lenet │ └── README.md ├── resnet-34 │ └── README.md ├── resnet-50 │ └── README.md └── README.md ├── non-linearities ├── relu │ ├── README.md │ ├── ReLU.png │ └── ReLU.py ├── sigmoid │ ├── README.md │ ├── sigmoid.jpg │ └── sigmoid.py ├── tanh │ ├── README.md │ ├── tanh.png │ └── tanh.py ├── LeakyReLU │ ├── README.md │ ├── LeakyReLU.png │ └── LeakyReLU.py └── README.md ├── normalization ├── batchnorm │ └── README.md ├── layernorm │ └── README.md └── README.md ├── README.md └── LICENSE /losses/hinge/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /losses/aam-softmax/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /losses/am-softmax/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /losses/softmax/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /optimizers/adagrad/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /optimizers/adam/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /optimizers/rmsprop/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /optimizers/sgd/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /architectures/VGG-19/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /architectures/alexnet/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /architectures/lenet/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /architectures/resnet-34/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /architectures/resnet-50/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /losses/cross-entropy/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /losses/kullback-leibler/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /losses/prototypical/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /non-linearities/relu/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /non-linearities/sigmoid/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /non-linearities/tanh/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /normalization/batchnorm/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /normalization/layernorm/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /losses/angular-prototypical/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /non-linearities/LeakyReLU/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /losses/l1/formulae.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alishdipani/Machine-Learning-concepts/HEAD/losses/l1/formulae.png -------------------------------------------------------------------------------- /losses/l2/formulae.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alishdipani/Machine-Learning-concepts/HEAD/losses/l2/formulae.png -------------------------------------------------------------------------------- /optimizers/README.md: -------------------------------------------------------------------------------- 1 | # Optimizers 2 | - [ ] Stochastic Gradient Descent 3 | - [ ] Adam 4 | - [ ] AdaGrad 5 | - [ ] RMSprop -------------------------------------------------------------------------------- /non-linearities/relu/ReLU.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alishdipani/Machine-Learning-concepts/HEAD/non-linearities/relu/ReLU.png -------------------------------------------------------------------------------- /non-linearities/tanh/tanh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alishdipani/Machine-Learning-concepts/HEAD/non-linearities/tanh/tanh.png -------------------------------------------------------------------------------- /normalization/README.md: -------------------------------------------------------------------------------- 1 | # Normalization 2 | - [ ] Batch Normalization 3 | - [ ] Layer Normalization 4 | - [ ] Group Normalization -------------------------------------------------------------------------------- /non-linearities/sigmoid/sigmoid.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alishdipani/Machine-Learning-concepts/HEAD/non-linearities/sigmoid/sigmoid.jpg -------------------------------------------------------------------------------- /non-linearities/LeakyReLU/LeakyReLU.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alishdipani/Machine-Learning-concepts/HEAD/non-linearities/LeakyReLU/LeakyReLU.png -------------------------------------------------------------------------------- /architectures/README.md: -------------------------------------------------------------------------------- 1 | # Architectures 2 | - [ ] LeNet 3 | - [ ] AlexNet 4 | - [ ] VGG-19 5 | - [ ] ResNet-18 6 | - [ ] ResNet-34 7 | - [ ] ResNet-50 -------------------------------------------------------------------------------- /non-linearities/README.md: -------------------------------------------------------------------------------- 1 | # Non-Linearities 2 | - [x] Sigmoid 3 | - [x] Tanh 4 | - [x] ReLU 5 | - [x] LeakyReLU 6 | - [ ] ELU 7 | - [ ] pReLU 8 | - [ ] GELU -------------------------------------------------------------------------------- /losses/README.md: -------------------------------------------------------------------------------- 1 | # Losses 2 | - [x] Mean Squared Error Loss/ L2 Loss 3 | - [x] Mean Absolute Error/ L1 Loss 4 | - [ ] Cross Entropy Loss 5 | - [ ] Hinge Loss 6 | - [ ] Kullback-Leibler Loss 7 | - [ ] Softmax 8 | - [ ] AM-Softmax 9 | - [ ] AAM-Softmax 10 | - [ ] Prototypical Loss 11 | - [ ] Angular Prototypical Loss 12 | - [ ] Si-SDR 13 | - [ ] Si-SNR -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Machine-Learning-concepts 2 | This repository contains explanation for various Machine Learning and Deep Learning concepts with Python code implementations from scratch and/or using PyTorch. 3 | 4 | ## [Non-Linearities](non-linearities) 5 | 6 | ## [Optimizers](optimizers) 7 | 8 | ## [Losses](losses) 9 | 10 | ## [Architectures](architectures) 11 | 12 | ## [Normalization](normalization) 13 | -------------------------------------------------------------------------------- /non-linearities/relu/ReLU.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | 5 | def ReLU(x): 6 | return np.maximum(0, x) 7 | 8 | 9 | def generate_graph(x, save_graph=False): 10 | plt.plot(x, ReLU(x)) 11 | if save_graph: 12 | plt.savefig("ReLU.jpg") 13 | else: 14 | plt.show() 15 | 16 | 17 | def main(): 18 | print("Generating graph") 19 | save_graph = False # save the graph (true) or show (false) 20 | generate_graph(np.linspace(-10, 10, 100), save_graph=save_graph) 21 | 22 | 23 | if __name__ == "__main__": 24 | main() 25 | -------------------------------------------------------------------------------- /non-linearities/sigmoid/sigmoid.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | 5 | def sigmoid(x): 6 | return 1 / (1 + np.exp(-x)) 7 | 8 | 9 | def generate_graph(x, save_graph=False): 10 | plt.plot(x, sigmoid(x)) 11 | if save_graph: 12 | plt.savefig("sigmoid.jpg") 13 | else: 14 | plt.show() 15 | 16 | 17 | def main(): 18 | print("Generating graph") 19 | save_graph = False # save the graph (true) or show (false) 20 | generate_graph(np.linspace(-10, 10, 100), save_graph=save_graph) 21 | 22 | 23 | if __name__ == "__main__": 24 | main() 25 | -------------------------------------------------------------------------------- /non-linearities/tanh/tanh.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | def tanh(x): 5 | return ((np.exp(x))-(np.exp(-x)))/((np.exp(x))+(np.exp(-x))) 6 | 7 | def generate_graph(x, save_graph=False): 8 | plt.plot(x,tanh(x)) 9 | if save_graph: 10 | plt.savefig("tanh.jpg") 11 | else: 12 | plt.show() 13 | 14 | def main(): 15 | print("Generating graph") 16 | save_graph = False # save the graph (true) or show (false) 17 | generate_graph(np.linspace(-10,10,100),save_graph=save_graph) 18 | 19 | if __name__ == '__main__': 20 | main() 21 | 22 | 23 | -------------------------------------------------------------------------------- /non-linearities/LeakyReLU/LeakyReLU.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | 5 | def LeakyReLU(x): 6 | return np.where(x>0, x, x*0.01) 7 | 8 | 9 | 10 | def generate_graph(x, save_graph=False): 11 | plt.plot(x, LeakyReLU(x)) 12 | if save_graph: 13 | plt.savefig("LeakyReLU.jpg") 14 | else: 15 | plt.show() 16 | 17 | 18 | def main(): 19 | print("Generating graph") 20 | save_graph = False # save the graph (true) or show (false) 21 | generate_graph(np.linspace(-200, 50, 10), save_graph=save_graph) 22 | 23 | 24 | if __name__ == "__main__": 25 | main() 26 | -------------------------------------------------------------------------------- /losses/l1/README.md: -------------------------------------------------------------------------------- 1 | ## Formula 2 | 3 | ![formulae](./formulae.png) 4 | 5 | 6 | ## Standard Implementations in ML Frameworks 7 | - [PyTorch: L1Loss](https://pytorch.org/docs/stable/generated/torch.nn.L1Loss.html) 8 | - [Keras: MeanAbsoluteError](https://keras.io/api/losses/regression_losses/#meanabsoluteerror-class) 9 | - [TF 2.3: MAE](https://www.tensorflow.org/api_docs/python/tf/keras/losses/MAE) 10 | 11 | ## Other Resources 12 | - [Analytics Vidhya Blog](https://www.analyticsvidhya.com/blog/2019/08/detailed-guide-7-loss-functions-machine-learning-python-code/) 13 | - [L1 vs. L2 Loss function](http://rishy.github.io/ml/2015/07/28/l1-vs-l2-loss/) 14 | - [Differences between L1 and L2 as Loss Function and Regularization](http://www.chioka.in/differences-between-l1-and-l2-as-loss-function-and-regularization/) 15 | -------------------------------------------------------------------------------- /losses/l2/README.md: -------------------------------------------------------------------------------- 1 | ## Formula 2 | 3 | ![formulae](./formulae.png) 4 | 5 | 6 | ## Standard Implementations in ML Frameworks 7 | - [PyTorch: MSE Loss](https://pytorch.org/docs/stable/generated/torch.nn.MSELoss.html) 8 | - [Keras: MeanSquaredError](https://keras.io/api/losses/regression_losses/#meansquarederror-class) 9 | - [TF 2.3: MeanSquaredError](https://www.tensorflow.org/api_docs/python/tf/keras/losses/MeanSquaredError) 10 | 11 | ## Other Resources 12 | - [Analytics Vidhya Blog](https://www.analyticsvidhya.com/blog/2019/08/detailed-guide-7-loss-functions-machine-learning-python-code/) 13 | - [L1 vs. L2 Loss function](http://rishy.github.io/ml/2015/07/28/l1-vs-l2-loss/) 14 | - [Differences between L1 and L2 as Loss Function and Regularization](http://www.chioka.in/differences-between-l1-and-l2-as-loss-function-and-regularization/) 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Alish Dipani 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /losses/l1/L1.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def L1Loss(y_predicted, y_ground_truth, reduction="None"): 5 | """returns l1 loss between two arrays 6 | 7 | :param y_predicted: array of predicted values 8 | :type y_predicted: ndarray 9 | :param y_ground_truth: array of ground truth values 10 | :type y_ground_truth: ndarray 11 | :param reduction: reduction mode, defaults to "mean" 12 | :type reduction: str, optional 13 | :return: l1-loss 14 | :rtype: scalar if reduction is sum or mean, else ndarray 15 | """ 16 | # Calculate the absolute difference array 17 | absolute_difference = np.abs(y_predicted - y_ground_truth) 18 | # L1 distance is the reduced form of the absolute difference array 19 | if reduction == "sum": 20 | # Reduction can be done by summing up all the values in the difference array (this is known as "L1-Loss") 21 | l1_distance = np.sum(absolute_difference) 22 | return l1_distance 23 | elif reduction == "mean": 24 | # Reduction can also be done by taking the mean (this is known as "Mean Absolute Error") 25 | mean_absolute_error = np.mean(absolute_difference) 26 | return mean_absolute_error 27 | elif reduction == "None": 28 | return absolute_difference 29 | else: 30 | print('ValueError: reduction should be "sum" / "mean" / "None"') 31 | 32 | 33 | def main(): 34 | print("Initializing predicted and ground truth arrays:\n") 35 | print('(NOTE: Enter the values in a space-separated format. Ex: "5.36 1.02 2.03")') 36 | y_predicted = [ 37 | float(item) for item in input("Enter the predicted values: ").split() 38 | ] 39 | y_ground_truth = [ 40 | float(item) 41 | for item in input("Enter the corresponding ground truth values: ").split() 42 | ] 43 | assert len(y_predicted) == len( 44 | y_ground_truth 45 | ), "Number of predicted values {} and ground truth {} values should match".format( 46 | len(y_predicted), len(y_ground_truth) 47 | ) 48 | y_predicted = np.array(y_predicted) 49 | y_ground_truth = np.array(y_ground_truth) 50 | reduction = str(input('Enter the reduction mode: "sum" / "mean" / "None": ')) 51 | loss = L1Loss(y_predicted, y_ground_truth, reduction=reduction) 52 | print("L1-Loss with {}-reduction: {}".format(reduction, loss)) 53 | 54 | 55 | if __name__ == "__main__": 56 | main() 57 | -------------------------------------------------------------------------------- /losses/l2/L2.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def L2Loss(y_predicted, y_ground_truth, reduction="None"): 5 | """returns l2 loss between two arrays 6 | 7 | :param y_predicted: array of predicted values 8 | :type y_predicted: ndarray 9 | :param y_ground_truth: array of ground truth values 10 | :type y_ground_truth: ndarray 11 | :param reduction: reduction mode, defaults to "mean" 12 | :type reduction: str, optional 13 | :return: l2-loss 14 | :rtype: scalar if reduction is sum or mean, else ndarray 15 | """ 16 | # Calculate the difference array 17 | difference = y_predicted - y_ground_truth 18 | # Raise every difference value to the power of 2 19 | squared_difference = np.multiply(difference, difference) 20 | # L2 distance is the reduced form of the squared difference array 21 | if reduction == "sum": 22 | # Reduction can be done by summing up all the values in the difference array (this is known as "L2-Loss") 23 | l2_distance = np.sum(squared_difference) 24 | return l2_distance 25 | elif reduction == "mean": 26 | # Reduction can also be done by taking the mean (this is known as "Mean Squared Error") 27 | mean_squared_error = np.mean(squared_difference) 28 | return mean_squared_error 29 | elif reduction == "None": 30 | return squared_difference 31 | else: 32 | print('ValueError: reduction should be "sum" / "mean" / "None"') 33 | 34 | 35 | def main(): 36 | print("Initializing predicted and ground truth arrays:\n") 37 | print('(NOTE: Enter the values in a space-separated format. Ex: "5.36 1.02 2.03")') 38 | y_predicted = [ 39 | float(item) for item in input("Enter the predicted values: ").split() 40 | ] 41 | y_ground_truth = [ 42 | float(item) 43 | for item in input("Enter the corresponding ground truth values: ").split() 44 | ] 45 | assert len(y_predicted) == len( 46 | y_ground_truth 47 | ), "Number of predicted values {} and ground truth {} values should match".format( 48 | len(y_predicted), len(y_ground_truth) 49 | ) 50 | y_predicted = np.array(y_predicted) 51 | y_ground_truth = np.array(y_ground_truth) 52 | reduction = str(input('Enter the reduction mode: "sum" / "mean" / "None": ')) 53 | loss = L2Loss(y_predicted, y_ground_truth, reduction=reduction) 54 | print("L2-Loss with {}-reduction: {}".format(reduction, loss)) 55 | 56 | 57 | if __name__ == "__main__": 58 | main() 59 | --------------------------------------------------------------------------------