├── .gitignore ├── README.md ├── from_scratch.ipynb └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Byte-compiled / optimized / DLL files 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | cover/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | .pybuilder/ 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | # For a library or package, you might want to ignore these files since the code is 88 | # intended to run in multiple environments; otherwise, check them in: 89 | # .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 99 | __pypackages__/ 100 | 101 | # Celery stuff 102 | celerybeat-schedule 103 | celerybeat.pid 104 | 105 | # SageMath parsed files 106 | *.sage.py 107 | 108 | # Environments 109 | .env 110 | .venv 111 | env/ 112 | venv/ 113 | ENV/ 114 | env.bak/ 115 | venv.bak/ 116 | 117 | # Spyder project settings 118 | .spyderproject 119 | .spyproject 120 | 121 | # Rope project settings 122 | .ropeproject 123 | 124 | # mkdocs documentation 125 | /site 126 | 127 | # mypy 128 | .mypy_cache/ 129 | .dmypy.json 130 | dmypy.json 131 | 132 | # Pyre type checker 133 | .pyre/ 134 | 135 | # pytype static type analyzer 136 | .pytype/ 137 | 138 | # Cython debug symbols 139 | cython_debug/ 140 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Project Sophia: Make Neural Networks from Scratch 2 | 3 | ~~Plagiarized from~~ Inspired by [Andrej Karparthy's cs231n](http://cs231n.github.io/neural-networks-case-study/) and [Colah's Blog](http://colah.github.io/posts/2015-08-Backprop/). 4 | 5 | This notebook recreates neural network algorithms using only the linear algebra library `numpy` (uses other libraries for performance benchmark and visualization). The sample data is a 2-dimensional data with three classes in a form of spirals. 6 | 7 | The content consists of the following aspects of neural networks: 8 | 9 | * Weight initialization 10 | * Loss functions 11 | * Cross-entropy loss 12 | * Hinge loss 13 | * Regularization loss 14 | * Layers 15 | * Linear 16 | * Sigmoid 17 | * Tanh 18 | * ReLu 19 | * Swish 20 | * Softmax 21 | * SVM 22 | * Dropout 23 | * Batch Normalization 24 | * Convolution (TBD) 25 | * LSTM (TBD) 26 | * Embeddings (TBD) 27 | * Training 28 | * Linear > Softmax 29 | * Linear > SVM 30 | * [Linear > Activation]++ > [Linear > Output] 31 | * Optimization (TBD) 32 | * Validation (TBD) 33 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | #usual data science stuff 2 | import matplotlib 3 | import numpy as np 4 | import pandas as pd 5 | import matplotlib.pyplot as plt 6 | 7 | #utils 8 | import itertools 9 | 10 | #code from http://cs231n.github.io/neural-networks-case-study/ 11 | def plot_decision_boundary(pred_func,X,y): 12 | # Set min and max values and give it some padding 13 | x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5 14 | y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5 15 | h = 0.01 16 | # Generate a grid of points with distance h between them 17 | xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) 18 | # Predict the function value for the whole gid 19 | Z = pred_func(np.c_[xx.ravel(), yy.ravel()]) 20 | Z = Z.reshape(xx.shape) 21 | # Plot the contour and training examples 22 | plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral) 23 | plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Spectral) 24 | 25 | #code from http://cs231n.github.io/neural-networks-case-study/ 26 | def spiral_gen(N,D,K): 27 | #N = 100 # number of points per class 28 | #D = 2 # dimensionality 29 | #K = 3 # number of classes 30 | X = np.zeros((N*K,D)) # data matrix (each row = single example) 31 | y = np.zeros(N*K, dtype='uint8') # class labels 32 | for j in range(K): 33 | ix = range(N*j,N*(j+1)) 34 | r = np.linspace(0.0,1,N) # radius 35 | t = np.linspace(j*4,(j+1)*4,N) + np.random.randn(N)*0.2 # theta 36 | X[ix] = np.c_[r*np.sin(t), r*np.cos(t)] 37 | y[ix] = j 38 | # lets visualize the data: 39 | plt.scatter(X[:, 0], X[:, 1], c=y, s=40, cmap=plt.cm.Spectral) 40 | plt.show() 41 | return(X,y) 42 | 43 | #code from scikit-learn documentation 44 | def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues): 45 | plt.figure() 46 | plt.imshow(cm, interpolation='nearest', cmap=cmap) 47 | plt.title(title) 48 | plt.colorbar() 49 | tick_marks = np.arange(len(classes)) 50 | plt.xticks(tick_marks, classes, rotation=45) 51 | plt.yticks(tick_marks, classes) 52 | 53 | if normalize: 54 | cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] 55 | #print(cm) 56 | thresh = cm.max() / 2. 57 | for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): 58 | plt.text(j, i, cm[i, j], horizontalalignment="center", color="white" if cm[i, j] > thresh else "black") 59 | 60 | plt.tight_layout() 61 | plt.ylabel('True label') 62 | plt.xlabel('Predicted label') 63 | --------------------------------------------------------------------------------