├── machinelearning ├── .gitignore ├── Lesson1 │ ├── README.md │ ├── doraemon.jpg │ ├── scatter-marker.png │ └── iris.data ├── README.md └── Lesson2 │ ├── README.md │ ├── housing.names │ ├── SavingModel.ipynb │ ├── iris.data │ ├── Advertising.csv │ └── GradientDescentAlgorithm.ipynb ├── assignment1 ├── classwork │ ├── DSVC │ │ ├── __init__.py │ │ ├── classifiers │ │ │ ├── __init__.py │ │ │ └── k_nearest_neighbor.py │ │ ├── datasets │ │ │ └── .gitignore │ │ ├── vis_utils.py │ │ ├── gradient_check.py │ │ └── data_utils.py │ └── README.md └── README.md ├── assignment3 ├── classwork │ ├── DSVC │ │ ├── __init__.py │ │ ├── datasets │ │ │ └── .gitignore │ │ └── classifiers │ │ │ ├── __init__.py │ │ │ └── logistic_regression.py │ ├── README.md │ └── logistic_regression.ipynb └── README.md ├── assignment5 ├── classwork │ ├── DSVC │ │ ├── __init__.py │ │ ├── classifiers │ │ │ ├── __init__.py │ │ │ ├── linear_svm.py │ │ │ └── linear_classifier.py │ │ ├── datasets │ │ │ ├── .gitignore │ │ │ └── get_datasets.sh │ │ ├── vis_utils.py │ │ ├── gradient_check.py │ │ ├── features.py │ │ └── data_utils.py │ ├── README.md │ └── requirements.txt └── README.md ├── assignment6 ├── classwork │ ├── DSVC │ │ ├── __init__.py │ │ ├── classifiers │ │ │ ├── __init__.py │ │ │ └── neural_net.py │ │ ├── datasets │ │ │ └── .gitignore │ │ ├── vis_utils.py │ │ ├── gradient_check.py │ │ ├── features.py │ │ └── data_utils.py │ ├── .gitignore │ └── README.md └── README.md ├── assignment7 ├── classwork │ ├── DSVC │ │ ├── __init__.py │ │ ├── classifiers │ │ │ ├── __init__.py │ │ │ └── cnn.py │ │ ├── .gitignore │ │ ├── datasets │ │ │ ├── .gitignore │ │ │ └── get_datasets.sh │ │ ├── setup.py │ │ ├── im2col.py │ │ ├── vis_utils.py │ │ ├── layer_utils.py │ │ ├── gradient_check.py │ │ ├── im2col_cython.pyx │ │ ├── optim.py │ │ ├── data_utils.py │ │ └── fast_layers.py │ ├── .gitignore │ ├── kitten.jpg │ ├── puppy.jpg │ ├── requirements.txt │ ├── README.md │ └── Dropout.ipynb └── README.md ├── .gitignore ├── Course completion list ├── README.md ├── 2018 summer │ └── README.md └── 2019 spring │ └── README.md ├── assignment4 ├── README.md └── classwork │ ├── DSVC │ └── datasets │ │ └── tennis.csv.txt │ └── README.md ├── assignment8 └── README.md ├── assignment2 ├── classwork │ └── README.md └── README.md └── README.md /machinelearning/.gitignore: -------------------------------------------------------------------------------- 1 | .vscode -------------------------------------------------------------------------------- /assignment1/classwork/DSVC/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assignment3/classwork/DSVC/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assignment5/classwork/DSVC/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assignment6/classwork/DSVC/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assignment7/classwork/DSVC/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assignment7/classwork/DSVC/classifiers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assignment5/classwork/DSVC/classifiers/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /assignment6/classwork/DSVC/classifiers/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /assignment3/classwork/DSVC/datasets/.gitignore: -------------------------------------------------------------------------------- 1 | MNIST.csv 2 | -------------------------------------------------------------------------------- /assignment6/classwork/.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.pyc 3 | .env/* 4 | -------------------------------------------------------------------------------- /assignment7/classwork/.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.pyc 3 | .env/* 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .ipynb_checkpoints 3 | stuff 4 | *.pyc 5 | -------------------------------------------------------------------------------- /assignment7/classwork/DSVC/.gitignore: -------------------------------------------------------------------------------- 1 | build/* 2 | im2col_cython.c 3 | im2col_cython.so 4 | -------------------------------------------------------------------------------- /machinelearning/Lesson1/README.md: -------------------------------------------------------------------------------- 1 | ### Lesson1 2 | 3 | - [SVD](SVD.ipynb) 4 | - [PCA](PCA.ipynb) -------------------------------------------------------------------------------- /assignment1/classwork/DSVC/classifiers/__init__.py: -------------------------------------------------------------------------------- 1 | from DSVC.classifiers.k_nearest_neighbor import * -------------------------------------------------------------------------------- /assignment3/classwork/DSVC/classifiers/__init__.py: -------------------------------------------------------------------------------- 1 | from DSVC.classifiers.logistic_regression import * 2 | 3 | -------------------------------------------------------------------------------- /assignment7/classwork/kitten.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quinwu/DSVC/HEAD/assignment7/classwork/kitten.jpg -------------------------------------------------------------------------------- /assignment7/classwork/puppy.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quinwu/DSVC/HEAD/assignment7/classwork/puppy.jpg -------------------------------------------------------------------------------- /machinelearning/Lesson1/doraemon.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quinwu/DSVC/HEAD/machinelearning/Lesson1/doraemon.jpg -------------------------------------------------------------------------------- /machinelearning/Lesson1/scatter-marker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quinwu/DSVC/HEAD/machinelearning/Lesson1/scatter-marker.png -------------------------------------------------------------------------------- /Course completion list/README.md: -------------------------------------------------------------------------------- 1 | ### Course completion list 2 | 3 | 4 | - [2018 暑期课程](2018%20summer) 5 | 6 | - [2019 春季学期课程](2019%20spring) -------------------------------------------------------------------------------- /assignment1/classwork/DSVC/datasets/.gitignore: -------------------------------------------------------------------------------- 1 | cifar-10-batches-py/* 2 | tiny-imagenet-100-A* 3 | tiny-imagenet-100-B* 4 | tiny-100-A-pretrained/* 5 | -------------------------------------------------------------------------------- /assignment5/classwork/DSVC/datasets/.gitignore: -------------------------------------------------------------------------------- 1 | cifar-10-batches-py/* 2 | tiny-imagenet-100-A* 3 | tiny-imagenet-100-B* 4 | tiny-100-A-pretrained/* 5 | -------------------------------------------------------------------------------- /assignment6/classwork/DSVC/datasets/.gitignore: -------------------------------------------------------------------------------- 1 | cifar-10-batches-py/* 2 | tiny-imagenet-100-A* 3 | tiny-imagenet-100-B* 4 | tiny-100-A-pretrained/* 5 | -------------------------------------------------------------------------------- /assignment7/classwork/DSVC/datasets/.gitignore: -------------------------------------------------------------------------------- 1 | cifar-10-batches-py/* 2 | tiny-imagenet-100-A* 3 | tiny-imagenet-100-B* 4 | tiny-100-A-pretrained/* 5 | -------------------------------------------------------------------------------- /assignment5/classwork/DSVC/datasets/get_datasets.sh: -------------------------------------------------------------------------------- 1 | # Get CIFAR10 2 | wget http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz 3 | tar -xzvf cifar-10-python.tar.gz 4 | rm cifar-10-python.tar.gz 5 | -------------------------------------------------------------------------------- /assignment7/classwork/DSVC/datasets/get_datasets.sh: -------------------------------------------------------------------------------- 1 | # Get CIFAR10 2 | wget http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz 3 | tar -xzvf cifar-10-python.tar.gz 4 | rm cifar-10-python.tar.gz 5 | -------------------------------------------------------------------------------- /assignment4/README.md: -------------------------------------------------------------------------------- 1 | ## assignment4 2 | 3 | #### 内容 4 | 5 | - Decision Tree 6 | - Decision Tree Classification 7 | 8 | #### 作业 9 | 10 | - [Decision Tree Classification](classwork) 11 | 12 | -------------------------------------------------------------------------------- /machinelearning/README.md: -------------------------------------------------------------------------------- 1 | # Machine Learning 2 | 3 | 机器学习课程作业代码`Jupyter Notebook` 版本 4 | 5 | ### Lesson1 6 | 7 | [数据清洗](Lesson1/) 8 | 9 | ### Lesson2 10 | 11 | [回归问题](Lesson2/) 12 | 13 | ### Lesson3 14 | 15 | 决策树 -------------------------------------------------------------------------------- /assignment5/README.md: -------------------------------------------------------------------------------- 1 | ## assignment5 2 | 3 | ### 内容 4 | 5 | - SVM 6 | - SVM 硬间隔，软间隔 7 | - SVM 核函数 8 | - Hinge Loss 9 | 10 | 参考 11 | 12 | - [小记SVM](https://quinwu.github.io/2017/09/08/ML-SVM/) 13 | - 周志华老师 《机器学习》 14 | - 李航老师 《统计学习方法》 15 | 16 | ### 作业 17 | 18 | - [SVM 多分类任务](classwork) -------------------------------------------------------------------------------- /assignment7/README.md: -------------------------------------------------------------------------------- 1 | ## assignment7 2 | 3 | ### 内容 4 | 5 | - Neural Network modular layer design 6 | - [Batch Normalization](https://arxiv.org/abs/1502.03167) 7 | - [Dropout](http://jmlr.org/papers/v15/srivastava14a.html) 8 | 9 | ### 作业 10 | 11 | - [Neural Network modular layer design](classwork) 12 | - [Batch Normalization](classwork) 13 | - [Dropout](classwork) 14 | 15 | -------------------------------------------------------------------------------- /assignment7/classwork/DSVC/setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | from distutils.extension import Extension 3 | from Cython.Build import cythonize 4 | import numpy 5 | 6 | extensions = [ 7 | Extension('im2col_cython', ['im2col_cython.pyx'], 8 | include_dirs = [numpy.get_include()] 9 | ), 10 | ] 11 | 12 | setup( 13 | ext_modules = cythonize(extensions), 14 | ) 15 | -------------------------------------------------------------------------------- /machinelearning/Lesson2/README.md: -------------------------------------------------------------------------------- 1 | ### Lesson2 2 | 3 | - [Gradient Descent Algorithm](GradientDescentAlgorithm.ipynb) 4 | - [Linear Regression](LinearRegression.ipynb) 5 | - [Elastic Net](ElasticNet.ipynb) 6 | - [Iris](Iris.ipynb) 7 | - [Boston](Boston.ipynb) 8 | - [AUC Introduction](AUCIntroduction.ipynb) 9 | - [AUC](AUC.ipynb) 10 | - [AUC Iris](AUCIris.ipynb) 11 | - [Saving Model](SavingModel.ipynb) -------------------------------------------------------------------------------- /assignment8/README.md: -------------------------------------------------------------------------------- 1 | ## assignment8 2 | 3 | ### 内容 4 | 5 | - Convolution Neural Networks 6 | - Pooling, Stride, Padding 7 | 8 | 参考 9 | 10 | - [cs231n Convolutional Neural Networks](http://cs231n.github.io/convolutional-networks/) 11 | - [卷积神经网络 CNN 笔记(高级篇)](http://www.shuang0420.com/2017/04/25/%E5%8D%B7%E7%A7%AF%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%20CNN%20%E7%AC%94%E8%AE%B0(%E9%AB%98%E7%BA%A7%E7%AF%87)/) 12 | 13 | ### 作业 14 | 15 | - [Convolutional Networks CIFAR-10 图像分类](../assignment7/classwork) 16 | 17 | -------------------------------------------------------------------------------- /assignment4/classwork/DSVC/datasets/tennis.csv.txt: -------------------------------------------------------------------------------- 1 | outlook,temp,humidity,windy,play 2 | sunny,hot,high,false,no 3 | sunny,hot,high,true,no 4 | overcast,hot,high,false,yes 5 | rainy,mild,high,false,yes 6 | rainy,cool,normal,false,yes 7 | rainy,cool,normal,true,no 8 | overcast,cool,normal,true,yes 9 | sunny,mild,high,false,no 10 | sunny,cool,normal,false,yes 11 | rainy,mild,normal,false,yes 12 | sunny,mild,normal,true,yes 13 | overcast,mild,high,true,yes 14 | overcast,hot,normal,false,yes 15 | rainy,mild,high,true,no 16 | -------------------------------------------------------------------------------- /assignment6/README.md: -------------------------------------------------------------------------------- 1 | ## assignment6 2 | 3 | ### 内容 4 | 5 | - Neural Network 介绍 6 | - Two layers Neural Network 7 | 8 | 参考 9 | 10 | - [如何计算神经网络梯度](http://paper.kakapo.ml/?p=94) 11 | - [小记Neural Network](https://quinwu.github.io/2017/05/16/ML-Neural-Network/) 12 | - [小记Gradient Descend](https://quinwu.github.io/2017/09/02/ML-Gradient-Descent/) 批量梯度下降，随机梯度下降，mini-batch梯度下降 13 | - [一段关于神经网络的故事](https://iphysresearch.github.io/cs231n/cs231n_story_MLP.html) 14 | 15 | 16 | ### 作业 17 | 18 | - [Two layers Neural Network CIFAR-10图像分类](classwork) 19 | 20 | 21 | -------------------------------------------------------------------------------- /assignment4/classwork/README.md: -------------------------------------------------------------------------------- 1 | ## Decision Tree Classification 2 | 3 | ### 代码 4 | 5 | 作业的 Jupyter Notebook 模板在`decision_tree.ipynb`中给出 6 | 7 | ### 数据 8 | 9 | 数据在`/DSVC/datasets/`中给出 10 | 11 | ### 运行 12 | 13 | 在终端或命令行窗口中，选定`classwork/`的目录下（包含此README文件），运行下方的命令： 14 | 15 | ``` 16 | jupyter notebook 17 | ``` 18 | 19 | 这样就能够启动 Jupyter Notebook 软件，并在你的浏览器中打开`decision_tree.ipynb`文件。 20 | 21 | ### 作业提交 22 | 23 | ### 作业提交 24 | 25 | - 提交方式：邮件的标题为`姓名-assignment4`。 26 | - 格式说明： 27 | - 需要提交的文件：`decision_tree.ipynb`文件以及生成的`.html`文件和相关的资源文件、`decision_tree.ipynb`代码文件。 28 | - 将这些文件压缩成一个`.zip`文件， 命名与邮件标题相同。 29 | - 你提交的作业版本应与Github上的最新版本一致。 -------------------------------------------------------------------------------- /assignment6/classwork/README.md: -------------------------------------------------------------------------------- 1 | ## Two layers Neural Network CIFAR-10图像分类 2 | 3 | ### 作业文件 4 | 5 | 程序的Jupyter Notebook模板在`classwork/two_layer_net.ipynb`里给出。 6 | 7 | 需要实现的classifier在`classwork/DSVC/classifiers/nerual_net.py`里给出。 8 | 9 | ### 数据 10 | 11 | - CIFAR-10图像数据集（跟k-NN作业中的数据集相同） 12 | 13 | 讲k-NN作业中的CIFAR-10数据集数据文件放在路径`classwork/DSVC/datasets/`下即可。 14 | 15 | ### 运行 16 | 17 | 在终端或命令行窗口中，选定`classwork/`的目录下（包含此README文件），运行下方的命令： 18 | 19 | `jupyter notebook` 20 | 21 | 这样就能够启动jupyter notebook软件，并在你的浏览器中打开`two_layer_net.ipynb`文件。 22 | 23 | ### 任务 24 | 25 | - Two layers Neural Network CIFAR-10图像分类任务 26 | 27 | ### 作业提交 28 | 29 | - 提交方式：邮件的标题为`姓名-assignment6`。 30 | - 格式说明： 31 | - 需要提交的文件：`two_layer_net.ipynb`文件以及生成的`.html`文件和相关的资源文件、`neural_net.py`代码文件。 32 | - 将这些文件压缩成一个`.zip`文件， 命名与邮件标题相同。 33 | - 你提交的作业版本应与Github上的最新版本一致。 -------------------------------------------------------------------------------- /assignment2/classwork/README.md: -------------------------------------------------------------------------------- 1 | ## 波士顿房价预测 2 | 3 | #### 代码 4 | 5 | 代码的模版已经在`boston_housing.ipynb`文件中给出。你还会用到名为`housing.csv`的数据文件来完成这个项目。已经为你提供了一部分代码，但还有些功能需要你来实现才能以完成这个项目。 6 | 7 | #### 运行 8 | 9 | 在终端或命令行窗口中，选定`classwork/`的目录下（包含此README文件），运行下方的命令： 10 | 11 | `jupyter notebook boston_housing.ipynb` 12 | 13 | 这样就能够启动jupyter notebook软件，并在你的浏览器中打开文件。 14 | 15 | #### 数据 16 | 17 | 经过编辑的波士顿房价数据集有490个数据点，每个点有三个特征。这个数据集编辑自[加州大学欧文分校机器学习数据集库（数据集已下线）](https://archive.ics.uci.edu/ml/datasets.html). 18 | 19 | ###### 特征 20 | 21 | 1. `RM`: 住宅平均房间数量 22 | 2. `LSTAT`: 区域中被认为是低收入阶层的比率 23 | 3. `PTRATIO`: 镇上学生与教师数量比例 24 | 25 | ###### 目标变量 26 | 27 | 1. `MEDV`: 房屋的中值价格 28 | 29 | #### 作业提交 30 | 31 | - 提交方式：邮件的标题为`姓名-assignment2`。 32 | - 格式说明： 33 | - 需要提交的文件：`boston_housing.ipynb`文件以及生成的`.html`文件和相关的资源文件。 34 | - 将这些文件压缩成一个`.zip`文件， 命名与邮件标题相同。 35 | - 你提交的作业版本应与Github上的最新版本一致。 -------------------------------------------------------------------------------- /assignment3/README.md: -------------------------------------------------------------------------------- 1 | ## assignment3 2 | 3 | #### 内容 4 | 5 | - Logistics Regression 6 | - Logistics Regression MNIST 手写数字识别数据集分类任务 7 | 8 | 以下的内容可以帮助你更好的完成本次的作业： 9 | 10 | - [小记Logistics Regression](https://quinwu.github.io/2017/05/05/ML-Logistic-Regression/) 11 | - [Python 可视化 Logistics Regression效果](https://github.com/quinwu/ml_implementation/tree/master/Logistic-Regression/optimizeMinimize) 12 | 13 | #### 作业 14 | 15 | - Logistics Regression Python 可视化实现（optional） 16 | 17 | 独立使用Numpy、Matplotlib来完成Logistic Regression的可视化任务。数据集使用上面我给出的可视化效果链接中对应的数据集。在上面给出了我的实现代码来提供参考。 18 | 19 | - 第一个可视化任务中的Decision Boundary比较简单，不建议使用其他第三方包。 20 | - 第二个可视化任务中的Decision Boundary属于多项式较为复杂，可以使用scipy.optimize或者scikit-learn来完成。 21 | 22 | - [MNIST手写数字识别分类任务](classwork) 23 | 24 | **必须手动实现Logistics Regression，不得使用任何第三方库** 25 | 26 | - Logistics Regression完成MNIST数据集上的二分类任务。 27 | 28 | - Logistics Regression完成MNIST数据集上的多分类任务（one vs all ）。 29 | 30 | ​ 31 | 32 | -------------------------------------------------------------------------------- /assignment5/classwork/README.md: -------------------------------------------------------------------------------- 1 | ## SVM CIFAR-10图像分类 2 | 3 | ### 作业文件 4 | 5 | - 本次作业的python运行依赖环境在 `classwork/requirements.txt` 中给出 6 | 7 | - Jupyter Notebook 模板在 `classwork/svm.ipynb` 中给出 8 | 9 | - 需要实现的SVM分类器在 `classwork/DSVC/classifiers/liner_svm.py` 中给出 10 | 11 | - 本次作业需要用到的数据在 `classwork/DSVC/datasets/get_datasets.sh` 中给出 12 | 13 | ### 数据 14 | 15 | - CIFAR-10 图像数据集 16 | 17 | - Linux 下可以直接运行 `classwork/DSVC/datasets/get_datasets.sh` 脚本下载并解压 CIFAR-10 数据集 18 | 19 | 将下载好的 CIFAR-10 数据放在 `classwork/DSVB/datasets/` 下即可 20 | 21 | ### 运行 22 | 23 | 在终端或命令行窗口中，在该`classwork/`的目录下（包含此README文件），运行下方的命令： 24 | 25 | > jupyter notebook 26 | 27 | 这样就能够启动jupyter notebook软件，并在你的浏览器中打开`svm.ipynb`文件 28 | 29 | ### 任务 30 | 31 | - SVM CIFAR-10 图像分类任务 32 | 33 | ### 作业提交 34 | 35 | - 提交方式：邮件的标题为`姓名-assignment5`。 36 | - 格式说明： 37 | - 需要提交的文件：`svm.ipynb`文件以及生成的`.html`文件和相关的资源文件、`liner_svm.py`代码文件。 38 | - 将这些文件压缩成一个`.zip`文件， 命名与邮件标题相同。 39 | - 你提交的作业版本应与Github上的最新版本一致。 40 | 41 | -------------------------------------------------------------------------------- /assignment5/classwork/requirements.txt: -------------------------------------------------------------------------------- 1 | Cython==0.23.4 2 | Jinja2==2.8 3 | MarkupSafe==0.23 4 | Pillow==3.0.0 5 | Pygments==2.0.2 6 | appnope==0.1.0 7 | argparse==1.2.1 8 | backports-abc==0.4 9 | backports.ssl-match-hostname==3.5.0.1 10 | certifi==2015.11.20.1 11 | cycler==0.10.0 12 | decorator==4.0.6 13 | future==0.16.0 14 | gnureadline==6.3.3 15 | ipykernel==4.2.2 16 | ipython==4.0.1 17 | ipython-genutils==0.1.0 18 | ipywidgets==4.1.1 19 | jsonschema==2.5.1 20 | jupyter==1.0.0 21 | jupyter-client==4.1.1 22 | jupyter-console==4.0.3 23 | jupyter-core==4.0.6 24 | matplotlib==2.0.0 25 | mistune==0.7.1 26 | nbconvert==4.1.0 27 | nbformat==4.0.1 28 | notebook==4.0.6 29 | numpy==1.10.4 30 | path.py==8.1.2 31 | pexpect==4.0.1 32 | pickleshare==0.5 33 | ptyprocess==0.5 34 | pyparsing==2.0.7 35 | python-dateutil==2.4.2 36 | pytz==2015.7 37 | pyzmq==15.1.0 38 | qtconsole==4.1.1 39 | scipy==0.16.1 40 | simplegeneric==0.8.1 41 | singledispatch==3.4.0.3 42 | site==0.0.1 43 | six==1.10.0 44 | terminado==0.5 45 | tornado==4.3 46 | traitlets==4.0.0 47 | -------------------------------------------------------------------------------- /assignment7/classwork/requirements.txt: -------------------------------------------------------------------------------- 1 | Cython==0.25.2 2 | Jinja2==2.8 3 | MarkupSafe==0.23 4 | Pillow==3.0.0 5 | Pygments==2.0.2 6 | appnope==0.1.0 7 | argparse==1.2.1 8 | backports-abc==0.4 9 | backports.ssl-match-hostname==3.5.0.1 10 | certifi==2015.11.20.1 11 | cycler==0.10.0 12 | decorator==4.0.6 13 | future==0.16.0 14 | gnureadline==6.3.3 15 | h5py==2.7.0 16 | ipykernel==4.2.2 17 | ipython==4.0.1 18 | ipython-genutils==0.1.0 19 | ipywidgets==4.1.1 20 | jsonschema==2.5.1 21 | jupyter==1.0.0 22 | jupyter-client==4.1.1 23 | jupyter-console==4.0.3 24 | jupyter-core==4.0.6 25 | matplotlib==2.0.0 26 | mistune==0.7.1 27 | nbconvert==4.1.0 28 | nbformat==4.0.1 29 | nltk==3.2.2 30 | notebook==4.0.6 31 | numpy==1.12.1 32 | path.py==8.1.2 33 | pexpect==4.0.1 34 | pickleshare==0.5 35 | ptyprocess==0.5 36 | pyparsing==2.0.7 37 | python-dateutil==2.4.2 38 | pytz==2015.7 39 | pyzmq==15.1.0 40 | qtconsole==4.1.1 41 | scipy==0.19.0 42 | simplegeneric==0.8.1 43 | singledispatch==3.4.0.3 44 | site==0.0.1 45 | six==1.10.0 46 | terminado==0.5 47 | tornado==4.3 48 | traitlets==4.0.0 49 | -------------------------------------------------------------------------------- /assignment3/classwork/README.md: -------------------------------------------------------------------------------- 1 | ## MNIST 手写数字识别分类任务 2 | 3 | ### 代码 4 | 5 | 代码的Jupyter Notebook模板在`logistic_regression.ipynb`文件中给出。 6 | 7 | 需要实现的classifier在`classwork/DSVC/classifiers/logistic_regression.py`中给出。 8 | 9 | ### 数据 10 | 11 | - [MNIST手写数字数据集](http://yann.lecun.com/exdb/mnist/) 12 | 13 | 将数据存放在路径`classwork/DSVC/datasets/`下即可。 14 | 15 | ### 运行 16 | 17 | 在终端或命令行窗口中，选定`classwork/`的目录下（包含此README文件），运行下方的命令： 18 | 19 | `jupyter notebook ` 20 | 21 | 这样就能够启动jupyter notebook软件，并在你的浏览器中打开`logistic_regression.ipynb`文件。 22 | 23 | ### 任务 24 | 25 | The Jupyter Notebook `logistic_regressino.ipynb` will walk you through implementing the classifier. 26 | 27 | You should choose and extract features on your own. 28 | 29 | - Binary classification with Logistic Regression classifier 30 | - Multiclass classification with Logistic Regression classifier and "one vs all" 31 | 32 | ### 作业提交 33 | 34 | - 提交方式：邮件的标题为`姓名-assignment3`。 35 | - 格式说明： 36 | - 需要提交的文件：`logistic_regression.ipynb`文件以及生成的`.html`文件和相关的资源文件、`logistic_regression.py`代码文件。 37 | - 将这些文件压缩成一个`.zip`文件， 命名与邮件标题相同。 38 | - 你提交的作业版本应与Github上的最新版本一致。 39 | 40 | -------------------------------------------------------------------------------- /assignment2/README.md: -------------------------------------------------------------------------------- 1 | ## assignment2 2 | 3 | #### 准备工作 4 | 5 | 这个项目需要安装以下的Python函数库 6 | 7 | - [NumPy](http://www.numpy.org/) 8 | - [matplotlib](http://matplotlib.org/) 9 | - [scikit-learn](http://scikit-learn.org/stable/) 10 | 11 | 其中，**Numpy**、**Matplotlib**是必须的安装的（一般在第一次作业中安装了Anaconda后这两个包都是自动安装的），**scikit-learn**不是必须的。 12 | 13 | #### 内容 14 | 15 | - 通过学习线性回归算法来了解机器学习的一些基本的过程。 16 | - 利用线性回归来完成作业中波士顿房价数据集的拟合与预测。 17 | 18 | 以下的内容可以帮助你更好的完成这个作业： 19 | 20 | - [机器学习入门：线性回归及梯度下降](http://blog.csdn.net/xiazdong/article/details/7950084) 21 | - [小记Linear Regression](https://quinwu.github.io/2017/05/03/ML-Linear-Regression/) 22 | - [ A visual introduction to machine learning ](http://www.r2d3.us/visual-intro-to-machine-learning-part-1/) 23 | 24 | 25 | - 我自己用Python可视化实现的一个Linear Regression的效果[Linear Regression Python implement](https://github.com/quinwu/ml_implementation/tree/master/Linear-Regression) 26 | 27 | 给出这部分的代码仅在下面作业的时候作为一个参考，但是直接套用不一定会有很好的效果。 28 | 29 | #### 作业 30 | 31 | - Linear Regression 可视化（optional） 32 | 33 | 独立使用Numpy、Matplotlib 来完成Linear Regression的可视化任务，就像给出的我的实现效果一样。数据集可以参考我上面的可视化效果的数据集，也可以选择Boston House Price的数据集或者其他的。帮助理解Linear Regression的过程。 34 | 35 | - [波士顿房价预测](classwork) 36 | 37 | 在此项目中，我们将对为马萨诸塞州波士顿地区的房屋价格收集的数据应用本周学到的几个机器学习概念，以预测新房屋的销售价格。你首先将探索这些数据以获取数据集的重要特征和描述性统计信息。接下来，你要正确地将数据拆分为测试数据集和训练数据集，并确定适用于此问题的性能指标。然后，你将自己编写一个线性回归的模型，并使用不同的参数和训练集大小分析学习算法的性能图表。最后，你将根据一个新样本测试此模型并将预测的销售价格与你的统计数据进行比较。 38 | 39 | **必须手动实现Linear Regression，不得使用任何第三方库** 40 | 41 | 作业文件：`classwork/boston_housing.ipynb` -------------------------------------------------------------------------------- /Course completion list/2018 summer/README.md: -------------------------------------------------------------------------------- 1 | ### 2018 暑期 2 | 3 | | 组长 | GitHub | 4 | | :----- | :----------------------------------------------------------- | 5 | | 胡屹杉 | [https://github.com/huyisha](https://github.com/huyishan) | 6 | | 李萌 | [https://github.com/nuciot38](https://github.com/nuciot38) | 7 | | 王蕊芳 | [https://github.com/RuiFangW](https://github.com/RuiFangW) | 8 | | 许少伟 | [https://github.com/MeerkatX](https://github.com/MeerkatX) | 9 | | 杨博 | [https://github.com/Ybbbbbbbbbbb](https://github.com/Ybbbbbbbbbbb) | 10 | | 丁智慧 | [https://github.com/Source-00](https://github.com/Source-00) | 11 | | 李虹霞 | [https://github.com/DamDam123](https://github.com/DamDam123) | 12 | | 程谭 | [https://github.com/chengup](https://github.com/chengup) | 13 | | 王天锐 | [https://github.com/wangtianrui](https://github.com/wangtianrui) | 14 | | 赵励志 | [https://github.com/ZhaoLizz](https://github.com/ZhaoLizz) | 15 | | 李浩 | [https://github.com/wmpscc](https://github.com/wmpscc) | 16 | | 王宇飞 | [https://github.com/wangyufei1006](https://github.com/wangyufei1006) | 17 | | 张子珩 | [https://github.com/Talnex](https://github.com/Talnex) | 18 | | 杨可元 | [https://github.com/Bobbyyky](https://github.com/Bobbyyky) | 19 | | 陈帅宇 | [https://github.com/chenshuaiyu](https://github.com/chenshuaiyu) | 20 | | 张阳 | [https://github.com/annyangya](https://github.com/annyangya) | 21 | | 陈茹 | [https://github.com/king-ru](https://github.com/king-ru) | 22 | | 靳璐 | [https://github.com/jinlu1106](https://github.com/jinlu1106) | 23 | -------------------------------------------------------------------------------- /assignment1/classwork/README.md: -------------------------------------------------------------------------------- 1 | ## 《"智能移动互联网+"创新创业实践》人工智能模块作业任务 2 | 3 | 4 | 5 | ### CIFAR-10图像分类 6 | 7 | [The CIFAR-10 dataset](http://www.cs.toronto.edu/~kriz/cifar.html) consists of 60000 32x32 colour images in 10 classes, with 6000 images per class. There are 50000 training images and 10000 test images. 8 | [数据集下载链接](http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz) 9 | 10 | 下载到`cifar-10-python.tar.gz`文件后，解压。将解压后的文件夹拷贝到`/classwork/DSVC/datasets/`路径下即可。 11 | 12 | ### k-Nearest Neighbor classifier 13 | 14 | Jupyter Notebook文件 **knn.ipynb**将引导你来实现一个K-NN分类器。 15 | 16 | 使用数据集: CIFAR-10 17 | 18 | #### 作业任务 19 | 20 | - **实现k-NN算法**。 21 | - **通过使用部分向量化完成KNN算法**。 22 | - **通过使用全部矢量化来完成KNN算法**。 23 | - **通过交叉验证确定超参`k`的最佳值**。 24 | 25 | *暂时不考虑图像数据的特征提取以及其他更好的分类算法。* 26 | 27 | ## Optional （选做作业） 28 | This is your chance to show off! Try to get higher accuracy. 29 | 30 | Design and implement a new type of feature and select an algorithm, and use them for image classification on CIFAR-10. 31 | 32 | 尽可能的去调整参数得到最高的准确率值，可以尝试去使用一些新的特征选择算法，在CIFAR-10数据集上得到更高的分类准确率。 33 | 34 | ## 作业文件说明 35 | 36 | 上面的optional为选做题，可以选择其他的分类算法来训练CIFAR-10数据集。 37 | 38 | 所有的文件，真正需要去完成只有两个文件。 39 | 40 | - `knn-en.ipynb`（英文版）或者`knn-zh.ipynb`（中文版） 41 | - `classwork/DSVC/classifiers/k_nearest_neighbor.py` 42 | 43 | 这两个文件，其他的都是一些辅助验证的程序，不需要理会。 44 | 45 | - `.ipynb`文件只能通过`jupyter notebook `启动本地的`jupyter`服务器。默认端口是`8888`，在浏览器输入`localhost:8888`就可以。需要注意的是在`jupyter`的目录结构里找到你的`DSVC`项目文件的路径。 46 | - `k_nearest_neighbor.py`文件在本地可以选择任意的编辑器去填写相关的程序。 47 | 48 | ### 作业提交方式 49 | 50 | - 提交方式：邮件的标题为`姓名-assignment1`。 51 | - 格式说明： 52 | - 需要提交的文件：`knn.ipynb`文件以及生成的`.html`文件和相关的资源文件、`k_nearest_neighbor.py`代码文件。 53 | - 将这些文件压缩成一个`.zip`文件， 命名与邮件标题相同。 54 | - 你提交的作业版本应与Github上的最新版本一致。 -------------------------------------------------------------------------------- /Course completion list/2019 spring/README.md: -------------------------------------------------------------------------------- 1 | ### 2019 春季 2 | | 小组成员 | GitHub仓库 | 3 | | ------------------------------------ | ------------------------------------------------------------ | 4 | | 鲍骞月、张泽亿、景金龙、许瑞洋 | | 5 | | 孙旺、彭俊豪、石峰宇、张伟康 | https://github.com/ZakiZwk/LogisticsRegression | 6 | | 朱新杰、高歌、武杰、郑瑞东 | https://github.com/BadEric/boston_housing | 7 | | 赵远方、高天赐、柴可可、周瑞林 | https://github.com/yuanandfang/HandwrittenNumberRecognition | 8 | | 白思凡、郭春燕、左余林、申玉豪、连昕 | https://github.com/Haveoneriver/boston_housing | 9 | | 朱琳怡、宁浩芬、杨静、郑捷、张璐 | https://github.com/lisagreenzhu/xianxinghuigui/tree/master/classwork | 10 | | 董靖鑫、王君、王宇超、涂大祥 | https://github.com/Dongganggang/meachine-learning | 11 | | 钟昱明、高世雄、杨忠瑞、王熙为 | | 12 | | 李静涵、李开元、刘慧敏、陈亭文 | | 13 | | 樊琳卓、郝春燕、米嘉、乔嘉璇 | https://github.com/Anonymousflz/assignment | 14 | | 肖颖慧、刘泽宇、王璇、任拓 | https://github.com/helloworldxyh/Decision_Tree | 15 | | 胡蝶、李萌、李慧中、周婉琳、申彦坤 | https://github.com/miaopi/logistic-handwriting | 16 | | 席立天、刘嘉懿、药轶伦、王泽昕 | https://github.com/xin0v0/Boston | 17 | | 赵青、吴春红、蒋婷、李爱荣 | https://github.com/5047xc/jiangting-assignment4 | 18 | | 朱琳怡、宁浩芬 、杨静、郑捷、张璐 | https://github.com/lisagreenzhu/xianxinghuigui/tree/master/classwork | 19 | | 阮臻、李雅婷、张翔宇、史兰鑫 | https://github.com/Whale81/DecisionTreeAssignment | 20 | | 郭晋南、高祥、郑浪朗、周继美、罗哲轩 | https://github.com/swizLL/gitproject/tree/master/KNN/assignment1 | 21 | | 张一凡、刘诗蕊、梁亚亚、李虹宇 | https://github.com/shiliangyaya/decision--tree | 22 | | 谢帅康、候潇杰、闫昊、张林栋 | https://github.com/ShuaikangXie/MachineLearning.git | 23 | | 钟昱明、高世雄、杨忠瑞、王熙为 | https://github.com/zh62388/BostonHousing | 24 | 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Research Institute of Data Science and Vision Computing 2 | 3 | ### Introduction 4 | 5 | Research Institute of Data Science and Vision Computing 机器学习与深度学习课程作业 6 | 7 | ### Content 8 | 9 | [assignment1](assignment1) 10 | - 基础：Git、Python基础学习 11 | - 算法：k-NN 12 | - 作业：CIFAR-10 图像分类 13 | 14 | [assignment2](assignment2) 15 | - 算法：Linear Regression （线性回归） 16 | - 优化：梯度下降法 17 | - 作业：Boston House Price Predict 18 | 19 | [assignment3](assignment3) 20 | 21 | - 算法：Logistic Regression（对数几率回归），sigmoid 函数 22 | - 作业：MNIST 手写数字识别 23 | - Bonus：完成Softmax CIFAR-10图像分类，以及类比 Softmax与Logistic的关系。 24 | 25 | [assignment4](assignment4) 26 | 27 | - 算法：Decision Tree 28 | - 作业：Decision Tree 29 | 30 | [assignment5](assignment5) 31 | 32 | - SVM介绍 （参考书籍：周志华机器学习） 33 | - 算法：SVM Hinge Loss 34 | - 作业：CIFAR-10图像分类 35 | 36 | [assignment6](assignment6) 37 | 38 | - Neural Network介绍 39 | - forward pass 、 backpropagation介绍 40 | - 作业：two-layer Neural Network CIFAR-10图像分类 41 | 42 | [assignment7](assignment7) 43 | 44 | - Neural Network 模块化实现 45 | - batch normalization介绍 46 | - dropout介绍 47 | - 作业：改进 Neural Network 代码、Batch Normalization、Dropout 实现 48 | 49 | [assignment8](assignment8) 50 | 51 | - CNN 介绍 52 | - 概念：卷积, Pooling, Stride, Padding, Learning Rate, Momentum, Softmax, ReLU, BP, SGD, Cross-Entropy Loss。 53 | - 作业：CIFAR-10图像分类。 54 | 55 | 56 | ### Teacher 57 | 58 | | 教师 | - | 59 | | --------------------------------- | :---------------------------------: | 60 | | 秦品乐 | | 61 | | **助教** | **-** | 62 | | [武宽](https://github.com/quinwu) | [沈文祥](https://github.com/swxhss) | 63 | 64 | 65 | ### Participating students 66 | 67 | - [2018 Summer](Course%20completion%20list/2018%20summer) 68 | 69 | - [2019 Spring](Course%20completion%20list/2019%20spring) 70 | 71 | ### Reference & Acknowledgements 72 | 73 | 我们的课程作业内容主要参考到了以下相关课程，在此对以下相关内容的作者表示感谢。 74 | 75 | - [cs231n Convolutional Neural Networks for Visual Recognition](http://cs231n.github.io/) 76 | 77 | - [MIL 机器学习暑期研讨班](https://github.com/milLearningGroup/cama_summer_school_2017) 78 | -------------------------------------------------------------------------------- /assignment7/classwork/README.md: -------------------------------------------------------------------------------- 1 | In this assignment you will practice writing backpropagation code, and training 2 | Neural Networks and Convolutional Neural Networks. The goals of this assignment 3 | are as follows: 4 | 5 | - understand **Neural Networks** and how they are arranged in layered 6 | architectures 7 | - understand and be able to implement (vectorized) **backpropagation** 8 | - implement various **update rules** used to optimize Neural Networks 9 | - implement **batch normalization** for training deep networks 10 | - implement **dropout** to regularize networks 11 | - effectively **cross-validate** and find the best hyperparameters for Neural 12 | Network architecture 13 | - understand the architecture of **Convolutional Neural Networks** and train 14 | gain experience with training these models on data 15 | 16 | ### Neural Network modular 17 | 18 | The IPython notebook `FullyConnectedNets.ipynb` will introduce you to our modular layer design, and then use those layers to implement fully-connected networks of arbitrary depth. To optimize these models you will implement several popular update rules. 19 | 20 | - `FullyConnectedNets.ipynb` 21 | - `DSVC/layers.py` 22 | - `DSVC/layer_utils.py` 23 | - `DSVC/classfifiers/fc_net.py` 24 | - `DSVC/solver.py` 25 | - `DSVC/optim.py` 26 | 27 | ### Batch Normalization 28 | 29 | In the IPython notebook `BatchNormalization.ipynb` you will implement batch normalization, and use it to train deep fully-connected networks. 30 | 31 | - `BatchNormalization.ipynb` 32 | - `DSVC/layers.py` 33 | - `DSVC/classifiers/fc_net.py` 34 | 35 | ### Dropout 36 | 37 | The IPython notebook `Dropout.ipynb` will help you implement Dropout and explore its effects on model generalization. 38 | 39 | - `Dropout.ipynb` 40 | - `DSVC/layers.py` 41 | - `DSVC/classifiers/fc_net.py` 42 | 43 | ### Convolution Neural Networks 44 | 45 | In the IPython Notebook `ConvolutionalNetworks.ipynb` you will implement several new layers that are commonly used in convolutional networks. 46 | 47 | - `ConvolutionalNetworks.ipynb` 48 | - `DSVC/layers.py` 49 | - `DSVC/fast_layers.py` 50 | - `DSVC/layer_utils.py` 51 | - `DSVC/classifiers/cnn.py` -------------------------------------------------------------------------------- /machinelearning/Lesson2/housing.names: -------------------------------------------------------------------------------- 1 | 1. Title: Boston Housing Data 2 | 3 | 2. Sources: 4 | (a) Origin: This dataset was taken from the StatLib library which is 5 | maintained at Carnegie Mellon University. 6 | (b) Creator: Harrison, D. and Rubinfeld, D.L. 'Hedonic prices and the 7 | demand for clean air', J. Environ. Economics & Management, 8 | vol.5, 81-102, 1978. 9 | (c) Date: July 7, 1993 10 | 11 | 3. Past Usage: 12 | - Used in Belsley, Kuh & Welsch, 'Regression diagnostics ...', Wiley, 13 | 1980. N.B. Various transformations are used in the table on 14 | pages 244-261. 15 | - Quinlan,R. (1993). Combining Instance-Based and Model-Based Learning. 16 | In Proceedings on the Tenth International Conference of Machine 17 | Learning, 236-243, University of Massachusetts, Amherst. Morgan 18 | Kaufmann. 19 | 20 | 4. Relevant Information: 21 | 22 | Concerns housing values in suburbs of Boston. 23 | 24 | 5. Number of Instances: 506 25 | 26 | 6. Number of Attributes: 13 continuous attributes (including "class" 27 | attribute "MEDV"), 1 binary-valued attribute. 28 | 29 | 7. Attribute Information: 30 | 31 | 1. CRIM per capita crime rate by town 32 | 2. ZN proportion of residential land zoned for lots over 33 | 25,000 sq.ft. 34 | 3. INDUS proportion of non-retail business acres per town 35 | 4. CHAS Charles River dummy variable (= 1 if tract bounds 36 | river; 0 otherwise) 37 | 5. NOX nitric oxides concentration (parts per 10 million) 38 | 6. RM average number of rooms per dwelling 39 | 7. AGE proportion of owner-occupied units built prior to 1940 40 | 8. DIS weighted distances to five Boston employment centres 41 | 9. RAD index of accessibility to radial highways 42 | 10. TAX full-value property-tax rate per $10,000 43 | 11. PTRATIO pupil-teacher ratio by town 44 | 12. B 1000(Bk - 0.63)^2 where Bk is the proportion of blacks 45 | by town 46 | 13. LSTAT % lower status of the population 47 | 14. MEDV Median value of owner-occupied homes in $1000's 48 | 49 | 8. Missing Attribute Values: None. 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /assignment5/classwork/DSVC/vis_utils.py: -------------------------------------------------------------------------------- 1 | from past.builtins import xrange 2 | 3 | from math import sqrt, ceil 4 | import numpy as np 5 | 6 | def visualize_grid(Xs, ubound=255.0, padding=1): 7 | """ 8 | Reshape a 4D tensor of image data to a grid for easy visualization. 9 | 10 | Inputs: 11 | - Xs: Data of shape (N, H, W, C) 12 | - ubound: Output grid will have values scaled to the range [0, ubound] 13 | - padding: The number of blank pixels between elements of the grid 14 | """ 15 | (N, H, W, C) = Xs.shape 16 | grid_size = int(ceil(sqrt(N))) 17 | grid_height = H * grid_size + padding * (grid_size - 1) 18 | grid_width = W * grid_size + padding * (grid_size - 1) 19 | grid = np.zeros((grid_height, grid_width, C)) 20 | next_idx = 0 21 | y0, y1 = 0, H 22 | for y in xrange(grid_size): 23 | x0, x1 = 0, W 24 | for x in xrange(grid_size): 25 | if next_idx < N: 26 | img = Xs[next_idx] 27 | low, high = np.min(img), np.max(img) 28 | grid[y0:y1, x0:x1] = ubound * (img - low) / (high - low) 29 | # grid[y0:y1, x0:x1] = Xs[next_idx] 30 | next_idx += 1 31 | x0 += W + padding 32 | x1 += W + padding 33 | y0 += H + padding 34 | y1 += H + padding 35 | # grid_max = np.max(grid) 36 | # grid_min = np.min(grid) 37 | # grid = ubound * (grid - grid_min) / (grid_max - grid_min) 38 | return grid 39 | 40 | def vis_grid(Xs): 41 | """ visualize a grid of images """ 42 | (N, H, W, C) = Xs.shape 43 | A = int(ceil(sqrt(N))) 44 | G = np.ones((A*H+A, A*W+A, C), Xs.dtype) 45 | G *= np.min(Xs) 46 | n = 0 47 | for y in range(A): 48 | for x in range(A): 49 | if n < N: 50 | G[y*H+y:(y+1)*H+y, x*W+x:(x+1)*W+x, :] = Xs[n,:,:,:] 51 | n += 1 52 | # normalize to [0,1] 53 | maxg = G.max() 54 | ming = G.min() 55 | G = (G - ming)/(maxg-ming) 56 | return G 57 | 58 | def vis_nn(rows): 59 | """ visualize array of arrays of images """ 60 | N = len(rows) 61 | D = len(rows[0]) 62 | H,W,C = rows[0][0].shape 63 | Xs = rows[0][0] 64 | G = np.ones((N*H+N, D*W+D, C), Xs.dtype) 65 | for y in range(N): 66 | for x in range(D): 67 | G[y*H+y:(y+1)*H+y, x*W+x:(x+1)*W+x, :] = rows[y][x] 68 | # normalize to [0,1] 69 | maxg = G.max() 70 | ming = G.min() 71 | G = (G - ming)/(maxg-ming) 72 | return G 73 | 74 | 75 | 76 | -------------------------------------------------------------------------------- /assignment6/classwork/DSVC/vis_utils.py: -------------------------------------------------------------------------------- 1 | # from past.builtins import xrange 2 | 3 | from math import sqrt, ceil 4 | import numpy as np 5 | 6 | def visualize_grid(Xs, ubound=255.0, padding=1): 7 | """ 8 | Reshape a 4D tensor of image data to a grid for easy visualization. 9 | 10 | Inputs: 11 | - Xs: Data of shape (N, H, W, C) 12 | - ubound: Output grid will have values scaled to the range [0, ubound] 13 | - padding: The number of blank pixels between elements of the grid 14 | """ 15 | (N, H, W, C) = Xs.shape 16 | grid_size = int(ceil(sqrt(N))) 17 | grid_height = H * grid_size + padding * (grid_size - 1) 18 | grid_width = W * grid_size + padding * (grid_size - 1) 19 | grid = np.zeros((grid_height, grid_width, C)) 20 | next_idx = 0 21 | y0, y1 = 0, H 22 | for y in range(grid_size): 23 | x0, x1 = 0, W 24 | for x in range(grid_size): 25 | if next_idx < N: 26 | img = Xs[next_idx] 27 | low, high = np.min(img), np.max(img) 28 | grid[y0:y1, x0:x1] = ubound * (img - low) / (high - low) 29 | # grid[y0:y1, x0:x1] = Xs[next_idx] 30 | next_idx += 1 31 | x0 += W + padding 32 | x1 += W + padding 33 | y0 += H + padding 34 | y1 += H + padding 35 | # grid_max = np.max(grid) 36 | # grid_min = np.min(grid) 37 | # grid = ubound * (grid - grid_min) / (grid_max - grid_min) 38 | return grid 39 | 40 | def vis_grid(Xs): 41 | """ visualize a grid of images """ 42 | (N, H, W, C) = Xs.shape 43 | A = int(ceil(sqrt(N))) 44 | G = np.ones((A*H+A, A*W+A, C), Xs.dtype) 45 | G *= np.min(Xs) 46 | n = 0 47 | for y in range(A): 48 | for x in range(A): 49 | if n < N: 50 | G[y*H+y:(y+1)*H+y, x*W+x:(x+1)*W+x, :] = Xs[n,:,:,:] 51 | n += 1 52 | # normalize to [0,1] 53 | maxg = G.max() 54 | ming = G.min() 55 | G = (G - ming)/(maxg-ming) 56 | return G 57 | 58 | def vis_nn(rows): 59 | """ visualize array of arrays of images """ 60 | N = len(rows) 61 | D = len(rows[0]) 62 | H,W,C = rows[0][0].shape 63 | Xs = rows[0][0] 64 | G = np.ones((N*H+N, D*W+D, C), Xs.dtype) 65 | for y in range(N): 66 | for x in range(D): 67 | G[y*H+y:(y+1)*H+y, x*W+x:(x+1)*W+x, :] = rows[y][x] 68 | # normalize to [0,1] 69 | maxg = G.max() 70 | ming = G.min() 71 | G = (G - ming)/(maxg-ming) 72 | return G 73 | 74 | 75 | 76 | -------------------------------------------------------------------------------- /assignment7/classwork/DSVC/im2col.py: -------------------------------------------------------------------------------- 1 | from builtins import range 2 | import numpy as np 3 | 4 | 5 | def get_im2col_indices(x_shape, field_height, field_width, padding=1, stride=1): 6 | # First figure out what the size of the output should be 7 | N, C, H, W = x_shape 8 | assert (H + 2 * padding - field_height) % stride == 0 9 | assert (W + 2 * padding - field_height) % stride == 0 10 | out_height = (H + 2 * padding - field_height) / stride + 1 11 | out_width = (W + 2 * padding - field_width) / stride + 1 12 | 13 | i0 = np.repeat(np.arange(field_height), field_width) 14 | i0 = np.tile(i0, C) 15 | i1 = stride * np.repeat(np.arange(out_height), out_width) 16 | j0 = np.tile(np.arange(field_width), field_height * C) 17 | j1 = stride * np.tile(np.arange(out_width), out_height) 18 | i = i0.reshape(-1, 1) + i1.reshape(1, -1) 19 | j = j0.reshape(-1, 1) + j1.reshape(1, -1) 20 | 21 | k = np.repeat(np.arange(C), field_height * field_width).reshape(-1, 1) 22 | 23 | return (k, i, j) 24 | 25 | 26 | def im2col_indices(x, field_height, field_width, padding=1, stride=1): 27 | """ An implementation of im2col based on some fancy indexing """ 28 | # Zero-pad the input 29 | p = padding 30 | x_padded = np.pad(x, ((0, 0), (0, 0), (p, p), (p, p)), mode='constant') 31 | 32 | k, i, j = get_im2col_indices(x.shape, field_height, field_width, padding, 33 | stride) 34 | 35 | cols = x_padded[:, k, i, j] 36 | C = x.shape[1] 37 | cols = cols.transpose(1, 2, 0).reshape(field_height * field_width * C, -1) 38 | return cols 39 | 40 | 41 | def col2im_indices(cols, x_shape, field_height=3, field_width=3, padding=1, 42 | stride=1): 43 | """ An implementation of col2im based on fancy indexing and np.add.at """ 44 | N, C, H, W = x_shape 45 | H_padded, W_padded = H + 2 * padding, W + 2 * padding 46 | x_padded = np.zeros((N, C, H_padded, W_padded), dtype=cols.dtype) 47 | k, i, j = get_im2col_indices(x_shape, field_height, field_width, padding, 48 | stride) 49 | cols_reshaped = cols.reshape(C * field_height * field_width, -1, N) 50 | cols_reshaped = cols_reshaped.transpose(2, 0, 1) 51 | np.add.at(x_padded, (slice(None), k, i, j), cols_reshaped) 52 | if padding == 0: 53 | return x_padded 54 | return x_padded[:, :, padding:-padding, padding:-padding] 55 | 56 | pass 57 | -------------------------------------------------------------------------------- /assignment1/classwork/DSVC/vis_utils.py: -------------------------------------------------------------------------------- 1 | from math import sqrt, ceil 2 | import numpy as np 3 | 4 | 5 | def visualize_grid(Xs, ubound=255.0, padding=1): 6 | """ 7 | Reshape a 4D tensor of image data to a grid for easy visualization. 8 | 9 | Inputs: 10 | - Xs: Data of shape (N, H, W, C) 11 | - ubound: Output grid will have values scaled to the range [0, ubound] 12 | - padding: The number of blank pixels between elements of the grid 13 | """ 14 | (N, H, W, C) = Xs.shape 15 | grid_size = int(ceil(sqrt(N))) 16 | grid_height = H * grid_size + padding * (grid_size - 1) 17 | grid_width = W * grid_size + padding * (grid_size - 1) 18 | grid = np.zeros((grid_height, grid_width, C)) 19 | next_idx = 0 20 | y0, y1 = 0, H 21 | for y in xrange(grid_size): 22 | x0, x1 = 0, W 23 | for x in xrange(grid_size): 24 | if next_idx < N: 25 | img = Xs[next_idx] 26 | low, high = np.min(img), np.max(img) 27 | grid[y0:y1, x0:x1] = ubound * (img - low) / (high - low) 28 | # grid[y0:y1, x0:x1] = Xs[next_idx] 29 | next_idx += 1 30 | x0 += W + padding 31 | x1 += W + padding 32 | y0 += H + padding 33 | y1 += H + padding 34 | # grid_max = np.max(grid) 35 | # grid_min = np.min(grid) 36 | # grid = ubound * (grid - grid_min) / (grid_max - grid_min) 37 | return grid 38 | 39 | 40 | def vis_grid(Xs): 41 | """ visualize a grid of images """ 42 | (N, H, W, C) = Xs.shape 43 | A = int(ceil(sqrt(N))) 44 | G = np.ones((A*H+A, A*W+A, C), Xs.dtype) 45 | G *= np.min(Xs) 46 | n = 0 47 | for y in range(A): 48 | for x in range(A): 49 | if n < N: 50 | G[y*H+y:(y+1)*H+y, x*W+x:(x+1)*W+x, :] = Xs[n, :, :, :] 51 | n += 1 52 | # normalize to [0,1] 53 | maxg = G.max() 54 | ming = G.min() 55 | G = (G - ming)/(maxg-ming) 56 | return G 57 | 58 | 59 | def vis_nn(rows): 60 | """ visualize array of arrays of images """ 61 | N = len(rows) 62 | D = len(rows[0]) 63 | H, W, C = rows[0][0].shape 64 | Xs = rows[0][0] 65 | G = np.ones((N*H+N, D*W+D, C), Xs.dtype) 66 | for y in range(N): 67 | for x in range(D): 68 | G[y*H+y:(y+1)*H+y, x*W+x:(x+1)*W+x, :] = rows[y][x] 69 | # normalize to [0,1] 70 | maxg = G.max() 71 | ming = G.min() 72 | G = (G - ming)/(maxg-ming) 73 | return G 74 | -------------------------------------------------------------------------------- /assignment7/classwork/DSVC/vis_utils.py: -------------------------------------------------------------------------------- 1 | from builtins import range 2 | from past.builtins import xrange 3 | 4 | from math import sqrt, ceil 5 | import numpy as np 6 | 7 | def visualize_grid(Xs, ubound=255.0, padding=1): 8 | """ 9 | Reshape a 4D tensor of image data to a grid for easy visualization. 10 | 11 | Inputs: 12 | - Xs: Data of shape (N, H, W, C) 13 | - ubound: Output grid will have values scaled to the range [0, ubound] 14 | - padding: The number of blank pixels between elements of the grid 15 | """ 16 | (N, H, W, C) = Xs.shape 17 | grid_size = int(ceil(sqrt(N))) 18 | grid_height = H * grid_size + padding * (grid_size - 1) 19 | grid_width = W * grid_size + padding * (grid_size - 1) 20 | grid = np.zeros((grid_height, grid_width, C)) 21 | next_idx = 0 22 | y0, y1 = 0, H 23 | for y in range(grid_size): 24 | x0, x1 = 0, W 25 | for x in range(grid_size): 26 | if next_idx < N: 27 | img = Xs[next_idx] 28 | low, high = np.min(img), np.max(img) 29 | grid[y0:y1, x0:x1] = ubound * (img - low) / (high - low) 30 | # grid[y0:y1, x0:x1] = Xs[next_idx] 31 | next_idx += 1 32 | x0 += W + padding 33 | x1 += W + padding 34 | y0 += H + padding 35 | y1 += H + padding 36 | # grid_max = np.max(grid) 37 | # grid_min = np.min(grid) 38 | # grid = ubound * (grid - grid_min) / (grid_max - grid_min) 39 | return grid 40 | 41 | def vis_grid(Xs): 42 | """ visualize a grid of images """ 43 | (N, H, W, C) = Xs.shape 44 | A = int(ceil(sqrt(N))) 45 | G = np.ones((A*H+A, A*W+A, C), Xs.dtype) 46 | G *= np.min(Xs) 47 | n = 0 48 | for y in range(A): 49 | for x in range(A): 50 | if n < N: 51 | G[y*H+y:(y+1)*H+y, x*W+x:(x+1)*W+x, :] = Xs[n,:,:,:] 52 | n += 1 53 | # normalize to [0,1] 54 | maxg = G.max() 55 | ming = G.min() 56 | G = (G - ming)/(maxg-ming) 57 | return G 58 | 59 | def vis_nn(rows): 60 | """ visualize array of arrays of images """ 61 | N = len(rows) 62 | D = len(rows[0]) 63 | H,W,C = rows[0][0].shape 64 | Xs = rows[0][0] 65 | G = np.ones((N*H+N, D*W+D, C), Xs.dtype) 66 | for y in range(N): 67 | for x in range(D): 68 | G[y*H+y:(y+1)*H+y, x*W+x:(x+1)*W+x, :] = rows[y][x] 69 | # normalize to [0,1] 70 | maxg = G.max() 71 | ming = G.min() 72 | G = (G - ming)/(maxg-ming) 73 | return G 74 | -------------------------------------------------------------------------------- /assignment1/README.md: -------------------------------------------------------------------------------- 1 | ## assignment1 2 | 3 | #### 开发环境工具 4 | 5 | 在这第一个部分，大家要完成两个工具的安装，分别是 Anaconda 和 Jupyter notebook。 6 | 7 | - Python 8 | - Anaconda 是一个包含数据科学常用包的发行版本。它基于 conda ——一个包和环境管理器——衍生而来。你将使用 conda 创建环境，以便分隔使用不同 Python 版本和/或不同包的项目。你还将使用它在环境中安装、卸载和更新包。通过使用 Anaconda，使我处理数据的过程更加愉快。 9 | - 下载地址 [https://www.anaconda.com/download/](https://www.anaconda.com/download/) 10 | - 安装教程 [Windows](http://www.jianshu.com/p/cd35110f1ed0)，[Linux](http://blog.topspeedsnail.com/archives/9607) 11 | - Jupyter Notebook（此前被称为 IPython notebook）是一个交互式笔记本，支持运行 40 多种编程语言。Jupyter Notebook 的本质是一个 Web 应用程序，便于创建和共享文学化程序文档，支持实时代码，数学方程，可视化和 [markdown](https://baike.baidu.com/item/markdown)。 用途包括：数据清理和转换，数值模拟，统计建模，机器学习等等。 12 | - [快速入门教程](https://sdk.cn/news/4523) 13 | - Markdown 14 | - 项目文件里涉及到很多markdown(.md)格式的文件，可以选择下载[typora](https://typora.io/)来打开编辑Markdown文件。 15 | - IDE & 文本编辑器 16 | - Python IDE推荐 [PyCharm](https://www.jetbrains.com/pycharm/?fromMenu) 17 | - 也可以选择文本编辑器来修改.py文件 推荐[visual studio code](https://code.visualstudio.com/) 。（我个人倾向使用VScode） 18 | - 校园网下向Github通过ssh push更新不成功的解决办法 19 | - ssh使用22端口，本地网络为了网络安全会禁掉这个端口，可以选择把ssh改为443端口。[本地网络禁止 22 端口出站时怎样使用 Git+SSH](https://hyjk2000.github.io/2016/02/24/git-ssh-with-port-22-outbound-blocked.html) 20 | 21 | #### 基础学习 22 | 23 | - [Python教程](https://www.liaoxuefeng.com/wiki/0014316089557264a6b348958f449949df42a6d3a2e542c000) 24 | - [Git教程](https://www.liaoxuefeng.com/wiki/0013739516305929606dd18361248578c67b8067c8c017b000) 25 | - Python科学计算 [Numpy & Pandas](https://morvanzhou.github.io/tutorials/data-manipulation/np-pd/) 26 | - Python可视化（可以先作为了解） [Matplotlib](https://morvanzhou.github.io/tutorials/data-manipulation/plt/) 27 | - [cs231n-Python-numpy-tutorial](http://cs231n.github.io/python-numpy-tutorial/) 28 | 29 | 几个会用到的概念 30 | 31 | - [机器学习中的Bias(偏差)，Error(误差)，和Variance(方差)有什么区别和联系？](https://www.zhihu.com/question/27068705) 32 | - [K折交叉验证评估模型性能](https://github.com/basicv8vc/Python-Machine-Learning-zh/blob/master/%E7%AC%AC%E5%85%AD%E7%AB%A0/ch6_section2.md) 33 | - 机器学习通俗化系列文章 34 | - [写给大家看的机器学习书 第一篇](https://zhuanlan.zhihu.com/p/25328686) 35 | - [写给大家看的机器学习书 第二篇](https://zhuanlan.zhihu.com/p/25439997) 36 | - [写给大家看的机器学习书 第三篇](https://zhuanlan.zhihu.com/p/25358695) 37 | - [写给大家看的机器学习书 第四篇](https://zhuanlan.zhihu.com/p/25721582) 38 | - KNN 39 | - [Image Classification: Data-driven Approach, k-Nearest Neighbor, train/val/test splits](http://cs231n.github.io/classification/) 40 | - [A Complete Guide to K-Nearest-Neighbors with Applications in Python and R](https://kevinzakka.github.io/2016/07/13/k-nearest-neighbor/) 41 | 42 | #### 作业 43 | 44 | - [k-Nearest Neighbor classifier](classwork) 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /machinelearning/Lesson2/SavingModel.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Saving Model" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## 实验说明\n", 15 | "\n", 16 | "训练模型的保存和读取" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 1, 22 | "metadata": { 23 | "collapsed": true 24 | }, 25 | "outputs": [], 26 | "source": [ 27 | "import pandas as pd\n", 28 | "from sklearn.linear_model import LogisticRegression\n", 29 | "from sklearn.preprocessing import StandardScaler, PolynomialFeatures\n", 30 | "from sklearn.pipeline import Pipeline\n", 31 | "from sklearn.metrics import accuracy_score\n", 32 | "import os\n", 33 | "from sklearn.externals import joblib" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 2, 39 | "metadata": {}, 40 | "outputs": [ 41 | { 42 | "name": "stdout", 43 | "output_type": "stream", 44 | "text": [ 45 | "Load Model...\n", 46 | "y_hat = \n", 47 | " [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", 48 | " 0 0 0 0 0 0 0 0 0 0 0 0 0 2 2 2 1 2 1 2 1 2 1 1 1 1 1 1 2 1 1 1 1 1 1 1 1\n", 49 | " 2 2 2 2 1 1 1 1 1 1 1 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1 2 1 2 1 2 2 1 2 2 2 2\n", 50 | " 2 2 1 1 2 2 2 2 1 2 1 2 1 2 2 1 1 2 2 2 2 2 1 1 2 2 2 1 2 2 2 1 2 2 2 1 2\n", 51 | " 2 1]\n", 52 | "accuracy = 80.667%\n" 53 | ] 54 | } 55 | ], 56 | "source": [ 57 | "data = pd.read_csv('iris.data', header=None)\n", 58 | "x = data[[0, 1]]\n", 59 | "y = pd.Categorical(data[4]).codes\n", 60 | "if os.path.exists('iris.model'):\n", 61 | " print('Load Model...')\n", 62 | " lr = joblib.load('iris.model')\n", 63 | "else:\n", 64 | " print('Train Model...')\n", 65 | " lr = Pipeline([('sc', StandardScaler()),\n", 66 | " ('poly', PolynomialFeatures(degree=3)),\n", 67 | " ('clf', LogisticRegression()) ])\n", 68 | " lr.fit(x, y.ravel())\n", 69 | "y_hat = lr.predict(x)\n", 70 | "joblib.dump(lr, 'iris.model')\n", 71 | "print('y_hat = \\n', y_hat)\n", 72 | "print('accuracy = %.3f%%' % (100*accuracy_score(y, y_hat)))" 73 | ] 74 | } 75 | ], 76 | "metadata": { 77 | "kernelspec": { 78 | "display_name": "Python 3", 79 | "language": "python", 80 | "name": "python3" 81 | }, 82 | "language_info": { 83 | "codemirror_mode": { 84 | "name": "ipython", 85 | "version": 3 86 | }, 87 | "file_extension": ".py", 88 | "mimetype": "text/x-python", 89 | "name": "python", 90 | "nbconvert_exporter": "python", 91 | "pygments_lexer": "ipython3", 92 | "version": "3.6.3" 93 | } 94 | }, 95 | "nbformat": 4, 96 | "nbformat_minor": 2 97 | } 98 | -------------------------------------------------------------------------------- /assignment7/classwork/DSVC/layer_utils.py: -------------------------------------------------------------------------------- 1 | from DSVC.layers import * 2 | from DSVC.fast_layers import * 3 | 4 | 5 | def affine_relu_forward(x, w, b): 6 | """ 7 | Convenience layer that perorms an affine transform followed by a ReLU 8 | 9 | Inputs: 10 | - x: Input to the affine layer 11 | - w, b: Weights for the affine layer 12 | 13 | Returns a tuple of: 14 | - out: Output from the ReLU 15 | - cache: Object to give to the backward pass 16 | """ 17 | a, fc_cache = affine_forward(x, w, b) 18 | out, relu_cache = relu_forward(a) 19 | cache = (fc_cache, relu_cache) 20 | return out, cache 21 | 22 | 23 | def affine_relu_backward(dout, cache): 24 | """ 25 | Backward pass for the affine-relu convenience layer 26 | """ 27 | fc_cache, relu_cache = cache 28 | da = relu_backward(dout, relu_cache) 29 | dx, dw, db = affine_backward(da, fc_cache) 30 | return dx, dw, db 31 | 32 | 33 | def conv_relu_forward(x, w, b, conv_param): 34 | """ 35 | A convenience layer that performs a convolution followed by a ReLU. 36 | 37 | Inputs: 38 | - x: Input to the convolutional layer 39 | - w, b, conv_param: Weights and parameters for the convolutional layer 40 | 41 | Returns a tuple of: 42 | - out: Output from the ReLU 43 | - cache: Object to give to the backward pass 44 | """ 45 | a, conv_cache = conv_forward_fast(x, w, b, conv_param) 46 | out, relu_cache = relu_forward(a) 47 | cache = (conv_cache, relu_cache) 48 | return out, cache 49 | 50 | 51 | def conv_relu_backward(dout, cache): 52 | """ 53 | Backward pass for the conv-relu convenience layer. 54 | """ 55 | conv_cache, relu_cache = cache 56 | da = relu_backward(dout, relu_cache) 57 | dx, dw, db = conv_backward_fast(da, conv_cache) 58 | return dx, dw, db 59 | 60 | 61 | def conv_bn_relu_forward(x, w, b, gamma, beta, conv_param, bn_param): 62 | a, conv_cache = conv_forward_fast(x, w, b, conv_param) 63 | an, bn_cache = spatial_batchnorm_forward(a, gamma, beta, bn_param) 64 | out, relu_cache = relu_forward(an) 65 | cache = (conv_cache, bn_cache, relu_cache) 66 | return out, cache 67 | 68 | 69 | def conv_bn_relu_backward(dout, cache): 70 | conv_cache, bn_cache, relu_cache = cache 71 | dan = relu_backward(dout, relu_cache) 72 | da, dgamma, dbeta = spatial_batchnorm_backward(dan, bn_cache) 73 | dx, dw, db = conv_backward_fast(da, conv_cache) 74 | return dx, dw, db, dgamma, dbeta 75 | 76 | 77 | def conv_relu_pool_forward(x, w, b, conv_param, pool_param): 78 | """ 79 | Convenience layer that performs a convolution, a ReLU, and a pool. 80 | 81 | Inputs: 82 | - x: Input to the convolutional layer 83 | - w, b, conv_param: Weights and parameters for the convolutional layer 84 | - pool_param: Parameters for the pooling layer 85 | 86 | Returns a tuple of: 87 | - out: Output from the pooling layer 88 | - cache: Object to give to the backward pass 89 | """ 90 | a, conv_cache = conv_forward_fast(x, w, b, conv_param) 91 | s, relu_cache = relu_forward(a) 92 | out, pool_cache = max_pool_forward_fast(s, pool_param) 93 | cache = (conv_cache, relu_cache, pool_cache) 94 | return out, cache 95 | 96 | 97 | def conv_relu_pool_backward(dout, cache): 98 | """ 99 | Backward pass for the conv-relu-pool convenience layer 100 | """ 101 | conv_cache, relu_cache, pool_cache = cache 102 | ds = max_pool_backward_fast(dout, pool_cache) 103 | da = relu_backward(ds, relu_cache) 104 | dx, dw, db = conv_backward_fast(da, conv_cache) 105 | return dx, dw, db 106 | -------------------------------------------------------------------------------- /assignment5/classwork/DSVC/classifiers/linear_svm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from random import shuffle 3 | from past.builtins import xrange 4 | 5 | def svm_loss_naive(W, X, y, reg): 6 | """ 7 | Structured SVM loss function, naive implementation (with loops). 8 | 9 | Inputs have dimension D, there are C classes, and we operate on minibatches 10 | of N examples. 11 | 12 | Inputs: 13 | - W: A numpy array of shape (D, C) containing weights. 14 | - X: A numpy array of shape (N, D) containing a minibatch of data. 15 | - y: A numpy array of shape (N,) containing training labels; y[i] = c means 16 | that X[i] has label c, where 0 <= c < C. 17 | - reg: (float) regularization strength 18 | 19 | Returns a tuple of: 20 | - loss as single float 21 | - gradient with respect to weights W; an array of same shape as W 22 | """ 23 | dW = np.zeros(W.shape) # initialize the gradient as zero 24 | 25 | # compute the loss and the gradient 26 | num_classes = W.shape[1] 27 | num_train = X.shape[0] 28 | loss = 0.0 29 | for i in xrange(num_train): 30 | scores = X[i].dot(W) 31 | correct_class_score = scores[y[i]] 32 | for j in xrange(num_classes): 33 | if j == y[i]: 34 | continue 35 | margin = scores[j] - correct_class_score + 1 # note delta = 1 36 | if margin > 0: 37 | loss += margin 38 | 39 | # Right now the loss is a sum over all training examples, but we want it 40 | # to be an average instead so we divide by num_train. 41 | loss /= num_train 42 | 43 | # Add regularization to the loss. 44 | loss += reg * np.sum(W * W) 45 | 46 | ############################################################################# 47 | # TODO: # 48 | # Compute the gradient of the loss function and store it dW. # 49 | # Rather that first computing the loss and then computing the derivative, # 50 | # it may be simpler to compute the derivative at the same time that the # 51 | # loss is being computed. As a result you may need to modify some of the # 52 | # code above to compute the gradient. # 53 | ############################################################################# 54 | 55 | 56 | return loss, dW 57 | 58 | 59 | def svm_loss_vectorized(W, X, y, reg): 60 | """ 61 | Structured SVM loss function, vectorized implementation. 62 | 63 | Inputs and outputs are the same as svm_loss_naive. 64 | """ 65 | loss = 0.0 66 | dW = np.zeros(W.shape) # initialize the gradient as zero 67 | 68 | ############################################################################# 69 | # TODO: # 70 | # Implement a vectorized version of the structured SVM loss, storing the # 71 | # result in loss. # 72 | ############################################################################# 73 | pass 74 | ############################################################################# 75 | # END OF YOUR CODE # 76 | ############################################################################# 77 | 78 | 79 | ############################################################################# 80 | # TODO: # 81 | # Implement a vectorized version of the gradient for the structured SVM # 82 | # loss, storing the result in dW. # 83 | # # 84 | # Hint: Instead of computing the gradient from scratch, it may be easier # 85 | # to reuse some of the intermediate values that you used to compute the # 86 | # loss. # 87 | ############################################################################# 88 | pass 89 | ############################################################################# 90 | # END OF YOUR CODE # 91 | ############################################################################# 92 | 93 | return loss, dW 94 | -------------------------------------------------------------------------------- /assignment5/classwork/DSVC/gradient_check.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from past.builtins import xrange 3 | 4 | import numpy as np 5 | from random import randrange 6 | 7 | def eval_numerical_gradient(f, x, verbose=True, h=0.00001): 8 | """ 9 | a naive implementation of numerical gradient of f at x 10 | - f should be a function that takes a single argument 11 | - x is the point (numpy array) to evaluate the gradient at 12 | """ 13 | 14 | fx = f(x) # evaluate function value at original point 15 | grad = np.zeros_like(x) 16 | # iterate over all indexes in x 17 | it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite']) 18 | while not it.finished: 19 | 20 | # evaluate function at x+h 21 | ix = it.multi_index 22 | oldval = x[ix] 23 | x[ix] = oldval + h # increment by h 24 | fxph = f(x) # evalute f(x + h) 25 | x[ix] = oldval - h 26 | fxmh = f(x) # evaluate f(x - h) 27 | x[ix] = oldval # restore 28 | 29 | # compute the partial derivative with centered formula 30 | grad[ix] = (fxph - fxmh) / (2 * h) # the slope 31 | if verbose: 32 | print(ix, grad[ix]) 33 | it.iternext() # step to next dimension 34 | 35 | return grad 36 | 37 | 38 | def eval_numerical_gradient_array(f, x, df, h=1e-5): 39 | """ 40 | Evaluate a numeric gradient for a function that accepts a numpy 41 | array and returns a numpy array. 42 | """ 43 | grad = np.zeros_like(x) 44 | it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite']) 45 | while not it.finished: 46 | ix = it.multi_index 47 | 48 | oldval = x[ix] 49 | x[ix] = oldval + h 50 | pos = f(x).copy() 51 | x[ix] = oldval - h 52 | neg = f(x).copy() 53 | x[ix] = oldval 54 | 55 | grad[ix] = np.sum((pos - neg) * df) / (2 * h) 56 | it.iternext() 57 | return grad 58 | 59 | 60 | def eval_numerical_gradient_blobs(f, inputs, output, h=1e-5): 61 | """ 62 | Compute numeric gradients for a function that operates on input 63 | and output blobs. 64 | 65 | We assume that f accepts several input blobs as arguments, followed by a blob 66 | into which outputs will be written. For example, f might be called like this: 67 | 68 | f(x, w, out) 69 | 70 | where x and w are input Blobs, and the result of f will be written to out. 71 | 72 | Inputs: 73 | - f: function 74 | - inputs: tuple of input blobs 75 | - output: output blob 76 | - h: step size 77 | """ 78 | numeric_diffs = [] 79 | for input_blob in inputs: 80 | diff = np.zeros_like(input_blob.diffs) 81 | it = np.nditer(input_blob.vals, flags=['multi_index'], 82 | op_flags=['readwrite']) 83 | while not it.finished: 84 | idx = it.multi_index 85 | orig = input_blob.vals[idx] 86 | 87 | input_blob.vals[idx] = orig + h 88 | f(*(inputs + (output,))) 89 | pos = np.copy(output.vals) 90 | input_blob.vals[idx] = orig - h 91 | f(*(inputs + (output,))) 92 | neg = np.copy(output.vals) 93 | input_blob.vals[idx] = orig 94 | 95 | diff[idx] = np.sum((pos - neg) * output.diffs) / (2.0 * h) 96 | 97 | it.iternext() 98 | numeric_diffs.append(diff) 99 | return numeric_diffs 100 | 101 | 102 | def eval_numerical_gradient_net(net, inputs, output, h=1e-5): 103 | return eval_numerical_gradient_blobs(lambda *args: net.forward(), 104 | inputs, output, h=h) 105 | 106 | 107 | def grad_check_sparse(f, x, analytic_grad, num_checks=10, h=1e-5): 108 | """ 109 | sample a few random elements and only return numerical 110 | in this dimensions. 111 | """ 112 | 113 | for i in xrange(num_checks): 114 | ix = tuple([randrange(m) for m in x.shape]) 115 | 116 | oldval = x[ix] 117 | x[ix] = oldval + h # increment by h 118 | fxph = f(x) # evaluate f(x + h) 119 | x[ix] = oldval - h # increment by h 120 | fxmh = f(x) # evaluate f(x - h) 121 | x[ix] = oldval # reset 122 | 123 | grad_numerical = (fxph - fxmh) / (2 * h) 124 | grad_analytic = analytic_grad[ix] 125 | rel_error = abs(grad_numerical - grad_analytic) / (abs(grad_numerical) + abs(grad_analytic)) 126 | print('numerical: %f analytic: %f, relative error: %e' % (grad_numerical, grad_analytic, rel_error)) 127 | 128 | -------------------------------------------------------------------------------- /assignment6/classwork/DSVC/gradient_check.py: -------------------------------------------------------------------------------- 1 | # from __future__ import print_function 2 | # from past.builtins import xrange 3 | 4 | import numpy as np 5 | from random import randrange 6 | 7 | def eval_numerical_gradient(f, x, verbose=True, h=0.00001): 8 | """ 9 | a naive implementation of numerical gradient of f at x 10 | - f should be a function that takes a single argument 11 | - x is the point (numpy array) to evaluate the gradient at 12 | """ 13 | 14 | fx = f(x) # evaluate function value at original point 15 | grad = np.zeros_like(x) 16 | # iterate over all indexes in x 17 | it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite']) 18 | while not it.finished: 19 | 20 | # evaluate function at x+h 21 | ix = it.multi_index 22 | oldval = x[ix] 23 | x[ix] = oldval + h # increment by h 24 | fxph = f(x) # evalute f(x + h) 25 | x[ix] = oldval - h 26 | fxmh = f(x) # evaluate f(x - h) 27 | x[ix] = oldval # restore 28 | 29 | # compute the partial derivative with centered formula 30 | grad[ix] = (fxph - fxmh) / (2 * h) # the slope 31 | if verbose: 32 | print(ix, grad[ix]) 33 | it.iternext() # step to next dimension 34 | 35 | return grad 36 | 37 | 38 | def eval_numerical_gradient_array(f, x, df, h=1e-5): 39 | """ 40 | Evaluate a numeric gradient for a function that accepts a numpy 41 | array and returns a numpy array. 42 | """ 43 | grad = np.zeros_like(x) 44 | it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite']) 45 | while not it.finished: 46 | ix = it.multi_index 47 | 48 | oldval = x[ix] 49 | x[ix] = oldval + h 50 | pos = f(x).copy() 51 | x[ix] = oldval - h 52 | neg = f(x).copy() 53 | x[ix] = oldval 54 | 55 | grad[ix] = np.sum((pos - neg) * df) / (2 * h) 56 | it.iternext() 57 | return grad 58 | 59 | 60 | def eval_numerical_gradient_blobs(f, inputs, output, h=1e-5): 61 | """ 62 | Compute numeric gradients for a function that operates on input 63 | and output blobs. 64 | 65 | We assume that f accepts several input blobs as arguments, followed by a blob 66 | into which outputs will be written. For example, f might be called like this: 67 | 68 | f(x, w, out) 69 | 70 | where x and w are input Blobs, and the result of f will be written to out. 71 | 72 | Inputs: 73 | - f: function 74 | - inputs: tuple of input blobs 75 | - output: output blob 76 | - h: step size 77 | """ 78 | numeric_diffs = [] 79 | for input_blob in inputs: 80 | diff = np.zeros_like(input_blob.diffs) 81 | it = np.nditer(input_blob.vals, flags=['multi_index'], 82 | op_flags=['readwrite']) 83 | while not it.finished: 84 | idx = it.multi_index 85 | orig = input_blob.vals[idx] 86 | 87 | input_blob.vals[idx] = orig + h 88 | f(*(inputs + (output,))) 89 | pos = np.copy(output.vals) 90 | input_blob.vals[idx] = orig - h 91 | f(*(inputs + (output,))) 92 | neg = np.copy(output.vals) 93 | input_blob.vals[idx] = orig 94 | 95 | diff[idx] = np.sum((pos - neg) * output.diffs) / (2.0 * h) 96 | 97 | it.iternext() 98 | numeric_diffs.append(diff) 99 | return numeric_diffs 100 | 101 | 102 | def eval_numerical_gradient_net(net, inputs, output, h=1e-5): 103 | return eval_numerical_gradient_blobs(lambda *args: net.forward(), 104 | inputs, output, h=h) 105 | 106 | 107 | def grad_check_sparse(f, x, analytic_grad, num_checks=10, h=1e-5): 108 | """ 109 | sample a few random elements and only return numerical 110 | in this dimensions. 111 | """ 112 | 113 | for i in xrange(num_checks): 114 | ix = tuple([randrange(m) for m in x.shape]) 115 | 116 | oldval = x[ix] 117 | x[ix] = oldval + h # increment by h 118 | fxph = f(x) # evaluate f(x + h) 119 | x[ix] = oldval - h # increment by h 120 | fxmh = f(x) # evaluate f(x - h) 121 | x[ix] = oldval # reset 122 | 123 | grad_numerical = (fxph - fxmh) / (2 * h) 124 | grad_analytic = analytic_grad[ix] 125 | rel_error = abs(grad_numerical - grad_analytic) / (abs(grad_numerical) + abs(grad_analytic)) 126 | print('numerical: %f analytic: %f, relative error: %e' % (grad_numerical, grad_analytic, rel_error)) 127 | 128 | -------------------------------------------------------------------------------- /assignment1/classwork/DSVC/gradient_check.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from random import randrange 3 | 4 | 5 | def eval_numerical_gradient(f, x, verbose=True, h=0.00001): 6 | """ 7 | a naive implementation of numerical gradient of f at x 8 | - f should be a function that takes a single argument 9 | - x is the point (numpy array) to evaluate the gradient at 10 | """ 11 | 12 | fx = f(x) # evaluate function value at original point 13 | grad = np.zeros_like(x) 14 | # iterate over all indexes in x 15 | it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite']) 16 | while not it.finished: 17 | 18 | # evaluate function at x+h 19 | ix = it.multi_index 20 | oldval = x[ix] 21 | x[ix] = oldval + h # increment by h 22 | fxph = f(x) # evalute f(x + h) 23 | x[ix] = oldval - h 24 | fxmh = f(x) # evaluate f(x - h) 25 | x[ix] = oldval # restore 26 | 27 | # compute the partial derivative with centered formula 28 | grad[ix] = (fxph - fxmh) / (2 * h) # the slope 29 | if verbose: 30 | print ix, grad[ix] 31 | it.iternext() # step to next dimension 32 | 33 | return grad 34 | 35 | 36 | def eval_numerical_gradient_array(f, x, df, h=1e-5): 37 | """ 38 | Evaluate a numeric gradient for a function that accepts a numpy 39 | array and returns a numpy array. 40 | """ 41 | grad = np.zeros_like(x) 42 | it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite']) 43 | while not it.finished: 44 | ix = it.multi_index 45 | 46 | oldval = x[ix] 47 | x[ix] = oldval + h 48 | pos = f(x).copy() 49 | x[ix] = oldval - h 50 | neg = f(x).copy() 51 | x[ix] = oldval 52 | 53 | grad[ix] = np.sum((pos - neg) * df) / (2 * h) 54 | it.iternext() 55 | return grad 56 | 57 | 58 | def eval_numerical_gradient_blobs(f, inputs, output, h=1e-5): 59 | """ 60 | Compute numeric gradients for a function that operates on input 61 | and output blobs. 62 | 63 | We assume that f accepts several input blobs as arguments, followed by a blob 64 | into which outputs will be written. For example, f might be called like this: 65 | 66 | f(x, w, out) 67 | 68 | where x and w are input Blobs, and the result of f will be written to out. 69 | 70 | Inputs: 71 | - f: function 72 | - inputs: tuple of input blobs 73 | - output: output blob 74 | - h: step size 75 | """ 76 | numeric_diffs = [] 77 | for input_blob in inputs: 78 | diff = np.zeros_like(input_blob.diffs) 79 | it = np.nditer(input_blob.vals, flags=['multi_index'], 80 | op_flags=['readwrite']) 81 | while not it.finished: 82 | idx = it.multi_index 83 | orig = input_blob.vals[idx] 84 | 85 | input_blob.vals[idx] = orig + h 86 | f(*(inputs + (output,))) 87 | pos = np.copy(output.vals) 88 | input_blob.vals[idx] = orig - h 89 | f(*(inputs + (output,))) 90 | neg = np.copy(output.vals) 91 | input_blob.vals[idx] = orig 92 | 93 | diff[idx] = np.sum((pos - neg) * output.diffs) / (2.0 * h) 94 | 95 | it.iternext() 96 | numeric_diffs.append(diff) 97 | return numeric_diffs 98 | 99 | 100 | def eval_numerical_gradient_net(net, inputs, output, h=1e-5): 101 | return eval_numerical_gradient_blobs(lambda *args: net.forward(), 102 | inputs, output, h=h) 103 | 104 | 105 | def grad_check_sparse(f, x, analytic_grad, num_checks=10, h=1e-5): 106 | """ 107 | sample a few random elements and only return numerical 108 | in this dimensions. 109 | """ 110 | 111 | for i in xrange(num_checks): 112 | ix = tuple([randrange(m) for m in x.shape]) 113 | 114 | oldval = x[ix] 115 | x[ix] = oldval + h # increment by h 116 | fxph = f(x) # evaluate f(x + h) 117 | x[ix] = oldval - h # increment by h 118 | fxmh = f(x) # evaluate f(x - h) 119 | x[ix] = oldval # reset 120 | 121 | grad_numerical = (fxph - fxmh) / (2 * h) 122 | grad_analytic = analytic_grad[ix] 123 | rel_error = abs(grad_numerical - grad_analytic) / \ 124 | (abs(grad_numerical) + abs(grad_analytic)) 125 | print 'numerical: %f analytic: %f, relative error: %e' % (grad_numerical, grad_analytic, rel_error) 126 | -------------------------------------------------------------------------------- /assignment7/classwork/DSVC/gradient_check.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from builtins import range 3 | from past.builtins import xrange 4 | 5 | import numpy as np 6 | from random import randrange 7 | 8 | def eval_numerical_gradient(f, x, verbose=True, h=0.00001): 9 | """ 10 | a naive implementation of numerical gradient of f at x 11 | - f should be a function that takes a single argument 12 | - x is the point (numpy array) to evaluate the gradient at 13 | """ 14 | 15 | fx = f(x) # evaluate function value at original point 16 | grad = np.zeros_like(x) 17 | # iterate over all indexes in x 18 | it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite']) 19 | while not it.finished: 20 | 21 | # evaluate function at x+h 22 | ix = it.multi_index 23 | oldval = x[ix] 24 | x[ix] = oldval + h # increment by h 25 | fxph = f(x) # evalute f(x + h) 26 | x[ix] = oldval - h 27 | fxmh = f(x) # evaluate f(x - h) 28 | x[ix] = oldval # restore 29 | 30 | # compute the partial derivative with centered formula 31 | grad[ix] = (fxph - fxmh) / (2 * h) # the slope 32 | if verbose: 33 | print(ix, grad[ix]) 34 | it.iternext() # step to next dimension 35 | 36 | return grad 37 | 38 | 39 | def eval_numerical_gradient_array(f, x, df, h=1e-5): 40 | """ 41 | Evaluate a numeric gradient for a function that accepts a numpy 42 | array and returns a numpy array. 43 | """ 44 | grad = np.zeros_like(x) 45 | it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite']) 46 | while not it.finished: 47 | ix = it.multi_index 48 | 49 | oldval = x[ix] 50 | x[ix] = oldval + h 51 | pos = f(x).copy() 52 | x[ix] = oldval - h 53 | neg = f(x).copy() 54 | x[ix] = oldval 55 | 56 | grad[ix] = np.sum((pos - neg) * df) / (2 * h) 57 | it.iternext() 58 | return grad 59 | 60 | 61 | def eval_numerical_gradient_blobs(f, inputs, output, h=1e-5): 62 | """ 63 | Compute numeric gradients for a function that operates on input 64 | and output blobs. 65 | 66 | We assume that f accepts several input blobs as arguments, followed by a 67 | blob where outputs will be written. For example, f might be called like: 68 | 69 | f(x, w, out) 70 | 71 | where x and w are input Blobs, and the result of f will be written to out. 72 | 73 | Inputs: 74 | - f: function 75 | - inputs: tuple of input blobs 76 | - output: output blob 77 | - h: step size 78 | """ 79 | numeric_diffs = [] 80 | for input_blob in inputs: 81 | diff = np.zeros_like(input_blob.diffs) 82 | it = np.nditer(input_blob.vals, flags=['multi_index'], 83 | op_flags=['readwrite']) 84 | while not it.finished: 85 | idx = it.multi_index 86 | orig = input_blob.vals[idx] 87 | 88 | input_blob.vals[idx] = orig + h 89 | f(*(inputs + (output,))) 90 | pos = np.copy(output.vals) 91 | input_blob.vals[idx] = orig - h 92 | f(*(inputs + (output,))) 93 | neg = np.copy(output.vals) 94 | input_blob.vals[idx] = orig 95 | 96 | diff[idx] = np.sum((pos - neg) * output.diffs) / (2.0 * h) 97 | 98 | it.iternext() 99 | numeric_diffs.append(diff) 100 | return numeric_diffs 101 | 102 | 103 | def eval_numerical_gradient_net(net, inputs, output, h=1e-5): 104 | return eval_numerical_gradient_blobs(lambda *args: net.forward(), 105 | inputs, output, h=h) 106 | 107 | 108 | def grad_check_sparse(f, x, analytic_grad, num_checks=10, h=1e-5): 109 | """ 110 | sample a few random elements and only return numerical 111 | in this dimensions. 112 | """ 113 | 114 | for i in range(num_checks): 115 | ix = tuple([randrange(m) for m in x.shape]) 116 | 117 | oldval = x[ix] 118 | x[ix] = oldval + h # increment by h 119 | fxph = f(x) # evaluate f(x + h) 120 | x[ix] = oldval - h # increment by h 121 | fxmh = f(x) # evaluate f(x - h) 122 | x[ix] = oldval # reset 123 | 124 | grad_numerical = (fxph - fxmh) / (2 * h) 125 | grad_analytic = analytic_grad[ix] 126 | rel_error = (abs(grad_numerical - grad_analytic) / 127 | (abs(grad_numerical) + abs(grad_analytic))) 128 | print('numerical: %f analytic: %f, relative error: %e' 129 | %(grad_numerical, grad_analytic, rel_error)) 130 | -------------------------------------------------------------------------------- /machinelearning/Lesson1/iris.data: -------------------------------------------------------------------------------- 1 | 5.1,3.5,1.4,0.2,Iris-setosa 2 | 4.9,3.0,1.4,0.2,Iris-setosa 3 | 4.7,3.2,1.3,0.2,Iris-setosa 4 | 4.6,3.1,1.5,0.2,Iris-setosa 5 | 5.0,3.6,1.4,0.2,Iris-setosa 6 | 5.4,3.9,1.7,0.4,Iris-setosa 7 | 4.6,3.4,1.4,0.3,Iris-setosa 8 | 5.0,3.4,1.5,0.2,Iris-setosa 9 | 4.4,2.9,1.4,0.2,Iris-setosa 10 | 4.9,3.1,1.5,0.1,Iris-setosa 11 | 5.4,3.7,1.5,0.2,Iris-setosa 12 | 4.8,3.4,1.6,0.2,Iris-setosa 13 | 4.8,3.0,1.4,0.1,Iris-setosa 14 | 4.3,3.0,1.1,0.1,Iris-setosa 15 | 5.8,4.0,1.2,0.2,Iris-setosa 16 | 5.7,4.4,1.5,0.4,Iris-setosa 17 | 5.4,3.9,1.3,0.4,Iris-setosa 18 | 5.1,3.5,1.4,0.3,Iris-setosa 19 | 5.7,3.8,1.7,0.3,Iris-setosa 20 | 5.1,3.8,1.5,0.3,Iris-setosa 21 | 5.4,3.4,1.7,0.2,Iris-setosa 22 | 5.1,3.7,1.5,0.4,Iris-setosa 23 | 4.6,3.6,1.0,0.2,Iris-setosa 24 | 5.1,3.3,1.7,0.5,Iris-setosa 25 | 4.8,3.4,1.9,0.2,Iris-setosa 26 | 5.0,3.0,1.6,0.2,Iris-setosa 27 | 5.0,3.4,1.6,0.4,Iris-setosa 28 | 5.2,3.5,1.5,0.2,Iris-setosa 29 | 5.2,3.4,1.4,0.2,Iris-setosa 30 | 4.7,3.2,1.6,0.2,Iris-setosa 31 | 4.8,3.1,1.6,0.2,Iris-setosa 32 | 5.4,3.4,1.5,0.4,Iris-setosa 33 | 5.2,4.1,1.5,0.1,Iris-setosa 34 | 5.5,4.2,1.4,0.2,Iris-setosa 35 | 4.9,3.1,1.5,0.1,Iris-setosa 36 | 5.0,3.2,1.2,0.2,Iris-setosa 37 | 5.5,3.5,1.3,0.2,Iris-setosa 38 | 4.9,3.1,1.5,0.1,Iris-setosa 39 | 4.4,3.0,1.3,0.2,Iris-setosa 40 | 5.1,3.4,1.5,0.2,Iris-setosa 41 | 5.0,3.5,1.3,0.3,Iris-setosa 42 | 4.5,2.3,1.3,0.3,Iris-setosa 43 | 4.4,3.2,1.3,0.2,Iris-setosa 44 | 5.0,3.5,1.6,0.6,Iris-setosa 45 | 5.1,3.8,1.9,0.4,Iris-setosa 46 | 4.8,3.0,1.4,0.3,Iris-setosa 47 | 5.1,3.8,1.6,0.2,Iris-setosa 48 | 4.6,3.2,1.4,0.2,Iris-setosa 49 | 5.3,3.7,1.5,0.2,Iris-setosa 50 | 5.0,3.3,1.4,0.2,Iris-setosa 51 | 7.0,3.2,4.7,1.4,Iris-versicolor 52 | 6.4,3.2,4.5,1.5,Iris-versicolor 53 | 6.9,3.1,4.9,1.5,Iris-versicolor 54 | 5.5,2.3,4.0,1.3,Iris-versicolor 55 | 6.5,2.8,4.6,1.5,Iris-versicolor 56 | 5.7,2.8,4.5,1.3,Iris-versicolor 57 | 6.3,3.3,4.7,1.6,Iris-versicolor 58 | 4.9,2.4,3.3,1.0,Iris-versicolor 59 | 6.6,2.9,4.6,1.3,Iris-versicolor 60 | 5.2,2.7,3.9,1.4,Iris-versicolor 61 | 5.0,2.0,3.5,1.0,Iris-versicolor 62 | 5.9,3.0,4.2,1.5,Iris-versicolor 63 | 6.0,2.2,4.0,1.0,Iris-versicolor 64 | 6.1,2.9,4.7,1.4,Iris-versicolor 65 | 5.6,2.9,3.6,1.3,Iris-versicolor 66 | 6.7,3.1,4.4,1.4,Iris-versicolor 67 | 5.6,3.0,4.5,1.5,Iris-versicolor 68 | 5.8,2.7,4.1,1.0,Iris-versicolor 69 | 6.2,2.2,4.5,1.5,Iris-versicolor 70 | 5.6,2.5,3.9,1.1,Iris-versicolor 71 | 5.9,3.2,4.8,1.8,Iris-versicolor 72 | 6.1,2.8,4.0,1.3,Iris-versicolor 73 | 6.3,2.5,4.9,1.5,Iris-versicolor 74 | 6.1,2.8,4.7,1.2,Iris-versicolor 75 | 6.4,2.9,4.3,1.3,Iris-versicolor 76 | 6.6,3.0,4.4,1.4,Iris-versicolor 77 | 6.8,2.8,4.8,1.4,Iris-versicolor 78 | 6.7,3.0,5.0,1.7,Iris-versicolor 79 | 6.0,2.9,4.5,1.5,Iris-versicolor 80 | 5.7,2.6,3.5,1.0,Iris-versicolor 81 | 5.5,2.4,3.8,1.1,Iris-versicolor 82 | 5.5,2.4,3.7,1.0,Iris-versicolor 83 | 5.8,2.7,3.9,1.2,Iris-versicolor 84 | 6.0,2.7,5.1,1.6,Iris-versicolor 85 | 5.4,3.0,4.5,1.5,Iris-versicolor 86 | 6.0,3.4,4.5,1.6,Iris-versicolor 87 | 6.7,3.1,4.7,1.5,Iris-versicolor 88 | 6.3,2.3,4.4,1.3,Iris-versicolor 89 | 5.6,3.0,4.1,1.3,Iris-versicolor 90 | 5.5,2.5,4.0,1.3,Iris-versicolor 91 | 5.5,2.6,4.4,1.2,Iris-versicolor 92 | 6.1,3.0,4.6,1.4,Iris-versicolor 93 | 5.8,2.6,4.0,1.2,Iris-versicolor 94 | 5.0,2.3,3.3,1.0,Iris-versicolor 95 | 5.6,2.7,4.2,1.3,Iris-versicolor 96 | 5.7,3.0,4.2,1.2,Iris-versicolor 97 | 5.7,2.9,4.2,1.3,Iris-versicolor 98 | 6.2,2.9,4.3,1.3,Iris-versicolor 99 | 5.1,2.5,3.0,1.1,Iris-versicolor 100 | 5.7,2.8,4.1,1.3,Iris-versicolor 101 | 6.3,3.3,6.0,2.5,Iris-virginica 102 | 5.8,2.7,5.1,1.9,Iris-virginica 103 | 7.1,3.0,5.9,2.1,Iris-virginica 104 | 6.3,2.9,5.6,1.8,Iris-virginica 105 | 6.5,3.0,5.8,2.2,Iris-virginica 106 | 7.6,3.0,6.6,2.1,Iris-virginica 107 | 4.9,2.5,4.5,1.7,Iris-virginica 108 | 7.3,2.9,6.3,1.8,Iris-virginica 109 | 6.7,2.5,5.8,1.8,Iris-virginica 110 | 7.2,3.6,6.1,2.5,Iris-virginica 111 | 6.5,3.2,5.1,2.0,Iris-virginica 112 | 6.4,2.7,5.3,1.9,Iris-virginica 113 | 6.8,3.0,5.5,2.1,Iris-virginica 114 | 5.7,2.5,5.0,2.0,Iris-virginica 115 | 5.8,2.8,5.1,2.4,Iris-virginica 116 | 6.4,3.2,5.3,2.3,Iris-virginica 117 | 6.5,3.0,5.5,1.8,Iris-virginica 118 | 7.7,3.8,6.7,2.2,Iris-virginica 119 | 7.7,2.6,6.9,2.3,Iris-virginica 120 | 6.0,2.2,5.0,1.5,Iris-virginica 121 | 6.9,3.2,5.7,2.3,Iris-virginica 122 | 5.6,2.8,4.9,2.0,Iris-virginica 123 | 7.7,2.8,6.7,2.0,Iris-virginica 124 | 6.3,2.7,4.9,1.8,Iris-virginica 125 | 6.7,3.3,5.7,2.1,Iris-virginica 126 | 7.2,3.2,6.0,1.8,Iris-virginica 127 | 6.2,2.8,4.8,1.8,Iris-virginica 128 | 6.1,3.0,4.9,1.8,Iris-virginica 129 | 6.4,2.8,5.6,2.1,Iris-virginica 130 | 7.2,3.0,5.8,1.6,Iris-virginica 131 | 7.4,2.8,6.1,1.9,Iris-virginica 132 | 7.9,3.8,6.4,2.0,Iris-virginica 133 | 6.4,2.8,5.6,2.2,Iris-virginica 134 | 6.3,2.8,5.1,1.5,Iris-virginica 135 | 6.1,2.6,5.6,1.4,Iris-virginica 136 | 7.7,3.0,6.1,2.3,Iris-virginica 137 | 6.3,3.4,5.6,2.4,Iris-virginica 138 | 6.4,3.1,5.5,1.8,Iris-virginica 139 | 6.0,3.0,4.8,1.8,Iris-virginica 140 | 6.9,3.1,5.4,2.1,Iris-virginica 141 | 6.7,3.1,5.6,2.4,Iris-virginica 142 | 6.9,3.1,5.1,2.3,Iris-virginica 143 | 5.8,2.7,5.1,1.9,Iris-virginica 144 | 6.8,3.2,5.9,2.3,Iris-virginica 145 | 6.7,3.3,5.7,2.5,Iris-virginica 146 | 6.7,3.0,5.2,2.3,Iris-virginica 147 | 6.3,2.5,5.0,1.9,Iris-virginica 148 | 6.5,3.0,5.2,2.0,Iris-virginica 149 | 6.2,3.4,5.4,2.3,Iris-virginica 150 | 5.9,3.0,5.1,1.8,Iris-virginica 151 | 152 | -------------------------------------------------------------------------------- /machinelearning/Lesson2/iris.data: -------------------------------------------------------------------------------- 1 | 5.1,3.5,1.4,0.2,Iris-setosa 2 | 4.9,3.0,1.4,0.2,Iris-setosa 3 | 4.7,3.2,1.3,0.2,Iris-setosa 4 | 4.6,3.1,1.5,0.2,Iris-setosa 5 | 5.0,3.6,1.4,0.2,Iris-setosa 6 | 5.4,3.9,1.7,0.4,Iris-setosa 7 | 4.6,3.4,1.4,0.3,Iris-setosa 8 | 5.0,3.4,1.5,0.2,Iris-setosa 9 | 4.4,2.9,1.4,0.2,Iris-setosa 10 | 4.9,3.1,1.5,0.1,Iris-setosa 11 | 5.4,3.7,1.5,0.2,Iris-setosa 12 | 4.8,3.4,1.6,0.2,Iris-setosa 13 | 4.8,3.0,1.4,0.1,Iris-setosa 14 | 4.3,3.0,1.1,0.1,Iris-setosa 15 | 5.8,4.0,1.2,0.2,Iris-setosa 16 | 5.7,4.4,1.5,0.4,Iris-setosa 17 | 5.4,3.9,1.3,0.4,Iris-setosa 18 | 5.1,3.5,1.4,0.3,Iris-setosa 19 | 5.7,3.8,1.7,0.3,Iris-setosa 20 | 5.1,3.8,1.5,0.3,Iris-setosa 21 | 5.4,3.4,1.7,0.2,Iris-setosa 22 | 5.1,3.7,1.5,0.4,Iris-setosa 23 | 4.6,3.6,1.0,0.2,Iris-setosa 24 | 5.1,3.3,1.7,0.5,Iris-setosa 25 | 4.8,3.4,1.9,0.2,Iris-setosa 26 | 5.0,3.0,1.6,0.2,Iris-setosa 27 | 5.0,3.4,1.6,0.4,Iris-setosa 28 | 5.2,3.5,1.5,0.2,Iris-setosa 29 | 5.2,3.4,1.4,0.2,Iris-setosa 30 | 4.7,3.2,1.6,0.2,Iris-setosa 31 | 4.8,3.1,1.6,0.2,Iris-setosa 32 | 5.4,3.4,1.5,0.4,Iris-setosa 33 | 5.2,4.1,1.5,0.1,Iris-setosa 34 | 5.5,4.2,1.4,0.2,Iris-setosa 35 | 4.9,3.1,1.5,0.1,Iris-setosa 36 | 5.0,3.2,1.2,0.2,Iris-setosa 37 | 5.5,3.5,1.3,0.2,Iris-setosa 38 | 4.9,3.1,1.5,0.1,Iris-setosa 39 | 4.4,3.0,1.3,0.2,Iris-setosa 40 | 5.1,3.4,1.5,0.2,Iris-setosa 41 | 5.0,3.5,1.3,0.3,Iris-setosa 42 | 4.5,2.3,1.3,0.3,Iris-setosa 43 | 4.4,3.2,1.3,0.2,Iris-setosa 44 | 5.0,3.5,1.6,0.6,Iris-setosa 45 | 5.1,3.8,1.9,0.4,Iris-setosa 46 | 4.8,3.0,1.4,0.3,Iris-setosa 47 | 5.1,3.8,1.6,0.2,Iris-setosa 48 | 4.6,3.2,1.4,0.2,Iris-setosa 49 | 5.3,3.7,1.5,0.2,Iris-setosa 50 | 5.0,3.3,1.4,0.2,Iris-setosa 51 | 7.0,3.2,4.7,1.4,Iris-versicolor 52 | 6.4,3.2,4.5,1.5,Iris-versicolor 53 | 6.9,3.1,4.9,1.5,Iris-versicolor 54 | 5.5,2.3,4.0,1.3,Iris-versicolor 55 | 6.5,2.8,4.6,1.5,Iris-versicolor 56 | 5.7,2.8,4.5,1.3,Iris-versicolor 57 | 6.3,3.3,4.7,1.6,Iris-versicolor 58 | 4.9,2.4,3.3,1.0,Iris-versicolor 59 | 6.6,2.9,4.6,1.3,Iris-versicolor 60 | 5.2,2.7,3.9,1.4,Iris-versicolor 61 | 5.0,2.0,3.5,1.0,Iris-versicolor 62 | 5.9,3.0,4.2,1.5,Iris-versicolor 63 | 6.0,2.2,4.0,1.0,Iris-versicolor 64 | 6.1,2.9,4.7,1.4,Iris-versicolor 65 | 5.6,2.9,3.6,1.3,Iris-versicolor 66 | 6.7,3.1,4.4,1.4,Iris-versicolor 67 | 5.6,3.0,4.5,1.5,Iris-versicolor 68 | 5.8,2.7,4.1,1.0,Iris-versicolor 69 | 6.2,2.2,4.5,1.5,Iris-versicolor 70 | 5.6,2.5,3.9,1.1,Iris-versicolor 71 | 5.9,3.2,4.8,1.8,Iris-versicolor 72 | 6.1,2.8,4.0,1.3,Iris-versicolor 73 | 6.3,2.5,4.9,1.5,Iris-versicolor 74 | 6.1,2.8,4.7,1.2,Iris-versicolor 75 | 6.4,2.9,4.3,1.3,Iris-versicolor 76 | 6.6,3.0,4.4,1.4,Iris-versicolor 77 | 6.8,2.8,4.8,1.4,Iris-versicolor 78 | 6.7,3.0,5.0,1.7,Iris-versicolor 79 | 6.0,2.9,4.5,1.5,Iris-versicolor 80 | 5.7,2.6,3.5,1.0,Iris-versicolor 81 | 5.5,2.4,3.8,1.1,Iris-versicolor 82 | 5.5,2.4,3.7,1.0,Iris-versicolor 83 | 5.8,2.7,3.9,1.2,Iris-versicolor 84 | 6.0,2.7,5.1,1.6,Iris-versicolor 85 | 5.4,3.0,4.5,1.5,Iris-versicolor 86 | 6.0,3.4,4.5,1.6,Iris-versicolor 87 | 6.7,3.1,4.7,1.5,Iris-versicolor 88 | 6.3,2.3,4.4,1.3,Iris-versicolor 89 | 5.6,3.0,4.1,1.3,Iris-versicolor 90 | 5.5,2.5,4.0,1.3,Iris-versicolor 91 | 5.5,2.6,4.4,1.2,Iris-versicolor 92 | 6.1,3.0,4.6,1.4,Iris-versicolor 93 | 5.8,2.6,4.0,1.2,Iris-versicolor 94 | 5.0,2.3,3.3,1.0,Iris-versicolor 95 | 5.6,2.7,4.2,1.3,Iris-versicolor 96 | 5.7,3.0,4.2,1.2,Iris-versicolor 97 | 5.7,2.9,4.2,1.3,Iris-versicolor 98 | 6.2,2.9,4.3,1.3,Iris-versicolor 99 | 5.1,2.5,3.0,1.1,Iris-versicolor 100 | 5.7,2.8,4.1,1.3,Iris-versicolor 101 | 6.3,3.3,6.0,2.5,Iris-virginica 102 | 5.8,2.7,5.1,1.9,Iris-virginica 103 | 7.1,3.0,5.9,2.1,Iris-virginica 104 | 6.3,2.9,5.6,1.8,Iris-virginica 105 | 6.5,3.0,5.8,2.2,Iris-virginica 106 | 7.6,3.0,6.6,2.1,Iris-virginica 107 | 4.9,2.5,4.5,1.7,Iris-virginica 108 | 7.3,2.9,6.3,1.8,Iris-virginica 109 | 6.7,2.5,5.8,1.8,Iris-virginica 110 | 7.2,3.6,6.1,2.5,Iris-virginica 111 | 6.5,3.2,5.1,2.0,Iris-virginica 112 | 6.4,2.7,5.3,1.9,Iris-virginica 113 | 6.8,3.0,5.5,2.1,Iris-virginica 114 | 5.7,2.5,5.0,2.0,Iris-virginica 115 | 5.8,2.8,5.1,2.4,Iris-virginica 116 | 6.4,3.2,5.3,2.3,Iris-virginica 117 | 6.5,3.0,5.5,1.8,Iris-virginica 118 | 7.7,3.8,6.7,2.2,Iris-virginica 119 | 7.7,2.6,6.9,2.3,Iris-virginica 120 | 6.0,2.2,5.0,1.5,Iris-virginica 121 | 6.9,3.2,5.7,2.3,Iris-virginica 122 | 5.6,2.8,4.9,2.0,Iris-virginica 123 | 7.7,2.8,6.7,2.0,Iris-virginica 124 | 6.3,2.7,4.9,1.8,Iris-virginica 125 | 6.7,3.3,5.7,2.1,Iris-virginica 126 | 7.2,3.2,6.0,1.8,Iris-virginica 127 | 6.2,2.8,4.8,1.8,Iris-virginica 128 | 6.1,3.0,4.9,1.8,Iris-virginica 129 | 6.4,2.8,5.6,2.1,Iris-virginica 130 | 7.2,3.0,5.8,1.6,Iris-virginica 131 | 7.4,2.8,6.1,1.9,Iris-virginica 132 | 7.9,3.8,6.4,2.0,Iris-virginica 133 | 6.4,2.8,5.6,2.2,Iris-virginica 134 | 6.3,2.8,5.1,1.5,Iris-virginica 135 | 6.1,2.6,5.6,1.4,Iris-virginica 136 | 7.7,3.0,6.1,2.3,Iris-virginica 137 | 6.3,3.4,5.6,2.4,Iris-virginica 138 | 6.4,3.1,5.5,1.8,Iris-virginica 139 | 6.0,3.0,4.8,1.8,Iris-virginica 140 | 6.9,3.1,5.4,2.1,Iris-virginica 141 | 6.7,3.1,5.6,2.4,Iris-virginica 142 | 6.9,3.1,5.1,2.3,Iris-virginica 143 | 5.8,2.7,5.1,1.9,Iris-virginica 144 | 6.8,3.2,5.9,2.3,Iris-virginica 145 | 6.7,3.3,5.7,2.5,Iris-virginica 146 | 6.7,3.0,5.2,2.3,Iris-virginica 147 | 6.3,2.5,5.0,1.9,Iris-virginica 148 | 6.5,3.0,5.2,2.0,Iris-virginica 149 | 6.2,3.4,5.4,2.3,Iris-virginica 150 | 5.9,3.0,5.1,1.8,Iris-virginica 151 | 152 | -------------------------------------------------------------------------------- /assignment7/classwork/DSVC/classifiers/cnn.py: -------------------------------------------------------------------------------- 1 | from builtins import object 2 | import numpy as np 3 | 4 | from DSVC.layers import * 5 | from DSVC.fast_layers import * 6 | from DSVC.layer_utils import * 7 | 8 | 9 | class ThreeLayerConvNet(object): 10 | """ 11 | A three-layer convolutional network with the following architecture: 12 | 13 | conv - relu - 2x2 max pool - affine - relu - affine - softmax 14 | 15 | The network operates on minibatches of data that have shape (N, C, H, W) 16 | consisting of N images, each with height H and width W and with C input 17 | channels. 18 | """ 19 | 20 | def __init__(self, input_dim=(3, 32, 32), num_filters=32, filter_size=7, 21 | hidden_dim=100, num_classes=10, weight_scale=1e-3, reg=0.0, 22 | dtype=np.float32): 23 | """ 24 | Initialize a new network. 25 | 26 | Inputs: 27 | - input_dim: Tuple (C, H, W) giving size of input data 28 | - num_filters: Number of filters to use in the convolutional layer 29 | - filter_size: Size of filters to use in the convolutional layer 30 | - hidden_dim: Number of units to use in the fully-connected hidden layer 31 | - num_classes: Number of scores to produce from the final affine layer. 32 | - weight_scale: Scalar giving standard deviation for random initialization 33 | of weights. 34 | - reg: Scalar giving L2 regularization strength 35 | - dtype: numpy datatype to use for computation. 36 | """ 37 | self.params = {} 38 | self.reg = reg 39 | self.dtype = dtype 40 | 41 | ############################################################################ 42 | # TODO: Initialize weights and biases for the three-layer convolutional # 43 | # network. Weights should be initialized from a Gaussian with standard # 44 | # deviation equal to weight_scale; biases should be initialized to zero. # 45 | # All weights and biases should be stored in the dictionary self.params. # 46 | # Store weights and biases for the convolutional layer using the keys 'W1' # 47 | # and 'b1'; use keys 'W2' and 'b2' for the weights and biases of the # 48 | # hidden affine layer, and keys 'W3' and 'b3' for the weights and biases # 49 | # of the output affine layer. # 50 | ############################################################################ 51 | pass 52 | ############################################################################ 53 | # END OF YOUR CODE # 54 | ############################################################################ 55 | 56 | for k, v in self.params.items(): 57 | self.params[k] = v.astype(dtype) 58 | 59 | 60 | def loss(self, X, y=None): 61 | """ 62 | Evaluate loss and gradient for the three-layer convolutional network. 63 | 64 | Input / output: Same API as TwoLayerNet in fc_net.py. 65 | """ 66 | W1, b1 = self.params['W1'], self.params['b1'] 67 | W2, b2 = self.params['W2'], self.params['b2'] 68 | W3, b3 = self.params['W3'], self.params['b3'] 69 | 70 | # pass conv_param to the forward pass for the convolutional layer 71 | filter_size = W1.shape[2] 72 | conv_param = {'stride': 1, 'pad': (filter_size - 1) // 2} 73 | 74 | # pass pool_param to the forward pass for the max-pooling layer 75 | pool_param = {'pool_height': 2, 'pool_width': 2, 'stride': 2} 76 | 77 | scores = None 78 | ############################################################################ 79 | # TODO: Implement the forward pass for the three-layer convolutional net, # 80 | # computing the class scores for X and storing them in the scores # 81 | # variable. # 82 | ############################################################################ 83 | pass 84 | ############################################################################ 85 | # END OF YOUR CODE # 86 | ############################################################################ 87 | 88 | if y is None: 89 | return scores 90 | 91 | loss, grads = 0, {} 92 | ############################################################################ 93 | # TODO: Implement the backward pass for the three-layer convolutional net, # 94 | # storing the loss and gradients in the loss and grads variables. Compute # 95 | # data loss using softmax, and make sure that grads[k] holds the gradients # 96 | # for self.params[k]. Don't forget to add L2 regularization! # 97 | ############################################################################ 98 | pass 99 | ############################################################################ 100 | # END OF YOUR CODE # 101 | ############################################################################ 102 | 103 | return loss, grads 104 | -------------------------------------------------------------------------------- /assignment7/classwork/DSVC/im2col_cython.pyx: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | cimport numpy as np 3 | cimport cython 4 | 5 | # DTYPE = np.float64 6 | # ctypedef np.float64_t DTYPE_t 7 | 8 | ctypedef fused DTYPE_t: 9 | np.float32_t 10 | np.float64_t 11 | 12 | def im2col_cython(np.ndarray[DTYPE_t, ndim=4] x, int field_height, 13 | int field_width, int padding, int stride): 14 | cdef int N = x.shape[0] 15 | cdef int C = x.shape[1] 16 | cdef int H = x.shape[2] 17 | cdef int W = x.shape[3] 18 | 19 | cdef int HH = (H + 2 * padding - field_height) / stride + 1 20 | cdef int WW = (W + 2 * padding - field_width) / stride + 1 21 | 22 | cdef int p = padding 23 | cdef np.ndarray[DTYPE_t, ndim=4] x_padded = np.pad(x, 24 | ((0, 0), (0, 0), (p, p), (p, p)), mode='constant') 25 | 26 | cdef np.ndarray[DTYPE_t, ndim=2] cols = np.zeros( 27 | (C * field_height * field_width, N * HH * WW), 28 | dtype=x.dtype) 29 | 30 | # Moving the inner loop to a C function with no bounds checking works, but does 31 | # not seem to help performance in any measurable way. 32 | 33 | im2col_cython_inner(cols, x_padded, N, C, H, W, HH, WW, 34 | field_height, field_width, padding, stride) 35 | return cols 36 | 37 | 38 | @cython.boundscheck(False) 39 | cdef int im2col_cython_inner(np.ndarray[DTYPE_t, ndim=2] cols, 40 | np.ndarray[DTYPE_t, ndim=4] x_padded, 41 | int N, int C, int H, int W, int HH, int WW, 42 | int field_height, int field_width, int padding, int stride) except? -1: 43 | cdef int c, ii, jj, row, yy, xx, i, col 44 | 45 | for c in range(C): 46 | for yy in range(HH): 47 | for xx in range(WW): 48 | for ii in range(field_height): 49 | for jj in range(field_width): 50 | row = c * field_width * field_height + ii * field_height + jj 51 | for i in range(N): 52 | col = yy * WW * N + xx * N + i 53 | cols[row, col] = x_padded[i, c, stride * yy + ii, stride * xx + jj] 54 | 55 | 56 | 57 | def col2im_cython(np.ndarray[DTYPE_t, ndim=2] cols, int N, int C, int H, int W, 58 | int field_height, int field_width, int padding, int stride): 59 | cdef np.ndarray x = np.empty((N, C, H, W), dtype=cols.dtype) 60 | cdef int HH = (H + 2 * padding - field_height) / stride + 1 61 | cdef int WW = (W + 2 * padding - field_width) / stride + 1 62 | cdef np.ndarray[DTYPE_t, ndim=4] x_padded = np.zeros((N, C, H + 2 * padding, W + 2 * padding), 63 | dtype=cols.dtype) 64 | 65 | # Moving the inner loop to a C-function with no bounds checking improves 66 | # performance quite a bit for col2im. 67 | col2im_cython_inner(cols, x_padded, N, C, H, W, HH, WW, 68 | field_height, field_width, padding, stride) 69 | if padding > 0: 70 | return x_padded[:, :, padding:-padding, padding:-padding] 71 | return x_padded 72 | 73 | 74 | @cython.boundscheck(False) 75 | cdef int col2im_cython_inner(np.ndarray[DTYPE_t, ndim=2] cols, 76 | np.ndarray[DTYPE_t, ndim=4] x_padded, 77 | int N, int C, int H, int W, int HH, int WW, 78 | int field_height, int field_width, int padding, int stride) except? -1: 79 | cdef int c, ii, jj, row, yy, xx, i, col 80 | 81 | for c in range(C): 82 | for ii in range(field_height): 83 | for jj in range(field_width): 84 | row = c * field_width * field_height + ii * field_height + jj 85 | for yy in range(HH): 86 | for xx in range(WW): 87 | for i in range(N): 88 | col = yy * WW * N + xx * N + i 89 | x_padded[i, c, stride * yy + ii, stride * xx + jj] += cols[row, col] 90 | 91 | 92 | @cython.boundscheck(False) 93 | @cython.wraparound(False) 94 | cdef col2im_6d_cython_inner(np.ndarray[DTYPE_t, ndim=6] cols, 95 | np.ndarray[DTYPE_t, ndim=4] x_padded, 96 | int N, int C, int H, int W, int HH, int WW, 97 | int out_h, int out_w, int pad, int stride): 98 | 99 | cdef int c, hh, ww, n, h, w 100 | for n in range(N): 101 | for c in range(C): 102 | for hh in range(HH): 103 | for ww in range(WW): 104 | for h in range(out_h): 105 | for w in range(out_w): 106 | x_padded[n, c, stride * h + hh, stride * w + ww] += cols[c, hh, ww, n, h, w] 107 | 108 | 109 | def col2im_6d_cython(np.ndarray[DTYPE_t, ndim=6] cols, int N, int C, int H, int W, 110 | int HH, int WW, int pad, int stride): 111 | cdef np.ndarray x = np.empty((N, C, H, W), dtype=cols.dtype) 112 | cdef int out_h = (H + 2 * pad - HH) / stride + 1 113 | cdef int out_w = (W + 2 * pad - WW) / stride + 1 114 | cdef np.ndarray[DTYPE_t, ndim=4] x_padded = np.zeros((N, C, H + 2 * pad, W + 2 * pad), 115 | dtype=cols.dtype) 116 | 117 | col2im_6d_cython_inner(cols, x_padded, N, C, H, W, HH, WW, out_h, out_w, pad, stride) 118 | 119 | if pad > 0: 120 | return x_padded[:, :, pad:-pad, pad:-pad] 121 | return x_padded 122 | -------------------------------------------------------------------------------- /assignment5/classwork/DSVC/features.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from past.builtins import xrange 3 | 4 | import matplotlib 5 | import numpy as np 6 | from scipy.ndimage import uniform_filter 7 | 8 | 9 | def extract_features(imgs, feature_fns, verbose=False): 10 | """ 11 | Given pixel data for images and several feature functions that can operate on 12 | single images, apply all feature functions to all images, concatenating the 13 | feature vectors for each image and storing the features for all images in 14 | a single matrix. 15 | 16 | Inputs: 17 | - imgs: N x H X W X C array of pixel data for N images. 18 | - feature_fns: List of k feature functions. The ith feature function should 19 | take as input an H x W x D array and return a (one-dimensional) array of 20 | length F_i. 21 | - verbose: Boolean; if true, print progress. 22 | 23 | Returns: 24 | An array of shape (N, F_1 + ... + F_k) where each column is the concatenation 25 | of all features for a single image. 26 | """ 27 | num_images = imgs.shape[0] 28 | if num_images == 0: 29 | return np.array([]) 30 | 31 | # Use the first image to determine feature dimensions 32 | feature_dims = [] 33 | first_image_features = [] 34 | for feature_fn in feature_fns: 35 | feats = feature_fn(imgs[0].squeeze()) 36 | assert len(feats.shape) == 1, 'Feature functions must be one-dimensional' 37 | feature_dims.append(feats.size) 38 | first_image_features.append(feats) 39 | 40 | # Now that we know the dimensions of the features, we can allocate a single 41 | # big array to store all features as columns. 42 | total_feature_dim = sum(feature_dims) 43 | imgs_features = np.zeros((num_images, total_feature_dim)) 44 | imgs_features[0] = np.hstack(first_image_features).T 45 | 46 | # Extract features for the rest of the images. 47 | for i in xrange(1, num_images): 48 | idx = 0 49 | for feature_fn, feature_dim in zip(feature_fns, feature_dims): 50 | next_idx = idx + feature_dim 51 | imgs_features[i, idx:next_idx] = feature_fn(imgs[i].squeeze()) 52 | idx = next_idx 53 | if verbose and i % 1000 == 0: 54 | print('Done extracting features for %d / %d images' % (i, num_images)) 55 | 56 | return imgs_features 57 | 58 | 59 | def rgb2gray(rgb): 60 | """Convert RGB image to grayscale 61 | 62 | Parameters: 63 | rgb : RGB image 64 | 65 | Returns: 66 | gray : grayscale image 67 | 68 | """ 69 | return np.dot(rgb[...,:3], [0.299, 0.587, 0.144]) 70 | 71 | 72 | def hog_feature(im): 73 | """Compute Histogram of Gradient (HOG) feature for an image 74 | 75 | Modified from skimage.feature.hog 76 | http://pydoc.net/Python/scikits-image/0.4.2/skimage.feature.hog 77 | 78 | Reference: 79 | Histograms of Oriented Gradients for Human Detection 80 | Navneet Dalal and Bill Triggs, CVPR 2005 81 | 82 | Parameters: 83 | im : an input grayscale or rgb image 84 | 85 | Returns: 86 | feat: Histogram of Gradient (HOG) feature 87 | 88 | """ 89 | 90 | # convert rgb to grayscale if needed 91 | if im.ndim == 3: 92 | image = rgb2gray(im) 93 | else: 94 | image = np.at_least_2d(im) 95 | 96 | sx, sy = image.shape # image size 97 | orientations = 9 # number of gradient bins 98 | cx, cy = (8, 8) # pixels per cell 99 | 100 | gx = np.zeros(image.shape) 101 | gy = np.zeros(image.shape) 102 | gx[:, :-1] = np.diff(image, n=1, axis=1) # compute gradient on x-direction 103 | gy[:-1, :] = np.diff(image, n=1, axis=0) # compute gradient on y-direction 104 | grad_mag = np.sqrt(gx ** 2 + gy ** 2) # gradient magnitude 105 | grad_ori = np.arctan2(gy, (gx + 1e-15)) * (180 / np.pi) + 90 # gradient orientation 106 | 107 | n_cellsx = int(np.floor(sx / cx)) # number of cells in x 108 | n_cellsy = int(np.floor(sy / cy)) # number of cells in y 109 | # compute orientations integral images 110 | orientation_histogram = np.zeros((n_cellsx, n_cellsy, orientations)) 111 | for i in range(orientations): 112 | # create new integral image for this orientation 113 | # isolate orientations in this range 114 | temp_ori = np.where(grad_ori < 180 / orientations * (i + 1), 115 | grad_ori, 0) 116 | temp_ori = np.where(grad_ori >= 180 / orientations * i, 117 | temp_ori, 0) 118 | # select magnitudes for those orientations 119 | cond2 = temp_ori > 0 120 | temp_mag = np.where(cond2, grad_mag, 0) 121 | orientation_histogram[:,:,i] = uniform_filter(temp_mag, size=(cx, cy))[cx/2::cx, cy/2::cy].T 122 | 123 | return orientation_histogram.ravel() 124 | 125 | 126 | def color_histogram_hsv(im, nbin=10, xmin=0, xmax=255, normalized=True): 127 | """ 128 | Compute color histogram for an image using hue. 129 | 130 | Inputs: 131 | - im: H x W x C array of pixel data for an RGB image. 132 | - nbin: Number of histogram bins. (default: 10) 133 | - xmin: Minimum pixel value (default: 0) 134 | - xmax: Maximum pixel value (default: 255) 135 | - normalized: Whether to normalize the histogram (default: True) 136 | 137 | Returns: 138 | 1D vector of length nbin giving the color histogram over the hue of the 139 | input image. 140 | """ 141 | ndim = im.ndim 142 | bins = np.linspace(xmin, xmax, nbin+1) 143 | hsv = matplotlib.colors.rgb_to_hsv(im/xmax) * xmax 144 | imhist, bin_edges = np.histogram(hsv[:,:,0], bins=bins, density=normalized) 145 | imhist = imhist * np.diff(bin_edges) 146 | 147 | # return histogram 148 | return imhist 149 | 150 | 151 | pass 152 | -------------------------------------------------------------------------------- /assignment6/classwork/DSVC/features.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from past.builtins import xrange 3 | 4 | import matplotlib 5 | import numpy as np 6 | from scipy.ndimage import uniform_filter 7 | 8 | 9 | def extract_features(imgs, feature_fns, verbose=False): 10 | """ 11 | Given pixel data for images and several feature functions that can operate on 12 | single images, apply all feature functions to all images, concatenating the 13 | feature vectors for each image and storing the features for all images in 14 | a single matrix. 15 | 16 | Inputs: 17 | - imgs: N x H X W X C array of pixel data for N images. 18 | - feature_fns: List of k feature functions. The ith feature function should 19 | take as input an H x W x D array and return a (one-dimensional) array of 20 | length F_i. 21 | - verbose: Boolean; if true, print progress. 22 | 23 | Returns: 24 | An array of shape (N, F_1 + ... + F_k) where each column is the concatenation 25 | of all features for a single image. 26 | """ 27 | num_images = imgs.shape[0] 28 | if num_images == 0: 29 | return np.array([]) 30 | 31 | # Use the first image to determine feature dimensions 32 | feature_dims = [] 33 | first_image_features = [] 34 | for feature_fn in feature_fns: 35 | feats = feature_fn(imgs[0].squeeze()) 36 | assert len(feats.shape) == 1, 'Feature functions must be one-dimensional' 37 | feature_dims.append(feats.size) 38 | first_image_features.append(feats) 39 | 40 | # Now that we know the dimensions of the features, we can allocate a single 41 | # big array to store all features as columns. 42 | total_feature_dim = sum(feature_dims) 43 | imgs_features = np.zeros((num_images, total_feature_dim)) 44 | imgs_features[0] = np.hstack(first_image_features).T 45 | 46 | # Extract features for the rest of the images. 47 | for i in xrange(1, num_images): 48 | idx = 0 49 | for feature_fn, feature_dim in zip(feature_fns, feature_dims): 50 | next_idx = idx + feature_dim 51 | imgs_features[i, idx:next_idx] = feature_fn(imgs[i].squeeze()) 52 | idx = next_idx 53 | if verbose and i % 1000 == 0: 54 | print('Done extracting features for %d / %d images' % (i, num_images)) 55 | 56 | return imgs_features 57 | 58 | 59 | def rgb2gray(rgb): 60 | """Convert RGB image to grayscale 61 | 62 | Parameters: 63 | rgb : RGB image 64 | 65 | Returns: 66 | gray : grayscale image 67 | 68 | """ 69 | return np.dot(rgb[...,:3], [0.299, 0.587, 0.144]) 70 | 71 | 72 | def hog_feature(im): 73 | """Compute Histogram of Gradient (HOG) feature for an image 74 | 75 | Modified from skimage.feature.hog 76 | http://pydoc.net/Python/scikits-image/0.4.2/skimage.feature.hog 77 | 78 | Reference: 79 | Histograms of Oriented Gradients for Human Detection 80 | Navneet Dalal and Bill Triggs, CVPR 2005 81 | 82 | Parameters: 83 | im : an input grayscale or rgb image 84 | 85 | Returns: 86 | feat: Histogram of Gradient (HOG) feature 87 | 88 | """ 89 | 90 | # convert rgb to grayscale if needed 91 | if im.ndim == 3: 92 | image = rgb2gray(im) 93 | else: 94 | image = np.at_least_2d(im) 95 | 96 | sx, sy = image.shape # image size 97 | orientations = 9 # number of gradient bins 98 | cx, cy = (8, 8) # pixels per cell 99 | 100 | gx = np.zeros(image.shape) 101 | gy = np.zeros(image.shape) 102 | gx[:, :-1] = np.diff(image, n=1, axis=1) # compute gradient on x-direction 103 | gy[:-1, :] = np.diff(image, n=1, axis=0) # compute gradient on y-direction 104 | grad_mag = np.sqrt(gx ** 2 + gy ** 2) # gradient magnitude 105 | grad_ori = np.arctan2(gy, (gx + 1e-15)) * (180 / np.pi) + 90 # gradient orientation 106 | 107 | n_cellsx = int(np.floor(sx / cx)) # number of cells in x 108 | n_cellsy = int(np.floor(sy / cy)) # number of cells in y 109 | # compute orientations integral images 110 | orientation_histogram = np.zeros((n_cellsx, n_cellsy, orientations)) 111 | for i in range(orientations): 112 | # create new integral image for this orientation 113 | # isolate orientations in this range 114 | temp_ori = np.where(grad_ori < 180 / orientations * (i + 1), 115 | grad_ori, 0) 116 | temp_ori = np.where(grad_ori >= 180 / orientations * i, 117 | temp_ori, 0) 118 | # select magnitudes for those orientations 119 | cond2 = temp_ori > 0 120 | temp_mag = np.where(cond2, grad_mag, 0) 121 | orientation_histogram[:,:,i] = uniform_filter(temp_mag, size=(cx, cy))[cx/2::cx, cy/2::cy].T 122 | 123 | return orientation_histogram.ravel() 124 | 125 | 126 | def color_histogram_hsv(im, nbin=10, xmin=0, xmax=255, normalized=True): 127 | """ 128 | Compute color histogram for an image using hue. 129 | 130 | Inputs: 131 | - im: H x W x C array of pixel data for an RGB image. 132 | - nbin: Number of histogram bins. (default: 10) 133 | - xmin: Minimum pixel value (default: 0) 134 | - xmax: Maximum pixel value (default: 255) 135 | - normalized: Whether to normalize the histogram (default: True) 136 | 137 | Returns: 138 | 1D vector of length nbin giving the color histogram over the hue of the 139 | input image. 140 | """ 141 | ndim = im.ndim 142 | bins = np.linspace(xmin, xmax, nbin+1) 143 | hsv = matplotlib.colors.rgb_to_hsv(im/xmax) * xmax 144 | imhist, bin_edges = np.histogram(hsv[:,:,0], bins=bins, density=normalized) 145 | imhist = imhist * np.diff(bin_edges) 146 | 147 | # return histogram 148 | return imhist 149 | 150 | 151 | pass 152 | -------------------------------------------------------------------------------- /machinelearning/Lesson2/Advertising.csv: -------------------------------------------------------------------------------- 1 | ,TV,Radio,Newspaper,Sales 2 | 1,230.1,37.8,69.2,22.1 3 | 2,44.5,39.3,45.1,10.4 4 | 3,17.2,45.9,69.3,9.3 5 | 4,151.5,41.3,58.5,18.5 6 | 5,180.8,10.8,58.4,12.9 7 | 6,8.7,48.9,75,7.2 8 | 7,57.5,32.8,23.5,11.8 9 | 8,120.2,19.6,11.6,13.2 10 | 9,8.6,2.1,1,4.8 11 | 10,199.8,2.6,21.2,10.6 12 | 11,66.1,5.8,24.2,8.6 13 | 12,214.7,24,4,17.4 14 | 13,23.8,35.1,65.9,9.2 15 | 14,97.5,7.6,7.2,9.7 16 | 15,204.1,32.9,46,19 17 | 16,195.4,47.7,52.9,22.4 18 | 17,67.8,36.6,114,12.5 19 | 18,281.4,39.6,55.8,24.4 20 | 19,69.2,20.5,18.3,11.3 21 | 20,147.3,23.9,19.1,14.6 22 | 21,218.4,27.7,53.4,18 23 | 22,237.4,5.1,23.5,12.5 24 | 23,13.2,15.9,49.6,5.6 25 | 24,228.3,16.9,26.2,15.5 26 | 25,62.3,12.6,18.3,9.7 27 | 26,262.9,3.5,19.5,12 28 | 27,142.9,29.3,12.6,15 29 | 28,240.1,16.7,22.9,15.9 30 | 29,248.8,27.1,22.9,18.9 31 | 30,70.6,16,40.8,10.5 32 | 31,292.9,28.3,43.2,21.4 33 | 32,112.9,17.4,38.6,11.9 34 | 33,97.2,1.5,30,9.6 35 | 34,265.6,20,0.3,17.4 36 | 35,95.7,1.4,7.4,9.5 37 | 36,290.7,4.1,8.5,12.8 38 | 37,266.9,43.8,5,25.4 39 | 38,74.7,49.4,45.7,14.7 40 | 39,43.1,26.7,35.1,10.1 41 | 40,228,37.7,32,21.5 42 | 41,202.5,22.3,31.6,16.6 43 | 42,177,33.4,38.7,17.1 44 | 43,293.6,27.7,1.8,20.7 45 | 44,206.9,8.4,26.4,12.9 46 | 45,25.1,25.7,43.3,8.5 47 | 46,175.1,22.5,31.5,14.9 48 | 47,89.7,9.9,35.7,10.6 49 | 48,239.9,41.5,18.5,23.2 50 | 49,227.2,15.8,49.9,14.8 51 | 50,66.9,11.7,36.8,9.7 52 | 51,199.8,3.1,34.6,11.4 53 | 52,100.4,9.6,3.6,10.7 54 | 53,216.4,41.7,39.6,22.6 55 | 54,182.6,46.2,58.7,21.2 56 | 55,262.7,28.8,15.9,20.2 57 | 56,198.9,49.4,60,23.7 58 | 57,7.3,28.1,41.4,5.5 59 | 58,136.2,19.2,16.6,13.2 60 | 59,210.8,49.6,37.7,23.8 61 | 60,210.7,29.5,9.3,18.4 62 | 61,53.5,2,21.4,8.1 63 | 62,261.3,42.7,54.7,24.2 64 | 63,239.3,15.5,27.3,15.7 65 | 64,102.7,29.6,8.4,14 66 | 65,131.1,42.8,28.9,18 67 | 66,69,9.3,0.9,9.3 68 | 67,31.5,24.6,2.2,9.5 69 | 68,139.3,14.5,10.2,13.4 70 | 69,237.4,27.5,11,18.9 71 | 70,216.8,43.9,27.2,22.3 72 | 71,199.1,30.6,38.7,18.3 73 | 72,109.8,14.3,31.7,12.4 74 | 73,26.8,33,19.3,8.8 75 | 74,129.4,5.7,31.3,11 76 | 75,213.4,24.6,13.1,17 77 | 76,16.9,43.7,89.4,8.7 78 | 77,27.5,1.6,20.7,6.9 79 | 78,120.5,28.5,14.2,14.2 80 | 79,5.4,29.9,9.4,5.3 81 | 80,116,7.7,23.1,11 82 | 81,76.4,26.7,22.3,11.8 83 | 82,239.8,4.1,36.9,12.3 84 | 83,75.3,20.3,32.5,11.3 85 | 84,68.4,44.5,35.6,13.6 86 | 85,213.5,43,33.8,21.7 87 | 86,193.2,18.4,65.7,15.2 88 | 87,76.3,27.5,16,12 89 | 88,110.7,40.6,63.2,16 90 | 89,88.3,25.5,73.4,12.9 91 | 90,109.8,47.8,51.4,16.7 92 | 91,134.3,4.9,9.3,11.2 93 | 92,28.6,1.5,33,7.3 94 | 93,217.7,33.5,59,19.4 95 | 94,250.9,36.5,72.3,22.2 96 | 95,107.4,14,10.9,11.5 97 | 96,163.3,31.6,52.9,16.9 98 | 97,197.6,3.5,5.9,11.7 99 | 98,184.9,21,22,15.5 100 | 99,289.7,42.3,51.2,25.4 101 | 100,135.2,41.7,45.9,17.2 102 | 101,222.4,4.3,49.8,11.7 103 | 102,296.4,36.3,100.9,23.8 104 | 103,280.2,10.1,21.4,14.8 105 | 104,187.9,17.2,17.9,14.7 106 | 105,238.2,34.3,5.3,20.7 107 | 106,137.9,46.4,59,19.2 108 | 107,25,11,29.7,7.2 109 | 108,90.4,0.3,23.2,8.7 110 | 109,13.1,0.4,25.6,5.3 111 | 110,255.4,26.9,5.5,19.8 112 | 111,225.8,8.2,56.5,13.4 113 | 112,241.7,38,23.2,21.8 114 | 113,175.7,15.4,2.4,14.1 115 | 114,209.6,20.6,10.7,15.9 116 | 115,78.2,46.8,34.5,14.6 117 | 116,75.1,35,52.7,12.6 118 | 117,139.2,14.3,25.6,12.2 119 | 118,76.4,0.8,14.8,9.4 120 | 119,125.7,36.9,79.2,15.9 121 | 120,19.4,16,22.3,6.6 122 | 121,141.3,26.8,46.2,15.5 123 | 122,18.8,21.7,50.4,7 124 | 123,224,2.4,15.6,11.6 125 | 124,123.1,34.6,12.4,15.2 126 | 125,229.5,32.3,74.2,19.7 127 | 126,87.2,11.8,25.9,10.6 128 | 127,7.8,38.9,50.6,6.6 129 | 128,80.2,0,9.2,8.8 130 | 129,220.3,49,3.2,24.7 131 | 130,59.6,12,43.1,9.7 132 | 131,0.7,39.6,8.7,1.6 133 | 132,265.2,2.9,43,12.7 134 | 133,8.4,27.2,2.1,5.7 135 | 134,219.8,33.5,45.1,19.6 136 | 135,36.9,38.6,65.6,10.8 137 | 136,48.3,47,8.5,11.6 138 | 137,25.6,39,9.3,9.5 139 | 138,273.7,28.9,59.7,20.8 140 | 139,43,25.9,20.5,9.6 141 | 140,184.9,43.9,1.7,20.7 142 | 141,73.4,17,12.9,10.9 143 | 142,193.7,35.4,75.6,19.2 144 | 143,220.5,33.2,37.9,20.1 145 | 144,104.6,5.7,34.4,10.4 146 | 145,96.2,14.8,38.9,11.4 147 | 146,140.3,1.9,9,10.3 148 | 147,240.1,7.3,8.7,13.2 149 | 148,243.2,49,44.3,25.4 150 | 149,38,40.3,11.9,10.9 151 | 150,44.7,25.8,20.6,10.1 152 | 151,280.7,13.9,37,16.1 153 | 152,121,8.4,48.7,11.6 154 | 153,197.6,23.3,14.2,16.6 155 | 154,171.3,39.7,37.7,19 156 | 155,187.8,21.1,9.5,15.6 157 | 156,4.1,11.6,5.7,3.2 158 | 157,93.9,43.5,50.5,15.3 159 | 158,149.8,1.3,24.3,10.1 160 | 159,11.7,36.9,45.2,7.3 161 | 160,131.7,18.4,34.6,12.9 162 | 161,172.5,18.1,30.7,14.4 163 | 162,85.7,35.8,49.3,13.3 164 | 163,188.4,18.1,25.6,14.9 165 | 164,163.5,36.8,7.4,18 166 | 165,117.2,14.7,5.4,11.9 167 | 166,234.5,3.4,84.8,11.9 168 | 167,17.9,37.6,21.6,8 169 | 168,206.8,5.2,19.4,12.2 170 | 169,215.4,23.6,57.6,17.1 171 | 170,284.3,10.6,6.4,15 172 | 171,50,11.6,18.4,8.4 173 | 172,164.5,20.9,47.4,14.5 174 | 173,19.6,20.1,17,7.6 175 | 174,168.4,7.1,12.8,11.7 176 | 175,222.4,3.4,13.1,11.5 177 | 176,276.9,48.9,41.8,27 178 | 177,248.4,30.2,20.3,20.2 179 | 178,170.2,7.8,35.2,11.7 180 | 179,276.7,2.3,23.7,11.8 181 | 180,165.6,10,17.6,12.6 182 | 181,156.6,2.6,8.3,10.5 183 | 182,218.5,5.4,27.4,12.2 184 | 183,56.2,5.7,29.7,8.7 185 | 184,287.6,43,71.8,26.2 186 | 185,253.8,21.3,30,17.6 187 | 186,205,45.1,19.6,22.6 188 | 187,139.5,2.1,26.6,10.3 189 | 188,191.1,28.7,18.2,17.3 190 | 189,286,13.9,3.7,15.9 191 | 190,18.7,12.1,23.4,6.7 192 | 191,39.5,41.1,5.8,10.8 193 | 192,75.5,10.8,6,9.9 194 | 193,17.2,4.1,31.6,5.9 195 | 194,166.8,42,3.6,19.6 196 | 195,149.7,35.6,6,17.3 197 | 196,38.2,3.7,13.8,7.6 198 | 197,94.2,4.9,8.1,9.7 199 | 198,177,9.3,6.4,12.8 200 | 199,283.6,42,66.2,25.5 201 | 200,232.1,8.6,8.7,13.4 -------------------------------------------------------------------------------- /assignment7/classwork/DSVC/optim.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | """ 4 | This file implements various first-order update rules that are commonly used 5 | for training neural networks. Each update rule accepts current weights and the 6 | gradient of the loss with respect to those weights and produces the next set of 7 | weights. Each update rule has the same interface: 8 | 9 | def update(w, dw, config=None): 10 | 11 | Inputs: 12 | - w: A numpy array giving the current weights. 13 | - dw: A numpy array of the same shape as w giving the gradient of the 14 | loss with respect to w. 15 | - config: A dictionary containing hyperparameter values such as learning 16 | rate, momentum, etc. If the update rule requires caching values over many 17 | iterations, then config will also hold these cached values. 18 | 19 | Returns: 20 | - next_w: The next point after the update. 21 | - config: The config dictionary to be passed to the next iteration of the 22 | update rule. 23 | 24 | NOTE: For most update rules, the default learning rate will probably not 25 | perform well; however the default values of the other hyperparameters should 26 | work well for a variety of different problems. 27 | 28 | For efficiency, update rules may perform in-place updates, mutating w and 29 | setting next_w equal to w. 30 | """ 31 | 32 | 33 | def sgd(w, dw, config=None): 34 | """ 35 | Performs vanilla stochastic gradient descent. 36 | 37 | config format: 38 | - learning_rate: Scalar learning rate. 39 | """ 40 | if config is None: config = {} 41 | config.setdefault('learning_rate', 1e-2) 42 | 43 | w -= config['learning_rate'] * dw 44 | return w, config 45 | 46 | 47 | def sgd_momentum(w, dw, config=None): 48 | """ 49 | Performs stochastic gradient descent with momentum. 50 | 51 | config format: 52 | - learning_rate: Scalar learning rate. 53 | - momentum: Scalar between 0 and 1 giving the momentum value. 54 | Setting momentum = 0 reduces to sgd. 55 | - velocity: A numpy array of the same shape as w and dw used to store a 56 | moving average of the gradients. 57 | """ 58 | if config is None: config = {} 59 | config.setdefault('learning_rate', 1e-2) 60 | config.setdefault('momentum', 0.9) 61 | v = config.get('velocity', np.zeros_like(w)) 62 | 63 | next_w = None 64 | ########################################################################### 65 | # TODO: Implement the momentum update formula. Store the updated value in # 66 | # the next_w variable. You should also use and update the velocity v. # 67 | ########################################################################### 68 | pass 69 | ########################################################################### 70 | # END OF YOUR CODE # 71 | ########################################################################### 72 | config['velocity'] = v 73 | 74 | return next_w, config 75 | 76 | 77 | 78 | def rmsprop(x, dx, config=None): 79 | """ 80 | Uses the RMSProp update rule, which uses a moving average of squared 81 | gradient values to set adaptive per-parameter learning rates. 82 | 83 | config format: 84 | - learning_rate: Scalar learning rate. 85 | - decay_rate: Scalar between 0 and 1 giving the decay rate for the squared 86 | gradient cache. 87 | - epsilon: Small scalar used for smoothing to avoid dividing by zero. 88 | - cache: Moving average of second moments of gradients. 89 | """ 90 | if config is None: config = {} 91 | config.setdefault('learning_rate', 1e-2) 92 | config.setdefault('decay_rate', 0.99) 93 | config.setdefault('epsilon', 1e-8) 94 | config.setdefault('cache', np.zeros_like(x)) 95 | 96 | next_x = None 97 | ########################################################################### 98 | # TODO: Implement the RMSprop update formula, storing the next value of x # 99 | # in the next_x variable. Don't forget to update cache value stored in # 100 | # config['cache']. # 101 | ########################################################################### 102 | pass 103 | ########################################################################### 104 | # END OF YOUR CODE # 105 | ########################################################################### 106 | 107 | return next_x, config 108 | 109 | 110 | def adam(x, dx, config=None): 111 | """ 112 | Uses the Adam update rule, which incorporates moving averages of both the 113 | gradient and its square and a bias correction term. 114 | 115 | config format: 116 | - learning_rate: Scalar learning rate. 117 | - beta1: Decay rate for moving average of first moment of gradient. 118 | - beta2: Decay rate for moving average of second moment of gradient. 119 | - epsilon: Small scalar used for smoothing to avoid dividing by zero. 120 | - m: Moving average of gradient. 121 | - v: Moving average of squared gradient. 122 | - t: Iteration number. 123 | """ 124 | if config is None: config = {} 125 | config.setdefault('learning_rate', 1e-3) 126 | config.setdefault('beta1', 0.9) 127 | config.setdefault('beta2', 0.999) 128 | config.setdefault('epsilon', 1e-8) 129 | config.setdefault('m', np.zeros_like(x)) 130 | config.setdefault('v', np.zeros_like(x)) 131 | config.setdefault('t', 1) 132 | 133 | next_x = None 134 | ########################################################################### 135 | # TODO: Implement the Adam update formula, storing the next value of x in # 136 | # the next_x variable. Don't forget to update the m, v, and t variables # 137 | # stored in config. # 138 | ########################################################################### 139 | pass 140 | ########################################################################### 141 | # END OF YOUR CODE # 142 | ########################################################################### 143 | 144 | return next_x, config 145 | -------------------------------------------------------------------------------- /assignment5/classwork/DSVC/classifiers/linear_classifier.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import numpy as np 4 | from DSVC.classifiers.linear_svm import * 5 | from past.builtins import xrange 6 | 7 | 8 | class LinearClassifier(object): 9 | 10 | def __init__(self): 11 | self.W = None 12 | 13 | def train(self, X, y, learning_rate=1e-3, reg=1e-5, num_iters=100, 14 | batch_size=200, verbose=False): 15 | """ 16 | Train this linear classifier using stochastic gradient descent. 17 | 18 | Inputs: 19 | - X: A numpy array of shape (N, D) containing training data; there are N 20 | training samples each of dimension D. 21 | - y: A numpy array of shape (N,) containing training labels; y[i] = c 22 | means that X[i] has label 0 <= c < C for C classes. 23 | - learning_rate: (float) learning rate for optimization. 24 | - reg: (float) regularization strength. 25 | - num_iters: (integer) number of steps to take when optimizing 26 | - batch_size: (integer) number of training examples to use at each step. 27 | - verbose: (boolean) If true, print progress during optimization. 28 | 29 | Outputs: 30 | A list containing the value of the loss function at each training iteration. 31 | """ 32 | num_train, dim = X.shape 33 | # assume y takes values 0...K-1 where K is number of classes 34 | num_classes = np.max(y) + 1 35 | if self.W is None: 36 | # lazily initialize W 37 | self.W = 0.001 * np.random.randn(dim, num_classes) 38 | 39 | # Run stochastic gradient descent to optimize W 40 | loss_history = [] 41 | for it in xrange(num_iters): 42 | 43 | X_batch = None 44 | y_batch = None 45 | 46 | ######################################################################### 47 | # TODO: # 48 | # Sample batch_size elements from the training data and their # 49 | # corresponding labels to use in this round of gradient descent. # 50 | # Store the data in X_batch and their corresponding labels in # 51 | # y_batch; after sampling X_batch should have shape (dim, batch_size) # 52 | # and y_batch should have shape (batch_size,) # 53 | # # 54 | # Hint: Use np.random.choice to generate indices. Sampling with # 55 | # replacement is faster than sampling without replacement. # 56 | ######################################################################### 57 | ind = np.random.choice(num_train,batch_size) 58 | X_batch = X[ind] 59 | y_batch = y[ind] 60 | ######################################################################### 61 | # END OF YOUR CODE # 62 | ######################################################################### 63 | 64 | # evaluate loss and gradient 65 | loss, grad = self.loss(X_batch, y_batch, reg) 66 | loss_history.append(loss) 67 | # loss ,grad = None,None 68 | 69 | # perform parameter update 70 | ######################################################################### 71 | # TODO: # 72 | # Update the weights using the gradient and the learning rate. # 73 | ######################################################################### 74 | self.W += -learning_rate * grad 75 | ######################################################################### 76 | # END OF YOUR CODE # 77 | ######################################################################### 78 | 79 | if verbose and it % 100 == 0: 80 | print('iteration %d / %d: loss %f' % (it, num_iters, loss)) 81 | 82 | return loss_history 83 | 84 | def predict(self, X): 85 | """ 86 | Use the trained weights of this linear classifier to predict labels for 87 | data points. 88 | 89 | Inputs: 90 | - X: A numpy array of shape (N, D) containing training data; there are N 91 | training samples each of dimension D. 92 | 93 | Returns: 94 | - y_pred: Predicted labels for the data in X. y_pred is a 1-dimensional 95 | array of length N, and each element is an integer giving the predicted 96 | class. 97 | """ 98 | y_pred = np.zeros(X.shape[0]) 99 | ########################################################################### 100 | # TODO: # 101 | # Implement this method. Store the predicted labels in y_pred. # 102 | ########################################################################### 103 | scores = X.dot(self.W) 104 | y_pred = np.argmax(scores,axis = 1) 105 | ########################################################################### 106 | # END OF YOUR CODE # 107 | ########################################################################### 108 | return y_pred 109 | 110 | def loss(self, X_batch, y_batch, reg): 111 | """ 112 | Compute the loss function and its derivative. 113 | Subclasses will override this. 114 | 115 | Inputs: 116 | - X_batch: A numpy array of shape (N, D) containing a minibatch of N 117 | data points; each point has dimension D. 118 | - y_batch: A numpy array of shape (N,) containing labels for the minibatch. 119 | - reg: (float) regularization strength. 120 | 121 | Returns: A tuple containing: 122 | - loss as a single float 123 | - gradient with respect to self.W; an array of the same shape as W 124 | """ 125 | pass 126 | 127 | 128 | class LinearSVM(LinearClassifier): 129 | """ A subclass that uses the Multiclass SVM loss function """ 130 | 131 | def loss(self, X_batch, y_batch, reg): 132 | return svm_loss_vectorized(self.W, X_batch, y_batch, reg) 133 | -------------------------------------------------------------------------------- /assignment3/classwork/DSVC/classifiers/logistic_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import random 3 | import math 4 | 5 | class LogisticRegression(object): 6 | 7 | def __init__(self): 8 | self.w = None 9 | 10 | def loss(self, X_batch, y_batch): 11 | """ 12 | Compute the loss function and its derivative. 13 | Subclasses will override this. 14 | 15 | Inputs: 16 | - X_batch: A numpy array of shape (N, D) containing a minibatch of N 17 | data points; each point has dimension D. 18 | - y_batch: A numpy array of shape (N,) containing labels for the minibatch. 19 | 20 | Returns: A tuple containing: 21 | - loss as a single float 22 | - gradient with respect to self.W; an array of the same shape as W 23 | """ 24 | 25 | ######################################################################### 26 | # TODO: # 27 | # calculate the loss and the derivative # 28 | ######################################################################### 29 | pass 30 | ######################################################################### 31 | # END OF YOUR CODE # 32 | ######################################################################### 33 | 34 | def train(self, X, y, learning_rate=1e-3, num_iters=100, 35 | batch_size=200, verbose=True): 36 | 37 | """ 38 | Train this linear classifier using stochastic gradient descent. 39 | Inputs: 40 | - X: A numpy array of shape (N, D) containing training data; there are N 41 | training samples each of dimension D. 42 | - y: A numpy array of shape (N,) containing training labels; 43 | - learning_rate: (float) learning rate for optimization. 44 | - num_iters: (integer) number of steps to take when optimizing 45 | - batch_size: (integer) number of training examples to use at each step. 46 | - verbose: (boolean) If true, print progress during optimization. 47 | 48 | Outputs: 49 | A list containing the value of the loss function at each training iteration. 50 | """ 51 | num_train, dim = X.shape 52 | 53 | if self.w is None: 54 | self.w = 0.001 * np.random.randn(dim) 55 | 56 | loss_history = [] 57 | 58 | for it in xrange(num_iters): 59 | X_batch = None 60 | y_batch = None 61 | 62 | ######################################################################### 63 | # TODO: # 64 | # Sample batch_size elements from the training data and their # 65 | # corresponding labels to use in this round of gradient descent. # 66 | # Store the data in X_batch and their corresponding labels in # 67 | # y_batch; after sampling X_batch should have shape (batch_size, dim) # 68 | # and y_batch should have shape (batch_size,) # 69 | # # 70 | # Hint: Use np.random.choice to generate indices. Sampling with # 71 | # replacement is faster than sampling without replacement. # 72 | ######################################################################### 73 | pass 74 | ######################################################################### 75 | # END OF YOUR CODE # 76 | ######################################################################### 77 | 78 | # evaluate loss and gradient 79 | loss, grad = self.loss(X_batch, y_batch) 80 | loss_history.append(loss) 81 | 82 | # perform parameter update 83 | ######################################################################### 84 | # TODO: # 85 | # Update the weights using the gradient and the learning rate. # 86 | ######################################################################### 87 | pass 88 | ######################################################################### 89 | # END OF YOUR CODE # 90 | ######################################################################### 91 | 92 | if verbose and it % 100 == 0: 93 | print 'iteration %d / %d: loss %f' % (it, num_iters, loss) 94 | 95 | return loss_history 96 | 97 | def predict(self, X): 98 | """ 99 | Use the trained weights of this linear classifier to predict labels for 100 | data points. 101 | 102 | Inputs: 103 | - X: N x D array of training data. Each column is a D-dimensional point. 104 | 105 | Returns: 106 | - y_pred: Predicted labels for the data in X. y_pred is a 1-dimensional 107 | array of length N, and each element is an integer giving the predicted 108 | class. 109 | """ 110 | y_pred = np.zeros(X.shape[1]) 111 | ########################################################################### 112 | # TODO: # 113 | # Implement this method. Store the predicted labels in y_pred. # 114 | ########################################################################### 115 | pass 116 | ########################################################################### 117 | # END OF YOUR CODE # 118 | ########################################################################### 119 | return y_pred 120 | 121 | def one_vs_all(self, X, y, learning_rate=1e-3, num_iters=100, 122 | batch_size=200, verbose = True): 123 | """ 124 | Train this linear classifier using stochastic gradient descent. 125 | Inputs: 126 | - X: A numpy array of shape (N, D) containing training data; there are N 127 | training samples each of dimension D. 128 | - y: A numpy array of shape (N,) containing training labels; 129 | - learning_rate: (float) learning rate for optimization. 130 | - num_iters: (integer) number of steps to take when optimizing 131 | - batch_size: (integer) number of training examples to use at each step. 132 | - verbose: (boolean) If true, print progress during optimization. 133 | 134 | """ 135 | -------------------------------------------------------------------------------- /assignment1/classwork/DSVC/data_utils.py: -------------------------------------------------------------------------------- 1 | import pickle as pickle 2 | import numpy as np 3 | import os 4 | from scipy.misc import imread 5 | 6 | 7 | def load_CIFAR_batch(filename): 8 | """ load single batch of cifar """ 9 | with open(filename, 'rb') as f: 10 | # BarackBao 2019.5.23 fix 11 | datadict = pickle.load(f, encoding='iso-8859-1') 12 | X = datadict['data'] 13 | Y = datadict['labels'] 14 | X = X.reshape(10000, 3, 32, 32).transpose(0, 2, 3, 1).astype("float") 15 | Y = np.array(Y) 16 | return X, Y 17 | 18 | 19 | def load_CIFAR10(ROOT, num): 20 | """ load all of cifar """ 21 | xs = [] 22 | ys = [] 23 | for b in range(1, num): 24 | f = os.path.join(ROOT, 'data_batch_%d' % (b,)) 25 | X, Y = load_CIFAR_batch(f) 26 | xs.append(X) 27 | ys.append(Y) 28 | Xtr = np.concatenate(xs) 29 | Ytr = np.concatenate(ys) 30 | del X, Y 31 | Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch')) 32 | return Xtr, Ytr, Xte, Yte 33 | 34 | 35 | def load_tiny_imagenet(path, dtype=np.float32): 36 | """ 37 | Load TinyImageNet. Each of TinyImageNet-100-A, TinyImageNet-100-B, and 38 | TinyImageNet-200 have the same directory structure, so this can be used 39 | to load any of them. 40 | 41 | Inputs: 42 | - path: String giving path to the directory to load. 43 | - dtype: numpy datatype used to load the data. 44 | 45 | Returns: A tuple of 46 | - class_names: A list where class_names[i] is a list of strings giving the 47 | WordNet names for class i in the loaded dataset. 48 | - X_train: (N_tr, 3, 64, 64) array of training images 49 | - y_train: (N_tr,) array of training labels 50 | - X_val: (N_val, 3, 64, 64) array of validation images 51 | - y_val: (N_val,) array of validation labels 52 | - X_test: (N_test, 3, 64, 64) array of testing images. 53 | - y_test: (N_test,) array of test labels; if test labels are not available 54 | (such as in student code) then y_test will be None. 55 | """ 56 | # First load wnids 57 | with open(os.path.join(path, 'wnids.txt'), 'r') as f: 58 | wnids = [x.strip() for x in f] 59 | 60 | # Map wnids to integer labels 61 | wnid_to_label = {wnid: i for i, wnid in enumerate(wnids)} 62 | 63 | # Use words.txt to get names for each class 64 | with open(os.path.join(path, 'words.txt'), 'r') as f: 65 | wnid_to_words = dict(line.split('\t') for line in f) 66 | for wnid, words in wnid_to_words.iteritems(): 67 | wnid_to_words[wnid] = [w.strip() for w in words.split(',')] 68 | class_names = [wnid_to_words[wnid] for wnid in wnids] 69 | 70 | # Next load training data. 71 | X_train = [] 72 | y_train = [] 73 | for i, wnid in enumerate(wnids): 74 | if (i + 1) % 20 == 0: 75 | print('loading training data for synset %d / %d' % (i + 1, len(wnids))) 76 | # To figure out the filenames we need to open the boxes file 77 | boxes_file = os.path.join(path, 'train', wnid, '%s_boxes.txt' % wnid) 78 | with open(boxes_file, 'r') as f: 79 | filenames = [x.split('\t')[0] for x in f] 80 | num_images = len(filenames) 81 | 82 | X_train_block = np.zeros((num_images, 3, 64, 64), dtype=dtype) 83 | y_train_block = wnid_to_label[wnid] * \ 84 | np.ones(num_images, dtype=np.int64) 85 | for j, img_file in enumerate(filenames): 86 | img_file = os.path.join(path, 'train', wnid, 'images', img_file) 87 | img = imread(img_file) 88 | if img.ndim == 2: 89 | # grayscale file 90 | img.shape = (64, 64, 1) 91 | X_train_block[j] = img.transpose(2, 0, 1) 92 | X_train.append(X_train_block) 93 | y_train.append(y_train_block) 94 | 95 | # We need to concatenate all training data 96 | X_train = np.concatenate(X_train, axis=0) 97 | y_train = np.concatenate(y_train, axis=0) 98 | 99 | # Next load validation data 100 | with open(os.path.join(path, 'val', 'val_annotations.txt'), 'r') as f: 101 | img_files = [] 102 | val_wnids = [] 103 | for line in f: 104 | img_file, wnid = line.split('\t')[:2] 105 | img_files.append(img_file) 106 | val_wnids.append(wnid) 107 | num_val = len(img_files) 108 | y_val = np.array([wnid_to_label[wnid] for wnid in val_wnids]) 109 | X_val = np.zeros((num_val, 3, 64, 64), dtype=dtype) 110 | for i, img_file in enumerate(img_files): 111 | img_file = os.path.join(path, 'val', 'images', img_file) 112 | img = imread(img_file) 113 | if img.ndim == 2: 114 | img.shape = (64, 64, 1) 115 | X_val[i] = img.transpose(2, 0, 1) 116 | 117 | # Next load test images 118 | # Students won't have test labels, so we need to iterate over files in the 119 | # images directory. 120 | img_files = os.listdir(os.path.join(path, 'test', 'images')) 121 | X_test = np.zeros((len(img_files), 3, 64, 64), dtype=dtype) 122 | for i, img_file in enumerate(img_files): 123 | img_file = os.path.join(path, 'test', 'images', img_file) 124 | img = imread(img_file) 125 | if img.ndim == 2: 126 | img.shape = (64, 64, 1) 127 | X_test[i] = img.transpose(2, 0, 1) 128 | 129 | y_test = None 130 | y_test_file = os.path.join(path, 'test', 'test_annotations.txt') 131 | if os.path.isfile(y_test_file): 132 | with open(y_test_file, 'r') as f: 133 | img_file_to_wnid = {} 134 | for line in f: 135 | line = line.split('\t') 136 | img_file_to_wnid[line[0]] = line[1] 137 | y_test = [wnid_to_label[img_file_to_wnid[img_file]] 138 | for img_file in img_files] 139 | y_test = np.array(y_test) 140 | 141 | return class_names, X_train, y_train, X_val, y_val, X_test, y_test 142 | 143 | 144 | def load_models(models_dir): 145 | """ 146 | Load saved models from disk. This will attempt to unpickle all files in a 147 | directory; any files that give errors on unpickling (such as README.txt) will 148 | be skipped. 149 | 150 | Inputs: 151 | - models_dir: String giving the path to a directory containing model files. 152 | Each model file is a pickled dictionary with a 'model' field. 153 | 154 | Returns: 155 | A dictionary mapping model file names to models. 156 | """ 157 | models = {} 158 | for model_file in os.listdir(models_dir): 159 | with open(os.path.join(models_dir, model_file), 'rb') as f: 160 | try: 161 | models[model_file] = pickle.load(f)['model'] 162 | except pickle.UnpicklingError: 163 | continue 164 | return models 165 | -------------------------------------------------------------------------------- /machinelearning/Lesson2/GradientDescentAlgorithm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Gradient Descent Algorithm" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## 实验说明\n", 15 | "使用梯度下降算法拟合1-100之间实数值的平方根\n", 16 | "\n", 17 | "该方法可以拟合任何函数关系式，拟合的准确率与learning_rate（学习率）和epoch（迭代次数）相关\n", 18 | "\n", 19 | "下面实验计算了 y=x^2" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 1, 25 | "metadata": { 26 | "collapsed": true 27 | }, 28 | "outputs": [], 29 | "source": [ 30 | "import math" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 2, 36 | "metadata": {}, 37 | "outputs": [ 38 | { 39 | "name": "stdout", 40 | "output_type": "stream", 41 | "text": [ 42 | " 1的平方(近似)为：1.00000000，真实值是：1.00000000\n", 43 | " 2的平方(近似)为：4.00000000，真实值是：4.00000000\n", 44 | " 3的平方(近似)为：9.00000000，真实值是：9.00000000\n", 45 | " 4的平方(近似)为：16.00000000，真实值是：16.00000000\n", 46 | " 5的平方(近似)为：25.00000000，真实值是：25.00000000\n", 47 | " 6的平方(近似)为：36.00000000，真实值是：36.00000000\n", 48 | " 7的平方(近似)为：49.00000000，真实值是：49.00000000\n", 49 | " 8的平方(近似)为：64.00000000，真实值是：64.00000000\n", 50 | " 9的平方(近似)为：81.00000000，真实值是：81.00000000\n", 51 | " 10的平方(近似)为：100.00000000，真实值是：100.00000000\n", 52 | " 11的平方(近似)为：121.00000000，真实值是：121.00000000\n", 53 | " 12的平方(近似)为：144.00000000，真实值是：144.00000000\n", 54 | " 13的平方(近似)为：169.00000000，真实值是：169.00000000\n", 55 | " 14的平方(近似)为：196.00000000，真实值是：196.00000000\n", 56 | " 15的平方(近似)为：225.00000000，真实值是：225.00000000\n", 57 | " 16的平方(近似)为：256.00000000，真实值是：256.00000000\n", 58 | " 17的平方(近似)为：289.00000000，真实值是：289.00000000\n", 59 | " 18的平方(近似)为：324.00000000，真实值是：324.00000000\n", 60 | " 19的平方(近似)为：361.00000000，真实值是：361.00000000\n", 61 | " 20的平方(近似)为：400.00000000，真实值是：400.00000000\n", 62 | " 21的平方(近似)为：441.00000000，真实值是：441.00000000\n", 63 | " 22的平方(近似)为：484.00000000，真实值是：484.00000000\n", 64 | " 23的平方(近似)为：529.00000000，真实值是：529.00000000\n", 65 | " 24的平方(近似)为：576.00000000，真实值是：576.00000000\n", 66 | " 25的平方(近似)为：625.00000000，真实值是：625.00000000\n", 67 | " 26的平方(近似)为：676.00000000，真实值是：676.00000000\n", 68 | " 27的平方(近似)为：729.00000000，真实值是：729.00000000\n", 69 | " 28的平方(近似)为：784.00000000，真实值是：784.00000000\n", 70 | " 29的平方(近似)为：841.00000000，真实值是：841.00000000\n", 71 | " 30的平方(近似)为：900.00000000，真实值是：900.00000000\n", 72 | " 31的平方(近似)为：961.00000000，真实值是：961.00000000\n", 73 | " 32的平方(近似)为：1024.00000000，真实值是：1024.00000000\n", 74 | " 33的平方(近似)为：1089.00000000，真实值是：1089.00000000\n", 75 | " 34的平方(近似)为：1156.00000000，真实值是：1156.00000000\n", 76 | " 35的平方(近似)为：1225.00000000，真实值是：1225.00000000\n", 77 | " 36的平方(近似)为：1296.00000000，真实值是：1296.00000000\n", 78 | " 37的平方(近似)为：1369.00000000，真实值是：1369.00000000\n", 79 | " 38的平方(近似)为：1444.00000000，真实值是：1444.00000000\n", 80 | " 39的平方(近似)为：1521.00000000，真实值是：1521.00000000\n", 81 | " 40的平方(近似)为：1600.00000000，真实值是：1600.00000000\n", 82 | " 41的平方(近似)为：1681.00000000，真实值是：1681.00000000\n", 83 | " 42的平方(近似)为：1764.00000000，真实值是：1764.00000000\n", 84 | " 43的平方(近似)为：1849.00000000，真实值是：1849.00000000\n", 85 | " 44的平方(近似)为：1936.00000000，真实值是：1936.00000000\n", 86 | " 45的平方(近似)为：2025.00000000，真实值是：2025.00000000\n", 87 | " 46的平方(近似)为：2116.00000000，真实值是：2116.00000000\n", 88 | " 47的平方(近似)为：2209.00000000，真实值是：2209.00000000\n", 89 | " 48的平方(近似)为：2304.00000000，真实值是：2304.00000000\n", 90 | " 49的平方(近似)为：2401.00000000，真实值是：2401.00000000\n", 91 | " 50的平方(近似)为：2500.00000000，真实值是：2500.00000000\n", 92 | " 51的平方(近似)为：2601.00000000，真实值是：2601.00000000\n", 93 | " 52的平方(近似)为：2704.00000000，真实值是：2704.00000000\n", 94 | " 53的平方(近似)为：2809.00000000，真实值是：2809.00000000\n", 95 | " 54的平方(近似)为：2916.00000000，真实值是：2916.00000000\n", 96 | " 55的平方(近似)为：3025.00000000，真实值是：3025.00000000\n", 97 | " 56的平方(近似)为：3136.00000000，真实值是：3136.00000000\n", 98 | " 57的平方(近似)为：3249.00000000，真实值是：3249.00000000\n", 99 | " 58的平方(近似)为：3364.00000000，真实值是：3364.00000000\n", 100 | " 59的平方(近似)为：3481.00000000，真实值是：3481.00000000\n", 101 | " 60的平方(近似)为：3600.00000000，真实值是：3600.00000000\n", 102 | " 61的平方(近似)为：3721.00000000，真实值是：3721.00000000\n", 103 | " 62的平方(近似)为：3844.00000000，真实值是：3844.00000000\n", 104 | " 63的平方(近似)为：3969.00000000，真实值是：3969.00000000\n", 105 | " 64的平方(近似)为：4096.00000000，真实值是：4096.00000000\n", 106 | " 65的平方(近似)为：4224.99999999，真实值是：4225.00000000\n", 107 | " 66的平方(近似)为：4355.99999999，真实值是：4356.00000000\n", 108 | " 67的平方(近似)为：4488.99999999，真实值是：4489.00000000\n", 109 | " 68的平方(近似)为：4623.99999999，真实值是：4624.00000000\n", 110 | " 69的平方(近似)为：4760.99999999，真实值是：4761.00000000\n", 111 | " 70的平方(近似)为：4899.99999999，真实值是：4900.00000000\n", 112 | " 71的平方(近似)为：5040.99999999，真实值是：5041.00000000\n", 113 | " 72的平方(近似)为：5183.99999999，真实值是：5184.00000000\n", 114 | " 73的平方(近似)为：5328.99999999，真实值是：5329.00000000\n", 115 | " 74的平方(近似)为：5475.99999999，真实值是：5476.00000000\n", 116 | " 75的平方(近似)为：5624.99999999，真实值是：5625.00000000\n", 117 | " 76的平方(近似)为：5775.99999999，真实值是：5776.00000000\n", 118 | " 77的平方(近似)为：5928.99999999，真实值是：5929.00000000\n", 119 | " 78的平方(近似)为：6083.99999999，真实值是：6084.00000000\n", 120 | " 79的平方(近似)为：6240.99999999，真实值是：6241.00000000\n", 121 | " 80的平方(近似)为：6399.99999999，真实值是：6400.00000000\n", 122 | " 81的平方(近似)为：6560.99999999，真实值是：6561.00000000\n", 123 | " 82的平方(近似)为：6723.99999999，真实值是：6724.00000000\n", 124 | " 83的平方(近似)为：6888.99999999，真实值是：6889.00000000\n", 125 | " 84的平方(近似)为：7055.99999999，真实值是：7056.00000000\n", 126 | " 85的平方(近似)为：7224.99999999，真实值是：7225.00000000\n", 127 | " 86的平方(近似)为：7395.99999999，真实值是：7396.00000000\n", 128 | " 87的平方(近似)为：7568.99999999，真实值是：7569.00000000\n", 129 | " 88的平方(近似)为：7743.99999999，真实值是：7744.00000000\n", 130 | " 89的平方(近似)为：7920.99999999，真实值是：7921.00000000\n", 131 | " 90的平方(近似)为：8099.99999999，真实值是：8100.00000000\n", 132 | " 91的平方(近似)为：8280.99999998，真实值是：8281.00000000\n", 133 | " 92的平方(近似)为：8463.99999998，真实值是：8464.00000000\n", 134 | " 93的平方(近似)为：8648.99999998，真实值是：8649.00000000\n", 135 | " 94的平方(近似)为：8835.99999998，真实值是：8836.00000000\n", 136 | " 95的平方(近似)为：9024.99999997，真实值是：9025.00000000\n", 137 | " 96的平方(近似)为：9215.99999996，真实值是：9216.00000000\n", 138 | " 97的平方(近似)为：9408.99999995，真实值是：9409.00000000\n", 139 | " 98的平方(近似)为：9603.99999994，真实值是：9604.00000000\n", 140 | " 99的平方(近似)为：9800.99999992，真实值是：9801.00000000\n" 141 | ] 142 | } 143 | ], 144 | "source": [ 145 | "learning_rate = 0.01\n", 146 | "epoch = 500000\n", 147 | "for a in range(1, 100):\n", 148 | " cur = 0\n", 149 | " for i in range(epoch):\n", 150 | " cur -= learning_rate*(math.sqrt(cur)-a)\n", 151 | " print(' %d的平方(近似)为：%.8f，真实值是：%.8f' % (a, cur, a**2))" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": null, 157 | "metadata": { 158 | "collapsed": true 159 | }, 160 | "outputs": [], 161 | "source": [] 162 | } 163 | ], 164 | "metadata": { 165 | "kernelspec": { 166 | "display_name": "Python 3", 167 | "language": "python", 168 | "name": "python3" 169 | }, 170 | "language_info": { 171 | "codemirror_mode": { 172 | "name": "ipython", 173 | "version": 3 174 | }, 175 | "file_extension": ".py", 176 | "mimetype": "text/x-python", 177 | "name": "python", 178 | "nbconvert_exporter": "python", 179 | "pygments_lexer": "ipython3", 180 | "version": "3.6.3" 181 | } 182 | }, 183 | "nbformat": 4, 184 | "nbformat_minor": 2 185 | } 186 | -------------------------------------------------------------------------------- /assignment5/classwork/DSVC/data_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | from six.moves import cPickle as pickle 4 | import numpy as np 5 | import os 6 | from scipy.misc import imread 7 | import platform 8 | 9 | def load_pickle(f): 10 | version = platform.python_version_tuple() 11 | if version[0] == '2': 12 | return pickle.load(f) 13 | elif version[0] == '3': 14 | return pickle.load(f, encoding='latin1') 15 | raise ValueError("invalid python version: {}".format(version)) 16 | 17 | def load_CIFAR_batch(filename): 18 | """ load single batch of cifar """ 19 | with open(filename, 'rb') as f: 20 | datadict = load_pickle(f) 21 | X = datadict['data'] 22 | Y = datadict['labels'] 23 | X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float") 24 | Y = np.array(Y) 25 | return X, Y 26 | 27 | def load_CIFAR10(ROOT): 28 | """ load all of cifar """ 29 | xs = [] 30 | ys = [] 31 | for b in range(1,6): 32 | f = os.path.join(ROOT, 'data_batch_%d' % (b, )) 33 | X, Y = load_CIFAR_batch(f) 34 | xs.append(X) 35 | ys.append(Y) 36 | Xtr = np.concatenate(xs) 37 | Ytr = np.concatenate(ys) 38 | del X, Y 39 | Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch')) 40 | return Xtr, Ytr, Xte, Yte 41 | 42 | 43 | def get_CIFAR10_data(num_training=49000, num_validation=1000, num_test=1000, 44 | subtract_mean=True): 45 | """ 46 | Load the CIFAR-10 dataset from disk and perform preprocessing to prepare 47 | it for classifiers. These are the same steps as we used for the SVM, but 48 | condensed to a single function. 49 | """ 50 | # Load the raw CIFAR-10 data 51 | cifar10_dir = 'cs231n/datasets/cifar-10-batches-py' 52 | X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir) 53 | 54 | # Subsample the data 55 | mask = list(range(num_training, num_training + num_validation)) 56 | X_val = X_train[mask] 57 | y_val = y_train[mask] 58 | mask = list(range(num_training)) 59 | X_train = X_train[mask] 60 | y_train = y_train[mask] 61 | mask = list(range(num_test)) 62 | X_test = X_test[mask] 63 | y_test = y_test[mask] 64 | 65 | # Normalize the data: subtract the mean image 66 | if subtract_mean: 67 | mean_image = np.mean(X_train, axis=0) 68 | X_train -= mean_image 69 | X_val -= mean_image 70 | X_test -= mean_image 71 | 72 | # Transpose so that channels come first 73 | X_train = X_train.transpose(0, 3, 1, 2).copy() 74 | X_val = X_val.transpose(0, 3, 1, 2).copy() 75 | X_test = X_test.transpose(0, 3, 1, 2).copy() 76 | 77 | # Package data into a dictionary 78 | return { 79 | 'X_train': X_train, 'y_train': y_train, 80 | 'X_val': X_val, 'y_val': y_val, 81 | 'X_test': X_test, 'y_test': y_test, 82 | } 83 | 84 | 85 | def load_tiny_imagenet(path, dtype=np.float32, subtract_mean=True): 86 | """ 87 | Load TinyImageNet. Each of TinyImageNet-100-A, TinyImageNet-100-B, and 88 | TinyImageNet-200 have the same directory structure, so this can be used 89 | to load any of them. 90 | 91 | Inputs: 92 | - path: String giving path to the directory to load. 93 | - dtype: numpy datatype used to load the data. 94 | - subtract_mean: Whether to subtract the mean training image. 95 | 96 | Returns: A dictionary with the following entries: 97 | - class_names: A list where class_names[i] is a list of strings giving the 98 | WordNet names for class i in the loaded dataset. 99 | - X_train: (N_tr, 3, 64, 64) array of training images 100 | - y_train: (N_tr,) array of training labels 101 | - X_val: (N_val, 3, 64, 64) array of validation images 102 | - y_val: (N_val,) array of validation labels 103 | - X_test: (N_test, 3, 64, 64) array of testing images. 104 | - y_test: (N_test,) array of test labels; if test labels are not available 105 | (such as in student code) then y_test will be None. 106 | - mean_image: (3, 64, 64) array giving mean training image 107 | """ 108 | # First load wnids 109 | with open(os.path.join(path, 'wnids.txt'), 'r') as f: 110 | wnids = [x.strip() for x in f] 111 | 112 | # Map wnids to integer labels 113 | wnid_to_label = {wnid: i for i, wnid in enumerate(wnids)} 114 | 115 | # Use words.txt to get names for each class 116 | with open(os.path.join(path, 'words.txt'), 'r') as f: 117 | wnid_to_words = dict(line.split('\t') for line in f) 118 | for wnid, words in wnid_to_words.iteritems(): 119 | wnid_to_words[wnid] = [w.strip() for w in words.split(',')] 120 | class_names = [wnid_to_words[wnid] for wnid in wnids] 121 | 122 | # Next load training data. 123 | X_train = [] 124 | y_train = [] 125 | for i, wnid in enumerate(wnids): 126 | if (i + 1) % 20 == 0: 127 | print('loading training data for synset %d / %d' % (i + 1, len(wnids))) 128 | # To figure out the filenames we need to open the boxes file 129 | boxes_file = os.path.join(path, 'train', wnid, '%s_boxes.txt' % wnid) 130 | with open(boxes_file, 'r') as f: 131 | filenames = [x.split('\t')[0] for x in f] 132 | num_images = len(filenames) 133 | 134 | X_train_block = np.zeros((num_images, 3, 64, 64), dtype=dtype) 135 | y_train_block = wnid_to_label[wnid] * np.ones(num_images, dtype=np.int64) 136 | for j, img_file in enumerate(filenames): 137 | img_file = os.path.join(path, 'train', wnid, 'images', img_file) 138 | img = imread(img_file) 139 | if img.ndim == 2: 140 | ## grayscale file 141 | img.shape = (64, 64, 1) 142 | X_train_block[j] = img.transpose(2, 0, 1) 143 | X_train.append(X_train_block) 144 | y_train.append(y_train_block) 145 | 146 | # We need to concatenate all training data 147 | X_train = np.concatenate(X_train, axis=0) 148 | y_train = np.concatenate(y_train, axis=0) 149 | 150 | # Next load validation data 151 | with open(os.path.join(path, 'val', 'val_annotations.txt'), 'r') as f: 152 | img_files = [] 153 | val_wnids = [] 154 | for line in f: 155 | img_file, wnid = line.split('\t')[:2] 156 | img_files.append(img_file) 157 | val_wnids.append(wnid) 158 | num_val = len(img_files) 159 | y_val = np.array([wnid_to_label[wnid] for wnid in val_wnids]) 160 | X_val = np.zeros((num_val, 3, 64, 64), dtype=dtype) 161 | for i, img_file in enumerate(img_files): 162 | img_file = os.path.join(path, 'val', 'images', img_file) 163 | img = imread(img_file) 164 | if img.ndim == 2: 165 | img.shape = (64, 64, 1) 166 | X_val[i] = img.transpose(2, 0, 1) 167 | 168 | # Next load test images 169 | # Students won't have test labels, so we need to iterate over files in the 170 | # images directory. 171 | img_files = os.listdir(os.path.join(path, 'test', 'images')) 172 | X_test = np.zeros((len(img_files), 3, 64, 64), dtype=dtype) 173 | for i, img_file in enumerate(img_files): 174 | img_file = os.path.join(path, 'test', 'images', img_file) 175 | img = imread(img_file) 176 | if img.ndim == 2: 177 | img.shape = (64, 64, 1) 178 | X_test[i] = img.transpose(2, 0, 1) 179 | 180 | y_test = None 181 | y_test_file = os.path.join(path, 'test', 'test_annotations.txt') 182 | if os.path.isfile(y_test_file): 183 | with open(y_test_file, 'r') as f: 184 | img_file_to_wnid = {} 185 | for line in f: 186 | line = line.split('\t') 187 | img_file_to_wnid[line[0]] = line[1] 188 | y_test = [wnid_to_label[img_file_to_wnid[img_file]] for img_file in img_files] 189 | y_test = np.array(y_test) 190 | 191 | mean_image = X_train.mean(axis=0) 192 | if subtract_mean: 193 | X_train -= mean_image[None] 194 | X_val -= mean_image[None] 195 | X_test -= mean_image[None] 196 | 197 | return { 198 | 'class_names': class_names, 199 | 'X_train': X_train, 200 | 'y_train': y_train, 201 | 'X_val': X_val, 202 | 'y_val': y_val, 203 | 'X_test': X_test, 204 | 'y_test': y_test, 205 | 'class_names': class_names, 206 | 'mean_image': mean_image, 207 | } 208 | 209 | 210 | def load_models(models_dir): 211 | """ 212 | Load saved models from disk. This will attempt to unpickle all files in a 213 | directory; any files that give errors on unpickling (such as README.txt) will 214 | be skipped. 215 | 216 | Inputs: 217 | - models_dir: String giving the path to a directory containing model files. 218 | Each model file is a pickled dictionary with a 'model' field. 219 | 220 | Returns: 221 | A dictionary mapping model file names to models. 222 | """ 223 | models = {} 224 | for model_file in os.listdir(models_dir): 225 | with open(os.path.join(models_dir, model_file), 'rb') as f: 226 | try: 227 | models[model_file] = load_pickle(f)['model'] 228 | except pickle.UnpicklingError: 229 | continue 230 | return models 231 | -------------------------------------------------------------------------------- /assignment6/classwork/DSVC/data_utils.py: -------------------------------------------------------------------------------- 1 | # from __future__ import print_function 2 | 3 | # from six.moves import cPickle as pickle 4 | import pickle 5 | import numpy as np 6 | import os 7 | from scipy.misc import imread 8 | import platform 9 | 10 | def load_pickle(f): 11 | version = platform.python_version_tuple() 12 | if version[0] == '2': 13 | return pickle.load(f) 14 | elif version[0] == '3': 15 | return pickle.load(f, encoding='latin1') 16 | raise ValueError("invalid python version: {}".format(version)) 17 | 18 | def load_CIFAR_batch(filename): 19 | """ load single batch of cifar """ 20 | with open(filename, 'rb') as f: 21 | datadict = load_pickle(f) 22 | X = datadict['data'] 23 | Y = datadict['labels'] 24 | X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float") 25 | Y = np.array(Y) 26 | return X, Y 27 | 28 | def load_CIFAR10(ROOT): 29 | """ load all of cifar """ 30 | xs = [] 31 | ys = [] 32 | for b in range(1,6): 33 | f = os.path.join(ROOT, 'data_batch_%d' % (b, )) 34 | X, Y = load_CIFAR_batch(f) 35 | xs.append(X) 36 | ys.append(Y) 37 | Xtr = np.concatenate(xs) 38 | Ytr = np.concatenate(ys) 39 | del X, Y 40 | Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch')) 41 | return Xtr, Ytr, Xte, Yte 42 | 43 | 44 | def get_CIFAR10_data(num_training=49000, num_validation=1000, num_test=1000, 45 | subtract_mean=True): 46 | """ 47 | Load the CIFAR-10 dataset from disk and perform preprocessing to prepare 48 | it for classifiers. These are the same steps as we used for the SVM, but 49 | condensed to a single function. 50 | """ 51 | # Load the raw CIFAR-10 data 52 | cifar10_dir = 'cs231n/datasets/cifar-10-batches-py' 53 | X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir) 54 | 55 | # Subsample the data 56 | mask = list(range(num_training, num_training + num_validation)) 57 | X_val = X_train[mask] 58 | y_val = y_train[mask] 59 | mask = list(range(num_training)) 60 | X_train = X_train[mask] 61 | y_train = y_train[mask] 62 | mask = list(range(num_test)) 63 | X_test = X_test[mask] 64 | y_test = y_test[mask] 65 | 66 | # Normalize the data: subtract the mean image 67 | if subtract_mean: 68 | mean_image = np.mean(X_train, axis=0) 69 | X_train -= mean_image 70 | X_val -= mean_image 71 | X_test -= mean_image 72 | 73 | # Transpose so that channels come first 74 | X_train = X_train.transpose(0, 3, 1, 2).copy() 75 | X_val = X_val.transpose(0, 3, 1, 2).copy() 76 | X_test = X_test.transpose(0, 3, 1, 2).copy() 77 | 78 | # Package data into a dictionary 79 | return { 80 | 'X_train': X_train, 'y_train': y_train, 81 | 'X_val': X_val, 'y_val': y_val, 82 | 'X_test': X_test, 'y_test': y_test, 83 | } 84 | 85 | 86 | def load_tiny_imagenet(path, dtype=np.float32, subtract_mean=True): 87 | """ 88 | Load TinyImageNet. Each of TinyImageNet-100-A, TinyImageNet-100-B, and 89 | TinyImageNet-200 have the same directory structure, so this can be used 90 | to load any of them. 91 | 92 | Inputs: 93 | - path: String giving path to the directory to load. 94 | - dtype: numpy datatype used to load the data. 95 | - subtract_mean: Whether to subtract the mean training image. 96 | 97 | Returns: A dictionary with the following entries: 98 | - class_names: A list where class_names[i] is a list of strings giving the 99 | WordNet names for class i in the loaded dataset. 100 | - X_train: (N_tr, 3, 64, 64) array of training images 101 | - y_train: (N_tr,) array of training labels 102 | - X_val: (N_val, 3, 64, 64) array of validation images 103 | - y_val: (N_val,) array of validation labels 104 | - X_test: (N_test, 3, 64, 64) array of testing images. 105 | - y_test: (N_test,) array of test labels; if test labels are not available 106 | (such as in student code) then y_test will be None. 107 | - mean_image: (3, 64, 64) array giving mean training image 108 | """ 109 | # First load wnids 110 | with open(os.path.join(path, 'wnids.txt'), 'r') as f: 111 | wnids = [x.strip() for x in f] 112 | 113 | # Map wnids to integer labels 114 | wnid_to_label = {wnid: i for i, wnid in enumerate(wnids)} 115 | 116 | # Use words.txt to get names for each class 117 | with open(os.path.join(path, 'words.txt'), 'r') as f: 118 | wnid_to_words = dict(line.split('\t') for line in f) 119 | for wnid, words in wnid_to_words.iteritems(): 120 | wnid_to_words[wnid] = [w.strip() for w in words.split(',')] 121 | class_names = [wnid_to_words[wnid] for wnid in wnids] 122 | 123 | # Next load training data. 124 | X_train = [] 125 | y_train = [] 126 | for i, wnid in enumerate(wnids): 127 | if (i + 1) % 20 == 0: 128 | print('loading training data for synset %d / %d' % (i + 1, len(wnids))) 129 | # To figure out the filenames we need to open the boxes file 130 | boxes_file = os.path.join(path, 'train', wnid, '%s_boxes.txt' % wnid) 131 | with open(boxes_file, 'r') as f: 132 | filenames = [x.split('\t')[0] for x in f] 133 | num_images = len(filenames) 134 | 135 | X_train_block = np.zeros((num_images, 3, 64, 64), dtype=dtype) 136 | y_train_block = wnid_to_label[wnid] * np.ones(num_images, dtype=np.int64) 137 | for j, img_file in enumerate(filenames): 138 | img_file = os.path.join(path, 'train', wnid, 'images', img_file) 139 | img = imread(img_file) 140 | if img.ndim == 2: 141 | ## grayscale file 142 | img.shape = (64, 64, 1) 143 | X_train_block[j] = img.transpose(2, 0, 1) 144 | X_train.append(X_train_block) 145 | y_train.append(y_train_block) 146 | 147 | # We need to concatenate all training data 148 | X_train = np.concatenate(X_train, axis=0) 149 | y_train = np.concatenate(y_train, axis=0) 150 | 151 | # Next load validation data 152 | with open(os.path.join(path, 'val', 'val_annotations.txt'), 'r') as f: 153 | img_files = [] 154 | val_wnids = [] 155 | for line in f: 156 | img_file, wnid = line.split('\t')[:2] 157 | img_files.append(img_file) 158 | val_wnids.append(wnid) 159 | num_val = len(img_files) 160 | y_val = np.array([wnid_to_label[wnid] for wnid in val_wnids]) 161 | X_val = np.zeros((num_val, 3, 64, 64), dtype=dtype) 162 | for i, img_file in enumerate(img_files): 163 | img_file = os.path.join(path, 'val', 'images', img_file) 164 | img = imread(img_file) 165 | if img.ndim == 2: 166 | img.shape = (64, 64, 1) 167 | X_val[i] = img.transpose(2, 0, 1) 168 | 169 | # Next load test images 170 | # Students won't have test labels, so we need to iterate over files in the 171 | # images directory. 172 | img_files = os.listdir(os.path.join(path, 'test', 'images')) 173 | X_test = np.zeros((len(img_files), 3, 64, 64), dtype=dtype) 174 | for i, img_file in enumerate(img_files): 175 | img_file = os.path.join(path, 'test', 'images', img_file) 176 | img = imread(img_file) 177 | if img.ndim == 2: 178 | img.shape = (64, 64, 1) 179 | X_test[i] = img.transpose(2, 0, 1) 180 | 181 | y_test = None 182 | y_test_file = os.path.join(path, 'test', 'test_annotations.txt') 183 | if os.path.isfile(y_test_file): 184 | with open(y_test_file, 'r') as f: 185 | img_file_to_wnid = {} 186 | for line in f: 187 | line = line.split('\t') 188 | img_file_to_wnid[line[0]] = line[1] 189 | y_test = [wnid_to_label[img_file_to_wnid[img_file]] for img_file in img_files] 190 | y_test = np.array(y_test) 191 | 192 | mean_image = X_train.mean(axis=0) 193 | if subtract_mean: 194 | X_train -= mean_image[None] 195 | X_val -= mean_image[None] 196 | X_test -= mean_image[None] 197 | 198 | return { 199 | 'class_names': class_names, 200 | 'X_train': X_train, 201 | 'y_train': y_train, 202 | 'X_val': X_val, 203 | 'y_val': y_val, 204 | 'X_test': X_test, 205 | 'y_test': y_test, 206 | 'class_names': class_names, 207 | 'mean_image': mean_image, 208 | } 209 | 210 | 211 | def load_models(models_dir): 212 | """ 213 | Load saved models from disk. This will attempt to unpickle all files in a 214 | directory; any files that give errors on unpickling (such as README.txt) will 215 | be skipped. 216 | 217 | Inputs: 218 | - models_dir: String giving the path to a directory containing model files. 219 | Each model file is a pickled dictionary with a 'model' field. 220 | 221 | Returns: 222 | A dictionary mapping model file names to models. 223 | """ 224 | models = {} 225 | for model_file in os.listdir(models_dir): 226 | with open(os.path.join(models_dir, model_file), 'rb') as f: 227 | try: 228 | models[model_file] = load_pickle(f)['model'] 229 | except pickle.UnpicklingError: 230 | continue 231 | return models 232 | -------------------------------------------------------------------------------- /assignment1/classwork/DSVC/classifiers/k_nearest_neighbor.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class KNearestNeighbor(object): 5 | """ a kNN classifier with L2 distance """ 6 | 7 | def __init__(self): 8 | pass 9 | 10 | def train(self, X, y): 11 | """ 12 | Train the classifier. For k-nearest neighbors this is just 13 | memorizing the training data. 14 | 15 | Inputs: 16 | - X: A numpy array of shape (num_train, D) containing the training data 17 | consisting of num_train samples each of dimension D. 18 | - y: A numpy array of shape (N,) containing the training labels, where 19 | y[i] is the label for X[i]. 20 | """ 21 | self.X_train = X 22 | self.y_train = y 23 | 24 | def predict(self, X, k=1, num_loops=0): 25 | """ 26 | Predict labels for test data using this classifier. 27 | 28 | Inputs: 29 | - X: A numpy array of shape (num_test, D) containing test data consisting 30 | of num_test samples each of dimension D. 31 | - k: The number of nearest neighbors that vote for the predicted labels. 32 | - num_loops: Determines which implementation to use to compute distances 33 | between training points and testing points. 34 | 35 | Returns: 36 | - y: A numpy array of shape (num_test,) containing predicted labels for the 37 | test data, where y[i] is the predicted label for the test point X[i]. 38 | """ 39 | if num_loops == 0: 40 | dists = self.compute_distances_no_loops(X) 41 | elif num_loops == 1: 42 | dists = self.compute_distances_one_loop(X) 43 | elif num_loops == 2: 44 | dists = self.compute_distances_two_loops(X) 45 | else: 46 | raise ValueError('Invalid value %d for num_loops' % num_loops) 47 | 48 | return self.predict_labels(dists, k=k) 49 | 50 | def compute_distances_two_loops(self, X): 51 | """ 52 | Compute the distance between each test point in X and each training point 53 | in self.X_train using a nested loop over both the training data and the 54 | test data. 55 | 56 | Inputs: 57 | - X: A numpy array of shape (num_test, D) containing test data. 58 | 59 | Returns: 60 | - dists: A numpy array of shape (num_test, num_train) where dists[i, j] 61 | is the Euclidean distance between the ith test point and the jth training 62 | point. 63 | """ 64 | num_test = X.shape[0] 65 | num_train = self.X_train.shape[0] 66 | dists = np.zeros((num_test, num_train)) 67 | for i in range(num_test): 68 | for j in range(num_train): 69 | ##################################################################### 70 | # TODO: # 71 | # Compute the l2 distance between the ith test point and the jth # 72 | # training point, and store the result in dists[i, j]. You should # 73 | # not use a loop over dimension. # 74 | ##################################################################### 75 | pass 76 | ##################################################################### 77 | # END OF YOUR CODE # 78 | ##################################################################### 79 | return dists 80 | 81 | def compute_distances_one_loop(self, X): 82 | """ 83 | Compute the distance between each test point in X and each training point 84 | in self.X_train using a single loop over the test data. 85 | 86 | Input / Output: Same as compute_distances_two_loops 87 | """ 88 | num_test = X.shape[0] 89 | num_train = self.X_train.shape[0] 90 | dists = np.zeros((num_test, num_train)) 91 | for i in range(num_test): 92 | ####################################################################### 93 | # TODO: # 94 | # Compute the l2 distance between the ith test point and all training # 95 | # points, and store the result in dists[i, :]. # 96 | ####################################################################### 97 | pass 98 | ####################################################################### 99 | # END OF YOUR CODE # 100 | ####################################################################### 101 | return dists 102 | 103 | def compute_distances_no_loops(self, X): 104 | """ 105 | Compute the distance between each test point in X and each training point 106 | in self.X_train using no explicit loops. 107 | 108 | Input / Output: Same as compute_distances_two_loops 109 | """ 110 | num_test = X.shape[0] 111 | num_train = self.X_train.shape[0] 112 | dists = np.zeros((num_test, num_train)) 113 | ######################################################################### 114 | # TODO: # 115 | # Compute the l2 distance between all test points and all training # 116 | # points without using any explicit loops, and store the result in # 117 | # dists. # 118 | # # 119 | # You should implement this function using only basic array operations; # 120 | # in particular you should not use functions from scipy. # 121 | # # 122 | # HINT: Try to formulate the l2 distance using matrix multiplication # 123 | # and two broadcast sums. # 124 | ######################################################################### 125 | pass 126 | ######################################################################### 127 | # END OF YOUR CODE # 128 | ######################################################################### 129 | return dists 130 | 131 | def predict_labels(self, dists, k=1): 132 | """ 133 | Given a matrix of distances between test points and training points, 134 | predict a label for each test point. 135 | 136 | Inputs: 137 | - dists: A numpy array of shape (num_test, num_train) where dists[i, j] 138 | gives the distance betwen the ith test point and the jth training point. 139 | 140 | Returns: 141 | - y: A numpy array of shape (num_test,) containing predicted labels for the 142 | test data, where y[i] is the predicted label for the test point X[i]. 143 | """ 144 | num_test = dists.shape[0] 145 | y_pred = np.zeros(num_test) 146 | for i in range(num_test): 147 | # A list of length k storing the labels of the k nearest neighbors to 148 | # the ith test point. 149 | closest_y = [] 150 | ######################################################################### 151 | # TODO: # 152 | # Use the distance matrix to find the k nearest neighbors of the ith # 153 | # testing point, and use self.y_train to find the labels of these # 154 | # neighbors. Store these labels in closest_y. # 155 | # Hint: Look up the function numpy.argsort. # 156 | ######################################################################### 157 | pass 158 | ######################################################################### 159 | # TODO: # 160 | # Now that you have found the labels of the k nearest neighbors, you # 161 | # need to find the most common label in the list closest_y of labels. # 162 | # Store this label in y_pred[i]. Break ties by choosing the smaller # 163 | # label. # 164 | ######################################################################### 165 | pass 166 | ######################################################################### 167 | # END OF YOUR CODE # 168 | ######################################################################### 169 | 170 | return y_pred 171 | -------------------------------------------------------------------------------- /assignment7/classwork/DSVC/data_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | from builtins import range 4 | from six.moves import cPickle as pickle 5 | import numpy as np 6 | import os 7 | from scipy.misc import imread 8 | import platform 9 | 10 | def load_pickle(f): 11 | version = platform.python_version_tuple() 12 | if version[0] == '2': 13 | return pickle.load(f) 14 | elif version[0] == '3': 15 | return pickle.load(f, encoding='latin1') 16 | raise ValueError("invalid python version: {}".format(version)) 17 | 18 | def load_CIFAR_batch(filename): 19 | """ load single batch of cifar """ 20 | with open(filename, 'rb') as f: 21 | datadict = load_pickle(f) 22 | X = datadict['data'] 23 | Y = datadict['labels'] 24 | X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float") 25 | Y = np.array(Y) 26 | return X, Y 27 | 28 | def load_CIFAR10(ROOT): 29 | """ load all of cifar """ 30 | xs = [] 31 | ys = [] 32 | for b in range(1,6): 33 | f = os.path.join(ROOT, 'data_batch_%d' % (b, )) 34 | X, Y = load_CIFAR_batch(f) 35 | xs.append(X) 36 | ys.append(Y) 37 | Xtr = np.concatenate(xs) 38 | Ytr = np.concatenate(ys) 39 | del X, Y 40 | Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch')) 41 | return Xtr, Ytr, Xte, Yte 42 | 43 | 44 | def get_CIFAR10_data(num_training=49000, num_validation=1000, num_test=1000, 45 | subtract_mean=True): 46 | """ 47 | Load the CIFAR-10 dataset from disk and perform preprocessing to prepare 48 | it for classifiers. These are the same steps as we used for the SVM, but 49 | condensed to a single function. 50 | """ 51 | # Load the raw CIFAR-10 data 52 | cifar10_dir = 'DSVC/datasets/cifar-10-batches-py' 53 | X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir) 54 | 55 | # Subsample the data 56 | mask = list(range(num_training, num_training + num_validation)) 57 | X_val = X_train[mask] 58 | y_val = y_train[mask] 59 | mask = list(range(num_training)) 60 | X_train = X_train[mask] 61 | y_train = y_train[mask] 62 | mask = list(range(num_test)) 63 | X_test = X_test[mask] 64 | y_test = y_test[mask] 65 | 66 | # Normalize the data: subtract the mean image 67 | if subtract_mean: 68 | mean_image = np.mean(X_train, axis=0) 69 | X_train -= mean_image 70 | X_val -= mean_image 71 | X_test -= mean_image 72 | 73 | # Transpose so that channels come first 74 | X_train = X_train.transpose(0, 3, 1, 2).copy() 75 | X_val = X_val.transpose(0, 3, 1, 2).copy() 76 | X_test = X_test.transpose(0, 3, 1, 2).copy() 77 | 78 | # Package data into a dictionary 79 | return { 80 | 'X_train': X_train, 'y_train': y_train, 81 | 'X_val': X_val, 'y_val': y_val, 82 | 'X_test': X_test, 'y_test': y_test, 83 | } 84 | 85 | 86 | def load_tiny_imagenet(path, dtype=np.float32, subtract_mean=True): 87 | """ 88 | Load TinyImageNet. Each of TinyImageNet-100-A, TinyImageNet-100-B, and 89 | TinyImageNet-200 have the same directory structure, so this can be used 90 | to load any of them. 91 | 92 | Inputs: 93 | - path: String giving path to the directory to load. 94 | - dtype: numpy datatype used to load the data. 95 | - subtract_mean: Whether to subtract the mean training image. 96 | 97 | Returns: A dictionary with the following entries: 98 | - class_names: A list where class_names[i] is a list of strings giving the 99 | WordNet names for class i in the loaded dataset. 100 | - X_train: (N_tr, 3, 64, 64) array of training images 101 | - y_train: (N_tr,) array of training labels 102 | - X_val: (N_val, 3, 64, 64) array of validation images 103 | - y_val: (N_val,) array of validation labels 104 | - X_test: (N_test, 3, 64, 64) array of testing images. 105 | - y_test: (N_test,) array of test labels; if test labels are not available 106 | (such as in student code) then y_test will be None. 107 | - mean_image: (3, 64, 64) array giving mean training image 108 | """ 109 | # First load wnids 110 | with open(os.path.join(path, 'wnids.txt'), 'r') as f: 111 | wnids = [x.strip() for x in f] 112 | 113 | # Map wnids to integer labels 114 | wnid_to_label = {wnid: i for i, wnid in enumerate(wnids)} 115 | 116 | # Use words.txt to get names for each class 117 | with open(os.path.join(path, 'words.txt'), 'r') as f: 118 | wnid_to_words = dict(line.split('\t') for line in f) 119 | for wnid, words in wnid_to_words.items(): 120 | wnid_to_words[wnid] = [w.strip() for w in words.split(',')] 121 | class_names = [wnid_to_words[wnid] for wnid in wnids] 122 | 123 | # Next load training data. 124 | X_train = [] 125 | y_train = [] 126 | for i, wnid in enumerate(wnids): 127 | if (i + 1) % 20 == 0: 128 | print('loading training data for synset %d / %d' 129 | % (i + 1, len(wnids))) 130 | # To figure out the filenames we need to open the boxes file 131 | boxes_file = os.path.join(path, 'train', wnid, '%s_boxes.txt' % wnid) 132 | with open(boxes_file, 'r') as f: 133 | filenames = [x.split('\t')[0] for x in f] 134 | num_images = len(filenames) 135 | 136 | X_train_block = np.zeros((num_images, 3, 64, 64), dtype=dtype) 137 | y_train_block = wnid_to_label[wnid] * \ 138 | np.ones(num_images, dtype=np.int64) 139 | for j, img_file in enumerate(filenames): 140 | img_file = os.path.join(path, 'train', wnid, 'images', img_file) 141 | img = imread(img_file) 142 | if img.ndim == 2: 143 | ## grayscale file 144 | img.shape = (64, 64, 1) 145 | X_train_block[j] = img.transpose(2, 0, 1) 146 | X_train.append(X_train_block) 147 | y_train.append(y_train_block) 148 | 149 | # We need to concatenate all training data 150 | X_train = np.concatenate(X_train, axis=0) 151 | y_train = np.concatenate(y_train, axis=0) 152 | 153 | # Next load validation data 154 | with open(os.path.join(path, 'val', 'val_annotations.txt'), 'r') as f: 155 | img_files = [] 156 | val_wnids = [] 157 | for line in f: 158 | img_file, wnid = line.split('\t')[:2] 159 | img_files.append(img_file) 160 | val_wnids.append(wnid) 161 | num_val = len(img_files) 162 | y_val = np.array([wnid_to_label[wnid] for wnid in val_wnids]) 163 | X_val = np.zeros((num_val, 3, 64, 64), dtype=dtype) 164 | for i, img_file in enumerate(img_files): 165 | img_file = os.path.join(path, 'val', 'images', img_file) 166 | img = imread(img_file) 167 | if img.ndim == 2: 168 | img.shape = (64, 64, 1) 169 | X_val[i] = img.transpose(2, 0, 1) 170 | 171 | # Next load test images 172 | # Students won't have test labels, so we need to iterate over files in the 173 | # images directory. 174 | img_files = os.listdir(os.path.join(path, 'test', 'images')) 175 | X_test = np.zeros((len(img_files), 3, 64, 64), dtype=dtype) 176 | for i, img_file in enumerate(img_files): 177 | img_file = os.path.join(path, 'test', 'images', img_file) 178 | img = imread(img_file) 179 | if img.ndim == 2: 180 | img.shape = (64, 64, 1) 181 | X_test[i] = img.transpose(2, 0, 1) 182 | 183 | y_test = None 184 | y_test_file = os.path.join(path, 'test', 'test_annotations.txt') 185 | if os.path.isfile(y_test_file): 186 | with open(y_test_file, 'r') as f: 187 | img_file_to_wnid = {} 188 | for line in f: 189 | line = line.split('\t') 190 | img_file_to_wnid[line[0]] = line[1] 191 | y_test = [wnid_to_label[img_file_to_wnid[img_file]] 192 | for img_file in img_files] 193 | y_test = np.array(y_test) 194 | 195 | mean_image = X_train.mean(axis=0) 196 | if subtract_mean: 197 | X_train -= mean_image[None] 198 | X_val -= mean_image[None] 199 | X_test -= mean_image[None] 200 | 201 | return { 202 | 'class_names': class_names, 203 | 'X_train': X_train, 204 | 'y_train': y_train, 205 | 'X_val': X_val, 206 | 'y_val': y_val, 207 | 'X_test': X_test, 208 | 'y_test': y_test, 209 | 'class_names': class_names, 210 | 'mean_image': mean_image, 211 | } 212 | 213 | 214 | def load_models(models_dir): 215 | """ 216 | Load saved models from disk. This will attempt to unpickle all files in a 217 | directory; any files that give errors on unpickling (such as README.txt) 218 | will be skipped. 219 | 220 | Inputs: 221 | - models_dir: String giving the path to a directory containing model files. 222 | Each model file is a pickled dictionary with a 'model' field. 223 | 224 | Returns: 225 | A dictionary mapping model file names to models. 226 | """ 227 | models = {} 228 | for model_file in os.listdir(models_dir): 229 | with open(os.path.join(models_dir, model_file), 'rb') as f: 230 | try: 231 | models[model_file] = load_pickle(f)['model'] 232 | except pickle.UnpicklingError: 233 | continue 234 | return models 235 | 236 | 237 | def load_imagenet_val(num=None): 238 | """Load a handful of validation images from ImageNet. 239 | 240 | Inputs: 241 | - num: Number of images to load (max of 25) 242 | 243 | Returns: 244 | - X: numpy array with shape [num, 224, 224, 3] 245 | - y: numpy array of integer image labels, shape [num] 246 | - class_names: dict mapping integer label to class name 247 | """ 248 | imagenet_fn = 'DSVC/datasets/imagenet_val_25.npz' 249 | if not os.path.isfile(imagenet_fn): 250 | print('file %s not found' % imagenet_fn) 251 | print('Run the following:') 252 | print('cd DSVC/datasets') 253 | print('bash get_imagenet_val.sh') 254 | assert False, 'Need to download imagenet_val_25.npz' 255 | f = np.load(imagenet_fn) 256 | X = f['X'] 257 | y = f['y'] 258 | class_names = f['label_map'].item() 259 | if num is not None: 260 | X = X[:num] 261 | y = y[:num] 262 | return X, y, class_names 263 | -------------------------------------------------------------------------------- /assignment3/classwork/logistic_regression.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Logistic Regression exercise" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import random\n", 17 | "import numpy as np\n", 18 | "import matplotlib.pyplot as plt\n", 19 | "import pandas as pd\n", 20 | "\n", 21 | "%matplotlib inline\n", 22 | "plt.rcParams['figure.figsize'] = (15., 12.) # set default size of plots\n", 23 | "plt.rcParams['image.interpolation'] = 'nearest'\n", 24 | "plt.rcParams['image.cmap'] = 'gray'\n", 25 | "\n", 26 | "# Some more magic so that the notebook will reload external python modules;\n", 27 | "%load_ext autoreload\n", 28 | "%autoreload 2" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "# Read the data for you\n", 38 | "data = pd.read_csv('./DSVC/datasets/MNIST.csv',header=0).values # change the path by yourself\n", 39 | "imgs = data[0::,1::]\n", 40 | "labels = data[::,0]" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": null, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "# Visualize some examples from the dataset.\n", 50 | "# We show a few examples of training images from each class.\n", 51 | "classes = range(10)\n", 52 | "num_classes = len(classes)\n", 53 | "samples_per_class = 7\n", 54 | "for y, cls in enumerate(classes):\n", 55 | " idxs = np.flatnonzero(labels == y)\n", 56 | " idxs = np.random.choice(idxs, samples_per_class, replace=False)\n", 57 | " for i, idx in enumerate(idxs):\n", 58 | " plt_idx = i * num_classes + y + 1\n", 59 | " plt.subplot(samples_per_class, num_classes, plt_idx)\n", 60 | " plt.imshow(imgs[idx].reshape(28,28).astype('uint8'))\n", 61 | " plt.axis('off')\n", 62 | " if i == 0:\n", 63 | " plt.title(cls)\n", 64 | "plt.show()" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "metadata": {}, 70 | "source": [ 71 | "# Binary classification\n", 72 | "We use the Logistic Regression to classification handwritten digits wheather it's zero or not. If the handwritten digits is '0' , then the label is 0, otherwise, the label is 1." 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": null, 78 | "metadata": {}, 79 | "outputs": [], 80 | "source": [ 81 | "# transform the labels to binary\n", 82 | "for i in xrange(len(labels)):\n", 83 | " if labels[i] != 0:\n", 84 | " labels[i] = 1\n", 85 | " \n", 86 | "# 2/3 training set\n", 87 | "# 1/3 test set\n", 88 | "split_index = len(labels) * 2 / 3\n", 89 | "X_train = imgs[:split_index]\n", 90 | "y_train = labels[:split_index]\n", 91 | "X_test = imgs[split_index:]\n", 92 | "y_test = labels[split_index:]\n", 93 | "\n", 94 | "X_train = np.hstack([X_train, np.ones((X_train.shape[0], 1))])\n", 95 | "X_test = np.hstack([X_test, np.ones((X_test.shape[0], 1))])\n", 96 | "\n", 97 | "print X_train.shape\n", 98 | "print X_test.shape" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": null, 104 | "metadata": { 105 | "collapsed": true 106 | }, 107 | "outputs": [], 108 | "source": [ 109 | "X_train_feats = None # choose and extract features\n", 110 | "X_test_feats = None # choose and extract features" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": null, 116 | "metadata": {}, 117 | "outputs": [], 118 | "source": [ 119 | "# Visualize some examples from the dataset.\n", 120 | "# We show a few examples of training images from each class.\n", 121 | "classes = range(2)\n", 122 | "num_classes = len(classes)\n", 123 | "samples_per_class = 7\n", 124 | "for y, cls in enumerate(classes):\n", 125 | " idxs = np.flatnonzero(labels == y)\n", 126 | " idxs = np.random.choice(idxs, samples_per_class, replace=False)\n", 127 | " for i, idx in enumerate(idxs):\n", 128 | " plt_idx = i * num_classes + y + 1\n", 129 | " plt.subplot(samples_per_class, num_classes, plt_idx)\n", 130 | " plt.imshow(imgs[idx].reshape(28,28).astype('uint8'))\n", 131 | " plt.axis('off')\n", 132 | " if i == 0:\n", 133 | " plt.title(cls)\n", 134 | "plt.show()" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": null, 140 | "metadata": {}, 141 | "outputs": [], 142 | "source": [ 143 | "from DSVC.classifiers import LogisticRegression\n", 144 | "\n", 145 | "# Start training. \n", 146 | "# You should open DSVC/classifiers/logistic_regression.py and implement the function.\n", 147 | "# Then run this cell.\n", 148 | "\n", 149 | "classifier = LogisticRegression()\n", 150 | "loss_history = classifier.train(\n", 151 | " X_train_feats, \n", 152 | " y_train,\n", 153 | " learning_rate = 1e-3,\n", 154 | " num_iters = 500,\n", 155 | " batch_size = 64,\n", 156 | ")" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": null, 162 | "metadata": {}, 163 | "outputs": [], 164 | "source": [ 165 | "plt.plot(loss_history)" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": null, 171 | "metadata": { 172 | "collapsed": true 173 | }, 174 | "outputs": [], 175 | "source": [ 176 | "y_test_pred = classifier.predict(X_test_feats)\n", 177 | "print \"The accuracy socre is \", np.mean(y_test == y_test_pred)" 178 | ] 179 | }, 180 | { 181 | "cell_type": "markdown", 182 | "metadata": {}, 183 | "source": [ 184 | "you should get the accuracy higher than 96%.\n", 185 | "\n", 186 | "----\n", 187 | "\n", 188 | "### F1-Measure\n", 189 | "Notice that, if our model always output '1', totally ignoring the input X, we can get a accuracy 90%.So, in this assignment, accuracy is not efficient enough. \n", 190 | "\n", 191 | "We will use F1-Measure to evaluate our model.\n", 192 | "\n", 193 | "You may need this:\n", 194 | "[F1-Measure](https://baike.baidu.com/item/f-measure/913107?fr=aladdin)" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": null, 200 | "metadata": { 201 | "collapsed": true 202 | }, 203 | "outputs": [], 204 | "source": [ 205 | "# Calculate the precision(准确率), recall(召回率) and F1\n", 206 | "# important： We should consider label '0' as 'positive' here. \n", 207 | "# That means 'True positive' ==> '(y_test == 0) and (y_test_pred == 0)'\n", 208 | "\n", 209 | "#######Your code here########\n", 210 | "\n", 211 | "print precision\n", 212 | "print recall\n", 213 | "print 'F1:', precision*recall*2/(precision+recall)" 214 | ] 215 | }, 216 | { 217 | "cell_type": "markdown", 218 | "metadata": {}, 219 | "source": [ 220 | "you should get the F1 higher than 85%." 221 | ] 222 | }, 223 | { 224 | "cell_type": "markdown", 225 | "metadata": {}, 226 | "source": [ 227 | "# Multiclass classification\n", 228 | "\n", 229 | "Now, we use the Logistic Regression to classification handwritten digits. There are 10 class, from '0' to '9'.\n", 230 | "\n", 231 | "\n", 232 | "Hint: The method \"one vs all\" may helpful. [Here is the introduction to \"one vs all\"](https://msdn.microsoft.com/library/en-us/Dn905887.aspx). " 233 | ] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": null, 238 | "metadata": {}, 239 | "outputs": [], 240 | "source": [ 241 | "# Read the data for you\n", 242 | "data = pd.read_csv('./DSVC/datasets/MNIST.csv',header=0).values # change the path by yourself\n", 243 | "imgs = data[0::,1::]\n", 244 | "labels = data[::,0]\n", 245 | " \n", 246 | "# 2/3 training set\n", 247 | "# 1/3 test set\n", 248 | "split_index = len(labels) * 2 / 3\n", 249 | "X_train = imgs[:split_index]\n", 250 | "y_train = labels[:split_index]\n", 251 | "X_test = imgs[split_index:]\n", 252 | "y_test = labels[split_index:]\n", 253 | "\n", 254 | "X_train = np.hstack([X_train, np.ones((X_train.shape[0], 1))])\n", 255 | "X_test = np.hstack([X_test, np.ones((X_test.shape[0], 1))])\n", 256 | "\n", 257 | "print X_train.shape\n", 258 | "print X_test.shape" 259 | ] 260 | }, 261 | { 262 | "cell_type": "code", 263 | "execution_count": null, 264 | "metadata": { 265 | "collapsed": true 266 | }, 267 | "outputs": [], 268 | "source": [ 269 | "X_train_feats = None # choose and extract features\n", 270 | "X_test_feats = None # choose and extract features" 271 | ] 272 | }, 273 | { 274 | "cell_type": "code", 275 | "execution_count": null, 276 | "metadata": { 277 | "collapsed": true 278 | }, 279 | "outputs": [], 280 | "source": [ 281 | "# Start training. \n", 282 | "# You should update your code in DSVC/classifiers/logistic_regression.py .\n", 283 | "# Then run this cell.\n", 284 | "\n", 285 | "classifier = LogisticRegression()\n", 286 | "classifier.one_vs_all(\n", 287 | " X_train_feats, \n", 288 | " y_train,\n", 289 | " learning_rate = 1e-3,\n", 290 | " num_iters = 500,\n", 291 | " batch_size = 64,\n", 292 | ")" 293 | ] 294 | }, 295 | { 296 | "cell_type": "code", 297 | "execution_count": null, 298 | "metadata": {}, 299 | "outputs": [], 300 | "source": [ 301 | "# you may change your code in function `predict`\n", 302 | "y_test_pred = classifier.predict(X_test_feats)\n", 303 | "print \"The accruacy socre is \", np.mean(y_test == y_test_pred)" 304 | ] 305 | } 306 | ], 307 | "metadata": { 308 | "kernelspec": { 309 | "display_name": "Python 2", 310 | "language": "python", 311 | "name": "python2" 312 | }, 313 | "language_info": { 314 | "codemirror_mode": { 315 | "name": "ipython", 316 | "version": 2 317 | }, 318 | "file_extension": ".py", 319 | "mimetype": "text/x-python", 320 | "name": "python", 321 | "nbconvert_exporter": "python", 322 | "pygments_lexer": "ipython2", 323 | "version": "2.7.10" 324 | } 325 | }, 326 | "nbformat": 4, 327 | "nbformat_minor": 2 328 | } 329 | -------------------------------------------------------------------------------- /assignment7/classwork/DSVC/fast_layers.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import numpy as np 3 | try: 4 | from DSVC.im2col_cython import col2im_cython, im2col_cython 5 | from DSVC.im2col_cython import col2im_6d_cython 6 | except ImportError: 7 | print('run the following from the DSVC directory and try again:') 8 | print('python setup.py build_ext --inplace') 9 | print('You may also need to restart your iPython kernel') 10 | 11 | from DSVC.im2col import * 12 | 13 | 14 | def conv_forward_im2col(x, w, b, conv_param): 15 | """ 16 | A fast implementation of the forward pass for a convolutional layer 17 | based on im2col and col2im. 18 | """ 19 | N, C, H, W = x.shape 20 | num_filters, _, filter_height, filter_width = w.shape 21 | stride, pad = conv_param['stride'], conv_param['pad'] 22 | 23 | # Check dimensions 24 | assert (W + 2 * pad - filter_width) % stride == 0, 'width does not work' 25 | assert (H + 2 * pad - filter_height) % stride == 0, 'height does not work' 26 | 27 | # Create output 28 | out_height = (H + 2 * pad - filter_height) // stride + 1 29 | out_width = (W + 2 * pad - filter_width) // stride + 1 30 | out = np.zeros((N, num_filters, out_height, out_width), dtype=x.dtype) 31 | 32 | # x_cols = im2col_indices(x, w.shape[2], w.shape[3], pad, stride) 33 | x_cols = im2col_cython(x, w.shape[2], w.shape[3], pad, stride) 34 | res = w.reshape((w.shape[0], -1)).dot(x_cols) + b.reshape(-1, 1) 35 | 36 | out = res.reshape(w.shape[0], out.shape[2], out.shape[3], x.shape[0]) 37 | out = out.transpose(3, 0, 1, 2) 38 | 39 | cache = (x, w, b, conv_param, x_cols) 40 | return out, cache 41 | 42 | 43 | def conv_forward_strides(x, w, b, conv_param): 44 | N, C, H, W = x.shape 45 | F, _, HH, WW = w.shape 46 | stride, pad = conv_param['stride'], conv_param['pad'] 47 | 48 | # Check dimensions 49 | #assert (W + 2 * pad - WW) % stride == 0, 'width does not work' 50 | #assert (H + 2 * pad - HH) % stride == 0, 'height does not work' 51 | 52 | # Pad the input 53 | p = pad 54 | x_padded = np.pad(x, ((0, 0), (0, 0), (p, p), (p, p)), mode='constant') 55 | 56 | # Figure out output dimensions 57 | H += 2 * pad 58 | W += 2 * pad 59 | out_h = (H - HH) // stride + 1 60 | out_w = (W - WW) // stride + 1 61 | 62 | # Perform an im2col operation by picking clever strides 63 | shape = (C, HH, WW, N, out_h, out_w) 64 | strides = (H * W, W, 1, C * H * W, stride * W, stride) 65 | strides = x.itemsize * np.array(strides) 66 | x_stride = np.lib.stride_tricks.as_strided(x_padded, 67 | shape=shape, strides=strides) 68 | x_cols = np.ascontiguousarray(x_stride) 69 | x_cols.shape = (C * HH * WW, N * out_h * out_w) 70 | 71 | # Now all our convolutions are a big matrix multiply 72 | res = w.reshape(F, -1).dot(x_cols) + b.reshape(-1, 1) 73 | 74 | # Reshape the output 75 | res.shape = (F, N, out_h, out_w) 76 | out = res.transpose(1, 0, 2, 3) 77 | 78 | # Be nice and return a contiguous array 79 | # The old version of conv_forward_fast doesn't do this, so for a fair 80 | # comparison we won't either 81 | out = np.ascontiguousarray(out) 82 | 83 | cache = (x, w, b, conv_param, x_cols) 84 | return out, cache 85 | 86 | 87 | def conv_backward_strides(dout, cache): 88 | x, w, b, conv_param, x_cols = cache 89 | stride, pad = conv_param['stride'], conv_param['pad'] 90 | 91 | N, C, H, W = x.shape 92 | F, _, HH, WW = w.shape 93 | _, _, out_h, out_w = dout.shape 94 | 95 | db = np.sum(dout, axis=(0, 2, 3)) 96 | 97 | dout_reshaped = dout.transpose(1, 0, 2, 3).reshape(F, -1) 98 | dw = dout_reshaped.dot(x_cols.T).reshape(w.shape) 99 | 100 | dx_cols = w.reshape(F, -1).T.dot(dout_reshaped) 101 | dx_cols.shape = (C, HH, WW, N, out_h, out_w) 102 | dx = col2im_6d_cython(dx_cols, N, C, H, W, HH, WW, pad, stride) 103 | 104 | return dx, dw, db 105 | 106 | 107 | def conv_backward_im2col(dout, cache): 108 | """ 109 | A fast implementation of the backward pass for a convolutional layer 110 | based on im2col and col2im. 111 | """ 112 | x, w, b, conv_param, x_cols = cache 113 | stride, pad = conv_param['stride'], conv_param['pad'] 114 | 115 | db = np.sum(dout, axis=(0, 2, 3)) 116 | 117 | num_filters, _, filter_height, filter_width = w.shape 118 | dout_reshaped = dout.transpose(1, 2, 3, 0).reshape(num_filters, -1) 119 | dw = dout_reshaped.dot(x_cols.T).reshape(w.shape) 120 | 121 | dx_cols = w.reshape(num_filters, -1).T.dot(dout_reshaped) 122 | # dx = col2im_indices(dx_cols, x.shape, filter_height, filter_width, pad, stride) 123 | dx = col2im_cython(dx_cols, x.shape[0], x.shape[1], x.shape[2], x.shape[3], 124 | filter_height, filter_width, pad, stride) 125 | 126 | return dx, dw, db 127 | 128 | 129 | conv_forward_fast = conv_forward_strides 130 | conv_backward_fast = conv_backward_strides 131 | 132 | 133 | def max_pool_forward_fast(x, pool_param): 134 | """ 135 | A fast implementation of the forward pass for a max pooling layer. 136 | 137 | This chooses between the reshape method and the im2col method. If the pooling 138 | regions are square and tile the input image, then we can use the reshape 139 | method which is very fast. Otherwise we fall back on the im2col method, which 140 | is not much faster than the naive method. 141 | """ 142 | N, C, H, W = x.shape 143 | pool_height, pool_width = pool_param['pool_height'], pool_param['pool_width'] 144 | stride = pool_param['stride'] 145 | 146 | same_size = pool_height == pool_width == stride 147 | tiles = H % pool_height == 0 and W % pool_width == 0 148 | if same_size and tiles: 149 | out, reshape_cache = max_pool_forward_reshape(x, pool_param) 150 | cache = ('reshape', reshape_cache) 151 | else: 152 | out, im2col_cache = max_pool_forward_im2col(x, pool_param) 153 | cache = ('im2col', im2col_cache) 154 | return out, cache 155 | 156 | 157 | def max_pool_backward_fast(dout, cache): 158 | """ 159 | A fast implementation of the backward pass for a max pooling layer. 160 | 161 | This switches between the reshape method an the im2col method depending on 162 | which method was used to generate the cache. 163 | """ 164 | method, real_cache = cache 165 | if method == 'reshape': 166 | return max_pool_backward_reshape(dout, real_cache) 167 | elif method == 'im2col': 168 | return max_pool_backward_im2col(dout, real_cache) 169 | else: 170 | raise ValueError('Unrecognized method "%s"' % method) 171 | 172 | 173 | def max_pool_forward_reshape(x, pool_param): 174 | """ 175 | A fast implementation of the forward pass for the max pooling layer that uses 176 | some clever reshaping. 177 | 178 | This can only be used for square pooling regions that tile the input. 179 | """ 180 | N, C, H, W = x.shape 181 | pool_height, pool_width = pool_param['pool_height'], pool_param['pool_width'] 182 | stride = pool_param['stride'] 183 | assert pool_height == pool_width == stride, 'Invalid pool params' 184 | assert H % pool_height == 0 185 | assert W % pool_height == 0 186 | x_reshaped = x.reshape(N, C, H // pool_height, pool_height, 187 | W // pool_width, pool_width) 188 | out = x_reshaped.max(axis=3).max(axis=4) 189 | 190 | cache = (x, x_reshaped, out) 191 | return out, cache 192 | 193 | 194 | def max_pool_backward_reshape(dout, cache): 195 | """ 196 | A fast implementation of the backward pass for the max pooling layer that 197 | uses some clever broadcasting and reshaping. 198 | 199 | This can only be used if the forward pass was computed using 200 | max_pool_forward_reshape. 201 | 202 | NOTE: If there are multiple argmaxes, this method will assign gradient to 203 | ALL argmax elements of the input rather than picking one. In this case the 204 | gradient will actually be incorrect. However this is unlikely to occur in 205 | practice, so it shouldn't matter much. One possible solution is to split the 206 | upstream gradient equally among all argmax elements; this should result in a 207 | valid subgradient. You can make this happen by uncommenting the line below; 208 | however this results in a significant performance penalty (about 40% slower) 209 | and is unlikely to matter in practice so we don't do it. 210 | """ 211 | x, x_reshaped, out = cache 212 | 213 | dx_reshaped = np.zeros_like(x_reshaped) 214 | out_newaxis = out[:, :, :, np.newaxis, :, np.newaxis] 215 | mask = (x_reshaped == out_newaxis) 216 | dout_newaxis = dout[:, :, :, np.newaxis, :, np.newaxis] 217 | dout_broadcast, _ = np.broadcast_arrays(dout_newaxis, dx_reshaped) 218 | dx_reshaped[mask] = dout_broadcast[mask] 219 | dx_reshaped /= np.sum(mask, axis=(3, 5), keepdims=True) 220 | dx = dx_reshaped.reshape(x.shape) 221 | 222 | return dx 223 | 224 | 225 | def max_pool_forward_im2col(x, pool_param): 226 | """ 227 | An implementation of the forward pass for max pooling based on im2col. 228 | 229 | This isn't much faster than the naive version, so it should be avoided if 230 | possible. 231 | """ 232 | N, C, H, W = x.shape 233 | pool_height, pool_width = pool_param['pool_height'], pool_param['pool_width'] 234 | stride = pool_param['stride'] 235 | 236 | assert (H - pool_height) % stride == 0, 'Invalid height' 237 | assert (W - pool_width) % stride == 0, 'Invalid width' 238 | 239 | out_height = (H - pool_height) // stride + 1 240 | out_width = (W - pool_width) // stride + 1 241 | 242 | x_split = x.reshape(N * C, 1, H, W) 243 | x_cols = im2col(x_split, pool_height, pool_width, padding=0, stride=stride) 244 | x_cols_argmax = np.argmax(x_cols, axis=0) 245 | x_cols_max = x_cols[x_cols_argmax, np.arange(x_cols.shape[1])] 246 | out = x_cols_max.reshape(out_height, out_width, N, C).transpose(2, 3, 0, 1) 247 | 248 | cache = (x, x_cols, x_cols_argmax, pool_param) 249 | return out, cache 250 | 251 | 252 | def max_pool_backward_im2col(dout, cache): 253 | """ 254 | An implementation of the backward pass for max pooling based on im2col. 255 | 256 | This isn't much faster than the naive version, so it should be avoided if 257 | possible. 258 | """ 259 | x, x_cols, x_cols_argmax, pool_param = cache 260 | N, C, H, W = x.shape 261 | pool_height, pool_width = pool_param['pool_height'], pool_param['pool_width'] 262 | stride = pool_param['stride'] 263 | 264 | dout_reshaped = dout.transpose(2, 3, 0, 1).flatten() 265 | dx_cols = np.zeros_like(x_cols) 266 | dx_cols[x_cols_argmax, np.arange(dx_cols.shape[1])] = dout_reshaped 267 | dx = col2im_indices(dx_cols, (N * C, 1, H, W), pool_height, pool_width, 268 | padding=0, stride=stride) 269 | dx = dx.reshape(x.shape) 270 | 271 | return dx 272 | -------------------------------------------------------------------------------- /assignment7/classwork/Dropout.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat_minor": 0, 3 | "nbformat": 4, 4 | "cells": [ 5 | { 6 | "source": [ 7 | "# Dropout\n", 8 | "Dropout [1] is a technique for regularizing neural networks by randomly setting some features to zero during the forward pass. In this exercise you will implement a dropout layer and modify your fully-connected network to optionally use dropout.\n", 9 | "\n", 10 | "[1] Geoffrey E. Hinton et al, \"Improving neural networks by preventing co-adaptation of feature detectors\", arXiv 2012" 11 | ], 12 | "cell_type": "markdown", 13 | "metadata": { 14 | "editable": true, 15 | "deletable": true 16 | } 17 | }, 18 | { 19 | "execution_count": null, 20 | "cell_type": "code", 21 | "source": [ 22 | "# As usual, a bit of setup\n", 23 | "from __future__ import print_function\n", 24 | "import time\n", 25 | "import numpy as np\n", 26 | "import matplotlib.pyplot as plt\n", 27 | "from DSVC.classifiers.fc_net import *\n", 28 | "from DSVC.data_utils import get_CIFAR10_data\n", 29 | "from DSVC.gradient_check import eval_numerical_gradient, eval_numerical_gradient_array\n", 30 | "from DSVC.solver import Solver\n", 31 | "\n", 32 | "%matplotlib inline\n", 33 | "plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots\n", 34 | "plt.rcParams['image.interpolation'] = 'nearest'\n", 35 | "plt.rcParams['image.cmap'] = 'gray'\n", 36 | "\n", 37 | "# for auto-reloading external modules\n", 38 | "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n", 39 | "%load_ext autoreload\n", 40 | "%autoreload 2\n", 41 | "\n", 42 | "def rel_error(x, y):\n", 43 | " \"\"\" returns relative error \"\"\"\n", 44 | " return np.max(np.abs(x - y) / (np.maximum(1e-8, np.abs(x) + np.abs(y))))" 45 | ], 46 | "outputs": [], 47 | "metadata": { 48 | "collapsed": false, 49 | "editable": true, 50 | "deletable": true 51 | } 52 | }, 53 | { 54 | "execution_count": null, 55 | "cell_type": "code", 56 | "source": [ 57 | "# Load the (preprocessed) CIFAR10 data.\n", 58 | "\n", 59 | "data = get_CIFAR10_data()\n", 60 | "for k, v in data.items():\n", 61 | " print('%s: ' % k, v.shape)" 62 | ], 63 | "outputs": [], 64 | "metadata": { 65 | "collapsed": false, 66 | "editable": true, 67 | "deletable": true 68 | } 69 | }, 70 | { 71 | "source": [ 72 | "# Dropout forward pass\n", 73 | "In the file `DSVC/layers.py`, implement the forward pass for dropout. Since dropout behaves differently during training and testing, make sure to implement the operation for both modes.\n", 74 | "\n", 75 | "Once you have done so, run the cell below to test your implementation." 76 | ], 77 | "cell_type": "markdown", 78 | "metadata": { 79 | "editable": true, 80 | "deletable": true 81 | } 82 | }, 83 | { 84 | "execution_count": null, 85 | "cell_type": "code", 86 | "source": [ 87 | "np.random.seed(231)\n", 88 | "x = np.random.randn(500, 500) + 10\n", 89 | "\n", 90 | "for p in [0.3, 0.6, 0.75]:\n", 91 | " out, _ = dropout_forward(x, {'mode': 'train', 'p': p})\n", 92 | " out_test, _ = dropout_forward(x, {'mode': 'test', 'p': p})\n", 93 | "\n", 94 | " print('Running tests with p = ', p)\n", 95 | " print('Mean of input: ', x.mean())\n", 96 | " print('Mean of train-time output: ', out.mean())\n", 97 | " print('Mean of test-time output: ', out_test.mean())\n", 98 | " print('Fraction of train-time output set to zero: ', (out == 0).mean())\n", 99 | " print('Fraction of test-time output set to zero: ', (out_test == 0).mean())\n", 100 | " print()" 101 | ], 102 | "outputs": [], 103 | "metadata": { 104 | "collapsed": false, 105 | "editable": true, 106 | "deletable": true 107 | } 108 | }, 109 | { 110 | "source": [ 111 | "# Dropout backward pass\n", 112 | "In the file `DSVC/layers.py`, implement the backward pass for dropout. After doing so, run the following cell to numerically gradient-check your implementation." 113 | ], 114 | "cell_type": "markdown", 115 | "metadata": { 116 | "editable": true, 117 | "deletable": true 118 | } 119 | }, 120 | { 121 | "execution_count": null, 122 | "cell_type": "code", 123 | "source": [ 124 | "np.random.seed(231)\n", 125 | "x = np.random.randn(10, 10) + 10\n", 126 | "dout = np.random.randn(*x.shape)\n", 127 | "\n", 128 | "dropout_param = {'mode': 'train', 'p': 0.8, 'seed': 123}\n", 129 | "out, cache = dropout_forward(x, dropout_param)\n", 130 | "dx = dropout_backward(dout, cache)\n", 131 | "dx_num = eval_numerical_gradient_array(lambda xx: dropout_forward(xx, dropout_param)[0], x, dout)\n", 132 | "\n", 133 | "print('dx relative error: ', rel_error(dx, dx_num))" 134 | ], 135 | "outputs": [], 136 | "metadata": { 137 | "collapsed": false, 138 | "editable": true, 139 | "deletable": true 140 | } 141 | }, 142 | { 143 | "source": [ 144 | "# Fully-connected nets with Dropout\n", 145 | "In the file `DSVC/classifiers/fc_net.py`, modify your implementation to use dropout. Specificially, if the constructor the the net receives a nonzero value for the `dropout` parameter, then the net should add dropout immediately after every ReLU nonlinearity. After doing so, run the following to numerically gradient-check your implementation." 146 | ], 147 | "cell_type": "markdown", 148 | "metadata": { 149 | "editable": true, 150 | "deletable": true 151 | } 152 | }, 153 | { 154 | "execution_count": null, 155 | "cell_type": "code", 156 | "source": [ 157 | "np.random.seed(231)\n", 158 | "N, D, H1, H2, C = 2, 15, 20, 30, 10\n", 159 | "X = np.random.randn(N, D)\n", 160 | "y = np.random.randint(C, size=(N,))\n", 161 | "\n", 162 | "for dropout in [0, 0.25, 0.5]:\n", 163 | " print('Running check with dropout = ', dropout)\n", 164 | " model = FullyConnectedNet([H1, H2], input_dim=D, num_classes=C,\n", 165 | " weight_scale=5e-2, dtype=np.float64,\n", 166 | " dropout=dropout, seed=123)\n", 167 | "\n", 168 | " loss, grads = model.loss(X, y)\n", 169 | " print('Initial loss: ', loss)\n", 170 | "\n", 171 | " for name in sorted(grads):\n", 172 | " f = lambda _: model.loss(X, y)[0]\n", 173 | " grad_num = eval_numerical_gradient(f, model.params[name], verbose=False, h=1e-5)\n", 174 | " print('%s relative error: %.2e' % (name, rel_error(grad_num, grads[name])))\n", 175 | " print()" 176 | ], 177 | "outputs": [], 178 | "metadata": { 179 | "collapsed": false, 180 | "editable": true, 181 | "deletable": true 182 | } 183 | }, 184 | { 185 | "source": [ 186 | "# Regularization experiment\n", 187 | "As an experiment, we will train a pair of two-layer networks on 500 training examples: one will use no dropout, and one will use a dropout probability of 0.75. We will then visualize the training and validation accuracies of the two networks over time." 188 | ], 189 | "cell_type": "markdown", 190 | "metadata": { 191 | "editable": true, 192 | "deletable": true 193 | } 194 | }, 195 | { 196 | "execution_count": null, 197 | "cell_type": "code", 198 | "source": [ 199 | "# Train two identical nets, one with dropout and one without\n", 200 | "np.random.seed(231)\n", 201 | "num_train = 500\n", 202 | "small_data = {\n", 203 | " 'X_train': data['X_train'][:num_train],\n", 204 | " 'y_train': data['y_train'][:num_train],\n", 205 | " 'X_val': data['X_val'],\n", 206 | " 'y_val': data['y_val'],\n", 207 | "}\n", 208 | "\n", 209 | "solvers = {}\n", 210 | "dropout_choices = [0, 0.75]\n", 211 | "for dropout in dropout_choices:\n", 212 | " model = FullyConnectedNet([500], dropout=dropout)\n", 213 | " print(dropout)\n", 214 | "\n", 215 | " solver = Solver(model, small_data,\n", 216 | " num_epochs=25, batch_size=100,\n", 217 | " update_rule='adam',\n", 218 | " optim_config={\n", 219 | " 'learning_rate': 5e-4,\n", 220 | " },\n", 221 | " verbose=True, print_every=100)\n", 222 | " solver.train()\n", 223 | " solvers[dropout] = solver" 224 | ], 225 | "outputs": [], 226 | "metadata": { 227 | "scrolled": false, 228 | "collapsed": false, 229 | "editable": true, 230 | "deletable": true 231 | } 232 | }, 233 | { 234 | "execution_count": null, 235 | "cell_type": "code", 236 | "source": [ 237 | "# Plot train and validation accuracies of the two models\n", 238 | "\n", 239 | "train_accs = []\n", 240 | "val_accs = []\n", 241 | "for dropout in dropout_choices:\n", 242 | " solver = solvers[dropout]\n", 243 | " train_accs.append(solver.train_acc_history[-1])\n", 244 | " val_accs.append(solver.val_acc_history[-1])\n", 245 | "\n", 246 | "plt.subplot(3, 1, 1)\n", 247 | "for dropout in dropout_choices:\n", 248 | " plt.plot(solvers[dropout].train_acc_history, 'o', label='%.2f dropout' % dropout)\n", 249 | "plt.title('Train accuracy')\n", 250 | "plt.xlabel('Epoch')\n", 251 | "plt.ylabel('Accuracy')\n", 252 | "plt.legend(ncol=2, loc='lower right')\n", 253 | " \n", 254 | "plt.subplot(3, 1, 2)\n", 255 | "for dropout in dropout_choices:\n", 256 | " plt.plot(solvers[dropout].val_acc_history, 'o', label='%.2f dropout' % dropout)\n", 257 | "plt.title('Val accuracy')\n", 258 | "plt.xlabel('Epoch')\n", 259 | "plt.ylabel('Accuracy')\n", 260 | "plt.legend(ncol=2, loc='lower right')\n", 261 | "\n", 262 | "plt.gcf().set_size_inches(15, 15)\n", 263 | "plt.show()" 264 | ], 265 | "outputs": [], 266 | "metadata": { 267 | "collapsed": false, 268 | "editable": true, 269 | "deletable": true 270 | } 271 | }, 272 | { 273 | "source": [ 274 | "# Question\n", 275 | "Explain what you see in this experiment. What does it suggest about dropout?" 276 | ], 277 | "cell_type": "markdown", 278 | "metadata": { 279 | "editable": true, 280 | "deletable": true 281 | } 282 | }, 283 | { 284 | "source": [ 285 | "# Answer\n" 286 | ], 287 | "cell_type": "markdown", 288 | "metadata": { 289 | "editable": true, 290 | "deletable": true 291 | } 292 | } 293 | ], 294 | "metadata": { 295 | "kernelspec": { 296 | "display_name": "Python 2", 297 | "name": "python2", 298 | "language": "python" 299 | }, 300 | "language_info": { 301 | "mimetype": "text/x-python", 302 | "nbconvert_exporter": "python", 303 | "name": "python", 304 | "file_extension": ".py", 305 | "version": "2.7.12+", 306 | "pygments_lexer": "ipython2", 307 | "codemirror_mode": { 308 | "version": 2, 309 | "name": "ipython" 310 | } 311 | } 312 | } 313 | } -------------------------------------------------------------------------------- /assignment6/classwork/DSVC/classifiers/neural_net.py: -------------------------------------------------------------------------------- 1 | # from __future__ import print_function 2 | 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | # from past.builtins import xrange 6 | 7 | 8 | class TwoLayerNet(object): 9 | """ 10 | A two-layer fully-connected neural network. The net has an input dimension of 11 | N, a hidden layer dimension of H, and performs classification over C classes. 12 | We train the network with a softmax loss function and L2 regularization on the 13 | weight matrices. The network uses a ReLU nonlinearity after the first fully 14 | connected layer. 15 | 16 | In other words, the network has the following architecture: 17 | 18 | input - fully connected layer - ReLU - fully connected layer - softmax 19 | 20 | The outputs of the second fully-connected layer are the scores for each class. 21 | """ 22 | 23 | def __init__(self, input_size, hidden_size, output_size, std=1e-4): 24 | """ 25 | Initialize the model. Weights are initialized to small random values and 26 | biases are initialized to zero. Weights and biases are stored in the 27 | variable self.params, which is a dictionary with the following keys: 28 | 29 | W1: First layer weights; has shape (D, H) 30 | b1: First layer biases; has shape (H,) 31 | W2: Second layer weights; has shape (H, C) 32 | b2: Second layer biases; has shape (C,) 33 | 34 | Inputs: 35 | - input_size: The dimension D of the input data. 36 | - hidden_size: The number of neurons H in the hidden layer. 37 | - output_size: The number of classes C. 38 | """ 39 | self.params = {} 40 | self.params['W1'] = std * np.random.randn(input_size, hidden_size) 41 | self.params['b1'] = np.zeros(hidden_size) 42 | self.params['W2'] = std * np.random.randn(hidden_size, output_size) 43 | self.params['b2'] = np.zeros(output_size) 44 | 45 | def loss(self, X, y=None, reg=0.0): 46 | """ 47 | Compute the loss and gradients for a two layer fully connected neural 48 | network. 49 | 50 | Inputs: 51 | - X: Input data of shape (N, D). Each X[i] is a training sample. 52 | - y: Vector of training labels. y[i] is the label for X[i], and each y[i] is 53 | an integer in the range 0 <= y[i] < C. This parameter is optional; if it 54 | is not passed then we only return scores, and if it is passed then we 55 | instead return the loss and gradients. 56 | - reg: Regularization strength. 57 | 58 | Returns: 59 | If y is None, return a matrix scores of shape (N, C) where scores[i, c] is 60 | the score for class c on input X[i]. 61 | 62 | If y is not None, instead return a tuple of: 63 | - loss: Loss (data loss and regularization loss) for this batch of training 64 | samples. 65 | - grads: Dictionary mapping parameter names to gradients of those parameters 66 | with respect to the loss function; has the same keys as self.params. 67 | """ 68 | # Unpack variables from the params dictionary 69 | W1, b1 = self.params['W1'], self.params['b1'] 70 | W2, b2 = self.params['W2'], self.params['b2'] 71 | N, D = X.shape 72 | 73 | # Compute the forward pass 74 | scores = None 75 | ############################################################################# 76 | # TODO: Perform the forward pass, computing the class scores for the input. # 77 | # Store the result in the scores variable, which should be an array of # 78 | # shape (N, C). # 79 | ############################################################################# 80 | pass 81 | ############################################################################# 82 | # END OF YOUR CODE # 83 | ############################################################################# 84 | 85 | # If the targets are not given then jump out, we're done 86 | if y is None: 87 | return scores 88 | 89 | # Compute the loss 90 | loss = None 91 | ############################################################################# 92 | # TODO: Finish the forward pass, and compute the loss. This should include # 93 | # both the data loss and L2 regularization for W1 and W2. Store the result # 94 | # in the variable loss, which should be a scalar. Use the Softmax # 95 | # classifier loss. # 96 | ############################################################################# 97 | pass 98 | ############################################################################# 99 | # END OF YOUR CODE # 100 | ############################################################################# 101 | 102 | # Backward pass: compute gradients 103 | grads = {} 104 | ############################################################################# 105 | # TODO: Compute the backward pass, computing the derivatives of the weights # 106 | # and biases. Store the results in the grads dictionary. For example, # 107 | # grads['W1'] should store the gradient on W1, and be a matrix of same size # 108 | ############################################################################# 109 | pass 110 | ############################################################################# 111 | # END OF YOUR CODE # 112 | ############################################################################# 113 | 114 | return loss, grads 115 | 116 | def train(self, X, y, X_val, y_val, 117 | learning_rate=1e-3, learning_rate_decay=0.95, 118 | reg=5e-6, num_iters=100, 119 | batch_size=200, verbose=False): 120 | """ 121 | Train this neural network using stochastic gradient descent. 122 | 123 | Inputs: 124 | - X: A numpy array of shape (N, D) giving training data. 125 | - y: A numpy array f shape (N,) giving training labels; y[i] = c means that 126 | X[i] has label c, where 0 <= c < C. 127 | - X_val: A numpy array of shape (N_val, D) giving validation data. 128 | - y_val: A numpy array of shape (N_val,) giving validation labels. 129 | - learning_rate: Scalar giving learning rate for optimization. 130 | - learning_rate_decay: Scalar giving factor used to decay the learning rate 131 | after each epoch. 132 | - reg: Scalar giving regularization strength. 133 | - num_iters: Number of steps to take when optimizing. 134 | - batch_size: Number of training examples to use per step. 135 | - verbose: boolean; if true print progress during optimization. 136 | """ 137 | num_train = X.shape[0] 138 | iterations_per_epoch = max(num_train / batch_size, 1) 139 | 140 | # Use SGD to optimize the parameters in self.model 141 | loss_history = [] 142 | train_acc_history = [] 143 | val_acc_history = [] 144 | 145 | for it in range(num_iters): 146 | X_batch = None 147 | y_batch = None 148 | 149 | ######################################################################### 150 | # TODO: Create a random minibatch of training data and labels, storing # 151 | # them in X_batch and y_batch respectively. # 152 | ######################################################################### 153 | pass 154 | ######################################################################### 155 | # END OF YOUR CODE # 156 | ######################################################################### 157 | 158 | # Compute loss and gradients using the current minibatch 159 | loss, grads = self.loss(X_batch, y=y_batch, reg=reg) 160 | loss_history.append(loss) 161 | 162 | ######################################################################### 163 | # TODO: Use the gradients in the grads dictionary to update the # 164 | # parameters of the network (stored in the dictionary self.params) # 165 | # using stochastic gradient descent. You'll need to use the gradients # 166 | # stored in the grads dictionary defined above. # 167 | ######################################################################### 168 | pass 169 | ######################################################################### 170 | # END OF YOUR CODE # 171 | ######################################################################### 172 | 173 | if verbose and it % 100 == 0: 174 | print('iteration %d / %d: loss %f' % (it, num_iters, loss)) 175 | 176 | # Every epoch, check train and val accuracy and decay learning rate. 177 | if it % iterations_per_epoch == 0: 178 | # Check accuracy 179 | train_acc = (self.predict(X_batch) == y_batch).mean() 180 | val_acc = (self.predict(X_val) == y_val).mean() 181 | train_acc_history.append(train_acc) 182 | val_acc_history.append(val_acc) 183 | 184 | # Decay learning rate 185 | learning_rate *= learning_rate_decay 186 | 187 | return { 188 | 'loss_history': loss_history, 189 | 'train_acc_history': train_acc_history, 190 | 'val_acc_history': val_acc_history, 191 | } 192 | 193 | def predict(self, X): 194 | """ 195 | Use the trained weights of this two-layer network to predict labels for 196 | data points. For each data point we predict scores for each of the C 197 | classes, and assign each data point to the class with the highest score. 198 | 199 | Inputs: 200 | - X: A numpy array of shape (N, D) giving N D-dimensional data points to 201 | classify. 202 | 203 | Returns: 204 | - y_pred: A numpy array of shape (N,) giving predicted labels for each of 205 | the elements of X. For all i, y_pred[i] = c means that X[i] is predicted 206 | to have class c, where 0 <= c < C. 207 | """ 208 | y_pred = None 209 | 210 | ########################################################################### 211 | # TODO: Implement this function; it should be VERY simple! # 212 | ########################################################################### 213 | pass 214 | ########################################################################### 215 | # END OF YOUR CODE # 216 | ########################################################################### 217 | 218 | return y_pred 219 | --------------------------------------------------------------------------------