├── .gitignore ├── CMakeLists.txt ├── README.md ├── build.sh ├── data ├── cifar │ └── .keep └── mnist │ └── .keep ├── experiment ├── mnist │ ├── mnist_demo.cpp │ └── mnist_demo.hpp └── mnist_demo.sh ├── include ├── Layer.hpp ├── Net.hpp ├── Tensor.hpp ├── blas_function.hpp ├── convolution_layer.hpp ├── im2col.hpp ├── log_loss_layer.hpp ├── pooling_layer.hpp ├── rand_function.hpp ├── relu_layer.hpp └── softmax_layer.hpp ├── python ├── cifar-demo.py ├── cnn │ ├── __init__.py │ ├── conv_net.py │ ├── im2col.py │ └── layers.py ├── download_cifar.py ├── download_mnist.py ├── interfaces │ ├── layers.cpp │ └── pylayer_test.py ├── load_cifar.py ├── load_mnist.py ├── mnist-demo.py ├── model │ ├── __init__.py │ ├── initial_LeNet.py │ └── initial_cifarNet.py ├── rnn │ └── naive_rnn.py └── util │ ├── gaussian_kernel.py │ ├── test_layers.py │ └── util.py ├── src ├── Net.cpp ├── blas_function.cpp ├── convolution_layer.cpp ├── log_loss_layer.cpp ├── pooling_layer.cpp ├── rand_function.cpp ├── relu_layer.cpp └── softmax_layer.cpp └── test ├── test_conv_layer.cpp ├── test_pooling_layer.cpp ├── test_relu_layer.cpp └── test_softmax_log_loss_layer.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | code/.idea/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Sphinx documentation 58 | docs/_build/ 59 | 60 | # PyBuilder 61 | target/ 62 | 63 | #Ipython Notebook 64 | .ipynb_checkpoints 65 | 66 | # pyenv 67 | .python-version 68 | 69 | .idea 70 | sample_data 71 | data/ 72 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.3) 2 | project(PyConvNet) 3 | 4 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -lopenblas -O2") 5 | 6 | set(Open_BLAS_INCLUDE_SEARCH_PATHS $ENV{OpenBLAS_HOME}) 7 | 8 | set(Open_BLAS_LIB_SEARCH_PATHS $ENV{OpenBLAS_HOME}) 9 | 10 | find_path(OpenBLAS_INCLUDE_DIR NAMES cblas.h PATHS ${Open_BLAS_INCLUDE_SEARCH_PATHS}) 11 | find_library(OpenBLAS_LIB NAMES openblas PATHS ${Open_BLAS_LIB_SEARCH_PATHS}) 12 | 13 | # find_package(PythonInterp) 14 | # find_package(PythonLibs) 15 | # find_package(Boost COMPONENTS python) 16 | # link_libraries(${Boost_LIBRARIES} ${PYTHON_LIBRARIES}) 17 | link_libraries(${Boost_LIBRARIES}) 18 | 19 | # set(NUMPY_INCLUDE_DIRS $ENV{ANACONDA_HOME}/lib/python2.7/site-packages/numpy/core/include) 20 | 21 | # include_directories( include/ ${Boost_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIRS}) 22 | include_directories( include/ ${Boost_INCLUDE_DIRS}) 23 | 24 | set(SOURCE_FILES include/Layer.hpp experiment/mnist/mnist_demo.cpp include/im2col.hpp include/convolution_layer.hpp 25 | src/convolution_layer.cpp include/Tensor.hpp include/blas_function.hpp src/blas_function.cpp 26 | include/pooling_layer.hpp src/pooling_layer.cpp include/relu_layer.hpp src/relu_layer.cpp 27 | include/softmax_layer.hpp src/softmax_layer.cpp include/log_loss_layer.hpp src/log_loss_layer.cpp 28 | include/Net.hpp src/Net.cpp include/rand_function.hpp src/rand_function.cpp) 29 | # set(PYTHON_LAYER_FILES include/Layer.hpp include/im2col.hpp include/convolution_layer.hpp 30 | # src/convolution_layer.cpp include/Tensor.hpp include/blas_function.hpp src/blas_function.cpp 31 | # include/pooling_layer.hpp src/pooling_layer include/relu_layer.hpp src/relu_layer.cpp 32 | # python/interfaces/layers.cpp include/rand_function.hpp src/rand_function.cpp) 33 | 34 | set(TEST_CONV_FILES include/Layer.hpp include/im2col.hpp include/convolution_layer.hpp 35 | src/convolution_layer.cpp include/Tensor.hpp include/blas_function.hpp src/blas_function.cpp 36 | test/test_conv_layer.cpp include/rand_function.hpp src/rand_function.cpp) 37 | set(TEST_POOLING_FILES include/Layer.hpp include/Tensor.hpp include/pooling_layer.hpp 38 | src/pooling_layer.cpp test/test_pooling_layer.cpp include/rand_function.hpp src/rand_function.cpp) 39 | set(TEST_RELU_FILES include/Layer.hpp include/Tensor.hpp include/relu_layer.hpp src/relu_layer.cpp 40 | test/test_relu_layer.cpp include/rand_function.hpp src/rand_function.cpp) 41 | set(TEST_SOFTMAX_FILES include/Layer.hpp include/blas_function.hpp src/blas_function.cpp 42 | include/Tensor.hpp include/softmax_layer.hpp src/softmax_layer.cpp 43 | include/log_loss_layer.hpp src/log_loss_layer.cpp test/test_softmax_log_loss_layer.cpp 44 | include/rand_function.hpp src/rand_function.cpp) 45 | 46 | set(EXECUTABLE_OUTPUT_PATH bin/) 47 | 48 | add_executable(MnistDemo ${SOURCE_FILES}) 49 | add_executable(TestConv ${TEST_CONV_FILES}) 50 | add_executable(TestPool ${TEST_POOLING_FILES}) 51 | add_executable(TestReLU ${TEST_RELU_FILES}) 52 | add_executable(TestSoftMaxLogLoss ${TEST_SOFTMAX_FILES}) 53 | 54 | # python_add_module(pylayer ${PYTHON_LAYER_FILES}) 55 | 56 | target_link_libraries (MnistDemo ${OpenBLAS_LIB}) 57 | target_link_libraries (TestConv ${OpenBLAS_LIB}) 58 | target_link_libraries(TestSoftMaxLogLoss ${OpenBLAS_LIB}) 59 | # target_link_libraries (pylayer ${OpenBLAS_LIB}) 60 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyConvNet: CNN for Python 2 | **PyConvNet** is a python implementation of convolutional neural network. 3 | 4 | To train LeNet on MNIST dataset, just do as follow(you may need some python package such as numpy matplotlib): 5 | 6 | 1. cd python 7 | 2. python mnist_demo.py 8 | 9 | A C++ version ConNet is also available now. It is more faster than the Python version. To try the C++ implemention, you will need Openblas and boost installed, and then do as follow: 10 | 11 | 1. sh build.sh 12 | 2. cd experiment/ 13 | 3. run mnist_demo.sh 14 | 15 | Then the script will download the mnist dataset and train the lenet. 16 | 17 | This is a brief CNN tutorial (from [Jianxin Wu](http://cs.nju.edu.cn/wujx/)'s homepage): 18 | 19 | http://cs.nju.edu.cn/_upload/tpl/00/ed/237/template237/paper/CNN.pdf 20 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | export OpenBLAS_HOME=/path/to/your/OpenBLAS 2 | mkdir build 3 | cd build 4 | mkdir bin 5 | cmake .. 6 | make -j4 7 | -------------------------------------------------------------------------------- /data/cifar/.keep: -------------------------------------------------------------------------------- 1 | null -------------------------------------------------------------------------------- /data/mnist/.keep: -------------------------------------------------------------------------------- 1 | null -------------------------------------------------------------------------------- /experiment/mnist/mnist_demo.cpp: -------------------------------------------------------------------------------- 1 | # include 2 | # include 3 | # include 4 | # include 5 | # include 6 | 7 | # include "Layer.hpp" 8 | # include "Net.hpp" 9 | # include "mnist_demo.hpp" 10 | 11 | float train_data[784*60000]; 12 | float train_label[60000]; 13 | float test_data[784*10000]; 14 | float test_label[10000]; 15 | 16 | static timeb TimingMilliSeconds; 17 | 18 | void StartOfDuration() { 19 | ftime(&TimingMilliSeconds); 20 | } 21 | 22 | int EndOfDuration() { 23 | struct timeb now; 24 | ftime(&now); 25 | return int( (now.time-TimingMilliSeconds.time)*1000+(now.millitm-TimingMilliSeconds.millitm) ); 26 | } 27 | 28 | int main() { 29 | // loading mnist data 30 | read_mnist("../../data/mnist/train-images-idx3-ubyte", 31 | "../../data/mnist/train-labels-idx1-ubyte", 32 | train_data, train_label); 33 | read_mnist("../../data/mnist/t10k-images-idx3-ubyte", 34 | "../../data/mnist/t10k-labels-idx1-ubyte", 35 | test_data, test_label); 36 | 37 | int batch_size = 100; 38 | std::shared_ptr lenet = init_lenet(batch_size); 39 | int epochs = 5; 40 | float lr_arr[5] = {0.0001, 0.0001, 0.00001, 0.00001, 0.00001}; 41 | for (int epoch_idx = 0; epoch_idx < epochs; ++epoch_idx) { 42 | lenet->set_lr(lr_arr[epoch_idx]); 43 | for(int i = 0; i < (int)(60000 / batch_size); ++i) { 44 | float* batch_data_ptr = new float[batch_size * 28 * 28]; 45 | float* batch_label_ptr = new float[batch_size]; 46 | memcpy(batch_data_ptr, train_data + i * batch_size * 28 * 28, 47 | sizeof(float) * batch_size * 28 * 28); 48 | memcpy(batch_label_ptr, train_label + i * batch_size, sizeof(float) * batch_size); 49 | boost::shared_array data_batch(batch_data_ptr); 50 | boost::shared_array label_batch(batch_label_ptr); 51 | Tensor train_data_tensor(batch_size, 1, 28, 28); 52 | Tensor train_label_tensor(batch_size, 1, 1, 1); 53 | train_data_tensor.set_data(data_batch); 54 | train_label_tensor.set_data(label_batch); 55 | std::cout << "epoch: " << epoch_idx + 1 << ", iter: " << i << ", "; 56 | StartOfDuration(); 57 | lenet->train_batch(train_data_tensor, train_label_tensor); 58 | int msec = EndOfDuration(); 59 | std::cout << msec << "ms elapse\n"; 60 | } 61 | } 62 | 63 | // do testing 64 | float right_num = 0.f; 65 | for(int i = 0; i < (int)(10000 / batch_size); ++i) { 66 | float* batch_data_ptr = new float[batch_size * 28 * 28]; 67 | float* batch_label_ptr = new float[batch_size]; 68 | memcpy(batch_data_ptr, test_data + i * batch_size * 28 * 28, 69 | sizeof(float) * batch_size * 28 * 28); 70 | memcpy(batch_label_ptr, test_label + i * batch_size, sizeof(float) * batch_size); 71 | boost::shared_array data_batch(batch_data_ptr); 72 | boost::shared_array label_batch(batch_label_ptr); 73 | Tensor test_data_tensor(batch_size, 1, 28, 28); 74 | Tensor test_label_tensor(batch_size, 1, 1, 1); 75 | Tensor pred_label_tensor(batch_size, 1, 1, 1); 76 | test_data_tensor.set_data(data_batch); 77 | test_label_tensor.set_data(label_batch); 78 | std::cout << "test iter: " << i << " "; 79 | StartOfDuration(); 80 | lenet->test_batch(test_data_tensor, pred_label_tensor); 81 | int msec = EndOfDuration(); 82 | float* label_ptr = test_label_tensor.get_data().get(); 83 | float* pred_ptr = pred_label_tensor.get_data().get(); 84 | for(int j = 0; j < batch_size; ++j) { 85 | if (label_ptr[j] == pred_ptr[j]) 86 | ++right_num; 87 | } 88 | std::cout << msec << "ms elapse, batch acc: " << right_num / ((i+1) * batch_size) << std::endl; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /experiment/mnist/mnist_demo.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PYCONVNET_MNIST_DEMO_HPP 2 | #define PYCONVNET_MNIST_DEMO_HPP 3 | 4 | # include "Layer.hpp" 5 | # include "Net.hpp" 6 | # include "convolution_layer.hpp" 7 | # include "relu_layer.hpp" 8 | # include "pooling_layer.hpp" 9 | # include "softmax_layer.hpp" 10 | # include "log_loss_layer.hpp" 11 | 12 | // function to read mnist data 13 | int reverse_int (int i) 14 | { 15 | unsigned char ch1, ch2, ch3, ch4; 16 | ch1 = i&255; 17 | ch2 = (i >> 8)&255; 18 | ch3 = (i >> 16)&255; 19 | ch4 = (i >> 24)&255; 20 | return((int)ch1 << 24) + ((int)ch2 << 16) + ((int)ch3 << 8) + ch4; 21 | } 22 | 23 | void read_mnist(const char* data_path, const char* label_path, float* data, float* label) { 24 | std::ifstream data_file (data_path, std::ios::binary); 25 | if (data_file.is_open()) 26 | { 27 | int magic_number = 0; 28 | int number_of_images = 0; 29 | int n_rows = 0; 30 | int n_cols = 0; 31 | data_file.read((char*)&magic_number, sizeof(magic_number)); 32 | magic_number= reverse_int(magic_number); 33 | data_file.read((char*)&number_of_images, sizeof(number_of_images)); 34 | number_of_images = reverse_int(number_of_images); 35 | data_file.read((char*)&n_rows, sizeof(n_rows)); 36 | n_rows= reverse_int(n_rows); 37 | data_file.read((char*)&n_cols, sizeof(n_cols)); 38 | n_cols= reverse_int(n_cols); 39 | 40 | for(int i = 0; i < number_of_images; ++i) 41 | { 42 | for(int r = 0; r < n_rows; ++r) 43 | { 44 | for(int c = 0; c < n_cols; ++c) 45 | { 46 | unsigned char temp=0; 47 | data_file.read((char*)&temp,sizeof(temp)); 48 | data[i * n_rows * n_cols + r * n_cols + c]= (float)temp; 49 | } 50 | } 51 | } 52 | } 53 | data_file.close(); 54 | 55 | std::ifstream label_file (label_path, std::ios::binary); 56 | if (label_file.is_open()) 57 | { 58 | int magic_number = 0; 59 | int number_of_items = 0; 60 | label_file.read((char*)&magic_number, sizeof(magic_number)); 61 | magic_number= reverse_int(magic_number); 62 | label_file.read((char*)&number_of_items, sizeof(number_of_items)); 63 | number_of_items = reverse_int(number_of_items); 64 | 65 | for(int i = 0; i < number_of_items; ++i) 66 | { 67 | unsigned char temp=0; 68 | label_file.read((char*)&temp,sizeof(temp)); 69 | label[i]= (float)temp; 70 | } 71 | } 72 | label_file.close(); 73 | } 74 | 75 | std::shared_ptr init_lenet(int batch_size) { 76 | std::shared_ptr lenet(new Net()); 77 | std::shared_ptr conv1(new ConvolutionLayer(0, 0, 20, 1, 5, 5, 1, 1)); 78 | lenet->add_layer(conv1); 79 | lenet->add_param_layer_id(0); 80 | 81 | std::shared_ptr pool1(new PoolingLayer(0, 0, 2, 2, 2, 2)); 82 | lenet->add_layer(pool1); 83 | 84 | std::shared_ptr conv2(new ConvolutionLayer(0, 0, 50, 20, 5, 5, 1, 1)); 85 | lenet->add_layer(conv2); 86 | lenet->add_param_layer_id(2); 87 | 88 | std::shared_ptr pool2(new PoolingLayer(0, 0, 2, 2, 2, 2)); 89 | lenet->add_layer(pool2); 90 | 91 | std::shared_ptr conv3(new ConvolutionLayer(0, 0, 500, 50, 4, 4, 1, 1)); 92 | lenet->add_layer(conv3); 93 | lenet->add_param_layer_id(4); 94 | 95 | std::shared_ptr relu1(new ReLULayer()); 96 | lenet->add_layer(relu1); 97 | 98 | std::shared_ptr conv4(new ConvolutionLayer(0, 0, 10, 500, 1, 1, 1, 1)); 99 | lenet->add_layer(conv4); 100 | lenet->add_param_layer_id(6); 101 | 102 | std::shared_ptr softmax1(new SoftmaxLayer()); 103 | lenet->add_layer(softmax1); 104 | 105 | std::shared_ptr log1(new LogLoss()); 106 | lenet->add_layer(log1); 107 | 108 | 109 | 110 | Tensor data0(batch_size, 1, 28, 28); 111 | Tensor d_data0(batch_size, 1, 28, 28); 112 | std::vector v0; 113 | v0.push_back(data0); 114 | v0.push_back(d_data0); 115 | 116 | // conv1 117 | Tensor data1(batch_size, 20, 24, 24); 118 | Tensor d_data1(batch_size, 20, 24, 24); 119 | std::vector v1; 120 | v1.push_back(data1); 121 | v1.push_back(d_data1); 122 | 123 | // pool1 124 | Tensor data2(batch_size, 20, 12, 12); 125 | Tensor d_data2(batch_size, 20, 12, 12); 126 | std::vector v2; 127 | v2.push_back(data2); 128 | v2.push_back(d_data2); 129 | 130 | // conv2 131 | Tensor data3(batch_size, 50, 8, 8); 132 | Tensor d_data3(batch_size, 50, 8, 8); 133 | std::vector v3; 134 | v3.push_back(data3); 135 | v3.push_back(d_data3); 136 | 137 | // pool2 138 | Tensor data4(batch_size, 50, 4, 4); 139 | Tensor d_data4(batch_size, 50, 4, 4); 140 | std::vector v4; 141 | v4.push_back(data4); 142 | v4.push_back(d_data4); 143 | 144 | // conv3 145 | Tensor data5(batch_size, 500, 1, 1); 146 | Tensor d_data5(batch_size, 500, 1, 1); 147 | std::vector v5; 148 | v5.push_back(data5); 149 | v5.push_back(d_data5); 150 | 151 | // relu1 152 | Tensor data6(batch_size, 500, 1, 1); 153 | Tensor d_data6(batch_size, 500, 1, 1); 154 | std::vector v6; 155 | v6.push_back(data6); 156 | v6.push_back(d_data6); 157 | 158 | // conv4 159 | Tensor data7(batch_size, 10, 1, 1); 160 | Tensor d_data7(batch_size, 10, 1, 1); 161 | std::vector v7; 162 | v7.push_back(data7); 163 | v7.push_back(d_data7); 164 | 165 | // softmax1 166 | Tensor data8(batch_size, 10, 1, 1); 167 | Tensor d_data8(batch_size, 10, 1, 1); 168 | Tensor label(batch_size, 1, 1, 1); 169 | std::vector v8; 170 | v8.push_back(data8); 171 | v8.push_back(d_data8); 172 | v8.push_back(label); 173 | 174 | // log 175 | Tensor data9(1, 1, 1, 1); 176 | Tensor output9(batch_size, 1, 1, 1); 177 | 178 | std::vector v9; 179 | v9.push_back(data9); 180 | v9.push_back(output9); 181 | 182 | lenet->add_data(v0); 183 | lenet->add_data(v1); 184 | lenet->add_data(v2); 185 | lenet->add_data(v3); 186 | lenet->add_data(v4); 187 | lenet->add_data(v5); 188 | lenet->add_data(v6); 189 | lenet->add_data(v7); 190 | lenet->add_data(v8); 191 | lenet->add_data(v9); 192 | return lenet; 193 | } 194 | 195 | #endif //PYCONVNET_MNIST_DEMO_HPP 196 | -------------------------------------------------------------------------------- /experiment/mnist_demo.sh: -------------------------------------------------------------------------------- 1 | cd ../data/mnist/ 2 | 3 | wget http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz 4 | wget http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz 5 | wget http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz 6 | wget http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz 7 | 8 | gunzip -c train-images-idx3-ubyte.gz > train-images-idx3-ubyte 9 | gunzip -c t10k-images-idx3-ubyte.gz > t10k-images-idx3-ubyte 10 | gunzip -c train-labels-idx1-ubyte.gz > train-labels-idx1-ubyte 11 | gunzip -c t10k-labels-idx1-ubyte.gz > t10k-labels-idx1-ubyte 12 | 13 | cd ../../build/bin 14 | ./MnistDemo 15 | -------------------------------------------------------------------------------- /include/Layer.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PYCONVNET_LAYER_HPP 2 | #define PYCONVNET_LAYER_HPP 3 | 4 | # include 5 | # include "Tensor.hpp" 6 | 7 | class Layer { 8 | public: 9 | Layer() { 10 | // empty 11 | } 12 | virtual void params_update(float lr) = 0; 13 | virtual void forward(std::vector& input, std::vector& output) = 0; 14 | virtual void backward(std::vector& input, 15 | std::vector& output) = 0; 16 | }; 17 | 18 | #endif //PYCONVNET_LAYER_HPP 19 | -------------------------------------------------------------------------------- /include/Net.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PYCONVNET_NET_HPP 2 | #define PYCONVNET_NET_HPP 3 | 4 | # include 5 | # include "Layer.hpp" 6 | # include "Tensor.hpp" 7 | 8 | class Net { 9 | public: 10 | Net(); 11 | void add_layer(std::shared_ptr l); 12 | void set_lr(float lr); 13 | void add_param_layer_id(int idx); 14 | void add_data(std::vector& t); 15 | void forward_net(int start=-1, int end=-1); 16 | void backward_net(); 17 | void params_update(); 18 | void train_batch(Tensor& train_data, Tensor& train_label); 19 | void test_batch(Tensor& test_data, Tensor& pred_label); 20 | private: 21 | std::vector> layers_; // n layers 22 | std::vector> data_; // data and d_data 23 | std::vector params_layer_id; 24 | float lr; 25 | }; 26 | 27 | #endif //PYCONVNET_NET_HPP 28 | -------------------------------------------------------------------------------- /include/Tensor.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PYCONVNET_TENSOR_HPP 2 | #define PYCONVNET_TENSOR_HPP 3 | 4 | # include 5 | # include 6 | # include 7 | # include 8 | 9 | # include "blas_function.hpp" 10 | # include "rand_function.hpp" 11 | 12 | class Tensor { 13 | public: 14 | Tensor(int n, int c, int h, int w, float std = 0) : data(new float[n * c * h * w]){ 15 | N = n; 16 | C = c; 17 | H = h; 18 | W = w; 19 | if(std == 0) { 20 | memset(data.get(), 0, sizeof(float) * N * C * H * W); 21 | } else { 22 | gaussrand(0, std, data.get(), N*C*H*W); 23 | } 24 | } 25 | Tensor(const Tensor& T) { 26 | N = T.get_N(); 27 | C = T.get_C(); 28 | H = T.get_H(); 29 | W = T.get_W(); 30 | data = T.get_data(); 31 | } 32 | boost::shared_array get_data() const { 33 | return data; 34 | } 35 | void set_data(boost::shared_array data_input) { 36 | data = data_input; 37 | } 38 | int get_N() const { 39 | return N; 40 | } 41 | int get_C() const { 42 | return C; 43 | } 44 | int get_H() const { 45 | return H; 46 | } 47 | int get_W() const { 48 | return W; 49 | } 50 | int get_size() const { 51 | return N * C * H * W; 52 | } 53 | void add_Tensor(Tensor& t, const float coeff_a, const float coeff_b) { 54 | vector_add(this->get_data().get(), t.get_data().get(), this->get_data().get(), 55 | coeff_a, coeff_b, this->get_size()); 56 | } 57 | bool operator==( Tensor const& t ) const { 58 | return std::equal(this->get_data().get(), this->get_data().get() + N * C * H * W, 59 | t.get_data().get()); 60 | } 61 | bool operator!=( Tensor const& t ) const { 62 | return std::equal(this->get_data().get(), this->get_data().get() + N * C * H * W, 63 | t.get_data().get()); 64 | } 65 | Tensor& operator=(Tensor const& t) { 66 | if(this != &t) { 67 | this->N = t.N; 68 | this->C = t.C; 69 | this->H = t.H; 70 | this->W = t.W; 71 | this->data = t.get_data(); 72 | } 73 | return *this; 74 | } 75 | private: 76 | boost::shared_array data; 77 | int N, C, H, W; 78 | }; 79 | #endif //PYCONVNET_TENSOR_HPP 80 | -------------------------------------------------------------------------------- /include/blas_function.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PYCONVNET_BLAS_FUNCTION_HPP 2 | #define PYCONVNET_BLAS_FUNCTION_HPP 3 | 4 | extern "C" 5 | { 6 | # include "cblas.h" 7 | } 8 | 9 | // matrix multiplication 10 | void gemm(const CBLAS_TRANSPOSE TransA, const CBLAS_TRANSPOSE TransB, 11 | const int M, const int N, const int K, const float alpha, 12 | const float* A, const float* B, const float beta, float* C); 13 | 14 | // vector operation 15 | void vector_add(const float* A, const float* B, float* C, const float coeff_a, 16 | const float coeff_b, const int vector_size); 17 | 18 | void vector_sub(const float* A, const float* B, float* C, const int vector_size); 19 | 20 | void vector_mul(const float* A, const float* B, float* C, const int vector_size); 21 | 22 | void vector_exp(const float* A, float* B, const int vector_size); 23 | 24 | void vector_sub_scalar(float* A, float b, float* B, const int vector_size); 25 | 26 | void vector_div_scalar(float* A, float b, const int vector_size); 27 | 28 | void vector_mul_scalar(float* A, float b, const int vector_size); 29 | 30 | float vector_sum(float* A, const int vector_size); 31 | 32 | void vector_mul_scalar(float* A, float b, const int vector_size); 33 | 34 | void vector_scale(float* A, const int vector_size); 35 | 36 | #endif //PYCONVNET_BLAS_FUNCTION_HPP 37 | -------------------------------------------------------------------------------- /include/convolution_layer.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PYCONVNET_CONVOLUTION_LAYER_HPP 2 | #define PYCONVNET_CONVOLUTION_LAYER_HPP 3 | 4 | #include 5 | #include "Tensor.hpp" 6 | #include "Layer.hpp" 7 | 8 | class ConvolutionLayer: public Layer { 9 | public: 10 | ConvolutionLayer(const int pad_h_, const int pad_w_, 11 | const int filter_num, const int filter_channel, 12 | const int kernel_h_, const int kernel_w_, 13 | const int stride_h_, const int stride_w_): 14 | filter(filter_num, filter_channel, kernel_h_, kernel_w_, 0.01), 15 | bias(filter_num, 1, 1, 1), 16 | d_filter(filter_num, filter_channel, kernel_h_, kernel_w_), 17 | d_bias(filter_num, 1, 1, 1), 18 | pre_update_f(filter_num, filter_channel, kernel_h_, kernel_w_), 19 | pre_update_b(filter_num, 1, 1, 1) { 20 | pad_h = pad_h_; 21 | pad_w = pad_w_; 22 | kernel_h = kernel_h_; 23 | kernel_w = kernel_w_; 24 | stride_h = stride_h_; 25 | stride_w = stride_w_; 26 | 27 | }; 28 | // set_* function is used while testing layer 29 | void set_filter(Tensor& f) { 30 | filter = f; 31 | }; 32 | void set_bias(Tensor& b) { 33 | bias = b; 34 | }; 35 | void set_d_filter(Tensor& d_f) { 36 | d_filter = d_f; 37 | }; 38 | void set_d_bias(Tensor& d_b) { 39 | d_bias = d_b; 40 | }; 41 | 42 | void forward(std::vector& input, std::vector& output ); 43 | void backward(std::vector& input, 44 | std::vector& output); 45 | void params_update(float lr); 46 | 47 | private: 48 | Tensor filter; 49 | Tensor bias; 50 | Tensor d_filter; 51 | Tensor pre_update_f; 52 | Tensor d_bias; 53 | Tensor pre_update_b; 54 | int pad_h = 0; 55 | int pad_w = 0; 56 | int kernel_h = 0; 57 | int kernel_w = 0; 58 | int stride_h = 0; 59 | int stride_w = 0; 60 | }; 61 | #endif //PYCONVNET_CONVOLUTION_LAYER_HPP 62 | -------------------------------------------------------------------------------- /include/im2col.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PYCONVNET_IM2COL_HPP 2 | #define PYCONVNET_IM2COL_HPP 3 | 4 | #include 5 | 6 | void im2col(const float* data_input, const int height, const int width, const int channels, 7 | const int kernel_h, const int kernel_w, const int pad_h, const int pad_w, 8 | const int stride_h, const int stride_w, float* data_output) { 9 | int h_outout = (height + 2 * pad_h - kernel_h) / stride_h + 1; 10 | int w_output = (width + 2 * pad_w - kernel_w) / stride_w + 1; 11 | int output_width = h_outout * w_output; 12 | for (int base_h = 0; base_h < h_outout; ++base_h) { 13 | for (int base_w = 0; base_w < w_output; ++base_w) { 14 | for (int channels_idx = 0; channels_idx < channels; ++channels_idx) { 15 | for (int kernel_h_idx = 0; kernel_h_idx < kernel_h; ++kernel_h_idx) { 16 | for (int kernel_w_idx = 0; kernel_w_idx < kernel_w; ++kernel_w_idx) { 17 | int original_h = base_h * stride_h + kernel_h_idx - pad_h; 18 | int original_w = base_w * stride_w + kernel_w_idx - pad_w; 19 | 20 | int output_row = channels_idx * kernel_h * kernel_w + 21 | kernel_h_idx * kernel_w + kernel_w_idx; 22 | int output_col = base_h * w_output + base_w; 23 | 24 | if (original_h < 0 || original_h >= height || 25 | original_w < 0 || original_w >= width) { 26 | data_output[output_row * output_width + output_col] = 0; 27 | } 28 | else { 29 | data_output[output_row * output_width + output_col] = 30 | data_input[channels_idx * width * height + 31 | original_h * width + original_w]; 32 | } 33 | } 34 | } 35 | } 36 | } 37 | } 38 | } 39 | 40 | void col2im(const float* data_col, const int height, const int width, const int channels, 41 | const int kernel_h, const int kernel_w, const int pad_h, const int pad_w, 42 | const int stride_h, const int stride_w, float* data_original) { 43 | memset(data_original, 0, sizeof(float) * height * width * channels); 44 | int data_col_height = channels * kernel_h * kernel_w; 45 | int data_col_width = ((height + 2 * pad_h - kernel_h) / stride_h + 1) * 46 | ((width + 2 * pad_w - kernel_w) / stride_w + 1); 47 | for (int row = 0; row < data_col_height; ++row) { 48 | for(int cols = 0; cols < data_col_width; ++cols) { 49 | int base_h = cols / ((width + 2 * pad_w - kernel_w) / stride_w + 1); 50 | int base_w = cols % ((width + 2 * pad_w - kernel_w) / stride_w + 1); 51 | int bias_channel = row / kernel_h / kernel_w; 52 | int bias_height = row % (kernel_h * kernel_w) / kernel_w; 53 | int bias_width = row % kernel_w; 54 | int original_h = base_h * stride_h + bias_height - pad_h; 55 | int original_w = base_w * stride_w + bias_width - pad_w; 56 | if (original_h < 0 || original_h >= height || 57 | original_w < 0 || original_w >= width) 58 | continue; 59 | data_original[bias_channel * height * width + original_h * width 60 | + original_w] += 61 | data_col[row * data_col_width + cols]; 62 | } 63 | } 64 | } 65 | 66 | #endif //PYCONVNET_IM2COL_HPP 67 | -------------------------------------------------------------------------------- /include/log_loss_layer.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PYCONVNET_SOFTMAX_LOG_LOSS_LAYER_HPP 2 | #define PYCONVNET_SOFTMAX_LOG_LOSS_LAYER_HPP 3 | 4 | # include "Layer.hpp" 5 | 6 | class LogLoss: public Layer { 7 | public: 8 | void params_update(float lr); 9 | void forward(std::vector& input, std::vector& output); 10 | void backward(std::vector &input, 11 | std::vector &output); 12 | }; 13 | #endif //PYCONVNET_SOFTMAX_LOG_LOSS_LAYER_HPP 14 | -------------------------------------------------------------------------------- /include/pooling_layer.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PYCONVNET_POOLING_LAYER_HPP 2 | #define PYCONVNET_POOLING_LAYER_HPP 3 | 4 | # include 5 | # include 6 | # include "Layer.hpp" 7 | 8 | class PoolingLayer: public Layer { 9 | public: 10 | PoolingLayer(const int pad_h_, const int pad_w_, 11 | const int kernel_h_, const int kernel_w_, 12 | const int stride_h_, const int stride_w_) { 13 | pad_h = pad_h_; 14 | pad_w = pad_w_; 15 | kernel_h = kernel_h_; 16 | kernel_w = kernel_w_; 17 | stride_h = stride_h_; 18 | stride_w = stride_w_; 19 | max_index_mask.clear(); 20 | }; 21 | void params_update(float lr); 22 | void forward(std::vector& input, std::vector& output ); 23 | void backward(std::vector& input, 24 | std::vector& output); 25 | 26 | private: 27 | int pad_h = 0; 28 | int pad_w = 0; 29 | int kernel_h = 0; 30 | int kernel_w = 0; 31 | int stride_h = 0; 32 | int stride_w = 0; 33 | std::vector max_index_mask; 34 | }; 35 | #endif //PYCONVNET_POOLING_LAYER_HPP 36 | -------------------------------------------------------------------------------- /include/rand_function.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PYCONVNET_RAND_FUNCTION_HPP 2 | #define PYCONVNET_RAND_FUNCTION_HPP 3 | 4 | void gaussrand(float mean, float stddev, float* rand_array, int array_len); 5 | 6 | #endif //PYCONVNET_RAND_FUNCTION_HPP 7 | -------------------------------------------------------------------------------- /include/relu_layer.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PYCONVNET_RELU_LAYER_HPP 2 | #define PYCONVNET_RELU_LAYER_HPP 3 | 4 | # include 5 | # include "Tensor.hpp" 6 | # include "Layer.hpp" 7 | 8 | class ReLULayer: public Layer { 9 | public: 10 | void params_update(float lr); 11 | void forward(std::vector& input, std::vector& output ); 12 | void backward(std::vector& input, 13 | std::vector& output); 14 | }; 15 | #endif //PYCONVNET_RELU_LAYER_HPP 16 | -------------------------------------------------------------------------------- /include/softmax_layer.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PYCONVNET_SOFTMAX_LAYER_HPP 2 | #define PYCONVNET_SOFTMAX_LAYER_HPP 3 | 4 | # include "Layer.hpp" 5 | 6 | class SoftmaxLayer: public Layer { 7 | public: 8 | void params_update(float lr); 9 | void forward(std::vector& input, std::vector& output); 10 | void backward(std::vector& input, 11 | std::vector& output); 12 | }; 13 | #endif //PYCONVNET_SOFTMAX_LAYER_HPP 14 | -------------------------------------------------------------------------------- /python/cifar-demo.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | import os 4 | import scipy.io as sio 5 | import numpy as np 6 | 7 | from load_cifar import load_cifar 8 | from model.initial_cifarNet import initial_cifar 9 | 10 | train_data = [] 11 | valid_data = [] 12 | train_labels = [] 13 | valid_labels = [] 14 | if os.path.isfile('../data/cifar/cifar.mat'): 15 | print 'read mat file: %s' % ('../data/cifar/cifar.mat') 16 | data = sio.loadmat('../data/cifar/cifar.mat') 17 | train_data = data['train_data'] 18 | valid_data = data['valid_data'] 19 | train_labels = data['train_labels'] 20 | valid_labels = data['valid_labels'] 21 | else: 22 | train_data, valid_data, train_labels, valid_labels = load_cifar() 23 | 24 | cnn = initial_cifar() 25 | lr = np.logspace(-2, -3, 20) 26 | cnn.train(train_data, train_labels, lr, epoch=20, batch_size=32) 27 | 28 | res = cnn.predict(valid_data) 29 | res = res.reshape(valid_labels.shape) 30 | 31 | print 'Accuracy is: %f' % (np.sum(res == valid_labels) / float(np.max(valid_labels.shape))) 32 | -------------------------------------------------------------------------------- /python/cnn/__init__.py: -------------------------------------------------------------------------------- 1 | # empty file -------------------------------------------------------------------------------- /python/cnn/conv_net.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | from layers import * 4 | import time 5 | 6 | class ConvNet: 7 | def __init__(self): 8 | self.layers = [] 9 | 10 | def add_layer(self, layer_type, layer_params): 11 | if layer_type == 'conv': 12 | HF = layer_params['HF'] 13 | WF = layer_params['WF'] 14 | DF = layer_params['DF'] 15 | NF = layer_params['NF'] 16 | l_weights = np.random.normal(0, layer_params['var'], (HF, WF, DF, NF)) 17 | l_bias = np.zeros((1, 1, 1, NF)) 18 | layer_params = {'type': 'conv', 19 | 'weights': l_weights, 20 | 'bias': l_bias, 21 | 'stride': layer_params['stride'], 22 | 'pad': layer_params['pad'], 23 | 'input': None, 24 | 'output': None, 25 | 'grad': None} 26 | self.layers.append(layer_params) 27 | elif layer_type == 'max_pooling': 28 | layer_params = {'type': 'max_pooling', 29 | 'stride': layer_params['stride'], 30 | 'HF': layer_params['HF'], 31 | 'WF': layer_params['WF'], 32 | 'pad': layer_params['pad'], 33 | 'input': None, 34 | 'output': None, 35 | 'grad': None} 36 | self.layers.append(layer_params) 37 | elif layer_type == 'relu': 38 | layer_params = {'type': 'relu', 39 | 'input': None, 40 | 'output': None, 41 | 'grad': None} 42 | self.layers.append(layer_params) 43 | elif layer_type == 'softmax-loss': 44 | layer_params = {'type': 'softmax-loss', 45 | 'input': None, 46 | 'output': None, 47 | 'grad': None} 48 | self.layers.append(layer_params) 49 | else: 50 | print 'unkonw layer type!\n' 51 | exit(1) 52 | 53 | def forward(self, data, label=[]): 54 | for idx, each_layer in enumerate(self.layers): 55 | if idx == 0: 56 | each_layer['input'] = data 57 | else: 58 | each_layer['input'] = self.layers[idx - 1]['output'] 59 | if each_layer['type'] == 'conv': 60 | params = {'stride': each_layer['stride'], 61 | 'pad': each_layer['pad']} 62 | params['pad'] = each_layer['pad'] 63 | each_layer['output'] = conv_forward(each_layer['input'], 64 | each_layer['weights'], 65 | each_layer['bias'], params) 66 | elif each_layer['type'] == 'max_pooling': 67 | params = {'stride': each_layer['stride'], 68 | 'HF': each_layer['HF'], 69 | 'WF': each_layer['WF'], 70 | 'pad': each_layer['pad'] 71 | } 72 | each_layer['output'] = max_pooling_forward(each_layer['input'], 73 | params) 74 | elif each_layer['type'] == 'relu': 75 | each_layer['output'] = relu_forward(each_layer['input']) 76 | elif each_layer['type'] == 'softmax-loss': 77 | if len(label) == 0: 78 | each_layer['output'] = softmax(each_layer['input']) 79 | else: 80 | each_layer['output'] = softmax_loss_forward(each_layer['input'], label) 81 | return each_layer['output'] 82 | 83 | def backward(self, data, label, lr=0.01): 84 | for idx in reversed(np.arange(len(self.layers))): 85 | current_layer = self.layers[idx] 86 | if current_layer['type'] == 'softmax-loss': 87 | if idx != len(self.layers) - 1: 88 | print 'wrong architecture' 89 | exit(-1) 90 | self.layers[idx]['grad'] = softmax_loss_backward(self.layers[idx - 1]['output'], 91 | label) 92 | 93 | elif current_layer['type'] == 'conv': 94 | conv_param = {'stride': self.layers[idx]['stride'], 95 | 'pad': self.layers[idx]['pad']} 96 | self.layers[idx]['grad'] = conv_backward(self.layers[idx]['input'], 97 | self.layers[idx]['weights'], 98 | self.layers[idx]['bias'], 99 | conv_param, 100 | self.layers[idx + 1]['grad'][0]) 101 | self.layers[idx]['weights'] = self.layers[idx]['weights'] \ 102 | - lr * self.layers[idx]['grad'][1] 103 | # the learning rate of bias is 2 times of weights' 104 | self.layers[idx]['bias'] = self.layers[idx]['bias'] \ 105 | - 2 * lr * self.layers[idx]['grad'][2] 106 | elif current_layer['type'] == 'max_pooling': 107 | pooling_params = {'stride': self.layers[idx]['stride'], 108 | 'HF': self.layers[idx]['HF'], 109 | 'WF': self.layers[idx]['WF'], 110 | 'pad': self.layers[idx]['pad']} 111 | self.layers[idx]['grad'] = max_pooling_backward(self.layers[idx]['input'], 112 | self.layers[idx + 1]['grad'][0], 113 | pooling_params) 114 | elif current_layer['type'] == 'relu': 115 | self.layers[idx]['grad'] = relu_backward(self.layers[idx]['input'], self.layers[idx + 1]['grad'][0]) 116 | 117 | def predict(self, test_data, batch_size=50): 118 | _, _, _, N = test_data.shape 119 | prediction = np.zeros((1, N)) 120 | batch_num = np.ceil(float(N) / float(batch_size)) 121 | for batch_idx in np.arange(batch_num): 122 | sub_test_data = None 123 | sub_test_label= None 124 | if batch_idx == batch_num - 1: 125 | sub_test_data = test_data[:, :, :, batch_idx * batch_size:] 126 | self.forward(sub_test_data) 127 | prediction[0, batch_idx * batch_size:] = self.layers[-1]['output'] 128 | else: 129 | sub_test_data = test_data[:, :, :, batch_idx * batch_size:(batch_idx + 1) * batch_size] 130 | self.forward(sub_test_data) 131 | prediction[0, batch_idx * batch_size:(batch_idx + 1) * batch_size] = self.layers[-1]['output'] 132 | return prediction 133 | 134 | def train(self, train_data, train_label, lr, epoch=20, batch_size=100): 135 | H, W, D, N = train_data.shape 136 | _, N_l = train_label.shape 137 | assert N == N_l, 'Wrong data input!' 138 | # shuffle train_data 139 | shuffle_idx = np.arange(N) 140 | train_data = train_data[:, :, :, shuffle_idx] 141 | train_label = train_label[:, shuffle_idx] 142 | error_list = [] 143 | for epoch_idx in np.arange(epoch): 144 | batch_num = np.ceil(float(N) / float(batch_size)) 145 | for batch_idx in np.arange(batch_num): 146 | # start timing 147 | start_t = time.clock() 148 | sub_train_data = None 149 | sub_train_label= None 150 | if batch_idx == batch_num - 1: 151 | sub_train_data = train_data[:, :, :, batch_idx * batch_size:] 152 | sub_train_label = train_label[:, batch_idx * batch_size:] 153 | else: 154 | sub_train_data = train_data[:, :, :, batch_idx * batch_size:(batch_idx + 1) * batch_size] 155 | sub_train_label = train_label[:, batch_idx * batch_size:(batch_idx + 1) * batch_size] 156 | 157 | loss = self.forward(sub_train_data, sub_train_label) 158 | error_list.append(loss) 159 | 160 | self.backward(sub_train_data, sub_train_label, lr[epoch_idx]) 161 | # end timing 162 | end_t = time.clock() 163 | print 'epoch: %d, batch: %d, time: %f sec, obj: %f' % (epoch_idx, 164 | batch_idx, 165 | end_t - start_t, 166 | np.mean(np.array(error_list))) 167 | -------------------------------------------------------------------------------- /python/cnn/im2col.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | 3 | import numpy as np 4 | 5 | def im2col_index(x_shape, HF, WF, pad, stride): 6 | # get input size 7 | H, W, D, N = x_shape 8 | # get output size 9 | out_h = 0 10 | out_w = 0 11 | if type(pad) is int: 12 | out_h = (H + 2 * pad - HF) / stride + 1 13 | out_w = (W + 2 * pad - WF) / stride + 1 14 | else: 15 | out_h = (H + pad[0] + pad[1] - HF) / stride + 1 16 | out_w = (W + pad[2] + pad[3] - WF) / stride + 1 17 | # for row index, compute the first index of the first HF * WF block 18 | r0 = np.repeat(np.arange(HF), WF) 19 | r0 = np.tile(r0, D) 20 | # then compute the bias of each block 21 | r_bias = stride * np.repeat(np.arange(out_h), out_w) 22 | # then the row index is the r0 + r_bias 23 | r = r0.reshape(-1, 1) + r_bias.reshape(1, -1) 24 | 25 | # the same to the col index 26 | c0 = np.tile(np.arange(WF), HF * D) 27 | c_bias = stride * np.tile(np.arange(out_w), out_h) 28 | c = c0.reshape(-1, 1) + c_bias.reshape(1, -1) 29 | 30 | # then the dimension index 31 | d = np.repeat(np.arange(D), HF * WF).reshape(-1, 1) 32 | 33 | return (r, c, d) 34 | 35 | def im2col(x, HF, WF, pad, stride): 36 | # padding 37 | x_padded = None 38 | if type(pad) is int: 39 | x_padded = np.pad(x, ((pad, pad), (pad, pad), (0, 0), (0, 0)), mode='constant') 40 | else: 41 | x_padded = np.pad(x, ((pad[0], pad[1]), (pad[2], pad[3]), (0, 0), (0, 0)), mode='constant') 42 | r, c, d = im2col_index(x.shape, HF, WF, pad, stride) 43 | cols = x_padded[r, c, d, :] 44 | cols = cols.reshape(HF * WF * x.shape[2], -1) 45 | return cols 46 | 47 | def col2im(cols, x_shape, HF, WF, pad, stride): 48 | # get input size 49 | H, W, D, N = x_shape 50 | H_padded = 0 51 | W_padded = 0 52 | if type(pad) is int: 53 | H_padded, W_padded = H + 2 * pad, W + 2 * pad 54 | else: 55 | H_padded, W_padded = H + pad[0] + pad[1], W + pad[2] + pad[3] 56 | x_padded = np.zeros((H_padded, W_padded, D, N), dtype=cols.dtype) 57 | r, c, d = im2col_index(x_shape, HF, WF, pad, stride) 58 | cols_reshaped = cols.reshape((HF * WF * D, -1, N)) 59 | np.add.at(x_padded, (r, c, d, slice(None)), cols_reshaped) 60 | if pad == 0: 61 | return x_padded 62 | elif type(pad) is int: 63 | return x_padded[pad:-pad, pad:-pad, :, :] 64 | else: 65 | return x_padded[pad[0]:-pad[1], pad[2]:-pad[3], :, :] 66 | -------------------------------------------------------------------------------- /python/cnn/layers.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | 3 | import pylayer 4 | 5 | import numpy as np 6 | from im2col import im2col 7 | from im2col import col2im 8 | 9 | def conv_forward(x, w, b, params): 10 | # get convolution parameters 11 | stride = params['stride'] 12 | pad = params['pad'] 13 | # get input size 14 | H, W, D, N = x.shape 15 | HF, WF, DF, NF = w.shape 16 | _, _, DB, NB = b.shape 17 | # check input size 18 | assert D == DF, 'dimension does not work' 19 | assert NF == NB, 'batch size does not work' 20 | # check params 21 | assert (H + 2 * pad - HF) % stride == 0, 'pad and stride do not work' 22 | assert (W + 2 * pad - WF) % stride == 0, 'pad and stride do not work' 23 | # get output size 24 | HO = (H + 2 * pad - HF) / stride + 1 25 | WO = (W + 2 * pad - WF) / stride + 1 26 | x_col = im2col(x, HF, WF, pad, stride) 27 | w_col = w.transpose(3, 0, 1, 2).reshape((NF, -1)) 28 | output_col = w_col.dot(x_col) + b.reshape(-1, 1) 29 | output_col = output_col.reshape((NF, HO, WO, N)) 30 | output_col = output_col.transpose(1, 2, 0, 3) 31 | return output_col 32 | 33 | # def conv_forward(x, w, b, params): 34 | # # get convolution parameters 35 | # stride = params['stride'] 36 | # pad = params['pad'] 37 | # # get input size 38 | # H, W, D, N = x.shape 39 | # HF, WF, DF, NF = w.shape 40 | # 41 | # _, _, DB, NB = b.shape 42 | # 43 | # stride_h, stride_w = stride, stride 44 | # pad_h, pad_w = pad, pad 45 | # kernel_h, kernel_w = HF, WF 46 | # # check input size 47 | # assert D == DF, 'dimension does not work' 48 | # assert NF == NB, 'batch size does not work' 49 | # # check params 50 | # assert (H + 2 * pad - HF) % stride == 0, 'pad and stride do not work' 51 | # assert (W + 2 * pad - WF) % stride == 0, 'pad and stride do not work' 52 | # # get output size 53 | # HO = (H + 2 * pad - HF) / stride + 1 54 | # WO = (W + 2 * pad - WF) / stride + 1 55 | # 56 | # # initial data 57 | # input_data = x.transpose(3, 2, 0, 1).astype(np.float32) 58 | # input_data = input_data.copy(order='C') 59 | # input_tensor = pylayer.PyTensor(N, D, H, W) 60 | # input_tensor.init_from_numpy(input_data) 61 | # 62 | # filter_data = w.transpose(3, 2, 0, 1).astype(np.float32) 63 | # filter_data = filter_data.copy(order='C') 64 | # filter_tensor = pylayer.PyTensor(NF, DF, HF, WF) 65 | # filter_tensor.init_from_numpy(filter_data) 66 | # 67 | # bias_data = b.transpose(3, 2, 0, 1).astype(np.float32) 68 | # bias_data = bias_data.copy(order='C') 69 | # bias_tensor = pylayer.PyTensor(NB, 1, 1, 1) 70 | # bias_tensor.init_from_numpy(bias_data) 71 | # 72 | # output_data = np.zeros([N, NF, HO, WO], dtype=np.float32) 73 | # output_data = output_data.copy(order='C') 74 | # output_tensor = pylayer.PyTensor(N, NF, HO, WO) 75 | # output_tensor.init_from_numpy(output_data) 76 | # 77 | # input_vec = pylayer.PyTensorVec() 78 | # input_vec[:] = [input_tensor, filter_tensor, bias_tensor] 79 | # 80 | # output_vec = pylayer.PyTensorVec() 81 | # output_vec[:] = [output_tensor] 82 | # 83 | # L1 = pylayer.PyConvolutionLayer(pad_h, pad_w, kernel_h, kernel_w, stride_h, stride_w) 84 | # L1.forward(input_vec, output_vec) 85 | # 86 | # output_vec[0].return_numpy(output_data) 87 | # output_data = output_data.transpose(2, 3, 1, 0) 88 | # return output_data 89 | 90 | 91 | def conv_backward(x, w, b, conv_param, dout): 92 | HF, WF, DF, NF = w.shape 93 | x_col = im2col(x, HF, WF, conv_param['pad'], conv_param['stride']) 94 | w_col = w.transpose(3, 0, 1, 2).reshape((NF, -1)) 95 | db = np.sum(dout, axis=(0, 1, 3)) 96 | dout = dout.transpose(2, 0, 1, 3) 97 | dout = dout.reshape((w_col.shape[0], x_col.shape[-1])) 98 | dx_col = w_col.T.dot(dout) 99 | dw_col = dout.dot(x_col.T) 100 | 101 | dx = col2im(dx_col, x.shape, HF, WF, conv_param['pad'], conv_param['stride']) 102 | dw = dw_col.reshape((dw_col.shape[0], HF, WF, DF)) 103 | dw = dw.transpose(1, 2, 3, 0) 104 | 105 | return [dx, dw, db] 106 | 107 | def max_pooling_forward(x, pool_params): 108 | # get max-pooling parameters 109 | stride = pool_params['stride'] 110 | HF = pool_params['HF'] 111 | WF = pool_params['WF'] 112 | pad = pool_params['pad'] 113 | # get input size 114 | H, W, D, N = x.shape 115 | x_reshaped = x.reshape(H, W, 1, -1) 116 | # get output size 117 | HO = 0 118 | WO = 0 119 | if type(pad) is int: 120 | HO = (H + 2 * pad - HF) / stride + 1 121 | WO = (W + 2 * pad - WF) / stride + 1 122 | else: 123 | HO = (H + pad[0] + pad[1] - HF) / stride + 1 124 | WO = (W + pad[2] + pad[3] - WF) / stride + 1 125 | x_col = im2col(x_reshaped, HF, WF, pad, stride) 126 | x_col_argmax = np.argmax(x_col, axis=0) 127 | x_col_max = x_col[x_col_argmax, np.arange(x_col.shape[1])] 128 | out = x_col_max.reshape((HO, WO, D, N)) 129 | return out 130 | 131 | def max_pooling_backward(x, dout, pool_params): 132 | H, W, D, N = x.shape 133 | x_reshaped = x.reshape(H, W, 1, -1) 134 | x_col = im2col(x_reshaped, pool_params['HF'], 135 | pool_params['WF'], pool_params['pad'], pool_params['stride']) 136 | x_col_argmax = np.argmax(x_col, axis=0) 137 | dx_col = np.zeros_like(x_col) 138 | dx_col[x_col_argmax, np.arange(x_col.shape[1])] = dout.ravel() 139 | dx_shaped = col2im(dx_col, x_reshaped.shape, pool_params['HF'], pool_params['WF'], 140 | pool_params['pad'], stride=pool_params['stride']) 141 | dx = dx_shaped.reshape(x.shape) 142 | return [dx] 143 | 144 | def relu_forward(x): 145 | out = np.where(x > 0, x, 0) 146 | return out 147 | 148 | def relu_backward(x, dout): 149 | dx = np.where(x > 0, dout, 0) 150 | return [dx] 151 | 152 | def softmax_loss_forward(x, y): 153 | # x is the prediction(C * N), y is the label(1 * N) 154 | x_reshaped = x.reshape((x.shape[2], x.shape[3])) 155 | probs = np.exp(x_reshaped - np.max(x_reshaped, axis=0, keepdims=True)) 156 | probs /= np.sum(probs, axis=0, keepdims=True) 157 | N = x_reshaped.shape[1] 158 | loss = -np.sum(np.log(probs[y, np.arange(N)])) / N 159 | return loss 160 | 161 | def softmax_loss_backward(x, y): 162 | x_reshaped = x.reshape((x.shape[2], x.shape[3])) 163 | probs = np.exp(x_reshaped - np.max(x_reshaped, axis=0, keepdims=True)) 164 | probs /= np.sum(probs, axis=0, keepdims=True) 165 | dx = probs.copy() 166 | N = x_reshaped.shape[1] 167 | dx[y, np.arange(N)] -= 1 168 | dx /= N 169 | dx = dx.reshape((1, 1, dx.shape[0], dx.shape[1])) 170 | return [dx] 171 | 172 | def softmax(x): 173 | x_reshaped = x.reshape((x.shape[2], x.shape[3])) 174 | probs = np.exp(x_reshaped - np.max(x_reshaped, axis=0, keepdims=True)) 175 | probs /= np.sum(probs, axis=0, keepdims=True) 176 | return np.argmax(probs, axis=0) 177 | -------------------------------------------------------------------------------- /python/download_cifar.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | 3 | import urllib 4 | import sys 5 | import tarfile 6 | 7 | def call_back(num, size, all): 8 | per = 100.0 * num * size / all 9 | if per > 100: 10 | per = 100 11 | sys.stdout.write('\r%f%% completed' % per) 12 | sys.stdout.flush() 13 | 14 | def untar(file_name, dirs): 15 | t = tarfile.open(file_name) 16 | t.extractall(path=dirs) 17 | 18 | def download_cifar(): 19 | # download cifar dataset 20 | url = 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz' 21 | local = '../data/cifar/cifar-10-python.tar.gz' 22 | urllib.urlretrieve(url, local, call_back) 23 | untar(local, '../data/cifar') 24 | -------------------------------------------------------------------------------- /python/download_mnist.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | 3 | import urllib 4 | import sys 5 | 6 | def call_back(num, size, all): 7 | per = 100.0 * num * size / all 8 | if per > 100: 9 | per = 100 10 | sys.stdout.write('\r%f%% completed' % per) 11 | sys.stdout.flush() 12 | 13 | def download_mnist(): 14 | # download mnist dataset from lisa-lab website 15 | url = 'http://deeplearning.net/data/mnist/mnist.pkl.gz' 16 | local = '../data/mnist/mnist.pkl.gz' 17 | urllib.urlretrieve(url, local, call_back) 18 | -------------------------------------------------------------------------------- /python/interfaces/layers.cpp: -------------------------------------------------------------------------------- 1 | # include 2 | # include 3 | # include 4 | # include 5 | # include 6 | 7 | # include "convolution_layer.hpp" 8 | # include "pooling_layer.hpp" 9 | # include "relu_layer.hpp" 10 | # include "Tensor.hpp" 11 | 12 | typedef std::vector TensorVec; 13 | 14 | void init_from_numpy(Tensor& tensor_input, boost::python::object numpy_data) { 15 | PyArrayObject* data_arr = reinterpret_cast(numpy_data.ptr()); 16 | float* f_arr = static_cast(PyArray_DATA(data_arr)); 17 | if (!(PyArray_FLAGS(data_arr) & NPY_ARRAY_C_CONTIGUOUS)) { 18 | std::cout << "Input numpy data is not C contiguous" << std::endl; 19 | exit(-1); 20 | } 21 | int tensor_size = tensor_input.get_N() * tensor_input.get_C() * 22 | tensor_input.get_H() * tensor_input.get_W(); 23 | boost::shared_array data_ptr(new float[tensor_size]); 24 | std::copy(f_arr, f_arr + tensor_size, data_ptr.get()); 25 | tensor_input.set_data(data_ptr); 26 | } 27 | 28 | void return_numpy(Tensor& tensor_input, boost::python::object numpy_data) { 29 | PyArrayObject* data_arr = reinterpret_cast(numpy_data.ptr()); 30 | float* f_arr = static_cast(PyArray_DATA(data_arr)); 31 | if (!(PyArray_FLAGS(data_arr) & NPY_ARRAY_C_CONTIGUOUS)) { 32 | std::cout << "Input numpy data is not C contiguous" << std::endl; 33 | exit(-1); 34 | } 35 | boost::shared_array data_ptr = tensor_input.get_data(); 36 | int tensor_size = tensor_input.get_N() * tensor_input.get_C() * 37 | tensor_input.get_H() * tensor_input.get_W(); 38 | std::copy(data_ptr.get(), data_ptr.get() + tensor_size, f_arr); 39 | } 40 | 41 | BOOST_PYTHON_MODULE(pylayer) 42 | { 43 | using namespace boost::python; 44 | class_ >("PyFloatVector") 45 | .def(vector_indexing_suite, true>()); 46 | class_("PyTensor", init()) 47 | .def(init()) 48 | .def("init_from_numpy", &init_from_numpy) 49 | .def("return_numpy", &return_numpy) 50 | .add_property("N", &Tensor::get_N) 51 | .add_property("C", &Tensor::get_C) 52 | .add_property("H", &Tensor::get_H) 53 | .add_property("W", &Tensor::get_W); 54 | class_("PyTensorVec") 55 | .def(vector_indexing_suite >()); 56 | class_("PyConvolutionLayer", 57 | init()) 58 | .def("forward", &ConvolutionLayer::forward) 59 | .def("backward", &ConvolutionLayer::backward); 60 | class_("PyPoolingLayer", 61 | init()) 62 | .def("forward", &PoolingLayer::forward) 63 | .def("backward", &PoolingLayer::backward); 64 | class_("PyReLULayer") 65 | .def("forward", &ReLULayer::forward) 66 | .def("forward", &ReLULayer::backward); 67 | } 68 | -------------------------------------------------------------------------------- /python/interfaces/pylayer_test.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pylayer 3 | 4 | input_data = np.ones([200, 3, 50, 50], dtype=np.float32) 5 | input_tensor = pylayer.PyTensor(200, 3, 50, 50) 6 | input_tensor.init_from_numpy(input_data) 7 | 8 | filter_data = np.ones([2, 3, 3, 3], dtype=np.float32) 9 | filter_tensor = pylayer.PyTensor(2, 3, 3, 3) 10 | filter_tensor.init_from_numpy(filter_data) 11 | 12 | bias_data = np.ones([2, 1, 1, 1], dtype=np.float32) 13 | bias_tensor = pylayer.PyTensor(2, 1, 1, 1) 14 | bias_tensor.init_from_numpy(bias_data) 15 | 16 | output_data = np.ones([200, 2, 25, 25], dtype=np.float32) 17 | output_tensor = pylayer.PyTensor(200, 2, 25, 25) 18 | output_tensor.init_from_numpy(output_data) 19 | 20 | input_vec = pylayer.PyTensorVec() 21 | input_vec[:] = [input_tensor, filter_tensor, bias_tensor] 22 | 23 | output_vec = pylayer.PyTensorVec() 24 | output_vec[:] = [output_tensor] 25 | 26 | L1 = pylayer.PyConvolutionLayer(1, 1, 3, 3, 2, 2) 27 | L1.forward(input_vec, output_vec) 28 | output_vec[0].return_numpy(output_data) 29 | 30 | print output_data 31 | -------------------------------------------------------------------------------- /python/load_cifar.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | import os 4 | import cPickle 5 | import numpy as np 6 | import scipy.io as sio 7 | import download_cifar 8 | 9 | from sklearn.cross_validation import train_test_split 10 | 11 | def unpickle(file): 12 | fo = open(file, 'rb') 13 | dict = cPickle.load(fo) 14 | fo.close() 15 | return dict 16 | 17 | def load_data(): 18 | if os.path.exists('../data/cifar/cifar-10-python.tar.gz'): 19 | pass 20 | else: 21 | print 'download cifar dataset...' 22 | download_cifar.download_cifar() 23 | data = [] 24 | labels = [] 25 | for batch_idx in np.arange(5): 26 | data_path = '../data/cifar/cifar-10-batches-py/data_batch_%d' % (batch_idx + 1) 27 | data_batch = unpickle(data_path) 28 | sub_data = data_batch['data'] 29 | sub_labels = np.array(data_batch['labels']) 30 | if batch_idx == 0: 31 | data = sub_data 32 | labels = sub_labels 33 | else: 34 | data = np.vstack((data, sub_data)) 35 | labels = np.hstack((labels, sub_labels)) 36 | labels = labels.reshape((-1, 1)) 37 | return data, labels 38 | 39 | def load_cifar(): 40 | train_data, train_labels = load_data() 41 | train_data, valid_data, train_labels, valid_labels = train_test_split( 42 | train_data, train_labels, test_size=0.2, random_state=42) 43 | train_data = train_data.reshape((-1, 32, 32, 3)) 44 | train_data = np.transpose(train_data, (1, 2, 3, 0)) 45 | train_labels = np.transpose(train_labels, (1, 0)) 46 | valid_data = valid_data.reshape((-1, 32, 32, 3)) 47 | valid_data = np.transpose(valid_data, (1, 2, 3, 0)) 48 | valid_labels = np.transpose(valid_labels, (1, 0)) 49 | 50 | # remove mean 51 | print 'remove mean' 52 | mean_img = np.mean(train_data, axis=3, keepdims=True) 53 | train_data = train_data - mean_img 54 | # mean_img = np.mean(valid_data, axis=3, keepdims=True) 55 | valid_data = valid_data - mean_img 56 | 57 | # normalize by image mean and std 58 | print 'normalize by image mean and std' 59 | train_data = train_data.reshape((3072, -1)) 60 | valid_data = valid_data.reshape((3072, -1)) 61 | 62 | train_data = train_data - np.mean(train_data, axis=0, keepdims=True) 63 | valid_data = valid_data - np.mean(valid_data, axis=0, keepdims=True) 64 | 65 | train_std = np.std(train_data, axis=0, keepdims=True) 66 | train_data = train_data * np.mean(train_std) / np.maximum(train_std, 40 * np.ones_like(train_std)) 67 | train_data = train_data.reshape((32, 32, 3, -1)) 68 | 69 | valid_std = np.std(valid_data, axis=0, keepdims=True) 70 | valid_data = valid_data * np.mean(valid_std) / np.maximum(valid_std, 40 * np.ones_like(valid_std)) 71 | valid_data = valid_data.reshape((32, 32, 3, -1)) 72 | 73 | # # zca whitening 74 | # print 'zca whitening' 75 | # train_data = train_data.reshape((3072, -1)) 76 | # valid_data = valid_data.reshape((3072, -1)) 77 | # train_cov = train_data.dot(train_data.T) / train_data.shape[1] 78 | # D, V = np.linalg.eig(train_cov) 79 | # # D = D.reshape((-1, 1)) 80 | # # d2 = np.diag(D) 81 | # en = np.sqrt(np.mean(D)) 82 | # train_data = V.dot(np.diag(en / np.maximum(np.sqrt(D), 10 * np.ones_like(D)))).dot(V.T).dot(train_data) 83 | # valid_data = V.dot(np.diag(en / np.maximum(np.sqrt(D), 10 * np.ones_like(D)))).dot(V.T).dot(valid_data) 84 | # train_data = train_data.reshape((32, 32, 3, -1)) 85 | # valid_data = valid_data.reshape((32, 32, 3, -1)) 86 | 87 | # save to file 88 | sio.savemat('../data/cifar/cifar.mat', {'train_data': train_data, 89 | 'valid_data': valid_data, 90 | 'train_labels': train_labels, 91 | 'valid_labels': valid_labels}) 92 | print 'preprocess done' 93 | return train_data, valid_data, train_labels, valid_labels 94 | -------------------------------------------------------------------------------- /python/load_mnist.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | import os 4 | import numpy as np 5 | import cPickle 6 | import gzip 7 | import download_mnist 8 | 9 | def load_mnist(): 10 | if os.path.exists('../data/mnist/mnist.pkl.gz'): 11 | pass 12 | else: 13 | print 'download mnist dataset...' 14 | download_mnist.download_mnist() 15 | # Load the dataset 16 | f = gzip.open('../data/mnist/mnist.pkl.gz', 'rb') 17 | train_set, valid_set, test_set = cPickle.load(f) 18 | f.close() 19 | train_data = train_set[0] 20 | train_data = np.reshape(train_data, (-1, 28, 28, 1)) 21 | train_data = np.transpose(train_data, (1, 2, 3, 0)) * 255 22 | train_label = train_set[1] 23 | train_label = np.reshape(train_label, (1, -1)) 24 | 25 | valid_data = valid_set[0] 26 | valid_data = np.reshape(valid_data, (-1, 28, 28, 1)) 27 | valid_data = np.transpose(valid_data, (1, 2, 3, 0)) * 255 28 | valid_label = valid_set[1] 29 | valid_label = np.reshape(valid_label, (1, -1)) 30 | return train_data, train_label, valid_data, valid_label 31 | -------------------------------------------------------------------------------- /python/mnist-demo.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | import numpy as np 4 | 5 | from load_mnist import load_mnist 6 | from model.initial_LeNet import initial_LeNet 7 | 8 | train_data, train_label, valid_data, valid_label = load_mnist() 9 | cnn = initial_LeNet() 10 | lr = [0.01, 0.001] 11 | cnn.train(train_data, train_label, lr, epoch=2, batch_size=100) 12 | 13 | res = cnn.predict(valid_data) 14 | res = res.reshape(valid_label.shape) 15 | 16 | print 'Accuracy is: %f' % (np.sum(res == valid_label) / float(np.max(valid_label.shape))) 17 | -------------------------------------------------------------------------------- /python/model/__init__.py: -------------------------------------------------------------------------------- 1 | # empty file 2 | -------------------------------------------------------------------------------- /python/model/initial_LeNet.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append("..") 3 | from cnn.conv_net import ConvNet 4 | 5 | def initial_LeNet(): 6 | # initial LeNet 7 | cnn = ConvNet() 8 | 9 | conv1_params = {'HF': 5, 'WF': 5, 'DF': 1, 'NF': 20, 'stride': 1, 'pad': 0, 'var': 0.01} 10 | cnn.add_layer('conv', conv1_params) 11 | 12 | pooling1_params = {'HF': 2, 'WF': 2, 'stride': 2, 'pad': 0} 13 | cnn.add_layer('max_pooling', pooling1_params) 14 | 15 | conv2_params = {'HF': 5, 'WF': 5, 'DF': 20, 'NF': 50, 'stride': 1, 'pad': 0, 'var': 0.01} 16 | cnn.add_layer('conv', conv2_params) 17 | 18 | pooling2_params = {'HF': 2, 'WF': 2, 'stride': 2, 'pad': 0} 19 | cnn.add_layer('max_pooling', pooling2_params) 20 | 21 | conv3_params = {'HF': 4, 'WF': 4, 'DF': 50, 'NF': 500, 'stride': 1, 'pad': 0, 'var': 0.01} 22 | cnn.add_layer('conv', conv3_params) 23 | 24 | cnn.add_layer('relu', {}) 25 | 26 | conv4_params = {'HF': 1, 'WF': 1, 'DF': 500, 'NF': 10, 'stride': 1, 'pad': 0, 'var': 0.01} 27 | cnn.add_layer('conv', conv4_params) 28 | 29 | cnn.add_layer('softmax-loss', {}) 30 | 31 | return cnn 32 | -------------------------------------------------------------------------------- /python/model/initial_cifarNet.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append("..") 3 | from cnn.conv_net import ConvNet 4 | 5 | def initial_cifar(): 6 | # initial cifar net 7 | cnn = ConvNet() 8 | 9 | conv1_params = {'HF': 5, 'WF': 5, 'DF': 3, 'NF': 32, 'stride': 1, 'pad': 2, 'var': 0.01} 10 | cnn.add_layer('conv', conv1_params) 11 | pooling1_params = {'HF': 3, 'WF': 3, 'stride': 2, 'pad': [0, 1, 0, 1]} 12 | cnn.add_layer('max_pooling', pooling1_params) 13 | cnn.add_layer('relu', {}) 14 | 15 | conv2_params = {'HF': 5, 'WF': 5, 'DF': 32, 'NF': 32, 'stride': 1, 'pad': 2, 'var': 0.02} 16 | cnn.add_layer('conv', conv2_params) 17 | cnn.add_layer('relu', {}) 18 | pooling2_params = {'HF': 3, 'WF': 3, 'stride': 2, 'pad': [0, 1, 0, 1]} 19 | cnn.add_layer('max_pooling', pooling2_params) 20 | 21 | conv3_params = {'HF': 5, 'WF': 5, 'DF': 32, 'NF': 64, 'stride': 1, 'pad': 2, 'var': 0.03} 22 | cnn.add_layer('conv', conv3_params) 23 | cnn.add_layer('relu', {}) 24 | pooling3_params = {'HF': 3, 'WF': 3, 'stride': 2, 'pad': [0, 1, 0, 1]} 25 | cnn.add_layer('max_pooling', pooling3_params) 26 | 27 | conv4_params = {'HF': 4, 'WF': 4, 'DF': 64, 'NF': 64, 'stride': 1, 'pad': 0, 'var': 0.04} 28 | cnn.add_layer('conv', conv4_params) 29 | cnn.add_layer('relu', {}) 30 | 31 | conv5_params = {'HF': 1, 'WF': 1, 'DF': 64, 'NF': 10, 'stride': 1, 'pad': 0, 'var': 0.05} 32 | cnn.add_layer('conv', conv5_params) 33 | 34 | cnn.add_layer('softmax-loss', {}) 35 | 36 | return cnn 37 | -------------------------------------------------------------------------------- /python/rnn/naive_rnn.py: -------------------------------------------------------------------------------- 1 | 2 | class RNN: 3 | def __init__(self): 4 | self.layers = [] 5 | 6 | def add_layer(self, layer_params): 7 | 8 | -------------------------------------------------------------------------------- /python/util/gaussian_kernel.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.stats as st 3 | 4 | # this function is used to get gaussian 2d kernel 5 | 6 | def gkern(kernlen=5, nsig=1): 7 | # Returns a 2D Gaussian kernel array 8 | interval = (2*nsig+1.) / kernlen 9 | x = np.linspace(-nsig-interval / 2., nsig+interval / 2., kernlen + 1) 10 | kern1d = np.diff(st.norm.cdf(x)) 11 | kernel_raw = np.sqrt(np.outer(kern1d, kern1d)) 12 | kernel = kernel_raw / kernel_raw.sum() 13 | return kernel 14 | -------------------------------------------------------------------------------- /python/util/test_layers.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | 3 | # reading mnist dataset 4 | import cPickle 5 | import gzip 6 | import matplotlib.pyplot as plt 7 | import matplotlib.cm as cm 8 | import numpy as np 9 | 10 | import sys 11 | sys.path.append("..") 12 | from gaussian_kernel import gkern 13 | from cnn.layers import conv_forward 14 | from cnn.layers import conv_backward 15 | from cnn.layers import max_pooling_forward 16 | from cnn.layers import max_pooling_backward 17 | from cnn.layers import relu_forward 18 | from cnn.layers import relu_backward 19 | from cnn.layers import softmax_loss_forward 20 | from cnn.layers import softmax_loss_backward 21 | 22 | from util import numerical_gradient 23 | from util import numerical_gradient_loss 24 | from util import rel_error 25 | # Load the dataset 26 | f = gzip.open('../../data/mnist/mnist.pkl.gz', 'rb') 27 | train_set, valid_set, test_set = cPickle.load(f) 28 | f.close() 29 | # train_set, valid_set, test_set are all tuple with two elements: data and label 30 | 31 | # select one image and test the conv_forward layer by gaussian filtering 32 | im = train_set[0][0, :] 33 | im = im.reshape((28, 28)) * 255 34 | plt.subplot(1, 2, 1) 35 | plt.imshow(im.astype('uint8'), cmap=cm.Greys_r) 36 | 37 | g_k = gkern(4, 1) 38 | im = im.reshape(28, 28, 1, 1) 39 | filter = g_k.reshape(4, 4, 1, 1) 40 | b = np.zeros((1, 1, 1, 1)) 41 | params = {'stride': 2, 'pad': 2} 42 | res = conv_forward(im, filter, b, params) 43 | res = res.reshape(res.shape[0], res.shape[1]) 44 | plt.subplot(1, 2, 2) 45 | plt.imshow(res.astype('uint8'), cmap=cm.Greys_r) 46 | plt.show() 47 | 48 | # test conv_backward 49 | x = np.random.randn(5, 5, 3, 4) 50 | w = np.random.randn(3, 3, 3, 2) 51 | b = np.random.randn(1, 1, 1, 2) 52 | dout = np.random.randn(5, 5, 2, 4) 53 | conv_param = {'stride': 1, 'pad': 1} 54 | 55 | dx_num = numerical_gradient(lambda x: conv_forward(x, w, b, conv_param), x, dout) 56 | dw_num = numerical_gradient(lambda w: conv_forward(x, w, b, conv_param), w, dout) 57 | db_num = numerical_gradient(lambda b: conv_forward(x, w, b, conv_param), b, dout) 58 | 59 | out = conv_forward(x, w, b, conv_param) 60 | dx, dw, db = conv_backward(x, w, b, conv_param, dout) 61 | 62 | # Your errors should be around 1e-9' 63 | print 'Testing conv_backward function' 64 | print 'dx error: ', rel_error(dx, dx_num) 65 | print 'dw error: ', rel_error(dw, dw_num) 66 | print 'db error: ', rel_error(db, db_num) 67 | 68 | x_shape = (2, 3, 4, 4) 69 | x = np.linspace(-0.3, 0.4, num=np.prod(x_shape)).reshape(x_shape) 70 | pool_param = {'HF': 2, 'WF': 2, 'stride': 2} 71 | x = x.transpose(2, 3, 1, 0) 72 | out = max_pooling_forward(x, pool_param) 73 | 74 | correct_out = np.array([[[[-0.26315789, -0.24842105], 75 | [-0.20421053, -0.18947368]], 76 | [[-0.14526316, -0.13052632], 77 | [-0.08631579, -0.07157895]], 78 | [[-0.02736842, -0.01263158], 79 | [ 0.03157895, 0.04631579]]], 80 | [[[ 0.09052632, 0.10526316], 81 | [ 0.14947368, 0.16421053]], 82 | [[ 0.20842105, 0.22315789], 83 | [ 0.26736842, 0.28210526]], 84 | [[ 0.32631579, 0.34105263], 85 | [ 0.38526316, 0.4 ]]]]) 86 | correct_out = correct_out.transpose(2, 3, 1, 0) 87 | # Compare your output with ours. Difference should be around 1e-8. 88 | print 'Testing max_pooling_forward function:' 89 | print 'difference: ', rel_error(out, correct_out) 90 | 91 | x = np.random.randn(8, 8, 3, 2) 92 | dout = np.random.randn(4, 4, 3, 2) 93 | pool_param = {'HF': 2, 'WF': 2, 'stride': 2} 94 | 95 | dx_num = numerical_gradient(lambda x: max_pooling_forward(x, pool_param), x, dout) 96 | 97 | out = max_pooling_forward(x, pool_param) 98 | dx = max_pooling_backward(x, dout, pool_param) 99 | 100 | # Your error should be around 1e-12 101 | print 'Testing max_pool_backward_naive function:' 102 | print 'dx error: ', rel_error(dx, dx_num) 103 | 104 | x = np.linspace(-0.5, 0.5, num=12).reshape(3, 4) 105 | 106 | out = relu_forward(x) 107 | correct_out = np.array([[ 0., 0., 0., 0., ], 108 | [ 0., 0., 0.04545455, 0.13636364,], 109 | [ 0.22727273, 0.31818182, 0.40909091, 0.5, ]]) 110 | 111 | # Compare your output with ours. The error should be around 1e-8 112 | print 'Testing relu_forward function:' 113 | print 'difference: ', rel_error(out, correct_out) 114 | 115 | x = np.random.randn(10, 10) 116 | dout = np.random.randn(*x.shape) 117 | 118 | dx_num = numerical_gradient(lambda x: relu_forward(x), x, dout) 119 | 120 | dx = relu_backward(x, dout) 121 | 122 | # The error should be around 1e-12 123 | print 'Testing relu_backward function:' 124 | print 'dx error: ', rel_error(dx_num, dx) 125 | 126 | # num_classes, num_inputs = 10, 50 127 | # x = 0.001 * np.random.randn(num_classes, num_inputs) 128 | # y = np.random.randint(num_classes, size=num_inputs) 129 | # 130 | # dx_num = numerical_gradient_loss(lambda x: softmax_loss_forward(x, y), x, verbose=False) 131 | # loss = softmax_loss_forward(x, y) 132 | # dx = softmax_loss_backward(x, y) 133 | # 134 | # # Test softmax_loss function. Loss should be 2.3 and dx error should be 1e-8 135 | # print '\nTesting softmax_loss:' 136 | # print 'loss: ', loss 137 | # print 'dx error: ', rel_error(dx_num, dx) 138 | -------------------------------------------------------------------------------- /python/util/util.py: -------------------------------------------------------------------------------- 1 | # coding = utf-8 2 | 3 | import numpy as np 4 | 5 | # this two function are from stanford cs231n: http://cs231n.stanford.edu/syllabus.html 6 | # they are used to check gradient computing and get relative error 7 | 8 | def numerical_gradient(f, x, df, h=1e-5): 9 | # initialize grad 10 | grad = np.zeros_like(x) 11 | it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite']) 12 | while not it.finished: 13 | ix = it.multi_index 14 | oldval = x[ix] 15 | x[ix] = oldval + h 16 | pos = f(x) 17 | x[ix] = oldval - h 18 | neg = f(x) 19 | x[ix] = oldval 20 | 21 | grad[ix] = np.sum((pos - neg) * df) / (2 * h) 22 | it.iternext() 23 | return grad 24 | 25 | def rel_error(x, y): 26 | # returns relative error 27 | return np.max(np.abs(x - y) / (np.maximum(1e-8, np.abs(x) + np.abs(y)))) 28 | 29 | def numerical_gradient_loss(f, x, verbose=True, h=0.00001): 30 | # a naive implementation of numerical gradient of f at x 31 | # - f should be a function that takes a single argument 32 | # - x is the point (numpy array) to evaluate the gradient at 33 | 34 | fx = f(x) # evaluate function value at original point 35 | grad = np.zeros_like(x) 36 | # iterate over all indexes in x 37 | it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite']) 38 | while not it.finished: 39 | # evaluate function at x+h 40 | ix = it.multi_index 41 | oldval = x[ix] 42 | x[ix] = oldval + h # increment by h 43 | fxph = f(x) # evalute f(x + h) 44 | x[ix] = oldval - h 45 | fxmh = f(x) # evaluate f(x - h) 46 | x[ix] = oldval # restore 47 | 48 | # compute the partial derivative with centered formula 49 | grad[ix] = (fxph - fxmh) / (2 * h) # the slope 50 | if verbose: 51 | print ix, grad[ix] 52 | it.iternext() # step to next dimension 53 | 54 | return grad -------------------------------------------------------------------------------- /src/Net.cpp: -------------------------------------------------------------------------------- 1 | # include 2 | # include 3 | 4 | # include "Net.hpp" 5 | 6 | Net::Net() { 7 | this->lr = 0.0001; 8 | } 9 | 10 | void Net::set_lr(float lr) { 11 | this->lr = lr; 12 | } 13 | 14 | void Net::add_layer(std::shared_ptr l) { 15 | layers_.push_back(l); 16 | } 17 | 18 | void Net::add_param_layer_id(int idx) { 19 | this->params_layer_id.push_back(idx); 20 | } 21 | 22 | void Net::add_data(std::vector& t) { 23 | this->data_.push_back(t); 24 | } 25 | 26 | void Net::forward_net(int start, int end) { 27 | 28 | for (int layer_idx = start > 0 ? start:0; 29 | layer_idx < (end > 0 ? end:layers_.size()); ++layer_idx) { 30 | layers_[layer_idx]->forward(data_[layer_idx], data_[layer_idx + 1]); 31 | } 32 | } 33 | 34 | void Net::backward_net() { 35 | for (int layer_idx = layers_.size() - 1; layer_idx >= 0; --layer_idx) { 36 | layers_[layer_idx]->backward(data_[layer_idx], data_[layer_idx + 1]); 37 | } 38 | } 39 | 40 | void Net::params_update() { 41 | for(int idx = 0; idx < params_layer_id.size(); ++idx) { 42 | this->layers_[params_layer_id[idx]]->params_update(this->lr); 43 | } 44 | } 45 | 46 | void Net::train_batch(Tensor& train_data, Tensor& train_label) { 47 | // TODO 48 | // implement "=" operator of class Tensor 49 | this->data_[0][0] = train_data; 50 | this->data_[this->data_.size() - 2][2] = train_label; 51 | this->forward_net(); 52 | std::cout << "loss: " << *(data_[this->data_.size() - 1][0].get_data().get()) 53 | << " "; 54 | this->backward_net(); 55 | this->params_update(); 56 | } 57 | 58 | void Net::test_batch(Tensor &test_data, Tensor& pred_label) { 59 | this->data_[0][0] = test_data; 60 | this->forward_net(0, this->layers_.size() - 2); 61 | Tensor result = this->data_[this->data_.size() - 3][0]; 62 | int batch_size = result.get_N(); 63 | int channel_size = result.get_C(); 64 | assert(result.get_H() == 1 && result.get_W() == 1); 65 | 66 | float* res_ptr = result.get_data().get(); 67 | float* pred_label_ptr = pred_label.get_data().get(); 68 | for (int i = 0; i < batch_size; i++) { 69 | pred_label_ptr[i] = std::distance(res_ptr + channel_size * i, 70 | std::max_element(res_ptr + channel_size * i, 71 | res_ptr + channel_size * (i + 1))); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/blas_function.cpp: -------------------------------------------------------------------------------- 1 | # include 2 | #include 3 | 4 | # include "blas_function.hpp" 5 | 6 | void gemm(const CBLAS_TRANSPOSE TransA, const CBLAS_TRANSPOSE TransB, 7 | const int M, const int N, const int K, const float alpha, 8 | const float* A, const float* B, const float beta, float* C) { 9 | int lda = (TransA == CblasNoTrans) ? K:M; 10 | int ldb = (TransB == CblasNoTrans) ? N:K; 11 | cblas_sgemm(CblasRowMajor, TransA, TransB, M, N, K, alpha, A, lda, B, 12 | ldb, beta, C, N); 13 | } 14 | 15 | void vector_add(const float* A, const float* B, float* C, const float coeff_a, 16 | const float coeff_b, const int vector_size) { 17 | for(int i = 0; i < vector_size; ++i) { 18 | C[i] = A[i] * coeff_a + B[i] * coeff_b; 19 | } 20 | } 21 | 22 | void vector_sub(const float* A, const float* B, float* C, const int vector_size) { 23 | for(int i = 0; i < vector_size; ++i) { 24 | C[i] = A[i] - B[i]; 25 | } 26 | } 27 | 28 | void vector_mul(const float* A, const float* B, float* C, const int vector_size) { 29 | for(int i = 0; i < vector_size; ++i) { 30 | C[i] = A[i] * B[i]; 31 | } 32 | } 33 | 34 | void vector_exp(const float* A, float* B, const int vector_size) { 35 | for (int i = 0; i < vector_size; ++i) { 36 | B[i] = expf(A[i]); 37 | } 38 | } 39 | 40 | void vector_sub_scalar(float* A, float b, float* B, const int vector_size) { 41 | for (int i = 0; i < vector_size; ++i) { 42 | B[i] = A[i] - b; 43 | } 44 | } 45 | 46 | void vector_div_scalar(float* A, float b, const int vector_size) { 47 | for (int i = 0; i < vector_size; ++i) { 48 | A[i] = A[i] / b; 49 | } 50 | } 51 | 52 | void vector_mul_scalar(float* A, float b, const int vector_size) { 53 | for (int i = 0; i < vector_size; ++i) { 54 | A[i] = A[i] * b; 55 | } 56 | } 57 | 58 | float vector_sum(float* A, const int vector_size) { 59 | float sum = 0; 60 | for (int i = 0; i < vector_size; ++i) { 61 | sum += A[i]; 62 | } 63 | return sum; 64 | } 65 | 66 | void vector_scale(float* A, const int vector_size) { 67 | float max_item = *(std::max_element(A, A + vector_size)); 68 | float min_item = *(std::min_element(A, A + vector_size)); 69 | if (fabsf(max_item) < fabsf(min_item)) 70 | max_item = fabsf(min_item); 71 | if (max_item < 0.1) 72 | return; 73 | for (int i = 0; i < vector_size; ++i) { 74 | A[i] = A[i] / max_item; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/convolution_layer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "convolution_layer.hpp" 5 | #include "im2col.hpp" 6 | 7 | void ConvolutionLayer::forward(std::vector& input, 8 | std::vector& output ) { 9 | Tensor input_data(input[0]); 10 | Tensor output_data(output[0]); 11 | 12 | int N_in = input_data.get_N(); 13 | int height_in = input_data.get_H(); 14 | int width_in = input_data.get_W(); 15 | int channels_in = input_data.get_C(); 16 | 17 | int N_out = output_data.get_N(); 18 | int height_out = output_data.get_H(); 19 | int width_out = output_data.get_W(); 20 | int channels_out = output_data.get_C(); 21 | 22 | int N_filter = this->filter.get_N(); 23 | int channels_filter = this->filter.get_C(); 24 | 25 | assert(N_in == N_out); 26 | assert(channels_in==channels_filter); 27 | 28 | float* input_data_col = new float[kernel_h * kernel_w * channels_in * 29 | height_out * width_out]; 30 | float* input_data_ptr = input_data.get_data().get(); 31 | float* output_data_ptr = output_data.get_data().get(); 32 | 33 | float* ones_vector = new float[height_out * width_out]; 34 | std::fill(ones_vector, ones_vector + height_out * width_out, 1); 35 | for (int n = 0; n < N_in; n++) { 36 | im2col(input_data_ptr + n * channels_in * height_in * width_in, 37 | height_in, width_in, channels_in, kernel_h, 38 | kernel_w, pad_h, pad_w, stride_h, stride_w, input_data_col); 39 | gemm(CblasNoTrans, CblasNoTrans, N_filter, height_out * width_out, 40 | kernel_h * kernel_w * channels_in, 1, this->filter.get_data().get(), 41 | input_data_col, 0, 42 | output_data_ptr + n * channels_out * height_out * width_out); 43 | gemm(CblasNoTrans, CblasNoTrans, N_filter, height_out * width_out, 44 | 1, 1, this->bias.get_data().get(), ones_vector, 1, 45 | output_data_ptr + n * channels_out * height_out * width_out); 46 | 47 | } 48 | delete[] ones_vector; 49 | delete[] input_data_col; 50 | 51 | } 52 | 53 | void ConvolutionLayer::backward(std::vector& input, 54 | std::vector& output) { 55 | Tensor input_data(input[0]); 56 | Tensor d_input_data(input[1]); 57 | Tensor d_output_data(output[1]); 58 | 59 | int N_in = d_input_data.get_N(); 60 | int height_in = d_input_data.get_H(); 61 | int width_in = d_input_data.get_W(); 62 | int channels_in = d_input_data.get_C(); 63 | 64 | int N_filter = this->d_filter.get_N(); 65 | 66 | int height_out = d_output_data.get_H(); 67 | int width_out = d_output_data.get_W(); 68 | int channels_out = d_output_data.get_C(); 69 | 70 | float* d_input_data_ptr = d_input_data.get_data().get(); 71 | float* d_input_data_col_ptr = new float[kernel_h * kernel_w * channels_in * 72 | height_out * width_out]; 73 | float* input_data_ptr = input_data.get_data().get(); 74 | float* input_data_col_ptr = new float[kernel_h * kernel_w * channels_in * 75 | height_out * width_out]; 76 | float* d_output_date_ptr = d_output_data.get_data().get(); 77 | 78 | float* ones_vector = new float[height_out * width_out]; 79 | std::fill(ones_vector, ones_vector + height_out * width_out, 1); 80 | 81 | for (int n = 0; n < N_in; n++) { 82 | gemm(CblasTrans, CblasNoTrans, kernel_h * kernel_w * channels_in, 83 | height_out * width_out, N_filter, 1, this->filter.get_data().get(), 84 | d_output_date_ptr + n * channels_out * height_out * width_out, 85 | 0, d_input_data_col_ptr); 86 | col2im(d_input_data_col_ptr, height_in, width_in, channels_in, 87 | kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, 88 | d_input_data_ptr + n * channels_in * height_in * width_in); 89 | 90 | im2col(input_data_ptr + n * channels_in * height_in * width_in, 91 | height_in, width_in, channels_in, kernel_h, kernel_w, 92 | pad_h, pad_w, stride_h, stride_w, input_data_col_ptr); 93 | // accumulate the gradient each iteration 94 | gemm(CblasNoTrans, CblasTrans, N_filter, 95 | kernel_h * kernel_w * channels_in, height_out * width_out, 1, 96 | d_output_date_ptr + n * channels_out * height_out * width_out, 97 | input_data_col_ptr, 1, this->d_filter.get_data().get()); 98 | gemm(CblasNoTrans, CblasNoTrans, N_filter, 1, 99 | height_out * width_out, 1, 100 | d_output_date_ptr + n * channels_out * height_out * width_out, 101 | ones_vector, 1, this->d_bias.get_data().get()); 102 | } 103 | 104 | delete[] d_input_data_col_ptr; 105 | delete[] input_data_col_ptr; 106 | delete[] ones_vector; 107 | } 108 | 109 | void ConvolutionLayer::params_update(float lr) { 110 | vector_scale(this->d_filter.get_data().get(), this->d_filter.get_size()); 111 | vector_scale(this->d_bias.get_data().get(), this->d_bias.get_size()); 112 | 113 | this->pre_update_f.add_Tensor(this->d_filter, 0.9, -lr); 114 | this->filter.add_Tensor(this->pre_update_f, 1, 1); 115 | this->pre_update_b.add_Tensor(this->d_bias, 0.9, -2 * lr); 116 | this->bias.add_Tensor(this->pre_update_b, 1, 1); 117 | } -------------------------------------------------------------------------------- /src/log_loss_layer.cpp: -------------------------------------------------------------------------------- 1 | # include 2 | # include 3 | 4 | # include "log_loss_layer.hpp" 5 | 6 | void LogLoss::params_update(float lr) { 7 | // empty 8 | } 9 | void LogLoss::forward(std::vector &input, std::vector &output) { 10 | Tensor input_data(input[0]); 11 | Tensor label(input[2]); 12 | Tensor loss(output[0]); 13 | Tensor output_data(output[1]); 14 | 15 | 16 | float* input_data_ptr = input_data.get_data().get(); 17 | float* label_ptr = label.get_data().get(); 18 | float* output_data_ptr = output_data.get_data().get(); 19 | float* loss_ptr = loss.get_data().get(); 20 | 21 | loss_ptr[0] = 0; 22 | 23 | int N_in = input_data.get_N(); 24 | int C_in = input_data.get_C(); 25 | int H_in = input_data.get_H(); 26 | int W_in = input_data.get_W(); 27 | int size_in = C_in * H_in * W_in; 28 | 29 | assert(W_in == 1 && H_in == 1); 30 | 31 | for( int n = 0; n < N_in; ++n) { 32 | float gndth_prob = -logf(*(input_data_ptr + n * size_in + (int)(label_ptr[n])) + 1e-6); 33 | output_data_ptr[n] = *(input_data_ptr + n * size_in + (int)(label_ptr[n])) + 1e-6; 34 | loss_ptr[0] += gndth_prob / N_in; 35 | } 36 | } 37 | 38 | void LogLoss::backward(std::vector &input, 39 | std::vector &output) { 40 | Tensor output_data(output[1]); 41 | Tensor label(input[2]); 42 | Tensor d_input_data(input[1]); 43 | 44 | float* output_data_ptr = output_data.get_data().get(); 45 | float* label_ptr = label.get_data().get(); 46 | float* d_input_data_ptr = d_input_data.get_data().get(); 47 | 48 | int N_in = d_input_data.get_N(); 49 | int C_in = d_input_data.get_C(); 50 | int H_in = d_input_data.get_H(); 51 | int W_in = d_input_data.get_W(); 52 | int data_size = C_in * H_in * W_in; 53 | 54 | assert(W_in == 1 && H_in == 1); 55 | std::fill_n(d_input_data_ptr, d_input_data.get_size(), 0); 56 | for(int n = 0; n < N_in; ++n) { 57 | *(d_input_data_ptr + n * data_size + (int)(label_ptr[n])) = -1 / output_data_ptr[n] / N_in; 58 | } 59 | } -------------------------------------------------------------------------------- /src/pooling_layer.cpp: -------------------------------------------------------------------------------- 1 | # include 2 | # include 3 | # include 4 | # include 5 | 6 | void PoolingLayer::params_update(float lr) { 7 | // empty 8 | } 9 | 10 | void PoolingLayer::forward(std::vector &input, std::vector &output) { 11 | Tensor input_data(input[0]); 12 | Tensor output_data(output[0]); 13 | 14 | int N_in = input_data.get_N(); 15 | int C_in = input_data.get_C(); 16 | int H_in = input_data.get_H(); 17 | int W_in = input_data.get_W(); 18 | 19 | int N_out = output_data.get_N(); 20 | int C_out = output_data.get_C(); 21 | int H_out = output_data.get_H(); 22 | int W_out = output_data.get_W(); 23 | 24 | float* input_data_ptr = input_data.get_data().get(); 25 | float* output_data_ptr = output_data.get_data().get(); 26 | int* output_data_index = new int[N_out * C_out * H_out * W_out]; 27 | memset(output_data_index, 0, sizeof(int) * N_out * C_out * H_out * W_out); 28 | for (int n = 0; n < N_in; ++n) { 29 | for(int c = 0; c < C_in; ++c) { 30 | for(int h = -pad_h; h < H_in + pad_h - kernel_h; h += stride_h) { 31 | for(int w = -pad_w; w < W_in + pad_w - kernel_w; w += stride_w) { 32 | float current_max = -FLT_MAX; 33 | int max_index = 0; 34 | int base_pos = n * C_in * H_in * W_in + 35 | c * H_in * W_in + h * W_in + w; 36 | for(int bias_h = 0; bias_h < kernel_h; ++bias_h) { 37 | for(int bias_w = 0; bias_w < kernel_w; ++bias_w) { 38 | if(h + bias_h < 0 || w + bias_w < 0 || h + bias_h > H_in || 39 | w + bias_w > W_in) { 40 | continue; 41 | } 42 | int bias_pos = bias_h * W_in + bias_w; 43 | if(input_data_ptr[base_pos + bias_pos] > current_max) { 44 | max_index = bias_pos; 45 | current_max = input_data_ptr[base_pos + bias_pos]; 46 | } 47 | } 48 | } 49 | int dest_pos = n * C_out * H_out * W_out + 50 | c * H_out * W_out + (h + pad_h) / stride_h * W_out + 51 | (w + pad_w) / stride_w; 52 | output_data_ptr[dest_pos] = current_max; 53 | output_data_index[dest_pos] = max_index; 54 | } 55 | } 56 | } 57 | } 58 | this->max_index_mask.assign(output_data_index, output_data_index + N_out * C_out * H_out * W_out); 59 | delete[] output_data_index; 60 | } 61 | 62 | void PoolingLayer::backward(std::vector &input, std::vector &output) { 63 | Tensor d_input_data(input[1]); 64 | Tensor d_output_data(output[1]); 65 | 66 | int N_in = d_input_data.get_N(); 67 | int C_in = d_input_data.get_C(); 68 | int H_in = d_input_data.get_H(); 69 | int W_in = d_input_data.get_W(); 70 | 71 | int N_out = d_output_data.get_N(); 72 | int C_out = d_output_data.get_C(); 73 | int H_out = d_output_data.get_H(); 74 | int W_out = d_output_data.get_W(); 75 | 76 | float* d_input_data_ptr = d_input_data.get_data().get(); 77 | memset(d_input_data_ptr, 0, N_in * C_in * H_in * W_in * sizeof(float)); 78 | float* d_output_data_ptr = d_output_data.get_data().get(); 79 | 80 | for (int n = 0; n < N_out; ++n) { 81 | for(int c = 0; c < C_out; ++c) { 82 | for(int h = 0; h < H_out; ++h) { 83 | for(int w = 0; w < W_out; ++w) { 84 | int original_base_pos = n * C_in * H_in * W_in + 85 | c * H_in * W_in + (h * stride_h - pad_h) * H_in + 86 | w * stride_w - pad_w; 87 | int original_bias_pos = this->max_index_mask[n * C_out * H_out * W_out + 88 | c * H_out * W_out + h * W_out + w]; 89 | d_input_data_ptr[original_base_pos + original_bias_pos] += 90 | d_output_data_ptr[n * C_out * H_out * W_out + 91 | c * H_out * W_out + h * W_out + w]; 92 | } 93 | } 94 | } 95 | } 96 | } 97 | 98 | -------------------------------------------------------------------------------- /src/rand_function.cpp: -------------------------------------------------------------------------------- 1 | # include 2 | # include "rand_function.hpp" 3 | 4 | void gaussrand(float mean, float stddev, float* rand_array, int array_len) { 5 | std::default_random_engine generator; 6 | std::normal_distribution distribution(mean, stddev); 7 | for(int idx = 0; idx < array_len; ++idx) { 8 | rand_array[idx] = distribution(generator); 9 | } 10 | } 11 | 12 | -------------------------------------------------------------------------------- /src/relu_layer.cpp: -------------------------------------------------------------------------------- 1 | # include "relu_layer.hpp" 2 | 3 | void ReLULayer::params_update(float lr) { 4 | // empty 5 | } 6 | 7 | void ReLULayer::forward(std::vector &input, std::vector &output) { 8 | Tensor input_data(input[0]); 9 | Tensor output_data(output[0]); 10 | 11 | float* input_data_ptr = input_data.get_data().get(); 12 | float* output_data_ptr = output_data.get_data().get(); 13 | 14 | int N_in = input_data.get_N(); 15 | int C_in = input_data.get_C(); 16 | int H_in = input_data.get_H(); 17 | int W_in = input_data.get_W(); 18 | int data_size = N_in * C_in * H_in * W_in; 19 | for(int index = 0; index < data_size; ++index) { 20 | output_data_ptr[index] = input_data_ptr[index] > 0 ? input_data_ptr[index] : 0; 21 | } 22 | } 23 | 24 | void ReLULayer::backward(std::vector &input, std::vector &output) { 25 | Tensor input_data(input[0]); 26 | Tensor d_input_data(input[1]); 27 | Tensor d_output_data(output[1]); 28 | 29 | float* input_data_ptr = input_data.get_data().get(); 30 | float* d_input_data_ptr = d_input_data.get_data().get(); 31 | float* d_output_data_ptr = d_output_data.get_data().get(); 32 | int data_size = d_input_data.get_size(); 33 | 34 | for(int index = 0; index < data_size; ++index) { 35 | d_input_data_ptr[index] = input_data_ptr[index] > 0 ? d_output_data_ptr[index] : 0; 36 | } 37 | } 38 | 39 | -------------------------------------------------------------------------------- /src/softmax_layer.cpp: -------------------------------------------------------------------------------- 1 | # include 2 | # include 3 | # include 4 | 5 | # include "blas_function.hpp" 6 | # include "softmax_layer.hpp" 7 | # include "Tensor.hpp" 8 | 9 | void SoftmaxLayer::params_update(float lr) { 10 | // empty 11 | } 12 | 13 | void SoftmaxLayer::forward(std::vector &input, std::vector &output) { 14 | Tensor input_data(input[0]); 15 | Tensor output_data(output[0]); 16 | 17 | float* input_data_ptr = input_data.get_data().get(); 18 | float* output_data_ptr = output_data.get_data().get(); 19 | 20 | int N_in = input_data.get_N(); 21 | int C_in = input_data.get_C(); 22 | int H_in = input_data.get_H(); 23 | int W_in = input_data.get_W(); 24 | int size_in = C_in * H_in * W_in; 25 | 26 | assert(W_in == 1 && H_in == 1); 27 | 28 | for( int n = 0; n < N_in; ++n) { 29 | float max_ele = *(std::max_element(input_data_ptr + n * size_in, 30 | input_data_ptr + (n + 1) * size_in)); 31 | vector_sub_scalar(input_data_ptr + n * size_in, max_ele, 32 | output_data_ptr + n * size_in, size_in); 33 | 34 | vector_exp(output_data_ptr + n * size_in, 35 | output_data_ptr + n * size_in, size_in); 36 | 37 | float sum_exp = vector_sum(output_data_ptr + n * size_in, size_in); 38 | 39 | vector_div_scalar(output_data_ptr + n * size_in, 40 | sum_exp, size_in); 41 | } 42 | } 43 | 44 | void SoftmaxLayer::backward(std::vector &input, 45 | std::vector &output) { 46 | Tensor d_input_data(input[1]); 47 | Tensor output_data(output[0]); 48 | Tensor d_output_data(output[1]); 49 | 50 | float* output_data_ptr = output_data.get_data().get(); 51 | float* d_input_data_ptr = d_input_data.get_data().get(); 52 | float* d_output_data_ptr = d_output_data.get_data().get(); 53 | 54 | int N_in = d_input_data.get_N(); 55 | int C_in = d_input_data.get_C(); 56 | int H_in = d_input_data.get_H(); 57 | int W_in = d_input_data.get_W(); 58 | int data_size = C_in * H_in * W_in; 59 | 60 | assert(W_in == 1 && H_in == 1); 61 | 62 | for(int n = 0; n < N_in; ++n) { 63 | vector_mul(d_output_data_ptr + n * data_size, 64 | output_data_ptr + n * data_size, 65 | d_input_data_ptr + n * data_size, data_size); 66 | float dy_dot_y = 0; 67 | gemm(CblasNoTrans, CblasNoTrans, 1, 1, data_size, 1, 68 | d_output_data_ptr + n * data_size, 69 | output_data_ptr + n * data_size, 0, &dy_dot_y); 70 | gemm(CblasNoTrans, CblasNoTrans, 1, data_size, 1, -1, 71 | &dy_dot_y, output_data_ptr + n * data_size, 1, 72 | d_input_data_ptr + n * data_size); 73 | } 74 | } -------------------------------------------------------------------------------- /test/test_conv_layer.cpp: -------------------------------------------------------------------------------- 1 | # include 2 | # include 3 | #include 4 | #include 5 | # include "Tensor.hpp" 6 | # include "convolution_layer.hpp" 7 | 8 | using namespace std; 9 | 10 | void read_matrix(const char* filename, float* data) { 11 | string value; 12 | float f_value; 13 | int num = 0; 14 | 15 | //read data 16 | ifstream data_file(filename); 17 | if (!data_file.is_open()) { 18 | cout << filename << " NOT FOUND" << endl; 19 | exit(-1); 20 | } 21 | while(data_file.good()) { 22 | getline(data_file, value, ','); 23 | f_value = stof(value); 24 | *(data + num) = f_value; 25 | ++num; 26 | } 27 | } 28 | 29 | void get_sample_data(float* X, float* F, float* B, float* Y, 30 | float* d_X, float* d_F, float* d_B, float* d_Y) { 31 | read_matrix("../../test/sample_data/Conv/X.csv", X); 32 | read_matrix("../../test/sample_data/Conv/F.csv", F); 33 | read_matrix("../../test/sample_data/Conv/B.csv", B); 34 | read_matrix("../../test/sample_data/Conv/Y.csv", Y); 35 | read_matrix("../../test/sample_data/Conv/d_X.csv", d_X); 36 | read_matrix("../../test/sample_data/Conv/d_F.csv", d_F); 37 | read_matrix("../../test/sample_data/Conv/d_B.csv", d_B); 38 | read_matrix("../../test/sample_data/Conv/d_Y.csv", d_Y); 39 | } 40 | 41 | bool check_eq(float* pred, float* gndth, int size) { 42 | for(int iter = 0; iter < size; ++iter) { 43 | if(fabs(pred[iter] - gndth[iter]) / 44 | fabs(gndth[iter] + 1e-6) > 0.1) { 45 | return false; 46 | } 47 | } 48 | return true; 49 | } 50 | 51 | void test_conv_layer() { 52 | Tensor input_tensor(100, 32, 16, 16); 53 | Tensor d_input_tensor_pred(100, 32, 16, 16); 54 | Tensor d_input_tensor_gndth(100, 32, 16, 16); 55 | 56 | Tensor filter_tensor(32, 32, 5, 5); 57 | Tensor d_filter_tensor_pred(32, 32, 5, 5); 58 | Tensor d_filter_tensor_gndth(32, 32, 5, 5); 59 | 60 | Tensor bias_tensor(32, 1, 1, 1); 61 | Tensor d_bias_tensor_pred(32, 1, 1, 1); 62 | Tensor d_bias_tensor_gndth(32, 1, 1, 1); 63 | 64 | Tensor output_tensor_pred(100, 32, 8, 8); 65 | Tensor output_tensor_gndth(100, 32, 8, 8); 66 | Tensor d_output_tensor(100, 32, 8, 8); 67 | 68 | get_sample_data(input_tensor.get_data().get(), filter_tensor.get_data().get(), 69 | bias_tensor.get_data().get(), output_tensor_gndth.get_data().get(), 70 | d_input_tensor_gndth.get_data().get(), d_filter_tensor_gndth.get_data().get(), 71 | d_bias_tensor_gndth.get_data().get(), d_output_tensor.get_data().get()); 72 | 73 | int pad_h = 2, pad_w = 2, filter_num = 32, filter_channel = 32, 74 | kernel_h = 5, kernel_w = 5, stride_h = 2, stride_w = 2; 75 | 76 | ConvolutionLayer L1(pad_h, pad_w, 77 | filter_num, filter_channel, 78 | kernel_h, kernel_w, 79 | stride_h, stride_w); 80 | L1.set_filter(filter_tensor); 81 | L1.set_bias(bias_tensor); 82 | L1.set_d_filter(d_filter_tensor_pred); 83 | L1.set_d_bias(d_bias_tensor_pred); 84 | 85 | vector input_vector, output_vector; 86 | input_vector.push_back(input_tensor); 87 | input_vector.push_back(d_input_tensor_pred); 88 | 89 | output_vector.push_back(output_tensor_pred); 90 | output_vector.push_back(d_output_tensor); 91 | 92 | L1.forward(input_vector, output_vector); 93 | L1.backward(input_vector, output_vector); 94 | 95 | if (check_eq(output_tensor_pred.get_data().get(), 96 | output_tensor_gndth.get_data().get(), 100*32*8*8)) { 97 | cout<<"test successful\n"; 98 | } 99 | else { 100 | cout<<"test fail\n"; 101 | } 102 | 103 | if (check_eq(d_input_tensor_pred.get_data().get(), 104 | d_input_tensor_gndth.get_data().get(), 100*32*16*16)) { 105 | cout<<"test successful\n"; 106 | } 107 | else { 108 | cout<<"test fail\n"; 109 | } 110 | 111 | if (check_eq(d_filter_tensor_pred.get_data().get(), 112 | d_filter_tensor_gndth.get_data().get(), 32*32*5*5)) { 113 | cout<<"test successful\n"; 114 | } 115 | else { 116 | cout<<"test fail\n"; 117 | } 118 | 119 | if (check_eq(d_bias_tensor_pred.get_data().get(), 120 | d_bias_tensor_gndth.get_data().get(), 32)) { 121 | cout<<"test successful\n"; 122 | } 123 | else { 124 | cout<<"test fail\n"; 125 | } 126 | } 127 | 128 | int main() { 129 | test_conv_layer(); 130 | } 131 | 132 | -------------------------------------------------------------------------------- /test/test_pooling_layer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | # include 4 | #include "pooling_layer.hpp" 5 | 6 | using namespace std; 7 | 8 | void read_matrix(const char* filename, float* data) { 9 | string value; 10 | float f_value; 11 | int num = 0; 12 | 13 | //read data 14 | ifstream data_file(filename); 15 | if (!data_file.is_open()) { 16 | cout << filename << " NOT FOUND" << endl; 17 | exit(-1); 18 | } 19 | while(data_file.good()) { 20 | getline(data_file, value, ','); 21 | f_value = stof(value); 22 | *(data + num) = f_value; 23 | ++num; 24 | } 25 | } 26 | 27 | void get_sample_data(float* X, float* Y, 28 | float* d_X, float* d_Y) { 29 | read_matrix("../../test/sample_data/Pooling/X.csv", X); 30 | read_matrix("../../test/sample_data/Pooling/Y.csv", Y); 31 | read_matrix("../../test/sample_data/Pooling/d_X.csv", d_X); 32 | read_matrix("../../test/sample_data/Pooling/d_Y.csv", d_Y); 33 | } 34 | 35 | bool check_eq(float* pred, float* gndth, int size) { 36 | for(int iter = 0; iter < size; ++iter) { 37 | if(fabs(pred[iter] - gndth[iter]) / 38 | fabs(gndth[iter] + 1e-6) > 0.1) { 39 | return false; 40 | } 41 | } 42 | return true; 43 | } 44 | 45 | void test_pooling_layer() { 46 | Tensor input_tensor(100, 32, 16, 16); 47 | Tensor d_input_tensor_pred(100, 32, 16, 16); 48 | Tensor d_input_tensor_gndth(100, 32, 16, 16); 49 | 50 | Tensor output_tensor_pred(100, 32, 8, 8); 51 | Tensor output_tensor_gndth(100, 32, 8, 8); 52 | Tensor d_output_tensor(100, 32, 8, 8); 53 | 54 | get_sample_data(input_tensor.get_data().get(), output_tensor_gndth.get_data().get(), 55 | d_input_tensor_gndth.get_data().get(), d_output_tensor.get_data().get()); 56 | 57 | int pad_h = 1, pad_w = 1, kernel_h = 3, 58 | kernel_w = 3, stride_h = 2, stride_w = 2; 59 | 60 | PoolingLayer L1(pad_h, pad_w, 61 | kernel_h, kernel_w, 62 | stride_h, stride_w); 63 | vector input_vector, output_vector; 64 | 65 | input_vector.push_back(input_tensor); 66 | input_vector.push_back(d_input_tensor_pred); 67 | 68 | output_vector.push_back(output_tensor_pred); 69 | output_vector.push_back(d_output_tensor); 70 | 71 | L1.forward(input_vector, output_vector); 72 | L1.backward(input_vector, output_vector); 73 | 74 | if (check_eq(output_tensor_pred.get_data().get(), 75 | output_tensor_gndth.get_data().get(), 100*32*8*8)) { 76 | cout<<"test successful\n"; 77 | } 78 | else { 79 | cout<<"test fail\n"; 80 | } 81 | 82 | if (check_eq(d_input_tensor_pred.get_data().get(), 83 | d_input_tensor_gndth.get_data().get(), 100*32*16*16)) { 84 | cout<<"test successful\n"; 85 | } 86 | else { 87 | cout<<"test fail\n"; 88 | } 89 | } 90 | 91 | int main() { 92 | test_pooling_layer(); 93 | } -------------------------------------------------------------------------------- /test/test_relu_layer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "Tensor.hpp" 5 | #include "relu_layer.hpp" 6 | 7 | using namespace std; 8 | 9 | void read_matrix(const char* filename, float* data) { 10 | string value; 11 | float f_value; 12 | int num = 0; 13 | 14 | //read data 15 | ifstream data_file(filename); 16 | if (!data_file.is_open()) { 17 | cout << filename << " NOT FOUND" << endl; 18 | exit(-1); 19 | } 20 | while(data_file.good()) { 21 | getline(data_file, value, ','); 22 | f_value = stof(value); 23 | *(data + num) = f_value; 24 | ++num; 25 | } 26 | } 27 | 28 | void get_sample_data(float* X, float* Y, 29 | float* d_X, float* d_Y) { 30 | read_matrix("../../test/sample_data/ReLU/X.csv", X); 31 | read_matrix("../../test/sample_data/ReLU/Y.csv", Y); 32 | read_matrix("../../test/sample_data/ReLU/d_X.csv", d_X); 33 | read_matrix("../../test/sample_data/ReLU/d_Y.csv", d_Y); 34 | } 35 | 36 | bool check_eq(float* pred, float* gndth, int size) { 37 | for(int iter = 0; iter < size; ++iter) { 38 | if(fabs(pred[iter] - gndth[iter]) / 39 | fabs(gndth[iter] + 1e-6) > 0.1) { 40 | return false; 41 | } 42 | } 43 | return true; 44 | } 45 | 46 | void test_relu_layer() { 47 | Tensor input_tensor(100, 32, 16, 16); 48 | Tensor d_input_tensor_pred(100, 32, 16, 16); 49 | Tensor d_input_tensor_gndth(100, 32, 16, 16); 50 | 51 | Tensor output_tensor_pred(100, 32, 16, 16); 52 | Tensor output_tensor_gndth(100, 32, 16, 16); 53 | Tensor d_output_tensor(100, 32, 16, 16); 54 | 55 | get_sample_data(input_tensor.get_data().get(), output_tensor_gndth.get_data().get(), 56 | d_input_tensor_gndth.get_data().get(), d_output_tensor.get_data().get()); 57 | 58 | ReLULayer L1; 59 | vector input_vector, output_vector; 60 | 61 | input_vector.push_back(input_tensor); 62 | input_vector.push_back(d_input_tensor_pred); 63 | 64 | output_vector.push_back(output_tensor_pred); 65 | output_vector.push_back(d_output_tensor); 66 | 67 | L1.forward(input_vector, output_vector); 68 | L1.backward(input_vector, output_vector); 69 | 70 | if (check_eq(output_tensor_pred.get_data().get(), 71 | output_tensor_gndth.get_data().get(), 100*32*16*16)) { 72 | cout<<"test successful\n"; 73 | } 74 | else { 75 | cout<<"test fail\n"; 76 | } 77 | 78 | if (check_eq(d_input_tensor_pred.get_data().get(), 79 | d_input_tensor_gndth.get_data().get(), 100*32*16*16)) { 80 | cout<<"test successful\n"; 81 | } 82 | else { 83 | cout<<"test fail\n"; 84 | } 85 | } 86 | 87 | int main() { 88 | test_relu_layer(); 89 | } -------------------------------------------------------------------------------- /test/test_softmax_log_loss_layer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "Tensor.hpp" 5 | #include "softmax_layer.hpp" 6 | #include "log_loss_layer.hpp" 7 | 8 | using namespace std; 9 | 10 | void read_matrix(const char* filename, float* data, float coeff=1) { 11 | string value; 12 | float f_value; 13 | int num = 0; 14 | 15 | //read data 16 | ifstream data_file(filename); 17 | if (!data_file.is_open()) { 18 | cout << filename << " NOT FOUND" << endl; 19 | exit(-1); 20 | } 21 | while(data_file.good()) { 22 | getline(data_file, value, ','); 23 | f_value = stof(value); 24 | *(data + num) = f_value * coeff; 25 | ++num; 26 | } 27 | } 28 | 29 | void get_sample_data(float* X, float* L, float* Y, float* d_X) { 30 | read_matrix("../../test/sample_data/SoftMaxLogLoss/X.csv", X); 31 | read_matrix("../../test/sample_data/SoftMaxLogLoss/L.csv", L); 32 | // the test data is unnormalized, so we need add a coeff to normalize loss and grads 33 | read_matrix("../../test/sample_data/SoftMaxLogLoss/Y.csv", Y, 0.01); 34 | read_matrix("../../test/sample_data/SoftMaxLogLoss/d_X.csv", d_X, 0.01); 35 | } 36 | 37 | bool check_eq(float* pred, float* gndth, int size) { 38 | for(int iter = 0; iter < size; ++iter) { 39 | if(fabs(pred[iter] - gndth[iter]) / 40 | fabs(gndth[iter] + 1e-6) > 0.1) { 41 | return false; 42 | } 43 | } 44 | return true; 45 | } 46 | 47 | void test_softmax_log_loss_layer() { 48 | Tensor input_tensor(100, 10, 1, 1); 49 | Tensor label_tensor(100, 1, 1, 1); 50 | Tensor d_input_tensor_pred(100, 10, 1, 1); 51 | Tensor d_input_tensor_gndth(100, 10, 1, 1); 52 | 53 | Tensor intermediate_tensor(100, 10, 1, 1); 54 | Tensor d_intermediate_tensor(100, 10, 1, 1); 55 | 56 | Tensor output_tensor_pred(100, 1, 1, 1); 57 | 58 | Tensor loss_pred(1, 1, 1, 1); 59 | Tensor loss_gndth(1, 1, 1, 1); 60 | 61 | get_sample_data(input_tensor.get_data().get(), label_tensor.get_data().get(), 62 | loss_gndth.get_data().get(), 63 | d_input_tensor_gndth.get_data().get()); 64 | 65 | SoftmaxLayer L1; 66 | LogLoss L2; 67 | 68 | vector input_vector, intermediate_vector, output_vector; 69 | 70 | input_vector.push_back(input_tensor); 71 | input_vector.push_back(d_input_tensor_pred); 72 | 73 | intermediate_vector.push_back(intermediate_tensor); 74 | intermediate_vector.push_back(d_intermediate_tensor); 75 | intermediate_vector.push_back(label_tensor); 76 | 77 | output_vector.push_back(loss_pred); 78 | output_vector.push_back(output_tensor_pred); 79 | 80 | L1.forward(input_vector, intermediate_vector); 81 | L2.forward(intermediate_vector, output_vector); 82 | L2.backward(intermediate_vector, output_vector); 83 | L1.backward(input_vector, intermediate_vector); 84 | 85 | if (check_eq(loss_pred.get_data().get(), 86 | loss_gndth.get_data().get(), 1)) { 87 | cout<<"test successful\n"; 88 | } 89 | else { 90 | cout<<"test fail\n"; 91 | } 92 | 93 | if (check_eq(d_input_tensor_pred.get_data().get(), 94 | d_input_tensor_gndth.get_data().get(), 100*10*1*1)) { 95 | cout<<"test successful\n"; 96 | } 97 | else { 98 | cout<<"test fail\n"; 99 | } 100 | } 101 | 102 | int main() { 103 | test_softmax_log_loss_layer(); 104 | } --------------------------------------------------------------------------------