├── CMakeLists.txt ├── ImageNet Classification with Deep Convolutional Neural Networks.pdf ├── params ├── cifar10 │ ├── 1 │ ├── 4 │ ├── 7 │ ├── 8 │ ├── 1_bias │ ├── 4_bias │ ├── 7_bias │ └── 8_bias └── mnist │ ├── 1 │ ├── 4 │ ├── 7 │ ├── 8 │ ├── 1_bias │ ├── 4_bias │ ├── 7_bias │ └── 8_bias ├── readme.md ├── src ├── dataset │ ├── cifar10.cpp │ ├── cifar10.h │ ├── imagenet.cpp │ ├── imagenet.h │ ├── imagenet200.cpp │ ├── imagenet200.h │ ├── mnist.cpp │ └── mnist.h ├── layer │ ├── activation.cpp │ ├── activation.h │ ├── convolution.cpp │ ├── convolution.h │ ├── input.cpp │ ├── input.h │ ├── layer.cpp │ ├── layer.h │ ├── neuron.cpp │ ├── neuron.h │ ├── output.cu │ ├── output.h │ ├── pooling.cpp │ ├── pooling.h │ ├── relu.cpp │ ├── relu.h │ ├── softmax.cpp │ └── softmax.h ├── model │ ├── network.cpp │ └── network.h ├── utils │ ├── global.cpp │ ├── global.h │ ├── image.cpp │ ├── image.h │ ├── print.cpp │ ├── print.h │ ├── read_data.cpp │ ├── read_data.h │ ├── set_value.cu │ ├── set_value.h │ ├── utils.cpp │ ├── utils.h │ ├── write_data.cpp │ └── write_data.h └── xnet.cpp └── utils ├── ImageNet ├── cat2.file ├── cat2.txt ├── cat5.file ├── cat5.txt ├── loop.sh ├── resize.sh ├── run_sample.sh └── sample.sh ├── ImageNet200 ├── CMakeLists.txt └── main.cpp └── report └── train_error_curve.py /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 3.2) 2 | PROJECT(XNet) 3 | 4 | FIND_PACKAGE(CUDA REQUIRED) 5 | 6 | INCLUDE(FindCUDA) 7 | 8 | INCLUDE_DIRECTORIES(/usr/local/cuda/include) 9 | 10 | SET(CUDA_CUDNN_LIBRARY "/usr/local/cuda/lib64/libcudnn.so") 11 | 12 | SET(OpenCV_STATIC OFF) 13 | FIND_PACKAGE(OpenCV REQUIRED) 14 | INCLUDE_DIRECTORIES(${OpenCV_INCLUDE_DIRS}) 15 | 16 | FILE(GLOB SOURCES "src/*.cu" "src/*.cpp") 17 | FILE(GLOB UTILS "src/utils/*.cu" "src/utils/*.cpp") 18 | FILE(GLOB MODEL "src/model/*.cu" "src/model/*.cpp") 19 | FILE(GLOB LAYER "src/layer/*.cu" "src/layer/*.cpp") 20 | FILE(GLOB DATA "src/dataset/*.cu" "src/dataset/*.cpp") 21 | 22 | CUDA_ADD_EXECUTABLE(XNet ${SOURCES} ${UTILS} ${MODEL} ${LAYER} ${DATA}) 23 | 24 | LIST(APPEND CMAKE_CXX_FLAGS "-std=c++0x -O3 -ffast-math ") 25 | 26 | LIST(APPEND CUDA_NVCC_FLAGS --compiler-options -fno-strict-aliasing -lineinfo -use_fast_math -Xptxas -dlcm=cg) 27 | LIST(APPEND CUDA_NVCC_FLAGS -gencode arch=compute_30,code=sm_30) 28 | 29 | TARGET_LINK_LIBRARIES(XNet ${OpenCV_LIBS} ${CUDA_CUDART_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} ${CUDA_curand_LIBRARY} ${CUDA_CUDNN_LIBRARY}) -------------------------------------------------------------------------------- /ImageNet Classification with Deep Convolutional Neural Networks.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/ImageNet Classification with Deep Convolutional Neural Networks.pdf -------------------------------------------------------------------------------- /params/cifar10/1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/params/cifar10/1 -------------------------------------------------------------------------------- /params/cifar10/1_bias: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/params/cifar10/1_bias -------------------------------------------------------------------------------- /params/cifar10/4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/params/cifar10/4 -------------------------------------------------------------------------------- /params/cifar10/4_bias: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/params/cifar10/4_bias -------------------------------------------------------------------------------- /params/cifar10/7: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/params/cifar10/7 -------------------------------------------------------------------------------- /params/cifar10/7_bias: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/params/cifar10/7_bias -------------------------------------------------------------------------------- /params/cifar10/8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/params/cifar10/8 -------------------------------------------------------------------------------- /params/cifar10/8_bias: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/params/cifar10/8_bias -------------------------------------------------------------------------------- /params/mnist/1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/params/mnist/1 -------------------------------------------------------------------------------- /params/mnist/1_bias: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/params/mnist/1_bias -------------------------------------------------------------------------------- /params/mnist/4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/params/mnist/4 -------------------------------------------------------------------------------- /params/mnist/4_bias: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/params/mnist/4_bias -------------------------------------------------------------------------------- /params/mnist/7: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/params/mnist/7 -------------------------------------------------------------------------------- /params/mnist/7_bias: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/params/mnist/7_bias -------------------------------------------------------------------------------- /params/mnist/8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/params/mnist/8 -------------------------------------------------------------------------------- /params/mnist/8_bias: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msnh2012/XNet/eb03b36b09c7820de3c6373ff00235932df38450/params/mnist/8_bias -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # XNet: simple CuDNN wrapper 2 | 3 | XNet is still under development, some parts may not be completed yet. 4 | 5 | ## About 6 | 7 | This project is part of my undergraduate research project "Object classification with Convolutional Neural Network". It is inspired by Mr. Hinton's paper on 2012 which is included in the repository. Unlike [Caffe](http://caffe.berkeleyvision.org/), [Theano](http://deeplearning.net/software/theano/) or [Torch](http://torch.ch/), XNet is just a basic wrapper with few customizations and not very stable, but it is simple and easily understandable. You may want to use stable framework mentionned above for any production purposes. 8 | 9 | Traditionnal training example like MNIST and CIFAR will be added in the future. 10 | 11 | All suggestions are welcomed and hope that this piece of code is useful to you. 12 | 13 | ## Included features 14 | 15 | * Layers 16 | * Convolutional layer 17 | * Max-pooling layer 18 | * Fully connected layer (with ReLU or Softmax) 19 | * Activation layer 20 | * Input / Output layer 21 | * Neural Network 22 | * Adjustable layer dimension 23 | * Adjustable learning rate for each layer 24 | * Decreasing learning rate 25 | * Momentum and weight decay 26 | * Examples 27 | * MNIST 28 | * Test error rate: 0.5% - 1% 29 | * Real-time recognition on camera with OpenCV 30 | * CIFAR-10 31 | * Test error rate: around 25% (with data augmentation) 32 | 33 | ## How to use 34 | 35 | To run this code, you will need: 36 | 37 | 1. NVIDIA graphic card 38 | 2. Cuda Runtime 7.0 39 | 3. CuDNN library 3 40 | 4. C++ 11 compiler 41 | 5. OpenCV 3.0 (to use your camera) -------------------------------------------------------------------------------- /src/dataset/cifar10.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * cifar10.cpp 3 | * 4 | * Created on: Oct 11, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "cifar10.h" 9 | 10 | namespace cifar10 { 11 | 12 | const int label_count = 10; 13 | const int label_dim = 1; 14 | 15 | const int channel = 3; 16 | const int width = 32, height = 32; 17 | const int data_dim = channel * width * height; 18 | 19 | const int total_size = 5e4; 20 | const int test_size = 1e4; 21 | 22 | const string cifar10_file = "params/cifar10/"; 23 | 24 | int train() { 25 | string dir = "../Data/CIFAR10/data_batch_"; 26 | string extension = ".bin"; 27 | string test_file = "../Data/CIFAR10/test_batch.bin"; 28 | 29 | uint8_t* train = new uint8_t[total_size * (data_dim + label_dim)]; 30 | for (int i = 1; i <= 5; i++) { 31 | ifstream in(dir + to_string(i) + extension); 32 | utils::readBytes(in, train + (i - 1) * (data_dim + label_dim) * 10000, 33 | (data_dim + label_dim) * 10000); 34 | in.close(); 35 | } 36 | 37 | int validation_size = 1e4; 38 | int train_size = total_size * 2 - validation_size; // data size 39 | 40 | // use flip as data augmentation 41 | 42 | float* h_train_images = new float[total_size * data_dim * 2]; 43 | float* h_train_labels = new float[total_size * label_dim * 2]; 44 | 45 | int offset = 0; 46 | int augmentation = total_size * data_dim; 47 | for (int i = 0; i < total_size; i++) { 48 | h_train_labels[i] = (float)train[offset]; 49 | h_train_labels[total_size + i] = (float)train[offset]; 50 | for (int j = 0; j < data_dim; j++) 51 | h_train_images[i * data_dim + j] = (float)train[offset + 1 + j] / 255.0f; 52 | utils::flipImage(h_train_images + i * data_dim, 53 | h_train_images + i * data_dim + augmentation, width, height, channel); 54 | offset += data_dim + label_dim; 55 | } 56 | 57 | //utils::showImage(h_train_images + 32 * 32 * 3 * 85000, 32 ,32 ,3); 58 | 59 | int batch_size = 50; 60 | int iteration = 24; 61 | 62 | model::Network network(h_train_images, data_dim, h_train_labels, label_dim, 63 | train_size, validation_size, batch_size); 64 | network.PushInput(channel, height, width); // 3 32 32 65 | network.PushConvolution(48, 3, -8e-2f, 0.015f, 0.9f, 0.0005f); 66 | network.PushActivation(CUDNN_ACTIVATION_RELU); 67 | network.PushPooling(2, 2); 68 | network.PushConvolution(64, 3, -8e-2f, 0.015f, 0.9f, 0.0005f); 69 | network.PushActivation(CUDNN_ACTIVATION_RELU); 70 | network.PushPooling(2, 2); 71 | network.PushConvolution(64, 3, -8e-2f, 0.015f, 0.9f, 0.0005f); 72 | network.PushActivation(CUDNN_ACTIVATION_RELU); 73 | network.PushPooling(2, 2); 74 | network.PushReLU(800, 0.5, -6e-2f, 0.015f, 0.9f, 0.0005f); 75 | network.PushSoftmax(label_count, 0.25, -6e-2f, 0.015f, 0.9f, 0.0005f); 76 | network.PushOutput(label_count); 77 | network.PrintGeneral(); 78 | 79 | // train the model 80 | 81 | cout << "Train " << iteration << " times ..." << endl; 82 | network.Train(iteration, 0.001, 0.75); 83 | //network.SaveParams(cifar10_file); 84 | cout << "End of training ..." << endl; 85 | 86 | uint8_t* test = new uint8_t[test_size * (data_dim + label_dim)]; 87 | ifstream in_test(test_file); 88 | utils::readBytes(in_test, test, (data_dim + label_dim) * test_size); 89 | in_test.close(); 90 | 91 | float* h_test_images = new float[test_size * data_dim]; 92 | float* h_test_labels = new float[test_size * label_dim]; 93 | 94 | offset = 0; 95 | for (int i = 0; i < test_size; i++) { 96 | h_test_labels[i] = (float)test[offset]; 97 | for (int j = 0; j < data_dim; j++) 98 | h_test_images[i * data_dim + j] = (float)test[offset + 1 + j] / 255.0f; 99 | offset += data_dim + label_dim; 100 | } 101 | 102 | // test the model 103 | 104 | network.SwitchData(h_test_images, h_test_labels, test_size); 105 | 106 | cout << "Testing ..." << endl; 107 | float* h_test_labels_predict = new float[test_size]; 108 | network.Test(h_test_labels_predict); 109 | cout << "End of testing ..." << endl; 110 | vector errors; 111 | for (int i = 0; i < test_size; i++) { 112 | if (abs(h_test_labels_predict[i] - h_test_labels[i]) > 0.1) { 113 | errors.push_back(i); 114 | } 115 | } 116 | cout << "Error rate: " << (0.0 + errors.size()) / test_size * 100 << endl; 117 | 118 | delete[] h_test_labels_predict; 119 | 120 | delete[] h_test_images; 121 | delete[] h_test_labels; 122 | delete[] test; 123 | 124 | delete[] h_train_images; 125 | delete[] h_train_labels; 126 | delete[] train; 127 | 128 | return 0; 129 | } 130 | 131 | } 132 | -------------------------------------------------------------------------------- /src/dataset/cifar10.h: -------------------------------------------------------------------------------- 1 | /* 2 | * cifar10.h 3 | * 4 | * Created on: Oct 11, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #ifndef CIFAR10_H_ 9 | #define CIFAR10_H_ 10 | 11 | #include "../model/network.h" 12 | #include "../utils/read_data.h" 13 | #include "../utils/image.h" 14 | 15 | using namespace std; 16 | 17 | namespace cifar10 { 18 | 19 | int train(); 20 | 21 | } 22 | 23 | #endif /* CIFAR10_H_ */ 24 | -------------------------------------------------------------------------------- /src/dataset/imagenet.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * imagenet.cpp 3 | * 4 | * Created on: Oct 11, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "imagenet.h" 9 | 10 | namespace imagenet { 11 | 12 | struct Image { 13 | int number; 14 | int label; 15 | Image(): number(0), label(0) {} 16 | Image(int n, int l): number(n), label(l) {} 17 | }; 18 | 19 | string to_path(Image& image) { 20 | string label = to_string(image.label); 21 | while (label.length() < 8) 22 | label = "0" + label; 23 | string number = to_string(image.number); 24 | return "../Data/ImageNet/n" + label + "/n" + label + "_" + number + ".JPEG"; 25 | } 26 | 27 | int label_count = 5; // use a small part of ImageNet 28 | string label_file_path = "utils/ImageNet/cat" + to_string(label_count) + ".file"; 29 | 30 | vector images; // image list 31 | map label; // real label to scaled label, from 0 to label_dim - 1 32 | 33 | // read all image informations: folder and file name / number 34 | void prepare() { 35 | label.clear(); 36 | images.clear(); 37 | 38 | ifstream label_file(label_file_path); 39 | int n, m; 40 | int index = 0; 41 | while (true) { 42 | label_file >> n >> m; 43 | if (n == 0 && m == 0) // EOF 44 | break; 45 | images.push_back(Image(m, n)); 46 | if (label.find(n) == label.end()) 47 | label[n] = index++; 48 | } 49 | label_file.close(); 50 | 51 | assert(label.size() == label_count); 52 | } 53 | 54 | int train() { 55 | srand(time(NULL)); 56 | 57 | int batch_size = 2e1; 58 | int iteration_size = 1e4; // 50 batches in one iteration 59 | int iteration = 50; // 5e5 images 60 | 61 | // predefined dimensions 62 | int width = 256; 63 | int height = 256; 64 | int channel = 3; 65 | int data_dim = width * height * channel; 66 | int label_dim = 1; 67 | 68 | prepare(); 69 | 70 | iteration = 1; 71 | iteration_size = 100; 72 | batch_size = 1; 73 | 74 | float* h_train_images = new float[iteration_size * data_dim]; 75 | float* h_train_labels = new float[iteration_size * label_dim]; 76 | 77 | model::Network network(h_train_images, data_dim, h_train_labels, label_dim, 78 | iteration_size, 0, batch_size); 79 | network.PushInput(channel, height, width); // 3 256 256 80 | network.PushConvolution(48, 11, -20e-2f); 81 | network.PushActivation(CUDNN_ACTIVATION_RELU); 82 | network.PushPooling(2, 2); 83 | network.PushConvolution(128, 5, -20e-2f); 84 | network.PushActivation(CUDNN_ACTIVATION_RELU); 85 | network.PushPooling(2, 2); 86 | network.PushConvolution(192, 3, -20e-2f); 87 | network.PushActivation(CUDNN_ACTIVATION_RELU); 88 | network.PushConvolution(192, 3, -20e-2f); 89 | network.PushActivation(CUDNN_ACTIVATION_RELU); 90 | network.PushConvolution(128, 3, -20e-2f); 91 | network.PushActivation(CUDNN_ACTIVATION_RELU); 92 | network.PushPooling(2, 2); 93 | network.PushReLU(2048, 0, -20e-2f); 94 | network.PushReLU(2048, 0, -20e-2f); 95 | network.PushSoftmax(label_count, 0, -20e-2f); 96 | network.PushOutput(label_count); 97 | network.PrintGeneral(); 98 | 99 | for (int i = 0; i < iteration; i++) { 100 | // load iteration_size images randomly 101 | int r; 102 | int offset = 0; 103 | for (int j = 0; j < iteration_size; j++) { 104 | r = rand() % images.size(); 105 | Mat im = imread(to_path(images[r])); 106 | assert(data_dim == im.cols * im.rows * im.elemSize()); 107 | for (int k = 0; k < width * height; k++) { 108 | for (int c = 0; c < channel; c++) 109 | h_train_images[k + offset + c * width * height] 110 | = ((float)im.at(k).val[c]) / 255.; 111 | } 112 | h_train_labels[j] = label[images[r].label]; 113 | offset += data_dim; 114 | } 115 | 116 | // train the network several time 117 | network.Train(10); 118 | } 119 | 120 | //delete[] h_test_labels_predict; 121 | 122 | //delete[] h_test_images; 123 | //delete[] h_test_labels; 124 | //delete[] _test; 125 | 126 | delete[] h_train_images; 127 | delete[] h_train_labels; 128 | 129 | return 0; 130 | } 131 | 132 | } 133 | -------------------------------------------------------------------------------- /src/dataset/imagenet.h: -------------------------------------------------------------------------------- 1 | /* 2 | * imagenet.h 3 | * 4 | * Created on: Oct 11, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #ifndef IMAGENET_H_ 9 | #define IMAGENET_H_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "../model/network.h" 20 | #include "../utils/read_data.h" 21 | 22 | #include "opencv2/opencv.hpp" 23 | #include 24 | #include 25 | 26 | using namespace cv; 27 | using namespace std; 28 | 29 | namespace imagenet { 30 | 31 | int train(); 32 | 33 | } 34 | 35 | #endif /* IMAGENET_H_ */ 36 | -------------------------------------------------------------------------------- /src/dataset/imagenet200.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * imagenet200.cpp 3 | * 4 | * Created on: Oct 15, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "imagenet200.h" 9 | 10 | namespace imagenet200 { 11 | 12 | const int label_count = 10; // categories count 13 | const int images_per_category = 500 * 9; 14 | const int label_dim = 1; 15 | const int total_size = label_count * images_per_category; 16 | 17 | // image size 18 | const int channel = 3; 19 | const int width = 56, height = 56; 20 | const int data_dim = width * height * channel; 21 | 22 | string root_path = "../Data/ImageNet200/train/"; 23 | 24 | int train() { 25 | 26 | int train_size = total_size - label_count * 200; 27 | int val_size = total_size - train_size; 28 | 29 | // read data 30 | 31 | uint8_t* train_images = new uint8_t[total_size * data_dim]; 32 | ifstream in_images(root_path + "images.bin"); 33 | utils::readBytes(in_images, train_images, total_size * data_dim); 34 | in_images.close(); 35 | 36 | uint8_t* train_labels = new uint8_t[total_size * label_dim]; 37 | ifstream in_labels(root_path + "labels.bin"); 38 | utils::readBytes(in_labels, train_labels, total_size * label_dim); 39 | in_labels.close(); 40 | 41 | float* h_train_images = new float[total_size * data_dim]; 42 | float* h_train_labels = new float[total_size * label_dim]; 43 | 44 | for (int i = 0; i < total_size * data_dim; i++) 45 | h_train_images[i] = (float)train_images[i] / 255.0f; 46 | for (int i = 0; i < total_size * label_dim; i++) 47 | h_train_labels[i] = (float)train_labels[i]; 48 | 49 | //utils::showImage(h_train_images + 56 * 56 * 3 * i, 56 ,56 ,3); 50 | 51 | cout << "Training data loaded." << endl; 52 | 53 | // train network 54 | 55 | int batch_size = 50; 56 | int iteration = 24; 57 | 58 | model::Network network(h_train_images, data_dim, h_train_labels, label_dim, 59 | train_size, val_size, batch_size); 60 | network.PushInput(channel, height, width); 61 | network.PushConvolution(48, 5, -6e-2f, 0.02f, 0.9f, 0.001f); 62 | network.PushActivation(CUDNN_ACTIVATION_RELU); 63 | network.PushPooling(2, 2); 64 | network.PushConvolution(64, 5, -6e-2f, 0.02f, 0.9f, 0.001f); 65 | network.PushActivation(CUDNN_ACTIVATION_RELU); 66 | network.PushPooling(2, 2); 67 | network.PushConvolution(64, 4, -6e-2f, 0.02f, 0.9f, 0.001f); 68 | network.PushActivation(CUDNN_ACTIVATION_RELU); 69 | network.PushPooling(2, 2); 70 | network.PushReLU(800, 0.5, -4e-2f, 0.02f, 0.9f, 0.001f); 71 | network.PushSoftmax(label_count, 0.25, -4e-2f, 0.02f, 0.9f, 0.001f); 72 | network.PushOutput(label_count); 73 | network.PrintGeneral(); 74 | 75 | network.Train(iteration, 0, 0.2, 0.94, false); 76 | 77 | delete[] train_images; 78 | delete[] train_labels; 79 | 80 | delete[] h_train_images; 81 | delete[] h_train_labels; 82 | 83 | return 0; 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /src/dataset/imagenet200.h: -------------------------------------------------------------------------------- 1 | /* 2 | * imagenet200.h 3 | * 4 | * Created on: Oct 15, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #ifndef IMAGENET200_H_ 9 | #define IMAGENET200_H_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "../model/network.h" 20 | #include "../utils/read_data.h" 21 | #include "../utils/image.h" 22 | 23 | using namespace std; 24 | 25 | namespace imagenet200 { 26 | 27 | int train(); 28 | 29 | } 30 | 31 | #endif /* IMAGENET200_H_ */ 32 | -------------------------------------------------------------------------------- /src/dataset/mnist.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * mnist.cpp 3 | * 4 | * Created on: Oct 11, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "mnist.h" 9 | 10 | using namespace cv; 11 | using namespace std; 12 | 13 | namespace mnist { 14 | 15 | const int label_count = 10; 16 | const int label_dim = 1; 17 | 18 | const int channel = 1; 19 | 20 | string mnist_file = global::root + "params/mnist/"; 21 | 22 | int train() { 23 | string train_images_path = global::root + "../Data/MNIST/train-images.idx3-ubyte"; 24 | string train_labels_path = global::root + "../Data/MNIST/train-labels.idx1-ubyte"; 25 | string test_images_path = global::root + "../Data/MNIST/t10k-images.idx3-ubyte"; 26 | string test_labels_path = global::root + "../Data/MNIST/t10k-labels.idx1-ubyte"; 27 | 28 | int width = 28, height = 28; 29 | int train_size, test_size; 30 | 31 | cout << "Reading input data" << endl; 32 | 33 | // read train data 34 | ifstream train_images_file(train_images_path, ios::binary); 35 | train_images_file.seekg(4); 36 | utils::readInt(train_images_file, &train_size); 37 | utils::readInt(train_images_file, &height); 38 | utils::readInt(train_images_file, &width); 39 | uint8_t* train_images = new uint8_t[train_size * channel * height * width]; 40 | utils::readBytes(train_images_file, train_images, 41 | train_size * channel * height * width); 42 | train_images_file.close(); 43 | 44 | int data_dim = channel * width * height; 45 | 46 | // read train label 47 | ifstream train_labels_file(train_labels_path, ios::binary); 48 | train_labels_file.seekg(8); 49 | uint8_t* train_labels = new uint8_t[train_size]; 50 | utils::readBytes(train_labels_file, train_labels, train_size); 51 | train_labels_file.close(); 52 | 53 | cout << "Done. Training dataset size: " << train_size << endl; 54 | 55 | // transform data 56 | float* h_train_images = new float[train_size * channel * height * width]; 57 | float* h_train_labels = new float[train_size]; 58 | for (int i = 0; i < train_size * channel * height * width; i++) 59 | h_train_images[i] = (float)train_images[i] / 255.0f; 60 | for (int i = 0; i < train_size; i++) 61 | h_train_labels[i] = (float)train_labels[i]; 62 | 63 | int val_size = 10000; 64 | train_size -= val_size; // 50000 65 | 66 | // build LeNet 67 | 68 | int batch_size = 100; 69 | int iteration = 50; 70 | 71 | cout << "Batch size: " << batch_size << endl; 72 | 73 | model::Network network(h_train_images, data_dim, h_train_labels, label_dim, 74 | train_size, val_size, batch_size); 75 | network.PushInput(channel, height, width); // 1 28 28 76 | network.PushConvolution(32, 5, -10e-2f, 0.015f, 0.9f, 0.005f); 77 | network.PushActivation(CUDNN_ACTIVATION_RELU); 78 | network.PushPooling(2, 2); 79 | network.PushConvolution(64, 5, -10e-2f, 0.015f, 0.9f, 0.005f); 80 | network.PushActivation(CUDNN_ACTIVATION_RELU); 81 | network.PushPooling(2, 2); 82 | network.PushReLU(800, 0.6, -8e-2f, 0.02f, 0.15f, 0.005f); 83 | network.PushSoftmax(label_count, 0.25, -8e-2f, 0.015f, 0.9f, 0.005f); 84 | network.PushOutput(label_count); 85 | network.PrintGeneral(); 86 | 87 | // train the model 88 | 89 | cout << "Train " << iteration << " times ..." << endl; 90 | //network.ReadParams(mnist_file); 91 | network.Train(iteration, 0.01, 0.5, 0.99); // depend on the number of batch_size 92 | cout << "End of training ..." << endl; 93 | 94 | //network.SaveParams(mnist_file); 95 | 96 | // read test cases 97 | 98 | cout << "Reading test data" << endl; 99 | 100 | ifstream test_images_file(test_images_path, ios::binary); 101 | test_images_file.seekg(4); 102 | utils::readInt(test_images_file, &test_size); 103 | utils::readInt(test_images_file, &height); 104 | utils::readInt(test_images_file, &width); 105 | uint8_t* test_images = new uint8_t[test_size * channel * height * width]; 106 | test_images_file.read((char*)test_images, test_size * channel * height * width); 107 | test_images_file.close(); 108 | 109 | ifstream test_labels_file(test_labels_path, ios::binary); 110 | test_labels_file.seekg(8); 111 | uint8_t* test_labels = new uint8_t[test_size]; 112 | test_labels_file.read((char*)test_labels, test_size); 113 | test_labels_file.close(); 114 | 115 | cout << "Done. Test dataset size: " << test_size << endl; 116 | 117 | // transform test data 118 | float* h_test_images = new float[test_size * channel * height * width]; 119 | float* h_test_labels = new float[test_size]; 120 | for (int i = 0; i < test_size * channel * height * width; i++) 121 | h_test_images[i] = (float)test_images[i] / 255.0f; 122 | for (int i = 0; i < test_size; i++) 123 | h_test_labels[i] = (float)test_labels[i]; 124 | 125 | // test the model 126 | 127 | network.SwitchData(h_test_images, h_test_labels, test_size); 128 | 129 | cout << "Testing ..." << endl; 130 | float* h_test_labels_predict = new float[test_size]; 131 | network.Test(h_test_labels_predict); 132 | cout << "End of testing ..." << endl; 133 | vector errors; 134 | for (int i = 0; i < test_size; i++) { 135 | if (abs(h_test_labels_predict[i] - h_test_labels[i]) > 0.1) { 136 | errors.push_back(i); 137 | //cout << h_test_labels_predict[i] << ' ' << h_test_labels[i] << endl; 138 | } 139 | } 140 | cout << "Error rate: " << (0.0 + errors.size()) / test_size * 100 << endl; 141 | 142 | delete[] h_test_labels_predict; 143 | delete[] test_images; 144 | delete[] test_labels; 145 | delete[] h_test_images; 146 | delete[] h_test_labels; 147 | 148 | delete[] train_images; 149 | delete[] train_labels; 150 | 151 | delete[] h_train_images; 152 | delete[] h_train_labels; 153 | 154 | return 0; 155 | } 156 | 157 | void camera() { 158 | int channels = 1; 159 | int width = 28, height = 28; 160 | 161 | float* h_image = new float[channels * height * width]; 162 | float* h_label_predict = new float[1]; 163 | 164 | int data_dim = width * height * channels; 165 | model::Network network(h_image, data_dim, h_label_predict, 1, 1, 0, 1); 166 | network.PushInput(channels, height, width); // 1 28 28 167 | network.PushConvolution(20, 5, -10e-3f); 168 | network.PushActivation(CUDNN_ACTIVATION_RELU); 169 | network.PushPooling(2, 2); 170 | network.PushConvolution(50, 5, -10e-3f); 171 | network.PushActivation(CUDNN_ACTIVATION_RELU); 172 | network.PushPooling(2, 2); 173 | network.PushReLU(400, 0.5, -10e-3f); 174 | network.PushSoftmax(10, 0.5, -10e-3f); 175 | network.PushOutput(10); 176 | network.ReadParams(mnist_file); 177 | 178 | VideoCapture cap(0); // open the default camera 179 | if(!cap.isOpened()) // check if we succeeded 180 | return; 181 | 182 | Mat image; 183 | namedWindow("number", 1); 184 | int threshold = 128; 185 | createTrackbar("Threshold", "number", &threshold, 255); 186 | while (true) { 187 | Mat frame; 188 | cap >> frame; // get a new frame from camera 189 | Mat tmp; 190 | resize(frame, tmp, Size(28, 28)); 191 | cvtColor(tmp, image, CV_BGR2GRAY); 192 | for (int i = 0; i < channels * height * width; i++) { 193 | unsigned char _p = image.at(i / 28, i % 28); 194 | image.at(i / 28, i % 28) = _p < threshold ? 255 : 0; 195 | } 196 | imshow("number", image); 197 | for (int i = 0; i < channels * height * width; i++) { 198 | h_image[i] = image.at(i / 28, i % 28) / 255.0f; 199 | } 200 | network.Test(h_label_predict); 201 | cout << h_label_predict[0] << endl; 202 | if (waitKey(100) >= 0) 203 | break; 204 | } 205 | 206 | delete[] h_label_predict; 207 | delete[] h_image; 208 | 209 | } 210 | 211 | } 212 | -------------------------------------------------------------------------------- /src/dataset/mnist.h: -------------------------------------------------------------------------------- 1 | /* 2 | * mnist.h 3 | * 4 | * Created on: Oct 11, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #ifndef MNIST_H_ 9 | #define MNIST_H_ 10 | 11 | #include "../model/network.h" 12 | #include "../utils/read_data.h" 13 | #include "../utils/image.h" 14 | #include "../utils/global.h" 15 | 16 | #include "opencv2/opencv.hpp" 17 | 18 | namespace mnist { 19 | 20 | int train(); 21 | void camera(); 22 | 23 | } 24 | 25 | #endif /* MNIST_H_ */ 26 | -------------------------------------------------------------------------------- /src/layer/activation.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * activation.cpp 3 | * 4 | * Created on: Oct 5, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "activation.h" 9 | 10 | using namespace global; 11 | 12 | namespace layer { 13 | 14 | Activation::Activation(Layer* _prev, cudnnActivationMode_t _mode) : Layer(0) { 15 | prev = _prev; 16 | prev->next = this; 17 | 18 | mode = _mode; 19 | 20 | int _n, _c, _h, _w, _tmp; 21 | cudnnDataType_t _t; 22 | callCudnn(cudnnGetTensor4dDescriptor(prev->t_data, &_t, &_n, &_c, &_h, &_w, &_tmp, 23 | &_tmp, &_tmp, &_tmp)); 24 | batch = _n; 25 | data_size = _n * _c * _h * _w; 26 | callCudnn(cudnnCreateTensorDescriptor(&t_data)); 27 | callCudnn(cudnnSetTensor4dDescriptor(t_data, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, 28 | _n, _c, _h, _w)); 29 | callCuda(cudaMalloc(&data, sizeof(float) * data_size)); 30 | callCuda(cudaMalloc(&diff, sizeof(float) * data_size)); 31 | 32 | param_size = 0; 33 | param_bias_size = 0; 34 | } 35 | 36 | Activation::~Activation() { 37 | callCudnn(cudnnDestroyTensorDescriptor(t_data)); 38 | callCuda(cudaFree(data)); 39 | callCuda(cudaFree(diff)); 40 | } 41 | 42 | void Activation::forward(bool train) { 43 | float a = 1; 44 | float b = 0; 45 | callCudnn(cudnnActivationForward(cudnnHandle, mode, &a, prev->t_data, 46 | prev->data, &b, t_data, data)); 47 | } 48 | 49 | void Activation::backward() { 50 | float a = 1; 51 | float b = 0; 52 | callCudnn(cudnnActivationBackward(cudnnHandle, mode, &a, t_data, data, t_data, 53 | next->diff, prev->t_data, prev->data, &b, t_data, diff)); 54 | } 55 | 56 | void Activation::update() { 57 | // nothing 58 | } 59 | 60 | } /* namespace layer */ 61 | -------------------------------------------------------------------------------- /src/layer/activation.h: -------------------------------------------------------------------------------- 1 | /* 2 | * activation.h 3 | * 4 | * Created on: Oct 5, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #ifndef ACTIVATION_H_ 9 | #define ACTIVATION_H_ 10 | 11 | #include "layer.h" 12 | 13 | namespace layer { 14 | 15 | class Activation: public Layer { 16 | private: 17 | cudnnActivationMode_t mode; 18 | public: 19 | Activation(Layer* prev, cudnnActivationMode_t mode); 20 | virtual ~Activation(); 21 | 22 | void forward(bool train = true); 23 | void backward(); 24 | void update(); 25 | }; 26 | 27 | } /* namespace layer */ 28 | #endif /* ACTIVATION_H_ */ 29 | -------------------------------------------------------------------------------- /src/layer/convolution.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * convolution.c 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "convolution.h" 9 | 10 | using namespace global; 11 | 12 | namespace layer { 13 | 14 | Convolution::Convolution(Layer* _prev, int n ,int c, int kernel, float alpha, 15 | float sigma, float momentum, float weight_decay): 16 | Layer(alpha, momentum, weight_decay) { 17 | prev = _prev; 18 | prev->next = this; 19 | 20 | batch = n; 21 | 22 | callCudnn(cudnnCreateConvolutionDescriptor(&descriptor)); 23 | callCudnn(cudnnSetConvolution2dDescriptor(descriptor, 0, 0, 1, 1, 1, 1, 24 | CUDNN_CROSS_CORRELATION)); 25 | 26 | int _n, _c, _h, _w, _tmp; 27 | cudnnDataType_t _t; 28 | callCudnn(cudnnGetTensor4dDescriptor(prev->t_data, &_t, &_n, &_c, &_h, &_w, &_tmp, 29 | &_tmp, &_tmp, &_tmp)); 30 | callCudnn(cudnnCreateFilterDescriptor(&filter)); 31 | callCudnn(cudnnSetFilter4dDescriptor(filter, CUDNN_DATA_FLOAT, 32 | c, _c, kernel, kernel)); 33 | param_size = _c * c * kernel * kernel; 34 | callCuda(cudaMalloc(¶m, sizeof(float) * param_size)); 35 | callCuda(cudaMalloc(&gradient, sizeof(float) * param_size)); 36 | 37 | //utils::printGpuMatrix(param, param_size, _c * kernel, c * kernel, 8); 38 | 39 | int h = _h - kernel + 1; 40 | int w = _w - kernel + 1; 41 | 42 | callCudnn(cudnnCreateTensorDescriptor(&t_data)); 43 | callCudnn(cudnnSetTensor4dDescriptor(t_data, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, 44 | n, c, h, w)); 45 | data_size = n * c * h * w; 46 | callCuda(cudaMalloc(&data, sizeof(float) * data_size)); 47 | callCuda(cudaMalloc(&diff, sizeof(float) * prev->data_size)); 48 | 49 | callCudnn(cudnnCreateTensorDescriptor(&t_bias)); 50 | callCudnn(cudnnSetTensor4dDescriptor(t_bias, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, 51 | 1, c, 1, 1)); 52 | param_bias_size = c; 53 | callCuda(cudaMalloc(¶m_bias, sizeof(float) * param_bias_size)); 54 | callCuda(cudaMalloc(&gradient_bias, sizeof(float) * param_bias_size)); 55 | 56 | utils::setGpuNormalValue(param, param_size, 0, sigma); 57 | utils::setGpuNormalValue(param_bias, param_bias_size, 0, sigma); 58 | utils::setGpuValue(gradient, param_size, 0); 59 | utils::setGpuValue(gradient_bias, param_bias_size, 0); 60 | 61 | callCudnn(cudnnGetConvolutionForwardAlgorithm(cudnnHandle, prev->t_data, filter, 62 | descriptor, t_data, CUDNN_CONVOLUTION_FWD_PREFER_FASTEST, 0, &algo)); 63 | callCudnn(cudnnGetConvolutionForwardWorkspaceSize(cudnnHandle, 64 | prev->t_data, filter, descriptor, t_data, algo, &workspace_size)); 65 | callCuda(cudaMalloc(&workspace, workspace_size)); 66 | } 67 | 68 | Convolution::~Convolution() { 69 | callCudnn(cudnnDestroyFilterDescriptor(filter)); 70 | callCudnn(cudnnDestroyConvolutionDescriptor(descriptor)); 71 | callCudnn(cudnnDestroyTensorDescriptor(t_data)); 72 | callCudnn(cudnnDestroyTensorDescriptor(t_bias)); 73 | callCuda(cudaFree(data)); 74 | callCuda(cudaFree(diff)); 75 | callCuda(cudaFree(param)); 76 | callCuda(cudaFree(param_bias)); 77 | callCuda(cudaFree(gradient)); 78 | callCuda(cudaFree(gradient_bias)); 79 | callCuda(cudaFree(workspace)); 80 | } 81 | 82 | void Convolution::forward(bool train) { 83 | float a = 1; 84 | float b = 0; 85 | callCudnn(cudnnConvolutionForward(cudnnHandle, &a, prev->t_data, prev->data, filter, 86 | param, descriptor, algo, workspace, workspace_size, &b, t_data, data)); 87 | callCudnn(cudnnAddTensor(cudnnHandle, CUDNN_ADD_SAME_C, &a, t_bias, param_bias, 88 | &a, t_data, data)); 89 | } 90 | 91 | void Convolution::backward() { 92 | float a = alpha; // learning rate 93 | float b = momentum; // momentum 94 | callCudnn(cudnnConvolutionBackwardBias(cudnnHandle, &a, t_data, 95 | next->diff, &b, t_bias, gradient_bias)); 96 | callCudnn(cudnnConvolutionBackwardFilter(cudnnHandle, &a, prev->t_data, 97 | prev->data, t_data, next->diff, descriptor, &b, filter, gradient)); 98 | a = 1; 99 | b = 0; 100 | callCudnn(cudnnConvolutionBackwardData(cudnnHandle, &a, filter, 101 | param, t_data, next->diff, descriptor, &b, prev->t_data, diff)); 102 | } 103 | 104 | void Convolution::update() { 105 | float a = 1 - weight_decay; 106 | callCuda(cublasSaxpy(cublasHandle, param_size, &a, gradient, 1, param, 1)); 107 | callCuda(cublasSaxpy(cublasHandle, param_bias_size, &a, 108 | gradient_bias, 1, param_bias, 1)); 109 | } 110 | 111 | } 112 | -------------------------------------------------------------------------------- /src/layer/convolution.h: -------------------------------------------------------------------------------- 1 | /* 2 | * convolution.cuh 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #ifndef CONVOLUTION_CUH_ 9 | #define CONVOLUTION_CUH_ 10 | 11 | #include "layer.h" 12 | 13 | namespace layer { 14 | 15 | class Convolution : public Layer { 16 | private: 17 | cudnnFilterDescriptor_t filter; 18 | cudnnConvolutionDescriptor_t descriptor; 19 | cudnnTensorDescriptor_t t_bias; 20 | cudnnConvolutionFwdAlgo_t algo; 21 | size_t workspace_size; // extra size for computing 22 | void* workspace; // pointer to the extra size 23 | float* tmp_data; 24 | float* tmp_diff; 25 | public: 26 | Convolution(Layer* _prev, int n ,int c, int kernel, float alpha, 27 | float sigma = 0.01f, float momentum = 0.9f, float weight_decay = 0); 28 | virtual ~Convolution(); 29 | void forward(bool train = true); 30 | void backward(); 31 | void update(); 32 | }; 33 | 34 | } /* namespace layer */ 35 | #endif /* CONVOLUTION_CUH_ */ 36 | -------------------------------------------------------------------------------- /src/layer/input.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * input.cu 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "input.h" 9 | 10 | namespace layer { 11 | 12 | Input::Input(int n, int c, int h, int w, float* _data) : Layer() { 13 | prev = NULL; 14 | 15 | batch = n; 16 | callCudnn(cudnnCreateTensorDescriptor(&t_data)); 17 | callCudnn(cudnnSetTensor4dDescriptor(t_data, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, 18 | n, c, h, w)); 19 | data_size = n * c * h * w; 20 | data = _data; 21 | 22 | param_size = 0; 23 | param_bias_size = 0; 24 | } 25 | 26 | Input::~Input() { 27 | callCudnn(cudnnDestroyTensorDescriptor(t_data)); 28 | data = NULL; 29 | } 30 | 31 | void Input::forward(bool train) { 32 | // nothing 33 | } 34 | 35 | void Input::backward() { 36 | // nothing 37 | } 38 | 39 | void Input::update() { 40 | // nothing 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/layer/input.h: -------------------------------------------------------------------------------- 1 | /* 2 | * input.cuh 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #ifndef INPUT_CUH_ 9 | #define INPUT_CUH_ 10 | 11 | #include "layer.h" 12 | 13 | namespace layer { 14 | 15 | class Input: public Layer { 16 | public: 17 | Input(int n, int c, int h, int w, float* _data); 18 | virtual ~Input(); 19 | void forward(bool train = true); 20 | void backward(); 21 | void update(); 22 | }; 23 | 24 | } /* namespace layer */ 25 | #endif /* INPUT_CUH_ */ 26 | -------------------------------------------------------------------------------- /src/layer/layer.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * layer.cu 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "layer.h" 9 | 10 | namespace layer { 11 | 12 | Layer::Layer(float alpha, float momentum, float weight_decay): 13 | alpha(alpha), momentum(momentum), weight_decay(weight_decay) { 14 | 15 | } 16 | 17 | Layer::~Layer() {} 18 | 19 | void Layer::adjust_learning(float scale) { 20 | alpha *= scale; 21 | } 22 | 23 | 24 | } 25 | 26 | -------------------------------------------------------------------------------- /src/layer/layer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * layer.h 3 | * 4 | * Layer class, this class is abstract, it provides basic layer members like 5 | * data and some methods. All layers should extend this class. 6 | * 7 | * Created on: Sep 20, 2015 8 | * Author: lyx 9 | */ 10 | 11 | #ifndef LAYER_H_ 12 | #define LAYER_H_ 13 | 14 | #include 15 | #include 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "../utils/global.h" 23 | #include "../utils/set_value.h" 24 | #include "../utils/print.h" 25 | #include "../utils/utils.h" 26 | 27 | namespace layer { 28 | 29 | class Layer { 30 | public: 31 | Layer(float alpha = 0, float momentum = 0.9f, float weight_decay = 0); 32 | virtual ~Layer(); 33 | 34 | // three virtual method that all layers should have 35 | virtual void forward(bool train = true) = 0; 36 | virtual void backward() = 0; 37 | virtual void update() = 0; 38 | 39 | void adjust_learning(float scale); // change the learning rate 40 | 41 | Layer* prev; // previous layer 42 | Layer* next; // next layer 43 | cudnnTensorDescriptor_t t_data; // output dimension 44 | float* data; // output 45 | int data_size; // output size 46 | float* diff; // difference for the previous layer 47 | float* param; // parameters 48 | int param_size; // parameters count 49 | float* param_bias; // bias parameters for some layers 50 | int param_bias_size; // bias parameters count 51 | float* gradient; // gradient of parameters 52 | float* gradient_bias; // gradient of bias parameters 53 | int batch; // batch size 54 | float alpha; // learning rate 55 | float momentum; // momentum of gradient 56 | float weight_decay; // weight decay rate 57 | }; 58 | 59 | } /* namespace layer */ 60 | #endif /* LAYER_H_ */ 61 | -------------------------------------------------------------------------------- /src/layer/neuron.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * neuron.cu 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "neuron.h" 9 | 10 | using namespace global; 11 | 12 | namespace layer { 13 | 14 | Neuron::Neuron(Layer* _prev, int _output_size, float dropout_rate, float alpha, 15 | float sigma, float momentum, float weight_decay): 16 | Layer(alpha, momentum, weight_decay) { 17 | prev = _prev; 18 | prev->next = this; 19 | 20 | int _n, _c, _h, _w, _tmp; 21 | cudnnDataType_t _t; 22 | callCudnn(cudnnGetTensor4dDescriptor(prev->t_data, &_t, &_n, &_c, &_h, &_w, &_tmp, 23 | &_tmp, &_tmp, &_tmp)); 24 | batch = _n; 25 | input_size = _c * _h * _w; 26 | output_size = _output_size; 27 | callCudnn(cudnnCreateTensorDescriptor(&t_data)); 28 | callCudnn(cudnnSetTensor4dDescriptor(t_data, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, 29 | batch, output_size, 1, 1)); 30 | data_size = batch * output_size; 31 | callCuda(cudaMalloc(&data, sizeof(float) * data_size)); 32 | callCuda(cudaMalloc(&tmp_data, sizeof(float) * data_size)); 33 | callCuda(cudaMalloc(&diff, sizeof(float) * prev->data_size)); 34 | callCuda(cudaMalloc(&tmp_diff, sizeof(float) * data_size)); 35 | 36 | param_size = input_size * output_size; 37 | param_bias_size = output_size; 38 | callCuda(cudaMalloc(¶m, sizeof(float) * param_size)); 39 | callCuda(cudaMalloc(¶m_bias, sizeof(float) * param_bias_size)); 40 | callCuda(cudaMalloc(&gradient, sizeof(float) * param_size)); 41 | callCuda(cudaMalloc(&gradient_bias, sizeof(float) * param_bias_size)); 42 | 43 | utils::setGpuNormalValue(param, param_size, 0, sigma); 44 | utils::setGpuNormalValue(param_bias, param_bias_size, 0, sigma); 45 | utils::setGpuValue(gradient, param_size, 0); 46 | utils::setGpuValue(gradient_bias, param_bias_size, 0); 47 | 48 | callCuda(cudaMalloc(&one, sizeof(float) * batch)); 49 | utils::setGpuValue(one, batch, 1); 50 | 51 | this->dropout_rate = dropout_rate; 52 | } 53 | 54 | Neuron::~Neuron() { 55 | callCudnn(cudnnDestroyTensorDescriptor(t_data)); 56 | callCuda(cudaFree(data)); 57 | callCuda(cudaFree(tmp_data)); 58 | callCuda(cudaFree(diff)); 59 | callCuda(cudaFree(tmp_diff)); 60 | callCuda(cudaFree(param)); 61 | callCuda(cudaFree(param_bias)); 62 | callCuda(cudaFree(gradient)); 63 | callCuda(cudaFree(gradient_bias)); 64 | callCuda(cudaFree(one)); 65 | } 66 | 67 | void Neuron::forward(bool train) { 68 | float a = 1; 69 | float b = 0; 70 | dropout(train); 71 | callCuda(cublasSgemm(cublasHandle, CUBLAS_OP_T, CUBLAS_OP_N, output_size, batch, 72 | input_size, &a, param, input_size, prev->data, input_size, &b, tmp_data, 73 | output_size)); 74 | callCuda(cublasSgemm(cublasHandle, CUBLAS_OP_N, CUBLAS_OP_N, output_size, batch, 75 | 1, &a, param_bias, output_size, one, 1, &a, tmp_data, output_size)); 76 | forward_activation(); 77 | } 78 | 79 | void Neuron::backward() { 80 | float a = alpha; 81 | float b = momentum; 82 | backward_activation(); 83 | callCuda(cublasSgemm(cublasHandle, CUBLAS_OP_N, CUBLAS_OP_T, input_size, 84 | output_size, batch, &a, prev->data, input_size, tmp_diff, output_size, 85 | &b, gradient, input_size)); 86 | callCuda(cublasSgemv(cublasHandle, CUBLAS_OP_N, output_size, batch, 87 | &a, tmp_diff, output_size, one, 1, &b, gradient_bias, 1)); 88 | a = 1; 89 | b = 0; 90 | callCuda(cublasSgemm(cublasHandle, CUBLAS_OP_N, CUBLAS_OP_N, input_size, 91 | batch, output_size, &a, param, input_size, tmp_diff, output_size, 92 | &b, diff, input_size)); 93 | } 94 | 95 | void Neuron::update() { 96 | float a = 1 - weight_decay; 97 | callCuda(cublasSaxpy(cublasHandle, param_size, &a, gradient, 1, param, 1)); 98 | callCuda(cublasSaxpy(cublasHandle, param_bias_size, &a, 99 | gradient_bias, 1, param_bias, 1)); 100 | } 101 | 102 | void Neuron::dropout(bool train) { 103 | if (train) 104 | utils::dropGpuValue(prev->data, prev->data_size, dropout_rate); 105 | else 106 | utils::scaleGpuValue(prev->data, prev->data_size, 1 - dropout_rate); 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /src/layer/neuron.h: -------------------------------------------------------------------------------- 1 | /* 2 | * neuron.cuh 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #ifndef NEURON_CUH_ 9 | #define NEURON_CUH_ 10 | 11 | #include "layer.h" 12 | 13 | namespace layer { 14 | 15 | // fully connected layer 16 | class Neuron : public Layer { 17 | private: 18 | float dropout_rate; 19 | protected: 20 | float* tmp_data; // z 21 | float* tmp_diff; // difference of z 22 | int input_size; // output size of previous layer 23 | int output_size; // output size 24 | float* one; // full one vector for bias 25 | void dropout(bool train); 26 | public: 27 | // data is output_size * batch 28 | Neuron(Layer* _prev, int _output_size, float dropout_rate, float alpha, 29 | float sigma = 0.01f, float momentum = 0.9f, float weight_decay = 0); 30 | virtual ~Neuron(); 31 | void forward(bool train = true); 32 | virtual void forward_activation() = 0; // support different activation function 33 | void backward(); 34 | virtual void backward_activation() = 0; 35 | void update(); 36 | }; 37 | 38 | } /* namespace layer */ 39 | #endif /* NEURON_CUH_ */ 40 | -------------------------------------------------------------------------------- /src/layer/output.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * output.cu 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "output.h" 9 | 10 | using namespace global; 11 | 12 | namespace layer { 13 | 14 | __global__ void softmaxLoss(const float *label, int label_dim, int batch, float *diff) 15 | { 16 | int idx = blockIdx.x * blockDim.x + threadIdx.x; 17 | if (idx >= batch) 18 | return; 19 | 20 | const int label_value = static_cast(label[idx]); 21 | 22 | diff[idx * label_dim + label_value] -= 1.0f; 23 | } 24 | 25 | __global__ void predict(const float *softmax, int label_dim, int batch, float *data) 26 | { 27 | int idx = blockIdx.x * blockDim.x + threadIdx.x; 28 | if (idx >= batch) 29 | return; 30 | 31 | int label_value = 0; 32 | float max = -1; 33 | for (int i = 0; i < label_dim; i++) { 34 | if (softmax[idx * label_dim + i] > max) { 35 | max = softmax[idx * label_dim + i]; 36 | label_value = i; 37 | } 38 | } 39 | 40 | data[idx] = (float)label_value; 41 | } 42 | 43 | Output::Output(Layer* _prev, float* _label, int _label_dim, int _batch) : Layer() { 44 | prev = _prev; 45 | prev->next = this; 46 | 47 | batch = _batch; 48 | label_dim = _label_dim; 49 | callCudnn(cudnnCreateTensorDescriptor(&t_data)); 50 | callCudnn(cudnnSetTensor4dDescriptor(t_data, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, 51 | batch, 1, 1, 1)); 52 | data_size = batch; 53 | callCuda(cudaMalloc(&data, sizeof(float) * data_size)); 54 | label = _label; 55 | 56 | callCuda(cudaMalloc(&diff, sizeof(float) * prev->data_size)); 57 | 58 | param_size = 0; 59 | param_bias_size = 0; 60 | } 61 | 62 | Output::~Output() { 63 | callCudnn(cudnnDestroyTensorDescriptor(t_data)); 64 | callCuda(cudaFree(data)); 65 | label = NULL; 66 | callCuda(cudaFree(diff)); 67 | } 68 | 69 | void Output::forward(bool train) { 70 | predict<<< (batch + 127) / 128, 128>>> (prev->data, label_dim, batch, data); 71 | } 72 | 73 | void Output::backward() { 74 | callCuda(cudaMemcpy(diff, prev->data, sizeof(float) * prev->data_size, 75 | cudaMemcpyDeviceToDevice)); 76 | softmaxLoss<<< (batch + 127) / 128, 128>>> (label, label_dim, batch, diff); 77 | } 78 | 79 | void Output::update() { 80 | // nothing 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /src/layer/output.h: -------------------------------------------------------------------------------- 1 | /* 2 | * output.cuh 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #ifndef OUTPUT_CUH_ 9 | #define OUTPUT_CUH_ 10 | 11 | #include "layer.h" 12 | 13 | namespace layer { 14 | 15 | class Output: public Layer { 16 | public: 17 | float* label; // real label 18 | int label_dim; // dimension of label (ex. 10 for digit recognition) 19 | public: 20 | Output(Layer* _prev, float* _label, int _label_dim, int _batch); 21 | virtual ~Output(); 22 | void forward(bool train = true); 23 | void backward(); 24 | void update(); 25 | }; 26 | 27 | } /* namespace layer */ 28 | #endif /* OUTPUT_CUH_ */ 29 | -------------------------------------------------------------------------------- /src/layer/pooling.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * pooling.cu 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "pooling.h" 9 | 10 | using namespace global; 11 | 12 | namespace layer { 13 | 14 | Pooling::Pooling(Layer* _prev, int size, int stride) { 15 | prev = _prev; 16 | prev->next = this; 17 | 18 | callCudnn(cudnnCreatePoolingDescriptor(&descriptor)); 19 | callCudnn(cudnnSetPooling2dDescriptor(descriptor, CUDNN_POOLING_MAX, 20 | size, size, 0, 0, stride, stride)); 21 | 22 | int _n, _c, _h, _w, _tmp; 23 | cudnnDataType_t _t; 24 | callCudnn(cudnnGetTensor4dDescriptor(prev->t_data, &_t, &_n, &_c, &_h, &_w, &_tmp, 25 | &_tmp, &_tmp, &_tmp)); 26 | batch = _n; 27 | callCudnn(cudnnCreateTensorDescriptor(&t_data)); 28 | callCudnn(cudnnSetTensor4dDescriptor(t_data, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, 29 | _n, _c, _h / stride, _w / stride)); 30 | data_size = _n * _c * (_h / stride) * (_w / stride); 31 | callCuda(cudaMalloc(&data, sizeof(float) * data_size)); 32 | callCuda(cudaMalloc(&diff, sizeof(float) * prev->data_size)); 33 | utils::setGpuValue(diff, prev->data_size, 0); 34 | 35 | param_size = 0; 36 | param_bias_size = 0; 37 | } 38 | 39 | Pooling::~Pooling() { 40 | callCudnn(cudnnDestroyPoolingDescriptor(descriptor)); 41 | callCudnn(cudnnDestroyTensorDescriptor(t_data)); 42 | callCuda(cudaFree(data)); 43 | callCuda(cudaFree(diff)); 44 | } 45 | 46 | void Pooling::forward(bool train) { 47 | float a = 1; 48 | float b = 0; 49 | callCudnn(cudnnPoolingForward(cudnnHandle, descriptor, &a, prev->t_data, 50 | prev->data, &b, t_data, data)); 51 | } 52 | 53 | void Pooling::backward() { 54 | float a = 1; 55 | float b = 0; 56 | callCudnn(cudnnPoolingBackward(cudnnHandle, descriptor, &a, 57 | t_data, data, t_data, next->diff, 58 | prev->t_data, prev->data, &b, prev->t_data, diff)); 59 | } 60 | 61 | void Pooling::update() { 62 | // nothing 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /src/layer/pooling.h: -------------------------------------------------------------------------------- 1 | /* 2 | * pooling.cuh 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #ifndef POOLING_CUH_ 9 | #define POOLING_CUH_ 10 | 11 | #include "layer.h" 12 | 13 | namespace layer { 14 | 15 | class Pooling : public Layer { 16 | private: 17 | cudnnPoolingDescriptor_t descriptor; 18 | public: 19 | Pooling(Layer* _prev, int size, int stride); 20 | virtual ~Pooling(); 21 | void forward(bool train = true); 22 | void backward(); 23 | void update(); 24 | }; 25 | 26 | } /* namespace layer */ 27 | #endif /* POOLING_CUH_ */ 28 | -------------------------------------------------------------------------------- /src/layer/relu.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * relu.cu 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "relu.h" 9 | 10 | using namespace global; 11 | 12 | namespace layer { 13 | 14 | ReLU::ReLU(Layer* _prev, int _output_size, float dropout_rate, float alpha, 15 | float sigma, float momentum, float weight_decay): 16 | Neuron(_prev, _output_size, dropout_rate, alpha, sigma) {} 17 | 18 | ReLU::~ReLU() {} 19 | 20 | void ReLU::forward_activation() { 21 | float a = 1; 22 | float b = 0; 23 | callCudnn(cudnnActivationForward(cudnnHandle, CUDNN_ACTIVATION_RELU, &a, 24 | t_data, tmp_data, &b, t_data, data)); 25 | } 26 | 27 | void ReLU::backward_activation() { 28 | float a = 1; 29 | float b = 0; 30 | callCudnn(cudnnActivationBackward(cudnnHandle, CUDNN_ACTIVATION_RELU, &a, 31 | t_data, data, t_data, next->diff, 32 | t_data, tmp_data, &b, t_data, tmp_diff)); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/layer/relu.h: -------------------------------------------------------------------------------- 1 | /* 2 | * relu.cuh 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #ifndef RELU_CUH_ 9 | #define RELU_CUH_ 10 | 11 | #include "neuron.h" 12 | 13 | namespace layer { 14 | 15 | class ReLU: public Neuron { 16 | public: 17 | ReLU(Layer* _prev, int _output_size, float dropout_rate, float alpha, 18 | float sigma = 0.01f, float momentum = 0.9f, float weight_decay = 0); 19 | virtual ~ReLU(); 20 | void forward_activation(); 21 | void backward_activation(); 22 | }; 23 | 24 | } /* namespace layer */ 25 | #endif /* RELU_CUH_ */ 26 | -------------------------------------------------------------------------------- /src/layer/softmax.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * softmax.cu 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "softmax.h" 9 | 10 | using namespace global; 11 | 12 | namespace layer { 13 | 14 | Softmax::Softmax(Layer* _prev, int _output_size, float dropout_rate, float alpha, 15 | float sigma, float momentum, float weight_decay): 16 | Neuron(_prev, _output_size, dropout_rate, alpha, sigma) {} 17 | 18 | Softmax::~Softmax() {} 19 | 20 | void Softmax::forward_activation() { 21 | float a = 1; 22 | float b = 0; 23 | callCudnn(cudnnSoftmaxForward(cudnnHandle, CUDNN_SOFTMAX_FAST, 24 | CUDNN_SOFTMAX_MODE_CHANNEL, &a, t_data, tmp_data, &b, t_data, data)); 25 | } 26 | 27 | void Softmax::backward_activation() { 28 | float a = 1.0 / batch; 29 | float b = 0; 30 | callCudnn(cudnnSoftmaxBackward(cudnnHandle, CUDNN_SOFTMAX_FAST, 31 | CUDNN_SOFTMAX_MODE_CHANNEL, &a, t_data, data, t_data, next->diff, 32 | &b, t_data, tmp_diff)); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/layer/softmax.h: -------------------------------------------------------------------------------- 1 | /* 2 | * softmax.cuh 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #ifndef SOFTMAX_CUH_ 9 | #define SOFTMAX_CUH_ 10 | 11 | #include "neuron.h" 12 | 13 | namespace layer { 14 | 15 | class Softmax: public Neuron { 16 | public: 17 | Softmax(Layer* _prev, int _output_size, float dropout_rate, float alpha, 18 | float sigma = 0.01f, float momentum = 0.9f, float weight_decay = 0); 19 | virtual ~Softmax(); 20 | void forward_activation(); 21 | void backward_activation(); 22 | }; 23 | 24 | } /* namespace layer */ 25 | #endif /* SOFTMAX_CUH_ */ 26 | -------------------------------------------------------------------------------- /src/model/network.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * network.cpp 3 | * 4 | * Created on: Sep 21, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "network.h" 9 | 10 | using namespace layer; 11 | using namespace std; 12 | 13 | namespace model { 14 | 15 | Network::Network(float* _data, int _data_dim, float* _label, int _label_dim, 16 | int count, int _val_size, int _batch) { 17 | h_data = _data; 18 | h_label = _label; 19 | size = count; 20 | val_size = _val_size; 21 | batch = _batch; 22 | data_dim = _data_dim; 23 | label_dim = _label_dim; 24 | train_error = 100; 25 | val_error = 100; 26 | lambda = 1; 27 | callCuda(cudaMalloc(&data, sizeof(float) * data_dim * batch)); 28 | callCuda(cudaMemcpy(data, h_data, sizeof(float) * data_dim * batch, 29 | cudaMemcpyHostToDevice)); 30 | callCuda(cudaMalloc(&label, sizeof(float) * label_dim * batch)); 31 | callCuda(cudaMemcpy(label, h_label, sizeof(float) * label_dim * batch, 32 | cudaMemcpyHostToDevice)); 33 | } 34 | 35 | Network::~Network() { 36 | h_data = NULL; 37 | h_label = NULL; 38 | callCuda(cudaFree(data)); 39 | callCuda(cudaFree(label)); 40 | for (Layer* l : layers) 41 | delete l; 42 | } 43 | 44 | pair Network::GetData() { 45 | return make_pair(data, label); 46 | } 47 | 48 | void Network::Train(int iteration, float half_time, float half_rate, 49 | float step_decrease, bool debug) { 50 | 51 | // train the network multiple times 52 | 53 | for (int k = 0; k < iteration && lambda > 5e-3; k++) { 54 | if (debug) 55 | for (int i = layers.size() - 1; i > 0; i--) { 56 | if (layers[i]->param_size != 0) 57 | utils::printGpuMax(layers[i]->param, layers[i]->param_size); 58 | } 59 | 60 | // divide the training set to small pieces 61 | 62 | int offset = 0; 63 | std::cout << "Iteration " << k + 1 << std::endl; 64 | for (int b = 0; b < size / batch; b++) { 65 | 66 | // choose a new piece and its labels 67 | 68 | callCuda(cudaMemcpy(data, h_data + offset * data_dim, 69 | sizeof(float) * data_dim * batch, cudaMemcpyHostToDevice)); 70 | callCuda(cudaMemcpy(label, h_label + offset * label_dim, 71 | sizeof(float) * label_dim * batch, cudaMemcpyHostToDevice)); 72 | 73 | // forward propagation 74 | 75 | for (int i = 0; i < layers.size() - 1; i++) 76 | layers[i]->forward(); 77 | 78 | // back propagation 79 | 80 | for (int i = layers.size() - 1; i > 0; i--) { 81 | layers[i]->backward(); 82 | layers[i]->update(); // update the parameters 83 | } 84 | offset += batch; 85 | 86 | } 87 | 88 | for (int i = layers.size() - 1; i > 0; i--) 89 | layers[i]->adjust_learning(step_decrease); 90 | 91 | // training error 92 | 93 | if (size > 0) { 94 | float* predict = new float[size]; 95 | offset = 0; 96 | for (int b = 0; b < size / batch; b++) { 97 | callCuda(cudaMemcpy(data, h_data + offset * data_dim, 98 | sizeof(float) * data_dim * batch, cudaMemcpyHostToDevice)); 99 | for (int i = 0; i < layers.size(); i++) 100 | layers[i]->forward(false); 101 | callCuda(cudaMemcpy(predict + offset * label_dim, 102 | layers[layers.size() - 1]->data, 103 | sizeof(float) * label_dim * batch, cudaMemcpyDeviceToHost)); 104 | offset += batch; 105 | } 106 | int errors = 0; 107 | for (int i = 0; i < size; i++) 108 | if (abs(h_label[i] - predict[i]) > 0.1) 109 | errors++; 110 | 111 | train_error = errors * 100.0 / size; 112 | std::cout << "Train error: " << train_error << std::endl; 113 | delete[] predict; 114 | } 115 | 116 | // validation error 117 | 118 | if (val_size > 0) { 119 | float* predict = new float[val_size]; 120 | offset = 0; 121 | for (int b = 0; b < val_size / batch; b++) { 122 | callCuda(cudaMemcpy(data, h_data + (size + offset) * data_dim, 123 | sizeof(float) * data_dim * batch, cudaMemcpyHostToDevice)); 124 | for (int i = 0; i < layers.size(); i++) 125 | layers[i]->forward(false); 126 | callCuda(cudaMemcpy(predict + offset * label_dim, 127 | layers[layers.size() - 1]->data, 128 | sizeof(float) * label_dim * batch, cudaMemcpyDeviceToHost)); 129 | offset += batch; 130 | } 131 | int errors = 0; 132 | for (int i = 0; i < val_size; i++) 133 | if (abs(h_label[size + i] - predict[i]) > 0.1) 134 | errors++; 135 | 136 | float prev_error = val_error; 137 | val_error = errors * 100.0 / val_size; 138 | std::cout << "Validation error: " << val_error << std::endl; 139 | 140 | // adjust the learning rate if the validation error stabilizes 141 | 142 | if ((prev_error - val_error) / prev_error < half_time) { 143 | lambda *= half_rate; 144 | std::cout << "-- Learning rate decreased --" << std::endl; 145 | for (int i = layers.size() - 1; i > 0; i--) 146 | layers[i]->adjust_learning(half_rate); 147 | } 148 | 149 | delete[] predict; 150 | } 151 | } 152 | 153 | } 154 | 155 | void Network::PushInput(int c, int h, int w) { 156 | Input* input = new Input(batch, c, h, w, data); 157 | layers.push_back(input); 158 | } 159 | 160 | void Network::PushOutput(int label_dim) { 161 | Output* output = new Output(layers.back(), label, label_dim, batch); 162 | layers.push_back(output); 163 | } 164 | 165 | void Network::PushConvolution(int c, int kernel, float alpha, float sigma, 166 | float momentum, float weight_decay) { 167 | Convolution* conv = new Convolution(layers.back(), batch, c, kernel, alpha, 168 | sigma, momentum, weight_decay); 169 | layers.push_back(conv); 170 | } 171 | 172 | void Network::PushPooling(int size, int stride) { 173 | Pooling* pool = new Pooling(layers.back(), size, stride); 174 | layers.push_back(pool); 175 | } 176 | 177 | void Network::PushActivation(cudnnActivationMode_t mode) { 178 | Activation* activation = new Activation(layers.back(), mode); 179 | layers.push_back(activation); 180 | } 181 | 182 | void Network::PushReLU(int output_size, float dropout_rate, float alpha, 183 | float sigma, float momentum, float weight_decay) { 184 | ReLU* relu = new ReLU(layers.back(), output_size, dropout_rate, alpha, 185 | sigma, momentum, weight_decay); 186 | layers.push_back(relu); 187 | } 188 | 189 | void Network::PushSoftmax(int output_size, float dropout_rate, float alpha, 190 | float sigma, float momentum, float weight_decay) { 191 | Softmax* softmax = new Softmax(layers.back(), output_size, dropout_rate, alpha, 192 | sigma, momentum, weight_decay); 193 | layers.push_back(softmax); 194 | } 195 | 196 | void Network::Pop() { 197 | Layer* tmp = layers.back(); 198 | layers.pop_back(); 199 | delete tmp; 200 | layers.back()->next = NULL; 201 | } 202 | 203 | void Network::SwitchData(float* h_data, float* h_label, int count) { 204 | // switch data without modifying the batch size 205 | size = count; 206 | this->h_data = h_data; 207 | this->h_label = h_label; 208 | } 209 | 210 | void Network::Test(float* label) { 211 | int offset = 0; 212 | for (int b = 0; b < size / batch; b++) { 213 | callCuda(cudaMemcpy(data, h_data + offset * data_dim, 214 | sizeof(float) * data_dim * batch, cudaMemcpyHostToDevice)); 215 | for (int i = 0; i < layers.size(); i++) 216 | layers[i]->forward(false); 217 | callCuda(cudaMemcpy(label + offset * label_dim, 218 | layers[layers.size() - 1]->data, 219 | sizeof(float) * label_dim * batch, cudaMemcpyDeviceToHost)); 220 | offset += batch; 221 | } 222 | } 223 | 224 | void Network::PrintGeneral() { 225 | std::cout << "Neural Network" << std::endl; 226 | std::cout << "Layers: " << layers.size() << std::endl; 227 | int i = 0; 228 | for (Layer* l : layers) 229 | std::cout << " - " << i++ << ' ' << l->data_size << ' ' << l->param_size << std::endl; 230 | } 231 | 232 | void Network::PrintData(int offset, int r, int c, int precision) { 233 | utils::printGpuMatrix(data + offset, r * c, r, c, precision); 234 | } 235 | 236 | void Network::ReadParams(std::string dir) { 237 | for (int i = 1; i < layers.size() - 1; i++) { 238 | if (layers[i]->param_size > 0) 239 | utils::readGPUMatrix(dir + std::to_string(i), layers[i]->param, layers[i]->param_size); 240 | if (layers[i]->param_bias_size > 0) 241 | utils::readGPUMatrix(dir + std::to_string(i) + "_bias", 242 | layers[i]->param_bias, layers[i]->param_bias_size); 243 | } 244 | } 245 | 246 | void Network::SaveParams(std::string dir) { 247 | for (int i = 1; i < layers.size() - 1; i++) { 248 | if (layers[i]->param_size > 0) 249 | utils::writeGPUMatrix(dir + std::to_string(i), layers[i]->param, 250 | layers[i]->param_size); 251 | if (layers[i]->param_bias_size > 0) 252 | utils::writeGPUMatrix(dir + std::to_string(i) + "_bias", 253 | layers[i]->param_bias, layers[i]->param_bias_size); 254 | } 255 | std::cout << "Params saved." << std::endl; 256 | } 257 | 258 | } /* namespace model */ 259 | -------------------------------------------------------------------------------- /src/model/network.h: -------------------------------------------------------------------------------- 1 | /* 2 | * network.h 3 | * 4 | * This class uses all layer class to construct a neural network, user should 5 | * have his data and label array in the host machine (and test data eventually). 6 | * By calling the constructor and layer inserter, user can build his own network 7 | * with adjusted parameters like layer size, learning rate, etc.. 8 | * 9 | * The learning method is gradient descent and the learning rate decreases when 10 | * validation error stabilizes. 11 | * 12 | * Created on: Sep 21, 2015 13 | * Author: lyx 14 | */ 15 | 16 | #ifndef NETWORK_H_ 17 | #define NETWORK_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "../layer/layer.h" 25 | #include "../layer/input.h" 26 | #include "../layer/output.h" 27 | #include "../layer/convolution.h" 28 | #include "../layer/pooling.h" 29 | #include "../layer/activation.h" 30 | #include "../layer/relu.h" 31 | #include "../layer/softmax.h" 32 | 33 | #include "../utils/read_data.h" 34 | #include "../utils/write_data.h" 35 | 36 | using namespace layer; 37 | 38 | namespace model { 39 | 40 | class Network { 41 | private: 42 | std::vector layers; // list of layers 43 | float* data; // input on device 44 | float* h_data; // input on host 45 | int data_dim; // dimension of one input 46 | float* label; // label on device 47 | float* h_label; // label on host 48 | int label_dim; // dimension of one label (usually 1) 49 | int size, val_size, batch; // whole size of data, batch size 50 | 51 | float train_error; // previous error rate 52 | float val_error; 53 | float lambda; // cumulative learning rate adjustment 54 | public: 55 | /* 56 | * Constructor 57 | * 58 | * data: pointer to host data array (include training and validation data) 59 | * data_dim: dimension of one single datum 60 | * label: pointer to host label array 61 | * label_dim: dimension of label, usually 1 62 | * train_size: training data count 63 | * val_size: validation data count 64 | * batch_size: batch size, common for all layers 65 | */ 66 | Network(float* data, int data_dim, float* label, int label_dim, 67 | int train_size, int val_size, int batch_size); 68 | 69 | virtual ~Network(); 70 | 71 | /* 72 | * Get data and label pointer in order to change it from the outside 73 | */ 74 | std::pair GetData(); 75 | 76 | /* 77 | * Train the network: the start may be slow, need to change sigma of initial 78 | * weight or adjust learning rate, etc. 79 | * 80 | * iteration: number of epoch 81 | * half_time: threshold for changing the learning rate 82 | * half_rate: learning rate adjustment 83 | * step_decrease: decrease learning rate by each batch 84 | * debug: debug mode (print some extra information) 85 | */ 86 | void Train(int iteration, float half_time = 0, float half_rate = 0.5, 87 | float step_decrease = 1, bool debug = false); 88 | 89 | /* 90 | * Insert input layer (should be called at first) 91 | * 92 | * c: channel number 93 | * h: height 94 | * w: width 95 | */ 96 | void PushInput(int c, int h, int w); 97 | 98 | /* 99 | * Insert output layer (should be called in the end) 100 | * 101 | * label_count: label dimension (example: 10 for digits) 102 | */ 103 | void PushOutput(int label_count); 104 | 105 | /* 106 | * Insert convolutional layer without activation layer (mode NCHW) 107 | * 108 | * c: channel of the layer 109 | * kernel: square kernel size 110 | * alpha: initial learning rate 111 | * sigma: initial weight distribution 112 | * momentum: momentum of gradient when learning 113 | * weight_decay: decay rate of parameters 114 | */ 115 | void PushConvolution(int c, int kernel, float alpha, float sigma = 0.01f, 116 | float momentum = 0.9f, float weight_decay = 0); 117 | 118 | /* 119 | * Insert max pooling layer 120 | * 121 | * size: size of the square pool 122 | * stride: overlapping 123 | */ 124 | void PushPooling(int size, int stride); 125 | 126 | /* 127 | * Insert activation layer (after convolutional layer) 128 | * 129 | * mode: activation function (CuDNN constant) 130 | */ 131 | void PushActivation(cudnnActivationMode_t mode); 132 | 133 | /* 134 | * Insert fully-connected layer with ReLU as activation function 135 | * 136 | * output_size: output size of the current layer 137 | * dropout_rate: rate of dropout 138 | * alpha: initial learning rate 139 | * sigma: initial weight distribution 140 | * momentum: momentum of gradient when learning 141 | * weight_decay: decay rate of parameters 142 | */ 143 | void PushReLU(int output_size, float dropout_rate, float alpha, 144 | float sigma = 0.01f, float momentum = 0.9f, float weight_decay = 0); 145 | 146 | /* 147 | * Insert fully-connected layer with Softmax as activation function 148 | * 149 | * output_size: output size of the current layer 150 | * dropout_rate: rate of dropout 151 | * alpha: initial learning rate 152 | * sigma: initial weight distribution 153 | * momentum: momentum of gradient when learning 154 | * weight_decay: decay rate of parameters 155 | */ 156 | void PushSoftmax(int output_size, float dropout_rate, float alpha, 157 | float sigma = 0.01f, float momentum = 0.9f, float weight_decay = 0); 158 | 159 | /* 160 | * Remove last layer 161 | */ 162 | void Pop(); 163 | 164 | /* 165 | * Switch the host data, used when data size is huge or when testing 166 | * 167 | * h_data: pointer to new data array 168 | * h_label: pointer to new label array 169 | * size: data size 170 | */ 171 | void SwitchData(float* h_data, float* h_label, int size); 172 | 173 | /* 174 | * Test the network, used after switching the test data 175 | * 176 | * h_label: label array to store the prediction 177 | */ 178 | void Test(float* h_label); 179 | 180 | /* 181 | * Print general information about layers 182 | */ 183 | void PrintGeneral(); 184 | 185 | /* 186 | * Print a datum 187 | * 188 | * offset: position of datum 189 | * r: row 190 | * c: column 191 | */ 192 | void PrintData(int offset, int r, int c, int precision); 193 | 194 | /* 195 | * Get parameters from files (one folder with multiple files), the actual 196 | * network structure should match the saved one 197 | * 198 | * dir: path to the folder 199 | */ 200 | void ReadParams(std::string dir); 201 | 202 | /* 203 | * Save parameters to files (one folder with multiple files) 204 | * 205 | * dir: path to the folder 206 | */ 207 | void SaveParams(std::string dir); 208 | }; 209 | 210 | } /* namespace model */ 211 | #endif /* NETWORK_H_ */ 212 | -------------------------------------------------------------------------------- /src/utils/global.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * global.cpp 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "global.h" 9 | 10 | namespace global { 11 | 12 | cudnnHandle_t cudnnHandle; 13 | cublasHandle_t cublasHandle; 14 | 15 | } /* namespace global */ 16 | -------------------------------------------------------------------------------- /src/utils/global.h: -------------------------------------------------------------------------------- 1 | /* 2 | * global.h 3 | * 4 | * Global instances such as CUDA handle 5 | * 6 | * Created on: Sep 20, 2015 7 | * Author: lyx 8 | */ 9 | 10 | #ifndef GLOBAL_H_ 11 | #define GLOBAL_H_ 12 | 13 | #include "cublas_v2.h" 14 | #include "cudnn.h" 15 | #include 16 | 17 | namespace global { 18 | 19 | extern cudnnHandle_t cudnnHandle; 20 | extern cublasHandle_t cublasHandle; 21 | const std::string root = "/home/lyx/workspace/cuda/XNet/"; 22 | 23 | } /* namespace global */ 24 | #endif /* GLOBAL_H_ */ 25 | -------------------------------------------------------------------------------- /src/utils/image.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * image.cpp 3 | * 4 | * Created on: Oct 18, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "image.h" 9 | 10 | namespace utils { 11 | 12 | void showImage(float* img, int width, int height, int channel, int offset) { 13 | img += offset * width * height * channel; 14 | Mat I(height, width, CV_8UC3, Scalar::all(0)); 15 | for (int i = 0; i < width; i++) 16 | for (int j = 0; j < height; j++) 17 | for (int c = 0; c < channel; c++) 18 | I.at(j, i).val[c] = 19 | uchar(img[width * height * c + width * j + i] * 255); 20 | imshow("Image", I); 21 | waitKey(0); 22 | } 23 | 24 | void flipImage(float* src, float* dest, int width, int height, int channel) { 25 | for (int i = 0; i < width; i++) 26 | for (int j = 0; j < height; j++) 27 | for (int c = 0; c < channel; c++) 28 | dest[width * height * c + width * j + i] = 29 | src[width * height * c + width * j + (width - i - 1)]; 30 | } 31 | 32 | void translationAugmentation(float* old_m, int old_n, int old_w, int old_h, 33 | int delta, float* m, int n, int c, int w, int h) { 34 | int _k = 0; 35 | for (int i = 0; i <= delta; i += delta) 36 | for (int j = 0; j <= delta; j += delta) 37 | for (int k = 0; k < old_n; k++) 38 | for (int x = 0; x < c; x++) 39 | for (int y = 0; y < h; y++) 40 | for (int z = 0; z < w; z++) { 41 | m[_k] = old_m[k * c * old_w * old_h + x * old_w * old_h + 42 | (y + i) * old_w + (z + j)]; 43 | _k++; 44 | } 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/utils/image.h: -------------------------------------------------------------------------------- 1 | /* 2 | * image.h 3 | * 4 | * Image utilities with data augmentation 5 | * 6 | * Created on: Oct 18, 2015 7 | * Author: lyx 8 | */ 9 | 10 | #ifndef IMAGE_H_ 11 | #define IMAGE_H_ 12 | 13 | #include "opencv2/opencv.hpp" 14 | 15 | using namespace cv; 16 | using namespace std; 17 | 18 | namespace utils { 19 | 20 | /* 21 | * Display the image with OpenCV 22 | * 23 | * img: pointer to images array 24 | * width: image width 25 | * height: image height 26 | * channel: image channel 27 | * offset: index of image in the array 28 | */ 29 | void showImage(float* img, int width, int height, int channel, int offset = 0); 30 | 31 | /* 32 | * Flip an image horizontally 33 | * 34 | * src: pointer of old image 35 | * dest: pointer of new image 36 | * width: image width 37 | * height: image height 38 | * channel: image channel 39 | */ 40 | void flipImage(float* src, float* dest, int width, int height, int channel); 41 | 42 | /* 43 | * Translate an image and get 4 cropped images 44 | * 45 | * old_m: pointer to old images 46 | * old_n: old images count 47 | * old_w: old image width 48 | * old_h: old image height 49 | * delta: crop size 50 | * m: pointer to new images array 51 | * n: new images count 52 | * c: image channel (does not change) 53 | * w: new image width 54 | * h: new image height 55 | */ 56 | void translationAugmentation(float* old_m, int old_n, int old_w, int old_h, 57 | int delta, float* m, int n, int c, int w, int h); 58 | 59 | } 60 | 61 | #endif /* IMAGE_H_ */ 62 | -------------------------------------------------------------------------------- /src/utils/print.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * print.cu 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "print.h" 9 | 10 | namespace utils { 11 | 12 | void printCpuMatrix(float* m, int n, int r, int c, int precision) { 13 | std::cout << "Size: " << r << " * " << c << std::endl; 14 | for (int i = 0; i < r; i++) { 15 | for (int j = 0; j < c; j++) { 16 | if (i + j * r >= n) 17 | break; 18 | std::cout << std::fixed << std::setprecision(precision) << m[i + j * r] << '\t'; 19 | } 20 | std::cout << std::endl; 21 | } 22 | std::cout << std::endl; 23 | } 24 | 25 | void printGpuMatrix(float* d_m, int n, int r, int c, int precision) { 26 | float* m = new float[n]; 27 | cublasGetVector(n, sizeof(*m), d_m, 1, m, 1); 28 | std::cout << "Size: " << r << " * " << c << std::endl; 29 | for (int i = 0; i < r; i++) { 30 | for (int j = 0; j < c; j++) 31 | std::cout << std::fixed << std::setprecision(precision) << m[i + j * r] << '\t'; 32 | std::cout << std::endl; 33 | } 34 | std::cout << std::endl; 35 | delete[] m; 36 | } 37 | 38 | void printGpuMax(float* d_m, int n, int precision) { 39 | int result; 40 | callCuda(cublasIsamax(global::cublasHandle, n, d_m, 1, &result)); 41 | float ans; 42 | callCuda(cudaMemcpy(&ans, d_m + result - 1, sizeof(float), cudaMemcpyDeviceToHost)); 43 | std::cout << std::fixed << std::setprecision(precision) << 44 | "Max: " << ans << std::endl; 45 | 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/utils/print.h: -------------------------------------------------------------------------------- 1 | /* 2 | * print.h 3 | * 4 | * Methods to print out host or device array 5 | * 6 | * Created on: Sep 8, 2015 7 | * Author: lyx 8 | */ 9 | 10 | #ifndef PRINT_H_ 11 | #define PRINT_H_ 12 | 13 | #include 14 | #include 15 | 16 | #include "cuda_runtime.h" 17 | #include "cublas_v2.h" 18 | 19 | #include "utils.h" 20 | #include "global.h" 21 | 22 | namespace utils { 23 | 24 | void printCpuMatrix(float* m, int n, int r, int c = 1, int precision = 2); 25 | void printGpuMatrix(float* d_m, int n, int r, int c = 1, int precision = 2); 26 | 27 | /* 28 | * Print the maximum of an array 29 | * 30 | * d_m: pointer to array 31 | * n: array length 32 | * precision: display precision 33 | */ 34 | void printGpuMax(float* d_m, int n, int precision = 4); 35 | 36 | } 37 | 38 | #endif /* PRINT_H_ */ 39 | -------------------------------------------------------------------------------- /src/utils/read_data.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * read_data.cpp 3 | * 4 | * Created on: Sep 28, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "read_data.h" 9 | 10 | namespace utils { 11 | 12 | void readInt(std::ifstream& stream, int* val) { 13 | // little endian 14 | for (int i = sizeof(int) - 1; i >= 0; i--) 15 | stream.read(((char*)val) + i, 1); 16 | } 17 | 18 | void readBytes(std::ifstream& stream, uint8_t* val, int n) { 19 | stream.read((char*)val, n); 20 | } 21 | 22 | void readFloat(std::ifstream& stream, float* val, int n) { 23 | stream.read((char*)val, sizeof(float) * n); 24 | } 25 | 26 | void readCPUMatrix(std::string dir, float* val, int n) { 27 | std::ifstream input(dir); 28 | readFloat(input, val, n); 29 | input.close(); 30 | } 31 | 32 | void readGPUMatrix(std::string dir, float* d_val, int n) { 33 | float* val = new float[n]; 34 | readCPUMatrix(dir, val, n); 35 | cudaMemcpy(d_val, val, sizeof(float) * n, cudaMemcpyHostToDevice); 36 | delete[] val; 37 | } 38 | 39 | } 40 | 41 | 42 | -------------------------------------------------------------------------------- /src/utils/read_data.h: -------------------------------------------------------------------------------- 1 | /* 2 | * read_data.h 3 | * 4 | * Created on: Sep 28, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #ifndef READ_DATA_H_ 9 | #define READ_DATA_H_ 10 | 11 | #include 12 | 13 | #include "cuda_runtime.h" 14 | #include "cublas_v2.h" 15 | 16 | namespace utils { 17 | 18 | void readInt(std::ifstream& stream, int* val); 19 | void readBytes(std::ifstream& stream, uint8_t* val, int n); 20 | void readFloat(std::ifstream& stream, float* val, int n); 21 | void readCPUMatrix(std::string dir, float* val, int n); 22 | void readGPUMatrix(std::string dir, float* d_val, int n); 23 | 24 | } 25 | 26 | #endif /* READ_DATA_H_ */ 27 | -------------------------------------------------------------------------------- /src/utils/set_value.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * set_value.cu 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "set_value.h" 9 | 10 | using namespace global; 11 | 12 | namespace utils { 13 | 14 | __global__ void setValue(float* x, int n, float val) { 15 | int i = blockDim.x * blockIdx.x + threadIdx.x; 16 | if (i < n) 17 | x[i] = val; 18 | } 19 | 20 | __global__ void dropout(float* x, float* t, int n, float threshold) { 21 | int i = blockDim.x * blockIdx.x + threadIdx.x; 22 | if (i < n) 23 | if (t[i] < threshold) 24 | x[i] = 0; 25 | } 26 | 27 | __global__ void scale(float* x, int n, float epsilon) { 28 | int i = blockDim.x * blockIdx.x + threadIdx.x; 29 | if (i < n) 30 | x[i] = x[i] * epsilon * 2 - epsilon; 31 | } 32 | 33 | void setGpuValue(float* x, int n, float val) { 34 | int threadsPerBlock = 256; 35 | int blocksPerGrid = (n + threadsPerBlock - 1) / threadsPerBlock; 36 | setValue<<>>(x, n, val); 37 | } 38 | 39 | void setGpuUniformValue(float* x, int n, float epsilon) { 40 | int threadsPerBlock = 256; 41 | int blocksPerGrid = (n + threadsPerBlock - 1) / threadsPerBlock; 42 | curandGenerator_t generator; 43 | curandCreateGenerator(&generator, CURAND_RNG_PSEUDO_MTGP32); 44 | curandSetPseudoRandomGeneratorSeed(generator, time(NULL)); 45 | curandGenerateUniform(generator, x, n); 46 | scale<<>>(x, n, epsilon); 47 | curandDestroyGenerator(generator); 48 | } 49 | 50 | void setGpuNormalValue(float* x, int n, float mean, float stddev) { 51 | curandGenerator_t generator; 52 | curandCreateGenerator(&generator, CURAND_RNG_PSEUDO_MTGP32); 53 | curandSetPseudoRandomGeneratorSeed(generator, time(NULL)); 54 | curandGenerateNormal(generator, x, n, mean, stddev); 55 | curandDestroyGenerator(generator); 56 | } 57 | 58 | void dropGpuValue(float *x, int n, float dropout_rate) { 59 | int threadsPerBlock = 256; 60 | int blocksPerGrid = (n + threadsPerBlock - 1) / threadsPerBlock; 61 | curandGenerator_t generator; 62 | curandCreateGenerator(&generator, CURAND_RNG_PSEUDO_MTGP32); 63 | curandSetPseudoRandomGeneratorSeed(generator, time(NULL)); 64 | float* t; 65 | cudaMalloc((void**)&t, sizeof(float) * n); 66 | curandGenerateUniform(generator, t, n); 67 | dropout<<>>(x, t, n, dropout_rate); 68 | cudaFree(t); 69 | curandDestroyGenerator(generator); 70 | } 71 | 72 | void scaleGpuValue(float *x, int n, float scale) { 73 | cublasSscal(cublasHandle, n, &scale, x, 1); 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /src/utils/set_value.h: -------------------------------------------------------------------------------- 1 | /* 2 | * set_value.h 3 | * 4 | * Utility functions to set or change value to host or device array 5 | * 6 | * Created on: Sep 8, 2015 7 | * Author: lyx 8 | */ 9 | 10 | #ifndef SET_VALUE_H_ 11 | #define SET_VALUE_H_ 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include "cuda_runtime.h" 18 | #include "cublas_v2.h" 19 | #include "curand.h" 20 | 21 | #include "utils.h" 22 | #include "global.h" 23 | #include "print.h" 24 | 25 | namespace utils { 26 | 27 | /* 28 | * Set one single value to all the array 29 | * 30 | * x: pointer to array 31 | * n: array length 32 | * val: value 33 | */ 34 | void setGpuValue(float* x, int n, float val); 35 | 36 | /* 37 | * Set uniformly distributed value 38 | * 39 | * x: pointer to array 40 | * n: array length 41 | * epsilon: value interval 42 | */ 43 | void setGpuUniformValue(float* x, int n, float epsilon); 44 | 45 | /* 46 | * Set Gaussian distributed value 47 | * 48 | * x: pointer to array 49 | * n: array length 50 | * mean: mean of Gaussian distribution 51 | * stddev: standard deviation of Gaussian distribution 52 | */ 53 | void setGpuNormalValue(float* x, int n, float mean, float stddev); 54 | 55 | /* 56 | * Set part of the array to zero (dropout) 57 | * 58 | * x: pointer to array 59 | * n: array length 60 | * dropout_rate: proportion of array to be set to 0 61 | */ 62 | void dropGpuValue(float* x, int n, float dropout_rate); 63 | 64 | /* 65 | * Scale the array 66 | * 67 | * x: pointer to array 68 | * n: array length 69 | * scale: scale rate 70 | */ 71 | void scaleGpuValue(float* x, int n, float scale); 72 | 73 | } 74 | 75 | #endif /* SET_VALUE_H_ */ 76 | -------------------------------------------------------------------------------- /src/utils/utils.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * utils.cpp 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "utils.h" 9 | 10 | namespace utils { 11 | 12 | 13 | 14 | } /* namespace global */ 15 | -------------------------------------------------------------------------------- /src/utils/utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * utils.h 3 | * 4 | * Created on: Sep 20, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #ifndef UTILS_H_ 9 | #define UTILS_H_ 10 | 11 | #include 12 | #include 13 | 14 | #include 15 | #include 16 | #include 17 | 18 | #define FatalError(s) do { \ 19 | std::stringstream _where, _message; \ 20 | _where << __FILE__ << ':' << __LINE__; \ 21 | _message << std::string(s) + "\n" << __FILE__ << ':' << __LINE__; \ 22 | std::cerr << _message.str() << "\nAborting...\n"; \ 23 | cudaDeviceReset(); \ 24 | exit(1); \ 25 | } while(0) 26 | 27 | #define callCudnn(status) do { \ 28 | std::stringstream _error; \ 29 | if (status != CUDNN_STATUS_SUCCESS) { \ 30 | _error << "CuDNN failure: " << cudnnGetErrorString(status); \ 31 | FatalError(_error.str()); \ 32 | } \ 33 | } while(0) 34 | 35 | #define callCurand(status) do { \ 36 | std::stringstream _error; \ 37 | if (status != CURAND_STATUS_SUCCESS) { \ 38 | _error << "CuRAND failure: " << status; \ 39 | FatalError(_error.str()); \ 40 | } \ 41 | } while(0) 42 | 43 | #define callCuda(status) do { \ 44 | std::stringstream _error; \ 45 | if (status != 0) { \ 46 | _error << "Cuda failure: " << status; \ 47 | FatalError(_error.str()); \ 48 | } \ 49 | } while(0) 50 | 51 | namespace utils { 52 | 53 | 54 | } /* namespace utils */ 55 | #endif /* UTILS_H_ */ 56 | -------------------------------------------------------------------------------- /src/utils/write_data.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * write_data.cpp 3 | * 4 | * Created on: Oct 3, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #include "write_data.h" 9 | 10 | namespace utils { 11 | 12 | void writeGPUMatrix(std::string dir, float* d_val, int n) { 13 | float* val = new float[n]; 14 | cudaMemcpy(val, d_val, sizeof(float) * n, cudaMemcpyDeviceToHost); 15 | std::ofstream out(dir); 16 | out.write((char*)val, sizeof(float) * n); 17 | out.close(); 18 | delete[] val; 19 | } 20 | 21 | void writeCPUMatrix(std::string dir, float* val, int n) { 22 | std::ofstream out(dir); 23 | out.write((char*)val, sizeof(float) * n); 24 | out.close(); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/utils/write_data.h: -------------------------------------------------------------------------------- 1 | /* 2 | * write_data.h 3 | * 4 | * Created on: Oct 3, 2015 5 | * Author: lyx 6 | */ 7 | 8 | #ifndef WRITE_DATA_H_ 9 | #define WRITE_DATA_H_ 10 | 11 | #include 12 | 13 | #include "cuda_runtime.h" 14 | #include "cublas_v2.h" 15 | 16 | #include "utils.h" 17 | 18 | namespace utils { 19 | 20 | void writeGPUMatrix(std::string dir, float* d_val, int n); 21 | void writeCPUMatrix(std::string dir, float* val, int n); 22 | 23 | } 24 | 25 | #endif /* WRITE_DATA_H_ */ 26 | -------------------------------------------------------------------------------- /src/xnet.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * xnet.cpp 3 | * 4 | * XNet is a simple CuDNN wrapper for deep learning 5 | * 6 | * Created on: Sep 20, 2015 7 | * Author: lyx 8 | */ 9 | 10 | #include "dataset/mnist.h" 11 | #include "dataset/cifar10.h" 12 | #include "dataset/imagenet200.h" 13 | #include "dataset/imagenet.h" 14 | 15 | using namespace std; 16 | 17 | int main() { 18 | cout << "XNet v1.0" << endl; 19 | callCuda(cublasCreate(&global::cublasHandle)); 20 | callCudnn(cudnnCreate(&global::cudnnHandle)); 21 | 22 | mnist::train(); 23 | //cifar10::train(); 24 | //imagenet200::train(); 25 | 26 | callCuda(cublasDestroy(global::cublasHandle)); 27 | callCudnn(cudnnDestroy(global::cudnnHandle)); 28 | cout << "End" << endl; 29 | return 0; 30 | } 31 | 32 | 33 | -------------------------------------------------------------------------------- /utils/ImageNet/cat2.file: -------------------------------------------------------------------------------- 1 | 01496331 10045 2 | 01496331 10047 3 | 01496331 10048 4 | 01496331 1010 5 | 01496331 10100 6 | 01496331 1037 7 | 01496331 10373 8 | 01496331 10438 9 | 01496331 1054 10 | 01496331 10606 11 | 01496331 10720 12 | 01496331 10734 13 | 01496331 10754 14 | 01496331 10903 15 | 01496331 10993 16 | 01496331 11015 17 | 01496331 11039 18 | 01496331 1104 19 | 01496331 11173 20 | 01496331 1119 21 | 01496331 1120 22 | 01496331 11242 23 | 01496331 11249 24 | 01496331 11332 25 | 01496331 1134 26 | 01496331 11391 27 | 01496331 1147 28 | 01496331 11480 29 | 01496331 11547 30 | 01496331 11594 31 | 01496331 11647 32 | 01496331 11655 33 | 01496331 11721 34 | 01496331 11732 35 | 01496331 11752 36 | 01496331 11758 37 | 01496331 11763 38 | 01496331 11853 39 | 01496331 11866 40 | 01496331 11875 41 | 01496331 11890 42 | 01496331 11989 43 | 01496331 12093 44 | 01496331 12107 45 | 01496331 12116 46 | 01496331 12164 47 | 01496331 12211 48 | 01496331 12280 49 | 01496331 12286 50 | 01496331 12312 51 | 01496331 12413 52 | 01496331 12414 53 | 01496331 12694 54 | 01496331 12737 55 | 01496331 12784 56 | 01496331 12814 57 | 01496331 12876 58 | 01496331 12885 59 | 01496331 12919 60 | 01496331 13019 61 | 01496331 13265 62 | 01496331 13322 63 | 01496331 13372 64 | 01496331 13407 65 | 01496331 13467 66 | 01496331 13516 67 | 01496331 13582 68 | 01496331 13654 69 | 01496331 1368 70 | 01496331 13822 71 | 01496331 13850 72 | 01496331 13911 73 | 01496331 13934 74 | 01496331 14032 75 | 01496331 14045 76 | 01496331 1411 77 | 01496331 1417 78 | 01496331 14225 79 | 01496331 1425 80 | 01496331 14320 81 | 01496331 14563 82 | 01496331 14584 83 | 01496331 14634 84 | 01496331 14714 85 | 01496331 14877 86 | 01496331 15030 87 | 01496331 15168 88 | 01496331 15173 89 | 01496331 15182 90 | 01496331 15239 91 | 01496331 15261 92 | 01496331 15294 93 | 01496331 1532 94 | 01496331 15364 95 | 01496331 15446 96 | 01496331 15475 97 | 01496331 15494 98 | 01496331 15507 99 | 01496331 15534 100 | 01496331 15536 101 | 01496331 15571 102 | 01496331 15577 103 | 01496331 15629 104 | 01496331 15688 105 | 01496331 15695 106 | 01496331 15801 107 | 01496331 15812 108 | 01496331 15848 109 | 01496331 15872 110 | 01496331 15906 111 | 01496331 15923 112 | 01496331 15992 113 | 01496331 16020 114 | 01496331 16224 115 | 01496331 16251 116 | 01496331 16346 117 | 01496331 16377 118 | 01496331 1660 119 | 01496331 16612 120 | 01496331 16970 121 | 01496331 16981 122 | 01496331 1703 123 | 01496331 17032 124 | 01496331 17187 125 | 01496331 17263 126 | 01496331 17308 127 | 01496331 17339 128 | 01496331 17373 129 | 01496331 17399 130 | 01496331 17411 131 | 01496331 17413 132 | 01496331 17422 133 | 01496331 17471 134 | 01496331 1758 135 | 01496331 17711 136 | 01496331 17713 137 | 01496331 17724 138 | 01496331 17746 139 | 01496331 17864 140 | 01496331 17869 141 | 01496331 17876 142 | 01496331 17882 143 | 01496331 17907 144 | 01496331 17979 145 | 01496331 18015 146 | 01496331 18218 147 | 01496331 18272 148 | 01496331 1831 149 | 01496331 18437 150 | 01496331 18501 151 | 01496331 18538 152 | 01496331 18571 153 | 01496331 18603 154 | 01496331 18622 155 | 01496331 18723 156 | 01496331 18738 157 | 01496331 18747 158 | 01496331 18825 159 | 01496331 18834 160 | 01496331 18847 161 | 01496331 18862 162 | 01496331 18994 163 | 01496331 19085 164 | 01496331 1913 165 | 01496331 19168 166 | 01496331 19233 167 | 01496331 19307 168 | 01496331 19346 169 | 01496331 19407 170 | 01496331 19514 171 | 01496331 19547 172 | 01496331 19550 173 | 01496331 19556 174 | 01496331 19586 175 | 01496331 19656 176 | 01496331 19684 177 | 01496331 19777 178 | 01496331 19803 179 | 01496331 19845 180 | 01496331 19886 181 | 01496331 19901 182 | 01496331 19907 183 | 01496331 19936 184 | 01496331 19953 185 | 01496331 19959 186 | 01496331 20036 187 | 01496331 2017 188 | 01496331 2025 189 | 01496331 20252 190 | 01496331 2052 191 | 01496331 2071 192 | 01496331 20778 193 | 01496331 2078 194 | 01496331 2079 195 | 01496331 20837 196 | 01496331 20851 197 | 01496331 20865 198 | 01496331 20899 199 | 01496331 20916 200 | 01496331 20930 201 | 01496331 20962 202 | 01496331 21049 203 | 01496331 21079 204 | 01496331 21114 205 | 01496331 21349 206 | 01496331 21391 207 | 01496331 21472 208 | 01496331 21482 209 | 01496331 21626 210 | 01496331 21710 211 | 01496331 21933 212 | 01496331 21942 213 | 01496331 21946 214 | 01496331 22079 215 | 01496331 2213 216 | 01496331 2222 217 | 01496331 22244 218 | 01496331 22259 219 | 01496331 22264 220 | 01496331 22345 221 | 01496331 2235 222 | 01496331 22355 223 | 01496331 22441 224 | 01496331 22631 225 | 01496331 22793 226 | 01496331 22808 227 | 01496331 22892 228 | 01496331 22970 229 | 01496331 22995 230 | 01496331 23039 231 | 01496331 23066 232 | 01496331 23080 233 | 01496331 23095 234 | 01496331 23133 235 | 01496331 23206 236 | 01496331 23232 237 | 01496331 23293 238 | 01496331 23471 239 | 01496331 23535 240 | 01496331 2359 241 | 01496331 2360 242 | 01496331 23623 243 | 01496331 23634 244 | 01496331 2367 245 | 01496331 23679 246 | 01496331 2368 247 | 01496331 23690 248 | 01496331 23726 249 | 01496331 2391 250 | 01496331 24176 251 | 01496331 2434 252 | 01496331 2455 253 | 01496331 24722 254 | 01496331 24825 255 | 01496331 24828 256 | 01496331 24843 257 | 01496331 25111 258 | 01496331 25183 259 | 01496331 25201 260 | 01496331 25215 261 | 01496331 25268 262 | 01496331 25279 263 | 01496331 25288 264 | 01496331 25327 265 | 01496331 25345 266 | 01496331 25380 267 | 01496331 25408 268 | 01496331 2543 269 | 01496331 25452 270 | 01496331 25473 271 | 01496331 2583 272 | 01496331 26148 273 | 01496331 26205 274 | 01496331 2629 275 | 01496331 263 276 | 01496331 26340 277 | 01496331 26433 278 | 01496331 26441 279 | 01496331 26726 280 | 01496331 26786 281 | 01496331 27026 282 | 01496331 2719 283 | 01496331 27447 284 | 01496331 27476 285 | 01496331 27667 286 | 01496331 27703 287 | 01496331 27918 288 | 01496331 28064 289 | 01496331 28114 290 | 01496331 28413 291 | 01496331 28460 292 | 01496331 28503 293 | 01496331 2858 294 | 01496331 2860 295 | 01496331 28667 296 | 01496331 29030 297 | 01496331 29113 298 | 01496331 29158 299 | 01496331 29203 300 | 01496331 29308 301 | 01496331 29524 302 | 01496331 29594 303 | 01496331 29665 304 | 01496331 29716 305 | 01496331 29794 306 | 01496331 29849 307 | 01496331 29876 308 | 01496331 29919 309 | 01496331 30043 310 | 01496331 30068 311 | 01496331 30274 312 | 01496331 30437 313 | 01496331 30585 314 | 01496331 3074 315 | 01496331 30942 316 | 01496331 31086 317 | 01496331 31125 318 | 01496331 31368 319 | 01496331 31421 320 | 01496331 31554 321 | 01496331 31637 322 | 01496331 31652 323 | 01496331 318 324 | 01496331 31876 325 | 01496331 31946 326 | 01496331 31998 327 | 01496331 32172 328 | 01496331 32223 329 | 01496331 3273 330 | 01496331 32804 331 | 01496331 32975 332 | 01496331 3304 333 | 01496331 33151 334 | 01496331 33236 335 | 01496331 33376 336 | 01496331 33617 337 | 01496331 3375 338 | 01496331 33801 339 | 01496331 33956 340 | 01496331 34020 341 | 01496331 34240 342 | 01496331 34251 343 | 01496331 34426 344 | 01496331 34468 345 | 01496331 34475 346 | 01496331 34826 347 | 01496331 3485 348 | 01496331 34862 349 | 01496331 34865 350 | 01496331 35047 351 | 01496331 3543 352 | 01496331 35446 353 | 01496331 35449 354 | 01496331 35504 355 | 01496331 35538 356 | 01496331 35664 357 | 01496331 35780 358 | 01496331 3581 359 | 01496331 3635 360 | 01496331 3685 361 | 01496331 3759 362 | 01496331 3823 363 | 01496331 3833 364 | 01496331 3886 365 | 01496331 3965 366 | 01496331 4045 367 | 01496331 4089 368 | 01496331 4131 369 | 01496331 4137 370 | 01496331 4208 371 | 01496331 4246 372 | 01496331 4320 373 | 01496331 4329 374 | 01496331 4505 375 | 01496331 4544 376 | 01496331 4583 377 | 01496331 4685 378 | 01496331 4710 379 | 01496331 4845 380 | 01496331 4987 381 | 01496331 5022 382 | 01496331 5099 383 | 01496331 5102 384 | 01496331 5254 385 | 01496331 5280 386 | 01496331 5302 387 | 01496331 5319 388 | 01496331 5376 389 | 01496331 5552 390 | 01496331 5617 391 | 01496331 5831 392 | 01496331 5902 393 | 01496331 5931 394 | 01496331 5957 395 | 01496331 6054 396 | 01496331 6068 397 | 01496331 612 398 | 01496331 6202 399 | 01496331 6237 400 | 01496331 63 401 | 01443537 10014 402 | 01443537 10034 403 | 01443537 10078 404 | 01443537 10086 405 | 01443537 10149 406 | 01443537 10171 407 | 01443537 10176 408 | 01443537 10213 409 | 01443537 10242 410 | 01443537 10268 411 | 01443537 1029 412 | 01443537 1034 413 | 01443537 10408 414 | 01443537 10446 415 | 01443537 10470 416 | 01443537 10482 417 | 01443537 10500 418 | 01443537 10547 419 | 01443537 10569 420 | 01443537 1062 421 | 01443537 1066 422 | 01443537 10712 423 | 01443537 10763 424 | 01443537 10779 425 | 01443537 10782 426 | 01443537 10788 427 | 01443537 1079 428 | 01443537 10791 429 | 01443537 10795 430 | 01443537 1103 431 | 01443537 11052 432 | 01443537 11078 433 | 01443537 11084 434 | 01443537 11165 435 | 01443537 11207 436 | 01443537 11213 437 | 01443537 1124 438 | 01443537 11251 439 | 01443537 11283 440 | 01443537 11292 441 | 01443537 11300 442 | 01443537 11348 443 | 01443537 11349 444 | 01443537 1135 445 | 01443537 11360 446 | 01443537 11472 447 | 01443537 11483 448 | 01443537 11513 449 | 01443537 11577 450 | 01443537 11582 451 | 01443537 11665 452 | 01443537 1167 453 | 01443537 1170 454 | 01443537 11709 455 | 01443537 11715 456 | 01443537 11773 457 | 01443537 11787 458 | 01443537 11932 459 | 01443537 1194 460 | 01443537 1196 461 | 01443537 12040 462 | 01443537 12079 463 | 01443537 12130 464 | 01443537 12138 465 | 01443537 1219 466 | 01443537 1220 467 | 01443537 12356 468 | 01443537 1238 469 | 01443537 12408 470 | 01443537 12433 471 | 01443537 12500 472 | 01443537 12525 473 | 01443537 12557 474 | 01443537 12571 475 | 01443537 12585 476 | 01443537 12593 477 | 01443537 12661 478 | 01443537 12715 479 | 01443537 12723 480 | 01443537 12749 481 | 01443537 12814 482 | 01443537 12829 483 | 01443537 12838 484 | 01443537 12860 485 | 01443537 129 486 | 01443537 12900 487 | 01443537 12932 488 | 01443537 12938 489 | 01443537 12965 490 | 01443537 12974 491 | 01443537 1298 492 | 01443537 12984 493 | 01443537 130 494 | 01443537 1301 495 | 01443537 13028 496 | 01443537 13038 497 | 01443537 1306 498 | 01443537 1308 499 | 01443537 13120 500 | 01443537 13121 501 | 01443537 1313 502 | 01443537 13159 503 | 01443537 13166 504 | 01443537 13200 505 | 01443537 13218 506 | 01443537 13298 507 | 01443537 1330 508 | 01443537 13343 509 | 01443537 13348 510 | 01443537 13386 511 | 01443537 13422 512 | 01443537 13475 513 | 01443537 13513 514 | 01443537 13517 515 | 01443537 13577 516 | 01443537 1359 517 | 01443537 1361 518 | 01443537 13623 519 | 01443537 13650 520 | 01443537 13658 521 | 01443537 13659 522 | 01443537 13668 523 | 01443537 13710 524 | 01443537 13733 525 | 01443537 1374 526 | 01443537 13755 527 | 01443537 13771 528 | 01443537 13785 529 | 01443537 13806 530 | 01443537 13830 531 | 01443537 13924 532 | 01443537 13947 533 | 01443537 14020 534 | 01443537 1403 535 | 01443537 14043 536 | 01443537 1415 537 | 01443537 14365 538 | 01443537 14393 539 | 01443537 1440 540 | 01443537 14436 541 | 01443537 14454 542 | 01443537 1450 543 | 01443537 1452 544 | 01443537 1453 545 | 01443537 14541 546 | 01443537 1472 547 | 01443537 14753 548 | 01443537 1478 549 | 01443537 14789 550 | 01443537 14803 551 | 01443537 14819 552 | 01443537 14827 553 | 01443537 14835 554 | 01443537 14850 555 | 01443537 14854 556 | 01443537 15077 557 | 01443537 1508 558 | 01443537 15164 559 | 01443537 15166 560 | 01443537 15253 561 | 01443537 15284 562 | 01443537 15344 563 | 01443537 15416 564 | 01443537 15450 565 | 01443537 1549 566 | 01443537 15549 567 | 01443537 15607 568 | 01443537 15608 569 | 01443537 15686 570 | 01443537 15790 571 | 01443537 1580 572 | 01443537 15846 573 | 01443537 15891 574 | 01443537 1594 575 | 01443537 15955 576 | 01443537 15961 577 | 01443537 15991 578 | 01443537 16 579 | 01443537 16031 580 | 01443537 16057 581 | 01443537 1610 582 | 01443537 16103 583 | 01443537 16219 584 | 01443537 16234 585 | 01443537 16282 586 | 01443537 163 587 | 01443537 16322 588 | 01443537 16351 589 | 01443537 16382 590 | 01443537 1640 591 | 01443537 16403 592 | 01443537 16422 593 | 01443537 16518 594 | 01443537 1655 595 | 01443537 16576 596 | 01443537 1659 597 | 01443537 16595 598 | 01443537 16612 599 | 01443537 16613 600 | 01443537 16635 601 | 01443537 16652 602 | 01443537 16691 603 | 01443537 16756 604 | 01443537 168 605 | 01443537 16806 606 | 01443537 16813 607 | 01443537 16849 608 | 01443537 16853 609 | 01443537 16856 610 | 01443537 16923 611 | 01443537 16939 612 | 01443537 16942 613 | 01443537 16944 614 | 01443537 1698 615 | 01443537 17008 616 | 01443537 17038 617 | 01443537 17105 618 | 01443537 1715 619 | 01443537 17158 620 | 01443537 17183 621 | 01443537 17187 622 | 01443537 17246 623 | 01443537 17252 624 | 01443537 17254 625 | 01443537 17266 626 | 01443537 17296 627 | 01443537 17309 628 | 01443537 17374 629 | 01443537 1743 630 | 01443537 1746 631 | 01443537 1753 632 | 01443537 17565 633 | 01443537 17586 634 | 01443537 1764 635 | 01443537 17661 636 | 01443537 17665 637 | 01443537 17680 638 | 01443537 1769 639 | 01443537 1772 640 | 01443537 1786 641 | 01443537 17967 642 | 01443537 17999 643 | 01443537 180 644 | 01443537 18065 645 | 01443537 18113 646 | 01443537 18146 647 | 01443537 18165 648 | 01443537 1824 649 | 01443537 1832 650 | 01443537 18372 651 | 01443537 18376 652 | 01443537 1844 653 | 01443537 18508 654 | 01443537 1853 655 | 01443537 18549 656 | 01443537 18592 657 | 01443537 1860 658 | 01443537 18601 659 | 01443537 1895 660 | 01443537 18974 661 | 01443537 18998 662 | 01443537 19008 663 | 01443537 19026 664 | 01443537 1903 665 | 01443537 19245 666 | 01443537 19267 667 | 01443537 19304 668 | 01443537 19320 669 | 01443537 19364 670 | 01443537 19391 671 | 01443537 1942 672 | 01443537 19470 673 | 01443537 19514 674 | 01443537 1955 675 | 01443537 19678 676 | 01443537 1970 677 | 01443537 19771 678 | 01443537 19810 679 | 01443537 19840 680 | 01443537 1987 681 | 01443537 19954 682 | 01443537 2 683 | 01443537 20078 684 | 01443537 20299 685 | 01443537 20349 686 | 01443537 20383 687 | 01443537 20388 688 | 01443537 20556 689 | 01443537 20585 690 | 01443537 2060 691 | 01443537 2068 692 | 01443537 20680 693 | 01443537 20685 694 | 01443537 20722 695 | 01443537 2084 696 | 01443537 20842 697 | 01443537 2095 698 | 01443537 20965 699 | 01443537 21060 700 | 01443537 21067 701 | 01443537 21104 702 | 01443537 2111 703 | 01443537 21111 704 | 01443537 21129 705 | 01443537 21162 706 | 01443537 2117 707 | 01443537 21198 708 | 01443537 21226 709 | 01443537 2125 710 | 01443537 21338 711 | 01443537 21388 712 | 01443537 2139 713 | 01443537 21396 714 | 01443537 21400 715 | 01443537 21407 716 | 01443537 21408 717 | 01443537 21563 718 | 01443537 21611 719 | 01443537 21724 720 | 01443537 21801 721 | 01443537 2186 722 | 01443537 21879 723 | 01443537 2188 724 | 01443537 2191 725 | 01443537 21959 726 | 01443537 21971 727 | 01443537 21982 728 | 01443537 2199 729 | 01443537 22108 730 | 01443537 22279 731 | 01443537 2228 732 | 01443537 2238 733 | 01443537 22476 734 | 01443537 2249 735 | 01443537 22538 736 | 01443537 22549 737 | 01443537 22563 738 | 01443537 2260 739 | 01443537 22644 740 | 01443537 2266 741 | 01443537 22695 742 | 01443537 22740 743 | 01443537 22742 744 | 01443537 22807 745 | 01443537 2290 746 | 01443537 2297 747 | 01443537 22976 748 | 01443537 22992 749 | 01443537 23046 750 | 01443537 23095 751 | 01443537 23107 752 | 01443537 23113 753 | 01443537 2312 754 | 01443537 23265 755 | 01443537 23289 756 | 01443537 2333 757 | 01443537 23353 758 | 01443537 23362 759 | 01443537 23440 760 | 01443537 23453 761 | 01443537 23550 762 | 01443537 23671 763 | 01443537 23686 764 | 01443537 23868 765 | 01443537 2394 766 | 01443537 2397 767 | 01443537 24148 768 | 01443537 24156 769 | 01443537 2419 770 | 01443537 24209 771 | 01443537 24223 772 | 01443537 24387 773 | 01443537 24417 774 | 01443537 24424 775 | 01443537 24437 776 | 01443537 2449 777 | 01443537 24495 778 | 01443537 24514 779 | 01443537 24539 780 | 01443537 24543 781 | 01443537 2456 782 | 01443537 247 783 | 01443537 24784 784 | 01443537 24801 785 | 01443537 24803 786 | 01443537 2486 787 | 01443537 2514 788 | 01443537 2534 789 | 01443537 2535 790 | 01443537 2549 791 | 01443537 2552 792 | 01443537 2554 793 | 01443537 2563 794 | 01443537 2568 795 | 01443537 2594 796 | 01443537 2625 797 | 01443537 2637 798 | 01443537 2665 799 | 01443537 2684 800 | 01443537 2711 801 | 0 0 802 | -------------------------------------------------------------------------------- /utils/ImageNet/cat2.txt: -------------------------------------------------------------------------------- 1 | 01496331 2 | 01443537 3 | 0 -------------------------------------------------------------------------------- /utils/ImageNet/cat5.file: -------------------------------------------------------------------------------- 1 | 01784675 10074 2 | 01784675 10102 3 | 01784675 10108 4 | 01784675 1011 5 | 01784675 10135 6 | 01784675 10149 7 | 01784675 1015 8 | 01784675 10226 9 | 01784675 10259 10 | 01784675 10273 11 | 01784675 10308 12 | 01784675 10331 13 | 01784675 10391 14 | 01784675 1050 15 | 01784675 10504 16 | 01784675 10514 17 | 01784675 10515 18 | 01784675 10533 19 | 01784675 10554 20 | 01784675 10649 21 | 01784675 10673 22 | 01784675 10696 23 | 01784675 10750 24 | 01784675 10823 25 | 01784675 10844 26 | 01784675 10867 27 | 01784675 109 28 | 01784675 1092 29 | 01784675 10971 30 | 01784675 10987 31 | 01784675 10995 32 | 01784675 11014 33 | 01784675 11044 34 | 01784675 11062 35 | 01784675 11142 36 | 01784675 11164 37 | 01784675 11235 38 | 01784675 11240 39 | 01784675 11313 40 | 01784675 1133 41 | 01784675 11339 42 | 01784675 11374 43 | 01784675 11432 44 | 01784675 11444 45 | 01784675 11448 46 | 01784675 11449 47 | 01784675 11452 48 | 01784675 11467 49 | 01784675 11501 50 | 01784675 11533 51 | 01784675 11539 52 | 01784675 1154 53 | 01784675 11566 54 | 01784675 11582 55 | 01784675 11588 56 | 01784675 11601 57 | 01784675 11609 58 | 01784675 11611 59 | 01784675 11632 60 | 01784675 11645 61 | 01784675 1167 62 | 01784675 11690 63 | 01784675 11704 64 | 01784675 11746 65 | 01784675 11815 66 | 01784675 11817 67 | 01784675 11845 68 | 01784675 11866 69 | 01784675 11885 70 | 01784675 11889 71 | 01784675 11902 72 | 01784675 11923 73 | 01784675 11953 74 | 01784675 11962 75 | 01784675 12099 76 | 01784675 12110 77 | 01784675 12151 78 | 01784675 12176 79 | 01784675 12207 80 | 01784675 12279 81 | 01784675 12288 82 | 01784675 12307 83 | 01784675 12328 84 | 01784675 1233 85 | 01784675 12353 86 | 01784675 12362 87 | 01784675 12446 88 | 01784675 12462 89 | 01784675 12477 90 | 01784675 12497 91 | 01784675 12509 92 | 01784675 12519 93 | 01784675 12635 94 | 01784675 12662 95 | 01784675 12673 96 | 01784675 12683 97 | 01784675 12730 98 | 01784675 1275 99 | 01784675 1279 100 | 01784675 12904 101 | 01784675 12915 102 | 01784675 12991 103 | 01784675 13007 104 | 01784675 13010 105 | 01784675 13116 106 | 01784675 13120 107 | 01784675 13143 108 | 01784675 13176 109 | 01784675 13196 110 | 01784675 13231 111 | 01784675 13265 112 | 01784675 13294 113 | 01784675 13337 114 | 01784675 13349 115 | 01784675 13350 116 | 01784675 13359 117 | 01784675 13414 118 | 01784675 13438 119 | 01784675 13445 120 | 01784675 13448 121 | 01784675 1347 122 | 01784675 13472 123 | 01784675 1350 124 | 01784675 13501 125 | 01784675 13507 126 | 01784675 13518 127 | 01784675 13521 128 | 01784675 13535 129 | 01784675 13582 130 | 01784675 13608 131 | 01784675 13616 132 | 01784675 13651 133 | 01784675 13653 134 | 01784675 13701 135 | 01784675 13731 136 | 01784675 13791 137 | 01784675 1384 138 | 01784675 13864 139 | 01784675 13873 140 | 01784675 1392 141 | 01784675 13928 142 | 01784675 13974 143 | 01784675 14071 144 | 01784675 14125 145 | 01784675 14140 146 | 01784675 14207 147 | 01784675 14309 148 | 01784675 14320 149 | 01784675 14378 150 | 01784675 144 151 | 01784675 14408 152 | 01784675 14439 153 | 01784675 14445 154 | 01784675 14447 155 | 01784675 1445 156 | 01784675 14458 157 | 01784675 14489 158 | 01784675 145 159 | 01784675 1452 160 | 01784675 14536 161 | 01784675 14543 162 | 01784675 14590 163 | 01784675 14594 164 | 01784675 14597 165 | 01784675 14614 166 | 01784675 1467 167 | 01784675 14715 168 | 01784675 14723 169 | 01784675 14776 170 | 01784675 14869 171 | 01784675 14949 172 | 01784675 1502 173 | 01784675 15038 174 | 01784675 1505 175 | 01784675 15099 176 | 01784675 15201 177 | 01784675 15208 178 | 01784675 15244 179 | 01784675 15257 180 | 01784675 15347 181 | 01784675 1536 182 | 01784675 15405 183 | 01784675 15440 184 | 01784675 15470 185 | 01784675 15515 186 | 01784675 15520 187 | 01784675 15583 188 | 01784675 15585 189 | 01784675 15613 190 | 01784675 15626 191 | 01784675 15674 192 | 01784675 15717 193 | 01784675 15784 194 | 01784675 15792 195 | 01784675 15852 196 | 01784675 15913 197 | 01784675 16 198 | 01784675 1602 199 | 01784675 16238 200 | 01784675 16345 201 | 02087046 1004 202 | 02087046 10592 203 | 02087046 108 204 | 02087046 11019 205 | 02087046 1206 206 | 02087046 133 207 | 02087046 1386 208 | 02087046 1431 209 | 02087046 1468 210 | 02087046 1482 211 | 02087046 1520 212 | 02087046 1577 213 | 02087046 1658 214 | 02087046 1666 215 | 02087046 1693 216 | 02087046 1703 217 | 02087046 1724 218 | 02087046 178 219 | 02087046 1792 220 | 02087046 194 221 | 02087046 1984 222 | 02087046 1989 223 | 02087046 2058 224 | 02087046 2062 225 | 02087046 2093 226 | 02087046 2105 227 | 02087046 2118 228 | 02087046 2127 229 | 02087046 2140 230 | 02087046 2141 231 | 02087046 2152 232 | 02087046 2158 233 | 02087046 2168 234 | 02087046 217 235 | 02087046 2175 236 | 02087046 2185 237 | 02087046 2193 238 | 02087046 2199 239 | 02087046 2233 240 | 02087046 224 241 | 02087046 2265 242 | 02087046 2276 243 | 02087046 2279 244 | 02087046 228 245 | 02087046 2315 246 | 02087046 2316 247 | 02087046 232 248 | 02087046 2341 249 | 02087046 2374 250 | 02087046 2409 251 | 02087046 2419 252 | 02087046 2485 253 | 02087046 2499 254 | 02087046 2551 255 | 02087046 2614 256 | 02087046 2619 257 | 02087046 2620 258 | 02087046 267 259 | 02087046 2744 260 | 02087046 2752 261 | 02087046 2792 262 | 02087046 2819 263 | 02087046 2834 264 | 02087046 2843 265 | 02087046 294 266 | 02087046 2957 267 | 02087046 296 268 | 02087046 2969 269 | 02087046 3089 270 | 02087046 309 271 | 02087046 3103 272 | 02087046 3201 273 | 02087046 3211 274 | 02087046 3254 275 | 02087046 3311 276 | 02087046 3331 277 | 02087046 336 278 | 02087046 3414 279 | 02087046 3439 280 | 02087046 3459 281 | 02087046 3462 282 | 02087046 3470 283 | 02087046 3471 284 | 02087046 3490 285 | 02087046 352 286 | 02087046 3543 287 | 02087046 3560 288 | 02087046 357 289 | 02087046 3608 290 | 02087046 3651 291 | 02087046 3706 292 | 02087046 3724 293 | 02087046 3735 294 | 02087046 3764 295 | 02087046 3817 296 | 02087046 3843 297 | 02087046 386 298 | 02087046 3883 299 | 02087046 3923 300 | 02087046 3944 301 | 02087046 3953 302 | 02087046 3977 303 | 02087046 3998 304 | 02087046 4001 305 | 02087046 4039 306 | 02087046 4058 307 | 02087046 406 308 | 02087046 4066 309 | 02087046 4102 310 | 02087046 4127 311 | 02087046 4129 312 | 02087046 4135 313 | 02087046 4153 314 | 02087046 4155 315 | 02087046 420 316 | 02087046 4235 317 | 02087046 430 318 | 02087046 4301 319 | 02087046 4315 320 | 02087046 4338 321 | 02087046 4346 322 | 02087046 4350 323 | 02087046 4370 324 | 02087046 4374 325 | 02087046 4387 326 | 02087046 4394 327 | 02087046 4397 328 | 02087046 4401 329 | 02087046 4402 330 | 02087046 4408 331 | 02087046 4409 332 | 02087046 4426 333 | 02087046 4455 334 | 02087046 4506 335 | 02087046 4525 336 | 02087046 4575 337 | 02087046 4614 338 | 02087046 4631 339 | 02087046 4647 340 | 02087046 4650 341 | 02087046 4673 342 | 02087046 4679 343 | 02087046 4710 344 | 02087046 4719 345 | 02087046 4748 346 | 02087046 4751 347 | 02087046 4793 348 | 02087046 4808 349 | 02087046 4809 350 | 02087046 4832 351 | 02087046 4851 352 | 02087046 4856 353 | 02087046 4860 354 | 02087046 4906 355 | 02087046 4987 356 | 02087046 5017 357 | 02087046 505 358 | 02087046 5191 359 | 02087046 5196 360 | 02087046 5203 361 | 02087046 5219 362 | 02087046 5224 363 | 02087046 5249 364 | 02087046 5267 365 | 02087046 5305 366 | 02087046 5306 367 | 02087046 5317 368 | 02087046 5325 369 | 02087046 5347 370 | 02087046 5352 371 | 02087046 536 372 | 02087046 5365 373 | 02087046 538 374 | 02087046 5383 375 | 02087046 5386 376 | 02087046 5407 377 | 02087046 5439 378 | 02087046 5499 379 | 02087046 554 380 | 02087046 5541 381 | 02087046 5544 382 | 02087046 5556 383 | 02087046 5626 384 | 02087046 5661 385 | 02087046 5672 386 | 02087046 568 387 | 02087046 5701 388 | 02087046 5704 389 | 02087046 5756 390 | 02087046 5802 391 | 02087046 5838 392 | 02087046 5840 393 | 02087046 5843 394 | 02087046 5847 395 | 02087046 5849 396 | 02087046 5854 397 | 02087046 5860 398 | 02087046 5887 399 | 02087046 5890 400 | 02087046 5917 401 | 02277742 10045 402 | 02277742 10074 403 | 02277742 10085 404 | 02277742 103 405 | 02277742 10327 406 | 02277742 10534 407 | 02277742 10630 408 | 02277742 10647 409 | 02277742 10879 410 | 02277742 10923 411 | 02277742 10935 412 | 02277742 10955 413 | 02277742 11019 414 | 02277742 11166 415 | 02277742 11223 416 | 02277742 11238 417 | 02277742 1128 418 | 02277742 11309 419 | 02277742 11334 420 | 02277742 11367 421 | 02277742 11453 422 | 02277742 11606 423 | 02277742 11636 424 | 02277742 11723 425 | 02277742 11747 426 | 02277742 11809 427 | 02277742 12034 428 | 02277742 12256 429 | 02277742 12262 430 | 02277742 12310 431 | 02277742 12428 432 | 02277742 12467 433 | 02277742 12471 434 | 02277742 12474 435 | 02277742 12529 436 | 02277742 12554 437 | 02277742 12582 438 | 02277742 12795 439 | 02277742 12891 440 | 02277742 12925 441 | 02277742 12955 442 | 02277742 12959 443 | 02277742 12990 444 | 02277742 12991 445 | 02277742 12996 446 | 02277742 13011 447 | 02277742 13115 448 | 02277742 13120 449 | 02277742 13147 450 | 02277742 13307 451 | 02277742 13327 452 | 02277742 13334 453 | 02277742 13411 454 | 02277742 13479 455 | 02277742 13545 456 | 02277742 13626 457 | 02277742 13630 458 | 02277742 13807 459 | 02277742 13815 460 | 02277742 13900 461 | 02277742 13924 462 | 02277742 13947 463 | 02277742 14084 464 | 02277742 14229 465 | 02277742 14375 466 | 02277742 14379 467 | 02277742 14472 468 | 02277742 14496 469 | 02277742 14546 470 | 02277742 14604 471 | 02277742 14613 472 | 02277742 14639 473 | 02277742 14689 474 | 02277742 14759 475 | 02277742 14832 476 | 02277742 14839 477 | 02277742 14879 478 | 02277742 14971 479 | 02277742 14973 480 | 02277742 14979 481 | 02277742 15037 482 | 02277742 15095 483 | 02277742 15177 484 | 02277742 15296 485 | 02277742 15301 486 | 02277742 15364 487 | 02277742 1542 488 | 02277742 15426 489 | 02277742 15432 490 | 02277742 15438 491 | 02277742 15547 492 | 02277742 15554 493 | 02277742 15618 494 | 02277742 15647 495 | 02277742 15691 496 | 02277742 15734 497 | 02277742 15844 498 | 02277742 15960 499 | 02277742 16137 500 | 02277742 16153 501 | 02277742 16204 502 | 02277742 16241 503 | 02277742 16284 504 | 02277742 16287 505 | 02277742 163 506 | 02277742 16305 507 | 02277742 16354 508 | 02277742 16550 509 | 02277742 16629 510 | 02277742 16650 511 | 02277742 16689 512 | 02277742 16756 513 | 02277742 16766 514 | 02277742 16767 515 | 02277742 16888 516 | 02277742 17026 517 | 02277742 17062 518 | 02277742 17094 519 | 02277742 17145 520 | 02277742 17153 521 | 02277742 17161 522 | 02277742 17192 523 | 02277742 17266 524 | 02277742 17347 525 | 02277742 17369 526 | 02277742 17446 527 | 02277742 17490 528 | 02277742 17513 529 | 02277742 17626 530 | 02277742 17705 531 | 02277742 17722 532 | 02277742 17745 533 | 02277742 17749 534 | 02277742 17782 535 | 02277742 17822 536 | 02277742 17825 537 | 02277742 17865 538 | 02277742 17925 539 | 02277742 17948 540 | 02277742 18016 541 | 02277742 18029 542 | 02277742 18068 543 | 02277742 18101 544 | 02277742 18194 545 | 02277742 18218 546 | 02277742 18222 547 | 02277742 18315 548 | 02277742 18358 549 | 02277742 18453 550 | 02277742 18478 551 | 02277742 18567 552 | 02277742 18671 553 | 02277742 18685 554 | 02277742 18687 555 | 02277742 18741 556 | 02277742 18745 557 | 02277742 18803 558 | 02277742 18816 559 | 02277742 18958 560 | 02277742 1902 561 | 02277742 19035 562 | 02277742 19104 563 | 02277742 19276 564 | 02277742 19290 565 | 02277742 19448 566 | 02277742 19498 567 | 02277742 19510 568 | 02277742 19539 569 | 02277742 19547 570 | 02277742 19824 571 | 02277742 19935 572 | 02277742 19947 573 | 02277742 19997 574 | 02277742 20225 575 | 02277742 20249 576 | 02277742 20257 577 | 02277742 20636 578 | 02277742 20647 579 | 02277742 20651 580 | 02277742 20659 581 | 02277742 20670 582 | 02277742 20707 583 | 02277742 20725 584 | 02277742 20814 585 | 02277742 20844 586 | 02277742 20899 587 | 02277742 20939 588 | 02277742 20964 589 | 02277742 20965 590 | 02277742 20979 591 | 02277742 2100 592 | 02277742 21109 593 | 02277742 21168 594 | 02277742 21171 595 | 02277742 21174 596 | 02277742 21179 597 | 02277742 21227 598 | 02277742 21237 599 | 02277742 21248 600 | 02277742 21250 601 | 02687992 10002 602 | 02687992 10069 603 | 02687992 10073 604 | 02687992 1012 605 | 02687992 10283 606 | 02687992 10305 607 | 02687992 10474 608 | 02687992 10712 609 | 02687992 10806 610 | 02687992 10869 611 | 02687992 10920 612 | 02687992 10945 613 | 02687992 10991 614 | 02687992 11065 615 | 02687992 11123 616 | 02687992 11202 617 | 02687992 11217 618 | 02687992 11480 619 | 02687992 11522 620 | 02687992 11531 621 | 02687992 11572 622 | 02687992 11662 623 | 02687992 11707 624 | 02687992 11851 625 | 02687992 11960 626 | 02687992 1213 627 | 02687992 12164 628 | 02687992 12189 629 | 02687992 12246 630 | 02687992 12433 631 | 02687992 12505 632 | 02687992 12572 633 | 02687992 1283 634 | 02687992 13098 635 | 02687992 13160 636 | 02687992 13201 637 | 02687992 13238 638 | 02687992 13605 639 | 02687992 13650 640 | 02687992 13722 641 | 02687992 13879 642 | 02687992 13923 643 | 02687992 1393 644 | 02687992 14582 645 | 02687992 14705 646 | 02687992 15289 647 | 02687992 15406 648 | 02687992 15871 649 | 02687992 15924 650 | 02687992 15948 651 | 02687992 16439 652 | 02687992 16603 653 | 02687992 16642 654 | 02687992 16772 655 | 02687992 16894 656 | 02687992 17000 657 | 02687992 17064 658 | 02687992 17145 659 | 02687992 17235 660 | 02687992 1775 661 | 02687992 17825 662 | 02687992 18607 663 | 02687992 18636 664 | 02687992 1987 665 | 02687992 2057 666 | 02687992 20740 667 | 02687992 20826 668 | 02687992 2103 669 | 02687992 21692 670 | 02687992 22137 671 | 02687992 22214 672 | 02687992 22319 673 | 02687992 22485 674 | 02687992 2249 675 | 02687992 22562 676 | 02687992 22664 677 | 02687992 2280 678 | 02687992 23940 679 | 02687992 24295 680 | 02687992 2468 681 | 02687992 2475 682 | 02687992 25130 683 | 02687992 25218 684 | 02687992 2578 685 | 02687992 2592 686 | 02687992 2608 687 | 02687992 26175 688 | 02687992 26215 689 | 02687992 2623 690 | 02687992 26237 691 | 02687992 2629 692 | 02687992 26795 693 | 02687992 26972 694 | 02687992 28611 695 | 02687992 29081 696 | 02687992 29124 697 | 02687992 29126 698 | 02687992 29520 699 | 02687992 2984 700 | 02687992 30397 701 | 02687992 3043 702 | 02687992 30480 703 | 02687992 3066 704 | 02687992 31116 705 | 02687992 31272 706 | 02687992 31830 707 | 02687992 32135 708 | 02687992 32390 709 | 02687992 32418 710 | 02687992 32603 711 | 02687992 32888 712 | 02687992 33003 713 | 02687992 3308 714 | 02687992 33155 715 | 02687992 33356 716 | 02687992 33485 717 | 02687992 33602 718 | 02687992 33764 719 | 02687992 34150 720 | 02687992 3419 721 | 02687992 34370 722 | 02687992 3504 723 | 02687992 35133 724 | 02687992 3562 725 | 02687992 3566 726 | 02687992 35770 727 | 02687992 36099 728 | 02687992 36162 729 | 02687992 36260 730 | 02687992 3629 731 | 02687992 36374 732 | 02687992 36565 733 | 02687992 3663 734 | 02687992 3700 735 | 02687992 37237 736 | 02687992 37318 737 | 02687992 37502 738 | 02687992 37761 739 | 02687992 38486 740 | 02687992 3875 741 | 02687992 38995 742 | 02687992 39224 743 | 02687992 39334 744 | 02687992 39825 745 | 02687992 4016 746 | 02687992 4139 747 | 02687992 4153 748 | 02687992 4300 749 | 02687992 4509 750 | 02687992 4608 751 | 02687992 4670 752 | 02687992 4837 753 | 02687992 5194 754 | 02687992 5360 755 | 02687992 5438 756 | 02687992 5466 757 | 02687992 5485 758 | 02687992 5595 759 | 02687992 5599 760 | 02687992 5626 761 | 02687992 5648 762 | 02687992 5714 763 | 02687992 592 764 | 02687992 6134 765 | 02687992 6170 766 | 02687992 6183 767 | 02687992 626 768 | 02687992 6277 769 | 02687992 6483 770 | 02687992 6554 771 | 02687992 6708 772 | 02687992 6797 773 | 02687992 6913 774 | 02687992 702 775 | 02687992 7339 776 | 02687992 7381 777 | 02687992 7393 778 | 02687992 7408 779 | 02687992 7435 780 | 02687992 7555 781 | 02687992 7680 782 | 02687992 7701 783 | 02687992 7853 784 | 02687992 7872 785 | 02687992 7876 786 | 02687992 8122 787 | 02687992 8149 788 | 02687992 8186 789 | 02687992 8316 790 | 02687992 8638 791 | 02687992 8689 792 | 02687992 8735 793 | 02687992 8889 794 | 02687992 8896 795 | 02687992 8914 796 | 02687992 8926 797 | 02687992 8949 798 | 02687992 9290 799 | 02687992 9446 800 | 02687992 947 801 | 03662601 10021 802 | 03662601 10049 803 | 03662601 10058 804 | 03662601 10106 805 | 03662601 10133 806 | 03662601 10144 807 | 03662601 10183 808 | 03662601 10262 809 | 03662601 10297 810 | 03662601 10327 811 | 03662601 10360 812 | 03662601 10425 813 | 03662601 10427 814 | 03662601 10463 815 | 03662601 10534 816 | 03662601 10609 817 | 03662601 10634 818 | 03662601 10687 819 | 03662601 10697 820 | 03662601 10734 821 | 03662601 1074 822 | 03662601 10894 823 | 03662601 1090 824 | 03662601 10923 825 | 03662601 10940 826 | 03662601 11104 827 | 03662601 11178 828 | 03662601 11306 829 | 03662601 11328 830 | 03662601 11358 831 | 03662601 11384 832 | 03662601 11462 833 | 03662601 1158 834 | 03662601 11583 835 | 03662601 1161 836 | 03662601 11739 837 | 03662601 11747 838 | 03662601 11749 839 | 03662601 11771 840 | 03662601 11820 841 | 03662601 11862 842 | 03662601 1187 843 | 03662601 11882 844 | 03662601 1191 845 | 03662601 11924 846 | 03662601 11926 847 | 03662601 11952 848 | 03662601 12073 849 | 03662601 12170 850 | 03662601 1219 851 | 03662601 12252 852 | 03662601 12253 853 | 03662601 1229 854 | 03662601 12290 855 | 03662601 12300 856 | 03662601 12308 857 | 03662601 12323 858 | 03662601 12329 859 | 03662601 12362 860 | 03662601 12367 861 | 03662601 12381 862 | 03662601 12425 863 | 03662601 12434 864 | 03662601 12483 865 | 03662601 12632 866 | 03662601 1266 867 | 03662601 12676 868 | 03662601 12932 869 | 03662601 12950 870 | 03662601 12982 871 | 03662601 12995 872 | 03662601 13009 873 | 03662601 1301 874 | 03662601 13081 875 | 03662601 13151 876 | 03662601 13200 877 | 03662601 13210 878 | 03662601 13244 879 | 03662601 13281 880 | 03662601 13321 881 | 03662601 13326 882 | 03662601 13565 883 | 03662601 13675 884 | 03662601 13715 885 | 03662601 13754 886 | 03662601 13834 887 | 03662601 13961 888 | 03662601 13963 889 | 03662601 14072 890 | 03662601 14094 891 | 03662601 14478 892 | 03662601 145 893 | 03662601 14818 894 | 03662601 15001 895 | 03662601 15033 896 | 03662601 15143 897 | 03662601 1524 898 | 03662601 15252 899 | 03662601 1530 900 | 03662601 15329 901 | 03662601 1541 902 | 03662601 15505 903 | 03662601 15566 904 | 03662601 1561 905 | 03662601 15656 906 | 03662601 1566 907 | 03662601 15661 908 | 03662601 15698 909 | 03662601 15763 910 | 03662601 15781 911 | 03662601 15915 912 | 03662601 16048 913 | 03662601 16076 914 | 03662601 16119 915 | 03662601 16128 916 | 03662601 1616 917 | 03662601 16366 918 | 03662601 1637 919 | 03662601 16417 920 | 03662601 1642 921 | 03662601 16425 922 | 03662601 16438 923 | 03662601 16481 924 | 03662601 16505 925 | 03662601 16513 926 | 03662601 16580 927 | 03662601 16694 928 | 03662601 16811 929 | 03662601 16825 930 | 03662601 16848 931 | 03662601 16883 932 | 03662601 169 933 | 03662601 16943 934 | 03662601 16988 935 | 03662601 16999 936 | 03662601 17001 937 | 03662601 17021 938 | 03662601 1711 939 | 03662601 17151 940 | 03662601 17156 941 | 03662601 17198 942 | 03662601 17209 943 | 03662601 17237 944 | 03662601 17352 945 | 03662601 17403 946 | 03662601 17421 947 | 03662601 17428 948 | 03662601 17465 949 | 03662601 17483 950 | 03662601 17485 951 | 03662601 17489 952 | 03662601 17492 953 | 03662601 17539 954 | 03662601 17624 955 | 03662601 1764 956 | 03662601 17647 957 | 03662601 17714 958 | 03662601 1776 959 | 03662601 1777 960 | 03662601 17793 961 | 03662601 17861 962 | 03662601 1789 963 | 03662601 1805 964 | 03662601 18100 965 | 03662601 18186 966 | 03662601 1834 967 | 03662601 1863 968 | 03662601 18733 969 | 03662601 1884 970 | 03662601 1895 971 | 03662601 19028 972 | 03662601 19079 973 | 03662601 19120 974 | 03662601 19470 975 | 03662601 19474 976 | 03662601 19482 977 | 03662601 1968 978 | 03662601 19771 979 | 03662601 19969 980 | 03662601 2000 981 | 03662601 20059 982 | 03662601 20078 983 | 03662601 20087 984 | 03662601 20110 985 | 03662601 20206 986 | 03662601 20295 987 | 03662601 2032 988 | 03662601 20381 989 | 03662601 2048 990 | 03662601 20511 991 | 03662601 20518 992 | 03662601 2053 993 | 03662601 2058 994 | 03662601 2061 995 | 03662601 2073 996 | 03662601 20734 997 | 03662601 20834 998 | 03662601 20887 999 | 03662601 2090 1000 | 03662601 21290 1001 | 0 0 -------------------------------------------------------------------------------- /utils/ImageNet/cat5.txt: -------------------------------------------------------------------------------- 1 | 01784675 2 | 02087046 3 | 02277742 4 | 02687992 5 | 03662601 6 | 0 -------------------------------------------------------------------------------- /utils/ImageNet/loop.sh: -------------------------------------------------------------------------------- 1 | # define the working directory 2 | Root=$1 3 | echo "Loop under $Root." 4 | 5 | # define the action script 6 | Action=$2 7 | 8 | # list all sub-folders 9 | Folders="$Root*/" 10 | 11 | # process all files 12 | for Folder in $Folders 13 | do 14 | echo $Folder; 15 | sh $Action $Folder; 16 | done 17 | -------------------------------------------------------------------------------- /utils/ImageNet/resize.sh: -------------------------------------------------------------------------------- 1 | # define the working directory 2 | Folder=$1 3 | echo "Resizing under $Folder." 4 | 5 | # list all files 6 | Files="$Folder*" 7 | 8 | # process all files 9 | for File in $Files 10 | do 11 | # echo $File; 12 | cp $File "$File.bak"; 13 | convert $File -resize 256x256^ -gravity center -extent 256x256^ $File; 14 | done 15 | -------------------------------------------------------------------------------- /utils/ImageNet/run_sample.sh: -------------------------------------------------------------------------------- 1 | # run the sample script to extract all file names 2 | sh sample.sh ../../../Data/ImageNet/ "cat$1"".txt" "cat$1"".file" 3 | 4 | -------------------------------------------------------------------------------- /utils/ImageNet/sample.sh: -------------------------------------------------------------------------------- 1 | # define the working directory 2 | Root=$1 3 | echo "Sampling under $Root" 4 | 5 | Dest=$3 6 | rm -f $Dest; 7 | 8 | # read category list 9 | Cat=$2 10 | while read Folder 11 | do 12 | echo $Folder; 13 | 14 | # list all files 15 | Files="$Root""n$Folder/*.JPEG"; 16 | 17 | # process all files 18 | for File in $Files 19 | do 20 | File=$(echo $File | cut -c35-); 21 | Label=$(echo $File | cut -c-8); 22 | Num=$(echo $File | cut -c10- | awk -F . '{ print $1 }'); 23 | echo "$Label $Num" >> $Dest; 24 | done 25 | done < $Cat 26 | 27 | echo "0 0" >> $Dest; 28 | -------------------------------------------------------------------------------- /utils/ImageNet200/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.2) 2 | project(ImageNet200) 3 | 4 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 5 | 6 | SET(CMAKE_FIND_LIBRARY_PREFIXES "lib64;lib") 7 | SET(CMAKE_FIND_LIBRARY_SUFFIXES ".so;.a") 8 | SET(CMAKE_SIZEOF_VOID_P "8") 9 | 10 | SET(OpenCV_STATIC OFF) 11 | FIND_PACKAGE(OpenCV REQUIRED) 12 | INCLUDE_DIRECTORIES(${OpenCV_INCLUDE_DIRS}) 13 | 14 | set(SOURCE_FILES main.cpp) 15 | add_executable(ImageNet200 ${SOURCE_FILES}) 16 | TARGET_LINK_LIBRARIES(ImageNet200 ${OpenCV_LIBS}) -------------------------------------------------------------------------------- /utils/ImageNet200/main.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * ImageNet 200 database construction 3 | * 4 | * This program translate pictures into packaged binary file 5 | */ 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include "opencv2/opencv.hpp" 12 | 13 | using namespace std; 14 | using namespace cv; 15 | 16 | const int crop_size = 8; // data augmentation 17 | const int crop_step = 4; 18 | 19 | const int width = 64 - crop_size; 20 | const int height = 64 - crop_size; 21 | const int channel = 3; // RGB 22 | const int data_dim = width * height * channel; 23 | 24 | vector label_string; 25 | vector image_countdown; 26 | 27 | const string root = "/home/lyx/workspace/cuda/Data/ImageNet200/"; // define workspace 28 | const string wnid_path = root + "wnids_10.txt"; // define categories 29 | 30 | string to_train_path(int label, int number = -1) { 31 | return root + "train/" + label_string[label] + "/images/" + 32 | label_string[label] + "_" + to_string(number) + ".JPEG"; 33 | } 34 | 35 | template 36 | int get_rand(T max) { 37 | return rand() % int(max); 38 | } 39 | 40 | void save_file(string name, uint8_t* data, int size) { 41 | ofstream out(name); 42 | out.write((char*)data, size); 43 | out.close(); 44 | } 45 | 46 | void get_train_images(int argc, char** argv) { 47 | 48 | int image_per_category = 500; 49 | int label_count = 10; // choose some categories 50 | 51 | if (argc >= 3) { 52 | label_count = atoi(argv[2]); 53 | image_per_category = atoi(argv[3]); 54 | } 55 | 56 | int augmented_rate = (crop_size / crop_step + 1) * (crop_size / crop_step + 1); 57 | 58 | int data_size = data_dim * label_count * image_per_category; 59 | int label_size = label_count * image_per_category; 60 | 61 | data_size *= augmented_rate; 62 | label_size *= augmented_rate; 63 | 64 | ifstream wnid_file(wnid_path); 65 | string tmp; 66 | for (int i = 0; i < label_count; i++) { 67 | wnid_file >> tmp; 68 | label_string.push_back(tmp); 69 | image_countdown.push_back(image_per_category - 1); 70 | } 71 | wnid_file.close(); 72 | 73 | uint8_t* bin_images = new uint8_t[data_size]; 74 | uint8_t* bin_labels = new uint8_t[label_size]; 75 | 76 | int index = 0; 77 | while (!label_string.empty()) { 78 | uint8_t category = uint8_t(get_rand(label_string.size())); // random image 79 | int image = image_countdown[category]; 80 | Mat _m = imread(to_train_path(category, image)); 81 | assert(data_dim == (_m.cols - crop_size) * (_m.rows - crop_size) * _m.elemSize()); 82 | for (int h = 0; h <= crop_size; h += crop_step) 83 | for (int w = 0; w <= crop_size; w += crop_step) { 84 | Mat m; 85 | if (rand() % 2 == 0) 86 | m = _m.clone(); 87 | else 88 | flip(_m, m, 1); 89 | int _index = data_dim * (index + (w / crop_step * (crop_size / crop_step + 1) + h / crop_step) * label_count * image_per_category); 90 | for (int i = 0; i < channel; i++) 91 | for (int j = 0; j < height; j++) 92 | for (int k = 0; k < width; k++) { 93 | Vec3b pixel = m.at(j + h, k + w); 94 | bin_images[_index] = (uint8_t) pixel[i]; 95 | _index++; 96 | } 97 | bin_labels[index + (w / crop_step * (crop_size / crop_step + 1) + h / crop_step) * label_count * image_per_category] = category; 98 | m.release(); 99 | } 100 | _m.release(); 101 | if (image == 0) { 102 | label_string.erase(label_string.begin() + category); 103 | image_countdown.erase(image_countdown.begin() + category); 104 | } 105 | else 106 | image_countdown[category]--; 107 | index++; 108 | } 109 | 110 | save_file(root + "train/images.bin", bin_images, data_size); 111 | save_file(root + "train/labels.bin", bin_labels, label_size); 112 | 113 | delete[] bin_images; 114 | delete[] bin_labels; 115 | 116 | cout << "Done" << endl; 117 | } 118 | 119 | void get_validation_images(int argc, char** argv) { 120 | 121 | } 122 | 123 | void get_test_images(int argc, char** argv) { 124 | 125 | } 126 | 127 | /* 128 | * program 0 10 500 129 | * program 1 500 130 | * program 2 500 131 | */ 132 | int main(int argc, char** argv) { 133 | switch (argv[1][0]) { 134 | case '0': 135 | get_train_images(argc, argv); 136 | break; 137 | case '1': 138 | get_validation_images(argc, argv); 139 | break; 140 | case '2': 141 | get_test_images(argc, argv); 142 | break; 143 | default: 144 | cout << "Error!" << endl; 145 | } 146 | return EXIT_SUCCESS; 147 | } -------------------------------------------------------------------------------- /utils/report/train_error_curve.py: -------------------------------------------------------------------------------- 1 | # Read the log file and generate training error and validation error 2 | 3 | import sys 4 | import matplotlib.pyplot as plt 5 | 6 | extension = "_error.png" 7 | 8 | try: 9 | assert(len(sys.argv) >= 2) 10 | except Exception: 11 | print("Oops! OPlease enter the file name.") 12 | sys.exit(0) 13 | 14 | src_path = sys.argv[1] 15 | last_point = src_path.rfind('.') 16 | dest_path = src_path[:last_point] + extension 17 | 18 | start = False 19 | start_train = False 20 | start_test = False 21 | iteration = 0 22 | 23 | train_err = [] 24 | val_err = [] 25 | test_err = 0 26 | 27 | with open (src_path, "r") as f: 28 | for line in [_line.rstrip('\n') for _line in f]: # Delete the last \n 29 | if line == "XNet v1.0": 30 | start = True 31 | if start: 32 | if start_test: 33 | pos = line.find(':') 34 | if pos >= 0: 35 | test_err = float(line[pos + 2:]) 36 | if not start_train and not start_test and line[0] == 'T': 37 | int_start = line.find(' ') + 1; 38 | int_end = line.find("times") - 1; 39 | iteration = int(line[int_start:int_end]) 40 | start_train = True 41 | elif start_train: 42 | if line[0] == 'T': 43 | error = float(line[line.find(':') + 2:]) 44 | train_err.append(error) 45 | if line[0] == 'V': 46 | error = float(line[line.find(':') + 2:]) 47 | val_err.append(error) 48 | if line[0] == 'D': 49 | start_train = False 50 | start_test = True 51 | 52 | x = range(iteration) 53 | 54 | skip = 3 # skip some first data which are too big 55 | 56 | plt.ylabel('Error rate (%)') 57 | plt.xlabel('Epoch') 58 | plt.annotate('test error', xy=(iteration * 9 // 10, test_err * 0.9), 59 | xytext=(iteration * 7 // 10, test_err * 0.7), 60 | arrowprops=dict(facecolor='black', shrink=0.1)) 61 | 62 | plt.plot(x[skip:], train_err[skip:], 'b', label='training error') 63 | plt.plot(x[skip:], val_err[skip:], 'r', label='validation error') 64 | plt.plot(x[iteration * 4 // 5:], 65 | [test_err] * (iteration - iteration * 4 // 5), 'g--') 66 | plt.legend(loc='upper right', shadow=True) 67 | 68 | plt.savefig(dest_path) 69 | plt.show() 70 | --------------------------------------------------------------------------------