├── .gitattributes ├── .gitignore ├── .gitmodules ├── .travis.yml ├── .travis └── install.sh ├── CMakeLists.txt ├── FindTBB.cmake ├── README.md ├── appveyor.yml ├── data ├── solitaire-ranks-images.idx3-ubyte ├── solitaire-ranks-labels.idx1-ubyte ├── solitaire-suits-images.idx3-ubyte ├── solitaire-suits-labels.idx1-ubyte ├── t10k-images.idx3-ubyte ├── t10k-labels.idx1-ubyte ├── train-images.idx3-ubyte └── train-labels.idx1-ubyte ├── examples ├── build.bat ├── caffe_converter │ ├── caffe_converter.cpp │ └── readme.md ├── cifar10 │ ├── readme.md │ └── train.cpp ├── main.cpp ├── mnist │ ├── build.bat │ ├── readme.md │ ├── test.cpp │ └── train.cpp └── wscript ├── test ├── test.cpp ├── test_average_pooling_layer.h ├── test_convolutional_layer.h ├── test_dropout_layer.h ├── test_fully_connected_layer.h ├── test_lrn_layer.h ├── test_max_pooling_layer.h ├── test_network.h └── testhelper.h ├── tiny_cnn ├── activations │ └── activation_function.h ├── config.h ├── io │ ├── caffe │ │ ├── caffe.proto │ │ ├── layer_factory.h │ │ └── layer_factory_impl.h │ ├── cifar10_parser.h │ ├── display.h │ ├── layer_factory.h │ └── mnist_parser.h ├── layers │ ├── average_pooling_layer.h │ ├── batchnorm_layer.h │ ├── binarynet_layer.h │ ├── bnn_conv_layer.h │ ├── bnn_fc_layer.h │ ├── bnn_output_layer.h │ ├── bnn_threshold_layer.h │ ├── chaninterleave_layer.h │ ├── convolutional_layer.h │ ├── dropout_layer.h │ ├── fully_connected_layer.h │ ├── input_layer.h │ ├── layer.h │ ├── layers.h │ ├── linear_layer.h │ ├── lrn_layer.h │ ├── max_pooling_layer.h │ ├── monitor_layer.h │ ├── offloaded_layer.h │ └── partial_connected_layer.h ├── lossfunctions │ └── loss_function.h ├── network.h ├── optimizers │ └── optimizer.h ├── tiny_cnn.h └── util │ ├── aligned_allocator.h │ ├── conv_kernel.h │ ├── deform.h │ ├── image.h │ ├── nn_error.h │ ├── product.h │ ├── util.h │ └── weight_init.h ├── vc ├── vc12 │ ├── tiny_cnn.sln │ ├── tiny_cnn.vcxproj │ └── tiny_cnn_test.vcxproj └── vc14 │ ├── tiny_cnn.sln │ ├── tiny_cnn.vcxproj │ └── tiny_cnn_test.vcxproj ├── waf └── wscript /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | *.sln merge=union 7 | *.csproj merge=union 8 | *.vbproj merge=union 9 | *.fsproj merge=union 10 | *.dbproj merge=union 11 | 12 | # Standard to msysgit 13 | *.doc diff=astextplain 14 | *.DOC diff=astextplain 15 | *.docx diff=astextplain 16 | *.DOCX diff=astextplain 17 | *.dot diff=astextplain 18 | *.DOT diff=astextplain 19 | *.pdf diff=astextplain 20 | *.PDF diff=astextplain 21 | *.rtf diff=astextplain 22 | *.RTF diff=astextplain 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ################# 2 | ## Eclipse 3 | ################# 4 | 5 | *.pydevproject 6 | .project 7 | .metadata 8 | bin/ 9 | tmp/ 10 | *.tmp 11 | *.bak 12 | *.swp 13 | *~.nib 14 | local.properties 15 | .classpath 16 | .settings/ 17 | .loadpath 18 | waf-* 19 | .waf-* 20 | 21 | # External tool builders 22 | .externalToolBuilders/ 23 | 24 | # Locally stored "Eclipse launch configurations" 25 | *.launch 26 | 27 | # CDT-specific 28 | .cproject 29 | 30 | # PDT-specific 31 | .buildpath 32 | 33 | 34 | ################# 35 | ## Visual Studio 36 | ################# 37 | 38 | ## Ignore Visual Studio temporary files, build results, and 39 | ## files generated by popular Visual Studio add-ons. 40 | 41 | # User-specific files 42 | *.suo 43 | *.user 44 | *.sln.docstates 45 | 46 | # Build results 47 | [Dd]ebug/ 48 | [Rr]elease/ 49 | *_i.c 50 | *_p.c 51 | *.ilk 52 | *.meta 53 | *.obj 54 | *.pch 55 | *.pdb 56 | *.pgc 57 | *.pgd 58 | *.rsp 59 | *.sbr 60 | *.tlb 61 | *.tli 62 | *.tlh 63 | *.tmp 64 | *.vspscc 65 | .builds 66 | *.dotCover 67 | *.tlog 68 | *.ipdb 69 | *.iobj 70 | *.idb 71 | *.lastbuildstate 72 | *.unsuccessfulbuild 73 | ## TODO: If you have NuGet Package Restore enabled, uncomment this 74 | #packages/ 75 | 76 | # Visual C++ cache files 77 | ipch/ 78 | *.aps 79 | *.ncb 80 | *.opensdf 81 | *.sdf 82 | 83 | # Visual Studio profiler 84 | *.psess 85 | *.vsp 86 | 87 | # ReSharper is a .NET coding add-in 88 | _ReSharper* 89 | 90 | # Installshield output folder 91 | [Ee]xpress 92 | 93 | # DocProject is a documentation generator add-in 94 | DocProject/buildhelp/ 95 | DocProject/Help/*.HxT 96 | DocProject/Help/*.HxC 97 | DocProject/Help/*.hhc 98 | DocProject/Help/*.hhk 99 | DocProject/Help/*.hhp 100 | DocProject/Help/Html2 101 | DocProject/Help/html 102 | 103 | # Click-Once directory 104 | publish 105 | 106 | # Others 107 | [Bb]in 108 | [Oo]bj 109 | sql 110 | TestResults 111 | *.Cache 112 | ClientBin 113 | stylecop.* 114 | ~$* 115 | *.dbmdl 116 | Generated_Code #added for RIA/Silverlight projects 117 | 118 | # Backup & report files from converting an old project file to a newer 119 | # Visual Studio version. Backup files are not needed, because we have git ;-) 120 | _UpgradeReport_Files/ 121 | Backup*/ 122 | UpgradeLog*.XML 123 | 124 | 125 | 126 | ############ 127 | ## Windows 128 | ############ 129 | 130 | # Windows image file caches 131 | Thumbs.db 132 | 133 | # Folder config file 134 | Desktop.ini 135 | 136 | 137 | ############# 138 | ## Python 139 | ############# 140 | 141 | *.py[co] 142 | 143 | # Packages 144 | *.egg 145 | *.egg-info 146 | dist 147 | build 148 | eggs 149 | parts 150 | bin 151 | var 152 | sdist 153 | develop-eggs 154 | .installed.cfg 155 | 156 | # Installer logs 157 | pip-log.txt 158 | 159 | # Unit test / coverage reports 160 | .coverage 161 | .tox 162 | 163 | #Translations 164 | *.mo 165 | 166 | #Mr Developer 167 | .mr.developer.cfg 168 | 169 | # Mac crap 170 | .DS_Store 171 | 172 | 173 | 174 | ################# 175 | ## Waf 176 | ################# 177 | build 178 | waf3* 179 | .lock-waf* 180 | 181 | ################# 182 | ## CMake 183 | ################# 184 | CMakeCache.txt 185 | CMakeFiles* 186 | ALL_BUILD* 187 | ZERO_CHECK* 188 | 189 | ################# 190 | ## Other 191 | ################# 192 | *.exe 193 | *.dll 194 | *.bmp 195 | *.bin 196 | *.pb.cc 197 | *.pb.h 198 | CMakeFiles* 199 | build* 200 | *.cmake 201 | 202 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "test/picotest"] 2 | path = test/picotest 3 | url = https://github.com/nyanp/picotest.git 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: cpp 3 | os: 4 | - linux 5 | - osx 6 | compiler: 7 | - gcc 8 | - clang 9 | cache: 10 | directories: 11 | - $HOME/opencv_3_1_0 # Cache is only available on travis container-based boxes 12 | addons: 13 | apt: 14 | sources: 15 | - ubuntu-toolchain-r-test 16 | - boost-latest 17 | packages: 18 | - gcc-4.8 19 | - g++-4.8 20 | - clang 21 | - cmake 22 | - git 23 | - libtbb-dev 24 | - libboost1.55-all-dev 25 | - libjpeg8-dev 26 | - libtiff4-dev 27 | - libjasper-dev 28 | - libpng12-dev 29 | - libgtk2.0-dev 30 | - libavcodec-dev 31 | - libavformat-dev 32 | - libswscale-dev 33 | - libv4l-dev 34 | - libatlas-base-dev 35 | - gfortran 36 | install: 37 | - bash -x .travis/install.sh 38 | script: 39 | - if [ "$TRAVIS_OS_NAME" == "linux" ] && [ "$CXX" == "g++" ]; then export CXX="g++-4.8" CC="gcc-4.8"; fi 40 | - if [ "$TRAVIS_OS_NAME" == "linux" ]; then export CPLUS_INCLUDE_PATH="$HOME/opencv_3_1_0/include"; fi 41 | - if [ "$TRAVIS_OS_NAME" == "linux" ]; then cmake -DUSE_TBB=ON -DUSE_AVX=OFF -DBUILD_TESTS=ON -DOpenCV_DIR="$HOME/opencv_3_1_0/share/OpenCV" .; fi 42 | - if [ "$TRAVIS_OS_NAME" == "osx" ]; then cmake -DUSE_TBB=ON -DUSE_AVX=OFF -DBUILD_TESTS=ON .; fi 43 | - make -j2 44 | - ./tiny_cnn_test 45 | branches: 46 | only: 47 | - master 48 | matrix: 49 | exclude: # On OSX g++ is a symlink to clang++ by default 50 | - os: osx 51 | compiler: gcc 52 | -------------------------------------------------------------------------------- /.travis/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | if [ "$TRAVIS_OS_NAME" == "osx" ]; then 5 | brew update >/dev/null 6 | [ -z $( brew tap | grep 'homebrew/science' ) ] && brew tap homebrew/science 7 | # TODO download homebrew-cache.tar.gz from s3 or similar to speed up the build 8 | # Travis OSX boxes don't provide caching unfortunately 9 | if [ -f "${HOME}/homebrew-cache/homebrew-cache.tar.gz" ]; then 10 | tar -xvzf "${HOME}/homebrew-cache/homebrew-cache.tar.gz" --directory /usr/local/Cellar 11 | brew link --force tbb boost cmake opencv3 jpeg libpng libtiff 12 | brew install pkg-config 13 | else 14 | [ -z "$( brew ls --versions tbb )" ] && brew install --c++11 tbb 15 | [ -z "$( brew ls --versions boost )" ] && brew install --c++11 boost 16 | [ -z "$( brew ls --versions cmake )" ] && brew install cmake 17 | [ -z "$( brew ls --versions opencv3 )" ] && brew install -v --c++11 --with-tbb --without-python --without-openexr --without-eigen --without-numpy opencv3 && brew link --force opencv3 # verbose flag prevents the build from going stale 18 | mkdir "${HOME}/homebrew-cache" 19 | tar -czvf "${HOME}/homebrew-cache/homebrew-cache.tar.gz" --directory /usr/local/Cellar tbb boost cmake opencv3 jpeg libpng libtiff 20 | fi 21 | elif [ "$TRAVIS_OS_NAME" == "linux" ]; then 22 | version_major=3 23 | version_minor=1 24 | version_patch=0 25 | OPENCV_DIR="${HOME}/opencv_${version_major}_${version_minor}_${version_patch}" 26 | if [ ! -d "${OPENCV_DIR}/lib" ]; then 27 | git clone https://github.com/Itseez/opencv.git 28 | cd opencv 29 | git checkout ${version_major}.${version_minor}.${version_patch} 30 | mkdir build 31 | cd build 32 | cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$OPENCV_DIR" .. 33 | make -j2 && make -j2 install 34 | else 35 | echo "Cached OpenCV ${1}.${2}.${3} found"; 36 | fi 37 | fi 38 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | init: 2 | - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) 3 | 4 | environment: 5 | home: C:\projects 6 | cmake: C:\projects\cmake-3.4.1-win32-x86\bin\cmake.exe 7 | 8 | version: '{branch}-{build}' 9 | 10 | os: Visual Studio 2015 11 | 12 | install: 13 | # Clone submodule 14 | - git submodule update --init --recursive 15 | # Get a recent CMake and our dependencies (Eigen, OpenCV, Boost): 16 | - cmd: cd %home% 17 | - ps: wget https://cmake.org/files/v3.4/cmake-3.4.1-win32-x86.zip -OutFile cmake.zip 18 | - cmd: 7z x cmake.zip -o"C:\projects" -y > nul # will extract to cmake-3.4.1-win32-x86\ 19 | - cmd: '%cmake% --version' 20 | 21 | # gradient-check test is too slow to execute on debug mode 22 | configuration: Release 23 | 24 | before_build: 25 | - cmd: mkdir build 26 | - cmd: cd build 27 | - cmd: '%cmake% -G "Visual Studio 14 Win64" -DUSE_SSE=ON -DBUILD_TESTS=ON -DBUILD_EXAMPLES=OFF -DBOOST_ROOT=C:\Libraries\boost_1_59_0 -DBOOST_LIBRARYDIR=C:\Libraries\boost_1_59_0\lib64-msvc-14.0 -DCMAKE_INSTALL_PREFIX=..\install ..\tiny-cnn' 28 | 29 | after_build: 30 | - cmd: 'cd C:\projects\build\bin\Release' 31 | - cmd: 'copy "C:\Libraries\boost_1_59_0\lib64-msvc-14.0\boost_system-vc140-mt-1_59.dll" .' 32 | - cmd: 'copy "C:\Libraries\boost_1_59_0\lib64-msvc-14.0\boost_filesystem-vc140-mt-1_59.dll" .' 33 | - cmd: tiny_cnn_test.exe 34 | 35 | build: 36 | project: C:\projects\build\tiny_cnn.sln 37 | 38 | on_finish: 39 | - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) 40 | -------------------------------------------------------------------------------- /data/solitaire-ranks-images.idx3-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xilinx/xilinx-tiny-cnn/64ee7a274e67afe664d787313c9cb5b80c0396ee/data/solitaire-ranks-images.idx3-ubyte -------------------------------------------------------------------------------- /data/solitaire-ranks-labels.idx1-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xilinx/xilinx-tiny-cnn/64ee7a274e67afe664d787313c9cb5b80c0396ee/data/solitaire-ranks-labels.idx1-ubyte -------------------------------------------------------------------------------- /data/solitaire-suits-images.idx3-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xilinx/xilinx-tiny-cnn/64ee7a274e67afe664d787313c9cb5b80c0396ee/data/solitaire-suits-images.idx3-ubyte -------------------------------------------------------------------------------- /data/solitaire-suits-labels.idx1-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xilinx/xilinx-tiny-cnn/64ee7a274e67afe664d787313c9cb5b80c0396ee/data/solitaire-suits-labels.idx1-ubyte -------------------------------------------------------------------------------- /data/t10k-images.idx3-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xilinx/xilinx-tiny-cnn/64ee7a274e67afe664d787313c9cb5b80c0396ee/data/t10k-images.idx3-ubyte -------------------------------------------------------------------------------- /data/train-images.idx3-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xilinx/xilinx-tiny-cnn/64ee7a274e67afe664d787313c9cb5b80c0396ee/data/train-images.idx3-ubyte -------------------------------------------------------------------------------- /data/train-labels.idx1-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xilinx/xilinx-tiny-cnn/64ee7a274e67afe664d787313c9cb5b80c0396ee/data/train-labels.idx1-ubyte -------------------------------------------------------------------------------- /examples/build.bat: -------------------------------------------------------------------------------- 1 | g++ -std=c++11 -Wall -Werror -Wextra -pedantic -I ../ -O3 main.cpp -o main 2 | -------------------------------------------------------------------------------- /examples/caffe_converter/caffe_converter.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2016, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #include 28 | #include 29 | #define CNN_USE_CAFFE_CONVERTER 30 | #include "tiny_cnn/tiny_cnn.h" 31 | 32 | using namespace tiny_cnn; 33 | using namespace tiny_cnn::activation; 34 | using namespace std; 35 | 36 | #include 37 | #include 38 | #include 39 | #include 40 | 41 | cv::Mat compute_mean(const string& mean_file, int width, int height) 42 | { 43 | caffe::BlobProto blob; 44 | detail::read_proto_from_binary(mean_file, &blob); 45 | 46 | vector channels; 47 | auto data = blob.mutable_data()->mutable_data(); 48 | 49 | for (int i = 0; i < blob.channels(); i++, data += blob.height() * blob.width()) 50 | channels.emplace_back(blob.height(), blob.width(), CV_32FC1, data); 51 | 52 | cv::Mat mean; 53 | cv::merge(channels, mean); 54 | 55 | return cv::Mat(cv::Size(width, height), mean.type(), cv::mean(mean)); 56 | } 57 | 58 | cv::ColorConversionCodes get_cvt_codes(int src_channels, int dst_channels) 59 | { 60 | assert(src_channels != dst_channels); 61 | 62 | if (dst_channels == 3) 63 | return src_channels == 1 ? cv::COLOR_GRAY2BGR : cv::COLOR_BGRA2BGR; 64 | else if (dst_channels == 1) 65 | return src_channels == 3 ? cv::COLOR_BGR2GRAY : cv::COLOR_BGRA2GRAY; 66 | else 67 | throw runtime_error("unsupported color code"); 68 | } 69 | 70 | void preprocess(const cv::Mat& img, 71 | const cv::Mat& mean, 72 | int num_channels, 73 | cv::Size geometry, 74 | vector* input_channels) 75 | { 76 | cv::Mat sample; 77 | 78 | // convert color 79 | if (img.channels() != num_channels) 80 | cv::cvtColor(img, sample, get_cvt_codes(img.channels(), num_channels)); 81 | else 82 | sample = img; 83 | 84 | // resize 85 | cv::Mat sample_resized; 86 | cv::resize(sample, sample_resized, geometry); 87 | 88 | cv::Mat sample_float; 89 | sample_resized.convertTo(sample_float, num_channels == 3 ? CV_32FC3 : CV_32FC1); 90 | 91 | // subtract mean 92 | if (mean.size().width > 0) { 93 | cv::Mat sample_normalized; 94 | cv::subtract(sample_float, mean, sample_normalized); 95 | cv::split(sample_normalized, *input_channels); 96 | } 97 | else { 98 | cv::split(sample_float, *input_channels); 99 | } 100 | } 101 | 102 | vector get_label_list(const string& label_file) 103 | { 104 | string line; 105 | ifstream ifs(label_file.c_str()); 106 | 107 | if (ifs.fail() || ifs.bad()) 108 | throw runtime_error("failed to open:" + label_file); 109 | 110 | vector lines; 111 | while (getline(ifs, line)) 112 | lines.push_back(line); 113 | 114 | return lines; 115 | } 116 | 117 | void test(const string& model_file, 118 | const string& trained_file, 119 | const string& mean_file, 120 | const string& label_file, 121 | const string& img_file) 122 | { 123 | auto labels = get_label_list(label_file); 124 | auto net = create_net_from_caffe_prototxt(model_file); 125 | reload_weight_from_caffe_protobinary(trained_file, net.get()); 126 | 127 | int channels = net->in_shape().depth_; 128 | int width = net->in_shape().width_; 129 | int height = net->in_shape().height_; 130 | 131 | cv::Mat img = cv::imread(img_file, -1); 132 | 133 | auto mean = compute_mean(mean_file, width, height); 134 | 135 | vector inputvec(width*height*channels); 136 | vector input_channels; 137 | 138 | for (int i = 0; i < channels; i++) 139 | input_channels.emplace_back(height, width, CV_32FC1, &inputvec[width*height*i]); 140 | 141 | preprocess(img, mean, 3, cv::Size(width, height), &input_channels); 142 | 143 | vector vec(inputvec.begin(), inputvec.end()); 144 | 145 | auto result = net->predict(vec); 146 | vector sorted(result.begin(), result.end()); 147 | 148 | int top_n = 5; 149 | partial_sort(sorted.begin(), sorted.begin()+top_n, sorted.end(), greater()); 150 | 151 | for (int i = 0; i < top_n; i++) { 152 | size_t idx = distance(result.begin(), find(result.begin(), result.end(), sorted[i])); 153 | cout << labels[idx] << "," << sorted[i] << endl; 154 | } 155 | } 156 | 157 | int main(int argc, char** argv) { 158 | int arg_channel = 1; 159 | string model_file = argv[arg_channel++]; 160 | string trained_file = argv[arg_channel++]; 161 | string mean_file = argv[arg_channel++]; 162 | string label_file = argv[arg_channel++]; 163 | string img_file = argv[arg_channel++]; 164 | 165 | try { 166 | test(model_file, trained_file, mean_file, label_file, img_file); 167 | } catch (const nn_error& e) { 168 | cout << e.what() << endl; 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /examples/caffe_converter/readme.md: -------------------------------------------------------------------------------- 1 | # Import Caffe Model to tiny-cnn 2 | tiny-cnn can import Caffe's trained models. 3 | 4 | ## Prerequisites for this example 5 | - Google protobuf 6 | - OpenCV 7 | 8 | ## Build 9 | 10 | 1 Use ```protoc``` to generte caffe.pb.cc and caffe.pb.h. 11 | ```bash 12 | cd tiny_cnn/io/caffe 13 | protoc caffe.proto --cpp_out=./ 14 | ``` 15 | 16 | 2 Compile ```tiny_cnn/io/caffe/caffe.pb.cc``` and ```examples/caffe_converter/caffe_converter.cpp``` and link them. 17 | 18 | ## Usage 19 | ```bash 20 | ./caffe_converter.bin [model-file] [trained-file] [mean-file] [label-file] [img-file] 21 | ``` 22 | 23 | In the [pre-trained CaffeNet](https://github.com/BVLC/caffe/tree/master/examples/cpp_classification) model, 24 | ``` 25 | ./caffe_converter.bin\ 26 | deploy.prototxt\ 27 | bvlc_reference_caffenet.caffemodel\ 28 | imagenet_mean.binaryproto\ 29 | synset_words.txt\ 30 | cat.jpg 31 | ``` 32 | 33 | ## Restrictions 34 | - tiny-cnn's converter only supports single input/single output network without branch. -------------------------------------------------------------------------------- /examples/cifar10/readme.md: -------------------------------------------------------------------------------- 1 | # Cifar-10 Classification Example 2 | 3 | [Cifar-10](http://www.cs.toronto.edu/~kriz/cifar.html) is a common dataset for object classification. 4 | The problem is to classify 32x32 RGB (thus 32x32x3=1024 dimensions) image into 10 classes: 5 | airplane, automobile, bird, cat, deer, dog, frog, horse, ship, and truck. 6 | 7 | This problem is more complex than [MNIST](http://yann.lecun.com/exdb/mnist/) classification task. 8 | This means network architecture for Cifar-10 tends to be larger (or/and deeper) than MNIST. 9 | (If you are a machine learning beginner, I recommend you to visit 10 | [MNIST example](https://github.com/nyanp/tiny-cnn/tree/master/examples/mnist) before this page.) 11 | 12 | ## Prerequisites for this example 13 | - Download and locate Cifar-10 binary version dataset 14 | 15 | ## Constructing Model 16 | 17 | ```cpp 18 | // specify loss-function and learning strategy 19 | network nn; 20 | 21 | typedef convolutional_layer conv; 22 | typedef max_pooling_layer pool; 23 | 24 | const int n_fmaps = 32; ///< number of feature maps for upper layer 25 | const int n_fmaps2 = 64; ///< number of feature maps for lower layer 26 | const int n_fc = 64; ///< number of hidden units in fully-connected layer 27 | 28 | nn << conv(32, 32, 5, 3, n_fmaps, padding::same) 29 | << pool(32, 32, n_fmaps, 2) 30 | << conv(16, 16, 5, n_fmaps, n_fmaps, padding::same) 31 | << pool(16, 16, n_fmaps, 2) 32 | << conv(8, 8, 5, n_fmaps, n_fmaps2, padding::same) 33 | << pool(8, 8, n_fmaps2, 2) 34 | << fully_connected_layer(4 * 4 * n_fmaps2, n_fc) 35 | << fully_connected_layer(n_fc, 10); 36 | 37 | ``` 38 | 39 | ## Loading Dataset 40 | ```cpp 41 | vector train_labels, test_labels; 42 | vector train_images, test_images; 43 | 44 | for (int i = 1; i <= 5; i++) { 45 | parse_cifar10(data_dir_path + "/data_batch_" + to_string(i) + ".bin", 46 | &train_images, &train_labels, -1.0, 1.0, 0, 0); 47 | } 48 | 49 | parse_cifar10(data_dir_path + "/test_batch.bin", 50 | &test_images, &test_labels, -1.0, 1.0, 0, 0); 51 | ``` 52 | 53 | # Grid Search 54 | One of the most important hyperparameter in deep learning is learning rate. 55 | To get stable and better result, let's try [grid search](https://en.wikipedia.org/wiki/Hyperparameter_optimization#Grid_search) 56 | for learning rate. The entire code for training cifar-10 is following: 57 | 58 | ```cpp 59 | #include 60 | #include "tiny_cnn/tiny_cnn.h" 61 | 62 | using namespace tiny_cnn; 63 | using namespace tiny_cnn::activation; 64 | 65 | template 66 | void construct_net(N& nn) { 67 | typedef convolutional_layer conv; 68 | typedef max_pooling_layer pool; 69 | 70 | const int n_fmaps = 32; ///< number of feature maps for upper layer 71 | const int n_fmaps2 = 64; ///< number of feature maps for lower layer 72 | const int n_fc = 64; ///< number of hidden units in fully-connected layer 73 | 74 | nn << conv(32, 32, 5, 3, n_fmaps, padding::same) 75 | << pool(32, 32, n_fmaps, 2) 76 | << conv(16, 16, 5, n_fmaps, n_fmaps, padding::same) 77 | << pool(16, 16, n_fmaps, 2) 78 | << conv(8, 8, 5, n_fmaps, n_fmaps2, padding::same) 79 | << pool(8, 8, n_fmaps2, 2) 80 | << fully_connected_layer(4 * 4 * n_fmaps2, n_fc) 81 | << fully_connected_layer(n_fc, 10); 82 | } 83 | 84 | void train_cifar10(string data_dir_path, double learning_rate, ostream& log) { 85 | // specify loss-function and learning strategy 86 | network nn; 87 | 88 | construct_net(nn); 89 | 90 | log << "learning rate:" << learning_rate << endl; 91 | 92 | cout << "load models..." << endl; 93 | 94 | // load cifar dataset 95 | vector train_labels, test_labels; 96 | vector train_images, test_images; 97 | 98 | for (int i = 1; i <= 5; i++) { 99 | parse_cifar10(data_dir_path + "/data_batch_" + to_string(i) + ".bin", 100 | &train_images, &train_labels, -1.0, 1.0, 0, 0); 101 | } 102 | 103 | parse_cifar10(data_dir_path + "/test_batch.bin", 104 | &test_images, &test_labels, -1.0, 1.0, 0, 0); 105 | 106 | cout << "start learning" << endl; 107 | 108 | progress_display disp(train_images.size()); 109 | timer t; 110 | const int n_minibatch = 10; ///< minibatch size 111 | const int n_train_epochs = 30; ///< training duration 112 | 113 | nn.optimizer().alpha *= sqrt(n_minibatch) * learning_rate; 114 | 115 | // create callback 116 | auto on_enumerate_epoch = [&]() { 117 | cout << t.elapsed() << "s elapsed." << endl; 118 | tiny_cnn::result res = nn.test(test_images, test_labels); 119 | log << res.num_success << "/" << res.num_total << endl; 120 | 121 | disp.restart(train_images.size()); 122 | t.restart(); 123 | }; 124 | 125 | auto on_enumerate_minibatch = [&]() { 126 | disp += minibatch_size; 127 | }; 128 | 129 | // training 130 | nn.train(train_images, train_labels, n_minibatch, n_train_epochs, on_enumerate_minibatch, on_enumerate_epoch); 131 | 132 | cout << "end training." << endl; 133 | 134 | // test and show results 135 | nn.test(test_images, test_labels).print_detail(cout); 136 | 137 | // save networks 138 | ofstream ofs("cifar-weights"); 139 | ofs << nn; 140 | } 141 | 142 | int main(int argc, char **argv) { 143 | if (argc != 3) { 144 | cerr << "Usage : " << argv[0] 145 | << "arg[0]: path_to_data (example:../data)" << endl; 146 | << "arg[1]: learning rate (example:0.01)" << endl; 147 | return -1; 148 | } 149 | train_cifar10(argv[1], stod(argv[2]), cout); 150 | } 151 | ``` 152 | 153 | compile this file and try various learning rate: 154 | 155 | ``` 156 | ./train your-cifar-10-data-directory 10.0 157 | ./train your-cifar-10-data-directory 1.0 158 | ./train your-cifar-10-data-directory 0.1 159 | ./train your-cifar-10-data-directory 0.01 160 | ``` 161 | 162 | >Note: 163 | >If training is too slow, change ```n_training_epochs```, ```n_fmaps``` and ```n_fmaps2``` variables to smaller value. 164 | 165 | If you see the following message, some network weights become infinite while training. 166 | Usually it implies too large learning rate. 167 | 168 | ``` 169 | [Warning]Detected infinite value in weight. stop learning. 170 | ``` 171 | 172 | You will get about 70% accuracy in learning rate=0.01. -------------------------------------------------------------------------------- /examples/cifar10/train.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2016, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #include 28 | #include "tiny_cnn/tiny_cnn.h" 29 | 30 | using namespace tiny_cnn; 31 | using namespace tiny_cnn::activation; 32 | using namespace std; 33 | 34 | template 35 | void construct_net(N& nn) { 36 | typedef convolutional_layer conv; 37 | typedef max_pooling_layer pool; 38 | 39 | const int n_fmaps = 32; ///< number of feature maps for upper layer 40 | const int n_fmaps2 = 64; ///< number of feature maps for lower layer 41 | const int n_fc = 64; ///< number of hidden units in fully-connected layer 42 | 43 | nn << conv(32, 32, 5, 3, n_fmaps, padding::same) 44 | << pool(32, 32, n_fmaps, 2) 45 | << conv(16, 16, 5, n_fmaps, n_fmaps, padding::same) 46 | << pool(16, 16, n_fmaps, 2) 47 | << conv(8, 8, 5, n_fmaps, n_fmaps2, padding::same) 48 | << pool(8, 8, n_fmaps2, 2) 49 | << fully_connected_layer(4 * 4 * n_fmaps2, n_fc) 50 | << fully_connected_layer(n_fc, 10); 51 | } 52 | 53 | void train_cifar10(string data_dir_path, double learning_rate, ostream& log) { 54 | // specify loss-function and learning strategy 55 | network nn; 56 | 57 | construct_net(nn); 58 | 59 | log << "learning rate:" << learning_rate << endl; 60 | 61 | cout << "load models..." << endl; 62 | 63 | // load cifar dataset 64 | vector train_labels, test_labels; 65 | vector train_images, test_images; 66 | 67 | for (int i = 1; i <= 5; i++) { 68 | parse_cifar10(data_dir_path + "/data_batch_" + to_string(i) + ".bin", 69 | &train_images, &train_labels, -1.0, 1.0, 0, 0); 70 | } 71 | 72 | parse_cifar10(data_dir_path + "/test_batch.bin", 73 | &test_images, &test_labels, -1.0, 1.0, 0, 0); 74 | 75 | cout << "start learning" << endl; 76 | 77 | progress_display disp(train_images.size()); 78 | timer t; 79 | const int n_minibatch = 10; ///< minibatch size 80 | const int n_train_epochs = 30; ///< training duration 81 | 82 | nn.optimizer().alpha *= sqrt(n_minibatch) * learning_rate; 83 | 84 | // create callback 85 | auto on_enumerate_epoch = [&]() { 86 | cout << t.elapsed() << "s elapsed." << endl; 87 | tiny_cnn::result res = nn.test(test_images, test_labels); 88 | log << res.num_success << "/" << res.num_total << endl; 89 | 90 | disp.restart(train_images.size()); 91 | t.restart(); 92 | }; 93 | 94 | auto on_enumerate_minibatch = [&]() { 95 | disp += n_minibatch; 96 | }; 97 | 98 | // training 99 | nn.train(train_images, train_labels, n_minibatch, n_train_epochs, on_enumerate_minibatch, on_enumerate_epoch); 100 | 101 | cout << "end training." << endl; 102 | 103 | // test and show results 104 | nn.test(test_images, test_labels).print_detail(cout); 105 | 106 | // save networks 107 | ofstream ofs("cifar-weights"); 108 | ofs << nn; 109 | } 110 | 111 | int main(int argc, char **argv) { 112 | if (argc != 3) { 113 | cerr << "Usage : " << argv[0] 114 | << "arg[0]: path_to_data (example:../data)" 115 | << "arg[1]: learning rate (example:0.01)" << endl; 116 | return -1; 117 | } 118 | train_cifar10(argv[1], stod(argv[2]), cout); 119 | } 120 | -------------------------------------------------------------------------------- /examples/mnist/build.bat: -------------------------------------------------------------------------------- 1 | g++ -std=c++11 -Wall -Werror -Wextra -pedantic -I ../../ -O3 train.cpp -o train 2 | g++ -std=c++11 -Wall -Werror -Wextra -pedantic -I ../../ -O3 test.cpp -o test -------------------------------------------------------------------------------- /examples/mnist/test.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #include 28 | #include 29 | /*#include 30 | #include 31 | #include */ 32 | #include "tiny_cnn/tiny_cnn.h" 33 | 34 | using namespace tiny_cnn; 35 | using namespace tiny_cnn::activation; 36 | using namespace std; 37 | 38 | // rescale output to 0-100 39 | template 40 | double rescale(double x) { 41 | Activation a; 42 | return 100.0 * (x - a.scale().first) / (a.scale().second - a.scale().first); 43 | } 44 | 45 | // convert tiny_cnn::image to cv::Mat and resize 46 | cv::Mat image2mat(image<>& img) { 47 | cv::Mat ori(img.height(), img.width(), CV_8U, &img.at(0, 0)); 48 | cv::Mat resized; 49 | cv::resize(ori, resized, cv::Size(), 3, 3, cv::INTER_AREA); 50 | return resized; 51 | } 52 | 53 | void convert_image(const std::string& imagefilename, 54 | double minv, 55 | double maxv, 56 | int w, 57 | int h, 58 | vec_t& data) { 59 | auto img = cv::imread(imagefilename, cv::IMREAD_GRAYSCALE); 60 | if (img.data == nullptr) return; // cannot open, or it's not an image 61 | 62 | cv::Mat_ resized; 63 | cv::resize(img, resized, cv::Size(w, h)); 64 | 65 | // mnist dataset is "white on black", so negate required 66 | std::transform(resized.begin(), resized.end(), std::back_inserter(data), 67 | [=](uint8_t c) { return (255 - c) * (maxv - minv) / 255.0 + minv; }); 68 | } 69 | 70 | 71 | void construct_net(network& nn) { 72 | // connection table [Y.Lecun, 1998 Table.1] 73 | #define O true 74 | #define X false 75 | static const bool tbl[] = { 76 | O, X, X, X, O, O, O, X, X, O, O, O, O, X, O, O, 77 | O, O, X, X, X, O, O, O, X, X, O, O, O, O, X, O, 78 | O, O, O, X, X, X, O, O, O, X, X, O, X, O, O, O, 79 | X, O, O, O, X, X, O, O, O, O, X, X, O, X, O, O, 80 | X, X, O, O, O, X, X, O, O, O, O, X, O, O, X, O, 81 | X, X, X, O, O, O, X, X, O, O, O, O, X, O, O, O 82 | }; 83 | #undef O 84 | #undef X 85 | 86 | // construct nets 87 | nn << convolutional_layer(32, 32, 5, 1, 6) // C1, 1@32x32-in, 6@28x28-out 88 | << average_pooling_layer(28, 28, 6, 2) // S2, 6@28x28-in, 6@14x14-out 89 | << convolutional_layer(14, 14, 5, 6, 16, 90 | connection_table(tbl, 6, 16)) // C3, 6@14x14-in, 16@10x10-in 91 | << average_pooling_layer(10, 10, 16, 2) // S4, 16@10x10-in, 16@5x5-out 92 | << convolutional_layer(5, 5, 5, 16, 120) // C5, 16@5x5-in, 120@1x1-out 93 | << fully_connected_layer(120, 10); // F6, 120-in, 10-out 94 | } 95 | 96 | void recognize(const std::string& dictionary, const std::string& filename) { 97 | network nn; 98 | 99 | construct_net(nn); 100 | 101 | // load nets 102 | ifstream ifs(dictionary.c_str()); 103 | ifs >> nn; 104 | 105 | // convert imagefile to vec_t 106 | vec_t data; 107 | convert_image(filename, -1.0, 1.0, 32, 32, data); 108 | 109 | // recognize 110 | auto res = nn.predict(data); 111 | vector > scores; 112 | 113 | // sort & print top-3 114 | for (int i = 0; i < 10; i++) 115 | scores.emplace_back(rescale(res[i]), i); 116 | 117 | sort(scores.begin(), scores.end(), greater>()); 118 | 119 | for (int i = 0; i < 3; i++) 120 | cout << scores[i].second << "," << scores[i].first << endl; 121 | 122 | // visualize outputs of each layer 123 | for (size_t i = 0; i < nn.depth(); i++) { 124 | auto out_img = nn[i]->output_to_image(); 125 | cv::imshow("layer:" + std::to_string(i), image2mat(out_img)); 126 | } 127 | // visualize filter shape of first convolutional layer 128 | auto weight = nn.at>(0).weight_to_image(); 129 | cv::imshow("weights:", image2mat(weight)); 130 | 131 | cv::waitKey(0); 132 | } 133 | 134 | int main(int argc, char** argv) { 135 | if (argc != 2) { 136 | cout << "please specify image file"; 137 | return 0; 138 | } 139 | recognize("LeNet-weights", argv[1]); 140 | } 141 | -------------------------------------------------------------------------------- /examples/mnist/train.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #include 28 | #include "tiny_cnn/tiny_cnn.h" 29 | 30 | using namespace tiny_cnn; 31 | using namespace tiny_cnn::activation; 32 | 33 | void construct_net(network& nn) { 34 | // connection table [Y.Lecun, 1998 Table.1] 35 | #define O true 36 | #define X false 37 | static const bool tbl[] = { 38 | O, X, X, X, O, O, O, X, X, O, O, O, O, X, O, O, 39 | O, O, X, X, X, O, O, O, X, X, O, O, O, O, X, O, 40 | O, O, O, X, X, X, O, O, O, X, X, O, X, O, O, O, 41 | X, O, O, O, X, X, O, O, O, O, X, X, O, X, O, O, 42 | X, X, O, O, O, X, X, O, O, O, O, X, O, O, X, O, 43 | X, X, X, O, O, O, X, X, O, O, O, O, X, O, O, O 44 | }; 45 | #undef O 46 | #undef X 47 | 48 | // construct nets 49 | nn << convolutional_layer(32, 32, 5, 1, 6) // C1, 1@32x32-in, 6@28x28-out 50 | << average_pooling_layer(28, 28, 6, 2) // S2, 6@28x28-in, 6@14x14-out 51 | << convolutional_layer(14, 14, 5, 6, 16, 52 | connection_table(tbl, 6, 16)) // C3, 6@14x14-in, 16@10x10-in 53 | << average_pooling_layer(10, 10, 16, 2) // S4, 16@10x10-in, 16@5x5-out 54 | << convolutional_layer(5, 5, 5, 16, 120) // C5, 16@5x5-in, 120@1x1-out 55 | << fully_connected_layer(120, 10); // F6, 120-in, 10-out 56 | } 57 | 58 | void train_lenet(std::string data_dir_path) { 59 | // specify loss-function and learning strategy 60 | network nn; 61 | 62 | construct_net(nn); 63 | 64 | std::cout << "load models..." << std::endl; 65 | 66 | // load MNIST dataset 67 | std::vector train_labels, test_labels; 68 | std::vector train_images, test_images; 69 | 70 | parse_mnist_labels(data_dir_path+"/train-labels.idx1-ubyte", 71 | &train_labels); 72 | parse_mnist_images(data_dir_path+"/train-images.idx3-ubyte", 73 | &train_images, -1.0, 1.0, 2, 2); 74 | parse_mnist_labels(data_dir_path+"/t10k-labels.idx1-ubyte", 75 | &test_labels); 76 | parse_mnist_images(data_dir_path+"/t10k-images.idx3-ubyte", 77 | &test_images, -1.0, 1.0, 2, 2); 78 | 79 | std::cout << "start training" << std::endl; 80 | 81 | progress_display disp(train_images.size()); 82 | timer t; 83 | int minibatch_size = 10; 84 | int num_epochs = 30; 85 | 86 | nn.optimizer().alpha *= std::sqrt(minibatch_size); 87 | 88 | // create callback 89 | auto on_enumerate_epoch = [&](){ 90 | std::cout << t.elapsed() << "s elapsed." << std::endl; 91 | tiny_cnn::result res = nn.test(test_images, test_labels); 92 | std::cout << res.num_success << "/" << res.num_total << std::endl; 93 | 94 | disp.restart(train_images.size()); 95 | t.restart(); 96 | }; 97 | 98 | auto on_enumerate_minibatch = [&](){ 99 | disp += minibatch_size; 100 | }; 101 | 102 | // training 103 | nn.train(train_images, train_labels, minibatch_size, num_epochs, 104 | on_enumerate_minibatch, on_enumerate_epoch); 105 | 106 | std::cout << "end training." << std::endl; 107 | 108 | // test and show results 109 | nn.test(test_images, test_labels).print_detail(std::cout); 110 | 111 | // save networks 112 | std::ofstream ofs("LeNet-weights"); 113 | ofs << nn; 114 | } 115 | 116 | int main(int argc, char **argv) { 117 | if (argc != 2) { 118 | std::cerr << "Usage : " << argv[0] 119 | << " path_to_data (example:../data)" << std::endl; 120 | return -1; 121 | } 122 | train_lenet(argv[1]); 123 | } 124 | -------------------------------------------------------------------------------- /examples/wscript: -------------------------------------------------------------------------------- 1 | import sys 2 | def build(bld): 3 | if sys.platform == 'Darwin': 4 | libcxx = 'c++' 5 | else: 6 | libcxx = 'stdc++' 7 | 8 | if sys.platform.startswith('win'): 9 | bld(features = 'cxx cprogram', 10 | source = 'main.cpp', 11 | target = 'main', 12 | lib = ['tbb'], 13 | libpath = ['../'], 14 | includes = ['.', '../', bld.env.TBB_ROOT]) 15 | else: 16 | bld(features = 'cxx cprogram', 17 | source = 'main.cpp', 18 | target = 'main', 19 | cflags = ['-Wall'], 20 | cxxflags = ['-std=c++0x', '-Wall', '-s', '-Ofast'], 21 | lib = [libcxx, 'tbb'], 22 | libpath = ['../'], 23 | includes = ['.', '../', bld.env.TBB_ROOT]) 24 | -------------------------------------------------------------------------------- /test/test.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #define _CRT_SECURE_NO_WARNINGS 28 | #include "picotest/picotest.h" 29 | #include "tiny_cnn/tiny_cnn.h" 30 | 31 | using namespace tiny_cnn::activation; 32 | #include "test_network.h" 33 | #include "test_average_pooling_layer.h" 34 | #include "test_dropout_layer.h" 35 | #include "test_max_pooling_layer.h" 36 | #include "test_fully_connected_layer.h" 37 | #include "test_convolutional_layer.h" 38 | #include "test_lrn_layer.h" 39 | 40 | 41 | int main(void) { 42 | return RUN_ALL_TESTS(); 43 | } 44 | -------------------------------------------------------------------------------- /test/test_average_pooling_layer.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2016, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "picotest/picotest.h" 29 | #include "testhelper.h" 30 | #include "tiny_cnn/tiny_cnn.h" 31 | 32 | namespace tiny_cnn { 33 | 34 | TEST(ave_pool, gradient_check) { // sigmoid - cross-entropy 35 | typedef cross_entropy loss_func; 36 | typedef activation::sigmoid activation; 37 | typedef network network; 38 | 39 | network nn; 40 | nn << fully_connected_layer(3, 8) 41 | << average_pooling_layer(4, 2, 1, 2); // 4x2 => 2x1 42 | 43 | vec_t a(3, 0.0); 44 | for (int i = 0; i < 3; i++) a[i] = i; 45 | label_t t = 0; 46 | 47 | nn.init_weight(); 48 | for (int i = 0; i < 24; i++) nn[0]->weight()[i] = i; 49 | 50 | EXPECT_TRUE(nn.gradient_check(&a, &t, 1, 1e-5, GRAD_CHECK_ALL)); 51 | } 52 | 53 | TEST(ave_pool, read_write) { 54 | average_pooling_layer l1(100, 100, 5, 2); 55 | average_pooling_layer l2(100, 100, 5, 2); 56 | 57 | l1.init_weight(); 58 | l2.init_weight(); 59 | 60 | serialization_test(l1, l2); 61 | } 62 | 63 | } // namespace tiny-cnn 64 | 65 | -------------------------------------------------------------------------------- /test/test_convolutional_layer.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "picotest/picotest.h" 29 | #include "testhelper.h" 30 | #include "tiny_cnn/tiny_cnn.h" 31 | 32 | namespace tiny_cnn { 33 | 34 | TEST(convolutional, fprop) { 35 | typedef network CNN; 36 | CNN nn; 37 | 38 | convolutional_layer l(5, 5, 3, 1, 2); 39 | 40 | vec_t in(25); 41 | 42 | ASSERT_EQ(l.weight().size(), 18); 43 | 44 | std::fill(l.bias().begin(), l.bias().end(), 0.0); 45 | std::fill(l.weight().begin(), l.weight().end(), 0.0); 46 | 47 | uniform_rand(in.begin(), in.end(), -1.0, 1.0); 48 | 49 | { 50 | const vec_t& out = l.forward_propagation(in, 0); 51 | 52 | for (auto o: out) 53 | EXPECT_DOUBLE_EQ(o, (tiny_cnn::float_t)0.5); 54 | 55 | } 56 | 57 | l.weight()[0] = 0.3; l.weight()[1] = 0.1; l.weight()[2] = 0.2; 58 | l.weight()[3] = 0.0; l.weight()[4] =-0.1; l.weight()[5] =-0.1; 59 | l.weight()[6] = 0.05; l.weight()[7] =-0.2; l.weight()[8] = 0.05; 60 | 61 | l.weight()[9] = 0.0; l.weight()[10] =-0.1; l.weight()[11] = 0.1; 62 | l.weight()[12] = 0.1; l.weight()[13] =-0.2; l.weight()[14] = 0.3; 63 | l.weight()[15] = 0.2; l.weight()[16] =-0.3; l.weight()[17] = 0.2; 64 | 65 | in[0] = 3; in[1] = 2; in[2] = 1; in[3] = 5; in[4] = 2; 66 | in[5] = 3; in[6] = 0; in[7] = 2; in[8] = 0; in[9] = 1; 67 | in[10] = 0; in[11] = 6; in[12] = 1; in[13] = 1; in[14] = 10; 68 | in[15] = 3; in[16] =-1; in[17] = 2; in[18] = 9; in[19] = 0; 69 | in[20] = 1; in[21] = 2; in[22] = 1; in[23] = 5; in[24] = 5; 70 | 71 | { 72 | const vec_t& out = l.forward_propagation(in, 0); 73 | 74 | EXPECT_NEAR(0.4875026, out[0], 1E-5); 75 | EXPECT_NEAR(0.8388910, out[1], 1E-5); 76 | EXPECT_NEAR(0.8099984, out[2], 1E-5); 77 | EXPECT_NEAR(0.7407749, out[3], 1E-5); 78 | EXPECT_NEAR(0.5000000, out[4], 1E-5); 79 | EXPECT_NEAR(0.1192029, out[5], 1E-5); 80 | EXPECT_NEAR(0.5986877, out[6], 1E-5); 81 | EXPECT_NEAR(0.7595109, out[7], 1E-5); 82 | EXPECT_NEAR(0.6899745, out[8], 1E-5); 83 | } 84 | 85 | 86 | } 87 | 88 | TEST(convolutional, bprop) { 89 | network nn; 90 | 91 | nn << convolutional_layer(5, 5, 3, 1, 1); 92 | 93 | vec_t a(25, 0.0), t(9, 0.0); 94 | std::vector data, train; 95 | 96 | for (int y = 0; y < 5; y++) { 97 | a[5*y+3] = 1.0; 98 | } 99 | 100 | for (int y = 0; y < 3; y++) { 101 | t[3*y+0] = 0.0; 102 | t[3*y+1] = 0.5; 103 | t[3*y+2] = 1.0; 104 | } 105 | 106 | for (int i = 0; i < 100; i++) { 107 | data.push_back(a); 108 | train.push_back(t); 109 | } 110 | 111 | nn.train(data, train); 112 | 113 | vec_t predicted = nn.predict(a); 114 | } 115 | 116 | TEST(convolutional, gradient_check) { // tanh - mse 117 | network nn; 118 | nn << convolutional_layer(5, 5, 3, 1, 1); 119 | 120 | vec_t a(25, 0.0); 121 | label_t t = 3; 122 | 123 | uniform_rand(a.begin(), a.end(), -1, 1); 124 | nn.init_weight(); 125 | EXPECT_TRUE(nn.gradient_check(&a, &t, 1, 1e-4, GRAD_CHECK_ALL)); 126 | } 127 | 128 | TEST(convolutional, gradient_check2) { // sigmoid - mse 129 | network nn; 130 | nn << convolutional_layer(5, 5, 3, 1, 1); 131 | 132 | vec_t a(25, 0.0); 133 | label_t t = 3; 134 | 135 | uniform_rand(a.begin(), a.end(), -1, 1); 136 | nn.init_weight(); 137 | EXPECT_TRUE(nn.gradient_check(&a, &t, 1, 1e-4, GRAD_CHECK_ALL)); 138 | } 139 | 140 | TEST(convolutional, gradient_check3) { // rectified - mse 141 | network nn; 142 | 143 | nn << convolutional_layer(5, 5, 3, 1, 1); 144 | 145 | vec_t a(25, 0.0); 146 | label_t t = 3; 147 | 148 | uniform_rand(a.begin(), a.end(), -1, 1); 149 | nn.init_weight(); 150 | EXPECT_TRUE(nn.gradient_check(&a, &t, 1, 1e-4, GRAD_CHECK_ALL)); 151 | } 152 | 153 | TEST(convolutional, gradient_check4) { // identity - mse 154 | network nn; 155 | 156 | nn << convolutional_layer(5, 5, 3, 1, 1); 157 | 158 | vec_t a(25, 0.0); 159 | label_t t = 3; 160 | 161 | uniform_rand(a.begin(), a.end(), -1, 1); 162 | nn.init_weight(); 163 | EXPECT_TRUE(nn.gradient_check(&a, &t, 1, 1e-4, GRAD_CHECK_ALL)); 164 | } 165 | 166 | TEST(convolutional, gradient_check5) { // sigmoid - cross-entropy 167 | network nn; 168 | 169 | nn << convolutional_layer(5, 5, 3, 1, 1); 170 | 171 | vec_t a(25, 0.0); 172 | label_t t = 3; 173 | 174 | uniform_rand(a.begin(), a.end(), -1, 1); 175 | nn.init_weight(); 176 | EXPECT_TRUE(nn.gradient_check(&a, &t, 1, 1e-4, GRAD_CHECK_ALL)); 177 | } 178 | 179 | TEST(convolutional, read_write) 180 | { 181 | convolutional_layer l1(5, 5, 3, 1, 1); 182 | convolutional_layer l2(5, 5, 3, 1, 1); 183 | 184 | l1.init_weight(); 185 | l2.init_weight(); 186 | 187 | serialization_test(l1, l2); 188 | } 189 | 190 | TEST(convolutional, read_write2) { 191 | #define O true 192 | #define X false 193 | static const bool connection[] = { 194 | O, X, X, X, O, O, 195 | O, O, X, X, X, O, 196 | O, O, O, X, X, X 197 | }; 198 | #undef O 199 | #undef X 200 | convolutional_layer layer1(14, 14, 5, 3, 6, connection_table(connection, 3, 6)); 201 | convolutional_layer layer2(14, 14, 5, 3, 6, connection_table(connection, 3, 6)); 202 | layer1.init_weight(); 203 | layer2.init_weight(); 204 | 205 | serialization_test(layer1, layer2); 206 | } 207 | 208 | 209 | } // namespace tiny-cnn 210 | -------------------------------------------------------------------------------- /test/test_dropout_layer.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "picotest/picotest.h" 29 | #include "testhelper.h" 30 | #include "tiny_cnn/tiny_cnn.h" 31 | #include 32 | 33 | namespace tiny_cnn { 34 | 35 | TEST(dropout, randomized) { 36 | int num_units = 10000; 37 | double dropout_rate = 0.1; 38 | dropout_layer l(num_units, dropout_rate, net_phase::train); 39 | vec_t v(num_units, 1.0); 40 | const bool *pmask; 41 | 42 | l.forward_propagation(v, 0); 43 | pmask = l.get_mask(); 44 | std::deque mask1(pmask, pmask + num_units); 45 | 46 | l.forward_propagation(v, 0); 47 | pmask = l.get_mask(); 48 | std::deque mask2(pmask, pmask + num_units); 49 | 50 | // mask should change for each fprop 51 | EXPECT_TRUE(is_different_container(mask1, mask2)); 52 | 53 | // dropout-rate should be around 0.1 54 | double margin_factor = 0.9; 55 | int num_true1 = std::count(mask1.begin(), mask1.end(), true); 56 | int num_true2 = std::count(mask2.begin(), mask2.end(), true); 57 | 58 | EXPECT_LE(num_units * dropout_rate * margin_factor, num_true1); 59 | EXPECT_GE(num_units * dropout_rate / margin_factor, num_true1); 60 | EXPECT_LE(num_units * dropout_rate * margin_factor, num_true2); 61 | EXPECT_GE(num_units * dropout_rate / margin_factor, num_true2); 62 | } 63 | 64 | TEST(dropout, read_write) { 65 | dropout_layer l1(1024, 0.5, net_phase::test); 66 | dropout_layer l2(1024, 0.5, net_phase::test); 67 | 68 | l1.init_weight(); 69 | l2.init_weight(); 70 | 71 | serialization_test(l1, l2); 72 | } 73 | 74 | TEST(dropout, gradient_check) { 75 | network nn; 76 | nn << dropout_layer(50, 0.5, net_phase::test); 77 | 78 | vec_t a(50, 0.0); 79 | label_t t = 9; 80 | 81 | uniform_rand(a.begin(), a.end(), -1, 1); 82 | nn.init_weight(); 83 | EXPECT_TRUE(nn.gradient_check(&a, &t, 1, 1e-4, GRAD_CHECK_ALL)); 84 | } 85 | 86 | } // namespace tiny-cnn 87 | -------------------------------------------------------------------------------- /test/test_fully_connected_layer.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "picotest/picotest.h" 29 | #include "testhelper.h" 30 | #include "tiny_cnn/tiny_cnn.h" 31 | 32 | namespace tiny_cnn { 33 | 34 | TEST(fully_connected, bprop) { 35 | network nn; 36 | 37 | nn << fully_connected_layer(3, 2); 38 | 39 | vec_t a(3), t(2), a2(3), t2(2); 40 | 41 | a[0] = 3.0; a[1] = 0.0; a[2] = -1.0; 42 | t[0] = 0.3; t[1] = 0.7; 43 | 44 | a2[0] = 0.2; a2[1] = 0.5; a2[2] = 4.0; 45 | t2[0] = 0.5; t2[1] = 0.1; 46 | 47 | std::vector data, train; 48 | 49 | for (int i = 0; i < 100; i++) { 50 | data.push_back(a); 51 | data.push_back(a2); 52 | train.push_back(t); 53 | train.push_back(t2); 54 | } 55 | nn.optimizer().alpha = 0.1; 56 | nn.train(data, train, 1, 10); 57 | 58 | vec_t predicted = nn.predict(a); 59 | 60 | EXPECT_NEAR(predicted[0], t[0], 1E-5); 61 | EXPECT_NEAR(predicted[1], t[1], 1E-5); 62 | 63 | predicted = nn.predict(a2); 64 | 65 | EXPECT_NEAR(predicted[0], t2[0], 1E-5); 66 | EXPECT_NEAR(predicted[1], t2[1], 1E-5); 67 | } 68 | 69 | TEST(fully_connected, bprop2) { 70 | network nn; 71 | 72 | nn << fully_connected_layer(4, 6) 73 | << fully_connected_layer(6, 3); 74 | 75 | vec_t a(4, 0.0), t(3, 0.0), a2(4, 0.0), t2(3, 0.0); 76 | 77 | a[0] = 3.0; a[1] = 1.0; a[2] = -1.0; a[3] = 4.0; 78 | t[0] = 0.3; t[1] = 0.7; t[2] = 0.3; 79 | 80 | a2[0] = 1.0; a2[1] = 0.0; a2[2] = 4.0; a2[3] = 2.0; 81 | t2[0] = 0.6; t2[1] = 0.0; t2[2] = 0.1; 82 | 83 | std::vector data, train; 84 | 85 | for (int i = 0; i < 100; i++) { 86 | data.push_back(a); 87 | data.push_back(a2); 88 | train.push_back(t); 89 | train.push_back(t2); 90 | } 91 | nn.optimizer().alpha = 0.01; 92 | nn.train(data, train, 1, 10); 93 | 94 | vec_t predicted = nn.predict(a); 95 | 96 | EXPECT_NEAR(predicted[0], t[0], 1E-4); 97 | EXPECT_NEAR(predicted[1], t[1], 1E-4); 98 | 99 | predicted = nn.predict(a2); 100 | 101 | EXPECT_NEAR(predicted[0], t2[0], 1E-4); 102 | EXPECT_NEAR(predicted[1], t2[1], 1E-4); 103 | } 104 | 105 | TEST(fully_connected, gradient_check) { 106 | network nn; 107 | nn << fully_connected_layer(50, 10); 108 | 109 | vec_t a(50, 0.0); 110 | label_t t = 9; 111 | 112 | uniform_rand(a.begin(), a.end(), -1, 1); 113 | nn.init_weight(); 114 | EXPECT_TRUE(nn.gradient_check(&a, &t, 1, 1e-4, GRAD_CHECK_ALL)); 115 | } 116 | 117 | TEST(fully_connected, read_write) 118 | { 119 | fully_connected_layer l1(100, 100); 120 | fully_connected_layer l2(100, 100); 121 | 122 | l1.init_weight(); 123 | l2.init_weight(); 124 | 125 | serialization_test(l1, l2); 126 | } 127 | 128 | } // namespace tiny-cnn 129 | -------------------------------------------------------------------------------- /test/test_lrn_layer.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "picotest/picotest.h" 29 | #include "testhelper.h" 30 | #include "tiny_cnn/tiny_cnn.h" 31 | 32 | namespace tiny_cnn { 33 | 34 | TEST(lrn, cross) { 35 | lrn_layer lrn(1, 1, 3, 4, /*alpha=*/1.5, /*beta=*/2.0, norm_region::across_channels); 36 | 37 | tiny_cnn::float_t in[4] = { -1.0, 3.0, 2.0, 5.0 }; 38 | tiny_cnn::float_t expected[4] = 39 | { 40 | -1.0/36.0, // -1.0 / (1+0.5*(1*1+3*3))^2 41 | 3.0/64.0, // 3.0 / (1+0.5*(1*1+3*3+2*2))^2 42 | 2.0/400.0, // 2.0 / (1+0.5*(3*3+2*2+5*5))^2 43 | 5.0/15.5/15.5 // 5.0 / (1+0.5*(2*2+5*5))^2 44 | }; 45 | 46 | auto out = lrn.forward_propagation(vec_t(in, in + 4), 0); 47 | 48 | EXPECT_FLOAT_EQ(expected[0], out[0]); 49 | EXPECT_FLOAT_EQ(expected[1], out[1]); 50 | EXPECT_FLOAT_EQ(expected[2], out[2]); 51 | EXPECT_FLOAT_EQ(expected[3], out[3]); 52 | } 53 | 54 | TEST(lrn, read_write) { 55 | lrn_layer l1(10, 10, 3, 4, 1.5, 2.0, norm_region::across_channels); 56 | lrn_layer l2(10, 10, 3, 4, 1.5, 2.0, norm_region::across_channels); 57 | 58 | l1.init_weight(); 59 | l2.init_weight(); 60 | 61 | serialization_test(l1, l2); 62 | } 63 | 64 | } // namespace tiny-cnn 65 | -------------------------------------------------------------------------------- /test/test_max_pooling_layer.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2016, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "picotest/picotest.h" 29 | #include "testhelper.h" 30 | #include "tiny_cnn/tiny_cnn.h" 31 | 32 | namespace tiny_cnn { 33 | 34 | TEST(max_pool, gradient_check) { // sigmoid - cross-entropy 35 | typedef cross_entropy loss_func; 36 | typedef activation::sigmoid activation; 37 | typedef network network; 38 | 39 | network nn; 40 | nn << fully_connected_layer(3, 8) 41 | << max_pooling_layer(4, 2, 1, 2); // 4x2 => 2x1 42 | 43 | vec_t a(3, 0.0); 44 | for (int i = 0; i < 3; i++) a[i] = i; 45 | label_t t = 0; 46 | 47 | nn.init_weight(); 48 | for (int i = 0; i < 24; i++) nn[0]->weight()[i] = i; 49 | 50 | EXPECT_TRUE(nn.gradient_check(&a, &t, 1, 1e-5, GRAD_CHECK_ALL)); 51 | } 52 | 53 | TEST(max_pool, read_write) { 54 | max_pooling_layer l1(100, 100, 5, 2); 55 | max_pooling_layer l2(100, 100, 5, 2); 56 | 57 | l1.init_weight(); 58 | l2.init_weight(); 59 | 60 | serialization_test(l1, l2); 61 | } 62 | 63 | } // namespace tiny-cnn 64 | -------------------------------------------------------------------------------- /test/testhelper.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2016, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include 29 | #include 30 | #include 31 | #include "picotest/picotest.h" 32 | #include "tiny_cnn/tiny_cnn.h" 33 | 34 | namespace tiny_cnn { 35 | 36 | template 37 | inline bool is_near_container(const Container& expected, const Container& actual, T abs_error) { 38 | auto i1 = std::begin(expected); 39 | auto i2 = std::begin(actual); 40 | 41 | for (; i1 != std::end(expected); ++i1, ++i2) { 42 | if(std::abs(*i1 - *i2) > abs_error) return false; 43 | } 44 | return true; 45 | } 46 | 47 | template 48 | inline bool is_different_container(const Container& expected, const Container& actual) { 49 | auto i1 = std::begin(expected); 50 | auto i2 = std::begin(actual); 51 | 52 | for (; i1 != std::end(expected); ++i1, ++i2) { 53 | if (*i1 != *i2) return true; 54 | } 55 | return false; 56 | } 57 | 58 | inline bool exists(const std::string& path) { 59 | if (FILE *file = std::fopen(path.c_str(), "r")) { 60 | fclose(file); 61 | return true; 62 | } else { 63 | return false; 64 | } 65 | } 66 | 67 | inline std::string unique_path() { 68 | std::string pattern = "%%%%-%%%%-%%%%-%%%%"; 69 | 70 | for (auto p = pattern.begin(); p != pattern.end(); ++p) { 71 | if (*p == '%') *p = (rand()%10)+'0'; 72 | } 73 | return exists(pattern) ? unique_path() : pattern; 74 | } 75 | 76 | vec_t forward_pass(layer_base& src, const vec_t& vec) { 77 | return src.forward_propagation(vec, 0); 78 | } 79 | 80 | template 81 | vec_t forward_pass(network& net, const vec_t& vec) { 82 | return net.predict(vec); 83 | } 84 | 85 | template 86 | void serialization_test(T& src, T& dst) 87 | { 88 | //EXPECT_FALSE(src.has_same_weights(dst, 1E-5)); 89 | 90 | std::string tmp_file_path = unique_path(); 91 | 92 | // write 93 | { 94 | std::ofstream ofs(tmp_file_path.c_str()); 95 | ofs << src; 96 | } 97 | 98 | // read 99 | { 100 | std::ifstream ifs(tmp_file_path.c_str()); 101 | ifs >> dst; 102 | } 103 | 104 | std::remove(tmp_file_path.c_str()); 105 | 106 | vec_t v(src.in_dim()); 107 | uniform_rand(v.begin(), v.end(), -1.0, 1.0); 108 | 109 | EXPECT_TRUE(src.has_same_weights(dst, 1E-5)); 110 | 111 | vec_t r1 = forward_pass(src, v); 112 | vec_t r2 = forward_pass(dst, v); 113 | 114 | EXPECT_TRUE(is_near_container(r1, r2, 1E-4)); 115 | } 116 | 117 | } // namespace tiny_cnn 118 | -------------------------------------------------------------------------------- /tiny_cnn/activations/activation_function.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "tiny_cnn/util/util.h" 29 | #include 30 | 31 | namespace tiny_cnn { 32 | namespace activation { 33 | 34 | class function { 35 | public: 36 | function() = default; 37 | function(const function &) = default; 38 | #ifndef CNN_DEFAULT_MOVE_CONSTRUCTOR_UNAVAILABLE 39 | function(function &&) = default; 40 | #endif 41 | function &operator =(const function &) = default; 42 | #ifndef CNN_DEFAULT_ASSIGNMENT_OPERATOR_UNAVAILABLE 43 | function &operator =(function &&) = default; 44 | #endif 45 | virtual ~function() = default; 46 | 47 | virtual float_t f(const vec_t& v, cnn_size_t index) const = 0; 48 | 49 | // dfi/dyi 50 | virtual float_t df(float_t y) const = 0; 51 | 52 | // dfi/dyk (k=0,1,..n) 53 | virtual vec_t df(const vec_t& y, cnn_size_t i) const { vec_t v(y.size(), 0); v[i] = df(y[i]); return v; } 54 | 55 | // target value range for learning 56 | virtual std::pair scale() const = 0; 57 | }; 58 | 59 | class identity : public function { 60 | public: 61 | using function::df; 62 | float_t f(const vec_t& v, cnn_size_t i) const override { return v[i]; } 63 | float_t df(float_t /*y*/) const override { return float_t(1); } 64 | std::pair scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); } 65 | }; 66 | 67 | class bnn_sign : public function { 68 | public: 69 | using function::df; 70 | float_t f(const vec_t& v, cnn_size_t i) const override { return v[i] > 0 ? 1 : -1; } 71 | float_t df(float_t /*y*/) const override { throw "Derivative of sign not implemented"; return float_t(1); } 72 | std::pair scale() const override { throw "Scaling of sign not implemented"; return std::make_pair(float_t(0.1), float_t(0.9)); } 73 | }; 74 | 75 | 76 | class sigmoid : public function { 77 | public: 78 | using function::df; 79 | float_t f(const vec_t& v, cnn_size_t i) const override { return float_t(1) / (float_t(1) + std::exp(-v[i])); } 80 | float_t df(float_t y) const override { return y * (float_t(1) - y); } 81 | std::pair scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); } 82 | }; 83 | 84 | class relu : public function { 85 | public: 86 | using function::df; 87 | float_t f(const vec_t& v, cnn_size_t i) const override { return std::max(float_t(0), v[i]); } 88 | float_t df(float_t y) const override { return y > float_t(0) ? float_t(1) : float_t(0); } 89 | std::pair scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); } 90 | }; 91 | 92 | typedef relu rectified_linear; // for compatibility 93 | 94 | class leaky_relu : public function { 95 | public: 96 | using function::df; 97 | float_t f(const vec_t& v, cnn_size_t i) const override { return (v[i] > float_t(0)) ? v[i] : float_t(0.01) * v[i]; } 98 | float_t df(float_t y) const override { return y > float_t(0) ? float_t(1) : float_t(0.01); } 99 | std::pair scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); } 100 | }; 101 | 102 | class elu : public function { 103 | public: 104 | using function::df; 105 | float_t f(const vec_t& v, cnn_size_t i) const override { return (v[i] float_t(0) ? float_t(1) : (float_t(1)+y)); } 107 | std::pair scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); } 108 | }; 109 | 110 | class softmax : public function { 111 | public: 112 | float_t f(const vec_t& v, cnn_size_t i) const override { 113 | float_t alpha = *std::max_element(v.begin(), v.end()); 114 | float_t numer = std::exp(v[i] - alpha); 115 | float_t denom = float_t(0); 116 | for (auto x : v) 117 | denom += std::exp(x - alpha); 118 | return numer / denom; 119 | } 120 | 121 | float_t df(float_t y) const override { 122 | return y * (float_t(1) - y); 123 | } 124 | 125 | virtual vec_t df(const vec_t& y, cnn_size_t index) const override { 126 | vec_t v(y.size(), 0); 127 | for (cnn_size_t i = 0; i < y.size(); i++) 128 | v[i] = (i == index) ? df(y[index]) : -y[i] * y[index]; 129 | 130 | return v; 131 | } 132 | 133 | std::pair scale() const override { return std::make_pair(float_t(0), float_t(1)); } 134 | }; 135 | 136 | class tan_h : public function { 137 | public: 138 | using function::df; 139 | float_t f(const vec_t& v, cnn_size_t i) const override { 140 | const float_t ep = std::exp(v[i]); 141 | const float_t em = std::exp(-v[i]); 142 | return (ep - em) / (ep + em); 143 | } 144 | 145 | // fast approximation of tanh (improve 2-3% speed in LeNet-5) 146 | /*float_t f(float_t x) const { 147 | const float_t x2 = x * x; 148 | x *= 1.0 + x2 * (0.1653 + x2 * 0.0097); 149 | return x / std::sqrt(1.0 + x * x);// invsqrt(static_cast(1.0 + x * x)); 150 | }*/ 151 | 152 | float_t df(float_t y) const override { return float_t(1) - sqr(y); } 153 | std::pair scale() const override { return std::make_pair(float_t(-0.8), float_t(0.8)); } 154 | 155 | private: 156 | /*float invsqrt(float x) const { 157 | float x2 = x * 0.5f; 158 | long i = *reinterpret_cast(&x); 159 | 160 | i = 0x5f3759df - (i >> 1); 161 | x = *reinterpret_cast(&i); 162 | x = x * (1.5f - (x2 * x * x)); 163 | return x; 164 | }*/ 165 | }; 166 | 167 | // s tan_h, but scaled to match the other functions 168 | class tan_hp1m2 : public function { 169 | public: 170 | using function::df; 171 | float_t f(const vec_t& v, cnn_size_t i) const override { 172 | const float_t ep = std::exp(v[i]); 173 | return ep / (ep + std::exp(-v[i])); 174 | } 175 | 176 | float_t df(float_t y) const override { return 2 * y *(float_t(1) - y); } 177 | std::pair scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); } 178 | }; 179 | 180 | } // namespace activation 181 | } // namespace tiny_cnn 182 | -------------------------------------------------------------------------------- /tiny_cnn/config.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include 29 | 30 | /** 31 | * define if you want to use intel TBB library 32 | */ 33 | //#define CNN_USE_TBB 34 | 35 | /** 36 | * define to enable avx vectorization 37 | */ 38 | //#define CNN_USE_AVX 39 | 40 | /** 41 | * define to enable sse2 vectorization 42 | */ 43 | //#define CNN_USE_SSE 44 | 45 | /** 46 | * define to enable OMP parallelization 47 | */ 48 | #define CNN_USE_OMP 49 | 50 | /** 51 | * define to use exceptions 52 | */ 53 | #define CNN_USE_EXCEPTIONS 54 | 55 | /** 56 | * number of task in batch-gradient-descent. 57 | * @todo automatic optimization 58 | */ 59 | #ifdef CNN_USE_OMP 60 | #define CNN_TASK_SIZE 100 61 | #else 62 | #define CNN_TASK_SIZE 8 63 | #endif 64 | 65 | namespace tiny_cnn { 66 | 67 | /** 68 | * calculation data type 69 | * you can change it to float, or user defined class (fixed point,etc) 70 | **/ 71 | typedef float float_t; 72 | 73 | /** 74 | * size of layer, model, data etc. 75 | * change to smaller type if memory footprint is severe 76 | **/ 77 | typedef std::size_t cnn_size_t; 78 | 79 | } 80 | -------------------------------------------------------------------------------- /tiny_cnn/io/caffe/layer_factory.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include 29 | #include 30 | #include 31 | #include "caffe.pb.h" 32 | 33 | #include "tiny_cnn/tiny_cnn.h" 34 | #include "tiny_cnn/io/caffe/layer_factory_impl.h" 35 | 36 | namespace tiny_cnn { 37 | 38 | /** 39 | * create whole network and load weights from caffe's netparameter 40 | * 41 | * @param layer [in] netparameter of caffemodel 42 | * @param data_shape [in] size of input data (width x height x channels) 43 | */ 44 | inline std::shared_ptr> 45 | create_net_from_caffe_net(const caffe::NetParameter& layer, const layer_shape_t& data_shape) 46 | { 47 | detail::caffe_layer_vector src_net(layer); 48 | shape_t shape; 49 | 50 | if (data_shape.size() > 0) { 51 | shape = data_shape; 52 | } 53 | else { 54 | if (layer.input_shape_size() == 0) 55 | throw std::runtime_error("input_shape not found in caffemodel. must specify input shape explicitly"); 56 | int depth = static_cast(layer.input_shape(0).dim(1)); 57 | int width = static_cast(layer.input_shape(0).dim(2)); 58 | int height = static_cast(layer.input_shape(0).dim(3)); 59 | shape = layer_shape_t(width, height, depth); 60 | } 61 | 62 | auto dst_net = std::make_shared>(layer.name()); 63 | 64 | for (size_t i = 0; i < src_net.size(); i++) { 65 | auto type = src_net[i].type(); 66 | 67 | if (detail::layer_skipped(type)) { 68 | continue; 69 | } 70 | 71 | if (!detail::layer_supported(type)) 72 | throw std::runtime_error("error: tiny-cnn does not support this layer type:" + type); 73 | 74 | shape_t shape_next = shape; 75 | auto layer = detail::create(src_net[i], shape, &shape_next); 76 | 77 | std::cout << "convert " << type << " => " << typeid(*layer).name() << std::endl; 78 | std::cout << " shape:" << shape_next << std::endl; 79 | 80 | dst_net->add(layer); 81 | shape = shape_next; 82 | } 83 | 84 | return dst_net; 85 | } 86 | 87 | 88 | /** 89 | * create whole network and load weights from caffe's netparameter 90 | * 91 | * @param layer [in] netparameter of caffemodel 92 | * @param data_shape [in] size of input data (width x height x channels) 93 | */ 94 | inline std::shared_ptr> 95 | create_net_from_caffe_protobinary(const std::string& caffebinarymodel, const layer_shape_t& data_shape) 96 | { 97 | caffe::NetParameter np; 98 | 99 | detail::read_proto_from_binary(caffebinarymodel, &np); 100 | return create_net_from_caffe_net(np, data_shape); 101 | } 102 | 103 | /** 104 | * create whole network and load weights from caffe's netparameter 105 | * 106 | * @param layer [in] netparameter of caffe prototxt 107 | */ 108 | inline std::shared_ptr> 109 | create_net_from_caffe_prototxt(const std::string& caffeprototxt) 110 | { 111 | caffe::NetParameter np; 112 | 113 | detail::read_proto_from_text(caffeprototxt, &np); 114 | return create_net_from_caffe_net(np, layer_shape_t()); 115 | } 116 | 117 | /** 118 | * reload network weights from caffe's netparameter 119 | * this must be called after the network layers are constructed 120 | * 121 | * @param layer [in] caffe's netparameter 122 | * @param net [out] tiny-cnn's network 123 | */ 124 | template 125 | inline void reload_weight_from_caffe_net(const caffe::NetParameter& layer, network *net) 126 | { 127 | detail::caffe_layer_vector src_net(layer); 128 | 129 | int tinycnn_layer_idx = 0; 130 | 131 | for (int caffe_layer_idx = 0; caffe_layer_idx < src_net.size(); caffe_layer_idx++) { 132 | auto type = src_net[caffe_layer_idx].type(); 133 | 134 | if (detail::layer_skipped(type) || !detail::layer_has_weights(type)) { 135 | continue; 136 | } 137 | 138 | if (!detail::layer_supported(type)) 139 | throw std::runtime_error("error: tiny-cnn does not support this layer type:" + type); 140 | 141 | while (tinycnn_layer_idx < net->depth() && !detail::layer_match(type, (*net)[tinycnn_layer_idx]->layer_type())) { 142 | tinycnn_layer_idx++; 143 | } 144 | if (tinycnn_layer_idx >= net->depth()) break; 145 | 146 | // load weight 147 | detail::load(src_net[caffe_layer_idx], (*net)[tinycnn_layer_idx++]); 148 | } 149 | } 150 | 151 | /** 152 | * reload network weights from caffe's netparameter 153 | * this must be called after the network layers are constructed 154 | * 155 | * @param caffebinary [in] caffe's trained model file(binary format) 156 | * @param net [out] tiny-cnn's network 157 | */ 158 | template 159 | inline void reload_weight_from_caffe_protobinary(const std::string& caffebinary, network *net) 160 | { 161 | caffe::NetParameter np; 162 | 163 | detail::read_proto_from_binary(caffebinary, &np); 164 | reload_weight_from_caffe_net(np, net); 165 | } 166 | 167 | } // namespace tiny_cnn -------------------------------------------------------------------------------- /tiny_cnn/io/cifar10_parser.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "tiny_cnn/util/util.h" 29 | #include 30 | #include 31 | #include 32 | 33 | #define CIFAR10_IMAGE_DEPTH (3) 34 | #define CIFAR10_IMAGE_WIDTH (32) 35 | #define CIFAR10_IMAGE_HEIGHT (32) 36 | #define CIFAR10_IMAGE_AREA (CIFAR10_IMAGE_WIDTH*CIFAR10_IMAGE_HEIGHT) 37 | #define CIFAR10_IMAGE_SIZE (CIFAR10_IMAGE_AREA*CIFAR10_IMAGE_DEPTH) 38 | 39 | 40 | namespace tiny_cnn { 41 | 42 | /** 43 | * parse CIFAR-10 database format images 44 | * 45 | * @param filename [in] filename of database(binary version) 46 | * @param train_images [out] parsed images 47 | * @param train_labels [out] parsed labels 48 | * @param scale_min [in] min-value of output 49 | * @param scale_max [in] max-value of output 50 | * @param x_padding [in] adding border width (left,right) 51 | * @param y_padding [in] adding border width (top,bottom) 52 | **/ 53 | inline void parse_cifar10(const std::string& filename, 54 | std::vector *train_images, 55 | std::vector *train_labels, 56 | float_t scale_min, 57 | float_t scale_max, 58 | int x_padding, 59 | int y_padding) 60 | { 61 | if (x_padding < 0 || y_padding < 0) 62 | throw nn_error("padding size must not be negative"); 63 | if (scale_min >= scale_max) 64 | throw nn_error("scale_max must be greater than scale_min"); 65 | 66 | std::ifstream ifs(filename.c_str(), std::ios::in | std::ios::binary); 67 | if (ifs.fail() || ifs.bad()) 68 | throw nn_error("failed to open file:" + filename); 69 | 70 | uint8_t label; 71 | std::vector buf(CIFAR10_IMAGE_SIZE); 72 | 73 | while (ifs.read((char*) &label, 1)) { 74 | vec_t img; 75 | 76 | if (!ifs.read((char*) &buf[0], CIFAR10_IMAGE_SIZE)) break; 77 | 78 | if (x_padding || y_padding) 79 | { 80 | int w = CIFAR10_IMAGE_WIDTH + 2 * x_padding; 81 | int h = CIFAR10_IMAGE_HEIGHT + 2 * y_padding; 82 | 83 | img.resize(w * h * CIFAR10_IMAGE_DEPTH, scale_min); 84 | 85 | for (int c = 0; c < CIFAR10_IMAGE_DEPTH; c++) { 86 | for (int y = 0; y < CIFAR10_IMAGE_HEIGHT; y++) { 87 | for (int x = 0; x < CIFAR10_IMAGE_WIDTH; x++) { 88 | img[c * w * h + (y + y_padding) * w + x + x_padding] 89 | = scale_min + (scale_max - scale_min) * buf[c * CIFAR10_IMAGE_AREA + y * CIFAR10_IMAGE_WIDTH + x] / 255; 90 | } 91 | } 92 | } 93 | } 94 | else 95 | { 96 | std::transform(buf.begin(), buf.end(), std::back_inserter(img), 97 | [=](unsigned char c) { return scale_min + (scale_max - scale_min) * c / 255; }); 98 | } 99 | 100 | train_images->push_back(img); 101 | train_labels->push_back(label); 102 | } 103 | } 104 | 105 | } // namespace tiny_cnn 106 | -------------------------------------------------------------------------------- /tiny_cnn/io/display.h: -------------------------------------------------------------------------------- 1 | //addapted from boost progress.hpp, made c++11 only// 2 | 3 | 4 | #ifndef PROGRESS_H 5 | #define PROGRESS_H 6 | 7 | #include // for ostream, cout, etc 8 | #include // for string 9 | #include 10 | 11 | namespace tiny_cnn { 12 | 13 | class timer 14 | { 15 | public: 16 | timer(): t1(std::chrono::high_resolution_clock::now()){}; 17 | double elapsed(){return std::chrono::duration_cast>(std::chrono::high_resolution_clock::now() - t1).count();} 18 | void restart(){t1 = std::chrono::high_resolution_clock::now();} 19 | void start(){t1 = std::chrono::high_resolution_clock::now();} 20 | void stop(){t2 = std::chrono::high_resolution_clock::now();} 21 | double total(){stop();return std::chrono::duration_cast>(t2 - t1).count();} 22 | ~timer(){} 23 | private: 24 | std::chrono::high_resolution_clock::time_point t1, t2; 25 | }; 26 | 27 | 28 | // progress_display --------------------------------------------------------// 29 | 30 | // progress_display displays an appropriate indication of 31 | // progress at an appropriate place in an appropriate form. 32 | 33 | class progress_display 34 | { 35 | public: 36 | explicit progress_display( unsigned long expected_count_, 37 | std::ostream & os = std::cout, 38 | const std::string & s1 = "\n", //leading strings 39 | const std::string & s2 = "", 40 | const std::string & s3 = "" ) 41 | // os is hint; implementation may ignore, particularly in embedded systems 42 | : m_os(os), m_s1(s1), m_s2(s2), m_s3(s3) { restart(expected_count_); } 43 | 44 | void restart( unsigned long expected_count_ ) 45 | // Effects: display appropriate scale 46 | // Postconditions: count()==0, expected_count()==expected_count_ 47 | { 48 | _count = _next_tic_count = _tic = 0; 49 | _expected_count = expected_count_; 50 | 51 | m_os << m_s1 << "0% 10 20 30 40 50 60 70 80 90 100%\n" 52 | << m_s2 << "|----|----|----|----|----|----|----|----|----|----|" 53 | << std::endl // endl implies flush, which ensures display 54 | << m_s3; 55 | if ( !_expected_count ) _expected_count = 1; // prevent divide by zero 56 | } // restart 57 | 58 | unsigned long operator+=( unsigned long increment ) 59 | // Effects: Display appropriate progress tic if needed. 60 | // Postconditions: count()== original count() + increment 61 | // Returns: count(). 62 | { 63 | if ( (_count += increment) >= _next_tic_count ) { display_tic(); } 64 | return _count; 65 | } 66 | 67 | unsigned long operator++() { return operator+=( 1 ); } 68 | unsigned long count() const { return _count; } 69 | unsigned long expected_count() const { return _expected_count; } 70 | 71 | private: 72 | std::ostream & m_os; // may not be present in all imps 73 | const std::string m_s1; // string is more general, safer than 74 | const std::string m_s2; // const char *, and efficiency or size are 75 | const std::string m_s3; // not issues 76 | 77 | unsigned long _count, _expected_count, _next_tic_count; 78 | unsigned int _tic; 79 | void display_tic() 80 | { 81 | // use of floating point ensures that both large and small counts 82 | // work correctly. static_cast<>() is also used several places 83 | // to suppress spurious compiler warnings. 84 | unsigned int tics_needed = 85 | static_cast( 86 | (static_cast(_count)/_expected_count)*50.0 ); 87 | do { m_os << '*' << std::flush; } while ( ++_tic < tics_needed ); 88 | _next_tic_count = 89 | static_cast((_tic/50.0)*_expected_count); 90 | if ( _count == _expected_count ) { 91 | if ( _tic < 51 ) m_os << '*'; 92 | m_os << std::endl; 93 | } 94 | } // display_tic 95 | 96 | progress_display &operator = (const progress_display &) = delete; 97 | }; 98 | 99 | } // namespace 100 | 101 | #endif -------------------------------------------------------------------------------- /tiny_cnn/io/layer_factory.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "tiny_cnn/util/util.h" 29 | #include "tiny_cnn/layers/fully_connected_layer.h" 30 | 31 | namespace tiny_cnn { 32 | 33 | /** 34 | * create multi-layer perceptron 35 | */ 36 | template 37 | network make_mlp(Iter first, Iter last) { 38 | typedef network net_t; 39 | net_t n; 40 | 41 | Iter next = first + 1; 42 | for (; next != last; ++first, ++next) 43 | n << fully_connected_layer(*first, *next); 44 | return n; 45 | } 46 | 47 | /** 48 | * create multi-layer perceptron 49 | */ 50 | template 51 | network make_mlp(const std::vector& units) { 52 | return make_mlp(units.begin(), units.end()); 53 | } 54 | 55 | 56 | } // namespace tiny_cnn 57 | -------------------------------------------------------------------------------- /tiny_cnn/layers/average_pooling_layer.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "tiny_cnn/util/util.h" 29 | #include "tiny_cnn/util/image.h" 30 | #include "tiny_cnn/layers/partial_connected_layer.h" 31 | #include "tiny_cnn/activations/activation_function.h" 32 | 33 | namespace tiny_cnn { 34 | 35 | 36 | template 37 | class average_pooling_layer : public partial_connected_layer { 38 | public: 39 | typedef partial_connected_layer Base; 40 | CNN_USE_LAYER_MEMBERS; 41 | 42 | average_pooling_layer(cnn_size_t in_width, cnn_size_t in_height, cnn_size_t in_channels, cnn_size_t pooling_size) 43 | : Base(in_width * in_height * in_channels, 44 | in_width * in_height * in_channels / sqr(pooling_size), 45 | in_channels, in_channels, float_t(1) / sqr(pooling_size)), 46 | stride_(pooling_size), 47 | in_(in_width, in_height, in_channels), 48 | out_(in_width/pooling_size, in_height/pooling_size, in_channels) 49 | { 50 | if ((in_width % pooling_size) || (in_height % pooling_size)) 51 | pooling_size_mismatch(in_width, in_height, pooling_size); 52 | 53 | init_connection(pooling_size); 54 | } 55 | 56 | average_pooling_layer(cnn_size_t in_width, cnn_size_t in_height, cnn_size_t in_channels, cnn_size_t pooling_size, cnn_size_t stride) 57 | : Base(in_width * in_height * in_channels, 58 | pool_out_dim(in_width, pooling_size, stride) * pool_out_dim(in_height, pooling_size, stride) * in_channels, 59 | in_channels, in_channels, float_t(1) / sqr(pooling_size)), 60 | stride_(stride), 61 | in_(in_width, in_height, in_channels), 62 | out_(pool_out_dim(in_width, pooling_size, stride), pool_out_dim(in_height, pooling_size, stride), in_channels) 63 | { 64 | // if ((in_width % pooling_size) || (in_height % pooling_size)) 65 | // pooling_size_mismatch(in_width, in_height, pooling_size); 66 | 67 | init_connection(pooling_size); 68 | } 69 | 70 | image<> output_to_image(size_t worker_index = 0) const override { 71 | return vec2image(output_[worker_index], out_); 72 | } 73 | 74 | index3d in_shape() const override { return in_; } 75 | index3d out_shape() const override { return out_; } 76 | std::string layer_type() const override { return "ave-pool"; } 77 | 78 | private: 79 | size_t stride_; 80 | 81 | static cnn_size_t pool_out_dim(cnn_size_t in_size, cnn_size_t pooling_size, cnn_size_t stride) { 82 | return (int)std::ceil(((double)in_size - pooling_size) / stride) + 1; 83 | } 84 | 85 | void init_connection(cnn_size_t pooling_size) { 86 | for (cnn_size_t c = 0; c < in_.depth_; ++c) 87 | for (cnn_size_t y = 0; y < in_.height_; y += stride_) 88 | for (cnn_size_t x = 0; x < in_.width_; x += stride_) 89 | connect_kernel(pooling_size, x, y, c); 90 | 91 | 92 | for (cnn_size_t c = 0; c < in_.depth_; ++c) 93 | for (cnn_size_t y = 0; y < out_.height_; ++y) 94 | for (cnn_size_t x = 0; x < out_.width_; ++x) 95 | this->connect_bias(c, out_.get_index(x, y, c)); 96 | } 97 | 98 | void connect_kernel(cnn_size_t pooling_size, cnn_size_t x, cnn_size_t y, cnn_size_t inc) { 99 | cnn_size_t dymax = std::min(pooling_size, in_.height_ - y); 100 | cnn_size_t dxmax = std::min(pooling_size, in_.width_ - x); 101 | cnn_size_t dstx = x / stride_; 102 | cnn_size_t dsty = y / stride_; 103 | 104 | for (cnn_size_t dy = 0; dy < dymax; ++dy) 105 | for (cnn_size_t dx = 0; dx < dxmax; ++dx) 106 | this->connect_weight( 107 | in_.get_index(x + dx, y + dy, inc), 108 | out_.get_index(dstx, dsty, inc), 109 | inc); 110 | } 111 | 112 | index3d in_; 113 | index3d out_; 114 | }; 115 | 116 | } // namespace tiny_cnn 117 | -------------------------------------------------------------------------------- /tiny_cnn/layers/batchnorm_layer.h: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | * Copyright (c) 2016, Xilinx, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * 3. Neither the name of the copyright holder nor the names of its 16 | * contributors may be used to endorse or promote products derived from 17 | * this software without specific prior written permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 23 | * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 | * OR BUSINESS INTERRUPTION). HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 | * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 29 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | * 31 | *****************************************************************************/ 32 | #pragma once 33 | #include "tiny_cnn/layers/layer.h" 34 | #include 35 | #include 36 | 37 | // right now functional during inference only 38 | // pre-trained batchnorm params must be manually set 39 | // TODO: 40 | // - implement proper batch normalization during training 41 | 42 | // during training, for each neuron output, batch normalization learns 43 | // four statistics from the data that passes through it. we keep all 44 | // four of these as part of the layer's weights, in the following order: 45 | // * all of the shifts (beta) 46 | // * all of the scales (gamma) 47 | // * all of the means (mu) 48 | // * all of the inverted stddevs (inv_std) 49 | 50 | namespace tiny_cnn { 51 | template 52 | class batchnorm_layer : public layer { 53 | public: 54 | typedef layer Base; 55 | CNN_USE_LAYER_MEMBERS; 56 | 57 | // channels: number of channels. each channel has a batchnorm parameter set. 58 | // dim: number of pixels/elements in each channel. 59 | batchnorm_layer(cnn_size_t channels, cnn_size_t dim = 1, std::string paramFile = "") 60 | : Base(dim*channels, dim*channels, 4*channels, 0), dim_(dim), channels_(channels) 61 | { 62 | if(paramFile != "") { 63 | loadFromBinaryFile(paramFile); 64 | } 65 | } 66 | 67 | void loadFromBinaryFile(std::string fileName) { 68 | // TODO this assumes the binary file always uses 4 bytes per batchnorm parameter 69 | 70 | std::ifstream wf(fileName, std::ios::binary | std::ios::in); 71 | for(unsigned int line = 0 ; line < Base::W_.size(); line++) { 72 | float e = 0; 73 | wf.read((char *)&e, sizeof(float)); 74 | W_[line] = e; 75 | } 76 | wf.close(); 77 | } 78 | 79 | size_t connection_size() const override { 80 | return in_size_; 81 | } 82 | 83 | size_t fan_in_size() const override { 84 | return dim_; 85 | } 86 | 87 | size_t fan_out_size() const override { 88 | return dim_; 89 | } 90 | 91 | const vec_t& forward_propagation(const vec_t& in, size_t index) override { 92 | vec_t &a = a_[index]; 93 | vec_t &out = output_[index]; 94 | 95 | for_i(parallelize_, channels_, [&](int ch) { 96 | for(unsigned int j = 0; j < dim_; j++) { 97 | unsigned int pos = ch*dim_ + j; 98 | a[pos] = gamma(ch) * (in[pos] - mean(ch)) * invstd(ch) + beta(ch); 99 | } 100 | }); 101 | 102 | for_i(parallelize_, out_size_, [&](int i) { 103 | out[i] = h_.f(a, i); 104 | }); 105 | CNN_LOG_VECTOR(out, "[bn]forward"); 106 | 107 | return next_ ? next_->forward_propagation(out, index) : out; 108 | } 109 | 110 | const vec_t& back_propagation(const vec_t& curr_delta, size_t index) override { 111 | throw "Not yet implemented"; 112 | return curr_delta; 113 | } 114 | 115 | const vec_t& back_propagation_2nd(const vec_t& current_delta2) override { 116 | throw "Not yet implemented"; 117 | return current_delta2; 118 | } 119 | 120 | std::string layer_type() const override { return "batchnorm"; } 121 | 122 | protected: 123 | unsigned int dim_; 124 | unsigned int channels_; 125 | inline float_t beta(unsigned int ind) { 126 | return W_[(channels_ * 0) + ind]; 127 | } 128 | 129 | inline float_t gamma(unsigned int ind) { 130 | return W_[(channels_ * 1) + ind]; 131 | } 132 | 133 | inline float_t mean(unsigned int ind) { 134 | return W_[(channels_ * 2) + ind]; 135 | } 136 | 137 | inline float_t invstd(unsigned int ind) { 138 | return W_[(channels_ * 3) + ind]; 139 | } 140 | 141 | }; 142 | 143 | } // namespace tiny_cnn 144 | -------------------------------------------------------------------------------- /tiny_cnn/layers/bnn_fc_layer.h: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | * Copyright (c) 2016, Xilinx, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * 3. Neither the name of the copyright holder nor the names of its 16 | * contributors may be used to endorse or promote products derived from 17 | * this software without specific prior written permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 23 | * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 | * OR BUSINESS INTERRUPTION). HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 | * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 29 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | * 31 | *****************************************************************************/ 32 | #pragma once 33 | #include "tiny_cnn/layers/layer.h" 34 | #include "tiny_cnn/util/product.h" 35 | #include 36 | #include 37 | #include 38 | 39 | namespace tiny_cnn { 40 | 41 | template 42 | class bnn_fc_layer : public layer { 43 | public: 44 | typedef layer Base; 45 | CNN_USE_LAYER_MEMBERS; 46 | 47 | bnn_fc_layer(cnn_size_t in_dim, cnn_size_t out_dim, 48 | bool usePopcount = false, bool rowMajorWeights = false, std::string binaryParamFile = "") 49 | : Base(in_dim, out_dim, size_t(in_dim) * out_dim, 0), Wbin_(in_dim*out_dim, false), 50 | usePopcount_(usePopcount), rowMajorWeights_(rowMajorWeights) { 51 | if(binaryParamFile != "") 52 | loadFromBinaryFile(binaryParamFile); 53 | } 54 | 55 | void loadFromBinaryFile(std::string fileName) { 56 | // TODO this assumes the binary file always uses 8 bytes per threshold entry 57 | 58 | // load weights 59 | std::ifstream wf(fileName, std::ios::binary | std::ios::in); 60 | if(!wf.is_open()) 61 | throw "Could not open file"; 62 | for(unsigned int line = 0 ; line < Wbin_.size(); line++) { 63 | unsigned long long e = 0; 64 | wf.read((char *)&e, sizeof(unsigned long long)); 65 | Wbin_[line] = e == 1 ? true : false; 66 | } 67 | wf.close(); 68 | } 69 | 70 | size_t connection_size() const override { 71 | return size_t(in_size_) * out_size_; 72 | } 73 | 74 | size_t fan_in_size() const override { 75 | return in_size_; 76 | } 77 | 78 | size_t fan_out_size() const override { 79 | return out_size_; 80 | } 81 | 82 | virtual void post_update() { 83 | // once the weights have been updated, update the binarized versions too 84 | float2bipolar(W_, Wbin_); 85 | } 86 | 87 | const vec_t& forward_propagation(const vec_t& in, size_t index) override { 88 | std::vector in_bin(in_size_, false); 89 | // explicitly binarize the input 90 | float2bipolar(in, in_bin); 91 | vec_t &a = a_[index]; 92 | vec_t &out = output_[index]; 93 | 94 | for_i(parallelize_, out_size_, [&](int i) { 95 | a[i] = float_t(0); 96 | for (cnn_size_t c = 0; c < in_size_; c++) { 97 | // multiplication for binarized values is basically XNOR (equals) 98 | // i.e. if two values have the same sign (pos-pos or neg-neg) 99 | // the mul. result will be positive, otherwise negative 100 | // when using the popcount mode, consider positive results only 101 | const unsigned int wInd = rowMajorWeights_ ? i*in_size_+c : c*out_size_+i; 102 | if(usePopcount_) 103 | a[i] += (Wbin_[wInd] == in_bin[c]) ? +1 : 0; 104 | else 105 | a[i] += (Wbin_[wInd] == in_bin[c]) ? +1 : -1; 106 | } 107 | }); 108 | 109 | for_i(parallelize_, out_size_, [&](int i) { 110 | out[i] = h_.f(a, i); 111 | }); 112 | CNN_LOG_VECTOR(out, "[bfc]forward"); 113 | 114 | return next_ ? next_->forward_propagation(out, index) : out; 115 | } 116 | 117 | const vec_t& back_propagation(const vec_t& curr_delta, size_t index) override { 118 | throw "Not yet implemented"; 119 | return curr_delta; 120 | } 121 | 122 | const vec_t& back_propagation_2nd(const vec_t& current_delta2) override { 123 | throw "Not yet implemented"; 124 | return current_delta2; 125 | } 126 | 127 | std::string layer_type() const override { return "bnn_fc_layer"; } 128 | 129 | protected: 130 | std::vector Wbin_; 131 | bool usePopcount_, rowMajorWeights_; 132 | 133 | // utility function to convert a vector of floats into a vector of bools, where the 134 | // output boolean represents the sign of the input value (false: negative, 135 | // true: positive) 136 | void float2bipolar(const vec_t & in, std::vector & out) { 137 | for(unsigned int i = 0; i < in.size(); i++) 138 | out[i] = in[i] >= 0 ? true : false; 139 | } 140 | 141 | }; 142 | 143 | } // namespace tiny_cnn 144 | -------------------------------------------------------------------------------- /tiny_cnn/layers/bnn_output_layer.h: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | * Copyright (c) 2016, Xilinx, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * 3. Neither the name of the copyright holder nor the names of its 16 | * contributors may be used to endorse or promote products derived from 17 | * this software without specific prior written permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 23 | * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 | * OR BUSINESS INTERRUPTION). HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 | * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 29 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | * 31 | *****************************************************************************/ 32 | #pragma once 33 | #include "tiny_cnn/layers/bnn_threshold_layer.h" 34 | 35 | namespace tiny_cnn { 36 | class bnn_output_layer : public bnn_threshold_layer { 37 | public: 38 | using bnn_threshold_layer::bnn_threshold_layer; 39 | 40 | const vec_t& forward_propagation(const vec_t& in, size_t index) override { 41 | vec_t &out = output_[index]; 42 | 43 | for(unsigned int ch = 0; ch < channels_; ch++) { 44 | for(unsigned int j = 0; j < dim_; j++) { 45 | unsigned int pos = ch*dim_ + j; 46 | out[pos] = in[pos] - thresholds_[ch]; 47 | 48 | if(invertOutput_[ch]) 49 | out[pos] = -out[pos]; 50 | } 51 | } 52 | 53 | return next_ ? next_->forward_propagation(out, index) : out; 54 | } 55 | }; 56 | } 57 | 58 | -------------------------------------------------------------------------------- /tiny_cnn/layers/bnn_threshold_layer.h: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | * Copyright (c) 2016, Xilinx, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * 3. Neither the name of the copyright holder nor the names of its 16 | * contributors may be used to endorse or promote products derived from 17 | * this software without specific prior written permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 23 | * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 | * OR BUSINESS INTERRUPTION). HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 | * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 29 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | * 31 | *****************************************************************************/ 32 | #pragma once 33 | #include "tiny_cnn/layers/layer.h" 34 | #include "tiny_cnn/activations/activation_function.h" 35 | #include "tiny_cnn/util/util.h" 36 | #include 37 | #include 38 | #include 39 | 40 | namespace tiny_cnn { 41 | class bnn_threshold_layer : public layer { 42 | public: 43 | typedef layer Base; 44 | 45 | // channels: number of channels. each channel has a separate threshold 46 | // dim: number of pixels/elements in each channel. 47 | bnn_threshold_layer(cnn_size_t channels, cnn_size_t dim = 1, std::string binaryParamFile = "") 48 | : Base(dim*channels, dim*channels, 0, 0), dim_(dim), channels_(channels), 49 | thresholds_(channels, 0), invertOutput_(channels, false) 50 | { 51 | // TODO re-enable parallelization -- need to support worker index in forward prop 52 | set_parallelize(false); 53 | if(binaryParamFile != "") 54 | loadFromBinaryFile(binaryParamFile); 55 | } 56 | 57 | void loadFromBinaryFile(std::string fileName) { 58 | // does not support setting invertOutput but should not be necessary anyway -- 59 | // bin weight files are generated by Python script that flips the weights when 60 | // inverted output is needed 61 | // TODO this assumes the binary file always uses 8 bytes per threshold entry 62 | 63 | // load thresholds 64 | std::ifstream tf(fileName, std::ios::binary | std::ios::in); 65 | if(!tf.is_open()) 66 | throw "Could not open file"; 67 | for(unsigned int line = 0 ; line < channels_; line++) { 68 | unsigned long long e = 0; 69 | tf.read((char *)&e, sizeof(unsigned long long)); 70 | thresholds_[line] = e; 71 | } 72 | tf.close(); 73 | } 74 | 75 | std::vector & thresholds() { 76 | return thresholds_; 77 | } 78 | 79 | std::vector & invertOutput() { 80 | return invertOutput_; 81 | } 82 | 83 | size_t connection_size() const override { 84 | return in_size_; 85 | } 86 | 87 | size_t fan_in_size() const override { 88 | return dim_; 89 | } 90 | 91 | size_t fan_out_size() const override { 92 | return dim_; 93 | } 94 | 95 | const vec_t& forward_propagation(const vec_t& in, size_t index) override { 96 | vec_t &out = output_[index]; 97 | 98 | for(unsigned int ch = 0; ch < channels_; ch++) { 99 | for(unsigned int j = 0; j < dim_; j++) { 100 | unsigned int pos = ch*dim_ + j; 101 | out[pos] = (in[pos] > thresholds_[ch] ? +1 : -1); 102 | 103 | if(invertOutput_[ch]) 104 | out[pos] = -out[pos]; 105 | } 106 | } 107 | 108 | return next_ ? next_->forward_propagation(out, index) : out; 109 | } 110 | 111 | const vec_t& back_propagation(const vec_t& curr_delta, size_t index) override { 112 | throw "Not yet implemented"; 113 | return curr_delta; 114 | } 115 | 116 | const vec_t& back_propagation_2nd(const vec_t& current_delta2) override { 117 | throw "Not yet implemented"; 118 | return current_delta2; 119 | } 120 | 121 | std::string layer_type() const override { return "bnn_threshold_layer"; } 122 | 123 | protected: 124 | unsigned int dim_; 125 | unsigned int channels_; 126 | 127 | 128 | std::vector thresholds_; 129 | std::vector invertOutput_; 130 | }; 131 | 132 | } // namespace tiny_cnn 133 | -------------------------------------------------------------------------------- /tiny_cnn/layers/chaninterleave_layer.h: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | * Copyright (c) 2016, Xilinx, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * 3. Neither the name of the copyright holder nor the names of its 16 | * contributors may be used to endorse or promote products derived from 17 | * this software without specific prior written permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 23 | * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 | * OR BUSINESS INTERRUPTION). HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 | * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 29 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | * 31 | *****************************************************************************/ 32 | #pragma once 33 | #include "tiny_cnn/util/util.h" 34 | 35 | // each channel's feature maps are either contiguous or interleaved, 36 | // and this layer switches between those two modes. does not touch 37 | // row major / column major order of the pixels. 38 | 39 | namespace tiny_cnn { 40 | 41 | 42 | template 43 | class chaninterleave_layer : public layer { 44 | public: 45 | CNN_USE_LAYER_MEMBERS; 46 | 47 | typedef layer Base; 48 | 49 | explicit chaninterleave_layer(cnn_size_t channels, cnn_size_t pixelsPerChan, bool deinterleave) 50 | : Base(channels*pixelsPerChan, channels*pixelsPerChan, 0, 0), channels_(channels), 51 | pixelsPerChan_(pixelsPerChan), deinterleave_(deinterleave) 52 | { 53 | 54 | } 55 | 56 | size_t param_size() const override { 57 | return 0; 58 | } 59 | 60 | size_t connection_size() const override { 61 | return this->in_size(); 62 | } 63 | 64 | size_t fan_in_size() const override { 65 | return 1; 66 | } 67 | 68 | size_t fan_out_size() const override { 69 | return 1; 70 | } 71 | 72 | std::string layer_type() const override { return "chaninterleave_layer"; } 73 | 74 | const vec_t& forward_propagation(const vec_t& in, size_t index) override { 75 | vec_t& a = a_[index]; 76 | vec_t& out = output_[index]; 77 | 78 | for(unsigned int c = 0; c < channels_; c++) { 79 | for(unsigned int pix = 0; pix < pixelsPerChan_; pix++) { 80 | if(deinterleave_) { 81 | out[c * pixelsPerChan_ + pix] = in[pix*channels_ + c]; 82 | } else { 83 | out[pix*channels_ + c] = in[c * pixelsPerChan_ + pix]; 84 | } 85 | } 86 | } 87 | 88 | return next_ ? next_->forward_propagation(out, index) : out; 89 | } 90 | 91 | virtual const vec_t& back_propagation(const vec_t& current_delta, size_t index) override { 92 | throw "Not implemented"; 93 | 94 | return current_delta; 95 | } 96 | 97 | const vec_t& back_propagation_2nd(const vec_t& current_delta2) override { 98 | throw "Not implemented"; 99 | 100 | return current_delta2; 101 | } 102 | 103 | protected: 104 | cnn_size_t channels_, pixelsPerChan_; 105 | bool deinterleave_; 106 | }; 107 | 108 | } // namespace tiny_cnn 109 | -------------------------------------------------------------------------------- /tiny_cnn/layers/dropout_layer.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "tiny_cnn/util/util.h" 29 | #include "tiny_cnn/layers/layer.h" 30 | #include 31 | 32 | namespace tiny_cnn { 33 | 34 | // normal 35 | class dropout_layer : public layer { 36 | public: 37 | typedef activation::identity Activation; 38 | CNN_USE_LAYER_MEMBERS; 39 | 40 | dropout_layer(cnn_size_t in_dim, float_t dropout_rate, net_phase phase = net_phase::train) 41 | : layer(in_dim, in_dim, 0, 0), 42 | phase_(phase), 43 | dropout_rate_(dropout_rate), 44 | scale_(float_t(1) / (float_t(1) - dropout_rate_)) 45 | { 46 | mask_ = new bool[in_size_ * CNN_TASK_SIZE]; 47 | std::fill(mask_, mask_ + (in_size_ * CNN_TASK_SIZE), false); 48 | } 49 | 50 | dropout_layer(const dropout_layer& obj) 51 | : layer(obj.in_size_, obj.in_size_, 0, 0), 52 | phase_(obj.phase_), 53 | dropout_rate_(obj.dropout_rate_), 54 | scale_(float_t(1) / (float_t(1) - dropout_rate_)) 55 | { 56 | mask_ = new bool[in_size_ * CNN_TASK_SIZE]; 57 | std::copy(obj.mask_, (obj.mask_ + (in_size_ * CNN_TASK_SIZE)), mask_); 58 | } 59 | 60 | dropout_layer(dropout_layer&& obj) 61 | : layer(obj.in_size_, obj.in_size_, 0, 0), 62 | phase_(obj.phase_), 63 | dropout_rate_(obj.dropout_rate_), 64 | scale_(float_t(1) / (float_t(1) - dropout_rate_)), 65 | mask_(obj.mask_) 66 | { 67 | obj.mask_ = nullptr; 68 | } 69 | 70 | virtual ~dropout_layer() 71 | { 72 | delete[] mask_; 73 | } 74 | 75 | dropout_layer& operator=(const dropout_layer& obj) 76 | { 77 | delete[] mask_; 78 | 79 | layer::operator=(obj); 80 | phase_ = obj.phase_; 81 | dropout_rate_ = obj.dropout_rate_; 82 | scale_ = obj.scale_; 83 | mask_ = new bool[in_size_ * CNN_TASK_SIZE]; 84 | std::copy(obj.mask_, (obj.mask_ + (obj.in_size_ * CNN_TASK_SIZE)), mask_); 85 | return *this; 86 | } 87 | 88 | dropout_layer& operator=(dropout_layer&& obj) 89 | { 90 | layer::operator=(obj); 91 | phase_ = obj.phase_; 92 | dropout_rate_ = obj.dropout_rate_; 93 | scale_ = obj.scale_; 94 | std::swap(mask_, obj.mask_); 95 | return *this; 96 | } 97 | 98 | void set_dropout_rate(float_t rate) 99 | { 100 | dropout_rate_ = rate; 101 | scale_ = float_t(1) / (float_t(1) - dropout_rate_); 102 | } 103 | 104 | ///< number of incoming connections for each output unit 105 | size_t fan_in_size() const override 106 | { 107 | return 1; 108 | } 109 | 110 | ///< number of outgoing connections for each input unit 111 | size_t fan_out_size() const override 112 | { 113 | return 1; 114 | } 115 | 116 | ///< number of connections 117 | size_t connection_size() const override 118 | { 119 | return in_size(); 120 | } 121 | 122 | const vec_t& back_propagation_2nd(const vec_t& in_raw) override 123 | { 124 | prev_delta2_ = in_raw; 125 | return prev_->back_propagation_2nd(prev_delta2_); 126 | } 127 | 128 | const vec_t& back_propagation(const vec_t& current_delta, size_t worker_index) override 129 | { 130 | vec_t& prev_delta = prev_delta_[worker_index]; 131 | bool* mask = &mask_[worker_index * CNN_TASK_SIZE]; 132 | 133 | for (size_t i = 0; i < current_delta.size(); i++) { 134 | prev_delta[i] = mask[i] * current_delta[i]; 135 | } 136 | return prev_->back_propagation(prev_delta, worker_index); 137 | } 138 | 139 | const vec_t& forward_propagation(const vec_t& in, size_t worker_index) override 140 | { 141 | vec_t& out = output_[worker_index]; 142 | vec_t& a = a_[worker_index]; 143 | bool* mask = &mask_[worker_index * CNN_TASK_SIZE]; 144 | 145 | if (phase_ == net_phase::train) { 146 | for (size_t i = 0; i < in.size(); i++) 147 | mask[i] = bernoulli(dropout_rate_); 148 | 149 | for (size_t i = 0; i < in.size(); i++) 150 | a[i] = out[i] = mask[i] * scale_ * in[i]; 151 | } 152 | else { 153 | for (size_t i = 0; i < in.size(); i++) 154 | a[i] = out[i] = in[i]; 155 | } 156 | return next_ ? next_->forward_propagation(out, worker_index) : out; 157 | } 158 | 159 | /** 160 | * set dropout-context (training-phase or test-phase) 161 | **/ 162 | void set_context(net_phase ctx) override 163 | { 164 | phase_ = ctx; 165 | } 166 | 167 | std::string layer_type() const override { return "dropout"; } 168 | 169 | const bool* get_mask() const { return mask_; } 170 | 171 | private: 172 | net_phase phase_; 173 | float_t dropout_rate_; 174 | float_t scale_; 175 | bool *mask_; 176 | }; 177 | 178 | } // namespace tiny_cnn 179 | -------------------------------------------------------------------------------- /tiny_cnn/layers/fully_connected_layer.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "tiny_cnn/layers/layer.h" 29 | #include "tiny_cnn/util/product.h" 30 | 31 | namespace tiny_cnn { 32 | 33 | template 34 | class fully_connected_layer : public layer { 35 | public: 36 | typedef layer Base; 37 | CNN_USE_LAYER_MEMBERS; 38 | 39 | fully_connected_layer(cnn_size_t in_dim, cnn_size_t out_dim, bool has_bias = true, std::string binaryParamFile = "") 40 | : Base(in_dim, out_dim, size_t(in_dim) * out_dim, has_bias ? out_dim : 0), has_bias_(has_bias) { 41 | if(binaryParamFile != "") { 42 | loadFromBinaryFile(binaryParamFile); 43 | } 44 | } 45 | 46 | void loadFromBinaryFile(std::string fileName) { 47 | // TODO this assumes the binary file always uses a float for each parameter 48 | 49 | std::ifstream wf(fileName, std::ios::binary | std::ios::in); 50 | if(!wf.is_open()) 51 | throw "Could not open file"; 52 | for(unsigned int line = 0 ; line < Base::W_.size(); line++) { 53 | float e = 0; 54 | wf.read((char *)&e, sizeof(float)); 55 | W_[line] = e; 56 | } 57 | if(has_bias_) { 58 | for(unsigned int line = 0 ; line < Base::b_.size(); line++) { 59 | float e = 0; 60 | wf.read((char *)&e, sizeof(float)); 61 | b_[line] = e; 62 | } 63 | } 64 | wf.close(); 65 | } 66 | 67 | size_t connection_size() const override { 68 | return size_t(in_size_) * out_size_ + size_t(has_bias_) * out_size_; 69 | } 70 | 71 | size_t fan_in_size() const override { 72 | return in_size_; 73 | } 74 | 75 | size_t fan_out_size() const override { 76 | return out_size_; 77 | } 78 | 79 | const vec_t& forward_propagation(const vec_t& in, size_t index) override { 80 | vec_t &a = a_[index]; 81 | vec_t &out = output_[index]; 82 | 83 | for_i(parallelize_, out_size_, [&](int i) { 84 | a[i] = float_t(0); 85 | for (cnn_size_t c = 0; c < in_size_; c++) { 86 | a[i] += W_[c*out_size_ + i] * in[c]; 87 | } 88 | 89 | if (has_bias_) 90 | a[i] += b_[i]; 91 | }); 92 | 93 | for_i(parallelize_, out_size_, [&](int i) { 94 | out[i] = h_.f(a, i); 95 | }); 96 | CNN_LOG_VECTOR(out, "[fc]forward"); 97 | 98 | return next_ ? next_->forward_propagation(out, index) : out; 99 | } 100 | 101 | const vec_t& back_propagation(const vec_t& curr_delta, size_t index) override { 102 | const vec_t& prev_out = prev_->output(static_cast(index)); 103 | const activation::function& prev_h = prev_->activation_function(); 104 | vec_t& prev_delta = prev_delta_[index]; 105 | vec_t& dW = dW_[index]; 106 | vec_t& db = db_[index]; 107 | 108 | for (cnn_size_t c = 0; c < this->in_size_; c++) { 109 | // propagate delta to previous layer 110 | // prev_delta[c] += current_delta[r] * W_[c * out_size_ + r] 111 | prev_delta[c] = vectorize::dot(&curr_delta[0], &W_[c*out_size_], out_size_); 112 | prev_delta[c] *= prev_h.df(prev_out[c]); 113 | } 114 | 115 | for_(parallelize_, 0, size_t(out_size_), [&](const blocked_range& r) { 116 | // accumulate weight-step using delta 117 | // dW[c * out_size + i] += current_delta[i] * prev_out[c] 118 | for (cnn_size_t c = 0; c < in_size_; c++) 119 | vectorize::muladd(&curr_delta[r.begin()], prev_out[c], r.end() - r.begin(), &dW[c*out_size_ + r.begin()]); 120 | 121 | if (has_bias_) { 122 | for (int i = r.begin(); i < r.end(); i++) 123 | db[i] += curr_delta[i]; 124 | } 125 | }); 126 | 127 | CNN_LOG_VECTOR(curr_delta, "[fc]curr_delta"); 128 | CNN_LOG_VECTOR(prev_delta, "[fc]prev_delta"); 129 | CNN_LOG_VECTOR(dW, "[fc]dW"); 130 | CNN_LOG_VECTOR(db, "[fc]db"); 131 | 132 | return prev_->back_propagation(prev_delta_[index], index); 133 | } 134 | 135 | const vec_t& back_propagation_2nd(const vec_t& current_delta2) override { 136 | const vec_t& prev_out = prev_->output(0); 137 | const activation::function& prev_h = prev_->activation_function(); 138 | 139 | for (cnn_size_t c = 0; c < in_size_; c++) 140 | for (cnn_size_t r = 0; r < out_size_; r++) 141 | Whessian_[c*out_size_ + r] += current_delta2[r] * sqr(prev_out[c]); 142 | 143 | if (has_bias_) { 144 | for (cnn_size_t r = 0; r < out_size_; r++) 145 | bhessian_[r] += current_delta2[r]; 146 | } 147 | 148 | for (cnn_size_t c = 0; c < in_size_; c++) { 149 | prev_delta2_[c] = float_t(0); 150 | 151 | for (cnn_size_t r = 0; r < out_size_; r++) 152 | prev_delta2_[c] += current_delta2[r] * sqr(W_[c*out_size_ + r]); 153 | 154 | prev_delta2_[c] *= sqr(prev_h.df(prev_out[c])); 155 | } 156 | CNN_LOG_VECTOR(current_delta2, "[fc]curr-delta2"); 157 | CNN_LOG_VECTOR(prev_delta2_, "[fc]prev-delta2"); 158 | 159 | return prev_->back_propagation_2nd(prev_delta2_); 160 | } 161 | 162 | std::string layer_type() const override { return "fully-connected"; } 163 | 164 | protected: 165 | bool has_bias_; 166 | }; 167 | 168 | } // namespace tiny_cnn 169 | -------------------------------------------------------------------------------- /tiny_cnn/layers/input_layer.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "tiny_cnn/layers/layer.h" 29 | 30 | namespace tiny_cnn { 31 | 32 | class input_layer : public layer { 33 | public: 34 | typedef activation::identity Activation; 35 | typedef layer Base; 36 | CNN_USE_LAYER_MEMBERS; 37 | 38 | input_layer() : Base(0, 0, 0, 0) {} 39 | 40 | cnn_size_t in_size() const override { return next_ ? next_->in_size(): static_cast(0); } 41 | 42 | index3d in_shape() const override { return next_ ? next_->in_shape() : index3d(0, 0, 0); } 43 | index3d out_shape() const override { return next_ ? next_->out_shape() : index3d(0, 0, 0); } 44 | std::string layer_type() const override { return next_ ? next_->layer_type() : "input"; } 45 | 46 | const vec_t& forward_propagation(const vec_t& in, size_t index) override { 47 | output_[index] = in; 48 | return next_ ? next_->forward_propagation(in, index) : output_[index]; 49 | } 50 | 51 | const vec_t& back_propagation(const vec_t& current_delta, size_t /*index*/) override { 52 | return current_delta; 53 | } 54 | 55 | const vec_t& back_propagation_2nd(const vec_t& current_delta2) override { 56 | return current_delta2; 57 | } 58 | 59 | size_t connection_size() const override { 60 | return in_size_; 61 | } 62 | 63 | size_t fan_in_size() const override { 64 | return 1; 65 | } 66 | 67 | size_t fan_out_size() const override { 68 | return 1; 69 | } 70 | }; 71 | 72 | } // namespace tiny_cnn 73 | -------------------------------------------------------------------------------- /tiny_cnn/layers/layers.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "tiny_cnn/layers/layer.h" 29 | #include "input_layer.h" 30 | 31 | namespace tiny_cnn { 32 | 33 | class layers { 34 | public: 35 | layers() { add(std::make_shared()); } 36 | 37 | layers(const layers& rhs) { construct(rhs); } 38 | 39 | layers& operator = (const layers& rhs) { 40 | layers_.clear(); 41 | construct(rhs); 42 | return *this; 43 | } 44 | 45 | void add(std::shared_ptr new_tail) { 46 | if (tail()) tail()->connect(new_tail); 47 | layers_.push_back(new_tail); 48 | } 49 | 50 | bool empty() const { return layers_.size() == 0; } 51 | 52 | layer_base* head() const { return empty() ? 0 : layers_[0].get(); } 53 | 54 | layer_base* tail() const { return empty() ? 0 : layers_[layers_.size() - 1].get(); } 55 | 56 | template 57 | const T& at(size_t index) const { 58 | const T* v = dynamic_cast(layers_[index + 1].get()); 59 | if (v) return *v; 60 | throw nn_error("failed to cast"); 61 | } 62 | 63 | const layer_base* operator [] (size_t index) const { 64 | return layers_[index + 1].get(); 65 | } 66 | 67 | layer_base* operator [] (size_t index) { 68 | return layers_[index + 1].get(); 69 | } 70 | 71 | void init_weight() { 72 | for (auto pl : layers_) 73 | pl->init_weight(); 74 | } 75 | 76 | bool is_exploded() const { 77 | for (auto pl : layers_) 78 | if (pl->is_exploded()) return true; 79 | return false; 80 | } 81 | 82 | void divide_hessian(int denominator) { 83 | for (auto pl : layers_) 84 | pl->divide_hessian(denominator); 85 | } 86 | 87 | template 88 | void update_weights(Optimizer *o, size_t worker_size, size_t batch_size) { 89 | for (auto pl : layers_) 90 | pl->update_weight(o, static_cast(worker_size), batch_size); 91 | } 92 | 93 | void set_parallelize(bool parallelize) { 94 | for (auto pl : layers_) 95 | pl->set_parallelize(parallelize); 96 | } 97 | 98 | // get depth(number of layers) of networks 99 | size_t depth() const { 100 | return layers_.size() - 1; // except input-layer 101 | } 102 | 103 | private: 104 | void construct(const layers& rhs) { 105 | add(std::make_shared()); 106 | for (size_t i = 1; i < rhs.layers_.size(); i++) 107 | add(rhs.layers_[i]); 108 | } 109 | 110 | std::vector> layers_; 111 | }; 112 | 113 | } // namespace tiny_cnn 114 | -------------------------------------------------------------------------------- /tiny_cnn/layers/linear_layer.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "tiny_cnn/util/util.h" 29 | #include 30 | 31 | extern bool g_log_softmax; 32 | 33 | 34 | namespace tiny_cnn { 35 | 36 | /** 37 | * f(x) = h(scale*x+bias) 38 | */ 39 | template 40 | class linear_layer : public layer { 41 | public: 42 | CNN_USE_LAYER_MEMBERS; 43 | 44 | typedef layer Base; 45 | 46 | explicit linear_layer(cnn_size_t dim, float_t scale = float_t(1), float_t bias = float_t(0)) 47 | : Base(dim, dim, 0, 0), 48 | scale_(scale), bias_(bias) {} 49 | 50 | size_t param_size() const override { 51 | return 0; 52 | } 53 | 54 | size_t connection_size() const override { 55 | return this->in_size(); 56 | } 57 | 58 | size_t fan_in_size() const override { 59 | return 1; 60 | } 61 | 62 | size_t fan_out_size() const override { 63 | return 1; 64 | } 65 | 66 | std::string layer_type() const override { return "linear"; } 67 | 68 | const vec_t& forward_propagation(const vec_t& in, size_t index) override { 69 | vec_t& a = a_[index]; 70 | vec_t& out = output_[index]; 71 | 72 | for_i(parallelize_, out_size_, [&](int i) { 73 | a[i] = scale_ * in[i] + bias_; 74 | }); 75 | for_i(parallelize_, out_size_, [&](int i) { 76 | out[i] = h_.f(a, i); 77 | }); 78 | 79 | return next_ ? next_->forward_propagation(out, index) : out; 80 | } 81 | 82 | virtual const vec_t& back_propagation(const vec_t& current_delta, size_t index) override { 83 | const vec_t& prev_out = prev_->output(index); 84 | const activation::function& prev_h = prev_->activation_function(); 85 | vec_t& prev_delta = prev_delta_[index]; 86 | 87 | for_i(parallelize_, out_size_, [&](int i) { 88 | prev_delta[i] = current_delta[i] * scale_ * prev_h.df(prev_out[i]); 89 | }); 90 | 91 | return prev_->back_propagation(prev_delta_[index], index); 92 | } 93 | 94 | const vec_t& back_propagation_2nd(const vec_t& current_delta2) override { 95 | const vec_t& prev_out = prev_->output(0); 96 | const activation::function& prev_h = prev_->activation_function(); 97 | 98 | for_i(parallelize_, out_size_, [&](int i) { 99 | prev_delta2_[i] = current_delta2[i] * sqr(scale_ * prev_h.df(prev_out[i])); 100 | }); 101 | 102 | return prev_->back_propagation_2nd(prev_delta2_); 103 | } 104 | 105 | protected: 106 | float_t scale_, bias_; 107 | }; 108 | 109 | } // namespace tiny_cnn 110 | -------------------------------------------------------------------------------- /tiny_cnn/layers/lrn_layer.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "tiny_cnn/util/util.h" 29 | #include 30 | 31 | namespace tiny_cnn { 32 | 33 | enum class norm_region { 34 | across_channels, 35 | within_channels 36 | }; 37 | 38 | /** 39 | * local response normalization 40 | */ 41 | template 42 | class lrn_layer : public layer { 43 | public: 44 | CNN_USE_LAYER_MEMBERS; 45 | 46 | typedef layer Base; 47 | 48 | lrn_layer(cnn_size_t in_width, cnn_size_t in_height, cnn_size_t local_size, cnn_size_t in_channels, 49 | float_t alpha, float_t beta, norm_region region = norm_region::across_channels) 50 | : Base(in_width*in_height*in_channels, in_width*in_height*in_channels, 0, 0), 51 | in_shape_(in_width, in_height, in_channels), size_(local_size), alpha_(alpha), beta_(beta), region_(region), in_square_(in_shape_.area()) {} 52 | 53 | size_t param_size() const override { 54 | return 0; 55 | } 56 | 57 | size_t connection_size() const override { 58 | return this->in_size() * size_; 59 | } 60 | 61 | size_t fan_in_size() const override { 62 | return size_; 63 | } 64 | 65 | size_t fan_out_size() const override { 66 | return size_; 67 | } 68 | 69 | std::string layer_type() const override { return "norm"; } 70 | 71 | const vec_t& forward_propagation(const vec_t& in, size_t index) override { 72 | vec_t& a = a_[index]; 73 | vec_t& out = output_[index]; 74 | 75 | if (region_ == norm_region::across_channels) { 76 | forward_across(in, a); 77 | } 78 | else { 79 | forward_within(in, a); 80 | } 81 | 82 | for_i(parallelize_, out_size_, [&](int i) { 83 | out[i] = h_.f(a, i); 84 | }); 85 | return next_ ? next_->forward_propagation(out, index) : out; 86 | } 87 | 88 | virtual const vec_t& back_propagation(const vec_t& current_delta, size_t index) override { 89 | CNN_UNREFERENCED_PARAMETER(current_delta); 90 | CNN_UNREFERENCED_PARAMETER(index); 91 | throw nn_error("not implemented"); 92 | } 93 | 94 | const vec_t& back_propagation_2nd(const vec_t& current_delta2) override { 95 | CNN_UNREFERENCED_PARAMETER(current_delta2); 96 | throw nn_error("not implemented"); 97 | } 98 | 99 | private: 100 | void forward_across(const vec_t& in, vec_t& out) { 101 | std::fill(in_square_.begin(), in_square_.end(), float_t(0)); 102 | 103 | for (cnn_size_t i = 0; i < size_ / 2; i++) { 104 | cnn_size_t idx = in_shape_.get_index(0, 0, i); 105 | add_square_sum(&in[idx], in_shape_.area(), &in_square_[0]); 106 | } 107 | 108 | cnn_size_t head = size_ / 2; 109 | long tail = ((long) head) - size_; 110 | cnn_size_t channels = in_shape_.depth_; 111 | const cnn_size_t wxh = in_shape_.area(); 112 | const float_t alpha_div_size = alpha_ / size_; 113 | 114 | for (cnn_size_t i = 0; i < channels; i++, head++, tail++) { 115 | if (head < channels) 116 | add_square_sum(&in[in_shape_.get_index(0, 0, head)], wxh, &in_square_[0]); 117 | 118 | if (tail >= 0) 119 | sub_square_sum(&in[in_shape_.get_index(0, 0, tail)], wxh, &in_square_[0]); 120 | 121 | float_t *dst = &out[in_shape_.get_index(0, 0, i)]; 122 | const float_t *src = &in[in_shape_.get_index(0, 0, i)]; 123 | for (cnn_size_t j = 0; j < wxh; j++) 124 | dst[j] = src[j] * std::pow(float_t(1) + alpha_div_size * in_square_[j], -beta_); 125 | } 126 | } 127 | 128 | void forward_within(const vec_t& in, vec_t& out) { 129 | CNN_UNREFERENCED_PARAMETER(in); 130 | CNN_UNREFERENCED_PARAMETER(out); 131 | throw nn_error("not implemented"); 132 | } 133 | 134 | void add_square_sum(const float_t *src, cnn_size_t size, float_t *dst) { 135 | for (cnn_size_t i = 0; i < size; i++) 136 | dst[i] += src[i] * src[i]; 137 | } 138 | 139 | void sub_square_sum(const float_t *src, cnn_size_t size, float_t *dst) { 140 | for (cnn_size_t i = 0; i < size; i++) 141 | dst[i] -= src[i] * src[i]; 142 | } 143 | 144 | layer_shape_t in_shape_; 145 | 146 | cnn_size_t size_; 147 | float_t alpha_, beta_; 148 | norm_region region_; 149 | 150 | vec_t in_square_; 151 | }; 152 | 153 | } // namespace tiny_cnn 154 | -------------------------------------------------------------------------------- /tiny_cnn/layers/monitor_layer.h: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | * Copyright (c) 2016, Xilinx, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * 3. Neither the name of the copyright holder nor the names of its 16 | * contributors may be used to endorse or promote products derived from 17 | * this software without specific prior written permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 23 | * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 | * OR BUSINESS INTERRUPTION). HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 | * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 29 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | * 31 | *****************************************************************************/ 32 | #pragma once 33 | 34 | // monitor layer - print out contents passing through 35 | 36 | #include "tiny_cnn/activations/activation_function.h" 37 | #include "tiny_cnn/layers/layer.h" 38 | #include "tiny_cnn/util/product.h" 39 | #include "tiny_cnn/util/util.h" 40 | #include 41 | #include 42 | #include 43 | 44 | namespace tiny_cnn { 45 | 46 | class monitor_layer : public layer { 47 | public: 48 | typedef layer Base; 49 | 50 | monitor_layer(cnn_size_t dim, std::string monitorName) : 51 | Base(dim, dim, 0, 0), monitorName_(monitorName) 52 | { 53 | // disable parallelization for monitor layers 54 | Base::set_parallelize(false); 55 | } 56 | 57 | std::string layer_type() const override { return "monitor"; } 58 | 59 | size_t param_size() const override { 60 | return 0; 61 | } 62 | 63 | size_t connection_size() const override { 64 | return 0; 65 | } 66 | 67 | size_t fan_in_size() const override { 68 | return in_size_; 69 | } 70 | 71 | size_t fan_out_size() const override { 72 | return out_size_; 73 | } 74 | 75 | // forward prop does nothing except calling the 76 | const vec_t& forward_propagation(const vec_t& in, size_t index) override { 77 | vec_t & out = output_[index]; 78 | 79 | std::cout << monitorName_ << std::endl; 80 | 81 | for(unsigned int i = 0; i < in.size(); i++) { 82 | out[i] = in[i]; 83 | std::cout << i << " " << in[i] << std::endl; 84 | } 85 | 86 | return next_ ? next_->forward_propagation(out, index) : out; 87 | } 88 | 89 | // offloaded layer is feedforward only, does not support training 90 | const vec_t& back_propagation(const vec_t& curr_delta, size_t index) override { 91 | throw "Not implemented"; 92 | return curr_delta; 93 | } 94 | 95 | const vec_t& back_propagation_2nd(const vec_t& current_delta2) override { 96 | throw "Not implemented"; 97 | return current_delta2; 98 | } 99 | 100 | protected: 101 | std::string monitorName_; 102 | 103 | }; 104 | 105 | } // namespace tiny_cnn 106 | -------------------------------------------------------------------------------- /tiny_cnn/layers/offloaded_layer.h: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | * Copyright (c) 2016, Xilinx, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * 3. Neither the name of the copyright holder nor the names of its 16 | * contributors may be used to endorse or promote products derived from 17 | * this software without specific prior written permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 23 | * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 | * OR BUSINESS INTERRUPTION). HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 | * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 29 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | * 31 | *****************************************************************************/ 32 | #pragma once 33 | 34 | // offloaded_layer -- simply calls a hook function every time its forward pass is called 35 | 36 | #include "tiny_cnn/activations/activation_function.h" 37 | #include "tiny_cnn/layers/layer.h" 38 | #include "tiny_cnn/util/product.h" 39 | #include "tiny_cnn/util/util.h" 40 | #include 41 | 42 | 43 | 44 | namespace tiny_cnn { 45 | 46 | typedef struct { 47 | cnn_size_t in_width; // input feature map width 48 | cnn_size_t in_height; // input feature map height 49 | cnn_size_t window_size; // convolution kernel window size 50 | cnn_size_t in_channels; // # input feature maps 51 | cnn_size_t out_channels; // # output feature maps 52 | } OffloadConvParams; 53 | 54 | #ifdef SOLITAIRE 55 | // function type for offload handling. args are (input, output, offloadID, conv params if any or 0, target set of weigths) 56 | typedef void (*OffloadHandler)(const vec_t &, vec_t &, unsigned int, OffloadConvParams *, unsigned int); 57 | 58 | class offloaded_layer : public layer { 59 | public: 60 | typedef layer Base; 61 | 62 | offloaded_layer(cnn_size_t in_dim, cnn_size_t out_dim, OffloadHandler handler, 63 | unsigned int offloadID, OffloadConvParams * convParams = 0, unsigned int targetSet = 0) : 64 | Base(in_dim, out_dim, 0, 0), offloadHandler_(handler), offloadID_(offloadID), 65 | offloadConvParams_(convParams), targetSet_(targetSet) 66 | 67 | #else 68 | // function type for offload handling. args are (input, output, offloadID, conv params if any or 0) 69 | typedef void (*OffloadHandler)(const vec_t &, vec_t &, unsigned int, OffloadConvParams *); 70 | 71 | class offloaded_layer : public layer { 72 | public: 73 | typedef layer Base; 74 | 75 | offloaded_layer(cnn_size_t in_dim, cnn_size_t out_dim, OffloadHandler handler, 76 | unsigned int offloadID, OffloadConvParams * convParams = 0) : 77 | Base(in_dim, out_dim, 0, 0), offloadHandler_(handler), offloadID_(offloadID), 78 | offloadConvParams_(convParams) 79 | #endif 80 | { 81 | // disable parallelization for offloaded layers 82 | Base::set_parallelize(false); 83 | } 84 | 85 | std::string layer_type() const override { return "offloaded"; } 86 | 87 | size_t param_size() const override { 88 | return 0; 89 | } 90 | 91 | size_t connection_size() const override { 92 | return 0; 93 | } 94 | 95 | size_t fan_in_size() const override { 96 | return in_size_; 97 | } 98 | 99 | size_t fan_out_size() const override { 100 | return out_size_; 101 | } 102 | 103 | // forward prop does nothing except calling the 104 | const vec_t& forward_propagation(const vec_t& in, size_t index) override { 105 | vec_t & out = output_[index]; 106 | #ifdef SOLITAIRE 107 | offloadHandler_(in, out, offloadID_, offloadConvParams_, targetSet_); 108 | #else 109 | offloadHandler_(in, out, offloadID_, offloadConvParams_); 110 | #endif 111 | return next_ ? next_->forward_propagation(out, index) : out; 112 | } 113 | 114 | // offloaded layer is feedforward only, does not support training 115 | const vec_t& back_propagation(const vec_t& curr_delta, size_t index) override { 116 | throw "Not implemented"; 117 | return curr_delta; 118 | } 119 | 120 | const vec_t& back_propagation_2nd(const vec_t& current_delta2) override { 121 | throw "Not implemented"; 122 | return current_delta2; 123 | } 124 | 125 | protected: 126 | OffloadHandler offloadHandler_; 127 | OffloadConvParams * offloadConvParams_; 128 | unsigned int offloadID_; 129 | #ifdef SOLITAIRE 130 | unsigned int targetSet_; 131 | #endif 132 | }; 133 | 134 | } // namespace tiny_cnn 135 | 136 | -------------------------------------------------------------------------------- /tiny_cnn/lossfunctions/loss_function.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "tiny_cnn/util/util.h" 29 | 30 | namespace tiny_cnn { 31 | 32 | // mean-squared-error loss function for regression 33 | class mse { 34 | public: 35 | static float_t f(float_t y, float_t t) { 36 | return (y - t) * (y - t) / 2; 37 | } 38 | 39 | static float_t df(float_t y, float_t t) { 40 | return y - t; 41 | } 42 | }; 43 | 44 | // cross-entropy loss function for (multiple independent) binary classifications 45 | class cross_entropy { 46 | public: 47 | static float_t f(float_t y, float_t t) { 48 | return -t * std::log(y) - (float_t(1) - t) * std::log(float_t(1) - y); 49 | } 50 | 51 | static float_t df(float_t y, float_t t) { 52 | return (y - t) / (y * (float_t(1) - y)); 53 | } 54 | }; 55 | 56 | // cross-entropy loss function for multi-class classification 57 | class cross_entropy_multiclass { 58 | public: 59 | static float_t f(float_t y, float_t t) { 60 | return -t * std::log(y); 61 | } 62 | 63 | static float_t df(float_t y, float_t t) { 64 | return - t / y; 65 | } 66 | }; 67 | 68 | template 69 | vec_t gradient(const vec_t& y, const vec_t& t) { 70 | vec_t grad(y.size()); 71 | assert(y.size() == t.size()); 72 | 73 | for (cnn_size_t i = 0; i < y.size(); i++) 74 | grad[i] = E::df(y[i], t[i]); 75 | 76 | return grad; 77 | } 78 | 79 | } // namespace tiny_cnn 80 | -------------------------------------------------------------------------------- /tiny_cnn/tiny_cnn.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | 29 | #include "config.h" 30 | #include "network.h" 31 | 32 | 33 | #include "layers/convolutional_layer.h" 34 | #include "layers/fully_connected_layer.h" 35 | #include "layers/average_pooling_layer.h" 36 | #include "layers/max_pooling_layer.h" 37 | #include "layers/linear_layer.h" 38 | #include "layers/lrn_layer.h" 39 | #include "layers/dropout_layer.h" 40 | #include "layers/linear_layer.h" 41 | #include "layers/batchnorm_layer.h" 42 | #include "layers/bnn_fc_layer.h" 43 | #include "layers/binarynet_layer.h" 44 | #include "layers/bnn_conv_layer.h" 45 | #include "layers/offloaded_layer.h" 46 | #include "layers/bnn_threshold_layer.h" 47 | #include "layers/bnn_output_layer.h" 48 | #include "layers/chaninterleave_layer.h" 49 | #include "layers/monitor_layer.h" 50 | 51 | #include "activations/activation_function.h" 52 | #include "lossfunctions/loss_function.h" 53 | #include "optimizers/optimizer.h" 54 | 55 | #include "util/weight_init.h" 56 | #include "util/image.h" 57 | #include "util/deform.h" 58 | #include "util/product.h" 59 | 60 | #include "io/mnist_parser.h" 61 | #include "io/cifar10_parser.h" 62 | #include "io/display.h" 63 | #include "io/layer_factory.h" 64 | 65 | #ifdef CNN_USE_CAFFE_CONVERTER 66 | // experimental / require google protobuf 67 | #include "io/caffe/layer_factory.h" 68 | #endif 69 | -------------------------------------------------------------------------------- /tiny_cnn/util/aligned_allocator.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2016, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include 29 | #ifdef _WIN32 30 | #include 31 | #endif 32 | #ifdef __MINGW32__ 33 | #include 34 | #endif 35 | #include "nn_error.h" 36 | 37 | namespace tiny_cnn { 38 | 39 | template 40 | class aligned_allocator { 41 | public: 42 | typedef T value_type; 43 | typedef T* pointer; 44 | typedef std::size_t size_type; 45 | typedef std::ptrdiff_t difference_type; 46 | typedef T& reference; 47 | typedef const T& const_reference; 48 | typedef const T* const_pointer; 49 | 50 | template 51 | struct rebind { 52 | typedef aligned_allocator other; 53 | }; 54 | 55 | aligned_allocator() {} 56 | 57 | template 58 | aligned_allocator(const aligned_allocator&) {} 59 | 60 | const_pointer address(const_reference value) const { 61 | return std::addressof(value); 62 | } 63 | 64 | pointer address(reference value) const { 65 | return std::addressof(value); 66 | } 67 | 68 | pointer allocate(size_type size, const void* = nullptr) { 69 | void* p = aligned_alloc(alignment, sizeof(T) * size); 70 | if (!p && size > 0) 71 | throw nn_error("failed to allocate"); 72 | return static_cast(p); 73 | } 74 | 75 | size_type max_size() const { 76 | return ~static_cast(0) / sizeof(T); 77 | } 78 | 79 | void deallocate(pointer ptr, size_type) { 80 | aligned_free(ptr); 81 | } 82 | 83 | template 84 | void construct(U* ptr, const V& value) { 85 | void* p = ptr; 86 | ::new(p) U(value); 87 | } 88 | 89 | #if defined(_MSC_VER) && _MSC_VER <= 1800 90 | // -vc2013 doesn't support variadic templates 91 | #else 92 | template 93 | void construct(U* ptr, Args&&... args) { 94 | void* p = ptr; 95 | ::new(p) U(std::forward(args)...); 96 | } 97 | #endif 98 | 99 | template 100 | void construct(U* ptr) { 101 | void* p = ptr; 102 | ::new(p) U(); 103 | } 104 | 105 | template 106 | void destroy(U* ptr) { 107 | ptr->~U(); 108 | } 109 | 110 | private: 111 | void* aligned_alloc(size_type align, size_type size) const { 112 | #if defined(_MSC_VER) 113 | return ::_aligned_malloc(size, align); 114 | #elif defined (__ANDROID__) 115 | return ::memalign(align, size); 116 | #elif defined (__MINGW32__) 117 | return _mm_malloc(size, align); 118 | #else // posix assumed 119 | void* p; 120 | if (::posix_memalign(&p, align, size) != 0) { 121 | p = 0; 122 | } 123 | return p; 124 | #endif 125 | } 126 | 127 | void aligned_free(pointer ptr) { 128 | #if defined(_MSC_VER) 129 | ::_aligned_free(ptr); 130 | #elif defined(__MINGW32__) 131 | ::free(ptr); 132 | #else 133 | ::free(ptr); 134 | #endif 135 | } 136 | }; 137 | 138 | template 139 | inline bool operator==(const aligned_allocator&, const aligned_allocator&) 140 | { 141 | return true; 142 | } 143 | 144 | template 145 | inline bool operator!=(const aligned_allocator&, const aligned_allocator&) 146 | { 147 | return false; 148 | } 149 | } -------------------------------------------------------------------------------- /tiny_cnn/util/deform.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "tiny_cnn/util/util.h" 29 | 30 | namespace tiny_cnn { 31 | 32 | inline vec_t corrupt(vec_t&& in, float_t corruption_level, float_t min_value) { 33 | for (size_t i = 0; i < in.size(); i++) 34 | if (bernoulli(corruption_level)) 35 | in[i] = min_value; 36 | return in; 37 | } 38 | 39 | 40 | } // namespace tiny_cnn 41 | -------------------------------------------------------------------------------- /tiny_cnn/util/nn_error.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2016, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include 29 | #include 30 | 31 | namespace tiny_cnn { 32 | 33 | /** 34 | * basic exception class for tiny-cnn 35 | **/ 36 | class nn_error : public std::exception { 37 | public: 38 | explicit nn_error(const std::string& msg) : msg_(msg) {} 39 | const char* what() const throw() override { return msg_.c_str(); } 40 | private: 41 | std::string msg_; 42 | }; 43 | 44 | } // namespace tiny_cnn 45 | -------------------------------------------------------------------------------- /tiny_cnn/util/weight_init.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2015, Taiga Nomi 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | * Neither the name of the nor the 13 | names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 17 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 20 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | #pragma once 28 | #include "tiny_cnn/util/util.h" 29 | 30 | namespace tiny_cnn { 31 | namespace weight_init { 32 | 33 | class function { 34 | public: 35 | virtual void fill(vec_t *weight, cnn_size_t fan_in, cnn_size_t fan_out) = 0; 36 | }; 37 | 38 | class scalable : public function { 39 | public: 40 | scalable(float_t value) : scale_(value) {} 41 | 42 | void scale(float_t value) { 43 | scale_ = value; 44 | } 45 | protected: 46 | float_t scale_; 47 | }; 48 | 49 | /** 50 | * Use fan-in and fan-out for scaling 51 | * 52 | * X Glorot, Y Bengio, 53 | * Understanding the difficulty of training deep feedforward neural networks 54 | * Proc. AISTATS 10, May 2010, vol.9, pp249-256 55 | **/ 56 | class xavier : public scalable { 57 | public: 58 | xavier() : scalable(float_t(6)) {} 59 | explicit xavier(float_t value) : scalable(value) {} 60 | 61 | void fill(vec_t *weight, cnn_size_t fan_in, cnn_size_t fan_out) override { 62 | const float_t weight_base = std::sqrt(scale_ / (fan_in + fan_out)); 63 | 64 | uniform_rand(weight->begin(), weight->end(), -weight_base, weight_base); 65 | } 66 | }; 67 | 68 | /** 69 | * Use fan-in(number of input weight for each neuron) for scaling 70 | * 71 | * Y LeCun, L Bottou, G B Orr, and K Muller, 72 | * Efficient backprop 73 | * Neural Networks, Tricks of the Trade, Springer, 1998 74 | **/ 75 | class lecun : public scalable { 76 | public: 77 | lecun() : scalable(float_t(1)) {} 78 | explicit lecun(float_t value) : scalable(value) {} 79 | 80 | void fill(vec_t *weight, cnn_size_t fan_in, cnn_size_t fan_out) override { 81 | CNN_UNREFERENCED_PARAMETER(fan_out); 82 | 83 | const float_t weight_base = scale_ / std::sqrt(float_t(fan_in)); 84 | 85 | uniform_rand(weight->begin(), weight->end(), -weight_base, weight_base); 86 | } 87 | }; 88 | 89 | class gaussian : public scalable { 90 | public: 91 | gaussian() : scalable(float_t(1)) {} 92 | explicit gaussian(float_t sigma) : scalable(sigma) {} 93 | 94 | void fill(vec_t *weight, cnn_size_t fan_in, cnn_size_t fan_out) override { 95 | CNN_UNREFERENCED_PARAMETER(fan_in); 96 | CNN_UNREFERENCED_PARAMETER(fan_out); 97 | 98 | gaussian_rand(weight->begin(), weight->end(), float_t(0), scale_); 99 | } 100 | }; 101 | 102 | class constant : public scalable { 103 | public: 104 | constant() : scalable(float_t(0)) {} 105 | explicit constant(float_t value) : scalable(value) {} 106 | 107 | void fill(vec_t *weight, cnn_size_t fan_in, cnn_size_t fan_out) override { 108 | CNN_UNREFERENCED_PARAMETER(fan_in); 109 | CNN_UNREFERENCED_PARAMETER(fan_out); 110 | 111 | std::fill(weight->begin(), weight->end(), scale_); 112 | } 113 | }; 114 | 115 | class he : public scalable { 116 | public: 117 | he() : scalable(float_t(2)) {} 118 | explicit he(float_t value) : scalable(value) {} 119 | 120 | void fill(vec_t *weight, cnn_size_t fan_in, cnn_size_t fan_out) override { 121 | CNN_UNREFERENCED_PARAMETER(fan_out); 122 | 123 | const float_t sigma = std::sqrt(scale_ /fan_in); 124 | 125 | gaussian_rand(weight->begin(), weight->end(), float_t(0), sigma); 126 | } 127 | }; 128 | 129 | } // namespace weight_init 130 | } // namespace tiny_cnn 131 | -------------------------------------------------------------------------------- /vc/vc12/tiny_cnn.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 2013 4 | VisualStudioVersion = 12.0.21005.1 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tiny_cnn", "tiny_cnn.vcxproj", "{C7B38F22-F235-45A5-834A-DE2167849C4C}" 7 | EndProject 8 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tiny_cnn_test", "tiny_cnn_test.vcxproj", "{680A21F8-5877-48EA-B313-CE23B12167ED}" 9 | EndProject 10 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{7ADECCDF-20AF-4B54-81E1-90DF1109D5BF}" 11 | EndProject 12 | Global 13 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 14 | Debug|Win32 = Debug|Win32 15 | Release|Win32 = Release|Win32 16 | EndGlobalSection 17 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 18 | {C7B38F22-F235-45A5-834A-DE2167849C4C}.Debug|Win32.ActiveCfg = Debug|Win32 19 | {C7B38F22-F235-45A5-834A-DE2167849C4C}.Debug|Win32.Build.0 = Debug|Win32 20 | {C7B38F22-F235-45A5-834A-DE2167849C4C}.Release|Win32.ActiveCfg = Release|Win32 21 | {C7B38F22-F235-45A5-834A-DE2167849C4C}.Release|Win32.Build.0 = Release|Win32 22 | {680A21F8-5877-48EA-B313-CE23B12167ED}.Debug|Win32.ActiveCfg = DebugTest|Win32 23 | {680A21F8-5877-48EA-B313-CE23B12167ED}.Debug|Win32.Build.0 = DebugTest|Win32 24 | {680A21F8-5877-48EA-B313-CE23B12167ED}.Release|Win32.ActiveCfg = ReleaseTest|Win32 25 | {680A21F8-5877-48EA-B313-CE23B12167ED}.Release|Win32.Build.0 = ReleaseTest|Win32 26 | EndGlobalSection 27 | GlobalSection(SolutionProperties) = preSolution 28 | HideSolutionNode = FALSE 29 | EndGlobalSection 30 | EndGlobal 31 | -------------------------------------------------------------------------------- /vc/vc12/tiny_cnn.vcxproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | Win32 7 | 8 | 9 | Release 10 | Win32 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | $(VCTargetsPath11) 45 | 46 | 47 | {C7B38F22-F235-45A5-834A-DE2167849C4C} 48 | cnn 49 | tiny_cnn 50 | 51 | 52 | 53 | Application 54 | true 55 | v120 56 | MultiByte 57 | 58 | 59 | Application 60 | false 61 | v120 62 | true 63 | MultiByte 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | Level3 79 | Disabled 80 | ..\..\ 81 | Default 82 | Default 83 | false 84 | Neither 85 | false 86 | AdvancedVectorExtensions 87 | 88 | 89 | true 90 | 91 | 92 | 93 | 94 | 95 | 96 | Level3 97 | Full 98 | true 99 | true 100 | ..\..\ 101 | AnySuitable 102 | Speed 103 | true 104 | true 105 | AdvancedVectorExtensions 106 | Fast 107 | _MBCS;CNN_USE_AVX;CNN_USE_TBB;NDEBUG;%(PreprocessorDefinitions) 108 | Sync 109 | MultiThreadedDLL 110 | ProgramDatabase 111 | /D_VARIADIC_MAX=10 %(AdditionalOptions) 112 | 113 | 114 | true 115 | true 116 | true 117 | false 118 | C:\Program Files\tbb41_20121003oss\lib\ia32\vc11 119 | 120 | 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /vc/vc12/tiny_cnn_test.vcxproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | DebugTest 6 | Win32 7 | 8 | 9 | ReleaseTest 10 | Win32 11 | 12 | 13 | 14 | $(VCTargetsPath11) 15 | 16 | 17 | {680A21F8-5877-48EA-B313-CE23B12167ED} 18 | test 19 | tiny_cnn_test 20 | 21 | 22 | 23 | Application 24 | true 25 | v120 26 | MultiByte 27 | 28 | 29 | Application 30 | false 31 | v120 32 | true 33 | MultiByte 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | $(SolutionDir)Test$(Configuration)\ 47 | 48 | 49 | $(SolutionDir)Test$(Configuration)\ 50 | 51 | 52 | 53 | Level3 54 | Disabled 55 | ..\..\ 56 | 57 | 58 | true 59 | 60 | 61 | 62 | 63 | 64 | 65 | Level3 66 | MaxSpeed 67 | true 68 | true 69 | ..\..\ 70 | _MBCS;CNN_USE_TBB;%(PreprocessorDefinitions) 71 | 72 | 73 | true 74 | true 75 | true 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | -------------------------------------------------------------------------------- /vc/vc14/tiny_cnn.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 14 4 | VisualStudioVersion = 14.0.23107.0 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tiny_cnn", "tiny_cnn.vcxproj", "{C7B38F22-F235-45A5-834A-DE2167849C4C}" 7 | EndProject 8 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tiny_cnn_test", "tiny_cnn_test.vcxproj", "{680A21F8-5877-48EA-B313-CE23B12167ED}" 9 | EndProject 10 | Global 11 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 12 | Debug|Win32 = Debug|Win32 13 | Release|Win32 = Release|Win32 14 | EndGlobalSection 15 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 16 | {C7B38F22-F235-45A5-834A-DE2167849C4C}.Debug|Win32.ActiveCfg = Debug|Win32 17 | {C7B38F22-F235-45A5-834A-DE2167849C4C}.Debug|Win32.Build.0 = Debug|Win32 18 | {C7B38F22-F235-45A5-834A-DE2167849C4C}.Release|Win32.ActiveCfg = Release|Win32 19 | {C7B38F22-F235-45A5-834A-DE2167849C4C}.Release|Win32.Build.0 = Release|Win32 20 | {680A21F8-5877-48EA-B313-CE23B12167ED}.Debug|Win32.ActiveCfg = DebugTest|Win32 21 | {680A21F8-5877-48EA-B313-CE23B12167ED}.Debug|Win32.Build.0 = DebugTest|Win32 22 | {680A21F8-5877-48EA-B313-CE23B12167ED}.Release|Win32.ActiveCfg = ReleaseTest|Win32 23 | {680A21F8-5877-48EA-B313-CE23B12167ED}.Release|Win32.Build.0 = ReleaseTest|Win32 24 | EndGlobalSection 25 | GlobalSection(SolutionProperties) = preSolution 26 | HideSolutionNode = FALSE 27 | EndGlobalSection 28 | EndGlobal 29 | -------------------------------------------------------------------------------- /vc/vc14/tiny_cnn.vcxproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | Win32 7 | 8 | 9 | Release 10 | Win32 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | {C7B38F22-F235-45A5-834A-DE2167849C4C} 46 | cnn 47 | tiny_cnn 48 | 8.1 49 | 50 | 51 | 52 | Application 53 | true 54 | v140 55 | MultiByte 56 | 57 | 58 | Application 59 | false 60 | v140 61 | true 62 | MultiByte 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | $(UniversalCRT_LibraryPath_x86);$(LibraryPath) 76 | $(SolutionDir)$(Configuration)\ 77 | $(Configuration)\ 78 | 79 | 80 | $(UniversalCRT_LibraryPath_x86);C:\opencv\build\lib\Release;$(VCInstallDir)lib;$(VCInstallDir)atlmfc\lib;$(WindowsSDK_LibraryPath_x86);C:\Program Files\tbb41_20121003oss\lib\ia32\vc11;C:\Program Files\boost\boost_1_59_0\stage\lib;C:\Program Files\boost\boost_1_55_0\stage\lib 81 | 82 | 83 | 84 | Level3 85 | Disabled 86 | ..\..\ 87 | Default 88 | Default 89 | false 90 | Neither 91 | false 92 | AdvancedVectorExtensions 93 | 94 | 95 | true 96 | 97 | 98 | 99 | 100 | 101 | 102 | Level3 103 | Full 104 | true 105 | true 106 | ..\..\ 107 | AnySuitable 108 | Speed 109 | true 110 | true 111 | AdvancedVectorExtensions 112 | Fast 113 | _MBCS;CNN_USE_AVX;CNN_USE_TBB;NDEBUG;%(PreprocessorDefinitions) 114 | Sync 115 | MultiThreadedDLL 116 | ProgramDatabase 117 | 118 | 119 | true 120 | true 121 | true 122 | false 123 | C:\Program Files\tbb41_20121003oss\lib\ia32\vc11 124 | $(OutDir)$(TargetName)$(TargetExt) 125 | 126 | 127 | 128 | 129 | 130 | -------------------------------------------------------------------------------- /vc/vc14/tiny_cnn_test.vcxproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | DebugTest 6 | Win32 7 | 8 | 9 | ReleaseTest 10 | Win32 11 | 12 | 13 | 14 | 15 | {680A21F8-5877-48EA-B313-CE23B12167ED} 16 | test 17 | tiny_cnn_test 18 | 8.1 19 | 20 | 21 | 22 | Application 23 | true 24 | v140 25 | MultiByte 26 | 27 | 28 | Application 29 | false 30 | v140 31 | true 32 | MultiByte 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | $(UniversalCRT_LibraryPath_x86);$(LibraryPath) 46 | $(SolutionDir)$(Configuration)_Test\ 47 | $(Configuration)_Test\ 48 | 49 | 50 | $(UniversalCRT_LibraryPath_x86);C:\opencv\build\lib\Release;$(VCInstallDir)lib;$(VCInstallDir)atlmfc\lib;$(WindowsSDK_LibraryPath_x86);C:\Program Files\tbb41_20121003oss\lib\ia32\vc11;C:\Program Files\boost\boost_1_59_0\stage\lib;C:\Program Files\boost\boost_1_55_0\stage\lib 51 | 52 | 53 | 54 | Level3 55 | Disabled 56 | ..\..\ 57 | true 58 | AdvancedVectorExtensions2 59 | /bigobj %(AdditionalOptions) 60 | 61 | 62 | true 63 | 64 | 65 | 66 | 67 | 68 | 69 | Level3 70 | MaxSpeed 71 | true 72 | true 73 | ..\..\ 74 | _MBCS;CNN_USE_TBB;%(PreprocessorDefinitions) 75 | /bigobj %(AdditionalOptions) 76 | 77 | 78 | true 79 | true 80 | true 81 | 82 | 83 | $(OutDir)$(TargetName)$(TargetExt) 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | -------------------------------------------------------------------------------- /waf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xilinx/xilinx-tiny-cnn/64ee7a274e67afe664d787313c9cb5b80c0396ee/waf -------------------------------------------------------------------------------- /wscript: -------------------------------------------------------------------------------- 1 | 2 | VERSION="0.0.1" 3 | APPNAME='tiny-cnn' 4 | 5 | srcdir = '.' 6 | blddir = 'build' 7 | 8 | import sys 9 | def options(opt): 10 | opt.load('compiler_cxx') 11 | opt.add_option('--TBB', 12 | action = 'store_true', 13 | default = False, 14 | help='enable TBB parallelization [default:False]') 15 | opt.add_option('--AVX', 16 | action = 'store_true', 17 | default = False, 18 | help='enable AVX vectorization [default:False]') 19 | opt.add_option('--SSE', 20 | action = 'store_true', 21 | default = False, 22 | help='enable SSE vectorization [default:False]') 23 | 24 | if sys.platform.startswith('win'): 25 | BOOST_ROOT_DEFAULT = 'C:/Program Files/boost/boost_1_51_0/' 26 | else: 27 | BOOST_ROOT_DEFAULT = '/usr/local/include/boost/' 28 | opt.add_option('--BOOST_ROOT', 29 | action = 'store', 30 | default = BOOST_ROOT_DEFAULT, 31 | help='root directory of boost [default:%s]' % BOOST_ROOT_DEFAULT) 32 | 33 | if sys.platform.startswith('win'): 34 | TBB_ROOT_DEFAULT = 'C:/Program Files/tbb/include' 35 | else: 36 | TBB_ROOT_DEFAULT = '/usr/local/include/tbb/' 37 | opt.add_option('--TBB_ROOT', 38 | action = 'store', 39 | default = TBB_ROOT_DEFAULT, 40 | help='root directory of TBB [default:%s]' % TBB_ROOT_DEFAULT) 41 | 42 | def configure(conf): 43 | conf.load('compiler_cxx') 44 | if conf.options.TBB: 45 | conf.define("CNN_USE_TBB", "") 46 | print "enable option:TBB(root=%s)" % conf.options.TBB_ROOT 47 | if conf.options.AVX: 48 | conf.define("CNN_USE_AVX", "") 49 | print("enable option:AVX") 50 | if conf.options.SSE: 51 | conf.define("CNN_USE_SSE", "") 52 | print("enable option:SSE") 53 | conf.env.TBB_ROOT = conf.options.TBB_ROOT 54 | conf.env.BOOST_ROOT = conf.options.BOOST_ROOT 55 | 56 | def build(bld): 57 | bld.recurse('examples') 58 | 59 | --------------------------------------------------------------------------------