├── .bazelrc ├── .clang-format ├── .travis.yml ├── BUILD.bazel ├── CMakeLists.txt ├── COPYING ├── CPPLINT.cfg ├── WORKSPACE ├── cmake ├── compiler-flags.cmake ├── gflags.cmake ├── glog.cmake ├── lint.cmake └── protobuf.cmake ├── cnn ├── BUILD.bazel ├── array │ ├── BUILD.bazel │ ├── array.cc │ ├── array.h │ ├── array_math.h │ ├── test_array.cc │ └── test_array_math.cc ├── autodiff │ ├── BUILD.bazel │ ├── jet.h │ └── test_jet.cc ├── proto │ ├── BUILD.bazel │ └── cnn.proto ├── readme.md └── workspace.bzl ├── doc ├── Doxyfile └── run-doxygen.sh ├── examples ├── CMakeLists.txt ├── linear_regression │ ├── CMakeLists.txt │ ├── main.cpp │ ├── model.prototxt │ └── optimizer.prototxt ├── logistic_regression │ ├── CMakeLists.txt │ ├── data │ │ ├── ex2.pdf │ │ ├── ex2data1.txt │ │ ├── ex2data2.txt │ │ ├── gnuplot-notes.md │ │ ├── plot1.gp │ │ └── plot2.gp │ ├── main.cpp │ ├── model.prototxt │ ├── optimizer.prototxt │ └── trained.prototxt └── mnist │ ├── CMakeLists.txt │ ├── data │ └── download_and_unzip_data.sh │ ├── loss-accuracy-iter.png │ ├── main.cpp │ ├── model_for_deploy.prototxt │ ├── model_for_train.prototxt │ ├── model_for_train.prototxt2 │ └── optimizer.prototxt ├── include └── cnn │ ├── array.hpp │ ├── array_math.hpp │ ├── batch_normalization_layer.hpp │ ├── common.hpp │ ├── convolution_layer.hpp │ ├── drop_out_layer.hpp │ ├── full_connected_layer.hpp │ ├── input_layer.hpp │ ├── io.hpp │ ├── jet.hpp │ ├── l2_loss_layer.hpp │ ├── layer.hpp │ ├── leaky_relu_layer.hpp │ ├── log_loss_layer.hpp │ ├── max_pooling_layer.hpp │ ├── network.hpp │ ├── optimizer.hpp │ ├── relu_layer.hpp │ ├── rng.hpp │ ├── softmax_layer.hpp │ └── softmax_with_log_loss_layer.hpp ├── proto ├── CMakeLists.txt ├── cnn.proto ├── model.prototxt ├── optimizer.prototxt └── trained.prototxt ├── readme.md ├── scripts └── cpp_lint.py ├── src ├── array.cpp ├── batch_normalization_layer.cpp ├── convolution_layer.cpp ├── drop_out_layer.cpp ├── full_connected_layer.cpp ├── input_layer.cpp ├── io.cpp ├── l2_loss_layer.cpp ├── layer.cpp ├── leaky_relu_layer.cpp ├── log_loss_layer.cpp ├── max_pooling_layer.cpp ├── network.cpp ├── optimizer.cpp ├── relu_layer.cpp ├── rng.cpp ├── softmax_layer.cpp └── softmax_with_log_loss_layer.cpp ├── test ├── CMakeLists.txt ├── gtest │ ├── CMakeLists.txt │ ├── gtest-all.cpp │ ├── gtest.h │ └── readme.md ├── test_array.cpp ├── test_array_math.cpp ├── test_batch_normalization_layer.cpp ├── test_convolution_layer.cpp ├── test_drop_out_layer.cpp ├── test_full_connected_layer.cpp ├── test_input_layer.cpp ├── test_io.cpp ├── test_jet.cpp ├── test_l2_loss_layer.cpp ├── test_layer.cpp ├── test_leaky_relu_layer.cpp ├── test_log_loss_layer.cpp ├── test_main.cpp ├── test_max_pooling_layer.cpp ├── test_network.cpp ├── test_optimizer.cpp ├── test_relu_layer.cpp ├── test_rng.cpp ├── test_softmax_layer.cpp └── test_softmax_with_log_loss_layer.cpp ├── third_party ├── BUILD.bazel └── google_style_guide.BUILD ├── tools ├── CMakeLists.txt ├── cnn.cpp └── lint │ ├── BUILD.bazel │ ├── cpplint.bzl │ └── readme.md └── travis ├── install-bazel.sh ├── mnist-test.sh └── script.sh /.bazelrc: -------------------------------------------------------------------------------- 1 | 2 | # modified from https://github.com/ApolloAuto/apollo/blob/master/tools/bazel.rc#L21 3 | # To enable the lint test, the BUILD *must* load the cpplint.bzl by having 4 | # 'load("//tools/lint:cpplint.bzl", "cpplint")' at the beginning and 'cpplint()' 5 | # at the end. 6 | test:cpplint --test_tag_filters=cpplint 7 | test:cpplint --build_tests_only 8 | test:cpplint --test_output=all 9 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | # -*- yaml -*- 2 | # modified from https://github.com/RobotLocomotion/drake/blob/master/.clang-format 3 | 4 | # This file determines clang-format's style settings; for details, refer to 5 | # http://clang.llvm.org/docs/ClangFormatStyleOptions.html 6 | 7 | BasedOnStyle: Google 8 | 9 | Language: Cpp 10 | 11 | # Force pointers to the type for C++. 12 | DerivePointerAlignment: false 13 | PointerAlignment: Left 14 | 15 | # Specify the #include statement order. This implements the order mandated by 16 | # the Google C++ Style Guide: related header, C headers, C++ headers, library 17 | # headers, and finally the project headers. 18 | # 19 | # To obtain updated lists of system headers used in the below expressions, see: 20 | # http://stackoverflow.com/questions/2027991/list-of-standard-header-files-in-c-and-c/2029106#2029106. 21 | IncludeCategories: 22 | # Spacers used by drake/tools/formatter.py. 23 | - Regex: '^$' 24 | Priority: 15 25 | - Regex: '^$' 26 | Priority: 25 27 | - Regex: '^$' 28 | Priority: 35 29 | - Regex: '^$' 30 | Priority: 45 31 | # C system headers. 32 | - Regex: '^[<"](aio|arpa/inet|assert|complex|cpio|ctype|curses|dirent|dlfcn|errno|fcntl|fenv|float|fmtmsg|fnmatch|ftw|glob|grp|iconv|inttypes|iso646|langinfo|libgen|limits|locale|math|monetary|mqueue|ndbm|netdb|net/if|netinet/in|netinet/tcp|nl_types|poll|pthread|pwd|regex|sched|search|semaphore|setjmp|signal|spawn|stdalign|stdarg|stdatomic|stdbool|stddef|stdint|stdio|stdlib|stdnoreturn|string|strings|stropts|sys/ipc|syslog|sys/mman|sys/msg|sys/resource|sys/select|sys/sem|sys/shm|sys/socket|sys/stat|sys/statvfs|sys/time|sys/times|sys/types|sys/uio|sys/un|sys/utsname|sys/wait|tar|term|termios|tgmath|threads|time|trace|uchar|ulimit|uncntrl|unistd|utime|utmpx|wchar|wctype|wordexp)\.h[">]$' 33 | Priority: 20 34 | # C++ system headers (as of C++17). 35 | - Regex: '^[<"](algorithm|any|array|atomic|bitset|cassert|ccomplex|cctype|cerrno|cfenv|cfloat|charconv|chrono|cinttypes|ciso646|climits|clocale|cmath|codecvt|complex|condition_variable|csetjmp|csignal|cstdalign|cstdarg|cstdbool|cstddef|cstdint|cstdio|cstdlib|cstring|ctgmath|ctime|cuchar|cwchar|cwctype|deque|exception|execution|filesystem|forward_list|fstream|functional|future|initializer_list|iomanip|ios|iosfwd|iostream|istream|iterator|limits|list|locale|map|memory|memory_resource|mutex|new|numeric|optional|ostream|queue|random|ratio|regex|scoped_allocator|set|shared_mutex|sstream|stack|stdexcept|streambuf|string|string_view|strstream|system_error|thread|tuple|type_traits|typeindex|typeinfo|unordered_map|unordered_set|utility|valarray|variant|vector)[">]$' 36 | Priority: 30 37 | # Other libraries' h files (with angles). 38 | - Regex: '^<' 39 | Priority: 40 40 | # Your project's h files. 41 | - Regex: '^"cnn' 42 | Priority: 50 43 | # Other libraries' h files (with quotes). 44 | - Regex: '^"' 45 | Priority: 41 46 | 47 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Travis CI for OpenCNN 2 | # 3 | # Copyright 2019. All Rights Reserved. 4 | # Author: csukuangfj@gmail.com (Fangjun Kuang) 5 | # Date: November 26, 2018 6 | # 7 | 8 | os: 9 | - linux 10 | - osx 11 | 12 | language: cpp 13 | 14 | compiler: 15 | - gcc 16 | - clang 17 | 18 | env: 19 | - CXX_FLAGS=-std=c++11 20 | 21 | # https://docs.travis-ci.com/user/installing-dependencies#Installing-Packages-with-the-APT-Addon 22 | addons: 23 | apt: 24 | sources: 25 | - ubuntu-toolchain-r-test 26 | - llvm-toolchain-precise-3.7 27 | packages: 28 | - git 29 | 30 | notifications: 31 | email: false 32 | 33 | sudo: false 34 | 35 | script: 36 | - bash travis/script.sh 37 | -------------------------------------------------------------------------------- /BUILD.bazel: -------------------------------------------------------------------------------- 1 | # Copyright 2019. All Rights Reserved. 2 | # Author: csukuangfj@gmail.com (Fangjun Kuang) 3 | 4 | package(default_visibility = ["//visibility:public"]) 5 | 6 | exports_files(["CPPLINT.cfg"]) 7 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | cmake_minimum_required(VERSION 3.0) 3 | project(cnn) 4 | 5 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) 6 | set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) 7 | 8 | # include(${PROJECT_SOURCE_DIR}/cmake/lint.cmake) 9 | include(${PROJECT_SOURCE_DIR}/cmake/gflags.cmake) 10 | include(${PROJECT_SOURCE_DIR}/cmake/glog.cmake) 11 | include(${PROJECT_SOURCE_DIR}/cmake/compiler-flags.cmake) 12 | include(${PROJECT_SOURCE_DIR}/cmake/protobuf.cmake) 13 | 14 | include_directories(include) 15 | 16 | add_subdirectory(proto) 17 | 18 | add_library( 19 | core STATIC 20 | src/io.cpp 21 | src/rng.cpp 22 | # src/array.cpp 23 | # src/layer.cpp 24 | # src/full_connected_layer.cpp 25 | # src/input_layer.cpp 26 | # src/network.cpp 27 | # src/l2_loss_layer.cpp 28 | # src/array_math.cpp 29 | # src/optimizer.cpp 30 | # src/softmax_layer.cpp 31 | # src/log_loss_layer.cpp 32 | # src/softmax_with_log_loss_layer.cpp 33 | ) 34 | target_link_libraries( 35 | core 36 | cnn_proto 37 | ${GLOG_LIBRARIES} 38 | ${GFLAGS_LIBRARIES} 39 | ) 40 | 41 | if(UNIX AND NOT APPLE) 42 | target_link_libraries( 43 | core 44 | -pthread 45 | ) 46 | endif() 47 | 48 | add_subdirectory(test) 49 | 50 | add_subdirectory(tools) 51 | add_subdirectory(examples) 52 | 53 | -------------------------------------------------------------------------------- /CPPLINT.cfg: -------------------------------------------------------------------------------- 1 | # Stop searching for additional config files. 2 | set noparent 3 | 4 | root = . 5 | -------------------------------------------------------------------------------- /WORKSPACE: -------------------------------------------------------------------------------- 1 | # Copyright 2019. All Rights Reserved. 2 | # Author: csukuangfj@gmail.com (Fangjun Kuang) 3 | 4 | workspace(name = "cnn") 5 | 6 | load("//cnn:workspace.bzl", "cnn_repositories") 7 | 8 | cnn_repositories() 9 | 10 | load( 11 | "@bazel_federation//:repositories.bzl", 12 | "rules_cc", 13 | "rules_java", 14 | "rules_python", 15 | ) 16 | 17 | #==================== rules_cc ==================== 18 | rules_cc() 19 | 20 | load("@bazel_federation//setup:rules_cc.bzl", "rules_cc_setup") 21 | 22 | rules_cc_setup() 23 | 24 | #==================== rules_java ==================== 25 | rules_java() 26 | 27 | load("@bazel_federation//setup:rules_java.bzl", "rules_java_setup") 28 | 29 | rules_java_setup() 30 | 31 | #==================== rules_python ==================== 32 | rules_python() 33 | 34 | load("@bazel_federation//setup:rules_python.bzl", "rules_python_setup") 35 | 36 | rules_python_setup() 37 | 38 | #==================== rules_proto ==================== 39 | 40 | load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") 41 | 42 | rules_proto_dependencies() 43 | 44 | rules_proto_toolchains() 45 | -------------------------------------------------------------------------------- /cmake/compiler-flags.cmake: -------------------------------------------------------------------------------- 1 | 2 | #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wextra -O3 -DNDEBUG") 3 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wextra -O2 -DNDEBUG") 4 | #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wextra -g") 5 | if (true) 6 | # refer to Useful GCC flags for C, http://stackoverflow.com/questions/3375697/useful-gcc-flags-for-c 7 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wpointer-arith -Wunreachable-code") 8 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wcast-align" ) 9 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wwrite-strings ") 10 | #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wstrict-overflow=5") 11 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wcast-qual -Wswitch-default -Wswitch-enum") 12 | #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow") 13 | #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wconversion") 14 | #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wundef") 15 | #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Waggregate-return") 16 | #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wstrict-prototypes") 17 | 18 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-float-equal") 19 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare") 20 | endif() 21 | -------------------------------------------------------------------------------- /cmake/gflags.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # GFLAGS_INCLUDE_DIRS 3 | # GFLAGS_LIBRARIES 4 | # 5 | 6 | find_path(GFLAGS_INCLUDE_DIRS 7 | gflags/gflags.h 8 | PATHS 9 | /usr/local/include 10 | $ENV{HOME}/software/gflags/include 11 | DOC "Path to gflags/gflags.h" 12 | ) 13 | if (NOT GFLAGS_INCLUDE_DIRS) 14 | message(FATAL_ERROR "Could not find gflags/gflags.h") 15 | endif() 16 | 17 | find_library(GFLAGS_LIBRARIES 18 | NAMES libgflags.so libgflags.dylib gflags 19 | PATHS 20 | /usr/local/lib 21 | /usr/lib/x86_64-linux-gnu 22 | $ENV{HOME}/software/gflags/lib 23 | DOC "Path to libgflags.so" 24 | ) 25 | if (NOT GFLAGS_LIBRARIES) 26 | message(FATAL_ERROR "Could not find libgflags.so") 27 | endif() 28 | 29 | include_directories(${GFLAGS_INCLUDE_DIRS}) 30 | message(STATUS "GFLAGS_INCLUDE_DIRS: ${GFLAGS_INCLUDE_DIRS}") 31 | message(STATUS "GFLAGS_LIBRARIES: ${GFLAGS_LIBRARIES}") 32 | 33 | #[[ 34 | brew install gflags 35 | ]] 36 | -------------------------------------------------------------------------------- /cmake/glog.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # GLOG_INCLUDE_DIRS 3 | # GLOG_LIBRARIES 4 | # 5 | 6 | find_path(GLOG_INCLUDE_DIRS 7 | glog/logging.h 8 | PATHS 9 | /usr/local/include 10 | $ENV{HOME}/software/glog/include 11 | DOC "Path to glog/logging.h" 12 | ) 13 | if (NOT GLOG_INCLUDE_DIRS) 14 | message(FATAL_ERROR "Could not find glog/logging.h") 15 | endif() 16 | 17 | find_library(GLOG_LIBRARIES 18 | NAMES libglog.so libglog.dylib glog 19 | PATHS 20 | /usr/local/lib 21 | /usr/lib/x86_64-linux-gnu 22 | $ENV{HOME}/software/glog/lib 23 | DOC "Path to libglog.so" 24 | ) 25 | if (NOT GLOG_LIBRARIES) 26 | message(FATAL_ERROR "Could not find libglog.so") 27 | endif() 28 | 29 | include_directories(${GLOG_INCLUDE_DIRS}) 30 | message(STATUS "GLOG_INCLUDE_DIRS: ${GLOG_INCLUDE_DIRS}") 31 | message(STATUS "GLOG_LIBRARIES: ${GLOG_LIBRARIES}") 32 | 33 | #[[ 34 | brew install glog 35 | ]] 36 | -------------------------------------------------------------------------------- /cmake/lint.cmake: -------------------------------------------------------------------------------- 1 | 2 | # modified from caffe/cmake/lint.cmake 3 | 4 | set(CMAKE_SOURCE_DIR ${PROJECT_SOURCE_DIR}) 5 | set(LINT_COMMAND ${CMAKE_SOURCE_DIR}/scripts/cpp_lint.py) 6 | set(LINT_OPTIONS "--filter=-whitespace/braces") 7 | set(SRC_FILE_EXTENSIONS h hpp hu c cpp cu cc) 8 | set(EXCLUDE_FILE_EXTENSTIONS pb.h pb.cc) 9 | set(LINT_DIRS include src) 10 | 11 | cmake_policy(SET CMP0009 NEW) # suppress cmake warning 12 | 13 | # find all files of interest 14 | foreach(ext ${SRC_FILE_EXTENSIONS}) 15 | foreach(dir ${LINT_DIRS}) 16 | file(GLOB_RECURSE FOUND_FILES ${CMAKE_SOURCE_DIR}/${dir}/*.${ext}) 17 | set(LINT_SOURCES ${LINT_SOURCES} ${FOUND_FILES}) 18 | endforeach() 19 | endforeach() 20 | 21 | # find all files that should be excluded 22 | foreach(ext ${EXCLUDE_FILE_EXTENSTIONS}) 23 | file(GLOB_RECURSE FOUND_FILES ${CMAKE_SOURCE_DIR}/*.${ext}) 24 | set(EXCLUDED_FILES ${EXCLUDED_FILES} ${FOUND_FILES}) 25 | endforeach() 26 | 27 | # exclude generated pb files 28 | # list(REMOVE_ITEM LINT_SOURCES ${EXCLUDED_FILES}) 29 | 30 | execute_process( 31 | COMMAND ${LINT_COMMAND} ${LINT_OPTIONS} ${LINT_SOURCES} 32 | ERROR_VARIABLE LINT_OUTPUT 33 | ERROR_STRIP_TRAILING_WHITESPACE 34 | ) 35 | 36 | string(REPLACE "\n" ";" LINT_OUTPUT ${LINT_OUTPUT}) 37 | 38 | list(GET LINT_OUTPUT -1 LINT_RESULT) 39 | list(REMOVE_AT LINT_OUTPUT -1) 40 | string(REPLACE " " ";" LINT_RESULT ${LINT_RESULT}) 41 | list(GET LINT_RESULT -1 NUM_ERRORS) 42 | if(NUM_ERRORS GREATER 0) 43 | foreach(msg ${LINT_OUTPUT}) 44 | string(FIND ${msg} "Done" result) 45 | if(result LESS 0) 46 | message(STATUS ${msg}) 47 | endif() 48 | endforeach() 49 | message(FATAL_ERROR "Lint found ${NUM_ERRORS} errors!") 50 | else() 51 | message(STATUS "Lint did not find any errors!") 52 | endif() 53 | 54 | -------------------------------------------------------------------------------- /cmake/protobuf.cmake: -------------------------------------------------------------------------------- 1 | 2 | #[==[ 3 | refer to 4 | https://cmake.org/cmake/help/v3.9/module/FindProtobuf.html 5 | ]==] 6 | include(FindProtobuf) 7 | find_package(Protobuf REQUIRED) 8 | 9 | message(STATUS "protobuf include dirs: ${Protobuf_INCLUDE_DIR}") 10 | message(STATUS "protobuf libs: ${Protobuf_LIBRARY}") 11 | 12 | include_directories(${Protobuf_INCLUDE_DIR}) 13 | include_directories(${CMAKE_CURRENT_BINARY_DIR}) 14 | 15 | -------------------------------------------------------------------------------- /cnn/BUILD.bazel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csukuangfj/OpenCNN/561936740226b6741e75237a68afd6bda6df4f7d/cnn/BUILD.bazel -------------------------------------------------------------------------------- /cnn/array/BUILD.bazel: -------------------------------------------------------------------------------- 1 | # Copyright 2019. All Rights Reserved. 2 | # Author: csukuangfj@gmail.com (Fangjun Kuang) 3 | 4 | package(default_visibility = ["//visibility:public"]) 5 | 6 | load("//tools/lint:cpplint.bzl", "cpplint") 7 | 8 | cc_library( 9 | name = "array", 10 | srcs = ["array.cc"], 11 | hdrs = ["array.h"], 12 | deps = [ 13 | "//cnn/autodiff:jet", 14 | "//cnn/proto:cnn_cc_proto", 15 | "@com_google_glog//:glog", 16 | ], 17 | ) 18 | 19 | cc_library( 20 | name = "array_math", 21 | hdrs = ["array_math.h"], 22 | deps = [ 23 | ":array", 24 | ], 25 | ) 26 | 27 | cc_test( 28 | name = "test_array", 29 | srcs = ["test_array.cc"], 30 | deps = [ 31 | ":array", 32 | "@com_google_googletest//:gtest_main", 33 | ], 34 | ) 35 | 36 | cc_test( 37 | name = "test_array_math", 38 | srcs = ["test_array_math.cc"], 39 | deps = [ 40 | ":array_math", 41 | "@com_google_googletest//:gtest_main", 42 | ], 43 | ) 44 | 45 | # cpplint() 46 | -------------------------------------------------------------------------------- /cnn/array/array.h: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #ifndef CNN_ARRAY_ARRAY_H_ 16 | #define CNN_ARRAY_ARRAY_H_ 17 | 18 | #include 19 | #include 20 | 21 | #include "cnn/proto/cnn.pb.h" 22 | 23 | namespace cnn { 24 | 25 | template 26 | class Array { 27 | public: 28 | Array(); 29 | ~Array(); 30 | Array(const Array&) = delete; 31 | Array& operator=(const Array&) = delete; 32 | 33 | Array(Array&&); 34 | Array& operator=(Array&&); 35 | 36 | void init(int n, int c, int h, int w); 37 | 38 | template 39 | void init_like(const Array& arr); 40 | 41 | template 42 | bool has_same_shape(const Array& arr) const; 43 | 44 | bool has_same_shape(const std::vector& vec) const; 45 | std::vector shape_vec() const { return {n_, c_, h_, w_}; } 46 | 47 | /** 48 | * Return the element at n*c_*h_*w_ + c*h_*w_ + h*w_ + w, 49 | * i.e., (n*c_ + c)*h_*w_ + h*w_ + w, 50 | * i.e., ((n*c_ + c)*h_ + h)*w_ + w 51 | */ 52 | const Dtype& at(int n, int c, int h, int w) const; 53 | Dtype& at(int n, int c, int h, int w); 54 | 55 | // no range check 56 | const Dtype& operator()(int n, int c, int h, int w) const; 57 | Dtype& operator()(int n, int c, int h, int w); 58 | 59 | const Dtype& operator[](int i) const; 60 | Dtype& operator[](int i); 61 | 62 | std::string shape_info() const; 63 | 64 | int n_; //!< number of batches 65 | int c_; //!< number of channels 66 | int h_; //!< image height, i.e., number of rows 67 | int w_; //!< image width, number of columns 68 | int total_; //!< n_*c_*h_*w_, number of elements 69 | 70 | Dtype* d_; //!< pointer to the data 71 | 72 | public: 73 | void from_proto(const ArrayProto& proto); 74 | void to_proto(ArrayProto* proto) const; 75 | }; 76 | 77 | extern template class Array; 78 | extern template class Array; 79 | 80 | } // namespace cnn 81 | 82 | #endif // CNN_ARRAY_ARRAY_H_ 83 | -------------------------------------------------------------------------------- /cnn/array/test_array_math.cc: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | 17 | #include "cnn/array/array_math.h" 18 | 19 | namespace cnn { 20 | 21 | template 22 | class ArrayMathTest : public ::testing::Test {}; 23 | 24 | using MyTypes = ::testing::Types; 25 | TYPED_TEST_CASE(ArrayMathTest, MyTypes); 26 | 27 | TYPED_TEST(ArrayMathTest, set_to) { 28 | int val = 100; 29 | { 30 | Array arr; 31 | arr.init(2, 3, 4, 5); 32 | set_to(&arr, val); 33 | for (int i = 0; i < arr.total_; i++) { 34 | EXPECT_EQ(arr[i], val); 35 | } 36 | 37 | val = 0; 38 | set_to(&arr, val); 39 | for (int i = 0; i < arr.total_; i++) { 40 | EXPECT_EQ(arr[i], val); 41 | } 42 | } 43 | 44 | { 45 | val = 200; 46 | Array arr; 47 | arr.init(2, 3, 4, 5); 48 | set_to(arr.total_, &arr[0], val); 49 | for (int i = 0; i < arr.total_; i++) { 50 | EXPECT_EQ(arr[i], val); 51 | } 52 | 53 | val = 0; 54 | set_to(arr.total_, &arr[0], val); 55 | for (int i = 0; i < arr.total_; i++) { 56 | EXPECT_EQ(arr[i], val); 57 | } 58 | } 59 | } 60 | 61 | TYPED_TEST(ArrayMathTest, ax_sub_by_squared) { 62 | Array arr1; 63 | Array arr2; 64 | 65 | arr1.init(1, 2, 3, 4); 66 | arr2.init(24, 1, 1, 1); 67 | 68 | set_to(&arr1, 1); 69 | set_to(&arr2, 1); 70 | 71 | double diff = 1000; 72 | 73 | diff = ax_sub_by_squared(arr1.total_, 1, &arr1(0, 0, 0, 0), 1, 74 | &arr2[0]); 75 | EXPECT_EQ(diff, 0); 76 | 77 | double sum = 0; 78 | 79 | sum = ax_sub_by_squared(arr1.total_, -1, &arr1(0, 0, 0, 0), 1, 80 | &arr2[0]); 81 | EXPECT_EQ(sum, 4 * arr1.total_); 82 | } 83 | 84 | TYPED_TEST(ArrayMathTest, ax_dot_by) { 85 | Array arr1; 86 | Array arr2; 87 | 88 | arr1.init(1, 1, 2, 3); 89 | arr2.init(1, 1, 2, 3); 90 | 91 | set_to(&arr1, 1); 92 | set_to(&arr2, 1); 93 | 94 | TypeParam sum = 0; 95 | sum = ax_dot_by(arr1.total_, 1, arr1.d_, 1, arr2.d_); 96 | EXPECT_EQ(sum, arr1.total_); 97 | 98 | sum = 0; 99 | set_to(&arr2, 2); 100 | sum = ax_dot_by(arr1.total_, -1, arr1.d_, 2, arr2.d_); 101 | EXPECT_EQ(sum, -arr1.total_ * 4); 102 | } 103 | 104 | TYPED_TEST(ArrayMathTest, scale_arr) { 105 | Array d; 106 | d.init(1, 2, 1, 1); 107 | d[0] = 2; 108 | d[1] = 3; 109 | scale_arr(2, d, &d); 110 | EXPECT_EQ(d[0], 4); 111 | EXPECT_EQ(d[1], 6); 112 | 113 | scale_arr(0.5, d, &d); 114 | EXPECT_EQ(d[0], 2); 115 | EXPECT_EQ(d[1], 3); 116 | 117 | scale_arr(2, 2, &d[0], &d[0]); 118 | EXPECT_EQ(d[0], 4); 119 | EXPECT_EQ(d[1], 6); 120 | } 121 | 122 | TYPED_TEST(ArrayMathTest, sum_arr) { 123 | Array d; 124 | d.init(1, 2, 1, 1); 125 | d[0] = -1; 126 | d[1] = 3; 127 | auto r = sum_arr(d); 128 | EXPECT_EQ(r, 2); 129 | } 130 | 131 | TYPED_TEST(ArrayMathTest, sum_arr2) { 132 | Array d; 133 | d.init(1, 2, 1, 1); 134 | d[0] = -2; 135 | d[1] = 3; 136 | auto r = sum_arr(d.total_, &d[0]); 137 | EXPECT_EQ(r, 1); 138 | } 139 | 140 | TYPED_TEST(ArrayMathTest, sum_squared_arr) { 141 | Array d; 142 | d.init(1, 2, 1, 1); 143 | d[0] = -1; 144 | d[1] = 3; 145 | auto r = sum_squared_arr(2, &d[0]); 146 | EXPECT_EQ(r, 10); 147 | } 148 | 149 | TYPED_TEST(ArrayMathTest, sub_scalar) { 150 | Array d; 151 | d.init(1, 2, 1, 1); 152 | d[0] = 10; 153 | d[1] = 5; 154 | 155 | sub_scalar(2, d, &d); 156 | EXPECT_EQ(d[0], 8); 157 | EXPECT_EQ(d[1], 3); 158 | } 159 | 160 | TYPED_TEST(ArrayMathTest, sub_scalar2) { 161 | Array d; 162 | d.init(1, 2, 1, 1); 163 | d[0] = 10; 164 | d[1] = 5; 165 | 166 | sub_scalar(d.total_, 3, &d[0], &d[0]); 167 | EXPECT_EQ(d[0], 7); 168 | EXPECT_EQ(d[1], 2); 169 | } 170 | 171 | } // namespace cnn 172 | -------------------------------------------------------------------------------- /cnn/autodiff/BUILD.bazel: -------------------------------------------------------------------------------- 1 | # Copyright 2019. All Rights Reserved. 2 | # Author: csukuangfj@gmail.com (Fangjun Kuang) 3 | 4 | package(default_visibility = ["//visibility:public"]) 5 | 6 | load("//tools/lint:cpplint.bzl", "cpplint") 7 | 8 | cc_library( 9 | name = "jet", 10 | hdrs = ["jet.h"], 11 | ) 12 | 13 | cc_test( 14 | name = "test_jet", 15 | srcs = ["test_jet.cc"], 16 | deps = [ 17 | ":jet", 18 | "@com_google_googletest//:gtest_main", 19 | ], 20 | ) 21 | 22 | # TODO(fangjun): it fails for bazel version > 0.26.0 23 | # cpplint() 24 | -------------------------------------------------------------------------------- /cnn/proto/BUILD.bazel: -------------------------------------------------------------------------------- 1 | # Copyright 2019. All Rights Reserved. 2 | # Author: csukuangfj@gmail.com (Fangjun Kuang) 3 | 4 | package(default_visibility = ["//visibility:public"]) 5 | 6 | load("@rules_proto//proto:defs.bzl", "proto_library") 7 | 8 | cc_proto_library( 9 | name = "cnn_cc_proto", 10 | deps = [":cnn_proto"], 11 | ) 12 | 13 | proto_library( 14 | name = "cnn_proto", 15 | srcs = ["cnn.proto"], 16 | ) 17 | -------------------------------------------------------------------------------- /cnn/proto/cnn.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2019. All Rights Reserved. 2 | // Author: csukuangfj@gmail.com (Fangjun Kuang) 3 | 4 | syntax = "proto2"; 5 | 6 | package cnn; 7 | 8 | message ArrayProto 9 | { 10 | optional int32 n = 1; // number of batches 11 | optional int32 c = 2; // number of channels 12 | optional int32 h = 3; // number of rows, i.e., height 13 | optional int32 w = 4; // number of columns, i.e., width 14 | repeated double d = 5; // data 15 | } 16 | 17 | message OptimizerProto 18 | { 19 | optional string model_filename = 1; 20 | optional double learning_rate = 2; 21 | optional int32 max_iteration_num = 3; // maximum number of iterations 22 | optional int32 print_interval = 4 [default = 1000]; // print after every multiple of this number of iterations 23 | optional string trained_filename = 5; // a binary file; if present, copy weights from it 24 | optional int32 snapshot_interval = 6; // save trained weights after every multiple of this number of iterations 25 | optional string snapshot_prefix = 7; // prefix of the snapshot 26 | } 27 | 28 | message NetworkProto 29 | { 30 | repeated LayerProto layer_proto = 1; 31 | } 32 | 33 | enum LayerType 34 | { 35 | INPUT = 0; // input layer 36 | FULL_CONNECTED = 1; // full connected layer 37 | L2_LOSS = 2; // l2 loss layer, for regression only 38 | SOFTMAX = 3; 39 | LOG_LOSS = 4; 40 | SOFTMAX_WITH_LOG_LOSS = 5; 41 | CONVOLUTION = 6; 42 | RELU = 7; 43 | MAX_POOLING = 8; 44 | DROP_OUT = 9; 45 | BATCH_NORMALIZATION = 10; 46 | LEAKY_RELU = 11; 47 | } 48 | 49 | // same as caffe 50 | enum Phase 51 | { 52 | TRAIN = 0; // during the train phase, we have to allocate gradient 53 | TEST = 1; // during the test case, no gradient is needed 54 | } 55 | 56 | message LayerProto 57 | { 58 | optional string name = 1; // can be arbitrary and may be not unique, for debug only 59 | optional LayerType type = 2; 60 | repeated string bottom = 3; // bottom/top name are globally unique 61 | repeated string top = 4; // bottom/top name are globally unique 62 | optional Phase phase = 5; // train or test, used from gradient space allocation 63 | repeated ArrayProto param = 6; // parameters for this layer 64 | 65 | optional InputLayerProto input_proto = 7; 66 | optional FullConnectedLayerProto fc_proto = 8; 67 | optional ConvolutionLayerProto conv_proto = 9; 68 | optional MaxPoolingLayerProto max_pooling_proto = 10; 69 | optional DropoutLayerProto dropout_proto = 11; 70 | optional BatchNormalizationLayerProto batch_normalization_proto = 12; 71 | optional LeakyReLULayerProto leaky_relu_proto = 13; 72 | } 73 | 74 | message InputLayerProto 75 | { 76 | optional int32 n = 1; 77 | optional int32 c = 2; 78 | optional int32 h = 3; 79 | optional int32 w = 4; 80 | } 81 | 82 | message FullConnectedLayerProto 83 | { 84 | optional int32 num_output = 1; // number of outputs 85 | } 86 | 87 | message ConvolutionLayerProto 88 | { 89 | optional int32 num_output = 1; 90 | optional int32 kernel_size = 2; // size of the square kernel 91 | // currently we assume implicit padding with stride 1 so 92 | // that the output size equals to the input size 93 | } 94 | 95 | message MaxPoolingLayerProto 96 | { 97 | optional int32 win_size = 1; // size of a square window 98 | optional int32 stride = 2; // stride of the window 99 | } 100 | 101 | message DropoutLayerProto 102 | { 103 | optional double keep_prob = 1; // the probability to retain the output 104 | } 105 | 106 | message BatchNormalizationLayerProto 107 | { 108 | // moving_average = momentum * moving_average + (1 - momentum) * mini_batch_average 109 | optional double momentum = 1 [default = 0.99]; 110 | } 111 | 112 | 113 | message LeakyReLULayerProto 114 | { 115 | // when input is less than 0, it returns alpha*input 116 | // tensorflow uses default value 0.2 117 | // keras uses 0.3 118 | // pytorch uses 0.01, which is the same as in the paper 119 | optional double alpha = 1 [default = 0.01]; 120 | } 121 | -------------------------------------------------------------------------------- /cnn/readme.md: -------------------------------------------------------------------------------- 1 | ../readme.md -------------------------------------------------------------------------------- /cnn/workspace.bzl: -------------------------------------------------------------------------------- 1 | # Copyright 2019. All Rights Reserved. 2 | # Author: csukuangfj@gmail.com (Fangjun Kuang) 3 | 4 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 5 | 6 | def cnn_repositories(): 7 | bazel_federation_commit_id = "130c84ec6d60f31b711400e8445a8d0d4a2b5de8" 8 | bazel_federation_sha256 = "9d4fdf7cc533af0b50f7dd8e58bea85df3b4454b7ae00056d7090eb98e3515cc" 9 | http_archive( 10 | name = "bazel_federation", 11 | sha256 = bazel_federation_sha256, 12 | strip_prefix = "bazel-federation-" + bazel_federation_commit_id, 13 | url = "https://github.com/bazelbuild/bazel-federation/archive/{}.zip".format(bazel_federation_commit_id), 14 | ) 15 | 16 | bazel_skylib_commit_id = "f130d7c388e6beeb77309ba4e421c8f783b91739" 17 | bazel_skylib_sha256 = "8eb5bce85cddd2f4e5232c94e57799de62b1671ce4f79f0f83e10e2d3b2e7986" 18 | http_archive( 19 | name = "bazel_skylib", 20 | sha256 = bazel_skylib_sha256, 21 | strip_prefix = "bazel-skylib-" + bazel_skylib_commit_id, 22 | url = "https://github.com/bazelbuild/bazel-skylib/archive/{}.zip".format(bazel_skylib_commit_id), 23 | ) 24 | 25 | http_archive( 26 | name = "com_google_googletest", 27 | sha256 = "9dc9157a9a1551ec7a7e43daea9a694a0bb5fb8bec81235d8a1e6ef64c716dcb", 28 | strip_prefix = "googletest-release-1.10.0", 29 | urls = ["https://github.com/google/googletest/archive/release-1.10.0.tar.gz"], 30 | ) 31 | 32 | google_style_guide_commit_id = "83a9e8d7ca3d47239cb0a7bf532de791e6df3ba6" 33 | google_style_guide_sha256 = "7ff8e886c1f48754e083137300a843896305fb0677e606f5abe234d3e68f70ee" 34 | http_archive( 35 | name = "google_style_guide", 36 | build_file = "//third_party:google_style_guide.BUILD", 37 | sha256 = google_style_guide_sha256, 38 | strip_prefix = "styleguide-" + google_style_guide_commit_id, 39 | urls = ["https://github.com/google/styleguide/archive/{}.tar.gz".format(google_style_guide_commit_id)], 40 | ) 41 | 42 | http_archive( 43 | name = "com_google_glog", 44 | patch_cmds = ["sed -i -e 's/glog_library()/glog_library(with_gflags=0)/' BUILD"], 45 | sha256 = "f28359aeba12f30d73d9e4711ef356dc842886968112162bc73002645139c39c", 46 | strip_prefix = "glog-0.4.0", 47 | urls = ["https://github.com/google/glog/archive/v0.4.0.tar.gz"], 48 | ) 49 | 50 | http_archive( 51 | name = "com_google_protobuf", 52 | sha256 = "e82ee5bdde198e0a1935e280748a86a7989474ea771418a2fd90f03e2e65b99b", 53 | strip_prefix = "protobuf-3.10.1", 54 | urls = [ 55 | "https://github.com/protocolbuffers/protobuf/releases/download/v3.10.1/protobuf-cpp-3.10.1.tar.gz", 56 | ], 57 | ) 58 | 59 | rules_proto_commit_id = "97d8af4dc474595af3900dd85cb3a29ad28cc313" 60 | rules_proto_sha256 = "602e7161d9195e50246177e7c55b2f39950a9cf7366f74ed5f22fd45750cd208" 61 | http_archive( 62 | name = "rules_proto", 63 | sha256 = rules_proto_sha256, 64 | strip_prefix = "rules_proto-" + rules_proto_commit_id, 65 | urls = [ 66 | "https://github.com/bazelbuild/rules_proto/archive/{}.tar.gz".format(rules_proto_commit_id), 67 | ], 68 | ) 69 | -------------------------------------------------------------------------------- /doc/run-doxygen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019. All Rights Reserved. 4 | # Author: fangjun.kuang@gmail.com (Fangjun Kuang) 5 | 6 | GA_ID=UA-152706513-1 7 | read -d "" ga_str << EOF 8 | 9 | 10 | 16 | EOF 17 | 18 | ga_str=$(echo -n $ga_str) # skip new line 19 | 20 | doxygen -w html header.html footer.html abc.css 21 | 22 | sed -e "s#\(^.*\)#$ga_str\1#" \ 23 | footer.html > footer-ga.html 24 | 25 | sed \ 26 | -e "s/^HTML_FOOTER.*=.*/HTML_FOOTER = footer-ga.html/" \ 27 | Doxyfile > Doxyfile-bak 28 | 29 | rm -rf ./doxygen 30 | 31 | doxygen Doxyfile-bak 32 | 33 | rm Doxyfile-bak header.html footer.html abc.css footer-ga.html 34 | -------------------------------------------------------------------------------- /examples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | add_subdirectory(linear_regression) 3 | add_subdirectory(logistic_regression) 4 | add_subdirectory(mnist) 5 | -------------------------------------------------------------------------------- /examples/linear_regression/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | add_executable( 3 | linear_regression 4 | main.cpp 5 | ) 6 | target_link_libraries( 7 | linear_regression 8 | core 9 | ) 10 | -------------------------------------------------------------------------------- /examples/linear_regression/main.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | #include "cnn/optimizer.hpp" 22 | 23 | template 24 | void data_callback(const std::vector*>& top) { 25 | // y = 5 + 10*x 26 | static std::vector, Dtype>> data{ 27 | {{11}, 115}, {{-14}, -135}, {{15}, 155}, {{6}, 65}, {{-18}, -175}, 28 | {{-8}, -75}, {{9}, 95}, {{-4}, -35}, {{18}, 185}, {{-1}, -5}, 29 | }; 30 | 31 | static int k = 0; 32 | 33 | int n = top[0]->n_; 34 | 35 | CHECK_LE(n, data.size()) 36 | << "the batch size cannot be larger than the dataset size"; 37 | 38 | int stride = top[0]->total_ / n; 39 | CHECK_EQ(stride, data[0].first.size()); 40 | 41 | for (int i = 0; i < n; i++) { 42 | if (k >= data.size()) { 43 | k = 0; 44 | } 45 | 46 | for (int j = 0; j < stride; j++) { 47 | top[0]->d_[i * stride + j] = (data[k].first)[j]; 48 | } 49 | 50 | if (top.size() == 2) { 51 | top[1]->d_[i] = data[k].second; 52 | } 53 | 54 | k++; 55 | } 56 | } 57 | 58 | int main(int /*argc*/, char* argv[]) { 59 | google::InitGoogleLogging(argv[0]); 60 | FLAGS_alsologtostderr = true; 61 | FLAGS_colorlogtostderr = true; 62 | 63 | std::string filename = "../examples/linear_regression/optimizer.prototxt"; 64 | 65 | cnn::Optimizer opt(filename); 66 | opt.register_data_callback(data_callback); 67 | opt.start_training(); 68 | 69 | return 0; 70 | } 71 | -------------------------------------------------------------------------------- /examples/linear_regression/model.prototxt: -------------------------------------------------------------------------------- 1 | 2 | layer_proto { 3 | name: "input" 4 | type: INPUT 5 | top: "data" 6 | top: "label" 7 | input_proto { 8 | n: 5 9 | c: 1 10 | h: 1 11 | w: 1 12 | } 13 | } 14 | 15 | layer_proto { 16 | name: "fc1" 17 | type: FULL_CONNECTED 18 | bottom: "data" 19 | top: "fc1" 20 | fc_proto { 21 | num_output: 1 22 | } 23 | } 24 | 25 | layer_proto { 26 | name: "l2_loss" 27 | type: L2_LOSS 28 | bottom: "fc1" 29 | bottom: "label" 30 | top: "loss" 31 | } 32 | -------------------------------------------------------------------------------- /examples/linear_regression/optimizer.prototxt: -------------------------------------------------------------------------------- 1 | 2 | model_filename: "../examples/linear_regression/model.prototxt" 3 | learning_rate: 0.01 4 | max_iteration_num: 400 5 | -------------------------------------------------------------------------------- /examples/logistic_regression/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | add_executable( 3 | logistic_regression 4 | main.cpp 5 | ) 6 | target_link_libraries( 7 | logistic_regression 8 | core 9 | ) 10 | -------------------------------------------------------------------------------- /examples/logistic_regression/data/ex2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csukuangfj/OpenCNN/561936740226b6741e75237a68afd6bda6df4f7d/examples/logistic_regression/data/ex2.pdf -------------------------------------------------------------------------------- /examples/logistic_regression/data/ex2data1.txt: -------------------------------------------------------------------------------- 1 | 34.62365962451697,78.0246928153624,0 2 | 30.28671076822607,43.89499752400101,0 3 | 35.84740876993872,72.90219802708364,0 4 | 60.18259938620976,86.30855209546826,1 5 | 79.0327360507101,75.3443764369103,1 6 | 45.08327747668339,56.3163717815305,0 7 | 61.10666453684766,96.51142588489624,1 8 | 75.02474556738889,46.55401354116538,1 9 | 76.09878670226257,87.42056971926803,1 10 | 84.43281996120035,43.53339331072109,1 11 | 95.86155507093572,38.22527805795094,0 12 | 75.01365838958247,30.60326323428011,0 13 | 82.30705337399482,76.48196330235604,1 14 | 69.36458875970939,97.71869196188608,1 15 | 39.53833914367223,76.03681085115882,0 16 | 53.9710521485623,89.20735013750205,1 17 | 69.07014406283025,52.74046973016765,1 18 | 67.94685547711617,46.67857410673128,0 19 | 70.66150955499435,92.92713789364831,1 20 | 76.97878372747498,47.57596364975532,1 21 | 67.37202754570876,42.83843832029179,0 22 | 89.67677575072079,65.79936592745237,1 23 | 50.534788289883,48.85581152764205,0 24 | 34.21206097786789,44.20952859866288,0 25 | 77.9240914545704,68.9723599933059,1 26 | 62.27101367004632,69.95445795447587,1 27 | 80.1901807509566,44.82162893218353,1 28 | 93.114388797442,38.80067033713209,0 29 | 61.83020602312595,50.25610789244621,0 30 | 38.78580379679423,64.99568095539578,0 31 | 61.379289447425,72.80788731317097,1 32 | 85.40451939411645,57.05198397627122,1 33 | 52.10797973193984,63.12762376881715,0 34 | 52.04540476831827,69.43286012045222,1 35 | 40.23689373545111,71.16774802184875,0 36 | 54.63510555424817,52.21388588061123,0 37 | 33.91550010906887,98.86943574220611,0 38 | 64.17698887494485,80.90806058670817,1 39 | 74.78925295941542,41.57341522824434,0 40 | 34.1836400264419,75.2377203360134,0 41 | 83.90239366249155,56.30804621605327,1 42 | 51.54772026906181,46.85629026349976,0 43 | 94.44336776917852,65.56892160559052,1 44 | 82.36875375713919,40.61825515970618,0 45 | 51.04775177128865,45.82270145776001,0 46 | 62.22267576120188,52.06099194836679,0 47 | 77.19303492601364,70.45820000180959,1 48 | 97.77159928000232,86.7278223300282,1 49 | 62.07306379667647,96.76882412413983,1 50 | 91.56497449807442,88.69629254546599,1 51 | 79.94481794066932,74.16311935043758,1 52 | 99.2725269292572,60.99903099844988,1 53 | 90.54671411399852,43.39060180650027,1 54 | 34.52451385320009,60.39634245837173,0 55 | 50.2864961189907,49.80453881323059,0 56 | 49.58667721632031,59.80895099453265,0 57 | 97.64563396007767,68.86157272420604,1 58 | 32.57720016809309,95.59854761387875,0 59 | 74.24869136721598,69.82457122657193,1 60 | 71.79646205863379,78.45356224515052,1 61 | 75.3956114656803,85.75993667331619,1 62 | 35.28611281526193,47.02051394723416,0 63 | 56.25381749711624,39.26147251058019,0 64 | 30.05882244669796,49.59297386723685,0 65 | 44.66826172480893,66.45008614558913,0 66 | 66.56089447242954,41.09209807936973,0 67 | 40.45755098375164,97.53518548909936,1 68 | 49.07256321908844,51.88321182073966,0 69 | 80.27957401466998,92.11606081344084,1 70 | 66.74671856944039,60.99139402740988,1 71 | 32.72283304060323,43.30717306430063,0 72 | 64.0393204150601,78.03168802018232,1 73 | 72.34649422579923,96.22759296761404,1 74 | 60.45788573918959,73.09499809758037,1 75 | 58.84095621726802,75.85844831279042,1 76 | 99.82785779692128,72.36925193383885,1 77 | 47.26426910848174,88.47586499559782,1 78 | 50.45815980285988,75.80985952982456,1 79 | 60.45555629271532,42.50840943572217,0 80 | 82.22666157785568,42.71987853716458,0 81 | 88.9138964166533,69.80378889835472,1 82 | 94.83450672430196,45.69430680250754,1 83 | 67.31925746917527,66.58935317747915,1 84 | 57.23870631569862,59.51428198012956,1 85 | 80.36675600171273,90.96014789746954,1 86 | 68.46852178591112,85.59430710452014,1 87 | 42.0754545384731,78.84478600148043,0 88 | 75.47770200533905,90.42453899753964,1 89 | 78.63542434898018,96.64742716885644,1 90 | 52.34800398794107,60.76950525602592,0 91 | 94.09433112516793,77.15910509073893,1 92 | 90.44855097096364,87.50879176484702,1 93 | 55.48216114069585,35.57070347228866,0 94 | 74.49269241843041,84.84513684930135,1 95 | 89.84580670720979,45.35828361091658,1 96 | 83.48916274498238,48.38028579728175,1 97 | 42.2617008099817,87.10385094025457,1 98 | 99.31500880510394,68.77540947206617,1 99 | 55.34001756003703,64.9319380069486,1 100 | 74.77589300092767,89.52981289513276,1 101 | -------------------------------------------------------------------------------- /examples/logistic_regression/data/ex2data2.txt: -------------------------------------------------------------------------------- 1 | 0.051267,0.69956,1 2 | -0.092742,0.68494,1 3 | -0.21371,0.69225,1 4 | -0.375,0.50219,1 5 | -0.51325,0.46564,1 6 | -0.52477,0.2098,1 7 | -0.39804,0.034357,1 8 | -0.30588,-0.19225,1 9 | 0.016705,-0.40424,1 10 | 0.13191,-0.51389,1 11 | 0.38537,-0.56506,1 12 | 0.52938,-0.5212,1 13 | 0.63882,-0.24342,1 14 | 0.73675,-0.18494,1 15 | 0.54666,0.48757,1 16 | 0.322,0.5826,1 17 | 0.16647,0.53874,1 18 | -0.046659,0.81652,1 19 | -0.17339,0.69956,1 20 | -0.47869,0.63377,1 21 | -0.60541,0.59722,1 22 | -0.62846,0.33406,1 23 | -0.59389,0.005117,1 24 | -0.42108,-0.27266,1 25 | -0.11578,-0.39693,1 26 | 0.20104,-0.60161,1 27 | 0.46601,-0.53582,1 28 | 0.67339,-0.53582,1 29 | -0.13882,0.54605,1 30 | -0.29435,0.77997,1 31 | -0.26555,0.96272,1 32 | -0.16187,0.8019,1 33 | -0.17339,0.64839,1 34 | -0.28283,0.47295,1 35 | -0.36348,0.31213,1 36 | -0.30012,0.027047,1 37 | -0.23675,-0.21418,1 38 | -0.06394,-0.18494,1 39 | 0.062788,-0.16301,1 40 | 0.22984,-0.41155,1 41 | 0.2932,-0.2288,1 42 | 0.48329,-0.18494,1 43 | 0.64459,-0.14108,1 44 | 0.46025,0.012427,1 45 | 0.6273,0.15863,1 46 | 0.57546,0.26827,1 47 | 0.72523,0.44371,1 48 | 0.22408,0.52412,1 49 | 0.44297,0.67032,1 50 | 0.322,0.69225,1 51 | 0.13767,0.57529,1 52 | -0.0063364,0.39985,1 53 | -0.092742,0.55336,1 54 | -0.20795,0.35599,1 55 | -0.20795,0.17325,1 56 | -0.43836,0.21711,1 57 | -0.21947,-0.016813,1 58 | -0.13882,-0.27266,1 59 | 0.18376,0.93348,0 60 | 0.22408,0.77997,0 61 | 0.29896,0.61915,0 62 | 0.50634,0.75804,0 63 | 0.61578,0.7288,0 64 | 0.60426,0.59722,0 65 | 0.76555,0.50219,0 66 | 0.92684,0.3633,0 67 | 0.82316,0.27558,0 68 | 0.96141,0.085526,0 69 | 0.93836,0.012427,0 70 | 0.86348,-0.082602,0 71 | 0.89804,-0.20687,0 72 | 0.85196,-0.36769,0 73 | 0.82892,-0.5212,0 74 | 0.79435,-0.55775,0 75 | 0.59274,-0.7405,0 76 | 0.51786,-0.5943,0 77 | 0.46601,-0.41886,0 78 | 0.35081,-0.57968,0 79 | 0.28744,-0.76974,0 80 | 0.085829,-0.75512,0 81 | 0.14919,-0.57968,0 82 | -0.13306,-0.4481,0 83 | -0.40956,-0.41155,0 84 | -0.39228,-0.25804,0 85 | -0.74366,-0.25804,0 86 | -0.69758,0.041667,0 87 | -0.75518,0.2902,0 88 | -0.69758,0.68494,0 89 | -0.4038,0.70687,0 90 | -0.38076,0.91886,0 91 | -0.50749,0.90424,0 92 | -0.54781,0.70687,0 93 | 0.10311,0.77997,0 94 | 0.057028,0.91886,0 95 | -0.10426,0.99196,0 96 | -0.081221,1.1089,0 97 | 0.28744,1.087,0 98 | 0.39689,0.82383,0 99 | 0.63882,0.88962,0 100 | 0.82316,0.66301,0 101 | 0.67339,0.64108,0 102 | 1.0709,0.10015,0 103 | -0.046659,-0.57968,0 104 | -0.23675,-0.63816,0 105 | -0.15035,-0.36769,0 106 | -0.49021,-0.3019,0 107 | -0.46717,-0.13377,0 108 | -0.28859,-0.060673,0 109 | -0.61118,-0.067982,0 110 | -0.66302,-0.21418,0 111 | -0.59965,-0.41886,0 112 | -0.72638,-0.082602,0 113 | -0.83007,0.31213,0 114 | -0.72062,0.53874,0 115 | -0.59389,0.49488,0 116 | -0.48445,0.99927,0 117 | -0.0063364,0.99927,0 118 | 0.63265,-0.030612,0 119 | -------------------------------------------------------------------------------- /examples/logistic_regression/data/gnuplot-notes.md: -------------------------------------------------------------------------------- 1 | 2 | To view available point types, type `test` in the command line. 3 | -------------------------------------------------------------------------------- /examples/logistic_regression/data/plot1.gp: -------------------------------------------------------------------------------- 1 | 2 | 3 | # refer to 4 | # https://stackoverflow.com/questions/9082807/gnuplot-plotting-points-with-color-based-values-in-one-column 5 | 6 | w01 = 0.176031 7 | w02 = -0.0095 8 | w11 = 0.19624 9 | w12 = 0.018 10 | 11 | w00 = 1.414 12 | w10 = 0.020 13 | 14 | f(x) =- ((w11 - w01)*x + (w10 - w00)) / (w12 - w02) 15 | 16 | #set xrange [30:100] 17 | #set yrange [30:100] 18 | set datafile separator "," 19 | plot "< awk -F ',' '{if($3 == \"1\") print}' ex2data1.txt" u 1:2 t "Admitted" w p pt 7 ps 1, \ 20 | "< awk -F ',' '{if($3 == \"0\") print}' ex2data1.txt" u 1:2 t "Not admitted" w p pt 5 ps 1, f(x); 21 | -------------------------------------------------------------------------------- /examples/logistic_regression/data/plot2.gp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csukuangfj/OpenCNN/561936740226b6741e75237a68afd6bda6df4f7d/examples/logistic_regression/data/plot2.gp -------------------------------------------------------------------------------- /examples/logistic_regression/main.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "cnn/optimizer.hpp" 23 | 24 | namespace { 25 | std::vector, double>> g_data; 26 | 27 | void load_data(const std::string& filename) { 28 | std::ifstream file(filename, std::ios::in); 29 | if (!file.is_open()) { 30 | LOG(FATAL) << "cannot open " << filename; 31 | } 32 | 33 | // we omit sanity check in the following 34 | std::string line; 35 | while (std::getline(file, line)) { 36 | std::stringstream ss(line); 37 | double s1, s2, label; 38 | 39 | ss >> s1; 40 | ss.ignore(); 41 | ss >> s2; 42 | ss.ignore(); 43 | ss >> label; 44 | ss.ignore(); 45 | 46 | g_data.push_back({{s1, s2}, label}); 47 | } 48 | } 49 | 50 | template 51 | void data_callback(const std::vector*>& top) { 52 | static int k = 0; 53 | int n = top[0]->n_; 54 | CHECK_LE(n, g_data.size()) 55 | << "the batch size cannot be larger than the dataset size"; 56 | 57 | int stride = top[0]->total_ / n; 58 | CHECK_EQ(stride, g_data[0].first.size()); 59 | 60 | for (int i = 0; i < n; i++) { 61 | if (k >= g_data.size()) { 62 | k = 0; 63 | } 64 | 65 | for (int j = 0; j < stride; j++) { 66 | top[0]->d_[i * stride + j] = (g_data[k].first)[j]; 67 | } 68 | 69 | if (top.size() == 2) { 70 | top[1]->d_[i] = g_data[k].second; 71 | } 72 | 73 | k++; 74 | } 75 | } 76 | 77 | } // namespace 78 | 79 | int main(int /*argc*/, char* argv[]) { 80 | google::InitGoogleLogging(argv[0]); 81 | FLAGS_alsologtostderr = true; 82 | FLAGS_colorlogtostderr = true; 83 | 84 | load_data("../examples/logistic_regression/data/ex2data1.txt"); 85 | CHECK_EQ(g_data.size(), 100) << "the file should contain 100 lines!"; 86 | 87 | std::string filename = "../examples/logistic_regression/trained.prototxt"; 88 | cnn::Network network(filename); 89 | network.reshape(); 90 | 91 | std::ostringstream ss; 92 | ss << "\n"; 93 | int correct = 0; 94 | int total = 0; 95 | 96 | network.get_data_top_mutable(0)[0]->d_[0] = 45; 97 | network.get_data_top_mutable(0)[0]->d_[1] = 85; 98 | 99 | network.perform_predication(); 100 | auto predictions = network.get_predications(); 101 | 102 | LOG(INFO) << "0: " << predictions[0]; 103 | LOG(INFO) << "1: " << predictions[1]; 104 | 105 | #if 1 106 | for (int i = 0; i < g_data.size(); i++) 107 | // for (int i = 0; i < 10; i++) 108 | { 109 | for (int j = 0; j < g_data[i].first.size(); j++) { 110 | network.get_data_top_mutable(0)[0]->d_[j] = g_data[i].first[j]; 111 | } 112 | network.perform_predication(); 113 | 114 | auto predictions = network.get_predications(); 115 | int res = predictions[1] >= predictions[0]; 116 | 117 | ss << "actual: " << res << "\n"; 118 | ss << "expected: " << g_data[i].second; 119 | total++; 120 | correct += res == int(g_data[i].second); 121 | 122 | ss << "\n\n"; 123 | } 124 | 125 | LOG(INFO) << ss.str(); 126 | LOG(INFO) << "accuracy: " << (float)correct / total; 127 | 128 | network.get_data_top_mutable(0)[0]->d_[0] = 45; 129 | network.get_data_top_mutable(0)[0]->d_[1] = 85; 130 | 131 | network.perform_predication(); 132 | predictions = network.get_predications(); 133 | 134 | LOG(INFO) << "0: " << predictions[0]; 135 | LOG(INFO) << "1: " << predictions[1]; 136 | #else 137 | std::string filename = "../examples/logistic_regression/optimizer.prototxt"; 138 | 139 | cnn::Optimizer opt(filename); 140 | opt.register_data_callback(data_callback); 141 | opt.start_training(); 142 | #endif 143 | 144 | return 0; 145 | } 146 | -------------------------------------------------------------------------------- /examples/logistic_regression/model.prototxt: -------------------------------------------------------------------------------- 1 | 2 | layer_proto { 3 | name: "input" 4 | type: INPUT 5 | top: "data" 6 | top: "label" 7 | input_proto { 8 | n: 100 9 | c: 1 10 | h: 1 11 | w: 2 12 | } 13 | } 14 | 15 | layer_proto { 16 | name: "fc1" 17 | type: FULL_CONNECTED 18 | bottom: "data" 19 | top: "fc1" 20 | fc_proto { 21 | num_output: 2 22 | } 23 | } 24 | 25 | layer_proto { 26 | name: "softmax_with_loss1" 27 | type: SOFTMAX_WITH_LOG_LOSS 28 | bottom: "fc1" 29 | bottom: "label" 30 | top: "loss" 31 | } 32 | 33 | -------------------------------------------------------------------------------- /examples/logistic_regression/optimizer.prototxt: -------------------------------------------------------------------------------- 1 | 2 | model_filename: "../examples/logistic_regression/model.prototxt" 3 | learning_rate: 0.0001 4 | max_iteration_num: 3000 5 | print_num: 1 6 | -------------------------------------------------------------------------------- /examples/logistic_regression/trained.prototxt: -------------------------------------------------------------------------------- 1 | layer_proto { 2 | name: "input" 3 | type: INPUT 4 | top: "data" 5 | top: "label" 6 | input_proto { 7 | n: 100 8 | c: 1 9 | h: 1 10 | w: 2 11 | } 12 | } 13 | layer_proto { 14 | name: "fc1" 15 | type: FULL_CONNECTED 16 | bottom: "data" 17 | top: "fc1" 18 | param { 19 | n: 1 20 | c: 1 21 | h: 2 22 | w: 2 23 | d: 0.1760312526239447 24 | d: -0.0095169208861774247 25 | d: 0.19624227744853706 26 | d: 0.0018157958571402518 27 | } 28 | param { 29 | n: 1 30 | c: 1 31 | h: 1 32 | w: 2 33 | d: 1.41450272773171 34 | d: 0.020617683211582919 35 | } 36 | fc_proto { 37 | num_output: 2 38 | } 39 | } 40 | layer_proto { 41 | name: "softmax_with_loss1" 42 | type: SOFTMAX 43 | bottom: "fc1" 44 | top: "loss" 45 | } 46 | -------------------------------------------------------------------------------- /examples/mnist/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | add_executable( 3 | mnist 4 | main.cpp 5 | ) 6 | target_link_libraries( 7 | mnist 8 | core 9 | ) 10 | -------------------------------------------------------------------------------- /examples/mnist/data/download_and_unzip_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Download mnist data and unzip them 5 | # 6 | 7 | CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 8 | 9 | base_url="http://yann.lecun.com/exdb/mnist" 10 | 11 | filenames=( 12 | train-images-idx3-ubyte 13 | train-labels-idx1-ubyte 14 | t10k-images-idx3-ubyte 15 | t10k-labels-idx1-ubyte 16 | ) 17 | 18 | function download_and_decompress() 19 | { 20 | filename="${CUR_DIR}/$1" 21 | 22 | if [ -e "${filename}" ]; then 23 | return 24 | fi 25 | 26 | if [ ! -e "${filename}.gz" ]; then 27 | url="${base_url}/$1.gz" 28 | echo "Downloading ${url} to ${filename}.gz" 29 | wget "${url}" -O "${filename}.gz" 30 | fi 31 | gunzip "${filename}.gz" 32 | 33 | } 34 | 35 | for filename in ${filenames[*]} 36 | do 37 | download_and_decompress ${filename} 38 | done 39 | 40 | 41 | -------------------------------------------------------------------------------- /examples/mnist/loss-accuracy-iter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csukuangfj/OpenCNN/561936740226b6741e75237a68afd6bda6df4f7d/examples/mnist/loss-accuracy-iter.png -------------------------------------------------------------------------------- /examples/mnist/model_for_deploy.prototxt: -------------------------------------------------------------------------------- 1 | 2 | layer_proto { 3 | name: "input" 4 | type: INPUT 5 | top: "data" 6 | input_proto { 7 | n: 1 8 | c: 1 9 | h: 28 10 | w: 28 11 | } 12 | } 13 | 14 | layer_proto { 15 | name: "conv1" 16 | type: CONVOLUTION 17 | bottom: "data" 18 | top: "conv1" 19 | conv_proto { 20 | num_output: 32 21 | kernel_size: 3 22 | } 23 | } 24 | 25 | layer_proto { 26 | name: "bn1" 27 | type: BATCH_NORMALIZATION 28 | bottom: "conv1" 29 | top: "bn1" 30 | } 31 | 32 | layer_proto { 33 | name: "relu1" 34 | type: RELU 35 | bottom: "bn1" 36 | top: "relu1" 37 | } 38 | 39 | layer_proto { 40 | name: "conv2" 41 | type: CONVOLUTION 42 | bottom: "relu1" 43 | top: "conv2" 44 | conv_proto { 45 | num_output: 32 46 | kernel_size: 3 47 | } 48 | } 49 | 50 | layer_proto { 51 | name: "bn2" 52 | type: BATCH_NORMALIZATION 53 | bottom: "conv2" 54 | top: "bn2" 55 | } 56 | 57 | layer_proto { 58 | name: "relu2" 59 | type: RELU 60 | bottom: "bn2" 61 | top: "relu2" 62 | } 63 | 64 | layer_proto { 65 | name: "pool1" 66 | type: MAX_POOLING 67 | bottom: "relu2" 68 | top: "pool1" 69 | max_pooling_proto { 70 | win_size: 2 71 | stride: 2 72 | } 73 | } 74 | 75 | layer_proto { 76 | name: "conv3" 77 | type: CONVOLUTION 78 | bottom: "pool1" 79 | top: "conv3" 80 | conv_proto { 81 | num_output: 64 82 | kernel_size: 3 83 | } 84 | } 85 | 86 | layer_proto { 87 | name: "bn3" 88 | type: BATCH_NORMALIZATION 89 | bottom: "conv3" 90 | top: "bn3" 91 | } 92 | 93 | layer_proto { 94 | name: "relu3" 95 | type: RELU 96 | bottom: "bn3" 97 | top: "relu3" 98 | } 99 | 100 | layer_proto { 101 | name: "conv4" 102 | type: CONVOLUTION 103 | bottom: "relu3" 104 | top: "conv4" 105 | conv_proto { 106 | num_output: 64 107 | kernel_size: 3 108 | } 109 | } 110 | 111 | layer_proto { 112 | name: "bn4" 113 | type: BATCH_NORMALIZATION 114 | bottom: "conv4" 115 | top: "bn4" 116 | } 117 | 118 | layer_proto { 119 | name: "relu4" 120 | type: RELU 121 | bottom: "bn4" 122 | top: "relu4" 123 | } 124 | 125 | layer_proto { 126 | name: "pool2" 127 | type: MAX_POOLING 128 | bottom: "relu4" 129 | top: "pool2" 130 | max_pooling_proto { 131 | win_size: 2 132 | stride: 2 133 | } 134 | } 135 | 136 | layer_proto { 137 | name: "fc1" 138 | type: FULL_CONNECTED 139 | bottom: "pool2" 140 | top: "fc1" 141 | fc_proto { 142 | num_output: 512 143 | } 144 | } 145 | 146 | layer_proto { 147 | name: "bn5" 148 | type: BATCH_NORMALIZATION 149 | bottom: "fc1" 150 | top: "bn5" 151 | } 152 | 153 | layer_proto { 154 | name: "relu5" 155 | type: RELU 156 | bottom: "bn5" 157 | top: "relu5" 158 | } 159 | 160 | layer_proto { 161 | name: "dropout1" 162 | type: DROP_OUT 163 | bottom: "relu5" 164 | top: "dropout1" 165 | dropout_proto { 166 | keep_prob: 0.8 167 | } 168 | } 169 | 170 | layer_proto { 171 | name: "fc2" 172 | type: FULL_CONNECTED 173 | bottom: "dropout1" 174 | top: "fc2" 175 | fc_proto { 176 | num_output: 10 177 | } 178 | } 179 | 180 | layer_proto { 181 | name: "softmax" 182 | type: SOFTMAX 183 | bottom: "fc2" 184 | top: "prob" 185 | } 186 | -------------------------------------------------------------------------------- /examples/mnist/model_for_train.prototxt: -------------------------------------------------------------------------------- 1 | 2 | layer_proto { 3 | name: "input" 4 | type: INPUT 5 | top: "data" 6 | top: "label" 7 | input_proto { 8 | n: 10 9 | c: 1 10 | h: 28 11 | w: 28 12 | } 13 | } 14 | 15 | layer_proto { 16 | name: "conv1" 17 | type: CONVOLUTION 18 | bottom: "data" 19 | top: "conv1" 20 | conv_proto { 21 | num_output: 20 22 | kernel_size: 5 23 | } 24 | } 25 | 26 | layer_proto { 27 | name: "pool1" 28 | type: MAX_POOLING 29 | bottom: "conv1" 30 | top: "pool1" 31 | max_pooling_proto { 32 | win_size: 2 33 | stride: 2 34 | } 35 | } 36 | 37 | layer_proto { 38 | name: "conv2" 39 | type: CONVOLUTION 40 | bottom: "pool1" 41 | top: "conv2" 42 | conv_proto { 43 | num_output: 50 44 | kernel_size: 5 45 | } 46 | } 47 | 48 | layer_proto { 49 | name: "pool2" 50 | type: MAX_POOLING 51 | bottom: "conv2" 52 | top: "pool2" 53 | max_pooling_proto { 54 | win_size: 2 55 | stride: 2 56 | } 57 | } 58 | 59 | layer_proto { 60 | name: "fc1" 61 | type: FULL_CONNECTED 62 | bottom: "pool2" 63 | top: "fc1" 64 | fc_proto { 65 | num_output: 500 66 | } 67 | } 68 | 69 | layer_proto { 70 | name: "relu1" 71 | type: RELU 72 | bottom: "fc1" 73 | top: "relu1" 74 | } 75 | 76 | layer_proto { 77 | name: "fc2" 78 | type: FULL_CONNECTED 79 | bottom: "relu1" 80 | top: "fc2" 81 | fc_proto { 82 | num_output: 10 83 | } 84 | } 85 | 86 | layer_proto { 87 | name: "softmax_log_loss" 88 | type: SOFTMAX_WITH_LOG_LOSS 89 | bottom: "fc2" 90 | bottom: "label" 91 | top: "loss" 92 | } 93 | -------------------------------------------------------------------------------- /examples/mnist/model_for_train.prototxt2: -------------------------------------------------------------------------------- 1 | 2 | layer_proto { 3 | name: "input" 4 | type: INPUT 5 | top: "data" 6 | top: "label" 7 | input_proto { 8 | n: 16 9 | c: 1 10 | h: 28 11 | w: 28 12 | } 13 | } 14 | 15 | layer_proto { 16 | name: "conv1" 17 | type: CONVOLUTION 18 | bottom: "data" 19 | top: "conv1" 20 | conv_proto { 21 | num_output: 32 22 | kernel_size: 3 23 | } 24 | } 25 | 26 | layer_proto { 27 | name: "bn1" 28 | type: BATCH_NORMALIZATION 29 | bottom: "conv1" 30 | top: "bn1" 31 | } 32 | 33 | layer_proto { 34 | name: "relu1" 35 | type: RELU 36 | bottom: "bn1" 37 | top: "relu1" 38 | } 39 | 40 | layer_proto { 41 | name: "conv2" 42 | type: CONVOLUTION 43 | bottom: "relu1" 44 | top: "conv2" 45 | conv_proto { 46 | num_output: 32 47 | kernel_size: 3 48 | } 49 | } 50 | 51 | layer_proto { 52 | name: "bn2" 53 | type: BATCH_NORMALIZATION 54 | bottom: "conv2" 55 | top: "bn2" 56 | } 57 | 58 | layer_proto { 59 | name: "relu2" 60 | type: RELU 61 | bottom: "bn2" 62 | top: "relu2" 63 | } 64 | 65 | layer_proto { 66 | name: "pool1" 67 | type: MAX_POOLING 68 | bottom: "relu2" 69 | top: "pool1" 70 | max_pooling_proto { 71 | win_size: 2 72 | stride: 2 73 | } 74 | } 75 | 76 | layer_proto { 77 | name: "conv3" 78 | type: CONVOLUTION 79 | bottom: "pool1" 80 | top: "conv3" 81 | conv_proto { 82 | num_output: 64 83 | kernel_size: 3 84 | } 85 | } 86 | 87 | layer_proto { 88 | name: "bn3" 89 | type: BATCH_NORMALIZATION 90 | bottom: "conv3" 91 | top: "bn3" 92 | } 93 | 94 | layer_proto { 95 | name: "relu3" 96 | type: RELU 97 | bottom: "bn3" 98 | top: "relu3" 99 | } 100 | 101 | layer_proto { 102 | name: "conv4" 103 | type: CONVOLUTION 104 | bottom: "relu3" 105 | top: "conv4" 106 | conv_proto { 107 | num_output: 64 108 | kernel_size: 3 109 | } 110 | } 111 | 112 | layer_proto { 113 | name: "bn4" 114 | type: BATCH_NORMALIZATION 115 | bottom: "conv4" 116 | top: "bn4" 117 | } 118 | 119 | layer_proto { 120 | name: "relu4" 121 | type: RELU 122 | bottom: "bn4" 123 | top: "relu4" 124 | } 125 | 126 | layer_proto { 127 | name: "pool2" 128 | type: MAX_POOLING 129 | bottom: "relu4" 130 | top: "pool2" 131 | max_pooling_proto { 132 | win_size: 2 133 | stride: 2 134 | } 135 | } 136 | 137 | layer_proto { 138 | name: "fc1" 139 | type: FULL_CONNECTED 140 | bottom: "pool2" 141 | top: "fc1" 142 | fc_proto { 143 | num_output: 512 144 | } 145 | } 146 | 147 | layer_proto { 148 | name: "bn5" 149 | type: BATCH_NORMALIZATION 150 | bottom: "fc1" 151 | top: "bn5" 152 | } 153 | 154 | layer_proto { 155 | name: "relu5" 156 | type: RELU 157 | bottom: "bn5" 158 | top: "relu5" 159 | } 160 | 161 | layer_proto { 162 | name: "dropout1" 163 | type: DROP_OUT 164 | bottom: "relu5" 165 | top: "dropout1" 166 | dropout_proto { 167 | keep_prob: 0.8 168 | } 169 | } 170 | 171 | layer_proto { 172 | name: "fc2" 173 | type: FULL_CONNECTED 174 | bottom: "dropout1" 175 | top: "fc2" 176 | fc_proto { 177 | num_output: 10 178 | } 179 | } 180 | 181 | layer_proto { 182 | name: "softmax_log_loss" 183 | type: SOFTMAX_WITH_LOG_LOSS 184 | bottom: "fc2" 185 | bottom: "label" 186 | top: "loss" 187 | } 188 | -------------------------------------------------------------------------------- /examples/mnist/optimizer.prototxt: -------------------------------------------------------------------------------- 1 | 2 | model_filename: "../examples/mnist/model_for_train.prototxt2" 3 | trained_filename: "trained-bin.prototxt" 4 | learning_rate: 0.01 5 | max_iteration_num: 20000 6 | print_interval: 1 7 | snapshot_interval: 500 8 | snapshot_prefix: "mnist-bin.prototxt" 9 | -------------------------------------------------------------------------------- /include/cnn/array.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | #include 19 | 20 | #include "proto/cnn.pb.h" 21 | 22 | namespace cnn { 23 | 24 | template 25 | class Array { 26 | public: 27 | Array(); 28 | ~Array(); 29 | Array(const Array&) = delete; 30 | Array& operator=(const Array&) = delete; 31 | 32 | Array(Array&&); 33 | Array& operator=(Array&&); 34 | 35 | void init(int n, int c, int h, int w); 36 | 37 | template 38 | void init_like(const Array& arr); 39 | 40 | template 41 | bool has_same_shape(const Array& arr) const; 42 | 43 | bool has_same_shape(const std::vector& vec) const; 44 | std::vector shape_vec() const { return {n_, c_, h_, w_}; } 45 | 46 | /** 47 | * Return the element at n*c_*h_*w_ + c*h_*w_ + h*w_ + w, 48 | * i.e., (n*c_ + c)*h_*w_ + h*w_ + w, 49 | * i.e., ((n*c_ + c)*h_ + h)*w_ + w 50 | */ 51 | const Dtype& at(int n, int c, int h, int w) const; 52 | Dtype& at(int n, int c, int h, int w); 53 | 54 | // no range check 55 | const Dtype& operator()(int n, int c, int h, int w) const; 56 | Dtype& operator()(int n, int c, int h, int w); 57 | 58 | const Dtype& operator[](int i) const; 59 | Dtype& operator[](int i); 60 | 61 | std::string shape_info() const; 62 | 63 | int n_; //!< number of batches 64 | int c_; //!< number of channels 65 | int h_; //!< image height, i.e., number of rows 66 | int w_; //!< image width, number of columns 67 | int total_; //!< n_*c_*h_*w_, number of elements 68 | 69 | Dtype* d_; //!< pointer to the data 70 | 71 | public: 72 | void from_proto(const ArrayProto& proto); 73 | void to_proto(ArrayProto* proto) const; 74 | }; 75 | 76 | } // namespace cnn 77 | 78 | #include "../../src/array.cpp" 79 | -------------------------------------------------------------------------------- /include/cnn/batch_normalization_layer.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | 19 | #include "cnn/layer.hpp" 20 | 21 | namespace cnn { 22 | /** 23 | * It has one bottom and one top. 24 | * 25 | * Both bottom[0] and top[0] have shape (N, C, H, W). 26 | * 27 | * Refer to the paper https://arxiv.org/pdf/1502.03167.pdf 28 | * 29 | * Feature maps, i.e., channels, are normalized over batches. 30 | * 31 | * @code 32 | * for c = 0:num_channels-1 33 | * total = 0 34 | * for n = 0:num_batches-1 35 | * add all pixels in channel c, batch n 36 | * accumulate the result in total 37 | * 38 | * average total 39 | * 40 | * for n = 0:num_batches-1 41 | * subtract total from all pixels in channel c, batch n 42 | * 43 | * total = 0 44 | * for n = 0:num_batches-1 45 | * add the square of all pixels in channel c, batch n 46 | * accumulate the result in total 47 | * 48 | * average total 49 | * stddev = sqrt(total + eps) 50 | * for n = 0:num_batches-1 51 | * divided by stddev of all pixels in channel c, batch n 52 | * @endcode 53 | * 54 | * param[0]: channel scale with shape (1, C, 1, 1) 55 | * param[1]: channel bias with shape (1, C, 1, 1) 56 | * param[2]: channel mean with shape (1, C, 1, 1) 57 | * param[3]: channel stddev with shape (1, C, 1, 1) 58 | */ 59 | template 60 | class BatchNormalizationLayer : public Layer { 61 | public: 62 | explicit BatchNormalizationLayer(const LayerProto&); 63 | 64 | void reshape(const std::vector*>& bottom, 65 | const std::vector*>& bottom_gradient, 66 | const std::vector*>& top, 67 | const std::vector*>& top_gradient) override; 68 | 69 | void fprop(const std::vector*>& bottom, 70 | const std::vector*>& top) override; 71 | 72 | void bprop(const std::vector*>& bottom, 73 | const std::vector*>& bottom_gradient, 74 | const std::vector*>& top, 75 | const std::vector*>& top_gradient) override; 76 | 77 | private: 78 | /** avoid dividing by 0 */ 79 | Dtype eps_ = 1e-5; 80 | 81 | /** moving_mean = moving_mean*momentum + mini_batch_mean*(1-momentum) */ 82 | Dtype momentum_; 83 | 84 | // the following variables are used only in the train phase 85 | Array x_minus_mu_; //!< saves x - mini_batch_mean 86 | Array mu_; //!< mini_batch_mean 87 | Array var_; //!< mini_batch_variance 88 | }; 89 | 90 | } // namespace cnn 91 | 92 | #include "../../src/batch_normalization_layer.cpp" 93 | -------------------------------------------------------------------------------- /include/cnn/common.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | 19 | #include "proto/cnn.pb.h" 20 | 21 | // TODO(fangjun): avoid define here! 22 | // we use the same threshold as caffe for computing log() 23 | #define g_log_threshold (1e-20) 24 | -------------------------------------------------------------------------------- /include/cnn/convolution_layer.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | 16 | #pragma once 17 | 18 | #include 19 | 20 | #include "cnn/layer.hpp" 21 | 22 | namespace cnn { 23 | /** 24 | * One input bottom[0] with shape (N, C, H, W) 25 | * and one output top[0] with shape (N, num_output, H, W) 26 | * 27 | */ 28 | template 29 | class ConvolutionLayer : public Layer { 30 | public: 31 | explicit ConvolutionLayer(const LayerProto&); 32 | 33 | void reshape(const std::vector*>& bottom, 34 | const std::vector*>& bottom_gradient, 35 | const std::vector*>& top, 36 | const std::vector*>& top_gradient) override; 37 | 38 | void fprop(const std::vector*>& bottom, 39 | const std::vector*>& top) override; 40 | 41 | void bprop(const std::vector*>& bottom, 42 | const std::vector*>& bottom_gradient, 43 | const std::vector*>& top, 44 | const std::vector*>& top_gradient) override; 45 | 46 | private: 47 | void one_channel_convolution(const Dtype* weight, const Dtype* src, 48 | int height, int width, Dtype* dst); 49 | 50 | void one_channel_bprop(const Dtype* weight, const Dtype* bottom, int height, 51 | int width, const Dtype* top_gradient, 52 | Dtype* bottom_gradient, Dtype* param_gradient); 53 | 54 | private: 55 | int num_output_; 56 | int kernel_size_; 57 | }; 58 | 59 | } // namespace cnn 60 | 61 | #include "../../src/convolution_layer.cpp" 62 | -------------------------------------------------------------------------------- /include/cnn/drop_out_layer.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | 19 | #include "cnn/layer.hpp" 20 | 21 | namespace cnn { 22 | /** 23 | * It has one bottom and one top. 24 | * 25 | * Both bottom[0] and top[0] have shape (N, C, H, W). 26 | * 27 | * During the train phase, 28 | * 29 | * top[0]->d_[i] = bottom[0]->d_[i] * mask_[i] / keep_prob_; 30 | * 31 | * where mask[i] is either 1 or 0. Its probability to be 1 is keep_prob_; 32 | * 33 | * During the test phase, 34 | * 35 | * top[0]->d_[i] = bottom[0]->d_[i]; 36 | * 37 | * We use inverted dropout here. 38 | * 39 | * Refer to 40 | * http://cs231n.github.io/neural-networks-2/#reg 41 | * 42 | * Dropout is short for DROP OUTput. 43 | * A similar word is DropConnect. 44 | */ 45 | template 46 | class DropoutLayer : public Layer { 47 | public: 48 | explicit DropoutLayer(const LayerProto&); 49 | 50 | void reshape(const std::vector*>& bottom, 51 | const std::vector*>& bottom_gradient, 52 | const std::vector*>& top, 53 | const std::vector*>& top_gradient) override; 54 | 55 | void fprop(const std::vector*>& bottom, 56 | const std::vector*>& top) override; 57 | 58 | void bprop(const std::vector*>& bottom, 59 | const std::vector*>& bottom_gradient, 60 | const std::vector*>& top, 61 | const std::vector*>& top_gradient) override; 62 | 63 | private: 64 | Dtype keep_prob_; 65 | Array mask_; 66 | }; 67 | 68 | } // namespace cnn 69 | 70 | #include "../../src/drop_out_layer.cpp" 71 | -------------------------------------------------------------------------------- /include/cnn/full_connected_layer.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | 19 | #include "cnn/layer.hpp" 20 | 21 | namespace cnn { 22 | /** 23 | * It has one input bottom[0] with shape (N, C, H, W) 24 | * and one output top[0] with shape (N, M, 1, 1), 25 | * where M is the number of output provided in the prototxt. 26 | * 27 | * param[0] has the shape (1, 1, M, K) where K equals to C*H*W 28 | * and param[1] has shape (1, 1, 1, M). 29 | * 30 | * param[0] contains weight parameters for inner product 31 | * and param[1] contains corresponding biases. 32 | */ 33 | template 34 | class FullConnectedLayer : public Layer { 35 | public: 36 | explicit FullConnectedLayer(const LayerProto&); 37 | 38 | void reshape(const std::vector*>& bottom, 39 | const std::vector*>& bottom_gradient, 40 | const std::vector*>& top, 41 | const std::vector*>& top_gradient) override; 42 | 43 | void fprop(const std::vector*>& bottom, 44 | const std::vector*>& top) override; 45 | 46 | void bprop(const std::vector*>& bottom, 47 | const std::vector*>& bottom_gradient, 48 | const std::vector*>& top, 49 | const std::vector*>& top_gradient) override; 50 | 51 | private: 52 | int num_output_; 53 | }; 54 | 55 | } // namespace cnn 56 | 57 | #include "../../src/full_connected_layer.cpp" 58 | -------------------------------------------------------------------------------- /include/cnn/input_layer.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | 19 | #include "cnn/layer.hpp" 20 | 21 | namespace cnn { 22 | 23 | template 24 | class InputLayer : public Layer { 25 | public: 26 | explicit InputLayer(const LayerProto&); 27 | 28 | void reshape(const std::vector*>& bottom, 29 | const std::vector*>& bottom_gradient, 30 | const std::vector*>& top, 31 | const std::vector*>& top_gradient) override; 32 | 33 | void fprop(const std::vector*>& input, 34 | const std::vector*>& output) override; 35 | 36 | void bprop(const std::vector*>&, 37 | const std::vector*>&, 38 | const std::vector*>&, 39 | const std::vector*>&) override {} 40 | 41 | private: 42 | int n_; 43 | int c_; 44 | int h_; 45 | int w_; 46 | }; 47 | 48 | } // namespace cnn 49 | 50 | #include "../../src/input_layer.cpp" 51 | -------------------------------------------------------------------------------- /include/cnn/io.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | 16 | #pragma once 17 | 18 | #include 19 | 20 | #include 21 | #include "cnn/array.hpp" 22 | 23 | namespace cnn { 24 | 25 | void read_proto_txt(const std::string& filename, 26 | google::protobuf::Message* message); 27 | 28 | void write_proto_txt(const std::string& filename, 29 | const google::protobuf::Message& message); 30 | 31 | void read_proto_bin(const std::string& filename, 32 | google::protobuf::Message* message); 33 | 34 | void write_proto_bin(const std::string& filename, 35 | const google::protobuf::Message& message); 36 | 37 | void string_to_proto(const std::string& filename, 38 | google::protobuf::Message* message); 39 | 40 | void write_pgm(const std::string& filename, const Array& img); 41 | 42 | void read_pgm(const std::string& filename, Array* img); 43 | 44 | } // namespace cnn 45 | -------------------------------------------------------------------------------- /include/cnn/l2_loss_layer.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | 19 | #include "cnn/layer.hpp" 20 | 21 | namespace cnn { 22 | /** 23 | * For regression only! 24 | * It computes 25 | * 26 | * \frac{1}{n} \sum_{i=0}^{n-1} (\mathrm{bottom}[0][i] - 27 | * \mathrm{bottom}[1][i])^2 28 | * 29 | * bottom[0] is the predication and bottom[1] is the ground truth. 30 | * Both of them have shape (N, C, H, W). 31 | */ 32 | template 33 | class L2LossLayer : public Layer { 34 | public: 35 | explicit L2LossLayer(const LayerProto&); 36 | 37 | /** 38 | * @param bottom bottom[0] is the predication and bottom[1] is the ground 39 | * truth; they must have the same shape 40 | * @param bottom_gradient the gradient for bottom[0]. 41 | * It is allocated only in the train phase 42 | * @param top the average squared loss between bottom[0] and bottom[1] 43 | * @param top_gradient it is not used and we always assume that the top 44 | * gradient is 1 45 | */ 46 | void reshape(const std::vector*>& bottom, 47 | const std::vector*>& bottom_gradient, 48 | const std::vector*>& top, 49 | const std::vector*>& top_gradient) override; 50 | 51 | void fprop(const std::vector*>& bottom, 52 | const std::vector*>& top) override; 53 | 54 | void bprop(const std::vector*>& bottom, 55 | const std::vector*>& bottom_gradient, 56 | const std::vector*>& top, 57 | const std::vector*>& top_gradient) override; 58 | 59 | private: 60 | Dtype loss_; 61 | }; 62 | 63 | } // namespace cnn 64 | 65 | #include "../../src/l2_loss_layer.cpp" 66 | -------------------------------------------------------------------------------- /include/cnn/layer.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | #include 19 | 20 | #include "proto/cnn.pb.h" 21 | 22 | #include "cnn/array.hpp" 23 | #include "cnn/array_math.hpp" 24 | 25 | namespace cnn { 26 | /** 27 | * 28 | * Every layer MUST implement the following functions 29 | * * reshape 30 | * * fprop 31 | * * bprop 32 | */ 33 | template 34 | class Layer { 35 | public: 36 | explicit Layer(const LayerProto&); 37 | static std::shared_ptr> create(const LayerProto&); 38 | 39 | const LayerProto& proto() const { return proto_; } 40 | LayerProto& proto() { return proto_; } 41 | 42 | std::vector*> mutable_param() { 43 | std::vector*> res; 44 | for (int i = 0; i < param_.size(); i++) { 45 | res.push_back(param_[i].get()); 46 | } 47 | return res; 48 | } 49 | 50 | std::vector*> param() const { 51 | std::vector*> res; 52 | for (int i = 0; i < param_.size(); i++) { 53 | res.push_back(param_[i].get()); 54 | } 55 | return res; 56 | } 57 | 58 | std::vector*> mutable_gradient() { 59 | std::vector*> res; 60 | for (int i = 0; i < gradient_.size(); i++) { 61 | res.push_back(gradient_[i].get()); 62 | } 63 | return res; 64 | } 65 | 66 | std::vector*> gradient() const { 67 | std::vector*> res; 68 | for (int i = 0; i < gradient_.size(); i++) { 69 | res.push_back(gradient_[i].get()); 70 | } 71 | return res; 72 | } 73 | 74 | void clear_gradient() { 75 | for (auto& g : gradient_) { 76 | if (g) { 77 | set_to(g.get(), 0); 78 | } 79 | } 80 | } 81 | 82 | void copy_trained_layer(const LayerProto& p); 83 | 84 | void update_parameters(int current_iter, double current_learning_rate); 85 | 86 | /** 87 | * At layer construction, we have no idea of the shape of its inputs, 88 | * so this function MUST be called after constructing the whole network. 89 | */ 90 | virtual void reshape(const std::vector*>& bottom, 91 | const std::vector*>& bottom_gradient, 92 | const std::vector*>& top, 93 | const std::vector*>& top_gradient) = 0; 94 | 95 | /** 96 | * forward propagation 97 | */ 98 | virtual void fprop(const std::vector*>& bottom, 99 | const std::vector*>& top) = 0; 100 | 101 | /** 102 | * backward propagation 103 | */ 104 | virtual void bprop(const std::vector*>& bottom, 105 | const std::vector*>& bottom_gradient, 106 | const std::vector*>& top, 107 | const std::vector*>& top_gradient) = 0; 108 | 109 | protected: 110 | std::vector>> param_; 111 | std::vector>> gradient_; 112 | std::vector>> history_gradient_; 113 | 114 | LayerProto proto_; 115 | 116 | private: 117 | Layer(const Layer&) = delete; 118 | Layer& operator=(const Layer&) = delete; 119 | }; 120 | 121 | } // namespace cnn 122 | 123 | #include "../../src/layer.cpp" 124 | -------------------------------------------------------------------------------- /include/cnn/leaky_relu_layer.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | 19 | #include "cnn/layer.hpp" 20 | 21 | namespace cnn { 22 | /** 23 | * It has one bottom and one top. 24 | * 25 | * Both bottom[0] and top[0] have shape (N, C, H, W). 26 | * 27 | * top[0]->d_[i] = max(0, bottom[0]->d_[i]) + alpha_ * min(0, bottom[0]->d_[i]); 28 | * 29 | * Refer to 30 | * - "Rectifier Nonlinearities Improve Neural Network Acoustic Models" 31 | * - https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf 32 | * 33 | * The purpose of leaky ReLU is to propagate gradient even if 34 | * the input is negative. 35 | * 36 | * The gradient for the negative input is a non-negative constant. 37 | * 38 | */ 39 | template 40 | class LeakyReLULayer : public Layer { 41 | public: 42 | explicit LeakyReLULayer(const LayerProto&); 43 | 44 | void reshape(const std::vector*>& bottom, 45 | const std::vector*>& bottom_gradient, 46 | const std::vector*>& top, 47 | const std::vector*>& top_gradient) override; 48 | 49 | void fprop(const std::vector*>& bottom, 50 | const std::vector*>& top) override; 51 | 52 | void bprop(const std::vector*>& bottom, 53 | const std::vector*>& bottom_gradient, 54 | const std::vector*>& top, 55 | const std::vector*>& top_gradient) override; 56 | 57 | private: 58 | Dtype alpha_; //!< greater than or equal to 0 59 | //!< It cannot be negative, otherwise a negative input 60 | //!< results in a positive output! 61 | }; 62 | 63 | } // namespace cnn 64 | 65 | #include "../../src/leaky_relu_layer.cpp" 66 | -------------------------------------------------------------------------------- /include/cnn/log_loss_layer.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | 19 | #include "cnn/layer.hpp" 20 | 21 | namespace cnn { 22 | /** 23 | * For classification only! 24 | * 25 | * It accepts two inputs: bottom[0] and bottom[1]. 26 | * 27 | * bottom[0] is the prediction with shape (N, C, H, W). 28 | * The meaning of the shape can be interpreted as follows: 29 | * (1) N is the batch size. (2) H and W can be considered 30 | * as the size of an image. (3) C represents the number of channels 31 | * of the image; in the case of multi-class classification, C 32 | * is the number of classes we have and the meaning of each pixel 33 | * in the channel k indicates the probability that this pixel belongs 34 | * to class k. 35 | * 36 | * The user has to ensure that the pixel values are in 37 | * the range of [0, 1] and it is normalized, i.e., sum to 1. 38 | * 39 | * This layer lays usually on top of the softmax layer. 40 | * 41 | * bottom[1] is the ground truth and has the shape (N, 1, H, W). 42 | * values of every element in bottom[1] have to be an integer 43 | * in the range [0, C-1]. 44 | */ 45 | template 46 | class LogLossLayer : public Layer { 47 | public: 48 | explicit LogLossLayer(const LayerProto&); 49 | 50 | void reshape(const std::vector*>& bottom, 51 | const std::vector*>& bottom_gradient, 52 | const std::vector*>& top, 53 | const std::vector*>& top_gradient) override; 54 | 55 | void fprop(const std::vector*>& bottom, 56 | const std::vector*>& top) override; 57 | 58 | void bprop(const std::vector*>& bottom, 59 | const std::vector*>& bottom_gradient, 60 | const std::vector*>& top, 61 | const std::vector*>& top_gradient) override; 62 | 63 | private: 64 | Dtype loss_; 65 | }; 66 | 67 | } // namespace cnn 68 | 69 | #include "../../src/log_loss_layer.cpp" 70 | -------------------------------------------------------------------------------- /include/cnn/max_pooling_layer.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | #include 19 | 20 | #include "cnn/layer.hpp" 21 | 22 | namespace cnn { 23 | /** 24 | * It has one bottom and one top. 25 | * 26 | * bottom[0] has shape (N, C, H, W) 27 | * 28 | * top[0] has shape (N, C, h, w) 29 | * 30 | * where 31 | * h = (H - win_size)/stride + 1 32 | * w = (W - win_size)/stride + 1 33 | */ 34 | template 35 | class MaxPoolingLayer : public Layer { 36 | public: 37 | explicit MaxPoolingLayer(const LayerProto&); 38 | 39 | void reshape(const std::vector*>& bottom, 40 | const std::vector*>& bottom_gradient, 41 | const std::vector*>& top, 42 | const std::vector*>& top_gradient) override; 43 | 44 | void fprop(const std::vector*>& bottom, 45 | const std::vector*>& top) override; 46 | 47 | void bprop(const std::vector*>& bottom, 48 | const std::vector*>& bottom_gradient, 49 | const std::vector*>& top, 50 | const std::vector*>& top_gradient) override; 51 | 52 | private: 53 | std::pair find_max_index(const Dtype* arr, int width, int h, 54 | int w) const; 55 | 56 | private: 57 | int win_size_; 58 | int stride_; 59 | 60 | Array> max_index_pair_; 61 | }; 62 | 63 | } // namespace cnn 64 | 65 | #include "../../src/max_pooling_layer.cpp" 66 | -------------------------------------------------------------------------------- /include/cnn/network.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "proto/cnn.pb.h" 23 | 24 | #include "cnn/array.hpp" 25 | #include "cnn/layer.hpp" 26 | 27 | namespace cnn { 28 | template 29 | class Network { 30 | public: 31 | explicit Network(const NetworkProto&); 32 | explicit Network(const std::string& filename, bool is_binary = false); 33 | void init(const std::string& filename, bool is_binary = false); 34 | void init(const NetworkProto&); 35 | 36 | void copy_trained_network(const std::string& filename, 37 | bool is_binary = false); 38 | 39 | const NetworkProto& proto() const { return proto_; } 40 | NetworkProto& proto() { return proto_; } 41 | 42 | void reshape(); 43 | /** forward propagation */ 44 | void fprop(); 45 | 46 | /** backward propagation */ 47 | void bprop(); 48 | 49 | void set_phase(Phase phase); 50 | Phase get_phase() const { return phase_; } 51 | 52 | Dtype get_loss() const { return get_data_top(layers_.size() - 1)[0]->d_[0]; } 53 | /** 54 | * we assume that the output of the last blob saves the predication 55 | */ 56 | 57 | void perform_predication(); 58 | std::vector get_predications() const { 59 | std::vector res; 60 | auto top = get_data_top(layers_.size() - 1)[0]; 61 | for (int i = 0; i < top->total_; i++) { 62 | res.push_back(top->d_[i]); 63 | } 64 | return res; 65 | } 66 | 67 | int get_batch_size() const { return layers_[0]->proto().input_proto().n(); } 68 | 69 | void save_network(const std::string& filename, bool is_binary = false); 70 | 71 | std::shared_ptr> layer(int i) const { return layers_[i]; } 72 | const std::vector>>& layers() const { 73 | return layers_; 74 | } 75 | std::vector>>& layers() { return layers_; } 76 | 77 | std::vector*> get_data_bottom(int i) const; 78 | std::vector*> get_data_top(int i) const; 79 | std::vector*> get_data_top_mutable(int i); 80 | 81 | std::vector*> get_gradient_bottom_mutable(int i); 82 | std::vector*> get_gradient_bottom(int i) const; 83 | std::vector*> get_gradient_top(int i) const; 84 | std::vector*> get_gradient_top_mutable(int i); 85 | 86 | void register_data_callback(void (*f)(const std::vector*>&)) { 87 | data_callback_ = f; 88 | } 89 | 90 | private: 91 | // add data to the map 92 | void add_data(const std::string& name, std::shared_ptr> arr); 93 | 94 | void add_gradient(const std::string& name, std::shared_ptr> arr); 95 | 96 | private: 97 | NetworkProto proto_; 98 | 99 | /** it saves the input and output of all layers in the network*/ 100 | std::map>> data_; 101 | std::map>> gradient_; 102 | 103 | std::vector>> layers_; 104 | 105 | std::function*>& top)> data_callback_; 106 | 107 | Phase phase_; 108 | }; 109 | 110 | } // namespace cnn 111 | 112 | #include "../../src/network.cpp" 113 | -------------------------------------------------------------------------------- /include/cnn/optimizer.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | #include "proto/cnn.pb.h" 22 | 23 | #include "cnn/network.hpp" 24 | 25 | namespace cnn { 26 | 27 | template 28 | class Optimizer { 29 | public: 30 | explicit Optimizer(const OptimizerProto& _proto); 31 | explicit Optimizer(const std::string& filename); 32 | void init(const OptimizerProto& _proto); 33 | 34 | void start_training(); 35 | 36 | void register_data_callback(void (*f)(const std::vector*>&)) { 37 | network_->register_data_callback(f); 38 | } 39 | 40 | private: 41 | void update_parameters(int current_iter); 42 | 43 | private: 44 | void print_parameters(); 45 | 46 | private: 47 | OptimizerProto proto_; 48 | 49 | std::shared_ptr> network_; 50 | }; 51 | 52 | } // namespace cnn 53 | 54 | #include "../../src/optimizer.cpp" 55 | -------------------------------------------------------------------------------- /include/cnn/relu_layer.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | 19 | #include "cnn/layer.hpp" 20 | 21 | namespace cnn { 22 | /** 23 | * It has one bottom and one top. 24 | * 25 | * Both bottom[0] and top[0] have shape (N, C, H, W). 26 | * 27 | * top[0]->d_[i] = max(0, bottom[0]->d_[i]); 28 | * 29 | */ 30 | template 31 | class ReLULayer : public Layer { 32 | public: 33 | explicit ReLULayer(const LayerProto&); 34 | 35 | void reshape(const std::vector*>& bottom, 36 | const std::vector*>& bottom_gradient, 37 | const std::vector*>& top, 38 | const std::vector*>& top_gradient) override; 39 | 40 | void fprop(const std::vector*>& bottom, 41 | const std::vector*>& top) override; 42 | 43 | void bprop(const std::vector*>& bottom, 44 | const std::vector*>& bottom_gradient, 45 | const std::vector*>& top, 46 | const std::vector*>& top_gradient) override; 47 | }; 48 | 49 | } // namespace cnn 50 | 51 | #include "../../src/relu_layer.cpp" 52 | -------------------------------------------------------------------------------- /include/cnn/rng.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | 19 | #include "cnn/array.hpp" 20 | 21 | namespace cnn { 22 | 23 | /** 24 | * Set the seed for the random number generator. 25 | * @param val the value of the seed. 26 | */ 27 | void set_seed(int val); 28 | 29 | /** 30 | * Return a uniformly distributed random value in the given 31 | * interval. Both ends are inclusive. 32 | * 33 | * @param low the lower bound, inclusive 34 | * @param high the upper bound, inclusive 35 | * @return a random value uniformly distributed in the interval [low, high]. 36 | */ 37 | int uniform(int low, int high); 38 | 39 | /** 40 | * Return a random boolean variable according to a bernoulli distribution. 41 | * 42 | * @param p the probability to return true 43 | * @return true with probability p, false with probability 1 - p. 44 | */ 45 | bool bernoulli(double p); 46 | 47 | /** 48 | * 49 | * Return a guassian distributed random 50 | * value. 51 | * @param mean mean of the gaussian 52 | * @param stddev standard deviation of the gaussian 53 | */ 54 | double gaussian(double mean, double stddev); 55 | 56 | /** 57 | * Fill the array with random values drawn from a gaussian 58 | * distribution with the given mean and standard deviation. 59 | * 60 | * @param arr the array to be filled 61 | * @param mean mean for the normal distribution 62 | * @param stddev standard deviation for the normal distribution 63 | * 64 | * @note variance is equal to the square of the standard deviation 65 | */ 66 | template 67 | void gaussian(Array* arr, Dtype mean, Dtype stddev) { 68 | for (int i = 0; i < arr->total_; i++) { 69 | arr->d_[i] = static_cast(gaussian(mean, stddev)); 70 | } 71 | } 72 | 73 | template 74 | void uniform(Array* arr, int low, int high) { 75 | for (int i = 0; i < arr->total_; i++) { 76 | arr->d_[i] = static_cast(uniform(low, high)); 77 | } 78 | } 79 | 80 | template 81 | void bernoulli(Array* arr, double p) { 82 | for (int i = 0; i < arr->total_; i++) { 83 | arr->d_[i] = static_cast(bernoulli(p)); 84 | } 85 | } 86 | 87 | } // namespace cnn 88 | -------------------------------------------------------------------------------- /include/cnn/softmax_layer.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | 19 | #include "cnn/layer.hpp" 20 | 21 | namespace cnn { 22 | 23 | /** 24 | * Compute softmax over channels. 25 | * 26 | * For example, for a color image with 3 channels, we compute softmax 27 | * for every pixel over its r, g, b channels such that 28 | * (r_i, g_i, b_i) = softmax(r_i, g_i, b_i), where 29 | * r_i is the red channel for pixel i and so on for g_i and b_i; 30 | * 31 | * softmax(a, b, c) = (e^a/(e^a+ e^b + e^c), e^b/(e^a+e^b+e^c), 32 | * e^c/(e^a+e^b+e^c)) 33 | * 34 | * if a = max(a, b, c), then 35 | * softmax(a, b, c) = softmax(a-a, b-a, c-a) = softmax(0, b-a, c-a) 36 | */ 37 | template 38 | class SoftmaxLayer : public Layer { 39 | public: 40 | explicit SoftmaxLayer(const LayerProto&); 41 | 42 | void reshape(const std::vector*>& bottom, 43 | const std::vector*>& bottom_gradient, 44 | const std::vector*>& top, 45 | const std::vector*>& top_gradient) override; 46 | 47 | void fprop(const std::vector*>& bottom, 48 | const std::vector*>& top) override; 49 | 50 | void bprop(const std::vector*>& bottom, 51 | const std::vector*>& bottom_gradient, 52 | const std::vector*>& top, 53 | const std::vector*>& top_gradient) override; 54 | 55 | private: 56 | Array buffer_; 57 | }; 58 | 59 | } // namespace cnn 60 | 61 | #include "../../src/softmax_layer.cpp" 62 | -------------------------------------------------------------------------------- /include/cnn/softmax_with_log_loss_layer.hpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #pragma once 16 | 17 | #include 18 | 19 | #include "cnn/layer.hpp" 20 | #include "cnn/softmax_layer.hpp" 21 | 22 | namespace cnn { 23 | /** 24 | * Softmax + log loss. 25 | * 26 | * Motivation: 27 | * -(1/y) * (y - y*y) does not equal to -(1 - y) 28 | * when y is close to 0 from a programmer's perspective 29 | * of view!!! I learn this by hard. 30 | * It causes gradient to be 0 when y is tiny, 31 | * i.e., the vanishing gradient problem. 32 | * 33 | * For classification only! 34 | * 35 | * It accepts two inputs: bottom[0] and bottom[1]. 36 | * 37 | * bottom[0] is the prediction with shape (N, C, H, W). 38 | * The meaning of the shape can be interpreted as follows: 39 | * (1) N is the batch size. (2) H and W can be considered 40 | * as the size of an image. (3) C represents the number of channels 41 | * of the image; in the case of multi-class classification, C 42 | * is the number of classes we have and the meaning of each pixel 43 | * in the channel k indicates the probability to the k-th category 44 | * this pixel belongs. 45 | * 46 | * The user has to ensure that the pixel values are in 47 | * the range of [0, 1] and it is normalized, i.e., sum to 1. 48 | * 49 | * This layer lays usually on top of the full connected layer. 50 | * 51 | * bottom[1] is the ground truth and has the shape (N, 1, H, W). 52 | * values of every element in bottom[1] have to be an integer 53 | * in the range [0, C-1]. 54 | */ 55 | template 56 | class SoftmaxWithLogLossLayer : public Layer { 57 | public: 58 | explicit SoftmaxWithLogLossLayer(const LayerProto&); 59 | 60 | void reshape(const std::vector*>& bottom, 61 | const std::vector*>& bottom_gradient, 62 | const std::vector*>& top, 63 | const std::vector*>& top_gradient) override; 64 | 65 | void fprop(const std::vector*>& bottom, 66 | const std::vector*>& top) override; 67 | 68 | void bprop(const std::vector*>& bottom, 69 | const std::vector*>& bottom_gradient, 70 | const std::vector*>& top, 71 | const std::vector*>& top_gradient) override; 72 | 73 | private: 74 | Dtype loss_; 75 | std::shared_ptr> softmax_layer_; 76 | Array softmax_top_; 77 | Array softmax_top_gradient_; 78 | Array softmax_bottom_gradient_; 79 | }; 80 | 81 | } // namespace cnn 82 | 83 | #include "../../src/softmax_with_log_loss_layer.cpp" 84 | -------------------------------------------------------------------------------- /proto/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | protobuf_generate_cpp( 3 | PROTO_SRCS 4 | PROTO_HDRS 5 | cnn.proto 6 | ) 7 | 8 | add_library( 9 | cnn_proto STATIC 10 | ${PROTO_SRCS} 11 | ${PROTO_HDRS} 12 | ) 13 | 14 | target_link_libraries( 15 | cnn_proto 16 | ${Protobuf_LIBRARY} 17 | ) 18 | 19 | -------------------------------------------------------------------------------- /proto/cnn.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cnn; 4 | 5 | message ArrayProto 6 | { 7 | optional int32 n = 1; // number of batches 8 | optional int32 c = 2; // number of channels 9 | optional int32 h = 3; // number of rows, i.e., height 10 | optional int32 w = 4; // number of columns, i.e., width 11 | repeated double d = 5; // data 12 | } 13 | 14 | message OptimizerProto 15 | { 16 | optional string model_filename = 1; 17 | optional double learning_rate = 2; 18 | optional int32 max_iteration_num = 3; // maximum number of iterations 19 | optional int32 print_interval = 4 [default = 1000]; // print after every multiple of this number of iterations 20 | optional string trained_filename = 5; // a binary file; if present, copy weights from it 21 | optional int32 snapshot_interval = 6; // save trained weights after every multiple of this number of iterations 22 | optional string snapshot_prefix = 7; // prefix of the snapshot 23 | } 24 | 25 | message NetworkProto 26 | { 27 | repeated LayerProto layer_proto = 1; 28 | } 29 | 30 | enum LayerType 31 | { 32 | INPUT = 0; // input layer 33 | FULL_CONNECTED = 1; // full connected layer 34 | L2_LOSS = 2; // l2 loss layer, for regression only 35 | SOFTMAX = 3; 36 | LOG_LOSS = 4; 37 | SOFTMAX_WITH_LOG_LOSS = 5; 38 | CONVOLUTION = 6; 39 | RELU = 7; 40 | MAX_POOLING = 8; 41 | DROP_OUT = 9; 42 | BATCH_NORMALIZATION = 10; 43 | LEAKY_RELU = 11; 44 | } 45 | 46 | // same as caffe 47 | enum Phase 48 | { 49 | TRAIN = 0; // during the train phase, we have to allocate gradient 50 | TEST = 1; // during the test case, no gradient is needed 51 | } 52 | 53 | message LayerProto 54 | { 55 | optional string name = 1; // can be arbitrary and may be not unique, for debug only 56 | optional LayerType type = 2; 57 | repeated string bottom = 3; // bottom/top name are globally unique 58 | repeated string top = 4; // bottom/top name are globally unique 59 | optional Phase phase = 5; // train or test, used from gradient space allocation 60 | repeated ArrayProto param = 6; // parameters for this layer 61 | 62 | optional InputLayerProto input_proto = 7; 63 | optional FullConnectedLayerProto fc_proto = 8; 64 | optional ConvolutionLayerProto conv_proto = 9; 65 | optional MaxPoolingLayerProto max_pooling_proto = 10; 66 | optional DropoutLayerProto dropout_proto = 11; 67 | optional BatchNormalizationLayerProto batch_normalization_proto = 12; 68 | optional LeakyReLULayerProto leaky_relu_proto = 13; 69 | } 70 | 71 | message InputLayerProto 72 | { 73 | optional int32 n = 1; 74 | optional int32 c = 2; 75 | optional int32 h = 3; 76 | optional int32 w = 4; 77 | } 78 | 79 | message FullConnectedLayerProto 80 | { 81 | optional int32 num_output = 1; // number of outputs 82 | } 83 | 84 | message ConvolutionLayerProto 85 | { 86 | optional int32 num_output = 1; 87 | optional int32 kernel_size = 2; // size of the square kernel 88 | // currently we assume implicit padding with stride 1 so 89 | // that the output size equals to the input size 90 | } 91 | 92 | message MaxPoolingLayerProto 93 | { 94 | optional int32 win_size = 1; // size of a square window 95 | optional int32 stride = 2; // stride of the window 96 | } 97 | 98 | message DropoutLayerProto 99 | { 100 | optional double keep_prob = 1; // the probability to retain the output 101 | } 102 | 103 | message BatchNormalizationLayerProto 104 | { 105 | // moving_average = momentum * moving_average + (1 - momentum) * mini_batch_average 106 | optional double momentum = 1 [default = 0.99]; 107 | } 108 | 109 | 110 | message LeakyReLULayerProto 111 | { 112 | // when input is less than 0, it returns alpha*input 113 | // tensorflow uses default value 0.2 114 | // keras uses 0.3 115 | // pytorch uses 0.01, which is the same as in the paper 116 | optional double alpha = 1 [default = 0.01]; 117 | } 118 | -------------------------------------------------------------------------------- /proto/model.prototxt: -------------------------------------------------------------------------------- 1 | 2 | layer_proto { 3 | name: "input" 4 | type: INPUT 5 | top: "data" 6 | top: "label" 7 | input_proto { 8 | n: 5 9 | c: 1 10 | h: 1 11 | w: 1 12 | } 13 | } 14 | 15 | layer_proto { 16 | name: "fc1" 17 | type: FULL_CONNECTED 18 | bottom: "data" 19 | top: "fc1" 20 | fc_proto { 21 | num_output: 1 22 | } 23 | } 24 | 25 | layer_proto { 26 | name: "l2_loss" 27 | type: L2_LOSS 28 | bottom: "fc1" 29 | bottom: "label" 30 | top: "loss" 31 | } 32 | -------------------------------------------------------------------------------- /proto/optimizer.prototxt: -------------------------------------------------------------------------------- 1 | 2 | model_filename: "../proto/model.prototxt" 3 | learning_rate: 0.01 4 | max_iteration_num: 400 5 | -------------------------------------------------------------------------------- /proto/trained.prototxt: -------------------------------------------------------------------------------- 1 | layer_proto { 2 | name: "input" 3 | type: INPUT 4 | top: "data" 5 | input_proto { 6 | n: 1 7 | c: 1 8 | h: 1 9 | w: 1 10 | } 11 | } 12 | layer_proto { 13 | name: "fc1" 14 | type: FULL_CONNECTED 15 | bottom: "data" 16 | top: "fc1" 17 | param { 18 | n: 1 19 | c: 1 20 | h: 1 21 | w: 1 22 | d: 10.001843607683776 23 | } 24 | param { 25 | n: 1 26 | c: 1 27 | h: 1 28 | w: 1 29 | d: 4.9333594877468165 30 | } 31 | fc_proto { 32 | num_output: 1 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | 2 | [![Build Status](https://travis-ci.com/csukuangfj/OpenCNN.svg?branch=dev)](https://travis-ci.com/csukuangfj/OpenCNN) 3 | 4 | 5 | *I am trying to port to [Bazel][10] and adding more documentation. It may break some 6 | of the existing functionalities; but it should be stable in three weeks*. 7 | 8 | [Documentation][9] 9 | 10 | # OpenCNN 11 | 12 | OpenCNN is a convolutional neural network framework implemented 13 | with C++11 from scratch. 14 | 15 | ## Table of contents 16 | 17 | - [Features](#features) 18 | - [Supported Layers](#supported-layers) 19 | - [Build](#build) 20 | - [Example with MNIST](#example-with-mnist) 21 | - [Usage](#usage) 22 | - [TODO](#todo) 23 | - [License](#License) 24 | 25 | ## Features 26 | - Easy to understand 27 | * Simply implemented and a good source for learning CNN 28 | - Easy to extend 29 | * Well defined interface for adding new layer types 30 | - Few dependencies 31 | * Depends only on [protobuf][1], [glog][2] and [gflags][3] 32 | - Fully tested 33 | * Every layer is covered by unit test with [googletest][4] 34 | * [autodiff][5] (in forward mode) is implemented to verify the correctness of forward/backward propagation 35 | - Pure C++ 36 | * If you are a big fan of C++ 37 | - Runs on CPU 38 | * No GPU is needed. 39 | * 95.21% accuracy on MNIST test dataset in 5000 iterations with a batch size of 16 40 | 41 | ## Supported Layers 42 | - convolutional 43 | - batch normalization 44 | - ReLU 45 | - leaky ReLU 46 | - max pooling 47 | - full connected 48 | - dropout 49 | - softmax 50 | - cross entropy loss (i.e., negative log loss) 51 | - softmax with cross entropy loss 52 | - L2 loss 53 | 54 | ## Build 55 | ### Install Dependencies on Linux (Ubuntu) 56 | 57 | ```sh 58 | sudo apt-get install libprotobuf-dev protobuf-compiler libgflags-dev libgoogle-glog-dev 59 | ``` 60 | 61 | ### Install Dependencies on Mac OS X 62 | 63 | ``` 64 | brew install gflags glog protobuf 65 | ``` 66 | 67 | ### Compile From Source 68 | 69 | ```sh 70 | git clone https://github.com/csukuangfj/OpenCNN.git 71 | cd OpenCNN 72 | mkdir build 73 | cd build 74 | cmake .. 75 | make 76 | ``` 77 | 78 | ### Run Unit Test 79 | 80 | ```sh 81 | cd OpenCNN/build 82 | ./gtest 83 | ``` 84 | 85 | It should pass all the test cases on your system. 86 | 87 | ## Example with MNIST 88 | We use the following network architecture for MNIST: 89 | 90 | | Layers | Description | 91 | |-----------------------|----------------------------------| 92 | | Input | dim: 1x28x28 | 93 | | Convolution-1 | num_output: 32, kernel_size: 3x3 | 94 | | Batch normalization-1 | | 95 | | ReLU-1 | | 96 | | Convolution-2 | num_output: 32, kernel_size: 3x3 | 97 | | Batch normalization-2 | | 98 | | ReLU-2 | | 99 | | Max pooling-1 | win_size: 2x2, stride: 2x2 | 100 | | Convolution-3 | num_output:64, kernel_size: 3x3 | 101 | | Batch normalization-3 | | 102 | | ReLU-3 | | 103 | | Convolution-4 | num_output: 64, kernel_size: 3x3 | 104 | | Batch normalization-4 | | 105 | | ReLU-4 | | 106 | | Max pooling-2 | win_size: 2x2, stride: 2x2 | 107 | | Full connected-1 | num_output: 512 | 108 | | Batch normalization-5 | | 109 | | ReLU-5 | | 110 | | Dropout-1 | keep_prob: 0.8 | 111 | | Full connected-2 | num_output: 10 | 112 | | Softmax with log loss | | 113 | 114 | During the training a batch size of 16 is used and the accuracy 115 | reaches 95.21% after 5000 iterations. The results for training loss and 116 | test accuracy are plotted in the following figure: 117 | 118 | ![training-loss-test-accuracy-versus-iterations][6] 119 | 120 | A pretrained model taken after 20000 iterations achieving an accuracy 121 | of 96.74% is provided in [OpenCNN-Models][8]. 122 | 123 | ## Usage 124 | Please refer to [examples/mnist][7] for how to use OpenCNN. 125 | 126 | More tutorials will be provided later. 127 | 128 | ## TODO 129 | - [ ] Add advanced optimizers 130 | - [ ] Add more layer types 131 | - [ ] Make code run faster 132 | - [ ] Tutorials and documentation 133 | 134 | 135 | ## License 136 | 137 | 138 | [10]: https://bazel.build/ 139 | [9]: https://csukuangfj.github.io/doc/OpenCNN/ 140 | [8]: https://github.com/csukuangfj/OpenCNN-Models/tree/master/mnist 141 | [7]: https://github.com/csukuangfj/OpenCNN/tree/master/examples/mnist 142 | [6]: /examples/mnist/loss-accuracy-iter.png 143 | [5]: https://en.wikipedia.org/wiki/Automatic_differentiation 144 | [4]: https://github.com/abseil/googletest 145 | [3]: https://github.com/gflags/gflags 146 | [2]: https://github.com/google/glog 147 | [1]: https://github.com/protocolbuffers/protobuf 148 | 149 | -------------------------------------------------------------------------------- /src/drop_out_layer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | 17 | #include 18 | 19 | #include "cnn/drop_out_layer.hpp" 20 | #include "cnn/rng.hpp" 21 | 22 | namespace cnn { 23 | 24 | template 25 | DropoutLayer::DropoutLayer(const LayerProto& _proto) 26 | : Layer(_proto) { 27 | const auto& p = _proto.dropout_proto(); 28 | keep_prob_ = p.keep_prob(); 29 | CHECK_GT(keep_prob_, 0); 30 | CHECK_LT(keep_prob_, 1); 31 | } 32 | 33 | template 34 | void DropoutLayer::reshape( 35 | const std::vector*>& bottom, 36 | const std::vector*>& bottom_gradient, 37 | const std::vector*>& top, 38 | const std::vector*>& top_gradient) { 39 | CHECK_EQ(bottom.size(), 1) << "Dropout accepts only 1 input"; 40 | CHECK_EQ(top.size(), 1) << "Dropout generates only 1 output"; 41 | 42 | top[0]->init_like(*bottom[0]); 43 | 44 | if (this->proto_.phase() == TRAIN) { 45 | CHECK_EQ(bottom_gradient.size(), 1); 46 | 47 | if (!bottom_gradient[0]->has_same_shape(*bottom[0])) { 48 | bottom_gradient[0]->init_like(*bottom[0]); 49 | } 50 | 51 | CHECK_EQ(top_gradient.size(), 1); 52 | top_gradient[0]->init_like(*top[0]); 53 | 54 | mask_.init_like(*top[0]); 55 | } 56 | } 57 | 58 | template 59 | void DropoutLayer::fprop(const std::vector*>& bottom, 60 | const std::vector*>& top) { 61 | const auto& b = *bottom[0]; 62 | auto& t = *top[0]; 63 | 64 | if (this->proto_.phase() == TRAIN) { 65 | bernoulli(&mask_, keep_prob_); 66 | for (int i = 0; i < b.total_; i++) { 67 | t[i] = b[i] * Dtype(mask_[i]) / keep_prob_; 68 | } 69 | } else // NOLINT 70 | { 71 | for (int i = 0; i < b.total_; i++) { 72 | t[i] = b[i]; 73 | } 74 | } 75 | } 76 | 77 | template 78 | void DropoutLayer::bprop( 79 | const std::vector*>& /*bottom*/, 80 | const std::vector*>& bottom_gradient, 81 | const std::vector*>& /*top*/, 82 | const std::vector*>& top_gradient) { 83 | auto& bg = *bottom_gradient[0]; 84 | const auto& tg = *top_gradient[0]; 85 | 86 | if (this->proto_.phase() == TRAIN) { 87 | for (int i = 0; i < bg.total_; i++) { 88 | bg[i] = tg[i] * Dtype(mask_[i]) / keep_prob_; 89 | } 90 | } else // NOLINT 91 | { 92 | for (int i = 0; i < bg.total_; i++) { 93 | bg[i] = tg[i]; 94 | } 95 | } 96 | } 97 | 98 | } // namespace cnn 99 | -------------------------------------------------------------------------------- /src/full_connected_layer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | 17 | #include 18 | 19 | #include "cnn/array_math.hpp" 20 | #include "cnn/full_connected_layer.hpp" 21 | #include "cnn/jet.hpp" 22 | #include "cnn/rng.hpp" 23 | 24 | namespace cnn { 25 | template 26 | FullConnectedLayer::FullConnectedLayer(const LayerProto& _proto) 27 | : Layer(_proto) { 28 | num_output_ = _proto.fc_proto().num_output(); 29 | } 30 | 31 | template 32 | void FullConnectedLayer::reshape( 33 | const std::vector*>& bottom, 34 | const std::vector*>& bottom_gradient, 35 | const std::vector*>& top, 36 | const std::vector*>& top_gradient) { 37 | CHECK_EQ(bottom.size(), 1); 38 | 39 | CHECK_EQ(top.size(), 1); 40 | 41 | int n = bottom[0]->n_; 42 | top[0]->init(n, num_output_, 1, 1); 43 | 44 | if (this->param_.empty()) { 45 | this->param_.resize(2); 46 | this->param_[0] = std::make_shared>(); 47 | this->param_[0]->init(1, 1, num_output_, bottom[0]->total_ / n); 48 | 49 | // TODO(fangjun): use other strategies 50 | gaussian(this->param_[0].get(), 0, 1); 51 | 52 | this->param_[1] = std::make_shared>(); 53 | this->param_[1]->init(1, 1, 1, num_output_); 54 | 55 | // TODO(fangjun): use other strategies 56 | gaussian(this->param_[1].get(), 0, 1); 57 | } else // NOLINT 58 | { 59 | CHECK_EQ(this->param_.size(), 2); 60 | 61 | CHECK_EQ(this->param_[0]->n_, 1); 62 | CHECK_EQ(this->param_[0]->c_, 1); 63 | CHECK_EQ(this->param_[0]->h_, num_output_); 64 | CHECK_EQ(this->param_[0]->w_, bottom[0]->total_ / n); 65 | 66 | CHECK(this->param_[1]->has_same_shape({1, 1, 1, num_output_})); 67 | } 68 | 69 | if (this->proto().phase() == TRAIN) { 70 | // gradient for parameters 71 | this->gradient_.resize(2); 72 | this->gradient_[0].reset(new Array); 73 | this->gradient_[1].reset(new Array); 74 | 75 | this->gradient_[0]->init_like(*this->param_[0]); 76 | this->gradient_[1]->init_like(*this->param_[1]); 77 | 78 | // gradient for the bottom input 79 | CHECK_EQ(bottom_gradient.size(), 1); 80 | if (!bottom_gradient[0]->has_same_shape(*bottom[0])) { 81 | bottom_gradient[0]->init_like(*bottom[0]); 82 | } 83 | 84 | // gradient for the top input 85 | CHECK_EQ(top_gradient.size(), 1); 86 | top_gradient[0]->init_like(*top[0]); 87 | } 88 | } 89 | 90 | template 91 | void FullConnectedLayer::fprop( 92 | const std::vector*>& bottom, 93 | const std::vector*>& top) { 94 | int n = bottom[0]->n_; 95 | for (int i = 0; i < n; i++) { 96 | for (int j = 0; j < num_output_; j++) { 97 | Dtype dot = ax_dot_by(this->param_[0]->w_, 1, 98 | &this->param_[0]->operator()(0, 0, j, 0), 1, 99 | &bottom[0]->operator()(i, 0, 0, 0)); 100 | top[0]->operator()(i, j, 0, 0) = dot + this->param_[1]->operator[](j); 101 | } 102 | } 103 | } 104 | 105 | template 106 | void FullConnectedLayer::bprop( 107 | const std::vector*>& bottom, 108 | const std::vector*>& bottom_gradient, 109 | const std::vector*>& top, 110 | const std::vector*>& top_gradient) { 111 | // compute parameter gradient 112 | auto& w = *this->param_[0]; 113 | auto& dw = *this->gradient_[0]; 114 | 115 | auto& db = *this->gradient_[1]; 116 | 117 | auto& x = *bottom[0]; 118 | auto& dx = *bottom_gradient[0]; 119 | 120 | auto& y = *top[0]; 121 | auto& dy = *top_gradient[0]; 122 | 123 | int stride = dw.w_; 124 | for (int n = 0; n < y.n_; n++) { 125 | for (int i = 0; i < num_output_; i++) { 126 | Dtype scale = dy(n, i, 0, 0); 127 | ax_plus_by(stride, scale, &x[n * stride], 1, &dw(0, 0, i, 0)); 128 | 129 | db.d_[i] += scale; 130 | 131 | ax_plus_by(stride, scale, &w(0, 0, i, 0), 1, &dx[n * stride]); 132 | } 133 | } 134 | } 135 | 136 | } // namespace cnn 137 | -------------------------------------------------------------------------------- /src/input_layer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | 17 | #include 18 | #include 19 | 20 | #include "cnn/input_layer.hpp" 21 | 22 | namespace cnn { 23 | template 24 | InputLayer::InputLayer(const LayerProto& _proto) : Layer(_proto) { 25 | const auto param = _proto.input_proto(); 26 | n_ = param.n(); 27 | c_ = param.c(); 28 | h_ = param.h(); 29 | w_ = param.w(); 30 | } 31 | 32 | template 33 | void InputLayer::reshape( 34 | const std::vector*>& /*bottom*/, 35 | const std::vector*>& /*bottom_gradient*/, 36 | const std::vector*>& top, 37 | const std::vector*>& /*top_gradient*/) { 38 | CHECK((top.size() == 1) || (top.size() == 2)); 39 | 40 | top[0]->init(n_, c_, h_, w_); 41 | if (top.size() == 2) { 42 | // resize the label 43 | top[1]->init(n_, 1, 1, 1); 44 | } 45 | } 46 | 47 | template 48 | void InputLayer::fprop( 49 | const std::vector*>& /*bottom*/, 50 | const std::vector*>& top) { 51 | // y = 5 + 10*x 52 | static std::vector, Dtype>> data{ 53 | {{11}, 115}, {{-14}, -135}, {{15}, 155}, {{6}, 65}, {{-18}, -175}, 54 | {{-8}, -75}, {{9}, 95}, {{-4}, -35}, {{18}, 185}, {{-1}, -5}, 55 | }; 56 | // TODO(fangjun) load data from here 57 | 58 | int n = top[0]->n_; 59 | int stride = top[0]->total_ / n; 60 | if (stride != data[0].first.size()) return; 61 | 62 | static int k = 0; 63 | CHECK_LE(n, data.size()) 64 | << "the batch size cannot be larger than the dataset size"; 65 | 66 | CHECK_EQ(stride, data[0].first.size()); 67 | 68 | for (int i = 0; i < n; i++) { 69 | if (k >= data.size()) { 70 | k = 0; 71 | } 72 | 73 | for (int j = 0; j < stride; j++) { 74 | top[0]->d_[i * stride + j] = (data[k].first)[j]; 75 | } 76 | 77 | if (top.size() == 2) { 78 | top[1]->d_[i] = data[k].second; 79 | } 80 | 81 | k++; 82 | } 83 | } 84 | 85 | } // namespace cnn 86 | -------------------------------------------------------------------------------- /src/io.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include // NOLINT 24 | #include 25 | 26 | #include "cnn/io.hpp" 27 | 28 | namespace cnn { 29 | // modified from 30 | // https://stackoverflow.com/questions/10842066/parse-in-text-file-for-google-protocol-buffer 31 | // https://github.com/BVLC/caffe/blob/master/src/caffe/util/io.cpp 32 | 33 | void read_proto_txt(const std::string& filename, 34 | google::protobuf::Message* message) { 35 | int fd = open(filename.c_str(), O_RDONLY); 36 | CHECK_GT(fd, 0) << "Failed to open " << filename; 37 | 38 | google::protobuf::io::FileInputStream fstream(fd); 39 | CHECK(google::protobuf::TextFormat::Parse(&fstream, message)) 40 | << "Failed to read " << filename; 41 | 42 | close(fd); 43 | } 44 | 45 | void write_proto_txt(const std::string& filename, 46 | const google::protobuf::Message& message) { 47 | int fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); 48 | CHECK_GT(fd, 0) << "Failed to create " << filename; 49 | 50 | google::protobuf::io::FileOutputStream fstream(fd); 51 | fstream.SetCloseOnDelete(true); 52 | CHECK(google::protobuf::TextFormat::Print(message, &fstream)) 53 | << "Failed to write " << filename; 54 | } 55 | 56 | // refer to 57 | // https://developers.google.com/protocol-buffers/docs/reference/cpp/google.protobuf.io.coded_stream 58 | void read_proto_bin(const std::string& filename, 59 | google::protobuf::Message* message) { 60 | int fd = open(filename.c_str(), O_RDONLY); 61 | CHECK_GT(fd, 0) << "Failed to open " << filename; 62 | 63 | google::protobuf::io::ZeroCopyInputStream* raw_input = 64 | new google::protobuf::io::FileInputStream(fd); 65 | google::protobuf::io::CodedInputStream coded_input(raw_input); 66 | CHECK(message->ParseFromCodedStream(&coded_input)) 67 | << "Failed to read " << filename; 68 | 69 | delete raw_input; 70 | close(fd); 71 | } 72 | 73 | // refer to https://github.com/BVLC/caffe/blob/master/src/caffe/util/io.cpp#L67 74 | void write_proto_bin(const std::string& filename, 75 | const google::protobuf::Message& message) { 76 | std::ofstream fstream(filename, 77 | std::ios::out | std::ios::trunc | std::ios::binary); 78 | CHECK(message.SerializeToOstream(&fstream)) << "Failed to write " << filename; 79 | } 80 | 81 | void string_to_proto(const std::string& model, 82 | google::protobuf::Message* message) { 83 | CHECK(google::protobuf::TextFormat::ParseFromString(model, message)); 84 | } 85 | 86 | // refer to http://netpbm.sourceforge.net/doc/pgm.html 87 | // for the spec of pgm 88 | void write_pgm(const std::string& filename, const Array& img) { 89 | std::ofstream of(filename, 90 | std::ios::out | std::ios::trunc | std::ios::binary); 91 | if (!of) { 92 | LOG(FATAL) << "cannot create file " << filename; 93 | } 94 | 95 | of << "P5" 96 | << "\n"; 97 | 98 | of << "# created with cnn" 99 | << "\n"; 100 | 101 | of << img.w_ << " " << img.h_ << "\n"; 102 | 103 | of << 255 << "\n"; 104 | 105 | of.write((const char*)&img[0], img.total_); 106 | } 107 | 108 | void read_pgm(const std::string& filename, Array* img) { 109 | // std::ifstream fs(filename, std::ios::binary); 110 | std::ifstream fs(filename); 111 | if (!fs) { 112 | LOG(FATAL) << "cannot read file " << filename; 113 | } 114 | 115 | std::string format; 116 | std::getline(fs, format); 117 | LOG(INFO) << "format is: " << format; 118 | 119 | std::string comments; 120 | std::getline(fs, comments); 121 | 122 | int width, height; 123 | fs >> width >> height; 124 | 125 | int max_val; 126 | fs >> max_val; 127 | 128 | CHECK_EQ(max_val, 255); 129 | 130 | fs.get(); // eat the new line \n 131 | 132 | img->init(1, 1, height, width); 133 | 134 | fs.read(reinterpret_cast(&img[0][0]), img->total_); 135 | } 136 | 137 | } // namespace cnn 138 | -------------------------------------------------------------------------------- /src/l2_loss_layer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | 17 | #include 18 | 19 | #include "cnn/array_math.hpp" 20 | #include "cnn/l2_loss_layer.hpp" 21 | 22 | namespace cnn { 23 | template 24 | L2LossLayer::L2LossLayer(const LayerProto& _proto) 25 | : Layer(_proto), loss_(0) {} 26 | 27 | template 28 | void L2LossLayer::reshape( 29 | const std::vector*>& bottom, 30 | const std::vector*>& bottom_gradient, 31 | const std::vector*>& top, 32 | const std::vector*>& /*top_gradient*/) { 33 | CHECK_EQ(bottom.size(), 2) << "It should have two inputs where " 34 | << "input[0] is the predication and " 35 | << "input[1] is the ground truth"; 36 | 37 | CHECK(bottom[0]->has_same_shape(*bottom[1])); 38 | 39 | CHECK_EQ(top.size(), 1); 40 | top[0]->init(1, 1, 1, 1); 41 | 42 | if (this->proto_.phase() == TRAIN) { 43 | CHECK_GE(bottom_gradient.size(), 1); 44 | 45 | if (!bottom_gradient[0]->has_same_shape(*bottom[0])) { 46 | bottom_gradient[0]->init_like(*(bottom[0])); 47 | } 48 | // we do not use the bottom_gradient[1] which is for the label 49 | } 50 | } 51 | 52 | template 53 | void L2LossLayer::fprop(const std::vector*>& bottom, 54 | const std::vector*>& top) { 55 | loss_ = ax_sub_by_squared(bottom[0]->total_, 1, bottom[0]->d_, 1, 56 | bottom[1]->d_); 57 | 58 | loss_ /= bottom[0]->total_; 59 | 60 | top[0]->d_[0] = loss_; 61 | } 62 | 63 | template 64 | void L2LossLayer::bprop( 65 | const std::vector*>& bottom, 66 | const std::vector*>& bottom_gradient, 67 | const std::vector*>& top, 68 | const std::vector*>& /*top_gradient*/) { 69 | // top[0] is the loss 70 | #if 0 71 | Dtype scale = top[0]->d_[0]; 72 | #else 73 | (void)top; 74 | Dtype scale = 1; 75 | #endif 76 | 77 | scale /= bottom[0]->total_; 78 | scale *= 2; // for the derivative of a squared loss 79 | // TODO(fangjun) it can be optimized, i.e., with lapack or blas. 80 | for (int i = 0; i < bottom[0]->total_; i++) { 81 | Dtype estimated_y = bottom[0]->d_[i]; 82 | Dtype true_y = bottom[1]->d_[i]; 83 | 84 | bottom_gradient[0]->d_[i] = scale * (estimated_y - true_y); 85 | } 86 | } 87 | 88 | } // namespace cnn 89 | -------------------------------------------------------------------------------- /src/leaky_relu_layer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | 17 | #include 18 | 19 | #include "cnn/jet.hpp" 20 | #include "cnn/relu_layer.hpp" 21 | 22 | namespace cnn { 23 | 24 | template 25 | LeakyReLULayer::LeakyReLULayer(const LayerProto& _proto) 26 | : Layer(_proto) { 27 | alpha_ = _proto.leaky_relu_proto().alpha(); 28 | CHECK_GE(alpha_, 0); 29 | } 30 | 31 | template 32 | void LeakyReLULayer::reshape( 33 | const std::vector*>& bottom, 34 | const std::vector*>& bottom_gradient, 35 | const std::vector*>& top, 36 | const std::vector*>& top_gradient) { 37 | CHECK_EQ(bottom.size(), 1) << "leaky relu accepts only 1 input"; 38 | CHECK_EQ(top.size(), 1) << "leaky relu generates only 1 output"; 39 | 40 | top[0]->init_like(*bottom[0]); 41 | 42 | if (this->proto_.phase() == TRAIN) { 43 | CHECK_EQ(bottom_gradient.size(), 1); 44 | 45 | if (!bottom_gradient[0]->has_same_shape(*bottom[0])) { 46 | bottom_gradient[0]->init_like(*bottom[0]); 47 | } 48 | 49 | CHECK_EQ(top_gradient.size(), 1); 50 | top_gradient[0]->init_like(*top[0]); 51 | } 52 | } 53 | 54 | template 55 | void LeakyReLULayer::fprop( 56 | const std::vector*>& bottom, 57 | const std::vector*>& top) { 58 | const auto& b = *bottom[0]; 59 | auto& t = *top[0]; 60 | for (int i = 0; i < b.total_; i++) { 61 | t[i] = ((b[i] >= Dtype(0)) + (b[i] < Dtype(0)) * alpha_) * b[i]; 62 | } 63 | } 64 | 65 | template 66 | void LeakyReLULayer::bprop( 67 | const std::vector*>& bottom, 68 | const std::vector*>& bottom_gradient, 69 | const std::vector*>& /*top*/, 70 | const std::vector*>& top_gradient) { 71 | const auto& b = *bottom[0]; 72 | auto& bg = *bottom_gradient[0]; 73 | 74 | const auto& tg = *top_gradient[0]; 75 | 76 | for (int i = 0; i < b.total_; i++) { 77 | bg[i] = tg[i] * ((b[i] >= Dtype(0)) + (b[i] < Dtype(0)) * alpha_); 78 | } 79 | } 80 | 81 | } // namespace cnn 82 | -------------------------------------------------------------------------------- /src/log_loss_layer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | 16 | #include 17 | #include 18 | 19 | #include "cnn/array_math.hpp" 20 | #include "cnn/common.hpp" 21 | #include "cnn/log_loss_layer.hpp" 22 | 23 | namespace cnn { 24 | 25 | template 26 | LogLossLayer::LogLossLayer(const LayerProto& _proto) 27 | : Layer(_proto), loss_(0) {} 28 | 29 | template 30 | void LogLossLayer::reshape( 31 | const std::vector*>& bottom, 32 | const std::vector*>& bottom_gradient, 33 | const std::vector*>& top, 34 | const std::vector*>& /*top_gradient*/) { 35 | CHECK_EQ(bottom.size(), 2) << "It should have two inputs where " 36 | << "input[0] is the predication and " 37 | << "input[1] is the ground truth"; 38 | 39 | CHECK_EQ(bottom[0]->n_, bottom[1]->n_); 40 | 41 | CHECK_EQ(bottom[1]->c_, 1); 42 | CHECK_EQ(bottom[0]->h_, bottom[1]->h_); 43 | CHECK_EQ(bottom[0]->w_, bottom[1]->w_); 44 | 45 | CHECK_EQ(top.size(), 1); 46 | top[0]->init(1, 1, 1, 1); 47 | 48 | if (this->proto_.phase() == TRAIN) { 49 | CHECK_GE(bottom_gradient.size(), 1); 50 | 51 | if (!bottom_gradient[0]->has_same_shape(*bottom[0])) { 52 | bottom_gradient[0]->init_like(*(bottom[0])); 53 | } 54 | // we do not use the bottom_gradient[1] which is for the label 55 | } 56 | } 57 | 58 | template 59 | void LogLossLayer::fprop(const std::vector*>& bottom, 60 | const std::vector*>& top) { 61 | loss_ = 0; 62 | 63 | const auto& b0 = *bottom[0]; 64 | const auto& b1 = *bottom[1]; 65 | 66 | for (int n = 0; n < b1.n_; n++) 67 | for (int h = 0; h < b1.h_; h++) 68 | for (int w = 0; w < b1.w_; w++) { 69 | auto label = b1(n, 0, h, w); // label for the ground truth 70 | auto p = b0(n, label, h, w); // probability for the predication 71 | p = std::max(p, Dtype(g_log_threshold)); 72 | p = std::min(p, Dtype(1)); 73 | 74 | // use cnn::log() here for Jet, in unit test only. 75 | loss_ += Dtype(-1) * log(p); 76 | } 77 | 78 | loss_ /= b1.total_; // take the average 79 | 80 | top[0]->d_[0] = loss_; 81 | } 82 | 83 | template 84 | void LogLossLayer::bprop( 85 | const std::vector*>& bottom, 86 | const std::vector*>& bottom_gradient, 87 | const std::vector*>& /*top*/, 88 | const std::vector*>& /*top_gradient*/) { 89 | Dtype scale = 1; 90 | 91 | const auto& b0 = *bottom[0]; 92 | const auto& b1 = *bottom[1]; 93 | auto& bg = *bottom_gradient[0]; 94 | 95 | scale /= Dtype(-1) * b1.total_; 96 | for (int n = 0; n < b1.n_; n++) 97 | for (int h = 0; h < b1.h_; h++) 98 | for (int w = 0; w < b1.w_; w++) { 99 | auto label = b1(n, 0, h, w); 100 | 101 | auto p = b0(n, label, h, w); // probability for the predication 102 | p = std::max(p, Dtype(g_log_threshold)); 103 | p = std::min(p, Dtype(1)); 104 | bg(n, label, h, w) = scale / p; 105 | } 106 | } 107 | 108 | } // namespace cnn 109 | -------------------------------------------------------------------------------- /src/max_pooling_layer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | 17 | #include 18 | #include 19 | 20 | #include "cnn/max_pooling_layer.hpp" 21 | 22 | namespace cnn { 23 | 24 | template 25 | MaxPoolingLayer::MaxPoolingLayer(const LayerProto& _proto) 26 | : Layer(_proto) { 27 | const auto& p = _proto.max_pooling_proto(); 28 | win_size_ = p.win_size(); 29 | stride_ = p.stride(); 30 | 31 | CHECK_GT(win_size_, 1) << "window size must be greater than 1"; 32 | 33 | CHECK_GT(stride_, 0) << "stride size must be greater than 0"; 34 | } 35 | 36 | template 37 | void MaxPoolingLayer::reshape( 38 | const std::vector*>& bottom, 39 | const std::vector*>& bottom_gradient, 40 | const std::vector*>& top, 41 | const std::vector*>& top_gradient) { 42 | CHECK_EQ(bottom.size(), 1) << "max pooling accepts only 1 input"; 43 | CHECK_EQ(top.size(), 1) << "max pooling generates only 1 output"; 44 | 45 | int h = (bottom[0]->h_ - win_size_) / stride_ + 1; 46 | int w = (bottom[0]->w_ - win_size_) / stride_ + 1; 47 | 48 | top[0]->init(bottom[0]->n_, bottom[0]->c_, h, w); 49 | 50 | max_index_pair_.init_like(*top[0]); 51 | 52 | if (this->proto_.phase() == TRAIN) { 53 | CHECK_EQ(bottom_gradient.size(), 1); 54 | 55 | if (!bottom_gradient[0]->has_same_shape(*bottom[0])) { 56 | bottom_gradient[0]->init_like(*bottom[0]); 57 | } 58 | 59 | CHECK_EQ(top_gradient.size(), 1); 60 | top_gradient[0]->init_like(*top[0]); 61 | } 62 | } 63 | 64 | template 65 | void MaxPoolingLayer::fprop( 66 | const std::vector*>& bottom, 67 | const std::vector*>& top) { 68 | const auto& b = *bottom[0]; 69 | auto& t = *top[0]; 70 | for (int n = 0; n < t.n_; n++) 71 | for (int c = 0; c < t.c_; c++) 72 | for (int h = 0; h < t.h_; h++) 73 | for (int w = 0; w < t.w_; w++) { 74 | auto p = 75 | find_max_index(&b(n, c, 0, 0), b.w_, h * stride_, w * stride_); 76 | 77 | t(n, c, h, w) = b(n, c, p.first, p.second); 78 | max_index_pair_(n, c, h, w) = p; 79 | } 80 | } 81 | 82 | template 83 | void MaxPoolingLayer::bprop( 84 | const std::vector*>& /*bottom*/, 85 | const std::vector*>& bottom_gradient, 86 | const std::vector*>& /*top*/, 87 | const std::vector*>& top_gradient) { 88 | auto& bg = *bottom_gradient[0]; 89 | const auto& tg = *top_gradient[0]; 90 | for (int n = 0; n < tg.n_; n++) 91 | for (int c = 0; c < tg.c_; c++) 92 | for (int h = 0; h < tg.h_; h++) 93 | for (int w = 0; w < tg.w_; w++) { 94 | const auto& p = max_index_pair_(n, c, h, w); 95 | bg(n, c, p.first, p.second) += tg(n, c, h, w); 96 | } 97 | } 98 | 99 | template 100 | std::pair MaxPoolingLayer::find_max_index(const Dtype* arr, 101 | int width, int h, 102 | int w) const { 103 | // find the index of the max value in the window 104 | // [h, h+win_size_) x [w, w+win_size_) 105 | Dtype max_val = arr[h * width + w]; 106 | int max_h = h; 107 | int max_w = w; 108 | 109 | for (int i = h; i < h + win_size_; i++) 110 | for (int j = w; j < w + win_size_; j++) { 111 | const auto& val = arr[i * width + j]; 112 | if (val > max_val) { 113 | max_val = val; 114 | max_h = i; 115 | max_w = j; 116 | } 117 | } 118 | return {max_h, max_w}; 119 | } 120 | 121 | } // namespace cnn 122 | -------------------------------------------------------------------------------- /src/optimizer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | 16 | #include 17 | 18 | #include // std::pow 19 | #include // NOLINT 20 | #include 21 | 22 | #include "cnn/array_math.hpp" 23 | #include "cnn/io.hpp" 24 | #include "cnn/optimizer.hpp" 25 | 26 | namespace cnn { 27 | template 28 | Optimizer::Optimizer(const OptimizerProto& _proto) { 29 | init(_proto); 30 | } 31 | 32 | template 33 | Optimizer::Optimizer(const std::string& filename) { 34 | OptimizerProto _proto; 35 | read_proto_txt(filename, &_proto); 36 | init(_proto); 37 | } 38 | 39 | template 40 | void Optimizer::init(const OptimizerProto& _proto) { 41 | proto_ = _proto; 42 | 43 | auto network_filename = proto_.model_filename(); 44 | network_.reset(new Network(network_filename)); 45 | if (proto_.has_trained_filename()) { 46 | network_->copy_trained_network(proto_.trained_filename(), true); 47 | } 48 | } 49 | 50 | template 51 | void Optimizer::start_training() { 52 | std::ofstream of("loss.txt"); 53 | int max_iter = proto_.max_iteration_num(); 54 | network_->reshape(); 55 | for (int i = 0; i < max_iter; i++) { 56 | network_->fprop(); 57 | network_->bprop(); 58 | update_parameters(i); 59 | 60 | if (i && !(i % proto_.print_interval())) { 61 | LOG(INFO) << "iter: " << i << "," 62 | << "loss is: " << network_->get_loss(); 63 | } 64 | of << i << "," << network_->get_loss() << "\n"; 65 | 66 | if (i && !(i % proto_.snapshot_interval())) { 67 | auto filename = proto_.snapshot_prefix() + "-" + std::to_string(i); 68 | network_->save_network(filename, true); 69 | } 70 | } 71 | 72 | LOG(INFO) << "iteration: " << max_iter; 73 | LOG(INFO) << "loss is: " << network_->get_loss(); 74 | print_parameters(); 75 | network_->save_network("trained-bin.prototxt", true); 76 | } 77 | 78 | template 79 | void Optimizer::update_parameters(int current_iter) { 80 | auto& layers = network_->layers(); 81 | int num_layers = layers.size(); 82 | 83 | Dtype learning_rate = proto_.learning_rate(); 84 | 85 | // TODO(fangjun): move the following options to proto 86 | static const double gamma = 0.0001; 87 | double base = 1 + gamma * current_iter; 88 | double exp = -0.75; 89 | learning_rate *= std::pow(base, exp); 90 | 91 | for (int i = 1; i < num_layers; i++) { 92 | layers[i]->update_parameters(current_iter, learning_rate); 93 | } 94 | } 95 | 96 | template 97 | void Optimizer::print_parameters() { 98 | auto& layers = network_->layers(); 99 | int num_layers = layers.size(); 100 | 101 | std::ostringstream ss; 102 | ss << "\n"; 103 | ss << "batch size is: " << network_->get_batch_size() << "\n"; 104 | // we skip the input layer since it has no parameters 105 | for (int i = 1; i < num_layers; i++) { 106 | #if 0 107 | auto param = layers[i]->mutable_param(); 108 | if (param.empty()) continue; 109 | ss << "parameters for layer: " << layers[i]->proto().name(); 110 | ss << "\n"; 111 | 112 | for (int j = 0; j < param.size(); j++) 113 | { 114 | for (int k = 0; k < param[j]->total_; k++) 115 | { 116 | ss << param[j]->d_[k] << " "; 117 | } 118 | ss << "\n"; 119 | } 120 | ss << " gradient for layer: " << layers[i]->proto().name() << "\n"; 121 | auto gradient = layers[i]->gradient(); 122 | for (const auto &g : gradient) 123 | { 124 | if (g->total_) 125 | { 126 | for (int i = 0; i < g->total_; i++) 127 | { 128 | ss << g->d_[i] << " "; 129 | } 130 | } 131 | ss << "\n"; 132 | } 133 | #endif 134 | } 135 | 136 | LOG(INFO) << ss.str(); 137 | } 138 | 139 | } // namespace cnn 140 | -------------------------------------------------------------------------------- /src/relu_layer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | 17 | #include 18 | 19 | #include "cnn/jet.hpp" 20 | #include "cnn/relu_layer.hpp" 21 | 22 | namespace cnn { 23 | 24 | template 25 | ReLULayer::ReLULayer(const LayerProto& _proto) : Layer(_proto) {} 26 | 27 | template 28 | void ReLULayer::reshape( 29 | const std::vector*>& bottom, 30 | const std::vector*>& bottom_gradient, 31 | const std::vector*>& top, 32 | const std::vector*>& top_gradient) { 33 | CHECK_EQ(bottom.size(), 1) << "relu accepts only 1 input"; 34 | CHECK_EQ(top.size(), 1) << "relu generates only 1 output"; 35 | 36 | top[0]->init_like(*bottom[0]); 37 | 38 | if (this->proto_.phase() == TRAIN) { 39 | CHECK_EQ(bottom_gradient.size(), 1); 40 | 41 | if (!bottom_gradient[0]->has_same_shape(*bottom[0])) { 42 | bottom_gradient[0]->init_like(*bottom[0]); 43 | } 44 | 45 | CHECK_EQ(top_gradient.size(), 1); 46 | top_gradient[0]->init_like(*top[0]); 47 | } 48 | } 49 | 50 | template 51 | void ReLULayer::fprop(const std::vector*>& bottom, 52 | const std::vector*>& top) { 53 | const auto& b = *bottom[0]; 54 | auto& t = *top[0]; 55 | for (int i = 0; i < b.total_; i++) { 56 | t[i] = max(b[i], Dtype(0)); // NOLINT 57 | } 58 | } 59 | 60 | template 61 | void ReLULayer::bprop( 62 | const std::vector*>& bottom, 63 | const std::vector*>& bottom_gradient, 64 | const std::vector*>& /*top*/, 65 | const std::vector*>& top_gradient) { 66 | const auto& b = *bottom[0]; 67 | auto& bg = *bottom_gradient[0]; 68 | 69 | const auto& tg = *top_gradient[0]; 70 | 71 | for (int i = 0; i < b.total_; i++) { 72 | bg[i] = tg[i] * (b[i] >= Dtype(0)); 73 | } 74 | } 75 | 76 | } // namespace cnn 77 | -------------------------------------------------------------------------------- /src/rng.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | 16 | #include 17 | 18 | #include "cnn/rng.hpp" 19 | 20 | namespace cnn { 21 | 22 | std::default_random_engine g_generator; 23 | 24 | void set_seed(int val) { g_generator.seed(val); } 25 | 26 | // refer to 27 | // http://www.cplusplus.com/reference/random/uniform_int_distribution/ 28 | int uniform(int low, int high) { 29 | std::uniform_int_distribution distribution(low, high); 30 | return distribution(g_generator); 31 | } 32 | 33 | // refer to 34 | // http://www.cplusplus.com/reference/random/normal_distribution/normal_distirbution/ 35 | double gaussian(double mean, double stddev) { 36 | std::normal_distribution distribution(mean, stddev); 37 | return distribution(g_generator); 38 | } 39 | 40 | // refer to 41 | // http://www.cplusplus.com/reference/random/bernoulli_distribution/bernoulli_distribution/ 42 | bool bernoulli(double p) { 43 | std::bernoulli_distribution distribution(p); 44 | return distribution(g_generator); 45 | } 46 | 47 | } // namespace cnn 48 | -------------------------------------------------------------------------------- /src/softmax_layer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | 17 | #include 18 | #include 19 | 20 | #include "cnn/softmax_layer.hpp" 21 | 22 | namespace cnn { 23 | 24 | template 25 | SoftmaxLayer::SoftmaxLayer(const LayerProto& _proto) 26 | : Layer(_proto) {} 27 | 28 | template 29 | void SoftmaxLayer::reshape( 30 | const std::vector*>& bottom, 31 | const std::vector*>& bottom_gradient, 32 | const std::vector*>& top, 33 | const std::vector*>& top_gradient) { 34 | CHECK_EQ(bottom.size(), 1) << "softmax accepts only 1 input"; 35 | CHECK_EQ(top.size(), 1) << "softmax generates only 1 output"; 36 | 37 | top[0]->init_like(*bottom[0]); 38 | 39 | if (this->proto_.phase() == TRAIN) { 40 | CHECK_EQ(bottom_gradient.size(), 1); 41 | 42 | if (!bottom_gradient[0]->has_same_shape(*bottom[0])) { 43 | bottom_gradient[0]->init_like(*bottom[0]); 44 | } 45 | 46 | CHECK_EQ(top_gradient.size(), 1); 47 | top_gradient[0]->init_like(*bottom[0]); 48 | } 49 | 50 | CHECK_GE(bottom[0]->c_, 2) 51 | << "we need at least two numbers to compute softmax!"; 52 | 53 | buffer_.init(1, 1, 1, bottom[0]->c_); 54 | } 55 | 56 | template 57 | void SoftmaxLayer::fprop(const std::vector*>& bottom, 58 | const std::vector*>& top) { 59 | const auto& b = *bottom[0]; 60 | auto& target = *top[0]; 61 | for (int n = 0; n < b.n_; n++) 62 | for (int h = 0; h < b.h_; h++) 63 | for (int w = 0; w < b.w_; w++) { 64 | Dtype max_val = b(n, 0, h, w); 65 | for (int c = 1; c < b.c_; c++) { 66 | max_val = (b(n, c, h, w) > max_val) ? b(n, c, h, w) : max_val; 67 | } 68 | 69 | for (int c = 0; c < b.c_; c++) { 70 | // we use cnn::exp() here for Jet (only in unit test) 71 | buffer_[c] = exp(b(n, c, h, w) - max_val); 72 | } 73 | 74 | Dtype den = sum_arr(buffer_); 75 | Dtype scale = Dtype(1) / den; 76 | scale_arr(scale, buffer_, &buffer_); 77 | for (int c = 0; c < b.c_; c++) { 78 | target(n, c, h, w) = buffer_[c]; 79 | } 80 | } 81 | } 82 | 83 | template 84 | void SoftmaxLayer::bprop( 85 | const std::vector*>& bottom, 86 | const std::vector*>& bottom_gradient, 87 | const std::vector*>& top, 88 | const std::vector*>& top_gradient) { 89 | auto& bg = *bottom_gradient[0]; 90 | const auto& tg = *top_gradient[0]; 91 | const auto& t = *top[0]; 92 | 93 | for (int n = 0; n < bottom[0]->n_; n++) 94 | for (int h = 0; h < bottom[0]->h_; h++) 95 | for (int w = 0; w < bottom[0]->w_; w++) 96 | for (int c = 0; c < bottom[0]->c_; c++) { 97 | Dtype yc = t(n, c, h, w); 98 | // TODO(fangjun) optimize it, i.e., eliminate the inner else condition 99 | // TODO(fangjun) implement copy_arr() to copy an array 100 | for (int i = 0; i < bottom[0]->c_; i++) { 101 | Dtype scale = tg(n, i, h, w); 102 | Dtype yi = t(n, i, h, w); 103 | bg(n, c, h, w) += scale * yi * ((i == c) - yc); 104 | #if 0 105 | if (i == c) 106 | { 107 | bg(n, c, h, w) += scale * (yi - yi*yc); 108 | } 109 | else // NOLINT 110 | { 111 | bg(n, c, h, w) += scale * (-yi * yc); 112 | } 113 | #endif 114 | } 115 | } 116 | } 117 | 118 | } // namespace cnn 119 | -------------------------------------------------------------------------------- /src/softmax_with_log_loss_layer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | 16 | #include 17 | #include 18 | 19 | #include "cnn/array_math.hpp" 20 | #include "cnn/common.hpp" 21 | #include "cnn/softmax_with_log_loss_layer.hpp" 22 | 23 | namespace cnn { 24 | 25 | template 26 | SoftmaxWithLogLossLayer::SoftmaxWithLogLossLayer( 27 | const LayerProto& _proto) 28 | : Layer(_proto), loss_(0) { 29 | LayerProto p; 30 | p.set_type(SOFTMAX); 31 | softmax_layer_ = Layer::create(p); 32 | } 33 | 34 | template 35 | void SoftmaxWithLogLossLayer::reshape( 36 | const std::vector*>& bottom, 37 | const std::vector*>& bottom_gradient, 38 | const std::vector*>& top, 39 | const std::vector*>& /*top_gradient*/) { 40 | CHECK_EQ(bottom.size(), 2) << "It should have two inputs where " 41 | << "input[0] is the predication and " 42 | << "input[1] is the ground truth"; 43 | 44 | CHECK_EQ(bottom[0]->n_, bottom[1]->n_); 45 | 46 | CHECK_EQ(bottom[1]->c_, 1); 47 | CHECK_EQ(bottom[0]->h_, bottom[1]->h_); 48 | CHECK_EQ(bottom[0]->w_, bottom[1]->w_); 49 | 50 | CHECK_EQ(top.size(), 1); 51 | top[0]->init(1, 1, 1, 1); 52 | 53 | softmax_layer_->reshape({bottom[0]}, {&softmax_bottom_gradient_}, 54 | {&softmax_top_}, {&softmax_top_gradient_}); 55 | 56 | if (this->proto_.phase() == TRAIN) { 57 | CHECK_GE(bottom_gradient.size(), 1); 58 | 59 | if (!bottom_gradient[0]->has_same_shape(*bottom[0])) { 60 | bottom_gradient[0]->init_like(*(bottom[0])); 61 | } 62 | // we do not use the bottom_gradient[1] which is for the label 63 | } 64 | } 65 | 66 | template 67 | void SoftmaxWithLogLossLayer::fprop( 68 | const std::vector*>& bottom, 69 | const std::vector*>& top) { 70 | softmax_layer_->fprop({bottom[0]}, {&softmax_top_}); 71 | 72 | loss_ = 0; 73 | 74 | const auto& b0 = softmax_top_; 75 | const auto& b1 = *bottom[1]; 76 | 77 | for (int n = 0; n < b1.n_; n++) 78 | for (int h = 0; h < b1.h_; h++) 79 | for (int w = 0; w < b1.w_; w++) { 80 | auto label = b1(n, 0, h, w); // label for the ground truth 81 | auto p = b0(n, label, h, w); // probability for the predication 82 | p = std::max(p, Dtype(g_log_threshold)); 83 | p = std::min(p, Dtype(1)); 84 | 85 | // use cnn::log() here for Jet in the unit test. 86 | loss_ += log(p); 87 | } 88 | 89 | loss_ /= Dtype(-1) * b1.total_; // take the average 90 | 91 | top[0]->d_[0] = loss_; 92 | } 93 | 94 | template 95 | void SoftmaxWithLogLossLayer::bprop( 96 | const std::vector*>& bottom, 97 | const std::vector*>& bottom_gradient, 98 | const std::vector*>& /*top*/, 99 | const std::vector*>& /*top_gradient*/) { 100 | const auto& b0 = *bottom[0]; 101 | const auto& b1 = *bottom[1]; 102 | auto& bg = *bottom_gradient[0]; 103 | 104 | Dtype scale = -1; 105 | scale /= b1.total_; 106 | 107 | for (int n = 0; n < b0.n_; n++) 108 | for (int h = 0; h < b0.h_; h++) 109 | for (int w = 0; w < b0.w_; w++) { 110 | auto label = b1(n, 0, h, w); 111 | for (int c = 0; c < b0.c_; c++) { 112 | bg(n, c, h, w) = scale * ((label == c) - softmax_top_(n, c, h, w)); 113 | } 114 | } 115 | } 116 | 117 | } // namespace cnn 118 | -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | include_directories(BEFORE ${CMAKE_CURRENT_LIST_DIR}) 3 | add_subdirectory(gtest) 4 | 5 | add_executable(gtest 6 | test_main.cpp 7 | test_array.cpp 8 | test_io.cpp 9 | test_layer.cpp 10 | test_input_layer.cpp 11 | test_network.cpp 12 | test_array_math.cpp 13 | test_l2_loss_layer.cpp 14 | test_full_connected_layer.cpp 15 | test_optimizer.cpp 16 | test_softmax_layer.cpp 17 | test_log_loss_layer.cpp 18 | test_softmax_with_log_loss_layer.cpp 19 | test_jet.cpp 20 | test_convolution_layer.cpp 21 | test_relu_layer.cpp 22 | test_max_pooling_layer.cpp 23 | test_rng.cpp 24 | test_drop_out_layer.cpp 25 | test_batch_normalization_layer.cpp 26 | test_leaky_relu_layer.cpp 27 | ) 28 | target_link_libraries( 29 | gtest 30 | gtest-all-in-one 31 | core 32 | ) 33 | -------------------------------------------------------------------------------- /test/gtest/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | add_library(gtest-all-in-one gtest-all.cpp) 3 | target_compile_definitions( 4 | gtest-all-in-one 5 | PUBLIC 6 | -DGTEST_HAS_PTHREAD=0 7 | -DGTEST_USE_OWN_TR1_TUPLE=1 8 | ) 9 | 10 | -------------------------------------------------------------------------------- /test/gtest/readme.md: -------------------------------------------------------------------------------- 1 | modified from https://github.com/BVLC/caffe/tree/master/src/gtest 2 | -------------------------------------------------------------------------------- /test/test_array_math.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | #include 17 | 18 | #include "cnn/array_math.hpp" 19 | 20 | namespace cnn { 21 | 22 | template 23 | class ArrayMathTest : public ::testing::Test {}; 24 | 25 | using MyTypes = ::testing::Types; 26 | TYPED_TEST_CASE(ArrayMathTest, MyTypes); 27 | 28 | TYPED_TEST(ArrayMathTest, set_to) { 29 | int val = 100; 30 | { 31 | Array arr; 32 | arr.init(2, 3, 4, 5); 33 | set_to(&arr, val); 34 | for (int i = 0; i < arr.total_; i++) { 35 | EXPECT_EQ(arr[i], val); 36 | } 37 | 38 | val = 0; 39 | set_to(&arr, val); 40 | for (int i = 0; i < arr.total_; i++) { 41 | EXPECT_EQ(arr[i], val); 42 | } 43 | } 44 | 45 | { 46 | val = 200; 47 | Array arr; 48 | arr.init(2, 3, 4, 5); 49 | set_to(arr.total_, &arr[0], val); 50 | for (int i = 0; i < arr.total_; i++) { 51 | EXPECT_EQ(arr[i], val); 52 | } 53 | 54 | val = 0; 55 | set_to(arr.total_, &arr[0], val); 56 | for (int i = 0; i < arr.total_; i++) { 57 | EXPECT_EQ(arr[i], val); 58 | } 59 | } 60 | } 61 | 62 | TYPED_TEST(ArrayMathTest, ax_sub_by_squared) { 63 | Array arr1; 64 | Array arr2; 65 | 66 | arr1.init(1, 2, 3, 4); 67 | arr2.init(24, 1, 1, 1); 68 | 69 | set_to(&arr1, 1); 70 | set_to(&arr2, 1); 71 | 72 | double diff = 1000; 73 | 74 | diff = ax_sub_by_squared(arr1.total_, 1, &arr1(0, 0, 0, 0), 1, 75 | &arr2[0]); 76 | EXPECT_EQ(diff, 0); 77 | 78 | double sum = 0; 79 | 80 | sum = ax_sub_by_squared(arr1.total_, -1, &arr1(0, 0, 0, 0), 1, 81 | &arr2[0]); 82 | EXPECT_EQ(sum, 4 * arr1.total_); 83 | } 84 | 85 | TYPED_TEST(ArrayMathTest, ax_dot_by) { 86 | Array arr1; 87 | Array arr2; 88 | 89 | arr1.init(1, 1, 2, 3); 90 | arr2.init(1, 1, 2, 3); 91 | 92 | set_to(&arr1, 1); 93 | set_to(&arr2, 1); 94 | 95 | TypeParam sum = 0; 96 | sum = ax_dot_by(arr1.total_, 1, arr1.d_, 1, arr2.d_); 97 | EXPECT_EQ(sum, arr1.total_); 98 | 99 | sum = 0; 100 | set_to(&arr2, 2); 101 | sum = ax_dot_by(arr1.total_, -1, arr1.d_, 2, arr2.d_); 102 | EXPECT_EQ(sum, -arr1.total_ * 4); 103 | } 104 | 105 | TYPED_TEST(ArrayMathTest, scale_arr) { 106 | Array d; 107 | d.init(1, 2, 1, 1); 108 | d[0] = 2; 109 | d[1] = 3; 110 | scale_arr(2, d, &d); 111 | EXPECT_EQ(d[0], 4); 112 | EXPECT_EQ(d[1], 6); 113 | 114 | scale_arr(0.5, d, &d); 115 | EXPECT_EQ(d[0], 2); 116 | EXPECT_EQ(d[1], 3); 117 | 118 | scale_arr(2, 2, &d[0], &d[0]); 119 | EXPECT_EQ(d[0], 4); 120 | EXPECT_EQ(d[1], 6); 121 | } 122 | 123 | TYPED_TEST(ArrayMathTest, sum_arr) { 124 | Array d; 125 | d.init(1, 2, 1, 1); 126 | d[0] = -1; 127 | d[1] = 3; 128 | auto r = sum_arr(d); 129 | EXPECT_EQ(r, 2); 130 | } 131 | 132 | TYPED_TEST(ArrayMathTest, sum_arr2) { 133 | Array d; 134 | d.init(1, 2, 1, 1); 135 | d[0] = -2; 136 | d[1] = 3; 137 | auto r = sum_arr(d.total_, &d[0]); 138 | EXPECT_EQ(r, 1); 139 | } 140 | 141 | TYPED_TEST(ArrayMathTest, sum_squared_arr) { 142 | Array d; 143 | d.init(1, 2, 1, 1); 144 | d[0] = -1; 145 | d[1] = 3; 146 | auto r = sum_squared_arr(2, &d[0]); 147 | EXPECT_EQ(r, 10); 148 | } 149 | 150 | TYPED_TEST(ArrayMathTest, sub_scalar) { 151 | Array d; 152 | d.init(1, 2, 1, 1); 153 | d[0] = 10; 154 | d[1] = 5; 155 | 156 | sub_scalar(2, d, &d); 157 | EXPECT_EQ(d[0], 8); 158 | EXPECT_EQ(d[1], 3); 159 | } 160 | 161 | TYPED_TEST(ArrayMathTest, sub_scalar2) { 162 | Array d; 163 | d.init(1, 2, 1, 1); 164 | d[0] = 10; 165 | d[1] = 5; 166 | 167 | sub_scalar(d.total_, 3, &d[0], &d[0]); 168 | EXPECT_EQ(d[0], 7); 169 | EXPECT_EQ(d[1], 2); 170 | } 171 | 172 | } // namespace cnn 173 | -------------------------------------------------------------------------------- /test/test_drop_out_layer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | #include 17 | 18 | #define private public 19 | 20 | #include "cnn/jet.hpp" 21 | #include "cnn/layer.hpp" 22 | 23 | namespace cnn { 24 | 25 | template 26 | class DropoutLayerTest : public ::testing::Test { 27 | void SetUp() override { 28 | LayerProto proto; 29 | proto.set_type(DROP_OUT); 30 | auto* p = proto.mutable_dropout_proto(); 31 | p->set_keep_prob(0.8); 32 | layer_ = Layer::create(proto); 33 | } 34 | 35 | protected: 36 | std::shared_ptr> layer_; 37 | 38 | Array bottom_; 39 | Array bottom_gradient_; 40 | Array top_; 41 | Array top_gradient_; 42 | }; 43 | 44 | using MyTypes = ::testing::Types; 45 | TYPED_TEST_CASE(DropoutLayerTest, MyTypes); 46 | 47 | TYPED_TEST(DropoutLayerTest, reshape_train_phase) { 48 | static constexpr int N = 2; 49 | static constexpr int C = 3; 50 | static constexpr int H = 4; 51 | static constexpr int W = 5; 52 | 53 | this->layer_->proto().set_phase(TRAIN); 54 | 55 | this->bottom_.init(N, C, H, W); 56 | this->layer_->reshape({&this->bottom_}, {&this->bottom_gradient_}, 57 | {&this->top_}, {&this->top_gradient_}); 58 | 59 | EXPECT_TRUE(this->bottom_gradient_.has_same_shape(this->bottom_)); 60 | 61 | EXPECT_TRUE(this->top_.has_same_shape(this->bottom_)); 62 | EXPECT_TRUE(this->top_gradient_.has_same_shape(this->bottom_)); 63 | 64 | EXPECT_TRUE(this->layer_->param().empty()); 65 | EXPECT_TRUE(this->layer_->gradient().empty()); 66 | 67 | auto* dropout_layer = 68 | dynamic_cast*>(this->layer_.get()); 69 | EXPECT_TRUE(dropout_layer->mask_.has_same_shape(this->top_)); 70 | } 71 | 72 | TYPED_TEST(DropoutLayerTest, reshape_test_phase) { 73 | static constexpr int N = 2; 74 | static constexpr int C = 3; 75 | static constexpr int H = 4; 76 | static constexpr int W = 5; 77 | 78 | this->layer_->proto().set_phase(TEST); 79 | 80 | this->bottom_.init(N, C, H, W); 81 | this->layer_->reshape({&this->bottom_}, {&this->bottom_gradient_}, 82 | {&this->top_}, {&this->top_gradient_}); 83 | 84 | EXPECT_TRUE(this->bottom_gradient_.has_same_shape({0, 0, 0, 0})); 85 | 86 | EXPECT_TRUE(this->top_.has_same_shape(this->bottom_)); 87 | EXPECT_TRUE(this->top_gradient_.has_same_shape({0, 0, 0, 0})); 88 | 89 | EXPECT_TRUE(this->layer_->param().empty()); 90 | EXPECT_TRUE(this->layer_->gradient().empty()); 91 | 92 | auto* dropout_layer = 93 | dynamic_cast*>(this->layer_.get()); 94 | EXPECT_TRUE(dropout_layer->mask_.has_same_shape({0, 0, 0, 0})); 95 | } 96 | 97 | // we do not need to test the fprop since it is implicitly 98 | // verified in the bprop below. 99 | TYPED_TEST(DropoutLayerTest, bprop_with_jet) { 100 | return; 101 | static constexpr int N = 2; 102 | static constexpr int C = 3; 103 | static constexpr int H = 4; 104 | static constexpr int W = 5; 105 | static constexpr int DIM = 1; 106 | 107 | using Type = Jet; 108 | 109 | LayerProto proto; 110 | proto.set_phase(TRAIN); 111 | proto.set_type(DROP_OUT); 112 | proto.mutable_dropout_proto()->set_keep_prob(0.5); 113 | auto layer = Layer::create(proto); 114 | 115 | Array bottom; 116 | Array bottom_gradient; 117 | Array top; 118 | Array top_gradient; 119 | 120 | bottom.init(N, C, H, W); 121 | 122 | uniform(&bottom, -100, 100); 123 | for (int i = 0; i < bottom.total_; i++) { 124 | bottom.d_[i].v_[0] = 1; 125 | } 126 | 127 | layer->reshape({&bottom}, {&bottom_gradient}, {&top}, {&top_gradient}); 128 | 129 | uniform(&top_gradient, -100, 100); 130 | layer->fprop({&bottom}, {&top}); 131 | layer->bprop({&bottom}, {&bottom_gradient}, {&top}, {&top_gradient}); 132 | 133 | for (int i = 0; i < bottom_gradient.total_; i++) { 134 | TypeParam expected = top[i].v_[0] * top_gradient[i].a_; 135 | if (expected == 0) { 136 | EXPECT_EQ(bottom_gradient[i], 0); 137 | } else { 138 | EXPECT_NEAR(bottom_gradient[i].a_ / expected, 1, 1e-4); 139 | } 140 | } 141 | } 142 | 143 | } // namespace cnn 144 | -------------------------------------------------------------------------------- /test/test_input_layer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | 16 | #include 17 | 18 | #define private public 19 | 20 | #include "cnn/layer.hpp" 21 | 22 | namespace cnn { 23 | 24 | template 25 | class InputLayerTest : public ::testing::Test {}; 26 | 27 | using MyTypes = ::testing::Types; 28 | TYPED_TEST_CASE(InputLayerTest, MyTypes); 29 | 30 | TYPED_TEST(InputLayerTest, create) { 31 | LayerProto proto; 32 | proto.set_type(INPUT); 33 | 34 | auto param = proto.mutable_input_proto(); 35 | param->set_n(2); 36 | param->set_c(3); 37 | param->set_h(4); 38 | param->set_w(5); 39 | 40 | InputLayer layer(proto); 41 | EXPECT_EQ(layer.n_, 2); 42 | EXPECT_EQ(layer.c_, 3); 43 | EXPECT_EQ(layer.h_, 4); 44 | EXPECT_EQ(layer.w_, 5); 45 | 46 | Array arr; 47 | std::vector*> vec{&arr}; 48 | layer.reshape({}, {}, vec, {}); 49 | 50 | EXPECT_EQ(arr.n_, 2); 51 | EXPECT_EQ(arr.c_, 3); 52 | EXPECT_EQ(arr.h_, 4); 53 | EXPECT_EQ(arr.w_, 5); 54 | } 55 | 56 | } // namespace cnn 57 | -------------------------------------------------------------------------------- /test/test_io.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | 17 | #include "cnn/array.hpp" 18 | #include "cnn/common.hpp" 19 | #include "cnn/io.hpp" 20 | 21 | #include "proto/cnn.pb.h" 22 | 23 | namespace cnn { 24 | 25 | TEST(io_test, txt) { 26 | ArrayProto arr; 27 | arr.set_n(1); 28 | arr.set_c(2); 29 | arr.set_h(3); 30 | arr.set_w(4); 31 | for (int i = 0; i < 24; i++) { 32 | arr.add_d(i); 33 | } 34 | write_proto_txt("a.txt", arr); 35 | 36 | ArrayProto arr2; 37 | read_proto_txt("a.txt", &arr2); 38 | 39 | EXPECT_EQ(arr.n(), arr2.n()); 40 | EXPECT_EQ(arr.c(), arr2.c()); 41 | EXPECT_EQ(arr.h(), arr2.h()); 42 | EXPECT_EQ(arr.w(), arr2.w()); 43 | EXPECT_EQ(arr.d_size(), arr2.d_size()); 44 | for (int i = 0; i < arr.d_size(); i++) { 45 | EXPECT_EQ(arr.d(i), arr2.d(i)); 46 | } 47 | } 48 | 49 | TEST(io_test, bin) { 50 | ArrayProto arr; 51 | arr.set_n(1); 52 | arr.set_c(2); 53 | arr.set_h(3); 54 | arr.set_w(4); 55 | for (int i = 0; i < 24; i++) { 56 | arr.add_d(i); 57 | } 58 | write_proto_bin("a.bin", arr); 59 | 60 | ArrayProto arr2; 61 | read_proto_bin("a.bin", &arr2); 62 | 63 | EXPECT_EQ(arr.n(), arr2.n()); 64 | EXPECT_EQ(arr.c(), arr2.c()); 65 | EXPECT_EQ(arr.h(), arr2.h()); 66 | EXPECT_EQ(arr.w(), arr2.w()); 67 | EXPECT_EQ(arr.d_size(), arr2.d_size()); 68 | for (int i = 0; i < arr.d_size(); i++) { 69 | EXPECT_EQ(arr.d(i), arr2.d(i)); 70 | } 71 | } 72 | 73 | TEST(io_test, string_to_proto) { 74 | const char* model = 75 | R"proto( 76 | layer_proto { 77 | name: "input" 78 | type: INPUT 79 | top: "data" 80 | top: "label" 81 | input_proto { n: 5 c: 1 h: 1 w: 1 } 82 | } 83 | )proto"; 84 | NetworkProto proto; 85 | string_to_proto(model, &proto); 86 | LOG(INFO) << proto.DebugString(); 87 | } 88 | 89 | TEST(io_test, write_read_pgm) { 90 | std::string filename = "abc.pgm"; 91 | Array a; 92 | a.init(1, 1, 50, 50); 93 | LOG(INFO) << "a.h: " << a.h_; 94 | for (int i = 0; i < a.h_; i++) { 95 | a(0, 0, i, i) = 255; 96 | } 97 | write_pgm(filename, a); 98 | 99 | Array b; 100 | read_pgm(filename, &b); 101 | 102 | EXPECT_TRUE(a.has_same_shape(b)); 103 | for (int i = 0; i < a.total_; i++) { 104 | EXPECT_EQ(a[i], b[i]); 105 | } 106 | } 107 | 108 | } // namespace cnn 109 | -------------------------------------------------------------------------------- /test/test_l2_loss_layer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | #include 17 | 18 | #include "cnn/array_math.hpp" 19 | #include "cnn/jet.hpp" 20 | #include "cnn/layer.hpp" 21 | 22 | namespace cnn { 23 | 24 | template 25 | class L2LossLayerTest : public ::testing::Test { 26 | void SetUp() override { 27 | LayerProto proto; 28 | proto.set_type(L2_LOSS); 29 | layer_ = Layer::create(proto); 30 | } 31 | 32 | protected: 33 | std::shared_ptr> layer_; 34 | 35 | Array bottom1_; //!< the prediction 36 | Array bottom2_; //!< the ground truth 37 | 38 | Array bottom1_gradient_; //!< gradient for the predication 39 | Array 40 | top_; //!< the loss, which is a scalar, i.e., with shape (1,1,1,1) 41 | }; 42 | 43 | using MyTypes = ::testing::Types; 44 | TYPED_TEST_CASE(L2LossLayerTest, MyTypes); 45 | 46 | TYPED_TEST(L2LossLayerTest, reshape_train_phase) { 47 | this->layer_->proto().set_phase(TRAIN); 48 | this->bottom1_.init(2, 3, 4, 5); 49 | this->bottom2_.init_like(this->bottom1_); 50 | 51 | this->layer_->reshape({&this->bottom1_, &this->bottom2_}, 52 | {&this->bottom1_gradient_}, {&this->top_}, {}); 53 | 54 | EXPECT_TRUE(this->bottom1_gradient_.has_same_shape(this->bottom1_)); 55 | EXPECT_TRUE(this->top_.has_same_shape({1, 1, 1, 1})); 56 | 57 | EXPECT_TRUE(this->layer_->param().empty()); 58 | EXPECT_TRUE(this->layer_->gradient().empty()); 59 | } 60 | 61 | TYPED_TEST(L2LossLayerTest, reshape_test_phase) { 62 | this->layer_->proto().set_phase(TEST); 63 | this->bottom1_.init(2, 3, 4, 5); 64 | this->bottom2_.init_like(this->bottom1_); 65 | 66 | this->layer_->reshape({&this->bottom1_, &this->bottom2_}, 67 | {&this->bottom1_gradient_}, {&this->top_}, {}); 68 | 69 | EXPECT_TRUE(this->top_.has_same_shape({1, 1, 1, 1})); 70 | EXPECT_TRUE(this->bottom1_gradient_.has_same_shape({0, 0, 0, 0})); 71 | 72 | EXPECT_TRUE(this->layer_->param().empty()); 73 | EXPECT_TRUE(this->layer_->gradient().empty()); 74 | } 75 | 76 | TYPED_TEST(L2LossLayerTest, fprop) { 77 | this->bottom1_.init(2, 3, 4, 5); 78 | this->bottom2_.init(2, 3, 4, 5); 79 | 80 | set_to(&this->bottom1_, 2); 81 | set_to(&this->bottom2_, 2); 82 | 83 | int n = uniform(1, this->bottom1_.total_); 84 | for (int i = 0; i < n; i++) { 85 | this->bottom2_[i] = 1; 86 | } 87 | 88 | this->layer_->reshape({&this->bottom1_, &this->bottom2_}, 89 | {&this->bottom1_gradient_}, {&this->top_}, {}); 90 | 91 | this->layer_->fprop({&this->bottom1_, &this->bottom2_}, {&this->top_}); 92 | EXPECT_NEAR(this->top_[0], TypeParam(n) / this->bottom1_.total_, 1e-4); 93 | } 94 | 95 | TYPED_TEST(L2LossLayerTest, bprop_with_jet) { 96 | static constexpr int N = 2; 97 | static constexpr int C = 3; 98 | static constexpr int H = 4; 99 | static constexpr int W = 5; 100 | static constexpr int DIM = N * C * H * W; 101 | 102 | using Type = Jet; 103 | 104 | LayerProto proto; 105 | proto.set_phase(TRAIN); 106 | proto.set_type(L2_LOSS); 107 | auto layer = Layer::create(proto); 108 | 109 | Array bottom1; 110 | Array bottom2; 111 | Array bottom1_gradient; 112 | Array top; 113 | 114 | bottom1.init(N, C, H, W); 115 | bottom2.init(N, C, H, W); 116 | 117 | uniform(&bottom1, -100, 100); 118 | uniform(&bottom2, -100, 100); 119 | for (int i = 0; i < bottom1.total_; i++) { 120 | bottom1.d_[i].v_[i] = 1; 121 | } 122 | 123 | layer->reshape({&bottom1, &bottom2}, {&bottom1_gradient}, {&top}, {}); 124 | 125 | layer->fprop({&bottom1, &bottom2}, {&top}); 126 | layer->bprop({&bottom1, &bottom2}, {&bottom1_gradient}, {&top}, {}); 127 | 128 | for (int i = 0; i < bottom1.total_; i++) { 129 | EXPECT_NEAR(bottom1_gradient[i].a_, top[0].v_[i], 1e-5); 130 | } 131 | } 132 | 133 | } // namespace cnn 134 | -------------------------------------------------------------------------------- /test/test_layer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | #include "cnn/layer.hpp" 17 | 18 | namespace cnn { 19 | 20 | template 21 | class LayerTest : public ::testing::Test {}; 22 | 23 | using MyTypes = ::testing::Types; 24 | TYPED_TEST_CASE(LayerTest, MyTypes); 25 | 26 | TYPED_TEST(LayerTest, create_fc) { 27 | LayerProto proto; 28 | proto.set_type(FULL_CONNECTED); 29 | auto res = Layer::create(proto); 30 | EXPECT_NE(res.get(), nullptr); 31 | } 32 | 33 | TYPED_TEST(LayerTest, create_input) { 34 | LayerProto proto; 35 | proto.set_type(INPUT); 36 | auto res = Layer::create(proto); 37 | EXPECT_NE(res.get(), nullptr); 38 | } 39 | 40 | TYPED_TEST(LayerTest, create_l2_loss) { 41 | LayerProto proto; 42 | proto.set_type(L2_LOSS); 43 | auto res = Layer::create(proto); 44 | EXPECT_NE(res.get(), nullptr); 45 | } 46 | 47 | TYPED_TEST(LayerTest, create_softmax) { 48 | LayerProto proto; 49 | proto.set_type(SOFTMAX); 50 | auto res = Layer::create(proto); 51 | EXPECT_NE(res.get(), nullptr); 52 | } 53 | 54 | TYPED_TEST(LayerTest, create_log_loss) { 55 | LayerProto proto; 56 | proto.set_type(LOG_LOSS); 57 | auto res = Layer::create(proto); 58 | EXPECT_NE(res.get(), nullptr); 59 | } 60 | 61 | TYPED_TEST(LayerTest, create_softmax_with_log_loss) { 62 | LayerProto proto; 63 | proto.set_type(SOFTMAX_WITH_LOG_LOSS); 64 | auto res = Layer::create(proto); 65 | EXPECT_NE(res.get(), nullptr); 66 | } 67 | 68 | TYPED_TEST(LayerTest, create_convolution) { 69 | LayerProto proto; 70 | proto.set_type(CONVOLUTION); 71 | auto* p = proto.mutable_conv_proto(); 72 | p->set_num_output(2); 73 | p->set_kernel_size(1); 74 | auto res = Layer::create(proto); 75 | EXPECT_NE(res.get(), nullptr); 76 | } 77 | 78 | TYPED_TEST(LayerTest, create_relu) { 79 | LayerProto proto; 80 | proto.set_type(RELU); 81 | auto res = Layer::create(proto); 82 | EXPECT_NE(res.get(), nullptr); 83 | } 84 | 85 | TYPED_TEST(LayerTest, create_max_pooling) { 86 | LayerProto proto; 87 | proto.set_type(MAX_POOLING); 88 | auto* p = proto.mutable_max_pooling_proto(); 89 | p->set_win_size(2); 90 | p->set_stride(2); 91 | auto res = Layer::create(proto); 92 | EXPECT_NE(res.get(), nullptr); 93 | } 94 | 95 | TYPED_TEST(LayerTest, create_dropout) { 96 | LayerProto proto; 97 | proto.set_type(DROP_OUT); 98 | auto* p = proto.mutable_dropout_proto(); 99 | p->set_keep_prob(0.8); 100 | auto res = Layer::create(proto); 101 | EXPECT_NE(res.get(), nullptr); 102 | } 103 | 104 | TYPED_TEST(LayerTest, create_batch_normalization) { 105 | LayerProto proto; 106 | proto.set_type(BATCH_NORMALIZATION); 107 | auto res = Layer::create(proto); 108 | EXPECT_NE(res.get(), nullptr); 109 | } 110 | 111 | TYPED_TEST(LayerTest, create_leaky_relu) { 112 | LayerProto proto; 113 | proto.set_type(LEAKY_RELU); 114 | proto.mutable_leaky_relu_proto()->set_alpha(0.3); 115 | auto res = Layer::create(proto); 116 | EXPECT_NE(res.get(), nullptr); 117 | } 118 | 119 | } // namespace cnn 120 | -------------------------------------------------------------------------------- /test/test_main.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | // Copyright 2006, Google Inc. 16 | // All rights reserved. 17 | // 18 | // Redistribution and use in source and binary forms, with or without 19 | // modification, are permitted provided that the following conditions are 20 | // met: 21 | // 22 | // * Redistributions of source code must retain the above copyright 23 | // notice, this list of conditions and the following disclaimer. 24 | // * Redistributions in binary form must reproduce the above 25 | // copyright notice, this list of conditions and the following disclaimer 26 | // in the documentation and/or other materials provided with the 27 | // distribution. 28 | // * Neither the name of Google Inc. nor the names of its 29 | // contributors may be used to endorse or promote products derived from 30 | // this software without specific prior written permission. 31 | // 32 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 33 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 34 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 35 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 36 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 37 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 38 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 39 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 40 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 41 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 42 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 43 | 44 | #include 45 | 46 | #include "gtest/gtest.h" 47 | 48 | GTEST_API_ int main(int argc, char** argv) { 49 | google::InitGoogleLogging(argv[0]); 50 | FLAGS_alsologtostderr = true; 51 | FLAGS_colorlogtostderr = true; 52 | 53 | // ::testing::GTEST_FLAG(filter) = "ArrayTest*"; 54 | // ::testing::GTEST_FLAG(filter) = "ArrayMathTest*"; 55 | // ::testing::GTEST_FLAG(filter) = "BatchNormalizationLayerTest*"; 56 | // ::testing::GTEST_FLAG(filter) = "BatchNormalizationLayerTest*bprop*"; 57 | // ::testing::GTEST_FLAG(filter) = "LayerTest*"; 58 | // ::testing::GTEST_FLAG(filter) = "ConvolutionLayerTest*"; 59 | // ::testing::GTEST_FLAG(filter) = "DropoutLayerTest*"; 60 | // ::testing::GTEST_FLAG(filter) = "FullConnectedLayerTest*"; 61 | // ::testing::GTEST_FLAG(filter) = "FullConnectedLayerTest*jet*"; 62 | // ::testing::GTEST_FLAG(filter) = "JetTest*"; 63 | // ::testing::GTEST_FLAG(filter) = "L2LossLayerTest*"; 64 | // ::testing::GTEST_FLAG(filter) = "L2LossLayer_Test*bprop*"; 65 | // ::testing::GTEST_FLAG(filter) = "LayerTest*"; 66 | // ::testing::GTEST_FLAG(filter) = "LeakyReLULayerTest*"; 67 | // ::testing::GTEST_FLAG(filter) = "LogLossLayerTest*"; 68 | // ::testing::GTEST_FLAG(filter) = "MaxPoolingLayerTest*"; 69 | // ::testing::GTEST_FLAG(filter) = "NetworkTest*"; 70 | // ::testing::GTEST_FLAG(filter) = "NetworkTest*fprop1*"; 71 | // ::testing::GTEST_FLAG(filter) = "NetworkTest*fprop2*"; 72 | // ::testing::GTEST_FLAG(filter) = "OptimizerTest*"; 73 | // ::testing::GTEST_FLAG(filter) = "ReLULayerTest*"; 74 | // ::testing::GTEST_FLAG(filter) = "RngTest*"; 75 | // ::testing::GTEST_FLAG(filter) = "SoftmaxLayerTest*fprop*"; 76 | // ::testing::GTEST_FLAG(filter) = "SoftmaxLayerTest*jet*"; 77 | // ::testing::GTEST_FLAG(filter) = "SoftmaxWithLogLossLayerTest*"; 78 | // 79 | // ::testing::GTEST_FLAG(filter) = "io_test*"; 80 | 81 | testing::InitGoogleTest(&argc, argv); 82 | 83 | LOG(INFO) << "Running main() from test_main.cpp"; 84 | 85 | return RUN_ALL_TESTS(); 86 | } 87 | -------------------------------------------------------------------------------- /test/test_optimizer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | #include 17 | 18 | #include "cnn/optimizer.hpp" 19 | 20 | namespace cnn { 21 | 22 | template 23 | class OptimizerTest : public ::testing::Test {}; 24 | 25 | using MyTypes = ::testing::Types; 26 | TYPED_TEST_CASE(OptimizerTest, MyTypes); 27 | 28 | TYPED_TEST(OptimizerTest, init) {} 29 | 30 | TYPED_TEST(OptimizerTest, training) {} 31 | 32 | } // namespace cnn 33 | -------------------------------------------------------------------------------- /test/test_relu_layer.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | #include 17 | 18 | #include "cnn/array_math.hpp" 19 | #include "cnn/jet.hpp" 20 | #include "cnn/layer.hpp" 21 | 22 | namespace cnn { 23 | 24 | template 25 | class ReLULayerTest : public ::testing::Test { 26 | void SetUp() override { 27 | LayerProto proto; 28 | proto.set_type(RELU); 29 | layer_ = Layer::create(proto); 30 | } 31 | 32 | protected: 33 | std::shared_ptr> layer_; 34 | 35 | Array bottom_; 36 | Array bottom_gradient_; 37 | Array top_; 38 | Array top_gradient_; 39 | }; 40 | 41 | using MyTypes = ::testing::Types; 42 | TYPED_TEST_CASE(ReLULayerTest, MyTypes); 43 | 44 | TYPED_TEST(ReLULayerTest, reshape_train_phase) { 45 | static constexpr int N = 2; 46 | static constexpr int C = 3; 47 | static constexpr int H = 4; 48 | static constexpr int W = 5; 49 | 50 | this->layer_->proto().set_phase(TRAIN); 51 | 52 | this->bottom_.init(N, C, H, W); 53 | this->layer_->reshape({&this->bottom_}, {&this->bottom_gradient_}, 54 | {&this->top_}, {&this->top_gradient_}); 55 | 56 | EXPECT_TRUE(this->bottom_gradient_.has_same_shape(this->bottom_)); 57 | 58 | EXPECT_TRUE(this->top_.has_same_shape(this->bottom_)); 59 | EXPECT_TRUE(this->top_gradient_.has_same_shape(this->bottom_)); 60 | 61 | EXPECT_TRUE(this->layer_->param().empty()); 62 | EXPECT_TRUE(this->layer_->gradient().empty()); 63 | } 64 | 65 | TYPED_TEST(ReLULayerTest, reshape_test_phase) { 66 | static constexpr int N = 2; 67 | static constexpr int C = 3; 68 | static constexpr int H = 4; 69 | static constexpr int W = 5; 70 | 71 | this->layer_->proto().set_phase(TEST); 72 | 73 | this->bottom_.init(N, C, H, W); 74 | this->layer_->reshape({&this->bottom_}, {&this->bottom_gradient_}, 75 | {&this->top_}, {&this->top_gradient_}); 76 | 77 | EXPECT_TRUE(this->bottom_gradient_.has_same_shape({0, 0, 0, 0})); 78 | 79 | EXPECT_TRUE(this->top_.has_same_shape(this->bottom_)); 80 | EXPECT_TRUE(this->top_gradient_.has_same_shape({0, 0, 0, 0})); 81 | 82 | EXPECT_TRUE(this->layer_->param().empty()); 83 | EXPECT_TRUE(this->layer_->gradient().empty()); 84 | } 85 | 86 | TYPED_TEST(ReLULayerTest, fprop) { 87 | static constexpr int N = 2; 88 | static constexpr int C = 3; 89 | static constexpr int H = 4; 90 | static constexpr int W = 5; 91 | 92 | this->layer_->proto().set_phase(TEST); 93 | 94 | this->bottom_.init(N, C, H, W); 95 | uniform(&this->bottom_, -100, 100); 96 | this->layer_->reshape({&this->bottom_}, {}, {&this->top_}, {}); 97 | 98 | this->layer_->fprop({&this->bottom_}, {&this->top_}); 99 | 100 | TypeParam expected; 101 | 102 | const auto& b = this->bottom_; 103 | const auto& t = this->top_; 104 | 105 | for (int i = 0; i < b.total_; i++) { 106 | expected = (b[i] >= 0) ? b[i] : 0; 107 | EXPECT_EQ(t[i], expected); 108 | } 109 | } 110 | 111 | TYPED_TEST(ReLULayerTest, bprop_with_jet) { 112 | static constexpr int N = 2; 113 | static constexpr int C = 3; 114 | static constexpr int H = 4; 115 | static constexpr int W = 5; 116 | static constexpr int DIM = 1; 117 | 118 | using Type = Jet; 119 | 120 | LayerProto proto; 121 | proto.set_phase(TRAIN); 122 | proto.set_type(RELU); 123 | auto layer = Layer::create(proto); 124 | 125 | Array bottom; 126 | Array bottom_gradient; 127 | Array top; 128 | Array top_gradient; 129 | 130 | bottom.init(N, C, H, W); 131 | 132 | uniform(&bottom, -100, 100); 133 | for (int i = 0; i < bottom.total_; i++) { 134 | bottom.d_[i].v_[0] = 1; 135 | } 136 | 137 | layer->reshape({&bottom}, {&bottom_gradient}, {&top}, {&top_gradient}); 138 | 139 | uniform(&top_gradient, -100, 100); 140 | layer->fprop({&bottom}, {&top}); 141 | layer->bprop({&bottom}, {&bottom_gradient}, {&top}, {&top_gradient}); 142 | 143 | for (int i = 0; i < bottom_gradient.total_; i++) { 144 | TypeParam expected = top[i].v_[0] * top_gradient[i].a_; 145 | EXPECT_EQ(bottom_gradient[i].a_, expected); 146 | } 147 | } 148 | 149 | } // namespace cnn 150 | -------------------------------------------------------------------------------- /test/test_rng.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | 17 | #include "cnn/rng.hpp" 18 | 19 | namespace cnn { 20 | 21 | template 22 | class RngTest : public ::testing::Test {}; 23 | 24 | using MyTypes = ::testing::Types; 25 | TYPED_TEST_CASE(RngTest, MyTypes); 26 | 27 | TYPED_TEST(RngTest, gaussian) { 28 | set_seed(200); 29 | Array arr; 30 | arr.init(100, 100, 10, 5); 31 | gaussian(&arr, 1, 5); 32 | TypeParam mean = 0; 33 | TypeParam var = 0; 34 | for (int i = 0; i < arr.total_; i++) { 35 | mean += arr[i]; 36 | } 37 | mean /= arr.total_; 38 | 39 | for (int i = 0; i < arr.total_; i++) { 40 | TypeParam diff = arr[i] - mean; 41 | var += diff * diff; 42 | } 43 | var /= arr.total_; 44 | EXPECT_NEAR(mean, 1, 0.01); 45 | EXPECT_NEAR(5 / std::sqrt(var), 1, 0.01); 46 | } 47 | 48 | TYPED_TEST(RngTest, uniform) { 49 | set_seed(100); 50 | Array arr; 51 | arr.init(100, 100, 10, 5); 52 | 53 | TypeParam low = -100; 54 | TypeParam high = 200; 55 | uniform(&arr, low, high); 56 | TypeParam mean = 0; 57 | TypeParam var = 0; 58 | for (int i = 0; i < arr.total_; i++) { 59 | mean += arr[i]; 60 | } 61 | mean /= arr.total_; 62 | 63 | for (int i = 0; i < arr.total_; i++) { 64 | TypeParam diff = arr[i] - mean; 65 | var += diff * diff; 66 | } 67 | var /= arr.total_; 68 | 69 | TypeParam expected_mean = (low + high) / 2; 70 | EXPECT_NEAR(mean, expected_mean, expected_mean * 1e-1); 71 | 72 | TypeParam expected_var = (high - low) * (high - low) / 12; 73 | EXPECT_NEAR(var, expected_var, expected_var * 1e-1); 74 | } 75 | 76 | TYPED_TEST(RngTest, bernoulli) { 77 | set_seed(1989); 78 | Array arr; 79 | arr.init(100, 100, 10, 5); 80 | 81 | double p = 0.8; 82 | 83 | bernoulli(&arr, p); 84 | 85 | TypeParam mean = 0; 86 | TypeParam var = 0; 87 | for (int i = 0; i < arr.total_; i++) { 88 | mean += arr[i]; 89 | } 90 | mean /= arr.total_; 91 | 92 | for (int i = 0; i < arr.total_; i++) { 93 | TypeParam diff = arr[i] - mean; 94 | var += diff * diff; 95 | } 96 | var /= arr.total_; 97 | 98 | TypeParam expected_mean = p; 99 | EXPECT_NEAR(mean, expected_mean, expected_mean * 1e-2); 100 | 101 | TypeParam expected_var = p * (1 - p); 102 | EXPECT_NEAR(var, expected_var, expected_var * 1e-2); 103 | } 104 | 105 | } // namespace cnn 106 | -------------------------------------------------------------------------------- /third_party/BUILD.bazel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csukuangfj/OpenCNN/561936740226b6741e75237a68afd6bda6df4f7d/third_party/BUILD.bazel -------------------------------------------------------------------------------- /third_party/google_style_guide.BUILD: -------------------------------------------------------------------------------- 1 | # copied from https://github.com/ApolloAuto/apollo/blob/master/third_party/google_styleguide.BUILD 2 | 3 | package(default_visibility = ["//visibility:public"]) 4 | 5 | licenses(["notice"]) 6 | 7 | # We can't set name="cpplint" here because that's the directory name so the 8 | # sandbox gets confused. We'll give it a private name with a public alias. 9 | py_binary( 10 | name = "cpplint_binary", 11 | srcs = ["cpplint/cpplint.py"], 12 | imports = ["cpplint"], 13 | main = "cpplint/cpplint.py", 14 | visibility = [], 15 | ) 16 | 17 | alias( 18 | name = "cpplint", 19 | actual = ":cpplint_binary", 20 | ) 21 | # exports_files(["cpplint/cpplint.py"]) 22 | -------------------------------------------------------------------------------- /tools/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | add_executable(cnn cnn.cpp) 3 | target_link_libraries(cnn core) 4 | -------------------------------------------------------------------------------- /tools/cnn.cpp: -------------------------------------------------------------------------------- 1 | /* --------------------------------------------------------------------- 2 | Copyright 2018-2019 Fangjun Kuang 3 | email: csukuangfj at gmail dot com 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a COPYING file of the GNU General Public License 13 | along with this program. If not, see 14 | ----------------------------------------------------------------- */ 15 | #include 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | #include "cnn/network.hpp" 22 | 23 | int main(int /*argc*/, char* argv[]) { 24 | google::InitGoogleLogging(argv[0]); 25 | FLAGS_alsologtostderr = true; 26 | FLAGS_colorlogtostderr = true; 27 | 28 | std::string filename = "../proto/trained.prototxt"; 29 | cnn::Network network(filename); 30 | network.reshape(); 31 | 32 | // y = 5 + 10x 33 | std::ostringstream ss; 34 | ss << "\n"; 35 | for (int i = 0; i < 10; i++) { 36 | network.get_data_top_mutable(0)[0]->d_[0] = i; 37 | network.perform_predication(); 38 | 39 | ss << "expect: " << (5 + 10 * i) << ", "; 40 | ss << "actual: " << network.get_predications()[0]; 41 | ss << "\n"; 42 | } 43 | 44 | LOG(INFO) << ss.str(); 45 | 46 | return 0; 47 | } 48 | -------------------------------------------------------------------------------- /tools/lint/BUILD.bazel: -------------------------------------------------------------------------------- 1 | # Copyright 2019. All Rights Reserved. 2 | # Author: csukuangfj@gmail.com (Fangjun Kuang) 3 | 4 | package( 5 | default_visibility = ["//visibility:public"], 6 | ) 7 | -------------------------------------------------------------------------------- /tools/lint/cpplint.bzl: -------------------------------------------------------------------------------- 1 | # -*- python -*- 2 | 3 | # modified from https://github.com/ApolloAuto/apollo/blob/master/tools/cpplint.bzl 4 | # also refer to https://github.com/RobotLocomotion/drake/blob/master/tools/lint/cpplint.bzl 5 | 6 | # From https://bazel.build/versions/master/docs/be/c-cpp.html#cc_library.srcs 7 | _SOURCE_EXTENSIONS = [source_ext for source_ext in """ 8 | .c 9 | .cc 10 | .cpp 11 | .cxx 12 | .c++.C 13 | .h 14 | .hh 15 | .hpp 16 | .hxx 17 | .inc 18 | """.split("\n") if len(source_ext)] 19 | 20 | # The cpplint.py command-line argument so it doesn't skip our files! 21 | _EXTENSIONS_ARGS = ["--extensions=" + ",".join( 22 | [ext[1:] for ext in _SOURCE_EXTENSIONS], 23 | )] 24 | 25 | def _extract_labels(srcs): 26 | """Convert a srcs= or hdrs= value to its set of labels.""" 27 | 28 | # Tuples are already labels. 29 | if type(srcs) == type(()): 30 | return list(srcs) 31 | return [] 32 | 33 | def _is_source_label(label): 34 | for extension in _SOURCE_EXTENSIONS: 35 | if label.endswith(extension): 36 | return True 37 | return False 38 | 39 | def _add_linter_rules(source_labels, source_filenames, name, data = None): 40 | # Common attributes for all of our py_test invocations. 41 | data = (data or []) 42 | size = "small" 43 | tags = ["cpplint"] 44 | 45 | # Google cpplint. 46 | cpplint_cfg = ["//:CPPLINT.cfg"] + native.glob(["CPPLINT.cfg"]) 47 | native.py_test( 48 | name = name + "_cpplint", 49 | srcs = ["@google_style_guide//:cpplint"], 50 | data = data + cpplint_cfg + source_labels, 51 | args = _EXTENSIONS_ARGS + source_filenames, 52 | main = "@google_style_guide//:cpplint/cpplint.py", 53 | size = size, 54 | tags = tags, 55 | ) 56 | 57 | def cpplint(data = None, extra_srcs = None): 58 | """For every rule in the BUILD file so far, adds a test rule that runs 59 | cpplint over the C++ sources listed in that rule. Thus, BUILD file authors 60 | should call this function at the *end* of every C++-related BUILD file. 61 | By default, only the CPPLINT.cfg from the project root and the current 62 | directory are used. Additional configs can be passed in as data labels. 63 | Sources that are not discoverable through the "sources so far" heuristic can 64 | be passed in as extra_srcs=[]. 65 | """ 66 | 67 | # Iterate over all rules. 68 | for rule in native.existing_rules().values(): 69 | # Extract the list of C++ source code labels and convert to filenames. 70 | candidate_labels = ( 71 | _extract_labels(rule.get("srcs", ())) + 72 | _extract_labels(rule.get("hdrs", ())) 73 | ) 74 | source_labels = [ 75 | label 76 | for label in candidate_labels 77 | if _is_source_label(label) 78 | ] 79 | source_filenames = ["$(location %s)" % x for x in source_labels] 80 | 81 | # Run the cpplint checker as a unit test. 82 | if len(source_filenames) > 0: 83 | _add_linter_rules(source_labels, source_filenames, rule["name"], data) 84 | 85 | # Lint all of the extra_srcs separately in a single rule. 86 | if extra_srcs: 87 | source_labels = extra_srcs 88 | source_filenames = ["$(location %s)" % x for x in source_labels] 89 | _add_linter_rules( 90 | source_labels, 91 | source_filenames, 92 | "extra_srcs_cpplint", 93 | data, 94 | ) 95 | -------------------------------------------------------------------------------- /tools/lint/readme.md: -------------------------------------------------------------------------------- 1 | 2 | # TODO 3 | 4 | Refer to https://github.com/RobotLocomotion/drake/tree/master/tools/lint 5 | for other various kinds of available lint tools. 6 | -------------------------------------------------------------------------------- /travis/install-bazel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019. All Rights Reserved. 4 | # Author: fangjun.kuang@gmail.com (Fangjun Kuang) 5 | 6 | BAZEL_VERSION=1.1.0 7 | 8 | if which bazel >/dev/null; then 9 | # get bazel version 10 | version=$(bazel version 2>/dev/null | grep "Build label" | awk '{print $NF}') 11 | if [[ $version == $BAZEL_VERSION ]]; then 12 | echo "bazel $BAZEL_VERSION has already been installed, skip" 13 | exit 0 14 | else 15 | echo "Replace bazel $version with $BAZEL_VERSION" 16 | fi 17 | fi 18 | 19 | os=$(uname -s | tr 'A-Z' 'a-z') 20 | if [[ $os != darwin && $os != linux ]]; then 21 | echo "We support only Darwin and Linux" 22 | echo "Current OS is: $(uname -s)" 23 | exit 1 24 | fi 25 | 26 | filename=bazel-$BAZEL_VERSION-installer-$os-x86_64.sh 27 | 28 | curl -L -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/$filename 29 | chmod +x $filename 30 | ./$filename --user 31 | 32 | rm $filename 33 | echo "Done. Installed to $(which bazel)" 34 | -------------------------------------------------------------------------------- /travis/mnist-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd build 4 | . ../examples/mnist/data/download_and_unzip_data.sh 5 | wget https://raw.githubusercontent.com/csukuangfj/OpenCNN-Models/master/mnist/mnist-bin.prototxt-20000 6 | ./mnist 0 7 | 8 | # you should see the test accuracy at the end of the output 9 | 10 | -------------------------------------------------------------------------------- /travis/script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019. All Rights Reserved. 4 | # Author: csukuangfj@gmail.com (Fangjun Kuang) 5 | 6 | CNN_ROOT_DIR=$(cd $(dirname ${BASH_SOURCE[0]})/.. && pwd) 7 | 8 | cd $CNN_ROOT_DIR 9 | 10 | ./travis/install-bazel.sh 11 | 12 | # bazel test --config=cpplint //cnn/... -s 13 | 14 | bazel test //cnn/... -s 15 | --------------------------------------------------------------------------------