├── .gitmodules ├── CMakeLists.txt ├── LICENSE ├── README.md ├── data └── test_data.bin ├── download_pack.sh ├── matlab ├── deepcompare.cpp └── make.m ├── opencv ├── CMakeLists.txt └── example.cpp ├── pytorch ├── README.md ├── create_dataset_file.py ├── eval.py └── requirements.txt ├── run_test.sh ├── src ├── loader.cpp ├── loader.h ├── test.cpp ├── wrapper.cpp └── wrapper.h ├── torch ├── extract_cpu.lua ├── extract_gpu.lua ├── match_cpu.lua └── match_gpu.lua └── training ├── FPR95Meter.lua ├── README.md ├── models ├── 2ch.lua ├── 2ch2stream.lua ├── 2chavg.lua ├── 2chdeep.lua ├── siam.lua ├── siam2stream.lua └── utils.lua └── train.lua /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "cunnproduction"] 2 | path = cunnproduction 3 | url = https://github.com/szagoruyko/cunnproduction 4 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.8 FATAL_ERROR) 2 | CMAKE_POLICY(VERSION 2.8) 3 | 4 | ADD_SUBDIRECTORY(cunnproduction) 5 | 6 | FIND_PACKAGE(CUDA 5.5 REQUIRED) 7 | FIND_PACKAGE(Torch) 8 | 9 | ADD_DEFINITIONS(-std=c++11 -fPIC) 10 | 11 | IF(DEFINED Torch_INSTALL_INCLUDE) 12 | SET(THC_INSTALL_INCLUDE ${Torch_INSTALL_INCLUDE}) 13 | SET(TH_INSTALL_INCLUDE ${Torch_INSTALL_INCLUDE}) 14 | SET(TH_INSTALL_LIB ${Torch_INSTALL_LIB}) 15 | SET(THC_INSTALL_LIB ${Torch_INSTALL_LIB}) 16 | SET(THCUNN_INSTALL_LIB ${Torch_INSTALL_LIB}/lua/5.1) 17 | ENDIF() 18 | 19 | INCLUDE_DIRECTORIES( 20 | ${THC_INSTALL_INCLUDE}/THC 21 | ${THC_INSTALL_INCLUDE} 22 | ${TH_INSTALL_INCLUDE}/TH 23 | ${TH_INSTALL_INCLUDE} 24 | ${CUDA_INCLUDE_DIRS} 25 | cunnproduction 26 | ) 27 | 28 | LINK_DIRECTORIES( 29 | ${TH_INSTALL_LIB} 30 | ${THC_INSTALL_LIB} 31 | ${THCUNN_INSTALL_LIB} 32 | ) 33 | 34 | ADD_LIBRARY(loader_static STATIC src/loader.cpp) 35 | ADD_LIBRARY(wrapper_static STATIC src/wrapper.cpp) 36 | 37 | ADD_EXECUTABLE(test_networks src/test.cpp) 38 | 39 | IF(WITH_OPENCV) 40 | ADD_SUBDIRECTORY(opencv) 41 | ENDIF() 42 | 43 | TARGET_LINK_LIBRARIES(test_networks cunnproduction_static TH THC loader_static) 44 | TARGET_LINK_LIBRARIES(loader_static TH THC cunnproduction_static) 45 | TARGET_LINK_LIBRARIES(wrapper_static TH THC cunnproduction_static loader_static) 46 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright © 2015 Ecole des Ponts, Universite Paris-Est 2 | 3 | All Rights Reserved. A license to use and copy this software and its documentation solely for your internal 4 | research and evaluation purposes, without fee and without a signed licensing agreement, is hereby granted 5 | upon your download of the software, through which you agree to the following: 6 | 1) the above copyright notice, this paragraph and the following three paragraphs will prominently appear 7 | in all internal copies and modifications; 8 | 2) no rights to sublicense or further distribute this software are granted; 9 | 3) no rights to modify this software are granted; and 10 | 4) no rights to assign this license are granted. 11 | 12 | Please Contact Prof. Nikos Komodakis, 13 | 6 Avenue Blaise Pascal - Cite Descartes, Champs-sur-Marne, 77455 Marne-la-Vallee cedex 2, France, Office B003 14 | Tel : +33164152173, Fax: +33164152186, nikos.komodakis@enpc.fr 15 | for commercial licensing opportunities, or for further distribution, modification or license rights. 16 | 17 | Created by Sergey Zagoruyko and Nikos Komodakis. http://imagine.enpc.fr/~komodakn/ 18 | 19 | IN NO EVENT SHALL ENPC, OR ITS EMPLOYEES, OFFICERS, AGENTS OR TRUSTEES (“COLLECTIVELY “ENPC PARTIES”) BE LIABLE 20 | TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY KIND , INCLUDING LOST 21 | PROFITS, ARISING OUT OF ANY CLAIM RESULTING FROM YOUR USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF ANY 22 | OF ENPC PARTIES HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH CLAIM OR DAMAGE. 23 | 24 | ENPC SPECIFICALLY DISCLAIMS ANY WARRANTIES OF ANY KIND REGARDING THE SOFTWARE, INCLUDING, BUT NOT LIMITED TO, 25 | NON-INFRINGEMENT, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, OR THE 26 | ACCURACY OR USEFULNESS, OR COMPLETENESS OF THE SOFTWARE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, 27 | PROVIDED HEREUNDER IS PROVIDED COMPLETELY "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE FURTHER DOCUMENTATION, 28 | MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. 29 | 30 | 31 | Please cite the paper below if you use this code in your research. 32 | 33 | Sergey Zagoruyko, Nikos Komodakis 34 | "Learning to Compare Image Patches via Convolutional Neural Networks", 35 | ArXiv:1504.03641. http://arxiv.org/abs/1504.03641 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Code for CVPR15 paper "Learning to Compare Image Patches via Convolutional Neural Networks" 2 | ----- 3 | This package allows researches to apply the described networks to match image patches and extract corresponding patches. 4 | 5 | We tried to make the code as easy to use as possible. The original models were trained with Torch ( http://torch.ch ) and we release them in Torch7 and binary formats with C++ bindings which do not require Torch installation. Thus we provide example code how to use the models in Torch, MATLAB and with OpenCV http://opencv.org 6 | 7 | CREDITS, LICENSE, CITATION 8 | ----- 9 | 10 | Copyright © 2015 Ecole des Ponts, Universite Paris-Est 11 | 12 | All Rights Reserved. A license to use and copy this software and its documentation solely for your internal 13 | research and evaluation purposes, without fee and without a signed licensing agreement, is hereby granted 14 | upon your download of the software, through which you agree to the following: 15 | 1) the above copyright notice, this paragraph and the following three paragraphs will prominently appear 16 | in all internal copies and modifications; 17 | 2) no rights to sublicense or further distribute this software are granted; 18 | 3) no rights to modify this software are granted; and 19 | 4) no rights to assign this license are granted. 20 | 21 | Please Contact Prof. Nikos Komodakis, 22 | 6 Avenue Blaise Pascal - Cite Descartes, Champs-sur-Marne, 77455 Marne-la-Vallee cedex 2, France for commercial licensing opportunities, or for further distribution, modification or license rights. 23 | 24 | Created by Sergey Zagoruyko and Nikos Komodakis. http://imagine.enpc.fr/~komodakn/ 25 | 26 | Please cite the paper below if you use this code in your research. 27 | 28 | Sergey Zagoruyko, Nikos Komodakis, 29 | "Learning to Compare Image Patches via Convolutional Neural Networks". http://www.cv-foundation.org/openaccess/content_cvpr_2015/papers/Zagoruyko_Learning_to_Compare_2015_CVPR_paper.pdf, bib: 30 | 31 | ``` 32 | @InProceedings{Zagoruyko_2015_CVPR, 33 | author = {Zagoruyko, Sergey and Komodakis, Nikos}, 34 | title = {Learning to Compare Image Patches via Convolutional Neural Networks}, 35 | booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, 36 | month = {June}, 37 | year = {2015} 38 | } 39 | ``` 40 | 41 | **Update 4** (April 2017) dead links for models and datasets fixed 42 | 43 | **Update 3** (July 2016) training code released 44 | 45 | **Update 2** (February 2016) caffe models released 46 | 47 | **Update 1** (January 2016) cudnn models removed because `cudnn.convert` was out 48 | 49 | 50 | ## Dataset 51 | 52 | The original dataset website is down, you can still download the files here: 53 | 54 |
55 |
56 |
57 | 58 | 59 | ### Models 60 | 61 | We provide the models in Torch7 and binary format. The table from the paper is here for convenience. 62 | 63 | **All models expect input patches to be in [0;1] range before mean subtraction.** 64 | 65 | **The models are not supposed to give outputs in [0;1] range, the outputs are not normalized** 66 | 67 | | Train set | Test set | 2ch | 2ch2stream | 2chdeep | siam | siam2stream | 68 | | --- | --- | :---: | :---: | :---: | :---: | :---: | 69 | | yosemite | notredame | 2.74 | **2.11** | 2.43 | 5.62 | 5.23 | 70 | | yosemite | liberty | 8.59 | **7.2** | 7.4 | 13.48 | 11.34 | 71 | | notredame | yosemite | 6.04 | **4.09** | 4.38 | 13.23 | 10.44 | 72 | | notredame | liberty | 6.04 | 4.85 | **4.56** | 8.77 | 6.45 | 73 | | liberty | yosemite | 7 | **5** | 6.18 | 14.76 | 9.39 | 74 | | liberty | notredame | 2.76 | **1.9** | 2.77 | 4.04 | 2.82 | 75 | 76 | 77 | Models in nn format can be loaded and used without CUDA support in Torch. To enable CUDA support ```model:cuda()``` call required. 78 | 79 | An archive with all models (binary and in torch format) is available at: 80 | 81 | ### Torch 82 | 83 | To install torch follow http://torch.ch/ 84 | Check torch folder for examples. 85 | Match patches on CPU: 86 | 87 | ```lua 88 | require 'nn' 89 | 90 | N = 76 -- the number of patches to match 91 | patches = torch.rand(N,2,64,64):float() 92 | 93 | -- load the network 94 | net = torch.load'../networks/2ch/2ch_liberty.t7' 95 | 96 | -- in place mean subtraction 97 | local p = patches:view(N,2,64*64) 98 | p:add(-p:mean(3):expandAs(p)) 99 | 100 | -- get the output similarities 101 | output = net:forward(patches) 102 | ``` 103 | 104 | Conversion to a faster `cudnn` backend is done by `cudnn.convert(net, cudnn)` function. 105 | 106 | ### C++ API 107 | 108 | The code was tested to work in Linux (Ubuntu 14.04) and OS X 10.10, although we release all the source code to enable usage in other operating systems. 109 | 110 | We release CUDA code for now, CPU code might be added in the future. To install it you need to have CUDA with the up-to-date CUDA driver (those are separate packages). 111 | 112 | Install TH and THC: 113 | 114 | ``` 115 | cd /tmp 116 | git clone https://github.com/torch/torch7.git 117 | cd torch7/lib/TH 118 | mkdir build; cd build 119 | cmake ..; make -j4 install 120 | cd /tmp 121 | git clone https://github.com/torch/cutorch.git; 122 | cd cutorch/lib/THC 123 | mkdir build; cd build 124 | cmake ..; make -j4 install 125 | ``` 126 | 127 | Clone and compile this repository it with: 128 | 129 | ``` 130 | git clone --recursive https://github.com/szagoruyko/cvpr15deepmatch 131 | cd cvpr15deepmatch 132 | mkdir build; cd build; 133 | cmake .. -DCMAKE_INSTALL_PREFIX=../install 134 | make -j4 install 135 | ``` 136 | 137 | Then you will have ```loadNetwork``` function defined in src/loader.h, which expects the state and the path to a network in binary format on input. A simple example: 138 | 139 | ```c++ 140 | THCState *state = (THCState*)malloc(sizeof(THCState)); 141 | THCudaInit(state); 142 | 143 | cunn::Sequential::Ptr net = loadNetwork(state, "networks/siam/siam_notredame.bin"); 144 | 145 | THCudaTensor *input = THCudaTensor_newWithSize4d(state, 128, 2, 64, 64); 146 | THCudaTensor *output = net->forward(input); // output is 128x1 similarity score tensor 147 | ``` 148 | 149 | Only 2D and 4D tensors accepted on input. 150 | 151 | Again, **all binary models expect input patches to be in [0;1] range before mean subtraction.** 152 | 153 | After you build everything and download the networks run test with ```run_test.sh```. It will download a small test_data.bin file. 154 | 155 | ### MATLAB 156 | 157 | Building Matlab bindings requires a little bit of user intervention. Open matlab/make.m file in Matlab and put your paths to Matlab and include/lib paths of TH and THC, then run ```>> make```. Mex file will be created. 158 | 159 | To initialize the interface do 160 | 161 | ``` 162 | deepcompare('init', 'networks/2ch/2ch_notredame.bin'); 163 | ``` 164 | To reset do 165 | 166 | ``` 167 | deepcompare('reset') 168 | ``` 169 | 170 | To propagate through the network: 171 | 172 | ``` 173 | deepcompare('forward', A) 174 | ``` 175 | ```A``` can be 2D, 3D or 4D array, which is converted inside to 2D or 4D array (Matlab is col-major and Torch is row-major so the array is transposed): 176 | 177 | | #dim | matlab dim | torch dim | 178 | | -- | -- | -- | 179 | | 2d | N x B | B x N | 180 | | 3d | 64 x 64 x N | 1 x N x 64 x 64 | 181 | | 4d | 64 x 64 x N x B | B x N x 64 x 64 | 182 | 183 | 2D or 4D tensor is returned. In case of full network propagation for example the output will be 2D: 1 x B, if input was B x 2 x 64 x 64. 184 | 185 | To set the number of GPU to be used (the numbering starts from 1): 186 | 187 | ``` 188 | deepcompare('set_device', 2) 189 | ``` 190 | Print the network structure: 191 | 192 | ``` 193 | deepcompare('print') 194 | ``` 195 | 196 | To enable splitting the computations of descriptor and decision parts in siamese networks we saved their binary parts. 197 | 198 | * siamese network: 199 | 200 | | Train Set | siam_desc | siam_decision | 201 | | --- | :---: | :---: | 202 | | yosemite | 3.47 MB,siam_desc_yosemite.bin | 1.00 MB,siam_decision_yosemite.bin | 203 | | notredame | 3.47 MB,siam_desc_notredame.bin | 1.0MB,siam_decision_notredame.bin | 204 | | liberty | 3.47 MB,siam_desc_liberty.bin | 1.00 MB,siam_decision_liberty.bin | 205 | 206 | * siam-2stream network: 207 | 208 | | Train Set | siam2stream_desc | siam2stream_decision | 209 | | --- | :---: | :---: | 210 | | yosemite | 9.16 MB,siam2stream_desc_yosemite.bin | 4.01 MB,siam2stream_decision_yosemite.bin | 211 | | notredame | 9.16 MB,siam2stream_desc_notredame.bin | 4.01 MB,siam2stream_decision_notredame.bin | 212 | | liberty | 9.16 MB,siam2stream_desc_liberty.bin | 4.01 MB,siam2stream_decision_liberty.bin | 213 | 214 | 215 | ### OpenCV 216 | 217 | OpenCV example is here to demonstrate how to use the deep CNN models to match image patches, how to preprocess the patches and use the proposed API. 218 | 219 | Depends on OpenCV 3.0. To build the example do 220 | 221 | ``` 222 | cd build; 223 | cmake -DWITH_OPENCV=ON -DOpenCV_DIR=/opt/opencv .; make -j8 224 | ``` 225 | Here ```/opt/opencv``` has to be a folder where OpenCV is built. If you have it installed, you don't need to add it, ```-DWITH_OPENCV=ON``` will be enough. 226 | 227 | To run the example download the images: 228 | 229 | ``` 230 | wget https://raw.githubusercontent.com/openMVG/ImageDataset_SceauxCastle/master/images/100_7100.JPG 231 | wget https://raw.githubusercontent.com/openMVG/ImageDataset_SceauxCastle/master/images/100_7101.JPG 232 | ``` 233 | 234 | and run it: 235 | 236 | ``` 237 | ./build/opencv/example networks/siam2stream/siam2stream_desc_notredame.bin 100_7100.JPG 100_7101.JPG 238 | ``` 239 | 240 | You have to use descriptor matching network in this example. Check the example code for explanation. 241 | 242 | 243 | ### CAFFE 244 | 245 | Thanks to the awesome @ajtulloch's [torch2caffe](https://github.com/facebook/fb-caffe-exts) models were converted to CAFFE format. Unfortunatelly only _siam_, _2ch_ and _2chdeep_ models could be converted at the time, other models will be converted as missing functionality is added to CAFFE. 246 | 247 | *Download link*: 248 | -------------------------------------------------------------------------------- /data/test_data.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szagoruyko/cvpr15deepcompare/4a850c8b4993e5c469904ee84309d53429d656ca/data/test_data.bin -------------------------------------------------------------------------------- /download_pack.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | wget https://s3.amazonaws.com/modelzoo-networks/cvpr2015matching_networks.tar.gz 4 | tar xvzf cvpr2015matching_networks.tar.gz 5 | -------------------------------------------------------------------------------- /matlab/deepcompare.cpp: -------------------------------------------------------------------------------- 1 | #include "mex.h" 2 | #include 3 | #include 4 | #include "wrapper.h" 5 | 6 | #define MEX_ARGS int nlhs, mxArray **plhs, int nrhs, const mxArray **prhs 7 | 8 | using namespace std; 9 | 10 | static std::shared_ptr net_; 11 | 12 | // Log and throw a Mex error 13 | inline void mex_error(const std::string &msg) { 14 | mexErrMsgTxt(msg.c_str()); 15 | } 16 | 17 | 18 | static void init(MEX_ARGS) 19 | { 20 | if(nrhs != 1) 21 | { 22 | ostringstream error_msg; 23 | error_msg << "Expected 1 arguments, got " << nrhs; 24 | mex_error(error_msg.str()); 25 | } 26 | 27 | char* param_file = mxArrayToString(prhs[0]); 28 | 29 | net_.reset(new Network(param_file)); 30 | mexPrintf(net_->tostring().c_str()); 31 | 32 | mxFree(param_file); 33 | } 34 | 35 | static void set_device(MEX_ARGS) 36 | { 37 | if (nrhs != 1) { 38 | ostringstream error_msg; 39 | error_msg << "Expected 1 argument, got " << nrhs; 40 | mex_error(error_msg.str()); 41 | } 42 | 43 | int device_id = static_cast(mxGetScalar(prhs[0])); 44 | net_->setDevice(device_id - 1); 45 | } 46 | 47 | static void forward(MEX_ARGS) 48 | { 49 | if (nrhs != 1) { 50 | ostringstream error_msg; 51 | error_msg << "Expected 1 argument, got " << nrhs; 52 | mex_error(error_msg.str()); 53 | } 54 | 55 | const mxArray *input_array = prhs[0]; 56 | if(!mxIsSingle(input_array)) 57 | mex_error("Expected single array"); 58 | 59 | const mwSize ndim = mxGetNumberOfDimensions(input_array); 60 | 61 | if(ndim > 4) 62 | mex_error("2D, 3D or 4D array expected"); 63 | 64 | // it is impossible to have a N1xN2xN3x1 array in Matlab! 65 | // but cunnproduction only expects 2d or 4d tensors 66 | // so have to do this: 67 | // 2d: N x B -> B x N 68 | // 3d: 64 x 64 x N -> 1 x N x 64 x 64 69 | // 4d: 64 x 64 x N x B -> B x N x 64 x 64 70 | 71 | mwSize tndim = ndim == 3 ? tndim : ndim; 72 | 73 | THLongStorage *input_sizes = THLongStorage_newWithSize(tndim); 74 | 75 | // have to invert the dimensions because torch is row-major 76 | // and matlab is col-major 77 | { 78 | const mwSize* size = mxGetDimensions(input_array); 79 | long *input_sizes_data = THLongStorage_data(input_sizes); 80 | if(ndim ==3) { 81 | input_sizes_data[0] = 1; 82 | for(mwSize i=0; iforward(input, output); 99 | 100 | mwSize dims[4]; 101 | const long noutput_dim = THFloatTensor_nDimension(output); 102 | for(long i=0; i < noutput_dim; ++i) 103 | dims[noutput_dim - i - 1] = output->size[i]; 104 | mxArray* output_array = mxCreateNumericArray(noutput_dim, dims, mxSINGLE_CLASS, mxREAL); 105 | 106 | memcpy(mxGetData(output_array), 107 | THFloatTensor_data(output), 108 | THFloatTensor_nElement(output) * sizeof(float)); 109 | 110 | plhs[0] = output_array; 111 | 112 | THFloatTensor_free(input); 113 | THFloatTensor_free(output); 114 | } 115 | 116 | static void print(MEX_ARGS) 117 | { 118 | if(net_) { 119 | mexPrintf(net_->tostring().c_str()); 120 | } 121 | } 122 | 123 | static void reset(MEX_ARGS) 124 | { 125 | if(net_) { 126 | net_.reset(); 127 | } 128 | } 129 | 130 | struct handler_registry { 131 | string cmd; 132 | void (*func)(MEX_ARGS); 133 | }; 134 | 135 | static handler_registry handlers[] = { 136 | // Public API functions 137 | { "forward", forward }, 138 | { "init", init }, 139 | { "set_device", set_device }, 140 | { "reset", reset }, 141 | { "print", print }, 142 | { "END", NULL }, 143 | }; 144 | 145 | 146 | void mexFunction(MEX_ARGS) { 147 | mexLock(); 148 | if (nrhs == 0) { 149 | mex_error("No API command given"); 150 | return; 151 | } 152 | 153 | { // Handle input command 154 | char *cmd = mxArrayToString(prhs[0]); 155 | bool dispatched = false; 156 | // Dispatch to cmd handler 157 | for (int i = 0; handlers[i].func != NULL; i++) { 158 | if (handlers[i].cmd.compare(cmd) == 0) { 159 | handlers[i].func(nlhs, plhs, nrhs-1, prhs+1); 160 | dispatched = true; 161 | break; 162 | } 163 | } 164 | if (!dispatched) { 165 | ostringstream error_msg; 166 | error_msg << "Unknown command '" << cmd << "'"; 167 | mex_error(error_msg.str()); 168 | } 169 | mxFree(cmd); 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /matlab/make.m: -------------------------------------------------------------------------------- 1 | % Uncomment this for OS X 2 | % and put your paths to TH, THC and Matlab 3 | %{ 4 | mex -I../src... 5 | -I/usr/local/include... 6 | -L/usr/local/lib... 7 | -L/usr/local/cuda/lib... 8 | -L/Applications/MATLAB_R2014a.app/bin/maci64/... 9 | ../build/libloader_static.a... 10 | ../build/libwrapper_static.a... 11 | ../build/cunnproduction/libcunnproduction_static.a... 12 | -lcudart -lTH -lTHC -lmex -lmat -lmx... 13 | deepcompare.cpp 14 | %} 15 | 16 | % Linux 17 | mex -I../src... 18 | -I/opt/rocks/distro/install/include... 19 | -L/opt/rocks/distro/install/lib... 20 | -L/usr/local/cuda/lib64... 21 | -L/usr/local/MATLAB/R2014a/bin/glnxa64... 22 | CXXFLAGS='-std=c++11 -fPIC'... 23 | ../build/libwrapper_static.a... 24 | ../build/libloader_static.a... 25 | ../build/cunnproduction/libcunnproduction_static.a... 26 | ../build/cunnproduction/libcunn.a... 27 | -lcudart -lTH -lTHC -lmex -lmat -lmx... 28 | deepcompare.cpp 29 | -------------------------------------------------------------------------------- /opencv/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.8 FATAL_ERROR) 2 | CMAKE_POLICY(VERSION 2.8) 3 | 4 | FIND_PACKAGE(OpenCV REQUIRED) 5 | 6 | INCLUDE_DIRECTORIES( 7 | ${OpenCV_INCLUDE_DIRS} 8 | ${CMAKE_SOURCE_DIR}/src 9 | ${CMAKE_SOURCE_DIR}/cunnproduction 10 | ) 11 | 12 | ADD_EXECUTABLE(example example.cpp) 13 | 14 | TARGET_LINK_LIBRARIES(example loader_static 15 | cunnproduction 16 | TH THC 17 | opencv_core opencv_highgui opencv_imgproc opencv_imgcodecs opencv_features2d 18 | ) 19 | -------------------------------------------------------------------------------- /opencv/example.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Sergey Zagoruyko, Nikos Komodakis 2 | // sergey.zagoruyko@imagine.enpc.fr, nikos.komodakis@enpc.fr 3 | // Ecole des Ponts ParisTech, Universite Paris-Est, IMAGINE 4 | // 5 | // The software is free to use only for non-commercial purposes. 6 | // IF YOU WOULD LIKE TO USE IT FOR COMMERCIAL PURPOSES, PLEASE CONTACT 7 | // Prof. Nikos Komodakis (nikos.komodakis@enpc.fr) 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include 15 | #include 16 | #include 17 | #include "loader.h" 18 | 19 | // defined by network architecture 20 | #define M 64 21 | 22 | // Given an image an coordinates+sizes of detected points 23 | // extract corresponding image patches with OpenCV functions 24 | // input image is in [0 255] range, 25 | // the patches are divided by 255 and mean-normalized 26 | // Note: depending on the type of applications you might want to use 27 | // orientation of detected region or inrease the cropped bounding box by some 28 | // constant 29 | // Another note: this is of course not the fastest way to extracted features. 30 | // The ultimate would be to use CUDA texture memory and process all the features 31 | // from am image in parallel 32 | void extractPatches(const cv::Mat& image, 33 | const std::vector& kp, 34 | std::vector& patches) 35 | { 36 | for(auto &it : kp) 37 | { 38 | cv::Mat patch(M, M, CV_32F); 39 | cv::Mat buf; 40 | // increase the size of the region to include some context 41 | cv::getRectSubPix(image, cv::Size(it.size*1.3, it.size*1.3), it.pt, buf); 42 | cv::Scalar m = cv::mean(buf); 43 | cv::resize(buf, patch, cv::Size(M,M)); 44 | patch.convertTo(patch, CV_32F, 1./255.); 45 | patch = patch.isContinuous() ? patch : patch.clone(); 46 | // mean subtraction is crucial! 47 | patches.push_back(patch - m[0]/255.); 48 | } 49 | } 50 | 51 | // Copy extracted patches to CUDA memory and run the network 52 | // One has to keep mind that GPU memory is limited and extracting too many patches 53 | // at once might cause troubles 54 | // So if you need to extract a lot of patches, an efficient way would be to 55 | // devide the set in smaller equal parts and preallocate CPU and GPU memory 56 | void extractDescriptors(THCState *state, 57 | cunn::Sequential::Ptr net, 58 | const std::vector& patches, 59 | cv::Mat& descriptors) 60 | { 61 | size_t batch_size = 128; 62 | size_t N = patches.size(); 63 | 64 | THFloatTensor *buffer = THFloatTensor_newWithSize4d(batch_size, 1, M, M); 65 | THCudaTensor *input = THCudaTensor_newWithSize4d(state, batch_size, 1, M, M); 66 | 67 | for(int j=0; j < ceil((float)N/batch_size); ++j) 68 | { 69 | float *data = THFloatTensor_data(buffer); 70 | size_t k = 0; 71 | for(size_t i = j*batch_size; i < std::min((j+1)*batch_size, N); ++i, ++k) 72 | memcpy(data + k*M*M, patches[i].data, sizeof(float) * M * M); 73 | 74 | // initialize 4D CUDA tensor and copy patches into it 75 | THCudaTensor_copyFloat(state, input, buffer); 76 | 77 | // propagate through the network 78 | THCudaTensor *output = net->forward(input); 79 | 80 | // copy descriptors back 81 | THFloatTensor *desc = THFloatTensor_newWithSize2d(output->size[0], output->size[1]); 82 | THFloatTensor_copyCuda(state, desc, output); 83 | 84 | size_t feature_dim = output->size[1]; 85 | if(descriptors.cols != feature_dim || descriptors.rows != N) 86 | descriptors.create(N, feature_dim, CV_32F); 87 | 88 | memcpy(descriptors.data + j * feature_dim * batch_size * sizeof(float), 89 | THFloatTensor_data(desc), 90 | sizeof(float) * feature_dim * k); 91 | 92 | THFloatTensor_free(desc); 93 | } 94 | 95 | THCudaTensor_free(state, input); 96 | THFloatTensor_free(buffer); 97 | } 98 | 99 | 100 | int main(int argc, char** argv) 101 | { 102 | THCState *state = (THCState*)malloc(sizeof(THCState)); 103 | THCudaInit(state); 104 | 105 | if(argc < 3) 106 | { 107 | std::cout << "arguments: [network] [image1] [image2]\n"; 108 | return 1; 109 | } 110 | 111 | const char *network_path = argv[1]; 112 | auto net = loadNetwork(state, network_path); 113 | 114 | // load the images 115 | cv::Mat ima = cv::imread(argv[2]); 116 | cv::Mat imb = cv::imread(argv[3]); 117 | 118 | if(ima.empty() || imb.empty()) 119 | { 120 | std::cout << "images not found\n"; 121 | return 1; 122 | } 123 | 124 | cv::Mat ima_gray, imb_gray; 125 | cv::cvtColor(ima, ima_gray, cv::COLOR_BGR2GRAY); 126 | cv::cvtColor(imb, imb_gray, cv::COLOR_BGR2GRAY); 127 | 128 | // Here we set min_area parameter to a bigger value, like that minimal size 129 | // of a patch will be around 11x11, because the network was trained on bigger patches 130 | // this parameter is important in practice 131 | cv::Ptr detector = cv::MSER::create(5, 620); 132 | std::vector kpa, kpb; 133 | detector->detect(ima_gray, kpa); 134 | detector->detect(imb_gray, kpb); 135 | std::cout << "image A MSER points detected: " << kpa.size() << std::endl; 136 | std::cout << "image B MSER points detected: " << kpb.size() << std::endl; 137 | 138 | std::vector patches_a, patches_b; 139 | extractPatches(ima_gray, kpa, patches_a); 140 | extractPatches(imb_gray, kpb, patches_b); 141 | 142 | cv::Mat descriptors_a, descriptors_b; 143 | extractDescriptors(state, net, patches_a, descriptors_a); 144 | extractDescriptors(state, net, patches_b, descriptors_b); 145 | 146 | cv::FlannBasedMatcher matcher; 147 | std::vector matches; 148 | matcher.match( descriptors_a, descriptors_b, matches ); 149 | 150 | double max_dist = 0; double min_dist = 100; 151 | 152 | //-- Quick calculation of max and min distances between keypoints 153 | for( int i = 0; i < descriptors_a.rows; i++ ) 154 | { double dist = matches[i].distance; 155 | if( dist < min_dist ) min_dist = dist; 156 | if( dist > max_dist ) max_dist = dist; 157 | } 158 | 159 | printf("-- Max dist : %f \n", max_dist ); 160 | printf("-- Min dist : %f \n", min_dist ); 161 | 162 | 163 | std::vector good_matches; 164 | for( int i = 0; i < descriptors_a.rows; i++ ) 165 | { if( matches[i].distance <= std::max(4*min_dist, 0.02) ) 166 | { good_matches.push_back( matches[i]); } 167 | } 168 | 169 | //-- Draw only "good" matches 170 | float f = 0.25; 171 | cv::resize(ima, ima, cv::Size(), f, f); 172 | cv::resize(imb, imb, cv::Size(), f, f); 173 | for(auto &it: kpa) { it.pt *= f; it.size *= f; } 174 | for(auto &it: kpb) { it.pt *= f; it.size *= f; } 175 | cv::Mat img_matches; 176 | cv::drawMatches( ima, kpa, imb, kpb, 177 | good_matches, img_matches, cv::Scalar::all(-1), cv::Scalar::all(-1), 178 | std::vector(), cv::DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS ); 179 | 180 | for(auto &it : kpa) 181 | cv::circle(ima, cv::Point(it.pt.x, it.pt.y), it.size, cv::Scalar(255,255,0)); 182 | for(auto &it : kpb) 183 | cv::circle(imb, cv::Point(it.pt.x, it.pt.y), it.size, cv::Scalar(255,255,0)); 184 | 185 | cv::imshow("matches", img_matches); 186 | //cv::imshow("keypoints image 1", ima); 187 | //cv::imshow("keypoints image 2", imb); 188 | cv::waitKey(); 189 | THCudaShutdown(state); 190 | 191 | return 0; 192 | } 193 | -------------------------------------------------------------------------------- /pytorch/README.md: -------------------------------------------------------------------------------- 1 | PyTorch DeepCompare 2 | ============ 3 | 4 | So far only evaluation code is supported. Run `eval.py` script to evaluate a model on a particular subset. 5 | You will need to use lua to convert dataset to the compatible format. `eval.py` has functional definitions 6 | for all models, so that you can easily grab the parameters and model function and reuse it in your own code. 7 | 8 | The numbers on Brown dataset are slightly different due to different evaluation code: 9 | 10 |
setssiam2stream_l2siam2stream2chsiamsiam_l22ch2stream
yosemite, notredame5.635.373.055.768.402.22
yosemite, liberty12.0211.039.0213.5818.907.48
notredame, yosemite12.8510.295.7312.6315.183.99
notredame, liberty7.936.195.868.6212.555.46
liberty, yosemite13.329.427.6515.2920.115.27
liberty, notredame5.193.163.024.436.051.88
11 | 12 | ## Installation 13 | 14 | Install PyTorch following instructions from , 15 | then run: 16 | 17 | ```bash 18 | pip install -r requirements.txt 19 | ``` 20 | 21 | And install `torchnet`: 22 | 23 | ```bash 24 | pip install git+https://github.com/pytorch/tnt.git@master 25 | ``` -------------------------------------------------------------------------------- /pytorch/create_dataset_file.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import os 3 | import argparse 4 | import numpy as np 5 | from tqdm import tqdm 6 | 7 | 8 | parser = argparse.ArgumentParser('DeepCompare data preparation') 9 | parser.add_argument('--data_dir', required=True, type=str) 10 | parser.add_argument('--test', action='store_true') 11 | 12 | 13 | def read_matches_file(filename): 14 | print(filename) 15 | data = np.loadtxt(filename, dtype=np.uint64) 16 | mask = data[:,1] == data[:,4] 17 | pairs = data[:,(0, 3)] 18 | return pairs[mask], pairs[np.logical_not(mask)] 19 | 20 | 21 | if __name__ == '__main__': 22 | opt = parser.parse_args() 23 | impath = opt.data_dir 24 | match_data = {name: read_matches_file(os.path.join(impath, name)) 25 | for name in os.listdir(impath) if name.startswith('m50_')} 26 | 27 | info = np.loadtxt(os.path.join(impath, 'info.txt'), dtype=np.uint64)[:,0] 28 | 29 | image_list = filter(lambda x: x.endswith('.bmp'), os.listdir(impath)) 30 | 31 | patches = [] 32 | for name in tqdm(sorted(image_list)): 33 | im = cv2.imread(os.path.join(impath, name), cv2.IMREAD_GRAYSCALE) 34 | patches.append(im.reshape(16,64,16,64).transpose(0,2,1,3).reshape(-1,64,64)) 35 | patches = np.concatenate(patches)[:info.size] 36 | mean = np.mean(patches, axis=(1,2)) 37 | 38 | np.save(arr={'patches': patches, 39 | 'mean': mean, 40 | 'info': info, 41 | 'match_data': match_data}, 42 | file=open(os.path.join(impath, 'data.npy'), 'w')) 43 | 44 | if opt.test: 45 | import torch 46 | from torch.utils.serialization import load_lua 47 | data_torch = load_lua(os.path.join(impath, 'data.t7')) 48 | 49 | print((torch.from_numpy(patches).float() - data_torch['patches'].float()).abs().max()) 50 | -------------------------------------------------------------------------------- /pytorch/eval.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os 3 | import sys 4 | import argparse 5 | from functools import partial 6 | from tqdm import tqdm 7 | import numpy as np 8 | import torch 9 | from torch.utils.serialization import load_lua 10 | from torchnet.dataset import ListDataset, ConcatDataset 11 | from torch.autograd import Variable 12 | import torch.nn.functional as F 13 | from sklearn import metrics 14 | from scipy import interpolate 15 | from torch.backends import cudnn 16 | cudnn.benchmark = True 17 | 18 | parser = argparse.ArgumentParser(description='DeepCompare PyTorch evaluation code') 19 | 20 | parser.add_argument('--model', default='2ch', type=str) 21 | parser.add_argument('--lua_model', default='', type=str, required=True) 22 | parser.add_argument('--nthread', default=4, type=int) 23 | parser.add_argument('--gpu_id', default='0', type=str) 24 | 25 | parser.add_argument('--batch_size', default=256, type=int) 26 | parser.add_argument('--test_set', default='liberty', type=str) 27 | parser.add_argument('--test_matches', default='m50_100000_100000_0.txt', type=str) 28 | 29 | 30 | def get_iterator(dataset, batch_size, nthread): 31 | def get_list_dataset(pair_type): 32 | ds = ListDataset(elem_list=dataset[pair_type], 33 | load=lambda idx: {'input': np.stack((dataset['patches'][v].astype(np.float32) 34 | - dataset['mean'][v]) / 256.0 for v in idx), 35 | 'target': 1 if pair_type == 'matches' else -1}) 36 | ds = ds.transform({'input': torch.from_numpy, 'target': lambda x: torch.LongTensor([x])}) 37 | 38 | return ds.batch(policy='include-last', batchsize=batch_size // 2) 39 | 40 | concat = ConcatDataset([get_list_dataset('matches'), 41 | get_list_dataset('nonmatches')]) 42 | 43 | return concat.parallel(batch_size=2, shuffle=False, num_workers=nthread) 44 | 45 | 46 | def conv2d(input, params, base, stride=1, padding=0): 47 | return F.conv2d(input, params[base + '.weight'], params[base + '.bias'], 48 | stride, padding) 49 | 50 | 51 | def linear(input, params, base): 52 | return F.linear(input, params[base + '.weight'], params[base + '.bias']) 53 | 54 | 55 | ##################### 2ch ##################### 56 | 57 | def deepcompare_2ch(input, params): 58 | o = conv2d(input, params, 'conv0', stride=3) 59 | o = F.max_pool2d(F.relu(o), 2, 2) 60 | o = conv2d(o, params, 'conv1') 61 | o = F.max_pool2d(F.relu(o), 2, 2) 62 | o = conv2d(o, params, 'conv2') 63 | o = F.relu(o).view(o.size(0), -1) 64 | return linear(o, params, 'fc') 65 | 66 | 67 | ##################### 2ch2stream ##################### 68 | 69 | def deepcompare_2ch2stream(input, params): 70 | 71 | def stream(input, name): 72 | o = conv2d(input, params, name + '.conv0') 73 | o = F.max_pool2d(F.relu(o), 2, 2) 74 | o = conv2d(o, params, name + '.conv1') 75 | o = F.max_pool2d(F.relu(o), 2, 2) 76 | o = conv2d(o, params, name + '.conv2') 77 | o = F.relu(o) 78 | o = conv2d(o, params, name + '.conv3') 79 | o = F.relu(o) 80 | return o.view(o.size(0), -1) 81 | 82 | o_fovea = stream(F.avg_pool2d(input, 2, 2), 'fovea') 83 | o_retina = stream(F.pad(input, (-16,) * 4), 'retina') 84 | o = linear(torch.cat([o_fovea, o_retina], dim=1), params, 'fc0') 85 | return linear(F.relu(o), params, 'fc1') 86 | 87 | 88 | ##################### siam ##################### 89 | 90 | def siam(patch, params): 91 | o = conv2d(patch, params, 'conv0', stride=3) 92 | o = F.max_pool2d(F.relu(o), 2, 2) 93 | o = conv2d(o, params, 'conv1') 94 | o = F.max_pool2d(F.relu(o), 2, 2) 95 | o = conv2d(o, params, 'conv2') 96 | o = F.relu(o) 97 | return o.view(o.size(0), -1) 98 | 99 | 100 | def deepcompare_siam(input, params): 101 | o = linear(torch.cat(map(partial(siam, params=params), input.split(1, dim=1)), 102 | dim=1), params, 'fc0') 103 | return linear(F.relu(o), params, 'fc1') 104 | 105 | 106 | def deepcompare_siam_l2(input, params): 107 | def single(patch): 108 | return F.normalize(siam(patch, params)) 109 | return - F.pairwise_distance(*map(single, input.split(1, dim=1))) 110 | 111 | 112 | ##################### siam2stream ##################### 113 | 114 | 115 | def siam_stream(patch, params, base): 116 | o = conv2d(patch, params, base + '.conv0', stride=2) 117 | o = F.max_pool2d(F.relu(o), 2, 2) 118 | o = conv2d(o, params, base + '.conv1') 119 | o = F.relu(o) 120 | o = conv2d(o, params, base + '.conv2') 121 | o = F.relu(o) 122 | o = conv2d(o, params, base + '.conv3') 123 | return o.view(o.size(0), -1) 124 | 125 | 126 | def streams(patch, params): 127 | o_retina = siam_stream(F.pad(patch, (-16,) * 4), params, 'retina') 128 | o_fovea = siam_stream(F.avg_pool2d(patch, 2, 2), params, 'fovea') 129 | return torch.cat([o_retina, o_fovea], dim=1) 130 | 131 | 132 | def deepcompare_siam2stream(input, params): 133 | embeddings = map(partial(streams, params=params), input.split(1, dim=1)) 134 | o = linear(torch.cat(embeddings, dim=1), params, 'fc0') 135 | o = F.relu(o) 136 | o = linear(o, params, 'fc1') 137 | return o 138 | 139 | 140 | def deepcompare_siam2stream_l2(input, params): 141 | def single(patch): 142 | return F.normalize(streams(patch, params)) 143 | return - F.pairwise_distance(*map(single, input.split(1, dim=1))) 144 | 145 | 146 | models = { 147 | '2ch': deepcompare_2ch, 148 | '2ch2stream': deepcompare_2ch2stream, 149 | 'siam': deepcompare_siam, 150 | 'siam_l2': deepcompare_siam_l2, 151 | 'siam2stream': deepcompare_siam2stream, 152 | 'siam2stream_l2': deepcompare_siam2stream_l2, 153 | } 154 | 155 | 156 | def main(args): 157 | opt = parser.parse_args(args) 158 | print('parsed options:', vars(opt)) 159 | 160 | os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpu_id 161 | if torch.cuda.is_available(): 162 | # to prevent opencv from initializing CUDA in workers 163 | torch.randn(8).cuda() 164 | os.environ['CUDA_VISIBLE_DEVICES'] = '' 165 | 166 | def load_provider(): 167 | print('Loading test data') 168 | 169 | p = np.load(opt.test_set)[()] 170 | 171 | for i, t in enumerate(['matches', 'nonmatches']): 172 | p[t] = p['match_data'][opt.test_matches][i] 173 | 174 | return p 175 | 176 | test_iter = get_iterator(load_provider(), opt.batch_size, opt.nthread) 177 | 178 | def cast(t): 179 | return t.cuda() if torch.cuda.is_available() else t 180 | 181 | f = models[opt.model] 182 | net = load_lua(opt.lua_model) 183 | 184 | if opt.model == '2ch': 185 | params = {} 186 | for j, i in enumerate([0, 3, 6]): 187 | params['conv%d.weight' % j] = net.get(i).weight 188 | params['conv%d.bias' % j] = net.get(i).bias 189 | params['fc.weight'] = net.get(9).weight 190 | params['fc.bias'] = net.get(9).bias 191 | elif opt.model == '2ch2stream': 192 | params = {} 193 | for j, branch in enumerate(['fovea', 'retina']): 194 | for k, layer in enumerate(map(net.get(0).get(j).get(1).get, [1, 4, 7, 9])): 195 | params['%s.conv%d.weight' % (branch, k)] = layer.weight 196 | params['%s.conv%d.bias' % (branch, k)] = layer.bias 197 | for k, layer in enumerate(map(net.get, [1, 3])): 198 | params['fc%d.weight' % k] = layer.weight 199 | params['fc%d.bias' % k] = layer.bias 200 | elif opt.model == 'siam' or opt.model == 'siam_l2': 201 | params = {} 202 | for k, layer in enumerate(map(net.get(0).get(0).get, [1, 4, 7])): 203 | params['conv%d.weight' % k] = layer.weight 204 | params['conv%d.bias' % k] = layer.bias 205 | for k, layer in enumerate(map(net.get, [1, 3])): 206 | params['fc%d.weight' % k] = layer.weight 207 | params['fc%d.bias' % k] = layer.bias 208 | elif opt.model == 'siam2stream' or opt.model == 'siam2stream_l2': 209 | params = {} 210 | for stream, name in zip(net.get(0).get(0).modules, ['retina', 'fovea']): 211 | for k, layer in enumerate(map(stream.get, [2, 5, 7, 9])): 212 | params['%s.conv%d.weight' % (name, k)] = layer.weight 213 | params['%s.conv%d.bias' % (name, k)] = layer.bias 214 | for k, layer in enumerate(map(net.get, [1, 3])): 215 | params['fc%d.weight' % k] = layer.weight 216 | params['fc%d.bias' % k] = layer.bias 217 | 218 | params = {k: Variable(cast(v)) for k, v in params.items()} 219 | 220 | def create_variables(sample): 221 | inputs = Variable(cast(sample['input'].float().view(-1, 2, 64, 64))) 222 | targets = Variable(cast(sample['target'].float().view(-1))) 223 | return inputs, targets 224 | 225 | test_outputs, test_targets = [], [] 226 | for sample in tqdm(test_iter, dynamic_ncols=True): 227 | inputs, targets = create_variables(sample) 228 | y = f(inputs, params) 229 | test_targets.append(sample['target'].view(-1)) 230 | test_outputs.append(y.data.cpu().view(-1)) 231 | 232 | fpr, tpr, thresholds = metrics.roc_curve(torch.cat(test_targets).numpy(), 233 | torch.cat(test_outputs).numpy(), pos_label=1) 234 | fpr95 = float(interpolate.interp1d(tpr, fpr)(0.95)) 235 | 236 | print('FPR95:', fpr95) 237 | 238 | return fpr95 239 | 240 | 241 | if __name__ == '__main__': 242 | main(sys.argv[1:]) 243 | -------------------------------------------------------------------------------- /pytorch/requirements.txt: -------------------------------------------------------------------------------- 1 | tqdm 2 | numpy 3 | sklearn 4 | scipy 5 | -------------------------------------------------------------------------------- /run_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ ! -f test_data.bin ]; then 4 | cp ./data/test_data.bin ./test_data.bin 5 | fi 6 | 7 | if [ ! -d networks ]; then 8 | echo 'run download_pack.sh before running run_test.sh' 9 | else 10 | ./build/test_networks 11 | fi 12 | -------------------------------------------------------------------------------- /src/loader.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Sergey Zagoruyko, Nikos Komodakis 2 | // sergey.zagoruyko@imagine.enpc.fr, nikos.komodakis@enpc.fr 3 | // Ecole des Ponts ParisTech, Universite Paris-Est, IMAGINE 4 | // 5 | // The software is free to use only for non-commercial purposes. 6 | // IF YOU WOULD LIKE TO USE IT FOR COMMERCIAL PURPOSES, PLEASE CONTACT 7 | // Prof. Nikos Komodakis (nikos.komodakis@enpc.fr) 8 | #include 9 | #include 10 | #include 11 | 12 | #define CHECK_EQ(CONDITION) \ 13 | do { \ 14 | if( !(CONDITION) ) \ 15 | printf("%d: assertion failed\n", __LINE__); \ 16 | } while (0) 17 | 18 | // Additional simple module for iris cropping 19 | namespace cunn { 20 | 21 | class CentralCrop : public Module { 22 | public: 23 | CentralCrop(THCState *state, int dimx, int dimy) : Module(state), dimx(dimx), dimy(dimy) {} 24 | ~CentralCrop() {} 25 | 26 | THCudaTensor* forward(THCudaTensor *input) 27 | { 28 | THCudaTensor* crop1 = THCudaTensor_newNarrow(state, input, 2, dimx/4, dimx/2); 29 | THCudaTensor* crop2 = THCudaTensor_newNarrow(state, crop1, 3, dimy/4, dimy/2); 30 | THCudaTensor_resizeAs(state, output, crop2); 31 | THCudaTensor_copy(state, output, crop2); 32 | THCudaTensor_free(state, crop2); 33 | THCudaTensor_free(state, crop1); 34 | return output; 35 | } 36 | 37 | int dimx, dimy; 38 | }; 39 | 40 | } 41 | 42 | // read weights for one module from file 43 | template 44 | void readParameters(THCState *state, T* casted, FILE* f) 45 | { 46 | THFloatTensor *weight = THFloatTensor_newWithSize1d(THCudaTensor_nElement(state, casted->weight)); 47 | THFloatTensor *bias = THFloatTensor_newWithSize1d(THCudaTensor_nElement(state, casted->bias)); 48 | int w_numel = 0, b_numel = 0; 49 | fread(&w_numel, sizeof(int), 1, f); 50 | fread(&b_numel, sizeof(int), 1, f); 51 | 52 | CHECK_EQ(THFloatTensor_nElement(weight) == w_numel); 53 | CHECK_EQ(THFloatTensor_nElement(bias) == b_numel); 54 | //std::cout << THFloatTensor_nElement(weight) << " " << w_numel << std::endl; 55 | //std::cout << THFloatTensor_nElement(bias) << " " << b_numel << std::endl; 56 | 57 | fread(THFloatTensor_data(weight), sizeof(float)*THFloatTensor_nElement(weight), 1, f); 58 | fread(THFloatTensor_data(bias), sizeof(float)*THFloatTensor_nElement(bias), 1, f); 59 | 60 | THCudaTensor_copyFloat(state, casted->weight, weight); 61 | THCudaTensor_copyFloat(state, casted->bias, bias); 62 | THFloatTensor_free(weight); 63 | THFloatTensor_free(bias); 64 | } 65 | 66 | cunn::Concat::Ptr createSiam2streamBranch(THCState *state) 67 | { 68 | std::vector size = {1, 64, 64}; 69 | cunn::Sequential::Ptr sub_branch1_1 = std::make_shared(state); 70 | sub_branch1_1->add(std::make_shared(state, size)); 71 | sub_branch1_1->add(std::make_shared(state, 64, 64)); 72 | sub_branch1_1->add(std::make_shared(state, 1,96, 4,4, 2,2)); 73 | sub_branch1_1->add(std::make_shared(state)); 74 | sub_branch1_1->add(std::make_shared(state, 2,2,2,2)); 75 | sub_branch1_1->add(std::make_shared(state, 96,192, 3,3)); 76 | sub_branch1_1->add(std::make_shared(state)); 77 | sub_branch1_1->add(std::make_shared(state, 192,256, 3,3)); 78 | sub_branch1_1->add(std::make_shared(state)); 79 | sub_branch1_1->add(std::make_shared(state, 256,256, 3,3)); 80 | sub_branch1_1->add(std::make_shared(state, std::vector(1,256))); 81 | 82 | cunn::Sequential::Ptr sub_branch1_2 = std::make_shared(state); 83 | sub_branch1_2->add(std::make_shared(state, size)); 84 | sub_branch1_2->add(std::make_shared(state, 2,2,2,2)); 85 | sub_branch1_2->add(std::make_shared(state, 1,96, 4,4, 2,2)); 86 | sub_branch1_2->add(std::make_shared(state)); 87 | sub_branch1_2->add(std::make_shared(state, 2,2,2,2)); 88 | sub_branch1_2->add(std::make_shared(state, 96,192, 3,3)); 89 | sub_branch1_2->add(std::make_shared(state)); 90 | sub_branch1_2->add(std::make_shared(state, 192,256, 3,3)); 91 | sub_branch1_2->add(std::make_shared(state)); 92 | sub_branch1_2->add(std::make_shared(state, 256,256, 3,3)); 93 | sub_branch1_2->add(std::make_shared(state, std::vector(1,256))); 94 | 95 | cunn::Concat::Ptr sub_branch1 = std::make_shared(state, 1); 96 | 97 | sub_branch1->add(sub_branch1_1); 98 | sub_branch1->add(sub_branch1_2); 99 | 100 | return sub_branch1; 101 | } 102 | 103 | // load the full network 104 | cunn::Sequential::Ptr 105 | loadNetwork(THCState* state, const char* filename) 106 | { 107 | cunn::Sequential::Ptr net = std::make_shared(state); 108 | 109 | FILE *f = fopen(filename, "rb"); 110 | if(f == NULL) 111 | { 112 | std::stringstream s; 113 | s << "file " << filename << " not found"; 114 | THError(s.str().c_str()); 115 | return net; 116 | } 117 | 118 | std::string net_type; 119 | { 120 | int type_size; 121 | fread(&type_size, sizeof(int), 1, f); 122 | std::vector type(type_size); 123 | fread(type.data(), sizeof(char) * type_size, 1, f); 124 | net_type = std::string(type.begin(), type.end()); 125 | } 126 | //std::cout << "Reading file with: " << net_type << std::endl; 127 | 128 | if(net_type == "2ch") 129 | { 130 | net->add(std::make_shared(state, 2,96, 7,7, 3,3)); 131 | net->add(std::make_shared(state)); 132 | net->add(std::make_shared(state, 2,2, 2,2)); 133 | net->add(std::make_shared(state, 96, 192, 5,5)); 134 | net->add(std::make_shared(state)); 135 | net->add(std::make_shared(state, 2,2, 2,2)); 136 | net->add(std::make_shared(state, 192, 256, 3,3)); 137 | net->add(std::make_shared(state)); 138 | net->add(std::make_shared(state, std::vector(1,256))); 139 | net->add(std::make_shared(state, 256, 1)); 140 | 141 | for(int i=0; i<4; ++i) 142 | { 143 | int net_j = 0; 144 | fread(&net_j, sizeof(int), 1, f); 145 | CHECK_EQ(net_j >= 1); 146 | //std::cout << "Reading weights of layer " << net_j << std::endl; 147 | auto module = net->get(net_j-1); 148 | if(i!=3) 149 | readParameters(state, (cunn::SpatialConvolutionMM*)module.get(), f); 150 | else 151 | readParameters(state, (cunn::Linear*)module.get(), f); 152 | } 153 | } 154 | else if(net_type == "2chdeep") 155 | { 156 | net->add(std::make_shared(state, 2,96, 4,4, 3,3)); 157 | net->add(std::make_shared(state)); 158 | net->add(std::make_shared(state, 96,96, 3,3)); 159 | net->add(std::make_shared(state)); 160 | net->add(std::make_shared(state, 96,96, 3,3)); 161 | net->add(std::make_shared(state)); 162 | net->add(std::make_shared(state, 96,96, 3,3)); 163 | net->add(std::make_shared(state)); 164 | 165 | net->add(std::make_shared(state, 2,2, 2,2, true)); 166 | 167 | net->add(std::make_shared(state, 96,192, 3,3)); 168 | net->add(std::make_shared(state)); 169 | net->add(std::make_shared(state, 192,192, 3,3)); 170 | net->add(std::make_shared(state)); 171 | net->add(std::make_shared(state, 192,192, 3,3)); 172 | net->add(std::make_shared(state)); 173 | 174 | net->add(std::make_shared(state, std::vector(1,192*2*2))); 175 | net->add(std::make_shared(state, 192*2*2, 1)); 176 | 177 | for(int i=0; i<8; ++i) 178 | { 179 | int net_j = 0; 180 | fread(&net_j, sizeof(int), 1, f); 181 | CHECK_EQ(net_j >= 1); 182 | //std::cout << "Reading weights of layer " << net_j << std::endl; 183 | auto module = net->get(net_j-1); 184 | if(i!=7) 185 | readParameters(state, (cunn::SpatialConvolutionMM*)module.get(), f); 186 | else 187 | readParameters(state, (cunn::Linear*)module.get(), f); 188 | } 189 | } 190 | else if(net_type == "2ch2stream") 191 | { 192 | int featureOut = 192*2*2; 193 | 194 | cunn::Sequential::Ptr branch1 = std::make_shared(state); 195 | branch1->add(std::make_shared(state, 2,2,2,2)); 196 | branch1->add(std::make_shared(state, 2, 96, 5,5)); 197 | branch1->add(std::make_shared(state)); 198 | branch1->add(std::make_shared(state, 2,2, 2,2)); 199 | branch1->add(std::make_shared(state, 96, 96, 3,3)); 200 | branch1->add(std::make_shared(state)); 201 | branch1->add(std::make_shared(state, 2,2, 2,2)); 202 | branch1->add(std::make_shared(state, 96, 192, 3,3)); 203 | branch1->add(std::make_shared(state)); 204 | branch1->add(std::make_shared(state, 192, 192, 3,3)); 205 | branch1->add(std::make_shared(state)); 206 | branch1->add(std::make_shared(state, std::vector(1,featureOut))); 207 | 208 | cunn::Sequential::Ptr branch2 = std::make_shared(state); 209 | branch2->add(std::make_shared(state, 64, 64)); 210 | branch2->add(std::make_shared(state, 2, 96, 5,5)); 211 | branch2->add(std::make_shared(state)); 212 | branch2->add(std::make_shared(state, 2,2, 2,2)); 213 | branch2->add(std::make_shared(state, 96, 96, 3,3)); 214 | branch2->add(std::make_shared(state)); 215 | branch2->add(std::make_shared(state, 2,2, 2,2)); 216 | branch2->add(std::make_shared(state, 96, 192, 3,3)); 217 | branch2->add(std::make_shared(state)); 218 | branch2->add(std::make_shared(state, 192, 192, 3,3)); 219 | branch2->add(std::make_shared(state)); 220 | branch2->add(std::make_shared(state, std::vector(1,featureOut))); 221 | 222 | cunn::Concat::Ptr concat = std::make_shared(state, 1); 223 | concat->add(branch1); 224 | concat->add(branch2); 225 | 226 | net->add(concat); 227 | 228 | net->add(std::make_shared(state, std::vector(1,featureOut*2))); 229 | net->add(std::make_shared(state, featureOut*2, featureOut)); 230 | net->add(std::make_shared(state)); 231 | net->add(std::make_shared(state, featureOut, 1)); 232 | 233 | for(int i=0; i<4; ++i) 234 | { 235 | int net_j = 0; 236 | fread(&net_j, sizeof(int), 1, f); 237 | //std::cout << "Reading weights of layer " << net_j << std::endl; 238 | auto module = branch1->get(net_j-1); 239 | readParameters(state, (cunn::SpatialConvolutionMM*)module.get(), f); 240 | } 241 | 242 | for(int i=0; i<4; ++i) 243 | { 244 | int net_j = 0; 245 | fread(&net_j, sizeof(int), 1, f); 246 | //std::cout << "Reading weights of layer " << net_j << std::endl; 247 | auto module = branch2->get(net_j-1); 248 | readParameters(state, (cunn::SpatialConvolutionMM*)module.get(), f); 249 | } 250 | 251 | int net_j = 0; 252 | fread(&net_j, sizeof(int), 1, f); 253 | readParameters(state, (cunn::Linear*)net->get(2).get(), f); 254 | fread(&net_j, sizeof(int), 1, f); 255 | readParameters(state, (cunn::Linear*)net->get(4).get(), f); 256 | } 257 | else if(net_type == "siam_desc") 258 | { 259 | net->add(std::make_shared(state, 1,96, 7,7, 3,3)); 260 | net->add(std::make_shared(state)); 261 | net->add(std::make_shared(state, 2,2,2,2)); 262 | net->add(std::make_shared(state, 96,192, 5,5)); 263 | net->add(std::make_shared(state)); 264 | net->add(std::make_shared(state, 2,2,2,2)); 265 | net->add(std::make_shared(state, 192,256, 3,3)); 266 | net->add(std::make_shared(state)); 267 | net->add(std::make_shared(state, std::vector(1,256))); 268 | 269 | for(int i=0; i<3; ++i) 270 | { 271 | int net_j = 0; 272 | fread(&net_j, sizeof(int), 1, f); 273 | //std::cout << "Reading weights of layer " << net_j << std::endl; 274 | auto module = net->get(net_j-2); 275 | readParameters(state, (cunn::SpatialConvolutionMM*)module.get(), f); 276 | } 277 | } 278 | else if(net_type == "siam") 279 | { 280 | std::vector size = {1, 64, 64}; 281 | cunn::Sequential::Ptr branch1 = std::make_shared(state); 282 | branch1->add(std::make_shared(state, size)); 283 | branch1->add(std::make_shared(state, 1,96, 7,7, 3,3)); 284 | branch1->add(std::make_shared(state)); 285 | branch1->add(std::make_shared(state, 2,2,2,2)); 286 | branch1->add(std::make_shared(state, 96,192, 5,5)); 287 | branch1->add(std::make_shared(state)); 288 | branch1->add(std::make_shared(state, 2,2,2,2)); 289 | branch1->add(std::make_shared(state, 192,256, 3,3)); 290 | branch1->add(std::make_shared(state)); 291 | branch1->add(std::make_shared(state, std::vector(1,256))); 292 | 293 | cunn::Sequential::Ptr branch2 = std::make_shared(state); 294 | branch2->add(std::make_shared(state, size)); 295 | branch2->add(std::make_shared(state, 1,96, 7,7, 3,3)); 296 | branch2->add(std::make_shared(state)); 297 | branch2->add(std::make_shared(state, 2,2,2,2)); 298 | branch2->add(std::make_shared(state, 96,192, 5,5)); 299 | branch2->add(std::make_shared(state)); 300 | branch2->add(std::make_shared(state, 2,2,2,2)); 301 | branch2->add(std::make_shared(state, 192,256, 3,3)); 302 | branch2->add(std::make_shared(state)); 303 | branch2->add(std::make_shared(state, std::vector(1,256))); 304 | 305 | cunn::Parallel::Ptr branches = std::make_shared(state, 1, 1); 306 | branches->add(branch1); 307 | branches->add(branch2); 308 | 309 | net->add(branches); 310 | net->add(std::make_shared(state, 256*2, 256*2)); 311 | net->add(std::make_shared(state)); 312 | net->add(std::make_shared(state, 256*2, 1)); 313 | 314 | for(int i=0; i<3; ++i) 315 | { 316 | int net_j = 0; 317 | fread(&net_j, sizeof(int), 1, f); 318 | //std::cout << "Reading weights of layer " << net_j << std::endl; 319 | auto casted1 = (cunn::SpatialConvolutionMM*)(branch1->get(net_j-1)).get(); 320 | auto casted2 = (cunn::SpatialConvolutionMM*)(branch2->get(net_j-1)).get(); 321 | readParameters(state, casted1, f); 322 | THCudaTensor_copy(state, casted2->weight, casted1->weight); 323 | THCudaTensor_copy(state, casted2->bias, casted1->bias); 324 | } 325 | int net_j = 0; 326 | fread(&net_j, sizeof(int), 1, f); 327 | readParameters(state, (cunn::Linear*)net->get(1).get(), f); 328 | fread(&net_j, sizeof(int), 1, f); 329 | readParameters(state, (cunn::Linear*)net->get(3).get(), f); 330 | } 331 | else if(net_type == "siam_decision") 332 | { 333 | net->add(std::make_shared(state, 256*2, 256*2)); 334 | net->add(std::make_shared(state)); 335 | net->add(std::make_shared(state, 256*2, 1)); 336 | 337 | int net_j = 0; 338 | fread(&net_j, sizeof(int), 1, f); 339 | readParameters(state, (cunn::Linear*)net->get(0).get(), f); 340 | fread(&net_j, sizeof(int), 1, f); 341 | readParameters(state, (cunn::Linear*)net->get(2).get(), f); 342 | } 343 | else if(net_type == "siam2stream") 344 | { 345 | cunn::Parallel::Ptr branches = std::make_shared(state, 1,1); 346 | auto branch1 = createSiam2streamBranch(state); 347 | auto branch2 = createSiam2streamBranch(state); 348 | branches->add(branch1); 349 | branches->add(branch2); 350 | auto sub_branch1_1 = (cunn::Sequential*)branch1->get(0).get(); 351 | auto sub_branch1_2 = (cunn::Sequential*)branch1->get(1).get(); 352 | auto sub_branch2_1 = (cunn::Sequential*)branch2->get(0).get(); 353 | auto sub_branch2_2 = (cunn::Sequential*)branch2->get(1).get(); 354 | 355 | net->add(branches); 356 | net->add(std::make_shared(state, 1024, 1024)); 357 | net->add(std::make_shared(state)); 358 | net->add(std::make_shared(state, 1024, 1)); 359 | 360 | for(int i=0; i<4; ++i) 361 | { 362 | int net_j = 0; 363 | fread(&net_j, sizeof(int), 1, f); 364 | auto casted1 = (cunn::SpatialConvolutionMM*)(sub_branch1_1->get(net_j-1)).get(); 365 | auto casted2 = (cunn::SpatialConvolutionMM*)(sub_branch2_1->get(net_j-1)).get(); 366 | readParameters(state, casted1, f); 367 | THCudaTensor_copy(state, casted2->weight, casted1->weight); 368 | THCudaTensor_copy(state, casted2->bias, casted1->bias); 369 | } 370 | for(int i=0; i<4; ++i) 371 | { 372 | int net_j = 0; 373 | fread(&net_j, sizeof(int), 1, f); 374 | auto casted1 = (cunn::SpatialConvolutionMM*)(sub_branch1_2->get(net_j-1)).get(); 375 | auto casted2 = (cunn::SpatialConvolutionMM*)(sub_branch2_2->get(net_j-1)).get(); 376 | readParameters(state, casted1, f); 377 | THCudaTensor_copy(state, casted2->weight, casted1->weight); 378 | THCudaTensor_copy(state, casted2->bias, casted1->bias); 379 | } 380 | int net_j = 0; 381 | fread(&net_j, sizeof(int), 1, f); 382 | readParameters(state, (cunn::Linear*)net->get(1).get(), f); 383 | fread(&net_j, sizeof(int), 1, f); 384 | readParameters(state, (cunn::Linear*)net->get(3).get(), f); 385 | } 386 | else if(net_type == "siam2stream_decision") 387 | { 388 | net->add(std::make_shared(state, 1024, 1024)); 389 | net->add(std::make_shared(state)); 390 | net->add(std::make_shared(state, 1024, 1)); 391 | 392 | int net_j = 0; 393 | fread(&net_j, sizeof(int), 1, f); 394 | readParameters(state, (cunn::Linear*)net->get(0).get(), f); 395 | fread(&net_j, sizeof(int), 1, f); 396 | readParameters(state, (cunn::Linear*)net->get(2).get(), f); 397 | } 398 | else if(net_type == "siam2stream_desc") 399 | { 400 | auto branch = createSiam2streamBranch(state); 401 | auto sub_branch1 = (cunn::Sequential*)branch->get(0).get(); 402 | auto sub_branch2 = (cunn::Sequential*)branch->get(1).get(); 403 | net->add(branch); 404 | 405 | for(int i=0; i<4; ++i) 406 | { 407 | int net_j = 0; 408 | fread(&net_j, sizeof(int), 1, f); 409 | auto casted = (cunn::SpatialConvolutionMM*)(sub_branch1->get(net_j-1)).get(); 410 | readParameters(state, casted, f); 411 | } 412 | for(int i=0; i<4; ++i) 413 | { 414 | int net_j = 0; 415 | fread(&net_j, sizeof(int), 1, f); 416 | auto casted = (cunn::SpatialConvolutionMM*)(sub_branch2->get(net_j-1)).get(); 417 | readParameters(state, casted, f); 418 | } 419 | } 420 | 421 | //std::cout << net->tostring() << std::endl; 422 | 423 | fclose(f); 424 | return net; 425 | } 426 | 427 | -------------------------------------------------------------------------------- /src/loader.h: -------------------------------------------------------------------------------- 1 | #ifndef CVPR15_MATCHER_LOADER_H 2 | #define CVPR15_MATCHER_LOADER_H 3 | // Copyright 2015 Sergey Zagoruyko, Nikos Komodakis 4 | // sergey.zagoruyko@imagine.enpc.fr, nikos.komodakis@enpc.fr 5 | // Ecole des Ponts ParisTech, Universite Paris-Est, IMAGINE 6 | // 7 | // The software is free to use only for non-commercial purposes. 8 | // IF YOU WOULD LIKE TO USE IT FOR COMMERCIAL PURPOSES, PLEASE CONTACT 9 | // Prof. Nikos Komodakis (nikos.komodakis@enpc.fr) 10 | #include 11 | 12 | cunn::Sequential::Ptr loadNetwork(THCState* state, const char* filename); 13 | 14 | #endif 15 | -------------------------------------------------------------------------------- /src/test.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Sergey Zagoruyko, Nikos Komodakis 2 | // sergey.zagoruyko@imagine.enpc.fr, nikos.komodakis@enpc.fr 3 | // Ecole des Ponts ParisTech, Universite Paris-Est, IMAGINE 4 | // 5 | // The software is free to use only for non-commercial purposes. 6 | // IF YOU WOULD LIKE TO USE IT FOR COMMERCIAL PURPOSES, PLEASE CONTACT 7 | // Prof. Nikos Komodakis (nikos.komodakis@enpc.fr) 8 | #include 9 | #include 10 | #include "loader.h" 11 | 12 | #define ANSI_COLOR_RED "\x1b[31m" 13 | #define ANSI_COLOR_GREEN "\x1b[32m" 14 | #define ANSI_COLOR_RESET "\x1b[0m" 15 | 16 | 17 | int main(int argc, char** argv) 18 | { 19 | THCState *state = (THCState*)malloc(sizeof(THCState)); 20 | THCudaInit(state); 21 | 22 | FILE *f = fopen("test_data.bin", "rb"); 23 | if(f == NULL) 24 | { 25 | std::cout << "test_data.bin file not found, exiting\n"; 26 | return 1; 27 | } 28 | int num = 0; 29 | int bs = 0; 30 | fread(&num, sizeof(int), 1, f); 31 | fread(&bs, sizeof(int), 1, f); 32 | 33 | // read test input 34 | THCudaTensor* input = THCudaTensor_newWithSize4d(state, bs,2,64,64); 35 | THFloatTensor *finput = THFloatTensor_newWithSize4d(bs,2,64,64); 36 | fread(THFloatTensor_data(finput), sizeof(float) * THFloatTensor_nElement(finput), 1, f); 37 | THCudaTensor_copyFloat(state, input, finput); 38 | 39 | for(int i=0; i < num; ++i) 40 | { 41 | // read network bin file path 42 | int namesize = 0; 43 | fread(&namesize, sizeof(int), 1, f); 44 | std::vector name_v(namesize); 45 | fread(name_v.data(), sizeof(char), namesize, f); 46 | std::string name(name_v.begin(), name_v.end()); 47 | 48 | // read test output 49 | int v[2]; 50 | fread(v, sizeof(int), 2, f); 51 | THFloatTensor *foutput = THFloatTensor_newWithSize2d(v[0], v[1]); 52 | fread(THFloatTensor_data(foutput), sizeof(float)*v[0]*v[1], 1, f); 53 | 54 | THCudaTensor* ref_output = THCudaTensor_newWithSize2d(state, v[0], v[1]); 55 | THCudaTensor_copyFloat(state, ref_output, foutput); 56 | 57 | // load the network 58 | auto net = loadNetwork(state, name.c_str()); 59 | 60 | THCudaTensor* output; 61 | 62 | if(name.find("_desc") != std::string::npos) 63 | { 64 | // for desc we have to do input:select(2,1) 65 | THCudaTensor* patch_src = THCudaTensor_newSelect(state, input, 1, 0); 66 | THCudaTensor* patch = THCudaTensor_newWithSize4d(state, bs, 1, 64, 64); 67 | 68 | THCudaTensor_copy(state, patch, patch_src); 69 | 70 | output = net->forward(patch); 71 | 72 | THCudaTensor_free(state, patch); 73 | THCudaTensor_free(state, patch_src); 74 | } 75 | else 76 | output = net->forward(input); 77 | 78 | if(THCudaTensor_dist(state, ref_output, output, 1) == 0) 79 | std::cout << ANSI_COLOR_GREEN << "PASSED" << ANSI_COLOR_RESET; 80 | else 81 | std::cout << ANSI_COLOR_RED << "FAILED" << ANSI_COLOR_RESET; 82 | std::cout << " " << name << std::endl; 83 | 84 | THCudaTensor_free(state, ref_output); 85 | THFloatTensor_free(foutput); 86 | } 87 | 88 | fclose(f); 89 | 90 | THFloatTensor_free(finput); 91 | THCudaTensor_free(state, input); 92 | 93 | THCudaShutdown(state); 94 | 95 | return 0; 96 | } 97 | -------------------------------------------------------------------------------- /src/wrapper.cpp: -------------------------------------------------------------------------------- 1 | #include "loader.h" 2 | #include "wrapper.h" 3 | 4 | struct Impl { 5 | cunn::Sequential::Ptr net; 6 | THCState *state; 7 | }; 8 | 9 | Network::Network(const char* filename) { 10 | THCState *state = (THCState*)malloc(sizeof(THCState)); 11 | THCudaInit(state); 12 | 13 | ptr = std::make_shared(); 14 | ptr->state = state; 15 | init(filename); 16 | } 17 | 18 | Network::~Network() { 19 | THCState* state = ptr->state; 20 | ptr.reset(); 21 | THCudaShutdown(state); 22 | } 23 | 24 | void Network::init(const char* filename) { 25 | ptr->net = loadNetwork(ptr->state, filename); 26 | } 27 | 28 | std::string Network::tostring() const { 29 | return ptr->net->tostring(); 30 | } 31 | 32 | void Network::setDevice(int i) const { 33 | cudaSetDevice(i); 34 | } 35 | 36 | void Network::reset() { 37 | ptr->net.reset(); 38 | } 39 | 40 | void forwardAnyNet(THCState *state, cunn::Module* net, 41 | THFloatTensor *input, THFloatTensor *output) { 42 | THLongStorage *input_size = THFloatTensor_newSizeOf(input); 43 | THCudaTensor *input_cuda = THCudaTensor_newWithSize(state, input_size, NULL); 44 | THCudaTensor_copyFloat(state, input_cuda, input); 45 | 46 | THCudaTensor* output_cuda = net->forward(input_cuda); 47 | 48 | THLongStorage *output_size = THCudaTensor_newSizeOf(state, output_cuda); 49 | THFloatTensor_resize(output, output_size, NULL); 50 | THFloatTensor_copyCuda(state, output, output_cuda); 51 | 52 | THCudaTensor_free(state, input_cuda); 53 | THLongStorage_free(input_size); 54 | THLongStorage_free(output_size); 55 | } 56 | 57 | void Network::forward(THFloatTensor *input, THFloatTensor *output) { 58 | forwardAnyNet(ptr->state, (cunn::Module*)ptr->net.get(), input, output); 59 | } 60 | -------------------------------------------------------------------------------- /src/wrapper.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | struct Impl; 4 | 5 | struct Network { 6 | Network(const char* filename); 7 | ~Network(); 8 | void init(const char* filename); 9 | std::string tostring() const; 10 | void reset(); 11 | void setDevice(int i) const; 12 | void forward(THFloatTensor *input, THFloatTensor *output); 13 | 14 | std::shared_ptr ptr; 15 | }; 16 | 17 | 18 | -------------------------------------------------------------------------------- /torch/extract_cpu.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | -- example of extracting descriptors on CPU 3 | 4 | N = 76 -- the number of patches to match 5 | patches = torch.rand(N,1,64,64):float() 6 | 7 | -- load the network 8 | net = torch.load'../networks/siam2stream/siam2stream_liberty_nn.t7' 9 | print(net) 10 | 11 | -- to extract the descriptors we need a branch of the first parallel module 12 | net = net:get(1):get(1) 13 | print(net) 14 | 15 | -- in place mean subtraction 16 | local p = patches:view(N,1,64*64) 17 | p:add(-p:mean(3):expandAs(p)) 18 | 19 | -- get the output descriptors 20 | output = net:forward(patches) 21 | 22 | -- print the size of the output tensor 23 | print(#output) 24 | -------------------------------------------------------------------------------- /torch/extract_gpu.lua: -------------------------------------------------------------------------------- 1 | require 'cunn' 2 | -- example of extracting descriptors on GPU 3 | 4 | N = 76 -- the number of patches to match 5 | patches = torch.rand(N,1,64,64):cuda() 6 | 7 | -- load the network 8 | net = torch.load'../networks/siam2stream/siam2stream_liberty_nn.t7':cuda() 9 | print(net) 10 | 11 | -- to extract the descriptors we need a branch of the first parallel module 12 | net = net:get(1):get(1) 13 | print(net) 14 | 15 | -- in place mean subtraction 16 | local p = patches:view(N,1,64*64) 17 | p:add(-p:mean(3):expandAs(p)) 18 | 19 | -- get the output descriptors 20 | output = net:forward(patches) 21 | 22 | -- print the size of the output tensor 23 | print(#output) 24 | 25 | -- OR if there is a lot of patches to extract descriptors 26 | -- it is better to split them to smaller batches 27 | batch_size = 128 28 | 29 | -- preallocate the output tensor, here we know the dimensionality of descriptor 30 | descriptors = torch.CudaTensor(N,512) 31 | descriptors_split = descriptors:split(batch_size) 32 | 33 | for i,v in ipairs(patches:split(batch_size)) do 34 | descriptors_split[i]:copy(net:forward(v)) 35 | end 36 | -------------------------------------------------------------------------------- /torch/match_cpu.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | 3 | N = 76 -- the number of patches to match 4 | patches = torch.rand(N,2,64,64):float() 5 | 6 | -- load the network 7 | net = torch.load'../networks/2ch/2ch_liberty_nn.t7' 8 | 9 | -- in place mean subtraction 10 | local p = patches:view(N,2,64*64) 11 | p:add(-p:mean(3):expandAs(p)) 12 | 13 | -- get the output similarities 14 | output = net:forward(patches) 15 | -------------------------------------------------------------------------------- /torch/match_gpu.lua: -------------------------------------------------------------------------------- 1 | require 'cunn' 2 | -- example of matching descriptors on GPU 3 | 4 | N = 76 -- the number of patches to match 5 | patches = torch.rand(N,2,64,64):cuda() 6 | 7 | -- load the network and move to cuda 8 | net = torch.load'../networks/2ch/2ch_liberty_nn.t7':cuda() 9 | 10 | -- in place mean subtraction 11 | local p = patches:view(N,2,64*64) 12 | p:add(-p:mean(3):expandAs(p)) 13 | 14 | -- get the output similarities 15 | output = net:forward(patches) 16 | 17 | 18 | -- OR if there is a lot of patches to match 19 | -- it is better to split them to smaller batches 20 | batch_size = 128 21 | 22 | -- preallocate the output tensor 23 | similarities = torch.CudaTensor(N) 24 | similarities_split = similarities:split(batch_size) 25 | 26 | for i,v in ipairs(patches:split(batch_size)) do 27 | similarities_split[i]:copy(net:forward(v)) 28 | end 29 | -------------------------------------------------------------------------------- /training/FPR95Meter.lua: -------------------------------------------------------------------------------- 1 | local tnt = require 'torchnet.env' 2 | local argcheck = require 'argcheck' 3 | 4 | local FPR95Meter = torch.class('tnt.FPR95Meter', 'tnt.Meter', tnt) 5 | 6 | FPR95Meter.__init = argcheck{ 7 | doc = [[ 8 | Compute false positive rate at 95% 9 | For more information see http://www.mathworks.com/help/nnet/ref/roc.html 10 | ]], 11 | {name="self", type="tnt.FPR95Meter"}, 12 | call = function(self) 13 | self:reset() 14 | end 15 | } 16 | 17 | FPR95Meter.reset = argcheck{ 18 | {name="self", type="tnt.FPR95Meter"}, 19 | call = function(self) 20 | self.targets = {} 21 | self.outputs = {} 22 | self.tpr, self.fpr = nil 23 | end 24 | } 25 | 26 | FPR95Meter.add = argcheck{ 27 | {name="self", type="tnt.FPR95Meter"}, 28 | {name="output", type="torch.*Tensor"}, 29 | {name="target", type="torch.*Tensor"}, 30 | call = function(self, output, target) 31 | if output:numel() ~= 1 then output = output:squeeze() end 32 | if target:numel() ~= 1 then target = target:squeeze() end 33 | table.insert(self.targets, target:float()) 34 | table.insert(self.outputs, output:float()) 35 | end 36 | } 37 | 38 | local roc = function(targets, outputs) 39 | local L,I = torch.sort(outputs, 1, true) 40 | local labels = targets:index(1,I) 41 | local TPR = torch.cumsum(labels:gt(0):float()) / labels:gt(0):float():sum() 42 | local FPR = torch.cumsum(labels:lt(0):float()) / labels:lt(0):float():sum() 43 | return TPR, FPR 44 | end 45 | 46 | 47 | FPR95Meter.value = argcheck{ 48 | {name="self", type="tnt.FPR95Meter"}, 49 | {name="t", type="number", opt=true}, 50 | call = function(self, t) 51 | local targets = torch.cat(self.targets) 52 | local outputs = torch.cat(self.outputs) 53 | self.tpr, self.fpr = roc(targets, outputs) 54 | local _,k = (self.tpr -0.95):abs():min(1) 55 | local FPR95 = self.fpr[k[1]] 56 | return FPR95 57 | end 58 | } 59 | -------------------------------------------------------------------------------- /training/README.md: -------------------------------------------------------------------------------- 1 | DeepCompare training code 2 | ========== 3 | 4 | This is training code for CVPR2015 paper "Learning to Compare Image Patches via Convolutional Neural Networks". http://www.cv-foundation.org/openaccess/content_cvpr_2015/papers/Zagoruyko_Learning_to_Compare_2015_CVPR_paper.pdf 5 | 6 | The original code was rewritten to support: 7 | * parallel data augmentation via torchnet 8 | * Multi-GPU 9 | * half precision (fp16) training 10 | * CPU training 11 | 12 | # Required rocks 13 | 14 | ``` 15 | luarocks install torchnet 16 | luarocks install cudnn 17 | ``` 18 | 19 | Cudnn is optional. 20 | 21 | Preprocessing code depends on OpenCV. To install it follow https://github.com/VisionLabs/torch-opencv 22 | 23 | # Data preprocessing 24 | 25 | First download Brown dataset http://www.cs.ubc.ca/~mbrown/patchdata/patchdata.html 26 | Then run preprocessing with this script https://gist.github.com/szagoruyko/569c2b713a2c40629ecbe75fb3c9d980 27 | 28 | ``` 29 | th create_dataset_file.lua notredame/ 30 | th create_dataset_file.lua yosemite/ 31 | th create_dataset_file.lua liberty/ 32 | ``` 33 | 34 | This will convert the data to torch format for faster loading and compute per-patch mean. 35 | 36 | # Testing existing models 37 | 38 | ``` 39 | testOnly=true train_set=liberty/data.t7 test_set=notredame/data.t7 model=2ch_liberty.t7 th train.lua 40 | ``` 41 | 42 | Only FPR95 value will be printed, to access TPR and FPR values see `fpr95meter.tpr` and `fpr95meter.fpr` tensors. 43 | It is easy to save them in torch/numpy/matlab format to visualize or compare with other methods. 44 | 45 | To test on CPU additionally set `data_type=torch.FloatTensor` 46 | 47 | # Training 48 | 49 | ``` 50 | train_set=liberty/data.t7 test_set=notredame/data.t7 model=siam th train.lua 51 | ``` 52 | 53 | I prefer to save logs in a separate folder, here is the script: 54 | 55 | ```bash 56 | export train_set=/opt/datasets/daisy/notredame/data.t7 57 | export test_set=/opt/datasets/daisy/liberty/data.t7 58 | 59 | export save_folder=logs/deepcompare_${model}_$RANDOM$RANDOM 60 | mkdir -p $save_folder 61 | 62 | th train.lua | tee $save_folder/log.txt 63 | ``` 64 | 65 | To train on multuple GPUs set: `nGPU=4` 66 | 67 | To train in half-presicion set: `data_type=torch.CudaHalfTensor` 68 | -------------------------------------------------------------------------------- /training/models/2ch.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | local utils = paths.dofile'./utils.lua' 3 | 4 | local model = nn.Sequential() 5 | 6 | local fSize = {2, 96, 192, 256, 256, 1} 7 | 8 | model:add(nn.SpatialConvolution(fSize[1], fSize[2], 7, 7, 3, 3)) 9 | model:add(nn.ReLU()) 10 | model:add(nn.SpatialMaxPooling(2,2,2,2)) 11 | model:add(nn.SpatialConvolution(fSize[2], fSize[3], 5, 5)) 12 | model:add(nn.ReLU()) 13 | model:add(nn.SpatialMaxPooling(2,2,2,2)) 14 | model:add(nn.SpatialConvolution(fSize[3], fSize[4], 3, 3)) 15 | model:add(nn.ReLU()) 16 | model:add(nn.View(-1):setNumInputDims(3)) 17 | model:add(nn.Linear(fSize[4], fSize[5])) 18 | model:add(nn.ReLU()) 19 | model:add(nn.Linear(fSize[5], fSize[6])) 20 | 21 | utils.MSRinit(model) 22 | utils.testModel(model) 23 | 24 | return model 25 | -------------------------------------------------------------------------------- /training/models/2ch2stream.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | local utils = paths.dofile'./utils.lua' 3 | 4 | local fSize = {2, 96, 192} 5 | local featuresOut = fSize[3]*2*2 6 | 7 | local fovea = nn.Sequential() 8 | :add(nn.Reshape(2,32,32)) 9 | :add(nn.SpatialConvolution(fSize[1], fSize[2], 5,5)) 10 | :add(nn.ReLU()) 11 | :add(nn.SpatialMaxPooling(2,2,2,2)) -- 15 12 | :add(nn.SpatialConvolution(fSize[2], fSize[2], 3,3)) 13 | :add(nn.ReLU()) 14 | :add(nn.SpatialMaxPooling(2,2,2,2)) -- 15 15 | :add(nn.SpatialConvolution(fSize[2], fSize[3], 3,3)) 16 | :add(nn.ReLU()) 17 | :add(nn.SpatialConvolution(fSize[3], fSize[3], 3,3)) 18 | :add(nn.ReLU()) 19 | :add(nn.Reshape(featuresOut)) 20 | 21 | local iris = fovea:clone() 22 | 23 | local model = nn.Sequential() 24 | :add(nn.Concat(2,2) 25 | :add(nn.Sequential() 26 | :add(nn.SpatialAveragePooling(2,2,2,2)) 27 | :add(fovea) 28 | ) 29 | :add(nn.Sequential() 30 | :add(nn.SpatialZeroPadding(-16,-16,-16,-16)) 31 | :add(iris) 32 | ) 33 | ) 34 | :add(nn.Linear(featuresOut*2,featuresOut)) 35 | :add(nn.ReLU()) 36 | :add(nn.Linear(featuresOut,1)) 37 | 38 | utils.MSRinit(model) 39 | utils.testModel(model) 40 | 41 | return model 42 | -------------------------------------------------------------------------------- /training/models/2chavg.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | 3 | local fSize = {2, 96, 192, 256, 256} 4 | local featuresOut = fSize[4] 5 | 6 | local model = nn.Sequential() 7 | model:add(nn.SpatialConvolution(fSize[1], fSize[2], 7, 7, 3, 3)) 8 | model:add(nn.ReLU()) 9 | model:add(nn.SpatialMaxPooling(2,2,2,2)) 10 | model:add(nn.SpatialConvolution(fSize[2], fSize[3], 5, 5)) 11 | model:add(nn.ReLU()) 12 | model:add(nn.SpatialConvolution(fSize[3], fSize[4], 3, 3)) 13 | model:add(nn.ReLU()) 14 | model:add(nn.SpatialConvolution(fSize[4], fSize[5], 1, 1)) 15 | model:add(nn.ReLU()) 16 | model:add(nn.SpatialAveragePooling(2,2,2,2)) 17 | model:add(nn.Reshape(featuresOut*2*2)) 18 | model:add(nn.Linear(featuresOut*2*2,1)) 19 | 20 | return model 21 | -------------------------------------------------------------------------------- /training/models/2chdeep.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | local utils = paths.dofile'./utils.lua' 3 | 4 | local model = nn.Sequential() 5 | 6 | local fSize = {2, 96, 192} 7 | local featuresOut = fSize[3]*2*2 8 | 9 | model:add(nn.SpatialConvolution(fSize[1], fSize[2], 4,4, 3,3)) 10 | model:add(nn.ReLU()) 11 | model:add(nn.SpatialConvolution(fSize[2], fSize[2], 3,3)) 12 | model:add(nn.ReLU()) 13 | model:add(nn.SpatialConvolution(fSize[2], fSize[2], 3,3)) 14 | model:add(nn.ReLU()) 15 | model:add(nn.SpatialConvolution(fSize[2], fSize[2], 3,3)) 16 | model:add(nn.ReLU()) 17 | model:add(nn.SpatialMaxPooling(2,2,2,2):ceil()) 18 | model:add(nn.SpatialConvolution(fSize[2], fSize[3], 3,3)) 19 | model:add(nn.ReLU()) 20 | model:add(nn.SpatialConvolution(fSize[3], fSize[3], 3,3)) 21 | model:add(nn.ReLU()) 22 | model:add(nn.SpatialConvolution(fSize[3], fSize[3], 3,3)) 23 | model:add(nn.ReLU()) 24 | model:add(nn.View(-1):setNumInputDims(3)) 25 | model:add(nn.Linear(featuresOut,featuresOut)) 26 | model:add(nn.ReLU()) 27 | model:add(nn.Linear(featuresOut,1)) 28 | 29 | utils.MSRinit(model) 30 | utils.testModel(model) 31 | 32 | return model 33 | -------------------------------------------------------------------------------- /training/models/siam.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | local utils = paths.dofile'./utils.lua' 3 | 4 | local fSize = {1, 96, 192, 256} 5 | local featuresOut = fSize[4] 6 | 7 | local desc = nn.Sequential() 8 | desc:add(nn.Reshape(1,64,64)) 9 | desc:add(nn.SpatialConvolution(fSize[1], fSize[2], 7, 7, 3, 3)) 10 | desc:add(nn.ReLU()) 11 | desc:add(nn.SpatialMaxPooling(2,2,2,2)) 12 | desc:add(nn.SpatialConvolution(fSize[2], fSize[3], 5, 5)) 13 | desc:add(nn.ReLU()) 14 | desc:add(nn.SpatialMaxPooling(2,2,2,2)) 15 | desc:add(nn.SpatialConvolution(fSize[3], fSize[4], 3, 3)) 16 | desc:add(nn.ReLU()) 17 | desc:add(nn.View(-1):setNumInputDims(3)) 18 | desc:add(nn.Contiguous()) 19 | 20 | local siamese = nn.Parallel(2,2) 21 | local siam = desc:clone() 22 | desc:share(siam, 'weight', 'bias', 'gradWeight', 'gradBias') 23 | siamese:add(desc) 24 | siamese:add(siam) 25 | 26 | local top = nn.Sequential() 27 | top:add(nn.Linear(featuresOut*2, featuresOut*2)) 28 | top:add(nn.ReLU()) 29 | top:add(nn.Linear(featuresOut*2, 1)) 30 | 31 | local model = nn.Sequential() 32 | :add(siamese) 33 | :add(top) 34 | 35 | utils.MSRinit(model) 36 | utils.testModel(model) 37 | 38 | return model 39 | -------------------------------------------------------------------------------- /training/models/siam2stream.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | 3 | local fSize = {1, 96, 192, 256} 4 | local featuresOut = fSize[4] 5 | 6 | local iris = nn.Sequential() 7 | iris:add(nn.Reshape(1,64,64)) 8 | iris:add(nn.SpatialZeroPadding(-16, -16, -16, -16)) 9 | iris:add(nn.SpatialConvolution(fSize[1], fSize[2], 4, 4, 2, 2)) 10 | iris:add(nn.ReLU()) 11 | iris:add(nn.SpatialMaxPooling(2,2,2,2)) 12 | iris:add(nn.SpatialConvolution(fSize[2], fSize[3], 3, 3)) 13 | iris:add(nn.ReLU()) 14 | iris:add(nn.SpatialConvolution(fSize[3], fSize[4], 3, 3)) 15 | iris:add(nn.ReLU()) 16 | iris:add(nn.SpatialConvolution(fSize[4], fSize[4], 3, 3)) 17 | iris:add(nn.Reshape(featuresOut)) 18 | 19 | local fovea = nn.Sequential() 20 | fovea:add(nn.Reshape(1,64,64)) 21 | fovea:add(nn.SpatialAveragePooling(2,2,2,2)) 22 | fovea:add(nn.SpatialConvolution(fSize[1], fSize[2], 4, 4, 2, 2)) 23 | fovea:add(nn.ReLU()) 24 | fovea:add(nn.SpatialMaxPooling(2,2,2,2)) 25 | fovea:add(nn.SpatialConvolution(fSize[2], fSize[3], 3, 3)) 26 | fovea:add(nn.ReLU()) 27 | fovea:add(nn.SpatialConvolution(fSize[3], fSize[4], 3, 3)) 28 | fovea:add(nn.ReLU()) 29 | fovea:add(nn.SpatialConvolution(fSize[4], fSize[4], 3, 3)) 30 | fovea:add(nn.Reshape(featuresOut)) 31 | 32 | local eye = nn.Concat(2,2) 33 | eye:add(iris):add(fovea) 34 | 35 | local siamese = nn.Parallel(2,2) 36 | local siam = eye:clone() 37 | eye:share(siam, 'weight', 'bias', 'gradWeight', 'gradBias') 38 | siamese:add(eye):add(siam) 39 | 40 | local model = nn.Sequential() 41 | model:add(siamese) 42 | model:add(nn.Linear(featuresOut*4, featuresOut*4)) 43 | model:add(nn.ReLU()) 44 | model:add(nn.Linear(featuresOut*4, featuresOut*4)) 45 | model:add(nn.ReLU()) 46 | model:add(nn.Linear(featuresOut*4, 1)) 47 | 48 | return model 49 | -------------------------------------------------------------------------------- /training/models/utils.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | 3 | local utils = {} 4 | 5 | function utils.testModel(net) 6 | local input = torch.randn(4,2,64,64):typeAs(net.output) 7 | local output = net:forward(input) 8 | local gradInput = net:backward(input, output:clone()) 9 | print{net.output, net.gradInput} 10 | end 11 | 12 | function utils.MSRinit(net) 13 | return net:apply(function(v) 14 | if torch.typename(v):find'Convolution' then 15 | local n = v.kW*v.kH*v.nOutputPlane 16 | v.weight:normal(0,math.sqrt(2/n)) 17 | v.bias:zero() 18 | end 19 | end) 20 | end 21 | 22 | function utils.makeDataParallelTable(model, nGPU) 23 | if nGPU > 1 then 24 | local gpus = torch.range(1, nGPU):totable() 25 | local fastest, benchmark = cudnn.fastest, cudnn.benchmark 26 | 27 | local dpt = nn.DataParallelTable(1, true, true) 28 | :add(model, gpus) 29 | :threads(function() 30 | local cudnn = require 'cudnn' 31 | cudnn.fastest, cudnn.benchmark = fastest, benchmark 32 | end) 33 | dpt.gradInput = nil 34 | 35 | model = dpt:cuda() 36 | end 37 | return model 38 | end 39 | 40 | return utils 41 | -------------------------------------------------------------------------------- /training/train.lua: -------------------------------------------------------------------------------- 1 | require 'optim' 2 | require 'image' 3 | require 'xlua' 4 | 5 | local json = require 'cjson' 6 | local tnt = require 'torchnet' 7 | local utils = require 'models.utils' 8 | require 'FPR95Meter' 9 | 10 | local opt = { 11 | save_folder = 'logs', 12 | batchSize = 256, 13 | learningRate = 0.1, 14 | weightDecay = 0.0005, 15 | momentum = 0.9, 16 | t0 = 1e+4, 17 | eta0 = 0.1, 18 | max_epoch = 1200, 19 | model = '2ch', 20 | optimMethod = 'asgd', 21 | backend = 'cudnn', 22 | train_set = 'notredame', 23 | test_set = 'liberty', 24 | train_matches = 'm50_500000_500000_0.txt', 25 | test_matches = 'm50_100000_100000_0.txt', 26 | nDonkeys = 12, 27 | manualSeed = 555, 28 | grad_clamp = 2, 29 | data_type = 'torch.CudaTensor', 30 | testOnly = false, 31 | nGPU = 1, 32 | } 33 | opt = xlua.envparams(opt) 34 | print(opt) 35 | 36 | 37 | -- Data loading -- 38 | 39 | local function loadProvider() 40 | print'Loading train and test data' 41 | local p = { 42 | train = torch.load(opt.train_set), 43 | test = torch.load(opt.test_set), 44 | } 45 | 46 | -- assign matches and nonmatches 47 | -- add 1 because original indexes are 0-based 48 | for i,v in ipairs{'matches', 'nonmatches'} do 49 | p.train[v] = p.train[1][opt.train_matches][v] + 1 50 | p.test[v] = p.test[1][opt.test_matches][v] + 1 51 | end 52 | return p 53 | end 54 | local provider = loadProvider() 55 | 56 | 57 | local function random_transform(x) 58 | local a = torch.random(6) 59 | for i=1,2 do 60 | local dst = x[i] 61 | if a == 1 then -- do nothing 62 | elseif a == 2 then image.rotate(dst, dst:clone(), math.pi/2) 63 | elseif a == 3 then image.rotate(dst, dst:clone(), math.pi) 64 | elseif a == 4 then image.rotate(dst, dst:clone(), -math.pi/2) 65 | elseif a == 5 then image.hflip(dst, dst:clone()) 66 | elseif a == 6 then image.vflip(dst, dst:clone()) 67 | end 68 | end 69 | if torch.random(0,1) == 1 then 70 | local x_hat = x:clone() 71 | x[1]:copy(x_hat[2]) 72 | x[2]:copy(x_hat[1]) 73 | end 74 | return x 75 | end 76 | 77 | local function getIterator(mode) 78 | return tnt.ParallelDatasetIterator{ 79 | nthread = opt.nDonkeys, 80 | init = function() 81 | require 'torchnet' 82 | require 'image' 83 | end, 84 | closure = function() 85 | local dataset = provider[mode] 86 | 87 | local function getListDataset(pair_type) 88 | local list_dataset = tnt.ListDataset{ 89 | list = dataset[pair_type], 90 | load = function(idx) 91 | local im = torch.FloatTensor(2,64,64) 92 | for i=1,2 do 93 | local mean = dataset.patches_mean[idx[i]][1] 94 | im[i]:copy(dataset.patches[idx[i]]):add(-mean):div(256) 95 | end 96 | return { 97 | input = im, 98 | target = torch.LongTensor{pair_type == 'matches' and 1 or -1}, 99 | } 100 | end, 101 | } 102 | if mode == 'train' then 103 | list_dataset = list_dataset:shuffle():transform{input = random_transform} 104 | end 105 | return tnt.BatchDataset{ 106 | dataset = list_dataset, 107 | policy = mode == 'train' and 'skip-last' or 'include-last', 108 | batchsize = opt.batchSize / 2, 109 | } 110 | end 111 | 112 | local concat = tnt.ConcatDataset{ 113 | datasets = { 114 | getListDataset'matches', 115 | getListDataset'nonmatches', 116 | } 117 | } 118 | 119 | local n = concat:size() 120 | local multi_idx = torch.range(1,n):view(2,-1):t():reshape(n) 121 | 122 | return concat 123 | :sample(function(dataset, idx) return multi_idx[idx] end) 124 | :batch(2) 125 | :transform{ 126 | input = function(x) return x:view(-1,2,64,64) end, 127 | target = function(y) return y:view(-1) end, 128 | } 129 | end, 130 | } 131 | end 132 | 133 | 134 | -- Network setup -- 135 | 136 | local function cast(x) return x:type(opt.data_type) end 137 | function log(t) print('json_stats: '..json.encode(tablex.merge(t,opt,true))) end 138 | 139 | local model = opt.testOnly and torch.load(opt.model) or paths.dofile('models/'..opt.model..'.lua') 140 | if opt.data_type:match'torch.Cuda.*Tensor' then 141 | require 'cudnn' 142 | require 'cunn' 143 | cudnn.convert(model, cudnn):cuda() 144 | cudnn.benchmark = true 145 | model = utils.makeDataParallelTable(model, opt.nGPU) 146 | end 147 | cast(model) 148 | print(model) 149 | 150 | local engine = tnt.OptimEngine() 151 | local criterion = cast(nn.MarginCriterion()) 152 | 153 | -- timers 154 | local train_timer = torch.Timer() 155 | local test_timer = torch.Timer() 156 | local data_timer = torch.Timer() 157 | 158 | -- meters 159 | local meter = tnt.AverageValueMeter() 160 | local confusion = optim.ConfusionMatrix(2) 161 | local data_time_meter = tnt.AverageValueMeter() 162 | local fpr95meter = tnt.FPR95Meter() 163 | 164 | local inputs = cast(torch.Tensor()) 165 | local targets = cast(torch.Tensor()) 166 | engine.hooks.onSample = function(state) 167 | if state.training then 168 | data_time_meter:add(data_timer:time().real) 169 | end 170 | inputs:resize(state.sample.input:size()):copy(state.sample.input) 171 | targets:resize(state.sample.target:size()):copy(state.sample.target) 172 | state.sample.input = inputs 173 | state.sample.target = targets 174 | end 175 | 176 | engine.hooks.onForwardCriterion = function(state) 177 | meter:add(state.criterion.output) 178 | confusion:batchAdd(state.network.output:gt(0):add(1), state.sample.target:gt(0):add(1)) 179 | if not state.training then 180 | fpr95meter:add(state.network.output, state.sample.target) 181 | end 182 | end 183 | 184 | local function test() 185 | engine:test{ 186 | network = model, 187 | iterator = getIterator'test', 188 | criterion = criterion, 189 | } 190 | confusion:updateValids() 191 | return fpr95meter:value() 192 | end 193 | 194 | engine.hooks.onStartEpoch = function(state) 195 | local epoch = state.epoch + 1 196 | print('==>'.." online epoch # " .. epoch .. ' [batchSize = ' .. opt.batchSize .. ']') 197 | meter:reset() 198 | confusion:zero() 199 | train_timer:reset() 200 | data_time_meter:reset() 201 | end 202 | 203 | engine.hooks.onEndEpoch = function(state) 204 | local train_loss = meter:value() 205 | confusion:updateValids() 206 | local train_acc = confusion.totalValid * 100 207 | local train_time = train_timer:time().real 208 | meter:reset() 209 | print(confusion) 210 | confusion:zero() 211 | test_timer:reset() 212 | fpr95meter:reset() 213 | 214 | local cache = state.params:clone() 215 | state.params:copy(state.optim.ax) 216 | 217 | local testFPR95 = test() 218 | 219 | state.params:copy(cache) 220 | 221 | log{ 222 | loss = train_loss, 223 | train_loss = train_loss, 224 | train_acc = train_acc, 225 | data_loading_time = data_time_meter:value(), 226 | epoch = state.epoch, 227 | test_acc = confusion.totalValid * 100, 228 | testFPR95 = testFPR95, 229 | lr = opt.learningRate, 230 | train_time = train_time, 231 | test_time = test_timer:time().real, 232 | } 233 | end 234 | 235 | engine.hooks.onUpdate = function(state) 236 | data_timer:reset() 237 | end 238 | 239 | if opt.testOnly then 240 | print('Testing on '..opt.test_set) 241 | print('FPR95:',test()) 242 | os.exit() 243 | end 244 | 245 | engine:train{ 246 | network = model, 247 | iterator = getIterator'train', 248 | criterion = criterion, 249 | optimMethod = optim[opt.optimMethod], 250 | config = tablex.deepcopy(opt), 251 | maxepoch = opt.max_epoch, 252 | } 253 | 254 | local modelpath = paths.concat(opt.save_folder, 'model.t7') 255 | print('Saving to '..modelpath) 256 | torch.save(modelpath, cudnn.convert(model, nn):float():clearState()) 257 | --------------------------------------------------------------------------------