├── .gitignore ├── CMakeLists.txt ├── LICENCE.txt ├── README.md ├── assets ├── cvc.cl ├── cvf.cl └── dispsel.cl ├── data ├── Art │ ├── disp1.png │ ├── disp5.png │ ├── dmin.txt │ ├── view1.png │ └── view5.png ├── Books │ ├── disp1.png │ ├── disp5.png │ ├── dmin.txt │ ├── view1.png │ └── view5.png ├── Cones │ ├── disp2.png │ ├── disp6.png │ ├── im2.png │ ├── im6.png │ ├── occ_and_discont.png │ └── occl.png ├── Dolls │ ├── disp1.png │ ├── disp5.png │ ├── dmin.txt │ ├── view1.png │ └── view5.png ├── Laundry │ ├── disp1.png │ ├── disp5.png │ ├── dmin.txt │ ├── view1.png │ └── view5.png ├── Moebius │ ├── disp1.png │ ├── disp5.png │ ├── dmin.txt │ ├── view1.png │ └── view5.png ├── Reindeer │ ├── disp1.png │ ├── disp5.png │ ├── dmin.txt │ ├── view1.png │ └── view5.png ├── Teddy │ ├── disp2.png │ ├── disp6.png │ ├── im2.png │ ├── im6.png │ ├── occ_and_discont.png │ └── occl.png ├── cpp-example-imagelist_creator_arm ├── cpp-example-imagelist_creator_x86 ├── extrinsics.yml ├── intrinsics.yml └── stereo_calib.xml ├── docs ├── de_bd.png └── de_examples.png ├── include ├── CVC.h ├── CVC_cl.h ├── CVF.h ├── CVF_cl.h ├── ComFunc.h ├── DispEst.h ├── DispSel.h ├── DispSel_cl.h ├── JointWMF.h ├── PP.h ├── StereoCalib.h ├── StereoMatch.h ├── args.hxx ├── fastguidedfilter.h └── oclUtil.h └── src ├── CVC.cpp ├── CVC_cl.cpp ├── CVF.cpp ├── CVF_cl.cpp ├── DispEst.cpp ├── DispSel.cpp ├── DispSel_cl.cpp ├── PP.cpp ├── StereoCalib.cpp ├── StereoMatch.cpp ├── fastguidedfilter.cpp ├── main.cpp └── oclUtil.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | build/ 3 | obj/ 4 | data/chessboard* 5 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR}) 2 | message(FATAL_ERROR "DO NOT BUILD in-tree.") 3 | endif() 4 | 5 | if(NOT CMAKE_BUILD_TYPE) 6 | set(CMAKE_BUILD_TYPE Release) 7 | endif() 8 | 9 | cmake_minimum_required (VERSION 3.2) 10 | project (PRiMEStereoMatch) 11 | 12 | include_directories(include) 13 | file(GLOB SOURCES "src/*.cpp") 14 | 15 | add_executable(PRiMEStereoMatch ${SOURCES}) 16 | set_property(TARGET PRiMEStereoMatch PROPERTY CXX_STANDARD 11) 17 | set(CMAKE_CXX_FLAGS_RELEASE "-O3") 18 | 19 | #pthread libraries 20 | set(THREADS_PREFER_PTHREAD_FLAG ON) 21 | find_package(Threads REQUIRED) 22 | if(Threads_FOUND) 23 | include_directories(${THREAD_INCLUDE_DIRS}) 24 | else(Threads_FOUND) 25 | message(FATAL_ERROR ">> Some form of threading library is requried for compilation, preferably PThreads.") 26 | endif(Threads_FOUND) 27 | 28 | #OpenCV libraries 29 | find_package(OpenCV REQUIRED COMPONENTS core calib3d imgproc highgui video) 30 | if(OpenCV_FOUND) 31 | include_directories(${OpenCV_INCLUDE_DIRS}) 32 | else(OpenCV_FOUND) 33 | message(">> OpenCV Required") 34 | endif(OpenCV_FOUND) 35 | 36 | #OpenCL libraries 37 | find_package(OpenCL REQUIRED) 38 | if(OpenCL_FOUND) 39 | include_directories(${OpenCL_INCLUDE_DIRS}) 40 | else(OpenCL_FOUND) 41 | message(">> OpenCL not found. OpenCL is required for GPGPU/FPGA/Accelerator use.") 42 | endif(OpenCL_FOUND) 43 | 44 | #OpenMP libraries 45 | find_package(OpenMP REQUIRED) 46 | if(OpenMP_FOUND) 47 | include_directories(${OpenMP_INCLUDE_DIRS}) 48 | message("Found OpenMP, setting Flags.") 49 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") 50 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") 51 | set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") 52 | else(OpenMP_FOUND) 53 | message(">> OpenMP not found. OpenMP is required for multi-threading of some intermediate processing.") 54 | endif(OpenMP_FOUND) 55 | 56 | #Can still compile without OpenCL 57 | if(OpenCV_FOUND AND OpenMP_FOUND) 58 | target_link_libraries(PRiMEStereoMatch ${CMAKE_THREAD_LIBS_INIT} ${OpenCV_LIBS} ${OpenCL_LIBRARIES} ${OpenMP_LIBRARIES}) 59 | else(OpenCV_FOUND AND OpenMP_FOUND) 60 | message(FATAL_ERROR ">> Some form of threading library is requried for compilation, preferably PThreads.") 61 | endif(OpenCV_FOUND AND OpenMP_FOUND) 62 | -------------------------------------------------------------------------------- /LICENCE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Charlie Leech, University of Southampton 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | 8 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 9 | 10 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 11 | 12 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PRiMEStereoMatch 2 | 3 | **Please use these citations in your publication if you use this work:** ([bibtex here](https://github.com/PRiME-project/PRiMEStereoMatch#license)) 4 | 5 | Charles Leech, Charan Kumar, Amit Acharyya, Sheng Yang, Geoff V. Merrett, and Bashir M. Al-Hashimi. 2017. Runtime Performance and Power Optimization of Parallel Disparity Estimation on Many-Core Platforms. ACM Transactions on Embedded Computing Systems (TECS) Volume 17 Issue 2, Article 41 (November 2017), 19 pages. DOI: https://doi.org/10.1145/3133560 6 | 7 | Leech, Charles (2018) [Runtime energy management of multi-core processors.](https://eprints.soton.ac.uk/422287/) University of Southampton, Doctoral Thesis, 293pp. 8 | 9 | --- 10 |

11 | Examples Image Pairs 12 |

13 | 14 | ## Theoretical Background 15 | 16 | A heterogeneous and fully parallel stereo matching algorithm for depth estimation. Stereo Matching is based on the disparity estimation algorithm, an algorithm designed to calculate 3D depth information about a scene from a pair of 2D images captured by a stereoscopic camera. The algorithm contains the following stages: 17 | 18 | * Cost Volume Construction - weighted absolute difference of colours and gradients function. 19 | * Cost Volume Filtering - Adaptive Support Weight (ADSW) Guided Image Filter (GIF) function. 20 | * Disparity Selection - Winner-Takes-All (WTA) minimum cost selection. 21 | * Post Processing - left-right occlusion check, invalid pixel replacement and weight-median filtering. 22 | 23 |

24 | Disparity estimation process block diagram 25 |

26 | 27 | ## Implementation Details 28 | 29 | * All stages of the algorithm have been developed in both C++ and OpenCL. 30 | * C++ parallelism is introduced via the POSIX threads (pthreads) library. Disparity level parallelism is supported, enabling up to 64 concurrent threads. 31 | * OpenCL parallelism is inherent through the concurrent execution of kernels on an OpenCL-compatible device. The optimum level of parallelism will be bounded by the platform & devices. 32 | * Support for live video disparity estimation using the OpenCV VideoCapture interface as well as static image computation. 33 | * Additional integration of the OpenCV Semi-Global Block Matching (SGBM) algorithm. 34 | 35 | ## Installation 36 | 37 | ### Prerequisites 38 | * Hardware: 39 | * Development Platform - preferably including devices supporting OpenCL 40 | * Stereo Camera - to use the algorithm in video mode - the [ZED Stereo Camera](https://www.stereolabs.com/) is used in our experimentation. 41 | * Software Libraries: 42 | * OpenCV 3.0 or later - [Installation in Linux instructions](https://docs.opencv.org/3.4.0/d7/d9f/tutorial_linux_install.html) 43 | * pthread library for non-OpenCL execution on the CPU 44 | * OpenCL Library for execution on the GPU 45 | * cmake v3.2, git 46 | 47 | ### Compilation 48 | * Clone repo to the platform: `git clone https://github.com/PRiME-project/PRiMEStereoMatch.git` 49 | * Enter the base directory: `cd PRiMEStereoMatch/`. 50 | * Create a build directory: `mkdir build` 51 | * Enter the build directory: `cd build` 52 | * Invoke cmake to build the compilation files: `cmake ..` (Two dots are required in order to reference the base directory) 53 | * Compile the project with the generated makefile: `make -jN`. 54 | * Set N to the number of simultaneous threads supported on your compilation platform, e.g. `make -j8`. 55 | 56 | ### Deployment 57 | * Run the application from the build dir: `./PRiMEStereoMatch ` 58 | * The program mode is selected with git-style commands. Valid commands include: 59 | * video 60 | * [optional] When specifying the video mode, the following arguments can be included: 61 | * --recal - recalculate the intrinsic and extrinsic parameters of the stereo camera. Previously captured chessboard images must be supplied if the RECAPTURE flag is not also set. 62 | * --recap - record chessboard image pairs in preparation for calibration. A chessboard image must be presented in front of the stereo camera and in full view of both cameras. Press the R key to capture a frame. The last frame captured is shown beneath the video stream. 63 | * image 64 | * [optional] When specifying the image mode, the following arguments can be included: 65 | * -l [i]left *image filename>* -r *right image filename* 66 | * -gt *ground truth filename* 67 | * A set of global options also exist, which must be specified for all modes: 68 | * -a (--alg=) - Set the default matching algorithm to run. It has options {STEREO_GIF, STEREO_SGBM}. This can also be toggled during executions. 69 | 70 | * For example, to run using a stereo camera, specify: 71 | * `./PRiMEStereoMatch video` 72 | * To run with calibration and capture beforehand, specify: 73 | * `./PRiMEStereoMatch video --recal --recap` 74 | * Image disparity estimation is achieved using: 75 | * `./PRiMEStereoMatch image -l left_img.png -r right_img.png` 76 | 77 | * The first time the application is deployed using a stereo camera, the --recal and --recap flags must be set in order to capture chessboard image to calculate the intrinsic and extrinsic parameters. 78 | * This process only needs to be repeated if the relative orientations of the left and right cameras are changed or a different resolution is specified. 79 | * Once the intrinsic and extrinsic parameters have been calucalted and saved to .yml files, the application can be re-run with the same camera without needing to recalibrate as the parameters will be loaded from these files. The files can be found in the data directory. 80 | 81 | ### Interactivity 82 | 83 | * Press h to display a help menu on the command line. This shows input and control options for the program which change the way the algorithm behaves for the next frame. 84 | * Control Options: 85 | * Matching Algorithm (a): STEREO_GIF or STEREO_SGBM 86 | * STEREO_GIF: 87 | * Numbers 1 - 8: (CPU only) change the number of simultaneous pthreads created 88 | * m: switch the computational mode between OpenCL (GPU) and pthreads (CPU) 89 | * t: switch the data type use for processing between 32-bit float and 8-bit char 90 | * STEREO_SGBM: 91 | * m: switch the computational mode between MODE_SGBM, MODE_HH and MODE_SGDM_3WAY 92 | 93 | ## Directory Structure 94 | 95 | ``` 96 | folders: 97 | assets - OpenCL kernel files 98 | data - program data including input images, stereo camera parameters, calibration images 99 | docs - images for the readme & wiki 100 | include - Project header files (h/hpp) 101 | src - Project source files (c/cpp) 102 | 103 | files: 104 | CMakeLists.txt - cmake project compilation file 105 | LICENCE.txt - license file 106 | README.md - this readme file 107 | ``` 108 | 109 | ## References 110 | 111 | ### Code 112 | 113 | Some components of the application are based on source code from the following locations: 114 | 115 | [rookiepig/CrossScaleStereo](https://github.com/rookiepig/CrossScaleStereo) - The basis for some C++ functions (GNU Public License) 116 | 117 | [atilimcetin/guided-filter](https://github.com/atilimcetin/guided-filter) - CPU-based GIF implementation using the Fast Guided Filter (MIT License) 118 | 119 | ### Literature 120 | 121 | The algorithm in this work is based in parts on those presented in the following publications: 122 | 123 | [Hosni2011CVPR]: C. Rhemann, A. Hosni, M. Bleyer, C. Rother, and M. Gelautz. Fast cost-volume filtering for visual correspondence and beyond. In CVPR, 2011 124 | 125 | [Hosni2011ICME]: A. Hosni, M. Bleyer, C. Rhemann, M. Gelautz and C. Rother, Real-time local stereo matching using guided image filtering, in Multimedia and Expo (ICME), 2011 IEEE International Conference on, Barcelona, 2011. 126 | 127 | [Ttofis2014]: C. Ttofis and T. Theocharides, High-quality real-time hardware stereo matching based on guided image filtering, in Design, Automation and Test in Europe Conference and Exhibition (DATE), Dresden, 2014. 128 | 129 | [He2012]: K. He, J. Sun and X. Tang, Guided Image Filtering, Pattern Analysis and Machine Intelligence, IEEE Transactions on, pp. 1397-1409, 02 October 2012. 130 | 131 | ## License 132 | 133 | This software is released under the BSD 3 Clause License. See LICENSE.txt for details. 134 | 135 | To cite this code in your work, please also include the following reference: 136 | 137 | Charles Leech, Charan Kumar, Amit Acharyya, Sheng Yang, Geoff V. Merrett, and Bashir M. Al-Hashimi. 2017. Runtime Performance and Power Optimization of Parallel Disparity Estimation on Many-Core Platforms. ACM Transactions on Embedded Computing Systems (TECS) Volume 17 Issue 2, Article 41 (November 2017), 19 pages. DOI: https://doi.org/10.1145/3133560 138 | 139 | Bibtex: 140 | ``` 141 | @article{Leech:2017:RPP:3160927.3133560, 142 | author = {Leech, Charles and Kumar, Charan and Acharyya, Amit and Yang, Sheng and Merrett, Geoff V. and Al-Hashimi, Bashir M.}, 143 | title = {Runtime Performance and Power Optimization of Parallel Disparity Estimation on Many-Core Platforms}, 144 | journal = {ACM Transactions on Embedded Computing Systems (TECS)}, 145 | issue_date = {January 2018}, 146 | volume = {17}, 147 | number = {2}, 148 | month = nov, 149 | year = {2017}, 150 | issn = {1539-9087}, 151 | pages = {41:1--41:19}, 152 | articleno = {41}, 153 | numpages = {19}, 154 | url = {http://doi.acm.org/10.1145/3133560}, 155 | doi = {10.1145/3133560}, 156 | acmid = {3133560}, 157 | publisher = {ACM}, 158 | address = {New York, NY, USA}, 159 | keywords = {Runtime management, computer vision, many-core platforms, power optimization}, 160 | } 161 | ``` 162 | -------------------------------------------------------------------------------- /assets/cvc.cl: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | cvc.cl - OpenCL Cost Volume Construction Kernel 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | All rights reserved. 8 | ---------------------------------------------------------------------------*/ 9 | //#pragma OPENCL EXTENSION cl_khr_fp64 : enable 10 | 11 | #define UCHAR_MAX 255 12 | #define USHRT_MAX 65535 13 | 14 | #define UC16M (uchar16)UCHAR_MAX 15 | #define US16M (ushort16)USHRT_MAX 16 | 17 | #define TAU_1_32F 0.028f 18 | #define TAU_2_32F 0.008f 19 | 20 | //cvc_uchar_v16: 21 | #define ALPHA_US ((ushort16)9/(ushort16)10) 22 | #define TAU_1_US16 (ushort16)1835 //0.028 * USHRT_MAX 23 | #define TAU_2_US16 (ushort16)524 //0.008 * USHRT_MAX 24 | 25 | //cvc_uchar_nv: 26 | #define ALPHA 0.9f 27 | #define TAU_1_US 1835 //0.028 * USHRT_MAX 28 | #define TAU_2_US 524 //0.008 * USHRT_MAX 29 | 30 | /** 31 | * \brief Cost Volume Construction kernel function. 32 | * \param[in] lImg - Left Input image data. 33 | * \param[in] rImg - Right Input image data. 34 | * \param[in] lGrdX - Left Input X dim gradient data. 35 | * \param[in] rGrdX - Right Input X dim gradient data. 36 | * \param[in] height - Height of the image. 37 | * \param[in] width - Width of the image. 38 | * \param[out] lcostVol - Calculated pixel cost for Cost Volume. 39 | * \param[out] rcostVol - Calculated pixel cost for Cost Volume. 40 | */ 41 | 42 | __kernel void cvc_uchar_vx(__global const uchar* lImgR, 43 | __global const uchar* lImgG, 44 | __global const uchar* lImgB, 45 | __global const uchar* rImgR, 46 | __global const uchar* rImgG, 47 | __global const uchar* rImgB, 48 | __global const uchar* lGrdX, 49 | __global const uchar* rGrdX, 50 | const int height, 51 | const int width, 52 | __global uchar* lcostVol, 53 | __global uchar* rcostVol) 54 | { 55 | /* [Kernel size] */ 56 | /* Each kernel calculates a single output pixels in the same row. 57 | * column (x) is in the range [0, width]. 58 | * row (y) is in the range [0, height]. */ 59 | 60 | // const int x = get_global_id(0); 61 | const int y = get_global_id(0); 62 | const int d = get_global_id(2); 63 | 64 | /* Offset calculates the position in the linear data for the row and the column. */ 65 | const int offset = y * width; 66 | const int costVol_offset = ((d * height) + y) * width; 67 | 68 | /* *************** Left to Right Cost Volume Construction ********************** */ 69 | ushort clrDiff, grdDiff; 70 | 71 | for(int x = 0; x < d; x++) 72 | { 73 | // three color diff at img boundary 74 | clrDiff = (abs(lImgR[offset + x] - UCHAR_MAX) 75 | + abs(lImgG[offset + x] - UCHAR_MAX) 76 | + abs(lImgB[offset + x] - UCHAR_MAX))/3; 77 | // gradient diff 78 | grdDiff = abs(lGrdX[offset + x] - UCHAR_MAX); 79 | 80 | clrDiff = clrDiff > TAU_1_US ? TAU_1_US : clrDiff; 81 | grdDiff = grdDiff > TAU_2_US ? TAU_2_US : grdDiff; 82 | lcostVol[costVol_offset + x] = (uchar)( ALPHA * clrDiff + (1-ALPHA) * grdDiff ); 83 | } 84 | for(int x = d; x < width; x++) 85 | { 86 | // three color diff 87 | clrDiff = (abs(lImgR[offset + x] - rImgR[offset + x - d]) 88 | + abs(lImgG[offset + x] - rImgG[offset + x - d]) 89 | + abs(lImgB[offset + x] - rImgB[offset + x - d]))/3; 90 | // gradient diff 91 | grdDiff = abs(lGrdX[offset + x] - rGrdX[offset + x - d]); 92 | 93 | clrDiff = clrDiff > TAU_1_US ? TAU_1_US : clrDiff; 94 | grdDiff = grdDiff > TAU_2_US ? TAU_2_US : grdDiff; 95 | lcostVol[costVol_offset + x] = (uchar)( ALPHA * clrDiff + (1-ALPHA) * grdDiff ); 96 | } 97 | 98 | /* *************** Right to Left Cost Volume Construction ********************** */ 99 | 100 | for(int x = 0; x < width - d; x++) 101 | { 102 | // three color diff 103 | clrDiff = (abs(rImgR[offset + x] - lImgR[offset + x + d]) 104 | + abs(rImgG[offset + x] - lImgG[offset + x + d]) 105 | + abs(rImgB[offset + x] - lImgB[offset + x + d])) * 0.333f; 106 | // gradient diff 107 | grdDiff = abs(rGrdX[offset] - lGrdX[offset + x + d]); 108 | 109 | clrDiff = clrDiff > TAU_1_US ? TAU_1_US : clrDiff; 110 | grdDiff = grdDiff > TAU_2_US ? TAU_2_US : grdDiff; 111 | rcostVol[costVol_offset + x] = (uchar)( ALPHA * clrDiff + (1-ALPHA) * grdDiff ); 112 | } 113 | for(int x = width - d; x < width; x++) 114 | { 115 | // three color diff at img boundary 116 | clrDiff = (abs(rImgR[offset + x] - UCHAR_MAX) 117 | + abs(rImgG[offset + x] - UCHAR_MAX) 118 | + abs(rImgB[offset + x] - UCHAR_MAX)) * 0.333f; 119 | // gradient diff 120 | grdDiff = abs(rGrdX[offset + x] - UCHAR_MAX); 121 | 122 | clrDiff = clrDiff > TAU_1_US ? TAU_1_US : clrDiff; 123 | grdDiff = grdDiff > TAU_2_US ? TAU_2_US : grdDiff; 124 | rcostVol[costVol_offset + x] = (uchar)( ALPHA * clrDiff + (1-ALPHA) * grdDiff ); 125 | } 126 | } 127 | 128 | /** 129 | * \brief Cost Volume Construction kernel function. 130 | * \param[in] lImg - Left Input image data. 131 | * \param[in] rImg - Right Input image data. 132 | * \param[in] lGrdX - Left Input X dim gradient data. 133 | * \param[in] rGrdX - Right Input X dim gradient data. 134 | * \param[in] height - Height of the image. 135 | * \param[in] width - Width of the image. 136 | * \param[out] lcostVol - Calculated pixel cost for Cost Volume. 137 | * \param[out] rcostVol - Calculated pixel cost for Cost Volume. 138 | */ 139 | 140 | __kernel void cvc_uchar_v16(__global const uchar* lImgR, 141 | __global const uchar* lImgG, 142 | __global const uchar* lImgB, 143 | __global const uchar* rImgR, 144 | __global const uchar* rImgG, 145 | __global const uchar* rImgB, 146 | __global const uchar* lGrdX, 147 | __global const uchar* rGrdX, 148 | const int height, 149 | const int width, 150 | __global uchar* lcostVol, 151 | __global uchar* rcostVol) 152 | { 153 | /* [Kernel size] */ 154 | /* Each kernel calculates a single output pixels in the same row. 155 | * column (x) is in the range [0, width]. 156 | * row (y) is in the range [0, height]. 157 | */ 158 | const int x = get_global_id(0) * 16; 159 | const int y = get_global_id(1); 160 | const int d = get_global_id(2); 161 | 162 | /* Offset calculates the position in the linear data for the row and the column. */ 163 | const int offset = (y * width) + x; 164 | const int costVol_offset = ((d * height) + y) * width + x; 165 | 166 | /* *************** Left to Right Cost Volume Construction ********************** */ 167 | // Set addresses for lImg, rImg, lGrdX and rGrdX. 168 | uchar16 lCR = vload16(0, lImgR + offset); 169 | uchar16 lCG = vload16(0, lImgG + offset); 170 | uchar16 lCB = vload16(0, lImgB + offset); 171 | uchar16 lGX = vload16(0, lGrdX + offset); 172 | uchar16 rCR, rCG, rCB, rGX; 173 | 174 | ushort16 clrDiff, grdDiff; 175 | 176 | if(x - d >= 0) 177 | { 178 | rCR = vload16(0, rImgR + offset - d); 179 | rCG = vload16(0, rImgG + offset - d); 180 | rCB = vload16(0, rImgB + offset - d); 181 | rGX = vload16(0, rGrdX + offset - d); 182 | 183 | // three color diff 184 | clrDiff = convert_ushort16(abs(lCR - rCR) + abs(lCG - rCG) + abs(lCB - rCB))/(ushort)3; 185 | // gradient diff 186 | grdDiff = convert_ushort16(abs(lGX - rGX)); 187 | } 188 | else 189 | { 190 | // three color diff at img boundary 191 | clrDiff = convert_ushort16(abs(lCR - UC16M) + abs(lCG - UC16M) + abs(lCB - UC16M))/(ushort)3; 192 | // gradient diff 193 | grdDiff = convert_ushort16(abs(lGX - UC16M)); 194 | } 195 | 196 | clrDiff = clrDiff > TAU_1_US16 ? TAU_1_US16 : clrDiff; 197 | grdDiff = grdDiff > TAU_2_US16 ? TAU_2_US16 : grdDiff; 198 | ushort16 cost = clrDiff/(ushort)9*(ushort)10 + grdDiff/(ushort)10; 199 | vstore16(convert_uchar16(cost), 0, lcostVol + costVol_offset); 200 | 201 | 202 | /* *************** Right to Left Cost Volume Construction ********************** */ 203 | // Set addresses for lImg, rImg, lGrdX and rGrdX. 204 | lCR = vload16(0, rImgR + offset); 205 | lCG = vload16(0, rImgG + offset); 206 | lCB = vload16(0, rImgB + offset); 207 | lGX = vload16(0, rGrdX + offset); 208 | 209 | clrDiff = 0; 210 | grdDiff = 0; 211 | 212 | if(x + d < width) 213 | { 214 | rCR = vload16(0, lImgR + offset + d); 215 | rCG = vload16(0, lImgG + offset + d); 216 | rCB = vload16(0, lImgB + offset + d); 217 | rGX = vload16(0, lGrdX + offset + d); 218 | 219 | // three color diff 220 | clrDiff = convert_ushort16(abs(lCR - rCR) + abs(lCG - rCG) + abs(lCB - rCB))/(ushort)3; 221 | // gradient diff 222 | grdDiff = convert_ushort16(abs(lGX - rGX)); 223 | } 224 | else 225 | { 226 | // three color diff at img boundary 227 | clrDiff = convert_ushort16(abs(lCR - UC16M) + abs(lCG - UC16M) + abs(lCB - UC16M))/(ushort)3; 228 | // gradient diff 229 | grdDiff = convert_ushort16(abs(lGX - UC16M)); 230 | } 231 | 232 | clrDiff = clrDiff > TAU_1_US16 ? TAU_1_US16 : clrDiff; 233 | grdDiff = grdDiff > TAU_2_US16 ? TAU_2_US16 : grdDiff; 234 | cost = clrDiff/(ushort)9*(ushort)10 + grdDiff/(ushort)10; 235 | vstore16(convert_uchar16(cost), 0, rcostVol + costVol_offset); 236 | } 237 | 238 | /** 239 | * \brief Cost Volume Construction kernel function. 240 | * \param[in] lImg - Left Input image data. 241 | * \param[in] rImg - Right Input image data. 242 | * \param[in] lGrdX - Left Input X dim gradient data. 243 | * \param[in] rGrdX - Right Input X dim gradient data. 244 | * \param[in] height - Height of the image. 245 | * \param[in] width - Width of the image. 246 | * \param[out] lcostVol - Calculated pixel cost for Cost Volume. 247 | * \param[out] rcostVol - Calculated pixel cost for Cost Volume. 248 | */ 249 | 250 | __kernel void cvc_uchar_nv(__global const uchar* lImgR, 251 | __global const uchar* lImgG, 252 | __global const uchar* lImgB, 253 | __global const uchar* rImgR, 254 | __global const uchar* rImgG, 255 | __global const uchar* rImgB, 256 | __global const uchar* lGrdX, 257 | __global const uchar* rGrdX, 258 | const int height, 259 | const int width, 260 | __global uchar* lcostVol, 261 | __global uchar* rcostVol) 262 | { 263 | /* [Kernel size] */ 264 | /* Each kernel calculates a single output pixels in the same row. 265 | * column (x) is in the range [0, width]. 266 | * row (y) is in the range [0, height]. */ 267 | 268 | const int x = get_global_id(0); 269 | const int y = get_global_id(1); 270 | const int d = get_global_id(2); 271 | 272 | /* Offset calculates the position in the linear data for the row and the column. */ 273 | const int offset = y * width + x; 274 | const int costVol_offset = ((d * height) + y) * width + x; 275 | 276 | /* *************** Left to Right Cost Volume Construction ********************** */ 277 | ushort clrDiff, grdDiff; 278 | 279 | if(x >= d) 280 | { 281 | // three color diff 282 | clrDiff = (abs(lImgR[offset] - rImgR[offset - d]) 283 | + abs(lImgG[offset] - rImgG[offset - d]) 284 | + abs(lImgB[offset] - rImgB[offset - d]))/3; 285 | // gradient diff 286 | grdDiff = abs(lGrdX[offset] - rGrdX[offset - d]); 287 | } 288 | else 289 | { 290 | // three color diff at img boundary 291 | clrDiff = (abs(lImgR[offset] - UCHAR_MAX) 292 | + abs(lImgG[offset] - UCHAR_MAX) 293 | + abs(lImgB[offset] - UCHAR_MAX))/3; 294 | // gradient diff 295 | grdDiff = abs(lGrdX[offset] - UCHAR_MAX); 296 | } 297 | 298 | clrDiff = clrDiff > TAU_1_US ? TAU_1_US : clrDiff; 299 | grdDiff = grdDiff > TAU_2_US ? TAU_2_US : grdDiff; 300 | lcostVol[costVol_offset] = (uchar)( ALPHA * clrDiff + (1-ALPHA) * grdDiff ); 301 | 302 | 303 | /* *************** Right to Left Cost Volume Construction ********************** */ 304 | clrDiff = 0; 305 | grdDiff = 0; 306 | 307 | if(x >= d) 308 | { 309 | // three color diff 310 | clrDiff = (abs(rImgR[offset] - lImgR[offset + d]) 311 | + abs(rImgG[offset] - lImgG[offset + d]) 312 | + abs(rImgB[offset] - lImgB[offset + d]))/3; 313 | // gradient diff 314 | grdDiff = abs(rGrdX[offset] - lGrdX[offset + d]); 315 | } 316 | else 317 | { 318 | // three color diff at img boundary 319 | clrDiff = (abs(rImgR[offset] - UCHAR_MAX) 320 | + abs(rImgG[offset] - UCHAR_MAX) 321 | + abs(rImgB[offset] - UCHAR_MAX))/3; 322 | // gradient diff 323 | grdDiff = abs(rGrdX[offset] - UCHAR_MAX); 324 | } 325 | 326 | clrDiff = clrDiff > TAU_1_US ? TAU_1_US : clrDiff; 327 | grdDiff = grdDiff > TAU_2_US ? TAU_2_US : grdDiff; 328 | rcostVol[costVol_offset] = (uchar)( ALPHA * clrDiff + (1-ALPHA) * grdDiff ); 329 | } 330 | 331 | /** 332 | * \brief Cost Volume Construction kernel function. 333 | * \param[in] lImg - Left Input image data. 334 | * \param[in] rImg - Right Input image data. 335 | * \param[in] lGrdX - Left Input X dim gradient data. 336 | * \param[in] rGrdX - Right Input X dim gradient data. 337 | * \param[in] height - Height of the image. 338 | * \param[in] width - Width of the image. 339 | * \param[out] lcostVol - Calculated pixel cost for Cost Volume. 340 | * \param[out] rcostVol - Calculated pixel cost for Cost Volume. 341 | */ 342 | __kernel void cvc_float_nv(__global const float* lImgR, 343 | __global const float* lImgG, 344 | __global const float* lImgB, 345 | __global const float* rImgR, 346 | __global const float* rImgG, 347 | __global const float* rImgB, 348 | __global const float* lGrdX, 349 | __global const float* rGrdX, 350 | const int height, 351 | const int width, 352 | __global float* lcostVol, 353 | __global float* rcostVol) 354 | { 355 | /* [Kernel size] */ 356 | /* 357 | * Each kernel calculates a single output pixels in the same row. 358 | * column (x) is in the range [0, width]. 359 | * row (y) is in the range [0, height]. 360 | */ 361 | const int x = get_global_id(0); 362 | const int y = get_global_id(1); 363 | const int d = get_global_id(2); 364 | 365 | /* Offset calculates the position in the linear data for the row and the column. */ 366 | const int offset = y * width + x; 367 | const int costVol_offset = ((d * height) + y) * width + x; 368 | 369 | /* *************** Left to Right Cost Volume Construction ********************** */ 370 | float clrDiff, grdDiff; 371 | 372 | if(x >= d) 373 | { 374 | // three color diff 375 | clrDiff = (fabs(lImgR[offset] - rImgR[offset - d]) 376 | + fabs(lImgG[offset] - rImgG[offset - d]) 377 | + fabs(lImgB[offset] - rImgB[offset - d]))/3; 378 | // gradient diff 379 | grdDiff = fabs(lGrdX[offset] - rGrdX[offset - d]); 380 | } 381 | else 382 | { 383 | // three color diff at img boundary 384 | clrDiff = (fabs(lImgR[offset] - 1.0f) 385 | + fabs(lImgG[offset] - 1.0f) 386 | + fabs(lImgB[offset] - 1.0f))/3; 387 | // gradient diff 388 | grdDiff = fabs(lGrdX[offset] - 1.0f); 389 | } 390 | 391 | clrDiff = clrDiff > TAU_1_32F ? TAU_1_32F : clrDiff; 392 | grdDiff = grdDiff > TAU_2_32F ? TAU_2_32F : grdDiff; 393 | lcostVol[costVol_offset] = ( ALPHA * clrDiff + (1-ALPHA) * grdDiff ); 394 | 395 | 396 | /* *************** Right to Left Cost Volume Construction ********************** */ 397 | clrDiff = 0; 398 | grdDiff = 0; 399 | 400 | if(x >= d) 401 | { 402 | // three color diff 403 | clrDiff = (fabs(rImgR[offset] - lImgR[offset + d]) 404 | + fabs(rImgG[offset] - lImgG[offset + d]) 405 | + fabs(rImgB[offset] - lImgB[offset + d]))/3; 406 | // gradient diff 407 | grdDiff = fabs(rGrdX[offset] - lGrdX[offset + d]); 408 | } 409 | else 410 | { 411 | // three color diff at img boundary 412 | clrDiff = (fabs(rImgR[offset] - 1.0f) 413 | + fabs(rImgG[offset] - 1.0f) 414 | + fabs(rImgB[offset] - 1.0f))/3; 415 | // gradient diff 416 | grdDiff = fabs(rGrdX[offset] - 1.0f); 417 | } 418 | 419 | clrDiff = clrDiff > TAU_1_32F ? TAU_1_32F : clrDiff; 420 | grdDiff = grdDiff > TAU_2_32F ? TAU_2_32F : grdDiff; 421 | rcostVol[costVol_offset] = ( ALPHA * clrDiff + (1-ALPHA) * grdDiff ); 422 | } 423 | 424 | /** 425 | * \brief Cost Volume Construction kernel function. 426 | * \param[in] lImg - Left Input image data. 427 | * \param[in] rImg - Right Input image data. 428 | * \param[in] lGrdX - Left Input X dim gradient data. 429 | * \param[in] rGrdX - Right Input X dim gradient data. 430 | * \param[in] height - Height of the image. 431 | * \param[in] width - Width of the image. 432 | * \param[out] lcostVol - Calculated pixel cost for Cost Volume. 433 | * \param[out] rcostVol - Calculated pixel cost for Cost Volume. 434 | */ 435 | __kernel void cvc_float_v4(__global const float* restrict lImg, 436 | __global const float* restrict rImg, 437 | __global const float* restrict lGrdX, 438 | __global const float* restrict rGrdX, 439 | const int height, 440 | const int width, 441 | __global float* restrict lcostVol, 442 | __global float* restrict rcostVol) 443 | { 444 | /* [Kernel size] */ 445 | /* 446 | * Each kernel calculates a single output pixels in the same row. 447 | * column (x) is in the range [0, width]. 448 | * row (y) is in the range [0, height]. 449 | */ 450 | const int x = get_global_id(0) * 4; 451 | const int y = get_global_id(1); 452 | const int d = get_global_id(2); 453 | 454 | /* Offset calculates the position in the linear data for the row and the column. */ 455 | int img_offset = y * width + x; 456 | const int costVol_offset = ((d * height) + y) * width + x; 457 | 458 | /// /* *************** Left to Right Cost Volume Construction ********************** */ 459 | // Set addresses for lImg, rImg, lGrdX and rGrdX. 460 | float16 rC, lC = vload16(0, lImg + (img_offset * 3)); 461 | float4 rGX, lGX = vload4(0, lGrdX + img_offset); 462 | 463 | float4 clrDiff = 0; 464 | float4 grdDiff = 0; 465 | 466 | if(x - d >= 0) 467 | { 468 | img_offset = y * width + x - d; 469 | rC = vload16(0, rImg + (img_offset * 3)); 470 | rGX = vload4(0, rGrdX + img_offset); 471 | 472 | // three color diff 473 | lC = fabs(lC - rC); 474 | // gradient diff 475 | grdDiff = fabs(lGX - rGX); 476 | } 477 | else 478 | { 479 | // three color diff at img boundary 480 | lC = fabs(lC - 1.0f); //BORDER_CONSTANT = 1.0 481 | // gradient diff 482 | grdDiff = fabs(lGX - 1.0f); //BORDER_CONSTANT = 1.0 483 | } 484 | clrDiff = (float4)(lC.s0 + lC.s1 + lC.s2, 485 | lC.s3 + lC.s4 + lC.s5, 486 | lC.s6 + lC.s7 + lC.s8, 487 | lC.s9 + lC.sa + lC.sb) * 0.3333333333f; 488 | 489 | clrDiff = clrDiff > 0.028f ? 0.028f : clrDiff; 490 | grdDiff = grdDiff > 0.008f ? 0.008f : grdDiff; 491 | vstore4(0.9f * clrDiff + 0.1f * grdDiff, 0, lcostVol + costVol_offset); //data, offset, addr 492 | // *(lcostVol + costVol_offset + 4) = 0.9 * clrDiff.s4 + 0.1 * grdDiff.s4; 493 | 494 | 495 | /// /* *************** Right to Left Cost Volume Construction ********************** */ 496 | // Set addresses for lImg, rImg, lGrdX and rGrdX. 497 | img_offset = y * width + x; 498 | lC = vload16(0, rImg + (img_offset * 3)); 499 | lGX = vload4(0, rGrdX + img_offset); 500 | 501 | grdDiff = 0; 502 | 503 | if(x + d < width) 504 | { 505 | img_offset = y * width + x + d; 506 | rC = vload16(0, lImg + (img_offset * 3)); 507 | rGX = vload4(0, lGrdX + img_offset); 508 | 509 | // three color diff 510 | lC = fabs(lC - rC); 511 | // gradient diff 512 | grdDiff = fabs(lGX - rGX); 513 | } 514 | else 515 | { 516 | // three color diff at img boundary 517 | lC = fabs(lC - 1.0f); //BORDER_CONSTANT = 1.0 518 | // gradient diff 519 | grdDiff = fabs(lGX - 1.0f); //BORDER_CONSTANT = 1.0 520 | } 521 | clrDiff = (float4)(lC.s0 + lC.s1 + lC.s2, 522 | lC.s3 + lC.s4 + lC.s5, 523 | lC.s6 + lC.s7 + lC.s8, 524 | lC.s9 + lC.sa + lC.sb) * 0.3333333333f; 525 | 526 | clrDiff = clrDiff > 0.028f ? 0.028f : clrDiff; 527 | grdDiff = grdDiff > 0.008f ? 0.008f : grdDiff; 528 | vstore4(0.9f * clrDiff + 0.1f * grdDiff, 0, rcostVol + costVol_offset); //data, offset, addr 529 | //*(rcostVol + costVol_offset + 4) = 0.9 * clrDiff.s4 + 0.1 * grdDiff.s4; 530 | } 531 | 532 | -------------------------------------------------------------------------------- /assets/cvf.cl: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | cvf.cl - OpenCL Cost Volume Filtering Kernels 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | All rights reserved. 8 | ---------------------------------------------------------------------------*/ 9 | 10 | //######################################################################################### 11 | //# Float Kernel Versions (_32F) 12 | //######################################################################################### 13 | 14 | /** 15 | * \brief Kernel function for element-wise multiplication of 3D and 3D Matricies. 16 | * \param[in] pIn_a - First Input Matrix. 17 | * \param[in] pIn_b - Second Input Matrix. 18 | * \param[in] width - Image Width. 19 | * \param[in] height - Image Height. 20 | * \param[out] pOut - Output Matrix. 21 | */ 22 | 23 | __kernel void EWMul_SameDim_32F(__global const float* pIn_a, 24 | __global const float* pIn_b, 25 | const int width, 26 | const int height, 27 | __global float* pOut) 28 | { 29 | const int x = get_global_id(0); 30 | const int y = get_global_id(1); 31 | const int d = get_global_id(2); 32 | 33 | const int offset = (((d * height) + y) * width) + x; 34 | 35 | pOut[offset] = pIn_a[offset] * pIn_b[offset]; 36 | } 37 | 38 | /** 39 | * \brief Kernel function for element-wise multiplication of 2D and 3D Matricies. 40 | * \param[in] pIn_a - First Input Matrix (2D). 41 | * \param[in] pIn_b - Second Input Matrix (3D). 42 | * \param[in] width - Image Width. 43 | * \param[in] height - Image Height. 44 | * \param[out] pOut - Output Matrix. 45 | */ 46 | 47 | __kernel void EWMul_DiffDim_32F(__global const float* pIn_a, 48 | __global const float* pIn_b, 49 | const int width, 50 | const int height, 51 | __global float* pOut) 52 | { 53 | const int x = get_global_id(0); 54 | const int y = get_global_id(1); 55 | const int d = get_global_id(2); 56 | 57 | const int offset2D = (y * width) + x; 58 | const int offset3D = (((d * height) + y) * width) + x; 59 | 60 | pOut[offset3D] = pIn_a[offset2D] * pIn_b[offset3D]; 61 | } 62 | 63 | /** 64 | * \brief Kernel function for element-wise division of 3D and 3D Matricies. 65 | * \param[in] pIn_a - First Input Matrix. 66 | * \param[in] pIn_b - Second Input Matrix. 67 | * \param[in] width - Image Width. 68 | * \param[in] height - Image Height. 69 | * \param[out] pOut - Output Matrix. 70 | */ 71 | 72 | __kernel void EWDiv_SameDim_32F(__global const float* pIn_a, 73 | __global const float* pIn_b, 74 | const int width, 75 | const int height, 76 | __global float* pOut) 77 | { 78 | const int x = get_global_id(0); 79 | const int y = get_global_id(1); 80 | const int d = get_global_id(2); 81 | 82 | const int offset = (((d * height) + y) * width) + x; 83 | 84 | pOut[offset] = pIn_a[offset] / pIn_b[offset]; 85 | } 86 | 87 | /** 88 | * \brief Kernel function for channel division of an RGB image 89 | * \param[in] pImg - 3-channel RGB Input image. 90 | * \param[in] width - Image Width. 91 | * \param[in] height - Image Height. 92 | * \param[out] pIr - Red channel output. 93 | * \param[out] pIg - Green channel output. 94 | * \param[out] pIb - Blue channel output. 95 | */ 96 | 97 | __kernel void Split_32F(__global const float* pImg, 98 | const int width, 99 | __global float* pIr, 100 | __global float* pIg, 101 | __global float* pIb) 102 | { 103 | const int x = get_global_id(0) * 12; 104 | const int y = get_global_id(1); 105 | 106 | const int offset = (y * width) + x; 107 | 108 | pIr[offset] = pImg[offset]; 109 | pIg[offset] = pImg[offset + 1]; 110 | pIb[offset] = pImg[offset + 2]; 111 | 112 | pIr[offset + 1] = pImg[offset + 3]; 113 | pIg[offset + 1] = pImg[offset + 4]; 114 | pIb[offset + 1] = pImg[offset + 5]; 115 | 116 | pIr[offset + 2] = pImg[offset + 6]; 117 | pIg[offset + 2] = pImg[offset + 7]; 118 | pIb[offset + 2] = pImg[offset + 8]; 119 | 120 | pIr[offset + 3] = pImg[offset + 9]; 121 | pIg[offset + 3] = pImg[offset + 10]; 122 | pIb[offset + 3] = pImg[offset + 11]; 123 | } 124 | 125 | /** 126 | * \brief Kernel function for the subtraction of 2 matricies 127 | * \param[in] pIn_a - First Input Matrix. 128 | * \param[in] pIn_b - Second Input Matrix. 129 | * \param[in] width - Image Width. 130 | * \param[in] height - Image Height. 131 | * \param[out] pOut - Output Matrix. 132 | */ 133 | 134 | __kernel void Subtract_32F(__global const float* pIn_a, 135 | __global const float* pIn_b, 136 | const int width, 137 | const int height, 138 | __global float* pOut) 139 | { 140 | const int x = get_global_id(0); 141 | const int y = get_global_id(1); 142 | const int d = get_global_id(2); 143 | 144 | const int offset = (((d * height) + y) * width) + x; 145 | 146 | pOut[offset] = pIn_a[offset] - pIn_b[offset]; 147 | } 148 | 149 | /** 150 | * \brief Kernel function for the addition of 2 matricies 151 | * \param[in] pIn_a - First Input Matrix. 152 | * \param[in] pIn_b - Second Input Matrix. 153 | * \param[in] width - Image Width. 154 | * \param[in] height - Image Height. 155 | * \param[out] pOut - Output Matrix. 156 | */ 157 | 158 | __kernel void Add_32F(__global const float* pIn_a, 159 | __global const float* pIn_b, 160 | const int width, 161 | const int height, 162 | __global float* pOut) 163 | { 164 | const int x = get_global_id(0); 165 | const int y = get_global_id(1); 166 | const int d = get_global_id(2); 167 | 168 | const int offset = (((d * height) + y) * width) + x; 169 | 170 | pOut[offset] = pIn_a[offset] + pIn_b[offset]; 171 | } 172 | 173 | /** 174 | * \brief Volume BoxFiltering kernel function. 175 | * \param[in] pIn - Input data. 176 | * \param[in] width - Image Width. 177 | * \param[in] height - Image Height. 178 | * \param[out] pOut - Output filtered data. 179 | */ 180 | __kernel void BoxFilter_32F(__global const float* pIn, 181 | const int width, 182 | const int height, 183 | __global float* pOut) 184 | { 185 | int load_rows = 9; 186 | 187 | /* [Kernel size] */ 188 | /* Each kernel calculates a single output pixel. 189 | * column (x) is in the range [0, width]. 190 | * row (y) is in the range [0, height]. 191 | * disparity (d) is in the range [0, maxDis]. 192 | */ 193 | const int x = get_global_id(0) * 16; 194 | const int y = get_global_id(1); 195 | const int d = get_global_id(2); 196 | 197 | const int offset = (((d * height) + y) * width) + x; 198 | 199 | float16 row[9][9]; 200 | float16 res; 201 | 202 | for (int r=0; r 2 | 3 | 4 | chessboard0L.png chessboard0R.png chessboard1L.png chessboard1R.png 5 | chessboard2L.png chessboard2R.png chessboard3L.png chessboard3R.png 6 | chessboard4L.png chessboard4R.png chessboard5L.png chessboard5R.png 7 | chessboard6L.png chessboard6R.png chessboard7L.png chessboard7R.png 8 | chessboard8L.png chessboard8R.png chessboard9L.png chessboard9R.png 9 | 10 | -------------------------------------------------------------------------------- /docs/de_bd.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRiME-project/PRiMEStereoMatch/3a28bb633cbe455286a48a39912d7a6c1f328063/docs/de_bd.png -------------------------------------------------------------------------------- /docs/de_examples.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRiME-project/PRiMEStereoMatch/3a28bb633cbe455286a48a39912d7a6c1f328063/docs/de_examples.png -------------------------------------------------------------------------------- /include/CVC.h: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | CVC.h - Cost Volume Construction Header 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | ---------------------------------------------------------------------------*/ 8 | #include "ComFunc.h" 9 | 10 | // CVPR 11 11 | #define BORDER_THRES 0.011764 12 | #define BC_32F 1.0 13 | #define BC_32UI UINT_MAX 14 | #define BC_8U UCHAR_MAX 15 | 16 | //#define TAU_1 0.7 17 | //#define TAU_2 0.2 18 | #define TAU_1_32F 0.028 19 | #define TAU_2_32F 0.008 20 | #define TAU_1_16U 1825 21 | #define TAU_2_16U 524 22 | 23 | #define ALPHA_32F 0.9f 24 | #define ALPHA_32UI (unsigned int)(0.9f*UINT_MAX) 25 | #define ALPHA_16U 0.9 26 | 27 | // 28 | // TAD + GRD for Cost Computation 29 | // 30 | class CVC 31 | { 32 | public: 33 | CVC(void); 34 | ~CVC(void); 35 | 36 | int preprocess(const Mat& Img, Mat& GrdX); 37 | 38 | static void *buildCV_left_thread(void *thread_arg); 39 | static void *buildCV_right_thread(void *thread_arg); 40 | 41 | int buildCV_left(const Mat& lImg, const Mat& rImg, const Mat& lGrdX, const Mat& rGrdX, const int d, Mat& costVol); 42 | int buildCV_right(const Mat& lImg, const Mat& rImg, const Mat& lGrdX, const Mat& rGrdX, const int d, Mat& costVol); 43 | }; 44 | 45 | //CVC thread data struct 46 | struct buildCV_TD{ 47 | Mat* lImg; 48 | Mat* rImg; 49 | Mat* lGrdX; 50 | Mat* rGrdX; 51 | int d; 52 | Mat* costVol; 53 | }; 54 | -------------------------------------------------------------------------------- /include/CVC_cl.h: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | CVC_cl.h - OpenCL Cost Volume Construction Header 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | All rights reserved. 8 | ---------------------------------------------------------------------------*/ 9 | #include "ComFunc.h" 10 | #include "oclUtil.h" 11 | 12 | #define FILE_CVC_PROG BASE_DIR "assets/cvc.cl" 13 | 14 | // 15 | // TAD + GRD for Cost Computation 16 | // 17 | class CVC_cl 18 | { 19 | public: 20 | 21 | //Data Variables 22 | Mat lGray, rGray; 23 | Mat lGrdX, rGrdX; 24 | int maxDis; 25 | Mat *lImgRGB, *rImgRGB; 26 | 27 | //OpenCL Variables 28 | cl_context* context; 29 | cl_command_queue* commandQueue; 30 | cl_program program; 31 | char kernel_name[128]; 32 | cl_kernel kernel; 33 | cl_int errorNumber; 34 | cl_event event; 35 | 36 | cl_int width, height, channels; 37 | size_t bufferSize_2D, bufferSize_3D; 38 | size_t globalWorksize[3]; 39 | 40 | CVC_cl(cl_context* context, cl_command_queue* commandQueue, cl_device_id device, Mat* I, const int d); 41 | ~CVC_cl(void); 42 | 43 | int buildCV(const Mat& lImg, const Mat& rImg, cl_mem* memoryObjects); 44 | }; 45 | -------------------------------------------------------------------------------- /include/CVF.h: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | CVF.h - Cost Volume Filter Header 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | All rights reserved. 8 | ---------------------------------------------------------------------------*/ 9 | #include "ComFunc.h" 10 | 11 | // 12 | // GIF for Cost Computation 13 | // 14 | class CVF 15 | { 16 | public: 17 | 18 | CVF(); 19 | ~CVF(); 20 | 21 | static void *filterCV_thread(void *thread_arg); 22 | int preprocess(const Mat& Img, Mat* Img_rgb, Mat* mean_Img, Mat* var_Img); 23 | int filterCV(const Mat* Img_rgb, const Mat* mean_Img, const Mat* var_Img, Mat& costVol); 24 | }; 25 | Mat GuidedFilter_cv(const Mat* rgb, const Mat* mean_I, const Mat* var_I, const Mat& p); 26 | 27 | //CVF thread data struct 28 | struct filterCV_TD{Mat* Img_rgb; Mat* mean_Img; Mat* var_Img; Mat* costVol;}; 29 | -------------------------------------------------------------------------------- /include/CVF_cl.h: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | CVF_cl.h - OpenCL Cost Volume Filter Header 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | All rights reserved. 8 | ---------------------------------------------------------------------------*/ 9 | #include "ComFunc.h" 10 | #include "oclUtil.h" 11 | 12 | #define FILE_CVF_PROG BASE_DIR "assets/cvf.cl" 13 | #define R_WIN 9 14 | 15 | // 16 | // GIF for Cost Computation 17 | // 18 | class CVF_cl 19 | { 20 | public: 21 | CVF_cl(cl_context* context, cl_command_queue* commandQueue, cl_device_id device, Mat* I, const int d); 22 | ~CVF_cl(void); 23 | 24 | int preprocess(cl_mem* Ir, cl_mem* Ig, cl_mem* Ib); 25 | int filterCV(cl_mem* cl_costVol); 26 | 27 | private: 28 | //OpenCL Variables 29 | cl_context* context; 30 | cl_command_queue* commandQueue; 31 | cl_program program; 32 | cl_kernel kernel_mmsd, kernel_mmdd, kernel_mdsd, kernel_bf; 33 | cl_kernel kernel_split, kernel_sub, kernel_add; 34 | cl_kernel kernel_centf, kernel_var; 35 | cl_kernel kernel_bfc_rows, kernel_bfc_cols; 36 | cl_int errorNumber; 37 | cl_event event; 38 | 39 | // int imgType; 40 | cl_int width, height, channels, maxDis; 41 | size_t bufferSize_2D, bufferSize_3D; 42 | 43 | size_t globalWorksize_3D[3], globalWorksize_2D[3]; 44 | size_t globalWorksize_bf_3D[3], globalWorksize_bf_2D[3]; 45 | size_t globalWorksize_bfc_3D[3], globalWorksize_bfc_2D[3]; 46 | size_t globalWorksize_split[2]; 47 | 48 | cl_mem *Ir, *Ig, *Ib; 49 | cl_mem mean_cv; 50 | cl_mem *Ixx, *mean_I, *mean_Ixx; 51 | cl_mem *var_I, *cov_Ip, *a; 52 | 53 | cl_mem tmp_3DA_r, tmp_3DA_g, tmp_3DA_b; 54 | cl_mem tmp_3DB_r, tmp_3DB_g, tmp_3DB_b; 55 | cl_mem bf2Dtmp, bf3Dtmp; 56 | 57 | int elementwiseMulSD(cl_mem *cl_in_a, cl_mem *cl_in_b, cl_mem *cl_out, size_t *globalworksize); 58 | int elementwiseMulDD(cl_mem *cl_in_a, cl_mem *cl_in_b, cl_mem *cl_out); 59 | int elementwiseDivSD(cl_mem *cl_in_a, cl_mem *cl_in_b, cl_mem *cl_out, size_t *globalworksize); 60 | int split(cl_mem* cl_Img, cl_mem* cl_Ir, cl_mem* cl_Ig, cl_mem* cl_Ib); 61 | int sub(cl_mem *cl_in_a, cl_mem *cl_in_b, cl_mem *cl_out, size_t *globalworksize); 62 | int add(cl_mem *cl_in_a, cl_mem *cl_in_b, cl_mem *cl_out, size_t *globalworksize); 63 | int central_filter(cl_mem *mean_I_in, cl_mem *mean_cv_io, cl_mem *var_I_in, cl_mem *cov_Ip_in, cl_mem *a_in, size_t *globalworksize); 64 | int preproc_maths(cl_mem *mean_I_in, cl_mem *mean_Ixx_in, cl_mem *var_I_out, size_t *globalworksize); 65 | // int boxfilter(cl_mem *cl_in, cl_mem *cl_out, size_t *globalworksize); 66 | int boxfilter(cl_mem *cl_in, cl_mem *cl_tmp, cl_mem *cl_out, size_t *globalworksize); 67 | }; 68 | -------------------------------------------------------------------------------- /include/ComFunc.h: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | ComFunc.h - Function and Header Linkage File 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | ---------------------------------------------------------------------------*/ 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | //POSIX Threads 26 | #include 27 | 28 | //OpenCL Header 29 | #define CL_USE_DEPRECATED_OPENCL_1_2_APIS 30 | #ifdef __APPLE__ 31 | #include 32 | #else 33 | #include 34 | #endif 35 | #include 36 | 37 | //OpenCV Header 38 | #include 39 | 40 | #define BASE_DIR "../" 41 | 42 | //Algorithm Definitions 43 | #define STEREO_SGBM 0 44 | #define STEREO_GIF 1 45 | 46 | #define OCV_DE 0 47 | #define OCL_DE 1 48 | 49 | #define GIF_R_WIN 8 50 | #define GIF_EPS 0.0001f 51 | 52 | #define MAX_CPU_THREADS 8 53 | #define MIN_CPU_THREADS 1 54 | 55 | #define MAX_GPU_THREADS 256 56 | #define MIN_GPU_THREADS 4 57 | 58 | #define OCL_STATS 0 59 | 60 | using namespace cv; 61 | 62 | #ifndef COMFUNC_H 63 | #define COMFUNC_H 64 | 65 | enum buff_id {CVC_LIMGR, CVC_LIMGG, CVC_LIMGB, CVC_RIMGR, CVC_RIMGG, CVC_RIMGB, CVC_LGRDX, CVC_RGRDX, CV_LCV, CV_RCV, DS_LDM, DS_RDM}; 66 | 67 | static float get_rt(){ 68 | struct timespec realtime; 69 | clock_gettime(CLOCK_MONOTONIC,&realtime); 70 | return (float)(realtime.tv_sec*1000000+realtime.tv_nsec/1000); 71 | } 72 | 73 | #endif // COMFUNC_H 74 | -------------------------------------------------------------------------------- /include/DispEst.h: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | DispEst.h - Disparity Estimation Class Header 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | ---------------------------------------------------------------------------*/ 8 | #include "ComFunc.h" 9 | #include "CVC.h" 10 | #include "CVC_cl.h" 11 | #include "CVF.h" 12 | #include "CVF_cl.h" 13 | #include "DispSel.h" 14 | #include "DispSel_cl.h" 15 | #include "PP.h" 16 | #include "oclUtil.h" 17 | #include "fastguidedfilter.h" 18 | // 19 | // Top-level Disparity Estimation Class 20 | // 21 | class DispEst 22 | { 23 | public: 24 | DispEst(cv::Mat l, cv::Mat r, const int d, int t, bool ocl); 25 | ~DispEst(void); 26 | 27 | //DispSel 28 | cv::Mat lDisMap; 29 | cv::Mat rDisMap; 30 | 31 | //Public Methods 32 | int setInputImages(cv::Mat l, cv::Mat r); 33 | int setThreads(unsigned int newThreads); 34 | void setSubsampleRate(unsigned int newRate) {subsample_rate = newRate;}; 35 | int printCV(void); 36 | 37 | int CostConst(); 38 | int CostConst_CPU(); 39 | int CostConst_GPU(); 40 | 41 | int CostFilter(); 42 | int CostFilter_CPU(); 43 | int CostFilter_GPU(); 44 | int CostFilter_FGF(); 45 | 46 | 47 | int DispSelect_CPU(); 48 | int DispSelect_GPU(); 49 | 50 | int PostProcess_CPU(); 51 | int PostProcess_GPU(); 52 | 53 | private: 54 | //Private Variable 55 | cv::Mat lImg; 56 | cv::Mat rImg; 57 | 58 | int hei; 59 | int wid; 60 | int maxDis; 61 | int threads; 62 | bool useOCL; 63 | unsigned int subsample_rate = 4; 64 | 65 | //CVC 66 | cv::Mat lGrdX; 67 | cv::Mat rGrdX; 68 | //CVC & CVF 69 | // Mat lcostVol_cvc; 70 | // Mat rcostVol_cvc; 71 | cv::Mat* lcostVol; 72 | cv::Mat* rcostVol; 73 | //CVF 74 | // Mat* lImg_rgb; 75 | // Mat* rImg_rgb; 76 | // Mat* mean_lImg; 77 | // Mat* mean_rImg; 78 | // Mat* var_lImg; 79 | // Mat* var_rImg; 80 | //PP 81 | cv::Mat lValid; 82 | cv::Mat rValid; 83 | 84 | CVC* constructor; 85 | CVF* filter; 86 | DispSel* selector; 87 | PP* postProcessor; 88 | 89 | CVC_cl* constructor_cl; 90 | CVF_cl* filter_cl; 91 | DispSel_cl* selector_cl; 92 | 93 | //OpenCL Variables 94 | cl_context context; 95 | cl_command_queue commandQueue; 96 | cl_device_id device; 97 | unsigned int numberOfMemoryObjects; 98 | cl_mem memoryObjects[12]; //OpenCL Memory Buffers 99 | cl_int errorNumber; 100 | cl_event event; 101 | 102 | cl_int width, height, channels; 103 | size_t bufferSize_2D_8UC1; //DispMap, 104 | size_t bufferSize_2D; //Img, Gray, GrdX, 105 | size_t bufferSize_3D; //costVol 106 | 107 | //Private Methods 108 | //None 109 | }; 110 | -------------------------------------------------------------------------------- /include/DispSel.h: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | DispSel.h - Disparity Selection Header 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | ---------------------------------------------------------------------------*/ 8 | #include "ComFunc.h" 9 | 10 | class DispSel 11 | { 12 | public: 13 | DispSel(); 14 | ~DispSel(); 15 | 16 | int CVSelect(cv::Mat* costVol, const unsigned int maxDis, cv::Mat& dispMap); 17 | int CVSelect_thread(cv::Mat* costVol, const unsigned int maxDis, cv::Mat& dispMap, int threads); 18 | }; 19 | 20 | struct DS_X_TD{Mat* costVol; cv::Mat* dispMap; int y; unsigned int maxDis;}; 21 | -------------------------------------------------------------------------------- /include/DispSel_cl.h: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | DispSel_cl.h - OpenCL Disparity Selection Header 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | All rights reserved. 8 | ---------------------------------------------------------------------------*/ 9 | #include "ComFunc.h" 10 | #include "oclUtil.h" 11 | 12 | #define FILE_DS_PROG BASE_DIR "assets/dispsel.cl" 13 | 14 | class DispSel_cl 15 | { 16 | public: 17 | // int imgType; 18 | const int maxDis; 19 | 20 | //OpenCL Variables 21 | cl_context* context; 22 | cl_command_queue* commandQueue; 23 | cl_program program; 24 | char kernel_name[128]; 25 | cl_kernel kernel; 26 | cl_int errorNumber; 27 | cl_event event; 28 | 29 | cl_int width, height; 30 | size_t bufferSize_2D_8UC1, bufferSize_3D_8UC1; 31 | size_t globalWorksize[2]; 32 | 33 | DispSel_cl(cl_context* context, cl_command_queue* commandQueue, cl_device_id device, Mat* I, const int d); 34 | ~DispSel_cl(void); 35 | 36 | int CVSelect(cl_mem* memoryObjects, Mat& ldispMap, Mat& rdispMap); 37 | }; 38 | 39 | -------------------------------------------------------------------------------- /include/JointWMF.h: -------------------------------------------------------------------------------- 1 | 2 | 3 | /***************************************************************/ 4 | /* 5 | * Distribution code Version 1.1 -- 09/21/2014 by Qi Zhang Copyright 2014, The Chinese University of Hong Kong. 6 | * 7 | * The Code is created based on the method described in the following paper 8 | * [1] "100+ Times Faster Weighted Median Filter", Qi Zhang, Li Xu, Jiaya Jia, IEEE Conference on 9 | * Computer Vision and Pattern Recognition (CVPR), 2014 10 | * 11 | * Due to the adaption for supporting mask and different types of input, this code is 12 | * slightly slower than the one claimed in the original paper. Please use 13 | * our executable on our website for performance comparison. 14 | * 15 | * The code and the algorithm are for non-comercial use only. 16 | * 17 | /***************************************************************/ 18 | 19 | 20 | #ifndef JOINT_WMF_H 21 | #define JOINT_WMF_H 22 | 23 | /***************************************************************/ 24 | /* 25 | * Standard IO library is required. 26 | * STL String library is required. 27 | * 28 | /***************************************************************/ 29 | #include 30 | #include 31 | 32 | /***************************************************************/ 33 | /* 34 | * OpenCV 2.4 is required. 35 | * The following code is already built on OpenCV 2.4.2. 36 | * 37 | /***************************************************************/ 38 | #include "opencv2/core/core.hpp" 39 | #include 40 | #include 41 | 42 | //Use the namespace of CV and STD 43 | using namespace std; 44 | using namespace cv; 45 | 46 | class JointWMF{ 47 | 48 | public: 49 | 50 | /***************************************************************/ 51 | /* Function: filter 52 | * 53 | * Description: filter implementation of joint-histogram weighted median framework 54 | * including clustering of feature image, adaptive quantization of input image. 55 | * 56 | * Input arguments: 57 | * I: input image (any # of channels). Accept only CV_32F and CV_8U type. 58 | * feature: the feature image ("F" in the paper). Accept only CV_8UC1 and CV_8UC3 type (the # of channels should be 1 or 3). 59 | * r: radius of filtering kernel, should be a positive integer. 60 | * sigma: filter range standard deviation for the feature image. 61 | * nI: # of quantization level of input image. (only when the input image is CV_32F type) 62 | * nF: # of clusters of feature value. (only when the feature image is 3-channel) 63 | * iter: # of filtering times/iterations. (without changing the feature map) 64 | * weightType: the type of weight definition, including: 65 | * exp: exp(-|I1-I2|^2/(2*sigma^2)) 66 | * iv1: (|I1-I2|+sigma)^-1 67 | * iv2: (|I1-I2|^2+sigma^2)^-1 68 | * cos: dot(I1,I2)/(|I1|*|I2|) 69 | * jac: (min(r1,r2)+min(g1,g2)+min(b1,b2))/(max(r1,r2)+max(g1,g2)+max(b1,b2)) 70 | * off: unweighted 71 | * mask: a 0-1 mask that has the same size with I. This mask is used to ignore the effect of some pixels. If the pixel value on mask is 0, 72 | * the pixel will be ignored when maintaining the joint-histogram. This is useful for applications like optical flow occlusion handling. 73 | * 74 | * Note: 75 | * 1. When feature image clustering (when F is 3-channel) OR adaptive quantization (when I is floating point image) is 76 | * performed, the result is an approximation. To increase the accuracy, using a larger "nI" or "nF" will help. 77 | * 78 | */ 79 | /***************************************************************/ 80 | 81 | static Mat filter(Mat &I, Mat &feature, int r, float sigma=25.5, int nI=256, int nF=256, int iter=1, string weightType="exp", Mat mask=Mat()){ 82 | 83 | Mat F = feature.clone(); 84 | 85 | //check validation 86 | assert(I.depth() == CV_32F || I.depth() == CV_8U); 87 | assert(F.depth() == CV_8U && (F.channels()==1 || F.channels()==3)); 88 | 89 | //declaration 90 | Mat result; 91 | 92 | //Preprocess I 93 | //OUTPUT OF THIS STEP: Is, iMap 94 | //If I is floating point image, "adaptive quantization" is done in from32FTo32S. 95 | //The mapping of floating value to integer value is stored in iMap (for each channel). 96 | //"Is" stores each channel of "I". The channels are converted to CV_32S type after this step. 97 | vector iMap(I.channels()); 98 | vector Is; 99 | { 100 | split(I,Is); 101 | for(int i=0;i<(int)Is.size();i++){ 102 | if(I.depth()==CV_32F){ 103 | iMap[i] = new float[nI]; 104 | from32FTo32S(Is[i],Is[i],nI,iMap[i]); 105 | } 106 | else if(I.depth()==CV_8U){ 107 | Is[i].convertTo(Is[i],CV_32S); 108 | } 109 | } 110 | } 111 | 112 | //Preprocess F 113 | //OUTPUT OF THIS STEP: F(new), wMap 114 | //If "F" is 3-channel image, "clustering feature image" is done in featureIndexing. 115 | //If "F" is 1-channel image, featureIndexing only does a type-casting on "F". 116 | //The output "F" is CV_32S type, containing indexes of feature values. 117 | //"wMap" is a 2D array that defines the distance between each pair of feature indexes. 118 | // wMap[i][j] is the weight between feature index "i" and "j". 119 | float **wMap; 120 | { 121 | featureIndexing(F, wMap, nF, sigma, weightType); 122 | } 123 | 124 | //Filtering - Joint-Histogram Framework 125 | { 126 | for(int i=0;i<(int)Is.size();i++){ 127 | for(int k=0;k(i); 223 | int *FPtr = F.ptr(i); 224 | uchar *maskPtr = mask.ptr(i); 225 | 226 | for(int j=downX;j<=upX;j++){ 227 | 228 | if(!maskPtr[j])continue; 229 | 230 | int fval = IPtr[j]; 231 | int *curHist = H[fval]; 232 | int gval = FPtr[j]; 233 | 234 | // Maintain necklace table of joint-histogram 235 | if(!curHist[gval] && gval){ 236 | int *curHf = Hf[fval]; 237 | int *curHb = Hb[fval]; 238 | 239 | int p1=0,p2=curHf[0]; 240 | curHf[p1]=gval; 241 | curHf[gval]=p2; 242 | curHb[p2]=gval; 243 | curHb[gval]=p1; 244 | } 245 | 246 | curHist[gval]++; 247 | 248 | // Maintain necklace table of BCB 249 | updateBCB(BCB[gval],BCBf,BCBb,gval,-1); 250 | } 251 | } 252 | } 253 | 254 | for(int y=0;y(y,x)[0]; 261 | float *fPtr = wMap[curIndex]; 262 | int &curMedianVal = medianVal; 263 | 264 | // Compute current balance 265 | int i=0; 266 | do{ 267 | balanceWeight += BCB[i]*fPtr[i]; 268 | i=BCBf[i]; 269 | }while(i); 270 | 271 | // Move cut-point to the left 272 | if(balanceWeight >= 0){ 273 | for(;balanceWeight >= 0 && curMedianVal;curMedianVal--){ 274 | float curWeight = 0; 275 | int *nextHist = H[curMedianVal]; 276 | int *nextHf = Hf[curMedianVal]; 277 | 278 | // Compute weight change by shift cut-point 279 | int i=0; 280 | do{ 281 | curWeight += (nextHist[i]<<1)*fPtr[i]; 282 | 283 | // Update BCB and maintain the necklace table of BCB 284 | updateBCB(BCB[i],BCBf,BCBb,i,-(nextHist[i]<<1)); 285 | 286 | i=nextHf[i]; 287 | }while(i); 288 | 289 | balanceWeight -= curWeight; 290 | } 291 | } 292 | // Move cut-point to the right 293 | else if(balanceWeight < 0){ 294 | for(;balanceWeight < 0 && curMedianVal != nI-1; curMedianVal++){ 295 | float curWeight = 0; 296 | int *nextHist = H[curMedianVal+1]; 297 | int *nextHf = Hf[curMedianVal+1]; 298 | 299 | // Compute weight change by shift cut-point 300 | int i=0; 301 | do{ 302 | curWeight += (nextHist[i]<<1)*fPtr[i]; 303 | 304 | // Update BCB and maintain the necklace table of BCB 305 | updateBCB(BCB[i],BCBf,BCBb,i,nextHist[i]<<1); 306 | 307 | i=nextHf[i]; 308 | }while(i); 309 | balanceWeight += curWeight; 310 | } 311 | } 312 | 313 | // Weighted median is found and written to the output image 314 | if(balanceWeight<0)outImg.ptr(y,x)[0] = curMedianVal+1; 315 | else outImg.ptr(y,x)[0] = curMedianVal; 316 | } 317 | 318 | // Update joint-histogram and BCB when local window is shifted. 319 | { 320 | int fval,gval,*curHist; 321 | // Add entering pixels into joint-histogram and BCB 322 | { 323 | int rownum = y + r + 1; 324 | if(rownum < rows){ 325 | 326 | int *inputImgPtr = I.ptr(rownum); 327 | int *guideImgPtr = F.ptr(rownum); 328 | uchar *maskPtr = mask.ptr(rownum); 329 | 330 | for(int j=downX;j<=upX;j++){ 331 | 332 | if(!maskPtr[j])continue; 333 | 334 | fval = inputImgPtr[j]; 335 | curHist = H[fval]; 336 | gval = guideImgPtr[j]; 337 | 338 | // Maintain necklace table of joint-histogram 339 | if(!curHist[gval] && gval){ 340 | int *curHf = Hf[fval]; 341 | int *curHb = Hb[fval]; 342 | 343 | int p1=0,p2=curHf[0]; 344 | curHf[gval]=p2; 345 | curHb[gval]=p1; 346 | curHf[p1]=curHb[p2]=gval; 347 | } 348 | 349 | curHist[gval]++; 350 | 351 | // Maintain necklace table of BCB 352 | updateBCB(BCB[gval],BCBf,BCBb,gval,((fval <= medianVal)<<1)-1); 353 | } 354 | } 355 | } 356 | 357 | // Delete leaving pixels into joint-histogram and BCB 358 | { 359 | int rownum = y - r; 360 | if(rownum >= 0){ 361 | 362 | int *inputImgPtr = I.ptr(rownum); 363 | int *guideImgPtr = F.ptr(rownum); 364 | uchar *maskPtr = mask.ptr(rownum); 365 | 366 | for(int j=downX;j<=upX;j++){ 367 | 368 | if(!maskPtr[j])continue; 369 | 370 | fval = inputImgPtr[j]; 371 | curHist = H[fval]; 372 | gval = guideImgPtr[j]; 373 | 374 | curHist[gval]--; 375 | 376 | // Maintain necklace table of joint-histogram 377 | if(!curHist[gval] && gval){ 378 | int *curHf = Hf[fval]; 379 | int *curHb = Hb[fval]; 380 | 381 | int p1=curHb[gval],p2=curHf[gval]; 382 | curHf[p1]=p2; 383 | curHb[p2]=p1; 384 | } 385 | 386 | // Maintain necklace table of BCB 387 | updateBCB(BCB[gval],BCBf,BCBb,gval,-((fval <= medianVal)<<1)+1); 388 | } 389 | } 390 | } 391 | } 392 | } 393 | 394 | } 395 | 396 | // Deallocate the memory 397 | { 398 | delete []BCB; 399 | delete []BCBf; 400 | delete []BCBb; 401 | int2D_release(H); 402 | int2D_release(Hf); 403 | int2D_release(Hb); 404 | } 405 | 406 | // end of the function 407 | return outImg; 408 | } 409 | 410 | private: 411 | 412 | static float get_rt(){ 413 | struct timespec realtime; 414 | clock_gettime(CLOCK_MONOTONIC,&realtime); 415 | return (float)(realtime.tv_sec*1000000+realtime.tv_nsec/1000); 416 | } 417 | 418 | /***************************************************************/ 419 | /* Function: updateBCB 420 | * Description: maintain the necklace table of BCB 421 | /***************************************************************/ 422 | static inline void updateBCB(int &num,int *f,int *b,int i,int v){ 423 | 424 | static int p1,p2; 425 | 426 | if(i){ 427 | if(!num){ // cell is becoming non-empty 428 | p2=f[0]; 429 | f[0]=i; 430 | f[i]=p2; 431 | b[p2]=i; 432 | b[i]=0; 433 | } 434 | else if(!(num+v)){// cell is becoming empty 435 | p1=b[i],p2=f[i]; 436 | f[p1]=p2; 437 | b[p2]=p1; 438 | } 439 | } 440 | 441 | // update the cell count 442 | num += v; 443 | } 444 | 445 | /***************************************************************/ 446 | /* Function: float2D 447 | * Description: allocate a 2D float array with dimension "dim1 x dim2" 448 | /***************************************************************/ 449 | static float** float2D(int dim1, int dim2){ 450 | float **ret = new float*[dim1]; 451 | ret[0] = new float[dim1*dim2]; 452 | for(int i=1;i ops; 503 | ops.push_back("exp"); 504 | ops.push_back("iv1"); 505 | ops.push_back("iv2"); 506 | ops.push_back("cos"); 507 | ops.push_back("jac"); 508 | ops.push_back("off"); 509 | 510 | // Get weight type number 511 | int numOfOps = (int)ops.size(); 512 | int op = 0; 513 | for(;op=numOfOps)op=0; 515 | 516 | /* For 1 channel feature image (uchar)*/ 517 | if(F.channels() == 1){ 518 | 519 | nF = 256; 520 | 521 | // Type-casting 522 | F.convertTo(FNew, CV_32S); 523 | 524 | // Computer weight map (weight between each pair of feature index) 525 | { 526 | wMap = float2D(nF,nF); 527 | float nSigmaI = sigmaI; 528 | float divider = (1.0f/(2*nSigmaI*nSigmaI)); 529 | 530 | for(int i=0;i64(6-bit) 547 | const int LOW_NUM = 256>>shift; 548 | static int hash[LOW_NUM][LOW_NUM][LOW_NUM]={0}; 549 | 550 | memset(hash,0,sizeof(hash)); 551 | 552 | // throw pixels into a 2D histogram 553 | int candCnt = 0; 554 | { 555 | 556 | int lowR,lowG,lowB; 557 | uchar *FPtr = F.ptr(); 558 | for(int i=0,i3=0;i>shift; 560 | lowG = FPtr[i3+1]>>shift; 561 | lowR = FPtr[i3+2]>>shift; 562 | 563 | if(hash[lowB][lowG][lowR]==0){ 564 | candCnt++; 565 | hash[lowB][lowG][lowR]=1; 566 | } 567 | } 568 | } 569 | 570 | nF = min(nF, candCnt); 571 | Mat samples(candCnt,3,CV_32F); 572 | 573 | //prepare for K-means 574 | { 575 | int top=0; 576 | for(int i=0;i(top)[0] = (float)i; 579 | samples.ptr(top)[1] = (float)j; 580 | samples.ptr(top)[2] = (float)k; 581 | top++; 582 | } 583 | } 584 | } 585 | 586 | //do K-means 587 | Mat labels; 588 | Mat centers; 589 | { 590 | kmeans(samples, nF, labels, TermCriteria(CV_TERMCRIT_ITER|CV_TERMCRIT_EPS, 0, 10000), KmeansAttempts, KMEANS_PP_CENTERS, centers ); 591 | } 592 | 593 | //make connection (i,j,k) <-> index 594 | { 595 | int top = 0; 596 | for(int i=0;i(top)[0]; 599 | top++; 600 | } 601 | } 602 | } 603 | 604 | // generate index map 605 | { 606 | FNew = Mat(F.size(),CV_32SC1); 607 | 608 | int lowR,lowG,lowB; 609 | uchar *FPtr = F.ptr(); 610 | for(int i=0,i3=0;i>shift; 612 | lowG = FPtr[i3+1]>>shift; 613 | lowR = FPtr[i3+2]>>shift; 614 | 615 | FNew.ptr()[i] = hash[lowB][lowG][lowR]; 616 | } 617 | } 618 | 619 | // Computer weight map (weight between each pair of feature index) 620 | { 621 | wMap = float2D(nF,nF); 622 | float nSigmaI = sigmaI/256.0f*LOW_NUM; 623 | float divider = (1.0f/(2*nSigmaI*nSigmaI)); 624 | 625 | float *length = new float[nF]; 626 | for(int i=0;i(i)[0]; 628 | float a1 = centers.ptr(i)[1]; 629 | float a2 = centers.ptr(i)[2]; 630 | length[i] = sqrt(a0*a0+a1*a1+a2*a2); 631 | } 632 | 633 | 634 | for(int i=0;i(i)[0], b0 = centers.ptr(j)[0]; 637 | float a1 = centers.ptr(i)[1], b1 = centers.ptr(j)[1]; 638 | float a2 = centers.ptr(i)[2], b2 = centers.ptr(j)[2]; 639 | float diff0 = a0-b0; 640 | float diff1 = a1-b1; 641 | float diff2 = a2-b2; 642 | 643 | if(op==0)wMap[i][j] = wMap[j][i] = exp(-(diff0*diff0+diff1*diff1+diff2*diff2)*divider); // EXP 2 644 | else if(op==2)wMap[i][j] = wMap[j][i] = 1.0f / (diff0*diff0+diff1*diff1+diff2*diff2+nSigmaI*nSigmaI); // IV2 645 | else if(op==1)wMap[i][j] = wMap[j][i] = 1.0f/(fabs(diff0)+fabs(diff1)+fabs(diff2)+nSigmaI);// IV1 646 | else if(op==3)wMap[i][j] = wMap[j][i] = (a0*b0+a1*b1+a2*b2)/(length[i]*length[j]); // COS 647 | else if(op==4)wMap[i][j] = wMap[j][i] = (min(a0,b0)+min(a1,b1)+min(a2,b2))/(max(a0,b0)+max(a1,b1)+max(a2,b2)); // Jacard 648 | else if(op==5)wMap[i][j] = wMap[j][i] = 1.0f; // Unweighted 649 | } 650 | } 651 | 652 | delete []length; 653 | 654 | } 655 | 656 | } 657 | 658 | //end of the function 659 | F = FNew; 660 | } 661 | 662 | /***************************************************************/ 663 | /* Function: from32FTo32S 664 | * Description: adaptive quantization for changing a floating-point 1D image to integer image. 665 | * The adaptive quantization strategy is based on binary search, which searches an 666 | * upper bound of quantization error. 667 | * The function also return a mapping between quantized value (32F) and quantized index (32S). 668 | * The mapping is used to convert integer image back to floating-point image after filtering. 669 | /***************************************************************/ 670 | static void from32FTo32S(Mat &img, Mat &outImg, int nI, float *mapping){ 671 | 672 | 673 | int rows = img.rows, cols = img.cols; 674 | int alls = rows * cols; 675 | 676 | float *imgPtr = img.ptr(); 677 | 678 | typedef pair pairFI; 679 | 680 | pairFI *data = (pairFI *)malloc(alls*sizeof(pairFI)); 681 | 682 | // Sort all pixels of the image by ascending order of pixel value 683 | { 684 | #pragma omp parallel for 685 | for(int i=0;i th){ 703 | float m = (r+l)*0.5f; 704 | bool suc = true; 705 | float base = (float)minVal; 706 | int cnt=0; 707 | for(int i=0;ibase+m){ 709 | cnt++; 710 | base = data[i].first; 711 | if(cnt==nI){ 712 | suc = false; 713 | break; 714 | } 715 | } 716 | } 717 | if(suc)r=m; 718 | else l=m; 719 | } 720 | 721 | Mat retImg(img.size(),CV_32SC1); 722 | int *retImgPtr = retImg.ptr(); 723 | 724 | // In the sorted list, divide pixel values into clusters according to the minimum error bound 725 | // Quantize each value to the median of its cluster 726 | // Also record the mapping of quantized value and quantized index. 727 | float base = (float)minVal; 728 | int baseI = 0; 729 | int cnt = 0; 730 | for(int i=0;i<=alls;i++){ 731 | if(i==alls || data[i].first>base+r){ 732 | mapping[cnt] = data[(baseI+i-1)>>1].first; //median 733 | if(i==alls)break; 734 | cnt++; 735 | base = data[i].first; 736 | baseI = i; 737 | } 738 | retImgPtr[data[i].second] = cnt; 739 | } 740 | 741 | free(data); 742 | 743 | //end of the function 744 | outImg = retImg; 745 | } 746 | 747 | /***************************************************************/ 748 | /* Function: from32STo32F 749 | * Description: convert the quantization index image back to the floating-point image accroding to the mapping 750 | /***************************************************************/ 751 | static void from32STo32F(Mat &img, Mat &outImg, float *mapping){ 752 | 753 | Mat retImg(img.size(),CV_32F); 754 | int rows = img.rows, cols = img.cols, alls = rows*cols; 755 | float *retImgPtr = retImg.ptr(); 756 | int *imgPtr = img.ptr(); 757 | 758 | // convert 32S index to 32F real value 759 | #pragma omp parallel for 760 | for(int i=0;i 38 | #include 39 | #include 40 | #include 41 | 42 | // Recalibrate 43 | #define FILE_INTRINSICS BASE_DIR "data/intrinsics.yml" 44 | #define FILE_EXTRINSICS BASE_DIR "data/extrinsics.yml" 45 | #define FILE_CALIB_XML BASE_DIR "data/stereo_calib.xml" 46 | // Recapture 47 | #define FILE_TEMPLATE_LEFT BASE_DIR "data/chessboard%dL.png" 48 | #define FILE_TEMPLATE_RIGHT BASE_DIR "data/chessboard%dR.png" 49 | 50 | struct StereoCameraProperties{ 51 | Mat cameraMatrix[2]; 52 | Mat distCoeffs[2]; 53 | Mat R, T, E, F; //rotation matrix, translation vector, essential matrix E=[T*R], fundamental matrix 54 | Mat R1, R2, P1, P2, Q; 55 | Size imgSize; 56 | Rect roi[2]; //region of interest 57 | }; 58 | 59 | int print_help(); 60 | 61 | void StereoCalib(const std::vector& imagelist, Size boardSize, StereoCameraProperties& props, bool useCalibrated, bool showRectified); 62 | 63 | bool readStringList( const std::string& filename, std::vector& l ); 64 | 65 | int calibrateCamera(int boardWidth, int boardHeight, StereoCameraProperties& props, std::string imagelistfn); 66 | -------------------------------------------------------------------------------- /include/StereoMatch.h: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | StereoMatch.h - Stereo Matching Application Header 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | ---------------------------------------------------------------------------*/ 8 | #ifndef STEREOMATCH_H 9 | #define STEREOMATCH_H 10 | 11 | #include "ComFunc.h" 12 | #include "StereoCalib.h" 13 | #include "DispEst.h" 14 | #include "args.hxx" 15 | 16 | #define DE_VIDEO 1 17 | #define DE_IMAGE 2 18 | 19 | #define NO_MASKS 0 20 | #define MASK_NONE 1 21 | #define MASK_NONOCC 2 22 | #define MASK_DISC 3 23 | 24 | #define DISPLAY 25 | //#define DEBUG_APP 26 | #define DEBUG_APP_MONITORS 27 | 28 | static std::vector dataset_names = std::vector{"Art", "Books", "Cones", "Dolls", "Laundry", "Moebius", "Teddy"}; 29 | 30 | struct Resolution{ 31 | unsigned int height; 32 | unsigned int width; 33 | }; 34 | 35 | class StereoMatch 36 | { 37 | public: 38 | StereoMatch(int argc, const char *argv[], int gotOpenCLDev); 39 | ~StereoMatch(void); 40 | 41 | int de_mode; 42 | int MatchingAlgorithm; 43 | int error_threshold; 44 | int mask_mode; 45 | int media_mode; 46 | cv::Mat display_container; 47 | 48 | int compute(float& de_time); 49 | int update_dataset(std::string dataset_name); 50 | bool user_dataset; 51 | 52 | //StereoSGBM Variables 53 | cv::Ptr ssgbm; 54 | 55 | //Stereo GIF Variables 56 | unsigned int subsample_rate = 4;; 57 | private: 58 | //Variables 59 | bool end_de, recaptureChessboards, recalibrate; 60 | int gotOCLDev; 61 | char cap_key; 62 | 63 | std::string left_img_filename, right_img_filename; 64 | std::string gt_img_filename, mask_occl_filename, mask_disc_filename; 65 | std::string curr_dataset; 66 | std::mutex input_data_m; 67 | bool ground_truth_data; 68 | int mask_mode_next; 69 | int scale_factor, scale_factor_next; 70 | 71 | //Display Variables 72 | cv::Mat leftInputImg, rightInputImg; 73 | cv::Mat leftDispMap, rightDispMap; 74 | cv::Mat gtDispMap, errDispMap; 75 | cv::Mat blankDispMap; 76 | 77 | //local disparity map containers 78 | cv::Mat lDispMap, rDispMap, eDispMap; 79 | cv::Mat errMask; 80 | 81 | //input values 82 | int maxDis; 83 | 84 | //stage & process time measurements 85 | double cvc_time, cvf_time, dispsel_time, pp_time; 86 | double cvc_time_avg, cvf_time_avg, dispsel_time_avg, pp_time_avg; 87 | unsigned int frame_count; 88 | 89 | //Frame Holders & Camera object 90 | cv::Mat lFrame, rFrame, vFrame; 91 | 92 | VideoCapture cap; 93 | //Image rectification maps 94 | cv::Mat mapl[2], mapr[2]; 95 | cv::Rect cropBox; 96 | cv::Mat lFrame_rec, rFrame_rec; 97 | cv::Mat gtFrame; 98 | 99 | //StereoSGBM Variables 100 | StereoCameraProperties camProps; 101 | double minVal, maxVal; 102 | double minVal_gt, maxVal_gt; 103 | cv::Mat imgDisparity16S; 104 | 105 | //StereoGIF Variables 106 | DispEst* SMDE; 107 | int num_threads; 108 | 109 | //Function prototypes 110 | int setCameraResolution(unsigned int height, unsigned int width); 111 | std::vector resolution_search(void); 112 | int stereoCameraSetup(void); 113 | int captureChessboards(void); 114 | int setupOpenCVSGBM(int, int); 115 | int update_display(void); 116 | int parse_cli(int argc, const char * argv[]); 117 | }; 118 | 119 | #endif //STEREOMATCH_H 120 | -------------------------------------------------------------------------------- /include/fastguidedfilter.h: -------------------------------------------------------------------------------- 1 | // Source: https://github.com/Sundrops/fast-guided-filter 2 | // Literature: https://arxiv.org/pdf/1505.00996.pdf 3 | 4 | #ifndef GUIDED_FILTER_H 5 | #define GUIDED_FILTER_H 6 | 7 | #include 8 | 9 | class FastGuidedFilterImpl; 10 | 11 | class FastGuidedFilter 12 | { 13 | public: 14 | FastGuidedFilter(const cv::Mat &I, int r, double eps,int s); 15 | ~FastGuidedFilter(); 16 | 17 | cv::Mat filter(const cv::Mat &p, int depth = -1) const; 18 | 19 | private: 20 | FastGuidedFilterImpl *impl_; 21 | }; 22 | 23 | cv::Mat fastGuidedFilter(const cv::Mat &I, const cv::Mat &p, int r, double eps, int s = 1,int depth = -1); 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /include/oclUtil.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This confidential and proprietary software may be used only as 3 | * authorised by a licensing agreement from ARM Limited 4 | * (C) COPYRIGHT 2013 ARM Limited 5 | * ALL RIGHTS RESERVED 6 | * The entire notice above must be reproduced on all authorised 7 | * copies and copies may only be made to the extent permitted 8 | * by a licensing agreement from ARM Limited. 9 | */ 10 | /*--------------------------------------------------------------------------- 11 | oclUtil.h - OpenCL Utility Function Header 12 | --------------------------------------------------------------------------- 13 | Co-Author: Charles Leech 14 | Email: cl19g10 [at] ecs.soton.ac.uk 15 | ---------------------------------------------------------------------------*/ 16 | #ifndef OCLUTIL_H 17 | #define OCLUTIL_H 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | //OpenCL Header 34 | #define CL_USE_DEPRECATED_OPENCL_1_2_APIS 35 | #ifdef __APPLE__ 36 | #include 37 | #else 38 | #include 39 | #endif 40 | #include 41 | 42 | /** 43 | * \file common.h 44 | * \brief Functions to simplify the use of OpenCL API. 45 | */ 46 | //Added functions: 47 | int openCLdevicepoll(void); 48 | 49 | /** 50 | * \brief Convert OpenCL error numbers to their string form. 51 | * \details Uses the error number definitions from cl.h. 52 | * \param[in] errorNumber The error number returned from an OpenCL command. 53 | * \return A name of the error. 54 | */ 55 | std::string errorNumberToString(cl_int errorNumber); 56 | 57 | /** 58 | * \brief Check an OpenCL error number for errors. 59 | * \details If errorNumber is not CL_SUCESS, the function will print the string form of the error number. 60 | * \param[in] errorNumber The error number returned from an OpenCL command. 61 | * \return False if errorNumber != CL_SUCCESS, true otherwise. 62 | */ 63 | inline bool checkSuccess(cl_int errorNumber) 64 | { 65 | if (errorNumber != CL_SUCCESS) 66 | { 67 | std::cerr << "OpenCL error: " << errorNumberToString(errorNumber) << std::endl; 68 | return false; 69 | } 70 | return true; 71 | } 72 | 73 | /** 74 | * \brief Print the profiling information associated with an OpenCL event. 75 | * \details Prints the time spent in the command queue, the time spent waiting before being submitted to a device, and the execution time. 76 | * \param[in] event The event to get profiling information for. 77 | * \return False if an error occurred, otherwise true. 78 | */ 79 | bool printProfilingInfo(cl_event event); 80 | 81 | /** 82 | * \brief Print a list of the 2D OpenCL image formats supported. 83 | * \param[in] context The OpenCL context to use. 84 | * \return False if an error occurred, otherwise true. 85 | */ 86 | bool printSupported2DImageFormats(cl_context context); 87 | 88 | /** 89 | * \brief Convert cl_channel_order values into their string form. 90 | * \details Uses the channel order definitions from cl.h. 91 | * \param[in] channelOrder The channel order value to convert. 92 | * \return The string form of the channel order. 93 | */ 94 | std::string imageChannelOrderToString(cl_channel_order channelOrder); 95 | 96 | /** 97 | * \brief Convert cl_channel_type values into their string form. 98 | * \details Uses the channel data type definitions from cl.h. 99 | * \param[in] channelDataType The channel data type value to convert. 100 | * \return The string form of the channel data type. 101 | */ 102 | std::string imageChannelDataTypeToString(cl_channel_type channelDataType); 103 | 104 | /** 105 | * \brief Release any OpenCL objects that have been created. 106 | * \details If any of the OpenCL objects passed in are not NULL, they will be freed using the appropriate OpenCL function. 107 | * \return False if an error occurred, otherwise true. 108 | */ 109 | bool cleanUpOpenCL(cl_context context, cl_command_queue commandQueue, cl_program program, cl_kernel kernel, cl_mem* memoryObjects, int numberOfMemoryObjects); 110 | 111 | /** 112 | * \brief Create an OpenCL context on a device on the first available platform. 113 | * \param[out] context Pointer to the created OpenCL context. 114 | * \return False if an error occurred, otherwise true. 115 | */ 116 | bool createContext(cl_context* context); 117 | 118 | /** 119 | * \brief Create an OpenCL context for a sub-device of a device on the first available platform. 120 | * \param[out] context Pointer to the created OpenCL context. 121 | * \return False if an error occurred, otherwise true. 122 | */ 123 | bool createSubDeviceContext(cl_context* context, cl_int numComputeUnits); 124 | 125 | /** 126 | * \brief Create an OpenCL command queue for a given context. 127 | * \param[in] context The OpenCL context to use. 128 | * \param[out] commandQueue The created OpenCL command queue. 129 | * \param[out] device The device in which the command queue is created. 130 | * \return False if an error occurred, otherwise true. 131 | */ 132 | bool createCommandQueue(cl_context context, cl_command_queue* commandQueue, cl_device_id* device); 133 | 134 | /** 135 | * \brief Create an OpenCL program from a given file and compile it. 136 | * \param[in] context The OpenCL context in use. 137 | * \param[in] device The OpenCL device to compile the kernel for. 138 | * \param[in] filename Name of the file containing the OpenCL kernel code to load. 139 | * \param[out] program The created OpenCL program object. 140 | * \return False if an error occurred, otherwise true. 141 | */ 142 | bool createProgram(cl_context context, cl_device_id device, std::string filename, cl_program* program); 143 | 144 | /** 145 | * \brief Query an OpenCL device to see if it supports an extension. 146 | * \param[in] device The device to query. 147 | * \param[in] extension The string name of the extension to query for. 148 | * \return True if the extension is supported on the given device, false otherwise. 149 | */ 150 | bool isExtensionSupported(cl_device_id device, std::string extension); 151 | 152 | void context_notify(const char *notify_message, const void *private_info, size_t cb, void *user_data); 153 | 154 | #endif 155 | -------------------------------------------------------------------------------- /src/CVC.cpp: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | CVC.cpp - Cost Volume Construction Code 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | ---------------------------------------------------------------------------*/ 8 | #include "CVC.h" 9 | 10 | CVC::CVC(void) 11 | { 12 | #ifdef DEBUG_APP 13 | std::cout << "Difference of Colours and Gradients method for Cost Computation." << std::endl; 14 | #endif // DEBUG_APP 15 | } 16 | CVC::~CVC(void) {} 17 | 18 | float myCostGrd(float* lC, float* rC, float* lG, float* rG) 19 | { 20 | // three color diff 21 | float clrDiff = fabs(lC[0] - rC[0]) + fabs(lC[1] - rC[1]) + fabs(lC[2] - rC[2]); 22 | // gradient diff 23 | float grdDiff = fabs(*lG - *rG); 24 | //clrDiff = clrDiff > TAU_1_32F ? TAU_1_32F : clrDiff; //TAU_1 0.028 25 | //grdDiff = grdDiff > TAU_2_32F ? TAU_2_32F : grdDiff; // TAU_2 0.008 26 | return ALPHA_32F * clrDiff + (1 - ALPHA_32F) * grdDiff; // ALPHA 0.9 27 | } 28 | 29 | // special handle for border region 30 | float myCostGrd(float* lC, float* lG) 31 | { 32 | // three color diff 33 | float clrDiff = fabs(lC[0] - BC_32F) + fabs(lC[1] - BC_32F) + fabs(lC[2] - BC_32F); 34 | // gradient diff 35 | float grdDiff = fabs(*lG - BC_32F); 36 | //clrDiff = clrDiff > TAU_1_32F ? TAU_1_32F : clrDiff; 37 | //grdDiff = grdDiff > TAU_2_32F ? TAU_2_32F : grdDiff; 38 | return ALPHA_32F * clrDiff + (1 - ALPHA_32F) * grdDiff; 39 | } 40 | 41 | int CVC::preprocess(const Mat& Img, Mat& GrdX) 42 | { 43 | cv::cvtColor(Img, GrdX, CV_RGB2GRAY); 44 | cv::Sobel(GrdX, GrdX, CV_32F, 1, 0, 1); 45 | return 0; 46 | } 47 | 48 | void *CVC::buildCV_left_thread(void *thread_arg) 49 | { 50 | struct buildCV_TD *t_data = static_cast(thread_arg); 51 | Mat *lImg = t_data->lImg; 52 | Mat *rImg = t_data->rImg; 53 | Mat *lGrdX = t_data->lGrdX; 54 | Mat *rGrdX = t_data->rGrdX; 55 | const int d = t_data->d; 56 | Mat* costVol = t_data->costVol; 57 | 58 | int hei = t_data->lImg->rows; 59 | int wid = t_data->lImg->cols; 60 | 61 | for( int y = 0; y < hei; ++y) { 62 | 63 | float* lData = ( float* ) lImg->ptr( y ); 64 | float* rData = ( float* ) rImg->ptr( y ); 65 | float* lGData = ( float* ) lGrdX->ptr( y ); 66 | float* rGData = ( float* ) rGrdX->ptr( y ); 67 | float* cost = ( float* ) costVol->ptr( y ); 68 | for( int x = d; x < wid; ++x) { 69 | float* lC = lData + 3 * x; 70 | float* rC = rData + 3 * ( x - d ); 71 | float* lG = lGData + x; 72 | float* rG = rGData + x - d; 73 | cost[x] = myCostGrd( lC, rC, lG, rG ); 74 | } 75 | for( int x = 0; x < d; ++x) { 76 | float* lC = lData + 3 * x; 77 | float* lG = lGData + x; 78 | cost[x] = myCostGrd( lC, lG ); 79 | } 80 | } 81 | return (void*)0; 82 | 83 | } 84 | 85 | void *CVC::buildCV_right_thread(void *thread_arg) 86 | { 87 | struct buildCV_TD *t_data; 88 | t_data = (struct buildCV_TD *) thread_arg; 89 | const Mat *lImg = t_data->lImg; 90 | const Mat *rImg = t_data->rImg; 91 | const Mat *lGrdX = t_data->lGrdX; 92 | const Mat *rGrdX = t_data->rGrdX; 93 | const int d = t_data->d; 94 | Mat* costVol = t_data->costVol; 95 | 96 | int hei = lImg->rows; 97 | int wid = lImg->cols; 98 | 99 | for( int y = 0; y < hei; ++y) { 100 | float* lData = ( float* ) lImg->ptr( y ); 101 | float* rData = ( float* ) rImg->ptr( y ); 102 | float* lGData = ( float* ) lGrdX->ptr( y ); 103 | float* rGData = ( float* ) rGrdX->ptr( y ); 104 | float* cost = ( float* ) costVol->ptr( y ); 105 | for( int x = 0; x < wid - d; ++x) { 106 | float* lC = lData + 3 * x; 107 | float* rC = rData + 3 * ( x + d ); 108 | float* lG = lGData + x; 109 | float* rG = rGData + x + d; 110 | cost[x] = myCostGrd( lC, rC, lG, rG ); 111 | } 112 | for( int x = wid - d; x < wid; ++x) { 113 | float* lC = lData + 3 * x; 114 | float* lG = lGData + x; 115 | cost[x] = myCostGrd( lC, lG ); 116 | } 117 | } 118 | 119 | return (void*)0; 120 | } 121 | 122 | int CVC::buildCV_left(const Mat& lImg, const Mat& rImg, const Mat& lGrdX, const Mat& rGrdX, const int d, Mat& costVol) 123 | { 124 | int wid = lImg.cols; 125 | int hei = lImg.rows; 126 | 127 | for(int y = 0; y < hei; ++y) 128 | { 129 | float* lData = (float*)lImg.ptr(y); 130 | float* rData = (float*)rImg.ptr(y); 131 | float* lGData = (float*)lGrdX.ptr(y); 132 | float* rGData = (float*)rGrdX.ptr(y); 133 | float* cost = (float*)costVol.ptr(y); 134 | 135 | for(int x = d; x < wid; ++x) { 136 | float* lC = lData + 3 * x; 137 | float* rC = rData + 3 * (x - d); 138 | float* lG = lGData + x; 139 | float* rG = rGData + x - d; 140 | cost[x] = myCostGrd(lC, rC, lG, rG); 141 | } 142 | for(int x = 0; x < d; ++x) { 143 | float* lC = lData + 3 * x; 144 | float* lG = lGData + x; 145 | cost[x] = myCostGrd(lC, lG); 146 | } 147 | } 148 | return 0; 149 | } 150 | 151 | int CVC::buildCV_right(const Mat& lImg, const Mat& rImg, const Mat& lGrdX, const Mat& rGrdX, const int d, Mat& costVol) 152 | { 153 | int wid = lImg.cols; 154 | int border = wid - d; 155 | int hei = lImg.rows; 156 | 157 | for(int y = 0; y < lImg.rows; ++y) 158 | { 159 | float* lData = (float*)lImg.ptr(y); 160 | float* rData = (float*)rImg.ptr(y); 161 | float* lGData = (float*)lGrdX.ptr(y); 162 | float* rGData = (float*)rGrdX.ptr(y); 163 | float* cost = (float*)costVol.ptr(y); 164 | 165 | for(int x = 0; x < border; ++x) { 166 | float* lC = lData + 3 * x; 167 | float* rC = rData + 3 * (x + d); 168 | float* lG = lGData + x; 169 | float* rG = rGData + x + d; 170 | cost[x] = myCostGrd(lC, rC, lG, rG); 171 | } 172 | for(int x = border; x < wid; ++x) { 173 | float* lC = lData + 3 * x; 174 | float* lG = lGData + x; 175 | cost[x] = myCostGrd(lC, lG); 176 | } 177 | } 178 | return 0; 179 | } 180 | -------------------------------------------------------------------------------- /src/CVC_cl.cpp: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | CVC_cl.cpp - OpenCL Cost Volume Construction Code 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | All rights reserved. 8 | ---------------------------------------------------------------------------*/ 9 | #include "CVC_cl.h" 10 | 11 | CVC_cl::CVC_cl(cl_context* context, cl_command_queue* commandQueue, cl_device_id device, 12 | Mat* I, const int d) : maxDis(d), 13 | context(context), commandQueue(commandQueue) 14 | { 15 | //printf("OpenCL Colours and Gradients method for Cost Computation\n"); 16 | 17 | //OpenCL Setup 18 | program = 0; 19 | kernel = 0; 20 | // imgType = I->type() & CV_MAT_DEPTH_MASK; 21 | 22 | if (!createProgram(*context, device, FILE_CVC_PROG, &program)) 23 | { 24 | cleanUpOpenCL(NULL, NULL, program, NULL, NULL, 0); 25 | std::cerr << "Failed to create OpenCL program." << __FILE__ << ":"<< __LINE__ << std::endl; 26 | } 27 | 28 | width = (cl_int)I->cols; 29 | height = (cl_int)I->rows; 30 | // channels = (cl_int)I->channels(); 31 | 32 | // if(imgType == CV_32F) 33 | // { 34 | strcpy(kernel_name, "cvc_float_nv"); 35 | //strcpy(kernel_name, "cvc_float_v4"); 36 | 37 | bufferSize_2D = width * height * sizeof(cl_float); 38 | bufferSize_3D = width * height * maxDis * sizeof(cl_float); 39 | 40 | //cvc_uchar_nv 41 | globalWorksize[0] = (size_t)width; 42 | //cvc_uchar_v4 43 | // globalWorksize[0] = (size_t)width/4; 44 | 45 | globalWorksize[1] = (size_t)height; 46 | globalWorksize[2] = (size_t)maxDis; 47 | // } 48 | // else if(imgType == CV_8U) 49 | // { 50 | // strcpy(kernel_name, "cvc_uchar_vx"); 51 | // //strcpy(kernel_name, "cvc_uchar_v16"); 52 | // //strcpy(kernel_name, "cvc_uchar_nv"); 53 | // 54 | // bufferSize_2D = width * height * sizeof(cl_uchar); 55 | // bufferSize_3D = width * height * maxDis * sizeof(cl_uchar); 56 | // 57 | // //cvc_uchar_vx 58 | // globalWorksize[0] = (size_t)height; 59 | // globalWorksize[1] = (size_t)1; 60 | // //cvc_uchar_v16 61 | //// globalWorksize[0] = (size_t)width/16; 62 | // //cvc_uchar_nv 63 | //// globalWorksize[0] = (size_t)width; 64 | //// globalWorksize[1] = (size_t)height; 65 | // 66 | // globalWorksize[2] = (size_t)maxDis; 67 | // 68 | // } 69 | // else{ 70 | // printf("CVC_cl: Error - Unrecognised data type in processing! (CVC_cl)\n"); 71 | // exit(1); 72 | // } 73 | kernel = clCreateKernel(program, kernel_name, &errorNumber); 74 | if (!checkSuccess(errorNumber)) 75 | { 76 | cleanUpOpenCL(NULL, NULL, NULL, NULL, NULL, 0); 77 | std::cerr << "Failed to create OpenCL kernel. " << __FILE__ << ":"<< __LINE__ << std::endl; 78 | exit(1); 79 | } 80 | else{ 81 | printf("CVC_cl: OpenCL kernels created.\n"); 82 | } 83 | 84 | /* An event to associate with the Kernel. Allows us to retreive profiling information later. */ 85 | event = 0; 86 | } 87 | CVC_cl::~CVC_cl(void) 88 | { 89 | /* Release OpenCL objects. */ 90 | cleanUpOpenCL(NULL, NULL, NULL, kernel, NULL, 0); 91 | } 92 | 93 | int CVC_cl::buildCV(const Mat& lImg, const Mat& rImg, cl_mem *memoryObjects) 94 | { 95 | lImgRGB = new Mat[lImg.channels()]; 96 | rImgRGB = new Mat[rImg.channels()]; 97 | split(lImg, lImgRGB); 98 | split(rImg, rImgRGB); 99 | 100 | cvtColor(lImg, lGray, CV_RGB2GRAY); 101 | cvtColor(rImg, rGray, CV_RGB2GRAY); 102 | 103 | /* Map the input memory objects to host side pointers. */ 104 | bool EnqueueMapBufferSuccess = true; 105 | // if(imgType == CV_32F) 106 | // { 107 | //Sobel filter to compute X gradient <-- investigate Mali Sobel OpenCL kernel 108 | Sobel( lGray, lGrdX, CV_32F, 1, 0, 1 ); // ex time 16 -17ms 109 | Sobel( rGray, rGrdX, CV_32F, 1, 0, 1 ); // for both 110 | lGrdX += 0.5; 111 | rGrdX += 0.5; 112 | 113 | cl_float *clbuffer_lImgRGB[3], *clbuffer_rImgRGB[3]; 114 | for (int i = 0; i < channels; i++) 115 | { 116 | clbuffer_lImgRGB[i] = (cl_float*)clEnqueueMapBuffer(*commandQueue, memoryObjects[i], CL_TRUE, CL_MAP_WRITE | CL_MAP_READ, 0, bufferSize_2D, 0, NULL, NULL, &errorNumber); 117 | EnqueueMapBufferSuccess &= checkSuccess(errorNumber); 118 | clbuffer_rImgRGB[i] = (cl_float*)clEnqueueMapBuffer(*commandQueue, memoryObjects[i+channels], CL_TRUE, CL_MAP_WRITE | CL_MAP_READ, 0, bufferSize_2D, 0, NULL, NULL, &errorNumber); 119 | EnqueueMapBufferSuccess &= checkSuccess(errorNumber); 120 | 121 | memcpy(clbuffer_lImgRGB[i], lImgRGB[i].data, bufferSize_2D); 122 | memcpy(clbuffer_rImgRGB[i], rImgRGB[i].data, bufferSize_2D); 123 | } 124 | // } 125 | // else if(imgType == CV_8U) 126 | // { 127 | // //Sobel filter to compute X gradient 128 | // Sobel( lGray, lGrdX, CV_8U, 1, 0, 1 ); 129 | // Sobel( rGray, rGrdX, CV_8U, 1, 0, 1 ); 130 | // lGrdX += 0.5; 131 | // rGrdX += 0.5; 132 | // 133 | // cl_uchar *clbuffer_lImgRGB[3], *clbuffer_rImgRGB[3]; 134 | // //Six 1-channel 2D buffers W*H 135 | // for (int i = 0; i < channels; i++) 136 | // { 137 | // clbuffer_lImgRGB[i] = (cl_uchar*)clEnqueueMapBuffer(*commandQueue, memoryObjects[i], CL_TRUE, CL_MAP_WRITE | CL_MAP_READ, 0, bufferSize_2D, 0, NULL, NULL, &errorNumber); 138 | // EnqueueMapBufferSuccess &= checkSuccess(errorNumber); 139 | // clbuffer_rImgRGB[i] = (cl_uchar*)clEnqueueMapBuffer(*commandQueue, memoryObjects[i+channels], CL_TRUE, CL_MAP_WRITE | CL_MAP_READ, 0, bufferSize_2D, 0, NULL, NULL, &errorNumber); 140 | // EnqueueMapBufferSuccess &= checkSuccess(errorNumber); 141 | // 142 | // memcpy(clbuffer_lImgRGB[i], lImgRGB[i].data, bufferSize_2D); 143 | // memcpy(clbuffer_rImgRGB[i], rImgRGB[i].data, bufferSize_2D); 144 | // } 145 | // } 146 | 147 | //Two 1-channel 2D buffers W*H 148 | cl_uchar *clbuffer_lGrdX = (cl_uchar*)clEnqueueMapBuffer(*commandQueue, memoryObjects[CVC_LGRDX], CL_TRUE, CL_MAP_WRITE | CL_MAP_READ, 0, bufferSize_2D, 0, NULL, NULL, &errorNumber); 149 | EnqueueMapBufferSuccess &= checkSuccess(errorNumber); 150 | cl_uchar *clbuffer_rGrdX = (cl_uchar*)clEnqueueMapBuffer(*commandQueue, memoryObjects[CVC_RGRDX], CL_TRUE, CL_MAP_WRITE | CL_MAP_READ, 0, bufferSize_2D, 0, NULL, NULL, &errorNumber); 151 | EnqueueMapBufferSuccess &= checkSuccess(errorNumber); 152 | if (!EnqueueMapBufferSuccess) 153 | { 154 | cleanUpOpenCL(NULL, NULL, program, NULL, NULL, 0); 155 | std::cerr << "Mapping memory objects failed " << __FILE__ << ":"<< __LINE__ << std::endl; 156 | } 157 | 158 | //printf("CVC_cl: Copying data to OpenCL memory space\n"); 159 | memcpy(clbuffer_lGrdX, lGrdX.data, bufferSize_2D); 160 | memcpy(clbuffer_rGrdX, rGrdX.data, bufferSize_2D); 161 | 162 | int arg_num = 0; 163 | /* Setup the kernel arguments. */ 164 | bool setKernelArgumentsSuccess = true; 165 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_mem), &memoryObjects[CVC_LIMGR])); 166 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_mem), &memoryObjects[CVC_LIMGG])); 167 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_mem), &memoryObjects[CVC_LIMGB])); 168 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_mem), &memoryObjects[CVC_RIMGR])); 169 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_mem), &memoryObjects[CVC_RIMGG])); 170 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_mem), &memoryObjects[CVC_RIMGB])); 171 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_mem), &memoryObjects[CVC_LGRDX])); 172 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_mem), &memoryObjects[CVC_RGRDX])); 173 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_int), &height)); 174 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_int), &width)); 175 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_mem), &memoryObjects[CV_LCV])); 176 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_mem), &memoryObjects[CV_RCV])); 177 | if (!setKernelArgumentsSuccess) 178 | { 179 | cleanUpOpenCL(NULL, NULL, NULL, NULL, NULL, 0); 180 | std::cerr << "Failed setting OpenCL kernel arguments. " << __FILE__ << ":"<< __LINE__ << std::endl; 181 | } 182 | 183 | if(OCL_STATS) printf("CVC_cl: Running CVC Kernels\n"); 184 | /* Enqueue the kernel */ 185 | if (!checkSuccess(clEnqueueNDRangeKernel(*commandQueue, kernel, 3, NULL, globalWorksize, NULL, 0, NULL, &event))) 186 | { 187 | cleanUpOpenCL(NULL, NULL, NULL, NULL, NULL, 0); 188 | std::cerr << "Failed enqueuing the kernel. " << __FILE__ << ":"<< __LINE__ << std::endl; 189 | return 1; 190 | } 191 | 192 | /* Wait for completion */ 193 | if (!checkSuccess(clFinish(*commandQueue))) 194 | { 195 | cleanUpOpenCL(NULL, NULL, NULL, NULL, NULL, 0); 196 | std::cerr << "Failed waiting for kernel execution to finish. " << __FILE__ << ":"<< __LINE__ << std::endl; 197 | return 1; 198 | } 199 | 200 | /* Print the profiling information for the event. */ 201 | if(OCL_STATS) printProfilingInfo(event); 202 | /* Release the event object. */ 203 | if (!checkSuccess(clReleaseEvent(event))) 204 | { 205 | cleanUpOpenCL(*context, *commandQueue, program, NULL, NULL, 0); 206 | std::cerr << "Failed releasing the event object. " << __FILE__ << ":"<< __LINE__ << std::endl; 207 | return 1; 208 | } 209 | 210 | return 0; 211 | } 212 | -------------------------------------------------------------------------------- /src/CVF.cpp: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | CVF.cpp - Cost Volume Filter Code 3 | - Guided Image Filter 4 | - Box Filter 5 | --------------------------------------------------------------------------- 6 | Co-Author: Charan Kumar 7 | Email: EE14MTECH01008@iith.ac.in 8 | Co-Author: Charles Leech 9 | Email: cl19g10 [at] ecs.soton.ac.uk 10 | Copyright (c) 2016 Charlie Leech, University of Southampton. 11 | ---------------------------------------------------------------------------*/ 12 | #include "CVF.h" 13 | 14 | CVF::CVF() 15 | { 16 | #ifdef DEBUG_APP 17 | std::cout << "Guided Image Filtering method for Cost Computation." << std::endl; 18 | #endif // DEBUG_APP 19 | } 20 | CVF::~CVF(){} 21 | 22 | int CVF::filterCV(const Mat* Img_rgb, const Mat* mean_Img, const Mat* var_Img, Mat& costVol) 23 | { 24 | costVol = GuidedFilter_cv(Img_rgb, mean_Img, var_Img, costVol); 25 | return 0; 26 | } 27 | 28 | void *CVF::filterCV_thread(void *thread_arg) 29 | { 30 | struct filterCV_TD *t_data; 31 | t_data = (struct filterCV_TD *) thread_arg; 32 | const Mat* Img_rgb = t_data->Img_rgb; 33 | const Mat* mean_Img = t_data->mean_Img; 34 | const Mat* var_Img = t_data->var_Img; 35 | const Mat pImg = *t_data->costVol; 36 | Mat* costVol = t_data->costVol; 37 | 38 | *costVol = GuidedFilter_cv(Img_rgb, mean_Img, var_Img, pImg); 39 | 40 | return (void*)0; 41 | } 42 | 43 | //Image channel division, boxfiltering and variance calculation can all be computed once for all disparities 44 | int CVF::preprocess(const Mat& Img, Mat* Img_rgb, Mat* mean_Img, Mat* var_Img) 45 | { 46 | Size r = Size(GIF_R_WIN,GIF_R_WIN); 47 | split( Img, Img_rgb ); 48 | 49 | for( int c = 0; c < 3; c ++ ) { 50 | boxFilter( Img_rgb[c], mean_Img[c], -1, r ); 51 | } 52 | 53 | // % variance of I in each local patch: the matrix Sigma in Eqn (14). 54 | // % Note the variance in each local patch is a 3x3 symmetric matrix: 55 | // % rr, rg, rb 56 | // % Sigma = rg, gg, gb 57 | // % rb, gb, bb 58 | Mat tmp; 59 | int varIdx = 0; 60 | for( int c = 0; c < 3; ++c) { 61 | for( int c_p = c; c_p < 3; c_p ++ ) { 62 | multiply( Img_rgb[ c ], Img_rgb[ c_p ], tmp ); 63 | boxFilter( tmp, var_Img[varIdx], -1, r ); 64 | multiply( mean_Img[ c ], mean_Img[ c_p ], tmp ); 65 | var_Img[ varIdx ] -= tmp; 66 | ++varIdx; 67 | } 68 | } 69 | return 0; 70 | } 71 | 72 | Mat GuidedFilter_cv(const Mat* rgb, const Mat* mean_I, const Mat* var_I, const Mat& p) 73 | { 74 | Size r = Size(GIF_R_WIN,GIF_R_WIN); 75 | 76 | int H = rgb[0].rows; 77 | int W = rgb[0].cols; 78 | // color guidence 79 | // image must in RGB format 80 | 81 | Mat mean_p; 82 | boxFilter( p, mean_p, -1, r ); 83 | 84 | Mat tmp; 85 | Mat mean_Ip[ 3 ]; 86 | for( int c = 0; c < 3; ++c) { 87 | multiply( rgb[ c ], p, tmp ); 88 | boxFilter( tmp, mean_Ip[c], -1, r ); 89 | } 90 | /*% covariance of (I, p) in each local patch.*/ 91 | Mat cov_Ip[ 3 ]; 92 | for( int c = 0; c < 3; ++c) { 93 | multiply( mean_I[ c ], mean_p, tmp ); 94 | cov_Ip[ c ] = mean_Ip[ c ] - tmp; 95 | } 96 | 97 | Mat a[ 3 ]; 98 | for(int c = 0; c < 3; ++c) 99 | { 100 | a[ c ] = Mat::zeros( H, W, CV_32FC1 ); 101 | } 102 | for(int y = 0; y < H; ++y) { 103 | float* vData[ 6 ]; 104 | for(int v = 0; v < 6; ++v) { 105 | vData[ v ] = ( float* ) var_I[ v ].ptr( y ); 106 | } 107 | float* cData[ 3 ]; 108 | for(int c = 0; c < 3; ++c) { 109 | cData[ c ] = ( float * ) cov_Ip[ c ].ptr( y ); 110 | } 111 | float* aData[ 3 ]; 112 | for(int c = 0; c < 3; ++c) { 113 | aData[ c ] = ( float* ) a[ c ].ptr( y ); 114 | } 115 | for(int x = 0; x < W; ++x) 116 | { 117 | float c0 = cData[ 0 ][ x ]; 118 | float c1 = cData[ 1 ][ x ]; 119 | float c2 = cData[ 2 ][ x ]; 120 | float a11 = vData[ 0 ][ x ] + GIF_EPS; 121 | float a12 = vData[ 1 ][ x ]; 122 | float a13 = vData[ 2 ][ x ]; 123 | float a21 = vData[ 1 ][ x ]; 124 | float a22 = vData[ 3 ][ x ] + GIF_EPS; 125 | float a23 = vData[ 4 ][ x ]; 126 | float a31 = vData[ 2 ][ x ]; 127 | float a32 = vData[ 4 ][ x ]; 128 | float a33 = vData[ 5 ][ x ] + GIF_EPS; 129 | float DET = a11 * ( a33 * a22 - a32 * a23 ) - 130 | a21 * ( a33 * a12 - a32 * a13 ) + 131 | a31 * ( a23 * a12 - a22 * a13 ); 132 | DET = 1 / DET; 133 | aData[ 0 ][ x ] = DET * ( 134 | c0 * ( a33 * a22 - a32 * a23 ) + 135 | c1 * ( a31 * a23 - a33 * a21 ) + 136 | c2 * ( a32 * a21 - a31 * a22 ) 137 | ); 138 | aData[ 1 ][ x ] = DET * ( 139 | c0 * ( a32 * a13 - a33 * a12 ) + 140 | c1 * ( a33 * a11 - a31 * a13 ) + 141 | c2 * ( a31 * a12 - a32 * a11 ) 142 | ); 143 | aData[ 2 ][ x ] = DET * ( 144 | c0 * ( a23 * a12 - a22 * a13 ) + 145 | c1 * ( a21 * a13 - a23 * a11 ) + 146 | c2 * ( a22 * a11 - a21 * a12 ) 147 | ); 148 | } 149 | } 150 | 151 | //Mat b = mean_p.clone(); 152 | for( int c = 0; c < 3; ++c) { 153 | multiply( a[ c ], mean_I[ c ], tmp ); 154 | mean_p -= tmp; 155 | } 156 | 157 | Mat q; 158 | boxFilter( mean_p, q, -1, r ); 159 | for( int c = 0; c < 3; ++c) { 160 | boxFilter( a[c], tmp, -1, r ); 161 | multiply( tmp, rgb[ c ], tmp ); 162 | q += tmp; 163 | } 164 | return q; 165 | } 166 | -------------------------------------------------------------------------------- /src/DispEst.cpp: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | DispEst.cpp - Disparity Estimation Class 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | ---------------------------------------------------------------------------*/ 8 | #include "DispEst.h" 9 | 10 | DispEst::DispEst(cv::Mat l, cv::Mat r, const int d, int t, bool ocl) 11 | : lImg(l), rImg(r), maxDis(d), threads(t), useOCL(ocl) 12 | { 13 | #ifdef DEBUG_APP 14 | std::cout << "Disparity Estimation for Depth Analysis in Stereo Vision Applications." << std::endl; 15 | #endif // DEBUG_APP 16 | 17 | hei = lImg.rows; 18 | wid = lImg.cols; 19 | 20 | //Global Image Type Checking 21 | if(lImg.type() == rImg.type()) 22 | { 23 | #ifdef DEBUG_APP 24 | printf("Data type = %d, CV_32F = %d, CV_8U = %d\n", (lImg.type() & CV_MAT_DEPTH_MASK), CV_32F, CV_8U); 25 | #endif // DEBUG_APP 26 | } else { 27 | printf("DE: Error - Left & Right images are of different types.\n"); 28 | exit(1); 29 | } 30 | 31 | lcostVol = new cv::Mat[maxDis]; 32 | rcostVol = new cv::Mat[maxDis]; 33 | for (int i = 0; i < maxDis; ++i) 34 | { 35 | lcostVol[i] = cv::Mat::zeros(hei, wid, CV_32FC1); 36 | rcostVol[i] = cv::Mat::zeros(hei, wid, CV_32FC1); 37 | } 38 | 39 | // lImg_rgb = new Mat[3]; 40 | // rImg_rgb = new Mat[3]; 41 | // mean_lImg = new Mat[3]; 42 | // mean_rImg = new Mat[3]; 43 | // var_lImg = new Mat[6]; 44 | // var_rImg = new Mat[6]; 45 | 46 | lDisMap = cv::Mat::zeros(hei, wid, CV_8UC1); 47 | rDisMap = cv::Mat::zeros(hei, wid, CV_8UC1); 48 | lValid = cv::Mat::zeros(hei, wid, CV_8UC1); 49 | rValid = cv::Mat::zeros(hei, wid, CV_8UC1); 50 | 51 | printf("Setting up pthreads function constructors\n"); 52 | constructor = new CVC(); 53 | filter = new CVF(); 54 | selector = new DispSel(); 55 | postProcessor = new PP(); 56 | 57 | if(useOCL) 58 | { 59 | printf("Setting up OpenCL Environment\n"); 60 | //OpenCL Setup 61 | context = 0; 62 | commandQueue = 0; 63 | device = 0; 64 | numberOfMemoryObjects = 12; 65 | for(int m = 0; m < (int)numberOfMemoryObjects; m++) 66 | memoryObjects[m] = 0; 67 | 68 | //cl_int numComputeUnits = 2; 69 | if (!createContext(&context)) 70 | //if (!createSubDeviceContext(&context, numComputeUnits)) //Device Fission is not supported on Xeon Phi 71 | { 72 | cleanUpOpenCL(context, commandQueue, NULL, NULL, memoryObjects, numberOfMemoryObjects); 73 | std::cerr << "Failed to create an OpenCL context. " << __FILE__ << ":"<< __LINE__ << std::endl; 74 | } 75 | if (!createCommandQueue(context, &commandQueue, &device)) 76 | { 77 | cleanUpOpenCL(context, commandQueue, NULL, NULL, memoryObjects, numberOfMemoryObjects); 78 | std::cerr << "Failed to create the OpenCL command queue. " << __FILE__ << ":"<< __LINE__ << std::endl; 79 | } 80 | 81 | width = (cl_int)wid; 82 | height = (cl_int)hei; 83 | channels = (cl_int)lImg.channels(); 84 | 85 | //OpenCL Buffers that are type dependent (in accending size order) 86 | // if(imgType == CV_32F) 87 | // { 88 | bufferSize_2D = width * height * sizeof(cl_float); 89 | bufferSize_3D = width * height * maxDis * sizeof(cl_float); 90 | // } 91 | // else if(imgType == CV_8U) 92 | // { 93 | // bufferSize_2D = width * height * sizeof(cl_uchar); 94 | // bufferSize_3D = width * height * maxDis * sizeof(cl_uchar); 95 | // } 96 | //OpenCL Buffers that are always required 97 | bufferSize_2D_8UC1 = width * height * sizeof(cl_uchar); 98 | 99 | /* Create buffers for the left and right images, gradient data, cost volume, and disparity maps. */ 100 | bool createMemoryObjectsSuccess = true; 101 | memoryObjects[CVC_LIMGR] = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, bufferSize_2D, NULL, &errorNumber); 102 | createMemoryObjectsSuccess &= checkSuccess(errorNumber); 103 | memoryObjects[CVC_LIMGG] = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, bufferSize_2D, NULL, &errorNumber); 104 | createMemoryObjectsSuccess &= checkSuccess(errorNumber); 105 | memoryObjects[CVC_LIMGB] = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, bufferSize_2D, NULL, &errorNumber); 106 | createMemoryObjectsSuccess &= checkSuccess(errorNumber); 107 | 108 | memoryObjects[CVC_RIMGR] = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, bufferSize_2D, NULL, &errorNumber); 109 | createMemoryObjectsSuccess &= checkSuccess(errorNumber); 110 | memoryObjects[CVC_RIMGG] = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, bufferSize_2D, NULL, &errorNumber); 111 | createMemoryObjectsSuccess &= checkSuccess(errorNumber); 112 | memoryObjects[CVC_RIMGB] = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, bufferSize_2D, NULL, &errorNumber); 113 | createMemoryObjectsSuccess &= checkSuccess(errorNumber); 114 | 115 | memoryObjects[CVC_LGRDX] = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, bufferSize_2D, NULL, &errorNumber); 116 | createMemoryObjectsSuccess &= checkSuccess(errorNumber); 117 | memoryObjects[CVC_RGRDX] = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, bufferSize_2D, NULL, &errorNumber); 118 | createMemoryObjectsSuccess &= checkSuccess(errorNumber); 119 | 120 | memoryObjects[CV_LCV] = clCreateBuffer(context, CL_MEM_READ_WRITE, bufferSize_3D, NULL, &errorNumber); 121 | createMemoryObjectsSuccess &= checkSuccess(errorNumber); 122 | memoryObjects[CV_RCV] = clCreateBuffer(context, CL_MEM_READ_WRITE, bufferSize_3D, NULL, &errorNumber); 123 | createMemoryObjectsSuccess &= checkSuccess(errorNumber); 124 | 125 | memoryObjects[DS_LDM] = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, bufferSize_2D_8UC1, NULL, &errorNumber); 126 | createMemoryObjectsSuccess &= checkSuccess(errorNumber); 127 | memoryObjects[DS_RDM] = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, bufferSize_2D_8UC1, NULL, &errorNumber); 128 | createMemoryObjectsSuccess &= checkSuccess(errorNumber); 129 | if (!createMemoryObjectsSuccess) 130 | { 131 | cleanUpOpenCL(context, commandQueue, NULL, NULL, memoryObjects, numberOfMemoryObjects); 132 | std::cerr << "Failed to create OpenCL buffers. " << __FILE__ << ":"<< __LINE__ << std::endl; 133 | } 134 | 135 | printf("Setting up OpenCL function constructors\n"); 136 | //OpenCL function constructors 137 | constructor_cl = new CVC_cl(&context, &commandQueue, device, &lImg, maxDis); 138 | filter_cl = new CVF_cl(&context, &commandQueue, device, &lImg, maxDis); 139 | selector_cl = new DispSel_cl(&context, &commandQueue, device, &lImg, maxDis); 140 | } 141 | 142 | printf("Construction Complete\n"); 143 | } 144 | 145 | DispEst::~DispEst(void) 146 | { 147 | delete [] lcostVol; 148 | delete [] rcostVol; 149 | delete constructor; 150 | delete filter; 151 | delete selector; 152 | delete postProcessor; 153 | 154 | if(useOCL){ 155 | delete constructor_cl; 156 | delete filter_cl; 157 | delete selector_cl; 158 | 159 | for(int m = 0; m < (int)numberOfMemoryObjects; ++m) 160 | clReleaseMemObject(memoryObjects[m]); 161 | } 162 | } 163 | 164 | int DispEst::setInputImages(cv::Mat leftImg, cv::Mat rightImg) 165 | { 166 | assert(leftImg.type() == rightImg.type()); 167 | lImg = leftImg; 168 | rImg = rightImg; 169 | return 0; 170 | } 171 | 172 | int DispEst::setThreads(unsigned int newThreads) 173 | { 174 | if(newThreads > MAX_CPU_THREADS) 175 | return -1; 176 | 177 | threads = newThreads; 178 | return 0; 179 | } 180 | 181 | int DispEst::printCV(void) 182 | { 183 | char filename[20]; 184 | int ret_val = 0; 185 | 186 | for (int i = 0; i < maxDis; ++i) 187 | { 188 | if(ret_val = sprintf(filename, "CV/lCV%d.png", i)) return ret_val; 189 | imwrite(filename, lcostVol[i]*1024*8); 190 | if(ret_val = sprintf(filename, "CV/rCV%d.png", i)) return ret_val; 191 | imwrite(filename, rcostVol[i]*1024*8); 192 | } 193 | return 0; 194 | } 195 | 196 | //############################################################################################################# 197 | //# Cost Volume Construction 198 | //############################################################################################################# 199 | int DispEst::CostConst() 200 | { 201 | int ret_val = 0; 202 | 203 | if(ret_val = constructor->preprocess(lImg, lGrdX)) 204 | return ret_val; 205 | if(ret_val = constructor->preprocess(rImg, rGrdX)) 206 | return ret_val; 207 | 208 | // Build Cost Volume 209 | #pragma omp parallel for 210 | for( int d = 0; d < maxDis; ++d) 211 | { 212 | constructor->buildCV_left(lImg, rImg, lGrdX, rGrdX, d, lcostVol[d]); 213 | } 214 | #pragma omp parallel for 215 | for( int d = 0; d < maxDis; ++d) 216 | { 217 | constructor->buildCV_right(rImg, lImg, rGrdX, lGrdX, d, rcostVol[d]); 218 | } 219 | return 0; 220 | } 221 | 222 | int DispEst::CostConst_CPU() 223 | { 224 | //Set up threads and thread attributes 225 | void *status; 226 | pthread_attr_t attr; 227 | pthread_attr_init(&attr); 228 | pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); 229 | pthread_t BCV_threads[maxDis]; 230 | buildCV_TD buildCV_TD_Array[maxDis]; 231 | 232 | constructor->preprocess(lImg, lGrdX); 233 | constructor->preprocess(rImg, rGrdX); 234 | 235 | for(int level = 0; level <= maxDis/threads; ++level) 236 | { 237 | //Handle remainder if threads is not power of 2. 238 | int block_size = (level < maxDis/threads) ? threads : (maxDis%threads); 239 | 240 | for(int iter=0; iter < block_size; ++iter) 241 | { 242 | int d = level*threads + iter; 243 | buildCV_TD_Array[d] = {&lImg, &rImg, &lGrdX, &rGrdX, d, &lcostVol[d]}; 244 | pthread_create(&BCV_threads[d], &attr, CVC::buildCV_left_thread, (void *)&buildCV_TD_Array[d]); 245 | } 246 | for(int iter=0; iter < block_size; ++iter) 247 | { 248 | int d = level*threads + iter; 249 | pthread_join(BCV_threads[d], &status); 250 | } 251 | } 252 | for(int level = 0; level <= maxDis/threads; ++level) 253 | { 254 | //Handle remainder if threads is not power of 2. 255 | int block_size = (level < maxDis/threads) ? threads : (maxDis%threads); 256 | 257 | for(int iter=0; iter < block_size; ++iter) 258 | { 259 | int d = level*threads + iter; 260 | buildCV_TD_Array[d] = {&rImg, &lImg, &rGrdX, &lGrdX, d, &rcostVol[d]}; 261 | pthread_create(&BCV_threads[d], &attr, CVC::buildCV_right_thread, (void *)&buildCV_TD_Array[d]); 262 | } 263 | for(int iter=0; iter < block_size; ++iter) 264 | { 265 | int d = level*threads + iter; 266 | pthread_join(BCV_threads[d], &status); 267 | } 268 | } 269 | return 0; 270 | } 271 | 272 | int DispEst::CostConst_GPU() 273 | { 274 | constructor_cl->buildCV(lImg, rImg, memoryObjects); 275 | return 0; 276 | } 277 | 278 | //############################################################################################################# 279 | //# Cost Volume Filtering 280 | //############################################################################################################# 281 | int DispEst::CostFilter_FGF() 282 | { 283 | FastGuidedFilter fgf_left(lImg, GIF_R_WIN, GIF_EPS, subsample_rate); 284 | FastGuidedFilter fgf_right(rImg, GIF_R_WIN, GIF_EPS, subsample_rate); 285 | 286 | #pragma omp parallel for 287 | for(int d = 0; d < maxDis; ++d){ 288 | lcostVol[d] = fgf_left.filter(lcostVol[d]); 289 | } 290 | 291 | #pragma omp parallel for 292 | for(int d = 0; d < maxDis; ++d){ 293 | rcostVol[d] = fgf_right.filter(rcostVol[d]); 294 | } 295 | return 0; 296 | } 297 | 298 | //TODO: Port FGF code to GPU 299 | int DispEst::CostFilter_GPU() 300 | { 301 | //printf("OpenCL Cost Filtering Underway...\n"); 302 | filter_cl->preprocess(&memoryObjects[CVC_LIMGR], &memoryObjects[CVC_LIMGG], &memoryObjects[CVC_LIMGB]); 303 | filter_cl->filterCV(&memoryObjects[CV_LCV]); 304 | filter_cl->preprocess(&memoryObjects[CVC_RIMGR], &memoryObjects[CVC_RIMGG], &memoryObjects[CVC_RIMGB]); 305 | filter_cl->filterCV(&memoryObjects[CV_RCV]); 306 | //printf("Filtering Complete\n"); 307 | return 0; 308 | } 309 | 310 | 311 | int DispEst::DispSelect_CPU() 312 | { 313 | //printf("Left Selection...\n"); 314 | selector->CVSelect(lcostVol, maxDis, lDisMap); 315 | //selector->CVSelect_thread(lcostVol, maxDis, lDisMap, threads); 316 | 317 | //printf("Right Selection...\n"); 318 | selector->CVSelect(rcostVol, maxDis, rDisMap); 319 | //selector->CVSelect_thread(rcostVol, maxDis, rDisMap, threads); 320 | return 0; 321 | } 322 | 323 | int DispEst::DispSelect_GPU() 324 | { 325 | //printf("Left & Right Selection...\n"); 326 | selector_cl->CVSelect(memoryObjects, lDisMap, rDisMap); 327 | return 0; 328 | } 329 | 330 | int DispEst::PostProcess_CPU() 331 | { 332 | //printf("Post Processing Underway...\n"); 333 | postProcessor->processDM(lImg, rImg, lDisMap, rDisMap, lValid, rValid, maxDis, threads); 334 | //printf("Post Processing Complete\n"); 335 | return 0; 336 | } 337 | 338 | int DispEst::PostProcess_GPU() 339 | { 340 | //printf("Post Processing Underway...\n"); 341 | postProcessor->processDM(lImg, rImg, lDisMap, rDisMap, lValid, rValid, maxDis, threads); 342 | //printf("Post Processing Complete\n"); 343 | return 0; 344 | } 345 | -------------------------------------------------------------------------------- /src/DispSel.cpp: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | DispSel.cpp - Disparity Selection Code 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | ---------------------------------------------------------------------------*/ 8 | #include "DispSel.h" 9 | 10 | DispSel::DispSel() 11 | { 12 | #ifdef DEBUG_APP 13 | std::cout << "Winner-Takes-All Disparity Selection." << std::endl; 14 | #endif // DEBUG_APP 15 | } 16 | DispSel::~DispSel() {} 17 | 18 | void *DS_X(void *thread_arg) 19 | { 20 | struct DS_X_TD *t_data; 21 | t_data = (struct DS_X_TD *) thread_arg; 22 | //Matricies 23 | cv::Mat* costVol = t_data->costVol; 24 | cv::Mat* dispMap = t_data->dispMap; 25 | //Variables 26 | int y = t_data->y; 27 | int maxDis = t_data->maxDis; 28 | 29 | int wid = dispMap->cols; 30 | unsigned char* dispData = (unsigned char*) dispMap->ptr(y); 31 | 32 | for(int x = 0; x < wid; ++x) 33 | { 34 | float minCost = DBL_MAX; 35 | int minDis = 0; 36 | 37 | for(int d = 1; d < maxDis; ++d) 38 | { 39 | float* costData = (float*)costVol[d].ptr(y); 40 | if(costData[x] < minCost) 41 | { 42 | minCost = costData[x]; 43 | minDis = d; 44 | } 45 | } 46 | dispData[x] = minDis; 47 | } 48 | return (void*)0; 49 | } 50 | 51 | int DispSel::CVSelect_thread(cv::Mat* costVol, const unsigned int maxDis, cv::Mat& dispMap, int threads) 52 | { 53 | unsigned int hei = dispMap.rows; 54 | 55 | //Set up threads for x-loop 56 | void* status; 57 | pthread_attr_t attr; 58 | pthread_attr_init(&attr); 59 | pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); 60 | pthread_t DS_X_threads[hei]; 61 | DS_X_TD DS_X_TD_Array[hei]; 62 | 63 | for(int level = 0; level <= hei/threads; ++level) 64 | { 65 | //Handle remainder if threads is not power of 2. 66 | int block_size = (level < hei/threads) ? threads : (hei%threads); 67 | 68 | for(int iter=0; iter < block_size; ++iter) 69 | { 70 | int d = level*threads + iter; 71 | DS_X_TD_Array[d] = {costVol, &dispMap, d, maxDis}; 72 | pthread_create(&DS_X_threads[d], &attr, DS_X, (void *)&DS_X_TD_Array[d]); 73 | } 74 | for(int iter=0; iter < block_size; ++iter) 75 | { 76 | int d = level*threads + iter; 77 | pthread_join(DS_X_threads[d], &status); 78 | } 79 | } 80 | return 0; 81 | } 82 | 83 | int DispSel::CVSelect(cv::Mat* costVol, const unsigned int maxDis, cv::Mat& dispMap) 84 | { 85 | unsigned int hei = dispMap.rows; 86 | unsigned int wid = dispMap.cols; 87 | 88 | #pragma omp parallel for 89 | for(unsigned int y = 0; y < hei; ++y) 90 | { 91 | for(unsigned int x = 0; x < wid; ++x) 92 | { 93 | float minCost = DBL_MAX; 94 | int minDis = 0; 95 | 96 | for(unsigned int d = 1; d < maxDis; ++d) 97 | { 98 | float* costData = (float*)costVol[d].ptr(y); 99 | if(costData[x] < minCost) 100 | { 101 | minCost = costData[x]; 102 | minDis = d; 103 | } 104 | } 105 | dispMap.at(y,x) = minDis; 106 | } 107 | } 108 | return 0; 109 | } 110 | -------------------------------------------------------------------------------- /src/DispSel_cl.cpp: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | DispSel_cl.cpp - OpenCL Disparity Selection Code 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | All rights reserved. 8 | ---------------------------------------------------------------------------*/ 9 | #include "DispSel_cl.h" 10 | 11 | DispSel_cl::DispSel_cl(cl_context* context, cl_command_queue* commandQueue, cl_device_id device, 12 | Mat* I, const int d) : maxDis(d), context(context), commandQueue(commandQueue) 13 | { 14 | //fprintf(stderr, "Winner-Takes-All Disparity Selection\n" ); 15 | 16 | //OpenCL Setup 17 | program = 0; 18 | // imgType = I->type() & CV_MAT_DEPTH_MASK; 19 | 20 | if (!createProgram(*context, device, FILE_DS_PROG, &program)) 21 | { 22 | cleanUpOpenCL(NULL, NULL, NULL, kernel, NULL, 0); 23 | std::cerr << "Failed to create OpenCL program." << __FILE__ << ":"<< __LINE__ << std::endl; 24 | } 25 | 26 | // if(imgType == CV_32F) 27 | // { 28 | strcpy(kernel_name, "dispsel_float"); 29 | //strcpy(kernel_name, "dispsel_double"); 30 | // } 31 | // else if(imgType == CV_8U) 32 | // { 33 | // strcpy(kernel_name, "dispsel_uchar"); 34 | // } 35 | // else{ 36 | // printf("DS_cl: Error - Unrecognised data type in processing! (DS_cl)\n"); 37 | // exit(1); 38 | // } 39 | kernel = clCreateKernel(program, kernel_name, &errorNumber); 40 | if (!checkSuccess(errorNumber)) 41 | { 42 | cleanUpOpenCL(NULL, NULL, NULL, kernel, NULL, 0); 43 | std::cerr << "Failed to create OpenCL kernel. " << __FILE__ << ":"<< __LINE__ << std::endl; 44 | } 45 | else{ 46 | printf("DispSel_cl: OpenCL kernels created.\n"); 47 | } 48 | 49 | width = (cl_int)I->cols; 50 | height = (cl_int)I->rows; 51 | 52 | //OpenCL Buffers in accending size order 53 | //bufferSize_2D_8UC1 = width * height * sizeof(cl_float); 54 | bufferSize_2D_8UC1 = width * height * sizeof(cl_char); 55 | 56 | /* An event to associate with the Kernel. Allows us to retreive profiling information later. */ 57 | event = 0; 58 | 59 | //Kernel size 60 | globalWorksize[0] = (size_t)width; 61 | globalWorksize[1] = (size_t)height; 62 | } 63 | DispSel_cl::~DispSel_cl(void) 64 | { 65 | /* Release OpenCL objects. */ 66 | cleanUpOpenCL(NULL, NULL, NULL, kernel, NULL, 0); 67 | } 68 | 69 | int DispSel_cl::CVSelect(cl_mem *memoryObjects, Mat& ldispMap, Mat& rdispMap) 70 | { 71 | 72 | cv::namedWindow("dispPreview", CV_WINDOW_AUTOSIZE); 73 | cv::imshow("dispPreview", ldispMap); 74 | cv::waitKey(0); 75 | 76 | int arg_num = 0; 77 | /* Setup the kernel arguments. */ 78 | bool setKernelArgumentsSuccess = true; 79 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_mem), &memoryObjects[CV_LCV])); 80 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_mem), &memoryObjects[CV_RCV])); 81 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_int), &height)); 82 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_int), &width)); 83 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_int), &maxDis)); 84 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_mem), &memoryObjects[DS_LDM])); 85 | setKernelArgumentsSuccess &= checkSuccess(clSetKernelArg(kernel, arg_num++, sizeof(cl_mem), &memoryObjects[DS_RDM])); 86 | if (!setKernelArgumentsSuccess) 87 | { 88 | cleanUpOpenCL(NULL, NULL, NULL, kernel, NULL, 0); 89 | std::cerr << "Failed setting OpenCL kernel arguments. " << __FILE__ << ":"<< __LINE__ << std::endl; 90 | } 91 | 92 | if(OCL_STATS) printf("DS_cl: Running DispSel Kernels\n"); 93 | /* Enqueue the kernel */ 94 | if (!checkSuccess(clEnqueueNDRangeKernel(*commandQueue, kernel, 2, NULL, globalWorksize, NULL, 0, NULL, &event))) 95 | { 96 | cleanUpOpenCL(NULL, NULL, NULL, kernel, NULL, 0); 97 | std::cerr << "Failed enqueuing the kernel. " << __FILE__ << ":"<< __LINE__ << std::endl; 98 | return 1; 99 | } 100 | 101 | /* Wait for completion */ 102 | if (!checkSuccess(clFinish(*commandQueue))) 103 | { 104 | cleanUpOpenCL(NULL, NULL, NULL, kernel, NULL, 0); 105 | std::cerr << "Failed waiting for kernel execution to finish. " << __FILE__ << ":"<< __LINE__ << std::endl; 106 | return 1; 107 | } 108 | 109 | /* Print the profiling information for the event. */ 110 | if(OCL_STATS) printProfilingInfo(event); 111 | /* Release the event object. */ 112 | if (!checkSuccess(clReleaseEvent(event))) 113 | { 114 | cleanUpOpenCL(*context, *commandQueue, program, kernel, NULL, 0); 115 | std::cerr << "Failed releasing the event object. " << __FILE__ << ":"<< __LINE__ << std::endl; 116 | return 1; 117 | } 118 | imshow("dispPreview", ldispMap); 119 | cv::waitKey(0); 120 | 121 | /* Map the output memory objects to host side pointers. */ 122 | bool EnqueueMapBufferSuccess = true; 123 | cl_char *clbuffer_lDispMap = (cl_char*)clEnqueueMapBuffer(*commandQueue, memoryObjects[DS_LDM], CL_TRUE, CL_MAP_READ, 0, bufferSize_2D_8UC1, 0, NULL, NULL, &errorNumber); 124 | EnqueueMapBufferSuccess &= checkSuccess(errorNumber); 125 | cl_char *clbuffer_rDispMap = (cl_char*)clEnqueueMapBuffer(*commandQueue, memoryObjects[DS_RDM], CL_TRUE, CL_MAP_READ, 0, bufferSize_2D_8UC1, 0, NULL, NULL, &errorNumber); 126 | EnqueueMapBufferSuccess &= checkSuccess(errorNumber); 127 | if (!EnqueueMapBufferSuccess) 128 | { 129 | cleanUpOpenCL(*context, *commandQueue, program, kernel, NULL, 0); 130 | std::cerr << "Mapping memory objects failed " << __FILE__ << ":"<< __LINE__ << std::endl; 131 | } 132 | 133 | memcpy(ldispMap.data, clbuffer_lDispMap, bufferSize_2D_8UC1); 134 | memcpy(rdispMap.data, clbuffer_rDispMap, bufferSize_2D_8UC1); 135 | 136 | imshow("dispPreview", ldispMap); 137 | cv::waitKey(0); 138 | 139 | // return 0; 140 | } 141 | -------------------------------------------------------------------------------- /src/PP.cpp: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | PP.cpp - Post Processing Code 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | All rights reserved. 8 | ---------------------------------------------------------------------------*/ 9 | #include "PP.h" 10 | 11 | PP::PP(void) 12 | { 13 | //printf( "L-R Consistency Check and Weighted-Median Filter Post-Processing\n" ); 14 | } 15 | PP::~PP(void) {} 16 | 17 | void lrCheck(Mat& lDis, Mat& rDis, Mat& lValid, Mat& rValid) 18 | { 19 | int hei = lDis.rows; 20 | int wid = lDis.cols; 21 | lValid = Scalar(0); //memset( lValid, 0, imgSize * sizeof( int ) ); 22 | rValid = Scalar(0); //memset( rValid, 0, imgSize * sizeof( int ) ); 23 | for( int y = 0; y < hei; y ++ ) { 24 | uchar* lDisData = ( uchar* ) lDis.ptr( y ); 25 | uchar* rDisData = ( uchar* ) rDis.ptr( y ); 26 | uchar* lValidData = ( uchar* ) lValid.ptr( y ); 27 | uchar* rValidData = ( uchar* ) rValid.ptr( y ); 28 | for( int x = 0; x < wid; x ++ ) { 29 | // check left image 30 | int lDep = lDisData[ x ]; 31 | // assert( ( x - lDep ) >= 0 && ( x - lDep ) < wid ); 32 | int rLoc = ( x - lDep + wid ) % wid; 33 | int rDep = rDisData[ rLoc ]; 34 | // disparity should not be zero 35 | if( lDep == rDep && lDep >= 2 ) { 36 | lValidData[x] = 1; 37 | } 38 | // check right image 39 | rDep = rDisData[ x ]; 40 | // assert( ( x + rDep ) >= 0 && ( x + rDep ) < wid ); 41 | int lLoc = ( x + rDep + wid ) % wid; 42 | lDep = lDisData[ lLoc ]; 43 | // disparity should not be zero 44 | if( rDep == lDep && rDep >= 2 ) { 45 | rValidData[x] = 1; 46 | } 47 | } 48 | } 49 | return; 50 | } 51 | 52 | void fillInv(Mat& lDis, Mat& rDis, Mat& lValid, Mat& rValid) 53 | { 54 | int hei = lDis.rows; 55 | int wid = lDis.cols; 56 | 57 | // fill left dep 58 | #pragma omp parallel for 59 | for( int y = 0; y < hei; y ++ ) { 60 | uchar* lDisData = ( uchar* ) lDis.ptr( y ); 61 | uchar* lValidData = ( uchar* ) lValid.ptr( y ); 62 | for( int x = 0; x < wid; x ++ ) { 63 | if( lValidData[x] == 0 ) { 64 | // find left first valid pixel 65 | int lFirst = x; 66 | int lFind = 0; 67 | while( lFirst >= 0 ) { 68 | if( lValidData[ lFirst ] ) { 69 | lFind = 1; 70 | break; 71 | } 72 | lFirst --; 73 | } 74 | int rFind = 0; 75 | // find right first valid pixel 76 | int rFirst = x; 77 | while( rFirst < wid ) { 78 | if( lValidData[ rFirst ] ) { 79 | rFind = 1; 80 | break; 81 | } 82 | rFirst ++; 83 | } 84 | // set x's depth to the lowest one 85 | if( lFind && rFind ) { 86 | if( lDisData[ lFirst ] <= lDisData[ rFirst ] ) { 87 | lDisData[ x ] = lDisData[ lFirst ]; 88 | } else { 89 | lDisData[ x ] = lDisData[ rFirst ]; 90 | } 91 | } else if( lFind ) { 92 | lDisData[ x ] = lDisData[ lFirst ]; 93 | } else if ( rFind ) { 94 | lDisData[ x ] = lDisData[ rFirst ]; 95 | } 96 | } 97 | } 98 | } 99 | // fill right dep 100 | #pragma omp parallel for 101 | for( int y = 0; y < hei; y ++ ) { 102 | uchar* rDisData = ( uchar* ) ( rDis.ptr( y ) ); 103 | uchar* rValidData = ( uchar* ) rValid.ptr( y ); 104 | for( int x = 0; x < wid; x ++ ) { 105 | if( rValidData[x] == 0 ) { 106 | // find left first valid pixel 107 | int lFirst = x; 108 | int lFind = 0; 109 | while( lFirst >= 0 ) { 110 | if( rValidData[ lFirst ] ) { 111 | lFind = 1; 112 | break; 113 | } 114 | lFirst --; 115 | } 116 | // find right first valid pixel 117 | int rFirst = x; 118 | int rFind = 0; 119 | while( rFirst < wid ) { 120 | if( rValidData[ rFirst ] ) { 121 | rFind = 1; 122 | break; 123 | } 124 | rFirst ++; 125 | } 126 | if( lFind && rFind ) { 127 | // set x's depth to the lowest one 128 | if( rDisData[ lFirst ] <= rDisData[ rFirst ] ) { 129 | rDisData[ x ] = rDisData[ lFirst ]; 130 | } else { 131 | rDisData[ x ] = rDisData[ rFirst ]; 132 | } 133 | } else if( lFind ) { 134 | rDisData[ x ] = rDisData[ lFirst ]; 135 | } else if ( rFind ) { 136 | rDisData[ x ] = rDisData[ rFirst ]; 137 | } 138 | 139 | } 140 | } 141 | } 142 | return; 143 | } 144 | 145 | void wgtMedian(const Mat& lImg, const Mat& rImg, Mat& lDis, Mat& rDis, Mat& lValid, Mat& rValid, const int maxDis) 146 | { 147 | int hei = lImg.rows; 148 | int wid = lImg.cols; 149 | int wndR = MED_SZ / 2; 150 | 151 | float* disHist = new float[maxDis]; 152 | 153 | // filter left 154 | for( int y = 0; y < hei; y ++ ) { 155 | uchar* lDisData = ( uchar* ) lDis.ptr( y ); 156 | float* pL = ( float* ) lImg.ptr( y ); 157 | uchar* lValidData = ( uchar* ) lValid.ptr( y ); 158 | for( int x = 0; x < wid; x ++ ) { 159 | if( lValidData[x] == 0 ) { 160 | // just filter invalid pixels 161 | memset( disHist, 0, sizeof( float ) * maxDis ); 162 | float sumWgt = 0.0f; 163 | // set disparity histogram by bilateral weight 164 | for( int wy = - wndR; wy <= wndR; wy ++ ) { 165 | int qy = ( y + wy + hei ) % hei; 166 | float* qL = ( float* ) lImg.ptr( qy ); 167 | uchar* qDisData = ( uchar* ) lDis.ptr( qy ); 168 | for( int wx = - wndR; wx <= wndR; wx ++ ) { 169 | int qx = ( x + wx + wid ) % wid; 170 | int qDep = qDisData[ qx ]; 171 | if( qDep != 0 ) { 172 | float disWgt = wx * wx + wy * wy; 173 | float clrWgt = 174 | ( pL[ 3 * x ] - qL[ 3 * qx ] ) * ( pL[ 3 * x ] - qL[ 3 * qx ] ) + 175 | ( pL[ 3 * x + 1 ] - qL[ 3 * qx + 1 ] ) * ( pL[ 3 * x + 1 ] - qL[ 3 * qx + 1 ] ) + 176 | ( pL[ 3 * x + 2 ] - qL[ 3 * qx + 2 ] ) * ( pL[ 3 * x + 2 ] - qL[ 3 * qx + 2 ] ); 177 | float biWgt = exp( - disWgt / ( SIG_DIS * SIG_DIS ) - clrWgt / ( SIG_CLR * SIG_CLR ) ); 178 | disHist[ qDep ] += biWgt; 179 | sumWgt += biWgt; 180 | } 181 | } 182 | } 183 | float halfWgt = sumWgt / 2.0f; 184 | sumWgt = 0.0f; 185 | int filterDep = 0; 186 | for( int d = 0; d < maxDis; d ++ ) { 187 | sumWgt += disHist[ d ]; 188 | if( sumWgt >= halfWgt ) { 189 | filterDep = d; 190 | break; 191 | } 192 | } 193 | // set new disparity 194 | lDisData[ x ] = filterDep; 195 | } 196 | } 197 | } 198 | 199 | // filter right depth 200 | for( int y = 0; y < hei; y ++ ) { 201 | uchar* rDisData = ( uchar* ) rDis.ptr( y ); 202 | float* pR = ( float* ) rImg.ptr( y ); 203 | uchar* rValidData = ( uchar* ) rValid.ptr( y ); 204 | for( int x = 0; x < wid; x ++ ) { 205 | if( rValidData[x] == 0 ) { 206 | // just filter invalid pixels 207 | memset( disHist, 0, sizeof( float ) * maxDis ); 208 | float sumWgt = 0.0f; 209 | // set disparity histogram by bilateral weight 210 | for( int wy = - wndR; wy <= wndR; wy ++ ) { 211 | int qy = ( y + wy + hei ) % hei; 212 | float* qR = ( float* ) rImg.ptr( qy ); 213 | uchar* qDisData = ( uchar* ) rDis.ptr( qy ); 214 | for( int wx = - wndR; wx <= wndR; wx ++ ) { 215 | int qx = ( x + wx + wid ) % wid; 216 | int qDep = qDisData[ qx ]; 217 | if( qDep != 0 ) { 218 | float disWgt = wx * wx + wy * wy; 219 | disWgt = sqrt( disWgt ); 220 | float clrWgt = 221 | ( pR[ 3 * x ] - qR[ 3 * qx ] ) * ( pR[ 3 * x ] - qR[ 3 * qx ] ) + 222 | ( pR[ 3 * x + 1 ] - qR[ 3 * qx + 1 ] ) * ( pR[ 3 * x + 1 ] - qR[ 3 * qx + 1 ] ) + 223 | ( pR[ 3 * x + 2 ] - qR[ 3 * qx + 2 ] ) * ( pR[ 3 * x + 2 ] - qR[ 3 * qx + 2 ] ); 224 | clrWgt = sqrt( clrWgt ); 225 | float biWgt = exp( - disWgt / ( SIG_DIS * SIG_DIS ) - clrWgt / ( SIG_CLR * SIG_CLR ) ); 226 | disHist[ qDep ] += biWgt; 227 | sumWgt += biWgt; 228 | } 229 | } 230 | } 231 | float halfWgt = sumWgt / 2.0f; 232 | sumWgt = 0.0f; 233 | int filterDep = 0; 234 | for( int d = 0; d < maxDis; d ++ ) { 235 | sumWgt += disHist[ d ]; 236 | if( sumWgt >= halfWgt ) { 237 | filterDep = d; 238 | break; 239 | } 240 | } 241 | // set new disparity 242 | rDisData[ x ] = filterDep; 243 | } 244 | } 245 | } 246 | return; 247 | } 248 | 249 | void *wgtMed_row(void *thread_arg) 250 | { 251 | //Passing function arguments from wgtMedian_thread 252 | struct WM_row_TD *t_data; 253 | t_data = (struct WM_row_TD *) thread_arg; 254 | //Matricies 255 | const Mat* Img = t_data->Img; 256 | Mat* Dis = t_data->Dis; 257 | //Pointers 258 | uchar *pValid = t_data->pValid; 259 | //Variables 260 | int y = t_data->y; 261 | int maxDis = t_data->maxDis; 262 | 263 | int wndR = MED_SZ / 2; 264 | int hei = Img->rows; 265 | int wid = Img->cols; 266 | 267 | float* disHist = new float[maxDis]; 268 | uchar* DisData = (uchar*) Dis->ptr(y); 269 | //32 bit float data 270 | // if(Img->type() == CV_32FC3) 271 | // { 272 | float* p = (float*) Img->ptr(y); 273 | for( int x = 0; x < wid; x ++ ) { 274 | if( pValid[x] == 0 ) { 275 | // just filter invalid pixels 276 | memset( disHist, 0, sizeof( float ) * maxDis ); 277 | float sumWgt = 0.0f; 278 | // set disparity histogram by bilateral weight 279 | for( int wy = - wndR; wy <= wndR; wy++ ) { 280 | int qy = ( y + wy + hei ) % hei; 281 | float* q = ( float* ) Img->ptr( qy ); 282 | uchar* qDisData = ( uchar* ) Dis->ptr( qy ); 283 | for( int wx = - wndR; wx <= wndR; wx ++ ) { 284 | int qx = ( x + wx + wid ) % wid; 285 | // invalid pixel also used 286 | int qDep = qDisData[ qx ]; 287 | if( qDep != 0 ) { 288 | float disWgt = wx * wx + wy * wy; 289 | float clrWgt = 290 | ( p[ 3 * x ] - q[ 3 * qx ] ) * ( p[ 3 * x ] - q[ 3 * qx ] ) + 291 | ( p[ 3 * x + 1 ] - q[ 3 * qx + 1 ] ) * ( p[ 3 * x + 1 ] - q[ 3 * qx + 1 ] ) + 292 | ( p[ 3 * x + 2 ] - q[ 3 * qx + 2 ] ) * ( p[ 3 * x + 2 ] - q[ 3 * qx + 2 ] ); 293 | float biWgt = exp( - disWgt / ( SIG_DIS * SIG_DIS ) - clrWgt / ( SIG_CLR * SIG_CLR ) ); 294 | disHist[ qDep ] += biWgt; 295 | sumWgt += biWgt; 296 | } 297 | } 298 | } 299 | float halfWgt = sumWgt / 2.0f; 300 | sumWgt = 0.0f; 301 | int filterDep = 0; 302 | for( int d = 0; d < maxDis; d ++ ) { 303 | sumWgt += disHist[ d ]; 304 | if( sumWgt >= halfWgt ) { 305 | filterDep = d; 306 | break; 307 | } 308 | } 309 | // set new disparity 310 | DisData[ x ] = filterDep; 311 | } 312 | } 313 | // } 314 | // //8 bit unsigned char data 315 | // else if(Img->type() == CV_8UC3) 316 | // { 317 | // uchar* p = (uchar*) Img->ptr(y); 318 | // for( int x = 0; x < wid; x ++ ) { 319 | // if( pValid[x] == 0 ) { 320 | // // just filter invalid pixels 321 | // memset( disHist, 0, sizeof( float ) * maxDis ); 322 | // float sumWgt = 0.0f; 323 | // // set disparity histogram by bilateral weight 324 | // for( int wy = - wndR; wy <= wndR; wy++ ) { 325 | // int qy = ( y + wy + hei ) % hei; 326 | // uchar* q = ( uchar* ) Img->ptr( qy ); 327 | // uchar* qDisData = ( uchar* ) Dis->ptr( qy ); 328 | // for( int wx = - wndR; wx <= wndR; wx ++ ) { 329 | // int qx = ( x + wx + wid ) % wid; 330 | // // invalid pixel also used 331 | // int qDep = qDisData[ qx ]; 332 | // if( qDep != 0 ) { 333 | // float disWgt = wx * wx + wy * wy; 334 | // float clrWgt = 335 | // ( p[ 3 * x ] - q[ 3 * qx ] ) * ( p[ 3 * x ] - q[ 3 * qx ] ) + 336 | // ( p[ 3 * x + 1 ] - q[ 3 * qx + 1 ] ) * ( p[ 3 * x + 1 ] - q[ 3 * qx + 1 ] ) + 337 | // ( p[ 3 * x + 2 ] - q[ 3 * qx + 2 ] ) * ( p[ 3 * x + 2 ] - q[ 3 * qx + 2 ] ); 338 | // float biWgt = exp( - disWgt / ( SIG_DIS * SIG_DIS ) - clrWgt / ( SIG_CLR * SIG_CLR ) ); 339 | // disHist[ qDep ] += biWgt; 340 | // sumWgt += biWgt; 341 | // } 342 | // } 343 | // } 344 | // float halfWgt = sumWgt / 2.0f; 345 | // sumWgt = 0.0f; 346 | // int filterDep = 0; 347 | // for( int d = 0; d < maxDis; d ++ ) { 348 | // sumWgt += disHist[ d ]; 349 | // if( sumWgt >= halfWgt ) { 350 | // filterDep = d; 351 | // break; 352 | // } 353 | // } 354 | // // set new disparity 355 | // DisData[ x ] = filterDep; 356 | // } 357 | // } 358 | // } 359 | // else{ 360 | // printf("PP: Error - Unrecognised data type in processing! (wgtMed_row)\n"); 361 | // exit(1); 362 | // } 363 | return (void*)0; 364 | } 365 | 366 | void wgtMedian_thread(const Mat& Img, Mat& Dis, Mat& Valid, const int maxDis, const int threads) 367 | { 368 | int hei = Img.rows; 369 | 370 | //Set up threads for x-loop 371 | void* status; 372 | pthread_attr_t attr; 373 | pthread_attr_init(&attr); 374 | pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); 375 | pthread_t WM_row_threads[hei]; 376 | WM_row_TD WM_row_TD_Array[hei]; 377 | 378 | for(int level = 0; level <= hei/threads; level ++) 379 | { 380 | //Handle remainder if threads is not power of 2. 381 | int block_size = (level < hei/threads) ? threads : (hei%threads); 382 | 383 | for(int iter=0; iter < block_size; iter++) 384 | { 385 | int d = level*threads + iter; 386 | uchar* ValidData = (uchar*) Valid.ptr(d); 387 | 388 | WM_row_TD_Array[d] = {&Img, &Dis, ValidData, d, maxDis}; 389 | pthread_create(&WM_row_threads[d], &attr, wgtMed_row, (void *)&WM_row_TD_Array[d]); 390 | //printf("WM Filtering Disparity Map @ y = %d\n", d); 391 | } 392 | for(int iter=0; iter < block_size; iter++) 393 | { 394 | int d = level*threads + iter; 395 | pthread_join(WM_row_threads[d], &status); 396 | //printf("Joining WM Filtering @ y = %d\n", d); 397 | } 398 | } 399 | return; 400 | } 401 | 402 | void PP::processDM(Mat& lImg, Mat& rImg, Mat& lDisMap, Mat& rDisMap, 403 | Mat& lValid, Mat& rValid, const int maxDis, int threads) 404 | { 405 | // lrCheck(lDisMap, rDisMap, lValid, rValid); 406 | // fillInv(lDisMap, rDisMap, lValid, rValid); 407 | 408 | // color image should be 3x3 median filtered 409 | // according to weightedMedianMatlab.m from CVPR11 410 | //wgtMedian( lImg, rImg, lDisMap, rDisMap, lValid, rValid, maxDis); 411 | 412 | // wgtMedian_thread(lImg, lDisMap, lValid, maxDis, threads); 413 | // wgtMedian_thread(rImg, rDisMap, rValid, maxDis, threads); 414 | //printf("Weighted-Median Filter Done\n"); 415 | 416 | Mat lImg_8UC3, rImg_8UC3; 417 | //use colour feature images: 418 | lImg.convertTo(lImg_8UC3, CV_8UC3, 255); 419 | rImg.convertTo(rImg_8UC3, CV_8UC3, 255); 420 | 421 | lDisMap = JointWMF::filter(lDisMap, lImg_8UC3, (int)MED_SZ/2); 422 | rDisMap = JointWMF::filter(rDisMap, rImg_8UC3, (int)MED_SZ/2); 423 | 424 | return; 425 | } 426 | -------------------------------------------------------------------------------- /src/StereoCalib.cpp: -------------------------------------------------------------------------------- 1 | /* This is sample from the OpenCV book. The copyright notice is below */ 2 | 3 | /* *************** License:************************** 4 | Oct. 3, 2008 5 | Right to use this code in any way you want without warranty, support or any guarantee of it working. 6 | 7 | BOOK: It would be nice if you cited it: 8 | Learning OpenCV: Computer Vision with the OpenCV Library 9 | by Gary Bradski and Adrian Kaehler 10 | Published by O'Reilly Media, October 3, 2008 11 | 12 | AVAILABLE AT: 13 | http://www.amazon.com/Learning-OpenCV-Computer-Vision-Library/dp/0596516134 14 | Or: http://oreilly.com/catalog/9780596516130/ 15 | ISBN-10: 0596516134 or: ISBN-13: 978-0596516130 16 | 17 | OPENCV WEBSITES: 18 | Homepage: http://opencv.org 19 | Online docs: http://docs.opencv.org 20 | Q&A forum: http://answers.opencv.org 21 | Issue tracker: http://code.opencv.org 22 | GitHub: https://github.com/Itseez/opencv/ 23 | ************************************************** */ 24 | /*--------------------------------------------------------------------------- 25 | StereoCalib.cpp - Stereo Camera Calibration Function Code 26 | --------------------------------------------------------------------------- 27 | Co-Author: Charles Leech 28 | Email: cl19g10 [at] ecs.soton.ac.uk 29 | ---------------------------------------------------------------------------*/ 30 | 31 | #include "StereoCalib.h" 32 | 33 | int print_help() 34 | { 35 | std::cout << 36 | " Given a list of chessboard images, the number of corners (nx, ny)\n" 37 | " on the chessboards, and a flag: useCalibrated for \n" 38 | " calibrated (0) or\n" 39 | " uncalibrated \n" 40 | " (1: use cvStereoCalibrate(), 2: compute fundamental\n" 41 | " matrix separately) stereo. \n" 42 | " Calibrate the cameras and display the\n" 43 | " rectified results along with the computed disparity images. \n" << std::endl; 44 | std::cout << "Usage:\n ./stereo_calib -w board_width -h board_height [-nr /*dot not view results*/] \n" << std::endl; 45 | return 0; 46 | } 47 | 48 | 49 | void StereoCalib(const std::vector& imagelist, Size boardSize, StereoCameraProperties& props, bool useCalibrated=true, bool showRectified=true) 50 | { 51 | if( imagelist.size() % 2 != 0 ) 52 | { 53 | std::cout << "Error: the image list contains odd (non-even) number of elements" << std::endl; 54 | return; 55 | } 56 | 57 | bool displayCorners = true; //false; 58 | const int maxScale = 2; 59 | const float squareSize = 1.f; // Set this to your actual square size 60 | // ARRAY AND VECTOR STORAGE: 61 | 62 | std::vector > imagePoints[2]; 63 | std::vector > objectPoints; 64 | Size imageSize; 65 | 66 | int i, j, k, nimages = (int)imagelist.size()/2; 67 | 68 | imagePoints[0].resize(nimages); 69 | imagePoints[1].resize(nimages); 70 | std::vector goodImageList; 71 | 72 | for( i = j = 0; i < nimages; i++ ) 73 | { 74 | for( k = 0; k < 2; k++ ) 75 | { 76 | std::string filename = std::string(BASE_DIR) + std::string("data/") + std::string(imagelist[i*2+k]); 77 | Mat img = imread(filename, 0); 78 | if(img.empty()) 79 | break; 80 | if( imageSize == Size() ) 81 | imageSize = img.size(); 82 | else if( img.size() != imageSize ) 83 | { 84 | std::cout << "The image " << filename << " has the size different from the first image size. Skipping the pair" << std::endl; 85 | break; 86 | } 87 | bool found = false; 88 | std::vector& corners = imagePoints[k][j]; 89 | for( int scale = 1; scale <= maxScale; scale++ ) 90 | { 91 | Mat timg; 92 | if( scale == 1 ) 93 | timg = img; 94 | else 95 | resize(img, timg, Size(), scale, scale); 96 | found = findChessboardCorners(timg, boardSize, corners, 97 | CALIB_CB_ADAPTIVE_THRESH | CALIB_CB_NORMALIZE_IMAGE); 98 | if( found ) 99 | { 100 | if( scale > 1 ) 101 | { 102 | Mat cornersMat(corners); 103 | cornersMat *= 1./scale; 104 | } 105 | break; 106 | } 107 | } 108 | if( displayCorners ) 109 | { 110 | cv::Mat cimg, cimg1; 111 | cv::cvtColor(img, cimg, COLOR_GRAY2BGR); 112 | cv::drawChessboardCorners(cimg, boardSize, corners, found); 113 | double sf = 640./MAX(img.rows, img.cols); 114 | resize(cimg, cimg1, Size(), sf, sf); 115 | cv::imshow("InputOutput", cimg1); 116 | char c = (char)waitKey(250); 117 | if( c == 27 || c == 'q' || c == 'Q' ) //Allow ESC to quit 118 | exit(-1); 119 | } 120 | else 121 | putchar('.'); 122 | if( !found ) 123 | break; 124 | cornerSubPix(img, corners, Size(11,11), Size(-1,-1), 125 | TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 126 | 30, 0.01)); 127 | } 128 | if( k == 2 ) 129 | { 130 | goodImageList.push_back(imagelist[i*2]); 131 | goodImageList.push_back(imagelist[i*2+1]); 132 | j++; 133 | } 134 | } 135 | std::cout << j << " pairs have been successfully detected.\n" << std::endl; 136 | nimages = j; 137 | 138 | if( nimages < 2 ) 139 | { 140 | std::cout << "Error: too few pairs to run the calibration\n" << std::endl; 141 | exit(1); 142 | } 143 | 144 | imagePoints[0].resize(nimages); 145 | imagePoints[1].resize(nimages); 146 | objectPoints.resize(nimages); 147 | 148 | for( i = 0; i < nimages; i++ ) 149 | { 150 | for( j = 0; j < boardSize.height; j++ ) 151 | for( k = 0; k < boardSize.width; k++ ) 152 | objectPoints[i].push_back(Point3f(k*squareSize, j*squareSize, 0)); 153 | } 154 | 155 | std::cout << "Running stereo calibration ..." << std::endl; 156 | 157 | Mat cameraMatrix[2], distCoeffs[2]; 158 | cameraMatrix[0] = Mat::eye(3, 3, CV_64F); 159 | cameraMatrix[1] = Mat::eye(3, 3, CV_64F); 160 | Mat R, T, E, F; 161 | 162 | double rms = cv::stereoCalibrate(objectPoints, imagePoints[0], imagePoints[1], 163 | cameraMatrix[0], distCoeffs[0], 164 | cameraMatrix[1], distCoeffs[1], 165 | imageSize, R, T, E, F, 166 | CALIB_FIX_ASPECT_RATIO + 167 | CALIB_ZERO_TANGENT_DIST + 168 | CALIB_SAME_FOCAL_LENGTH + 169 | CALIB_RATIONAL_MODEL + 170 | CALIB_FIX_K3 + CALIB_FIX_K4 + CALIB_FIX_K5, 171 | TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 100, 1e-5) ); 172 | std::cout << "done with RMS error = " << rms << std::endl; 173 | 174 | // CALIBRATION QUALITY CHECK 175 | // because the output fundamental matrix implicitly 176 | // includes all the output information, 177 | // we can check the quality of calibration using the 178 | // epipolar geometry constraint: m2^t*F*m1=0 179 | double err = 0; 180 | int npoints = 0; 181 | std::vector lines[2]; 182 | for( i = 0; i < nimages; i++ ) 183 | { 184 | int npt = (int)imagePoints[0][i].size(); 185 | Mat imgpt[2]; 186 | for( k = 0; k < 2; k++ ) 187 | { 188 | imgpt[k] = Mat(imagePoints[k][i]); 189 | undistortPoints(imgpt[k], imgpt[k], cameraMatrix[k], distCoeffs[k], Mat(), cameraMatrix[k]); 190 | computeCorrespondEpilines(imgpt[k], k+1, F, lines[k]); 191 | } 192 | for( j = 0; j < npt; j++ ) 193 | { 194 | double errij = fabs(imagePoints[0][i][j].x*lines[1][j][0] + 195 | imagePoints[0][i][j].y*lines[1][j][1] + lines[1][j][2]) + 196 | fabs(imagePoints[1][i][j].x*lines[0][j][0] + 197 | imagePoints[1][i][j].y*lines[0][j][1] + lines[0][j][2]); 198 | err += errij; 199 | } 200 | npoints += npt; 201 | } 202 | std::cout << "average reprojection err = " << err/npoints << std::endl; 203 | 204 | // save intrinsic parameters 205 | FileStorage fs(FILE_INTRINSICS, FileStorage::WRITE); 206 | if( fs.isOpened() ) 207 | { 208 | fs << "M1" << cameraMatrix[0] << "D1" << distCoeffs[0] << 209 | "M2" << cameraMatrix[1] << "D2" << distCoeffs[1]; 210 | fs.release(); 211 | } 212 | else{ 213 | std::cout << "Error: can not save the intrinsic parameters " << std::endl; 214 | } 215 | 216 | props.cameraMatrix[0] = cameraMatrix[0]; 217 | props.cameraMatrix[1] = cameraMatrix[1]; 218 | props.distCoeffs[0] = distCoeffs[0]; 219 | props.distCoeffs[1] = distCoeffs[1]; 220 | 221 | Mat R1, R2, P1, P2, Q; 222 | Rect validRoi[2]; 223 | 224 | stereoRectify(cameraMatrix[0], distCoeffs[0], 225 | cameraMatrix[1], distCoeffs[1], 226 | imageSize, R, T, R1, R2, P1, P2, Q, 227 | CALIB_ZERO_DISPARITY, 1, imageSize, &validRoi[0], &validRoi[1]); 228 | 229 | fs.open(FILE_EXTRINSICS, FileStorage::WRITE); 230 | if( fs.isOpened() ) 231 | { 232 | fs << "R" << R << "T" << T << "R1" << R1 << "R2" << R2 << "P1" << P1 << "P2" << P2 << "Q" << Q; 233 | fs.release(); 234 | } 235 | else{ 236 | std::cout << "Error: can not save the extrinsic parameters" << std::endl; 237 | } 238 | 239 | std::cout << "Valid ROI:" << std::endl; 240 | std::cout << "Left: " << validRoi[0] << std::endl; 241 | std::cout << "VRight: " << validRoi[1] << std::endl; 242 | 243 | props.R = R; 244 | props.T = T; 245 | props.R1 = R1; 246 | props.R2 = R2; 247 | props.P1 = P1; 248 | props.P2 = P2; 249 | props.Q = Q; 250 | props.imgSize = imageSize; 251 | props.roi[0] = validRoi[0]; 252 | props.roi[1] = validRoi[1]; 253 | 254 | // OpenCV can handle left-right 255 | // or up-down camera arrangements 256 | bool isVerticalStereo = fabs(P2.at(1, 3)) > fabs(P2.at(0, 3)); 257 | 258 | // COMPUTE AND DISPLAY RECTIFICATION 259 | if( !showRectified ) 260 | return; 261 | 262 | Mat rmap[2][2]; 263 | // IF BY CALIBRATED (BOUGUET'S METHOD) 264 | if( useCalibrated ) 265 | { 266 | // we already computed everything 267 | } 268 | // OR ELSE HARTLEY'S METHOD 269 | else 270 | // use intrinsic parameters of each camera, but 271 | // compute the rectification transformation directly 272 | // from the fundamental matrix 273 | { 274 | std::vector allimgpt[2]; 275 | for( k = 0; k < 2; k++ ) 276 | { 277 | for( i = 0; i < nimages; i++ ) 278 | std::copy(imagePoints[k][i].begin(), imagePoints[k][i].end(), back_inserter(allimgpt[k])); 279 | } 280 | F = findFundamentalMat(Mat(allimgpt[0]), Mat(allimgpt[1]), FM_8POINT, 0, 0); 281 | Mat H1, H2; 282 | stereoRectifyUncalibrated(Mat(allimgpt[0]), Mat(allimgpt[1]), F, imageSize, H1, H2, 3); 283 | 284 | R1 = cameraMatrix[0].inv()*H1*cameraMatrix[0]; 285 | R2 = cameraMatrix[1].inv()*H2*cameraMatrix[1]; 286 | P1 = cameraMatrix[0]; 287 | P2 = cameraMatrix[1]; 288 | } 289 | 290 | //Precompute maps for cv::remap() 291 | initUndistortRectifyMap(cameraMatrix[0], distCoeffs[0], R1, P1, imageSize, CV_16SC2, rmap[0][0], rmap[0][1]); 292 | initUndistortRectifyMap(cameraMatrix[1], distCoeffs[1], R2, P2, imageSize, CV_16SC2, rmap[1][0], rmap[1][1]); 293 | 294 | Mat canvas; 295 | double sf; 296 | int w, h; 297 | if( !isVerticalStereo ) 298 | { 299 | sf = 600./MAX(imageSize.width, imageSize.height); 300 | w = cvRound(imageSize.width*sf); 301 | h = cvRound(imageSize.height*sf); 302 | canvas.create(h, w*2, CV_8UC3); 303 | } 304 | else 305 | { 306 | sf = 300./MAX(imageSize.width, imageSize.height); 307 | w = cvRound(imageSize.width*sf); 308 | h = cvRound(imageSize.height*sf); 309 | canvas.create(h*2, w, CV_8UC3); 310 | } 311 | 312 | for( i = 0; i < nimages; i++ ) 313 | { 314 | for( k = 0; k < 2; k++ ) 315 | { 316 | std::string filename = std::string(BASE_DIR) + std::string("data/") + std::string(goodImageList[i*2+k]); 317 | Mat img = imread(filename, 0), rimg, cimg; 318 | remap(img, rimg, rmap[k][0], rmap[k][1], INTER_LINEAR); 319 | cvtColor(rimg, cimg, COLOR_GRAY2BGR); 320 | Mat canvasPart = !isVerticalStereo ? canvas(Rect(w*k, 0, w, h)) : canvas(Rect(0, h*k, w, h)); 321 | resize(cimg, canvasPart, canvasPart.size(), 0, 0, INTER_AREA); 322 | if( useCalibrated ) 323 | { 324 | Rect vroi(cvRound(validRoi[k].x*sf), cvRound(validRoi[k].y*sf), 325 | cvRound(validRoi[k].width*sf), cvRound(validRoi[k].height*sf)); 326 | rectangle(canvasPart, vroi, Scalar(0,0,255), 3, 8); 327 | } 328 | } 329 | 330 | if( !isVerticalStereo ) 331 | for( j = 0; j < canvas.rows; j += 16 ) 332 | line(canvas, Point(0, j), Point(canvas.cols, j), Scalar(0, 255, 0), 1, 8); 333 | else 334 | for( j = 0; j < canvas.cols; j += 16 ) 335 | line(canvas, Point(j, 0), Point(j, canvas.rows), Scalar(0, 255, 0), 1, 8); 336 | 337 | imshow("InputOutput", canvas); 338 | if(!i) 339 | { 340 | imwrite("rectifiedImage.png", canvas); 341 | } 342 | char c = (char)waitKey(); 343 | if( c == 27 || c == 'q' || c == 'Q' ) 344 | break; 345 | } 346 | //destroyWindow("rectified"); 347 | } 348 | 349 | bool readStringList( const std::string& filename, std::vector& l ) 350 | { 351 | l.resize(0); 352 | FileStorage fs(filename, FileStorage::READ); 353 | if( !fs.isOpened() ) 354 | return false; 355 | FileNode n = fs.getFirstTopLevelNode(); 356 | if( n.type() != FileNode::SEQ ) 357 | return false; 358 | FileNodeIterator it = n.begin(), it_end = n.end(); 359 | for( ; it != it_end; ++it ) 360 | l.push_back((std::string)*it); 361 | return true; 362 | } 363 | 364 | int calibrateCamera(int boardWidth, int boardHeight, StereoCameraProperties& props, std::string imagelistfn) 365 | { 366 | Size boardSize; 367 | boardSize.width = boardWidth; 368 | boardSize.height = boardHeight; 369 | bool useCalibrated = true; 370 | bool showRectified = true; 371 | 372 | //imagelistfn = "data/stereo_calib.xml"; 373 | 374 | std::vector imagelist; 375 | bool ok = readStringList(imagelistfn, imagelist); 376 | if(!ok || imagelist.empty()) 377 | { 378 | std::cout << "can not open " << imagelistfn << " or the string list is empty" << std::endl; 379 | return print_help(); 380 | } 381 | 382 | StereoCalib(imagelist, boardSize, props, useCalibrated, showRectified); 383 | return 0; 384 | } 385 | -------------------------------------------------------------------------------- /src/fastguidedfilter.cpp: -------------------------------------------------------------------------------- 1 | // Source: https://github.com/Sundrops/fast-guided-filter 2 | // Literature: https://arxiv.org/pdf/1505.00996.pdf 3 | #include "fastguidedfilter.h" 4 | 5 | static cv::Mat boxfilter(const cv::Mat &I, int r) 6 | { 7 | cv::Mat result; 8 | cv::blur(I, result, cv::Size(r, r)); 9 | return result; 10 | } 11 | 12 | static cv::Mat convertTo(const cv::Mat &mat, int depth) 13 | { 14 | if (mat.depth() == depth) 15 | return mat; 16 | 17 | cv::Mat result; 18 | mat.convertTo(result, depth); 19 | return result; 20 | } 21 | 22 | class FastGuidedFilterImpl 23 | { 24 | public: 25 | FastGuidedFilterImpl(int r, double eps,int s):r(r),eps(eps),s(s){} 26 | virtual ~FastGuidedFilterImpl() {} 27 | 28 | cv::Mat filter(const cv::Mat &p, int depth); 29 | 30 | protected: 31 | int Idepth,r,s; 32 | double eps; 33 | 34 | private: 35 | virtual cv::Mat filterSingleChannel(const cv::Mat &p) const = 0; 36 | }; 37 | 38 | class FastGuidedFilterMono : public FastGuidedFilterImpl 39 | { 40 | public: 41 | FastGuidedFilterMono(const cv::Mat &I, int r, double eps,int s); 42 | 43 | private: 44 | virtual cv::Mat filterSingleChannel(const cv::Mat &p) const; 45 | 46 | private: 47 | 48 | cv::Mat I,origI, mean_I, var_I; 49 | }; 50 | 51 | class FastGuidedFilterColor : public FastGuidedFilterImpl 52 | { 53 | public: 54 | FastGuidedFilterColor(const cv::Mat &I, int r, double eps,int s); 55 | 56 | private: 57 | virtual cv::Mat filterSingleChannel(const cv::Mat &p) const; 58 | 59 | private: 60 | std::vector origIchannels,Ichannels; 61 | cv::Mat mean_I_r, mean_I_g, mean_I_b; 62 | cv::Mat invrr, invrg, invrb, invgg, invgb, invbb; 63 | }; 64 | 65 | 66 | cv::Mat FastGuidedFilterImpl::filter(const cv::Mat &p, int depth) 67 | { 68 | cv::Mat p2 = convertTo(p, Idepth); 69 | cv::resize(p2 ,p2,cv::Size(p2.cols/s,p2.rows/s),0,0,CV_INTER_NN); 70 | cv::Mat result; 71 | if (p.channels() == 1) 72 | { 73 | result = filterSingleChannel(p2); 74 | } 75 | else 76 | { 77 | std::vector pc; 78 | cv::split(p2, pc); 79 | 80 | for (std::size_t i = 0; i < pc.size(); ++i) 81 | pc[i] = filterSingleChannel(pc[i]); 82 | 83 | cv::merge(pc, result); 84 | } 85 | 86 | return convertTo(result, depth == -1 ? p.depth() : depth); 87 | } 88 | 89 | FastGuidedFilterMono::FastGuidedFilterMono(const cv::Mat &origI, int r, double eps,int s):FastGuidedFilterImpl(r,eps,s) 90 | { 91 | 92 | if (origI.depth() == CV_32F || origI.depth() == CV_64F) 93 | this->origI = origI.clone(); 94 | else 95 | this->origI = convertTo(origI, CV_32F); 96 | cv::resize(this->origI ,I,cv::Size(this->origI.cols/s,this->origI.rows/s),0,0,CV_INTER_NN); 97 | Idepth = I.depth(); 98 | 99 | mean_I = boxfilter(I, r); 100 | cv::Mat mean_II = boxfilter(I.mul(I), r); 101 | var_I = mean_II - mean_I.mul(mean_I); 102 | } 103 | 104 | cv::Mat FastGuidedFilterMono::filterSingleChannel(const cv::Mat &p) const 105 | { 106 | 107 | cv::Mat mean_p = boxfilter(p, r); 108 | cv::Mat mean_Ip = boxfilter(I.mul(p), r); 109 | cv::Mat cov_Ip = mean_Ip - mean_I.mul(mean_p); // this is the covariance of (I, p) in each local patch. 110 | 111 | cv::Mat a = cov_Ip / (var_I + eps); 112 | cv::Mat b = mean_p - a.mul(mean_I); 113 | 114 | cv::Mat mean_a = boxfilter(a, r); 115 | cv::Mat mean_b = boxfilter(b, r); 116 | cv::resize(mean_a ,mean_a,cv::Size(origI.cols,origI.rows),0,0,CV_INTER_LINEAR); 117 | cv::resize(mean_b ,mean_b,cv::Size(origI.cols,origI.rows),0,0,CV_INTER_LINEAR); 118 | return mean_a.mul(origI) + mean_b; 119 | } 120 | 121 | FastGuidedFilterColor::FastGuidedFilterColor(const cv::Mat &origI, int r, double eps, int s):FastGuidedFilterImpl(r,eps,s)// : r(r), eps(eps) 122 | { 123 | 124 | cv::Mat I; 125 | if (origI.depth() == CV_32F || origI.depth() == CV_64F) 126 | I = origI.clone(); 127 | else 128 | I = convertTo(origI, CV_32F); 129 | Idepth = I.depth(); 130 | 131 | cv::split(I, origIchannels); 132 | cv::resize(I,I,cv::Size(I.cols/s,I.rows/s),0,0,CV_INTER_NN); 133 | cv::split(I, Ichannels); 134 | 135 | mean_I_r = boxfilter(Ichannels[0], r); 136 | mean_I_g = boxfilter(Ichannels[1], r); 137 | mean_I_b = boxfilter(Ichannels[2], r); 138 | 139 | // variance of I in each local patch: the matrix Sigma. 140 | // Note the variance in each local patch is a 3x3 symmetric matrix: 141 | // rr, rg, rb 142 | // Sigma = rg, gg, gb 143 | // rb, gb, bb 144 | cv::Mat var_I_rr = boxfilter(Ichannels[0].mul(Ichannels[0]), r) - mean_I_r.mul(mean_I_r) + eps; 145 | cv::Mat var_I_rg = boxfilter(Ichannels[0].mul(Ichannels[1]), r) - mean_I_r.mul(mean_I_g); 146 | cv::Mat var_I_rb = boxfilter(Ichannels[0].mul(Ichannels[2]), r) - mean_I_r.mul(mean_I_b); 147 | cv::Mat var_I_gg = boxfilter(Ichannels[1].mul(Ichannels[1]), r) - mean_I_g.mul(mean_I_g) + eps; 148 | cv::Mat var_I_gb = boxfilter(Ichannels[1].mul(Ichannels[2]), r) - mean_I_g.mul(mean_I_b); 149 | cv::Mat var_I_bb = boxfilter(Ichannels[2].mul(Ichannels[2]), r) - mean_I_b.mul(mean_I_b) + eps; 150 | 151 | // Inverse of Sigma + eps * I 152 | invrr = var_I_gg.mul(var_I_bb) - var_I_gb.mul(var_I_gb); 153 | invrg = var_I_gb.mul(var_I_rb) - var_I_rg.mul(var_I_bb); 154 | invrb = var_I_rg.mul(var_I_gb) - var_I_gg.mul(var_I_rb); 155 | invgg = var_I_rr.mul(var_I_bb) - var_I_rb.mul(var_I_rb); 156 | invgb = var_I_rb.mul(var_I_rg) - var_I_rr.mul(var_I_gb); 157 | invbb = var_I_rr.mul(var_I_gg) - var_I_rg.mul(var_I_rg); 158 | 159 | cv::Mat covDet = invrr.mul(var_I_rr) + invrg.mul(var_I_rg) + invrb.mul(var_I_rb); 160 | 161 | invrr /= covDet; 162 | invrg /= covDet; 163 | invrb /= covDet; 164 | invgg /= covDet; 165 | invgb /= covDet; 166 | invbb /= covDet; 167 | } 168 | 169 | cv::Mat FastGuidedFilterColor::filterSingleChannel(const cv::Mat &p) const 170 | { 171 | cv::Mat mean_p = boxfilter(p, r); 172 | 173 | cv::Mat mean_Ip_r = boxfilter(Ichannels[0].mul(p), r); 174 | cv::Mat mean_Ip_g = boxfilter(Ichannels[1].mul(p), r); 175 | cv::Mat mean_Ip_b = boxfilter(Ichannels[2].mul(p), r); 176 | 177 | // covariance of (I, p) in each local patch. 178 | cv::Mat cov_Ip_r = mean_Ip_r - mean_I_r.mul(mean_p); 179 | cv::Mat cov_Ip_g = mean_Ip_g - mean_I_g.mul(mean_p); 180 | cv::Mat cov_Ip_b = mean_Ip_b - mean_I_b.mul(mean_p); 181 | 182 | cv::Mat a_r = invrr.mul(cov_Ip_r) + invrg.mul(cov_Ip_g) + invrb.mul(cov_Ip_b); 183 | cv::Mat a_g = invrg.mul(cov_Ip_r) + invgg.mul(cov_Ip_g) + invgb.mul(cov_Ip_b); 184 | cv::Mat a_b = invrb.mul(cov_Ip_r) + invgb.mul(cov_Ip_g) + invbb.mul(cov_Ip_b); 185 | 186 | cv::Mat b = mean_p - a_r.mul(mean_I_r) - a_g.mul(mean_I_g) - a_b.mul(mean_I_b); 187 | 188 | cv::Mat mean_a_r = boxfilter(a_r, r); 189 | cv::Mat mean_a_g = boxfilter(a_g, r); 190 | cv::Mat mean_a_b = boxfilter(a_b, r); 191 | cv::Mat mean_b = boxfilter(b, r); 192 | cv::resize(mean_a_r ,mean_a_r,cv::Size(origIchannels[0].cols,origIchannels[0].rows),0,0,CV_INTER_LINEAR); 193 | cv::resize(mean_a_g ,mean_a_g,cv::Size(origIchannels[1].cols,origIchannels[1].rows),0,0,CV_INTER_LINEAR); 194 | cv::resize(mean_a_b ,mean_a_b,cv::Size(origIchannels[2].cols,origIchannels[2].rows),0,0,CV_INTER_LINEAR); 195 | cv::resize(mean_b,mean_b,cv::Size(origIchannels[2].cols,origIchannels[2].rows),0,0,CV_INTER_LINEAR); 196 | return (mean_a_r.mul(origIchannels[0]) +mean_a_g.mul(origIchannels[1]) +mean_a_b.mul(origIchannels[2]) + mean_b); 197 | 198 | } 199 | 200 | 201 | FastGuidedFilter::FastGuidedFilter(const cv::Mat &I, int r, double eps,int s) 202 | { 203 | CV_Assert(I.channels() == 1 || I.channels() == 3); 204 | 205 | if (I.channels() == 1) 206 | impl_ = new FastGuidedFilterMono(I, 2 * (r/s) + 1, eps,s); 207 | else 208 | impl_ = new FastGuidedFilterColor(I, 2 * (r/s) + 1, eps,s); 209 | } 210 | 211 | FastGuidedFilter::~FastGuidedFilter() 212 | { 213 | delete impl_; 214 | } 215 | 216 | cv::Mat FastGuidedFilter::filter(const cv::Mat &p, int depth) const 217 | { 218 | return impl_->filter(p, depth); 219 | } 220 | 221 | cv::Mat fastGuidedFilter(const cv::Mat &I, const cv::Mat &p, int r, double eps, int s,int depth) 222 | { 223 | return FastGuidedFilter(I, r, eps,s).filter(p, depth); 224 | } 225 | -------------------------------------------------------------------------------- /src/main.cpp: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------- 2 | main.cpp - central Disparity Estimation Code 3 | --------------------------------------------------------------------------- 4 | Author: Charles Leech 5 | Email: cl19g10 [at] ecs.soton.ac.uk 6 | Copyright (c) 2016 Charlie Leech, University of Southampton. 7 | ---------------------------------------------------------------------------*/ 8 | #include "ComFunc.h" 9 | #include "oclUtil.h" 10 | #include "StereoMatch.h" 11 | #include 12 | #include 13 | #include 14 | 15 | //Functions in main 16 | void getDepthMap(StereoMatch *sm); 17 | void HCI(StereoMatch *sm); 18 | 19 | //Global variables 20 | bool end_de = false; 21 | int nOpenCLDev = 0; 22 | int sgbm_mode = StereoSGBM::MODE_HH; 23 | 24 | int main(int argc, const char* argv[]) 25 | { 26 | //############################################################################################################# 27 | //# Introduction and Setup - poll for OpenCL devices 28 | //############################################################################################################# 29 | nOpenCLDev = openCLdevicepoll(); 30 | #ifdef DISPLAY 31 | namedWindow("InputOutput", CV_WINDOW_AUTOSIZE); 32 | #endif 33 | //############################################################################################################# 34 | //# Start Application Processes 35 | //############################################################################################################# 36 | printf("Starting Stereo Matching Application.\n"); 37 | StereoMatch *sm = new StereoMatch(argc, argv, nOpenCLDev); 38 | //printf("MAIN: Press h for help text.\n\n"); 39 | 40 | std::thread de_thread; 41 | //de_thread = std::thread(&StereoMatch::compute, sm, std::ref(de_time)); 42 | de_thread = std::thread(&getDepthMap, sm); 43 | 44 | #ifdef DISPLAY 45 | //User interface function 46 | HCI(sm); 47 | #else 48 | while(1){ 49 | std::this_thread::sleep_for (std::chrono::duration(1000)); 50 | } 51 | #endif 52 | //############################################################################################################# 53 | //# Clean up threads once quit signal called 54 | //############################################################################################################# 55 | end_de = true; 56 | printf("MAIN: Quit signal sent\n"); 57 | de_thread.join(); 58 | 59 | delete sm; 60 | printf("MAIN: Disparity Estimation Halted\n"); 61 | return 0; 62 | } 63 | 64 | void getDepthMap(StereoMatch *sm) 65 | { 66 | int ret = 0; 67 | float de_time; 68 | 69 | while(!(end_de || ret)){ 70 | ret = sm->compute(de_time); 71 | } 72 | return; 73 | } 74 | 75 | static void on_trackbar_err(int value, void* ptr) 76 | { 77 | printf("HCI: Error threshold set to %d.\n", value); 78 | } 79 | 80 | void HCI(StereoMatch *sm) 81 | { 82 | //User interface input handler 83 | char key = ' '; 84 | float de_time; 85 | int dataset_idx = 0; 86 | 87 | #ifdef DISPLAY 88 | imshow("InputOutput", sm->display_container); 89 | 90 | if(sm->media_mode == DE_IMAGE){ 91 | cv::createTrackbar("Error Threshold", "InputOutput", &sm->error_threshold, 64, on_trackbar_err); 92 | on_trackbar_err(sm->error_threshold, (void*)4); 93 | } 94 | #endif 95 | 96 | while(key != 'q') 97 | { 98 | switch(key) 99 | { 100 | case 'h': 101 | { 102 | printf("|-------------------------------------------------------------------|\n"); 103 | printf("| Input Options:\n"); 104 | printf("| h: Display this help text.\n"); 105 | printf("| q: Quit.\n"); 106 | printf("|-------------------------------------------------------------------|\n"); 107 | printf("| Control Options:\n"); 108 | printf("| 1-8: Change thread/core number.\n"); 109 | printf("| a: Switch matching algorithm: STEREO_GIF, STEREO_SGBM\n"); 110 | printf("| d: Cycle between images datasets:\n"); 111 | printf("| d: Art, Books, Cones, Dolls, Laundry, Moebius, Teddy.n"); 112 | printf("| m: Switch computation mode:\n"); 113 | printf("| m: STEREO_GIF: OpenCL <-> pthreads.\n"); 114 | printf("| m: STEREO_SGBM: MODE_SGBM, MODE_HH, MODE_SGBM_3WAY\n"); 115 | printf("| -/=: Increase or decrease the error threshold\n"); 116 | printf("|-------------------------------------------------------------------|\n"); 117 | printf("| Current Options:\n"); 118 | printf("| a: Matching Algorithm: %s\n", sm->MatchingAlgorithm ? "STEREO_GIF" : "STEREO_SGBM"); 119 | printf("| d: Dataset: %s\n", sm->MatchingAlgorithm ? "STEREO_GIF" : "STEREO_SGBM"); 120 | printf("| m: Computation mode: %s\n", sm->MatchingAlgorithm ? (sm->de_mode ? "OpenCL" : "pthreads") : ( 121 | sgbm_mode == StereoSGBM::MODE_HH ? "MODE_HH" : 122 | sgbm_mode == StereoSGBM::MODE_SGBM ? "MODE_SGBM" : "MODE_SGBM_3WAY" )); 123 | printf("| -/=: Error Threshold: %d\n", sm->error_threshold); 124 | printf("|-------------------------------------------------------------------|\n"); 125 | break; 126 | } 127 | case 'a': 128 | { 129 | sm->MatchingAlgorithm = sm->MatchingAlgorithm ? STEREO_SGBM : STEREO_GIF; 130 | printf("| a: Matching Algorithm Changed to: %s |\n", sm->MatchingAlgorithm ? "STEREO_GIF" : "STEREO_SGBM"); 131 | break; 132 | } 133 | case 'd': 134 | { 135 | if(sm->media_mode == DE_VIDEO){ 136 | printf("| d: Must be in image mode to use datasets.\n"); 137 | break; 138 | } 139 | if(sm->user_dataset){ 140 | printf("| d: User dataset has been specified.\n"); 141 | break; 142 | } 143 | dataset_idx = dataset_idx < dataset_names.size()-1 ? dataset_idx + 1 : 0; 144 | printf("| d: Dataset changed to: %s\n", dataset_names[dataset_idx].c_str()); 145 | sm->update_dataset(dataset_names[dataset_idx]); 146 | break; 147 | } 148 | case 'm': 149 | { 150 | if(sm->MatchingAlgorithm == STEREO_GIF){ 151 | if(nOpenCLDev){ 152 | sm->de_mode = sm->de_mode ? OCV_DE : OCL_DE; 153 | printf("| m: STEREO_GIF Matching Algorithm:\n"); 154 | printf("| m: Mode changed to %s |\n", sm->de_mode ? "OpenCL on the GPU" : "C++ & pthreads on the CPU"); 155 | } 156 | else{ 157 | printf("| m: Platform must contain an OpenCL compatible device to use OpenCL Mode.\n"); 158 | } 159 | } 160 | else if(sm->MatchingAlgorithm == STEREO_SGBM){ 161 | sgbm_mode = (sgbm_mode == StereoSGBM::MODE_HH ? StereoSGBM::MODE_SGBM : 162 | sgbm_mode == StereoSGBM::MODE_SGBM ? StereoSGBM::MODE_SGBM_3WAY : 163 | StereoSGBM::MODE_HH); 164 | sm->ssgbm->setMode(sgbm_mode); 165 | printf("| m: STEREO_GIF Matching Algorithm:\n"); 166 | printf("| m: Mode changed to %s |\n", sgbm_mode == StereoSGBM::MODE_HH ? "MODE_HH" : 167 | sgbm_mode == StereoSGBM::MODE_SGBM ? "MODE_SGBM" : 168 | "MODE_SGBM_3WAY"); 169 | } 170 | break; 171 | } 172 | case 'o': 173 | { 174 | if(sm->mask_mode == NO_MASKS){ 175 | printf("| o: Disparity error masks not provided for the chosen dataset.\n"); 176 | break; 177 | } 178 | sm->mask_mode = (sm->mask_mode == MASK_NONE ? MASK_NONOCC : 179 | sm->mask_mode == MASK_NONOCC ? MASK_DISC : 180 | MASK_NONE); 181 | printf("| o: Disparity error mask set to: %s |\n", sm->mask_mode == MASK_NONE ? "None" : 182 | sm->mask_mode == MASK_NONOCC ? "Nonocc" : 183 | "Disc"); 184 | break; 185 | } 186 | case 's': 187 | { 188 | sm->subsample_rate *= 2; 189 | if(sm->subsample_rate > 8) 190 | sm->subsample_rate = 2; 191 | printf("| =: Subsample rate changed to %d.\n", sm->subsample_rate); 192 | break; 193 | } 194 | } 195 | key = waitKey(1); 196 | } 197 | return; 198 | } 199 | --------------------------------------------------------------------------------