├── CMakeLists.txt ├── CMakeModules ├── FindEigen3.cmake ├── FindProtobuf.cmake └── FindTensorFlow.cmake ├── LICENSE ├── README.md ├── Tensorflow_build_instructions └── README.md ├── freeze_graph ├── README.md ├── freeze_graph.py └── monodepth.py ├── include └── monodepth │ ├── monodepth.h │ └── my_utils.h ├── model └── README.md ├── preview ├── dvso_final_map.gif ├── dvso_kitti_preview.gif ├── kitti_preview.gif ├── monodepth_preview.gif └── robotcar_preview.gif ├── run_tf_docker.sh ├── src ├── inference_monodepth.cpp └── monodepth.cpp └── tf.Dockerfile /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | SET(PROJECT_NAME monodepth) 2 | PROJECT(${PROJECT_NAME}) 3 | CMAKE_MINIMUM_REQUIRED (VERSION 2.8.3) 4 | SET(CMAKE_BUILD_TYPE Release) # Release, RelWithDebInfo 5 | SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/CMakeModules/") 6 | SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC") 7 | 8 | # Find package 9 | FIND_PACKAGE(OpenCV REQUIRED) 10 | FIND_PACKAGE(Eigen3 REQUIRED) 11 | 12 | # Find package for TensofFlow 13 | FIND_PACKAGE(Protobuf REQUIRED) 14 | FIND_PACKAGE(TensorFlow REQUIRED) 15 | 16 | INCLUDE_DIRECTORIES( 17 | include 18 | ${EIGEN3_INCLUDE_DIR} 19 | ${OpenCV_INCLUDE_DIRS} 20 | ${PROTOBUF_INCLUDE_DIRS} 21 | ${TensorFlow_INCLUDE_DIRS} 22 | ) 23 | 24 | # Create monodepth library 25 | ADD_LIBRARY(${PROJECT_NAME}_static STATIC src/monodepth.cpp) 26 | ADD_LIBRARY(${PROJECT_NAME} SHARED src/monodepth.cpp) 27 | # Link the libraries to the library 28 | TARGET_LINK_LIBRARIES(${PROJECT_NAME}_static 29 | ${OpenCV_LIBS} 30 | ${PROTOBUF_LIBRARIES} 31 | ${TensorFlow_LIBRARIES} 32 | ) 33 | TARGET_LINK_LIBRARIES(${PROJECT_NAME} 34 | ${OpenCV_LIBS} 35 | ${PROTOBUF_LIBRARIES} 36 | ${TensorFlow_LIBRARIES} 37 | ) 38 | 39 | 40 | # Make the static and shared libraries coexist in the output folder 41 | SET_TARGET_PROPERTIES(${PROJECT_NAME}_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) 42 | SET_TARGET_PROPERTIES(${PROJECT_NAME} PROPERTIES CLEAN_DIRECT_OUTPUT 1) 43 | 44 | # Make static library available for client 45 | # ('ARCHIVE' keyword required when building static library) 46 | INSTALL(TARGETS ${PROJECT_NAME}_static ${PROJECT_NAME} 47 | LIBRARY DESTINATION lib 48 | ARCHIVE DESTINATION lib) 49 | INSTALL(FILES include/monodepth/monodepth.h DESTINATION include/monodepth) 50 | 51 | # Create Executables (Test only) 52 | ADD_EXECUTABLE(inference_monodepth src/inference_monodepth.cpp) 53 | TARGET_LINK_LIBRARIES(inference_monodepth ${PROJECT_NAME}) 54 | -------------------------------------------------------------------------------- /CMakeModules/FindEigen3.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find Eigen3 lib 2 | # 3 | # This module supports requiring a minimum version, e.g. you can do 4 | # find_package(Eigen3 3.1.2) 5 | # to require version 3.1.2 or newer of Eigen3. 6 | # 7 | # Once done this will define 8 | # 9 | # EIGEN3_FOUND - system has eigen lib with correct version 10 | # EIGEN3_INCLUDE_DIR - the eigen include directory 11 | # EIGEN3_VERSION - eigen version 12 | # 13 | # This module reads hints about search locations from 14 | # the following enviroment variables: 15 | # 16 | # EIGEN3_ROOT 17 | # EIGEN3_ROOT_DIR 18 | 19 | # Copyright (c) 2006, 2007 Montel Laurent, 20 | # Copyright (c) 2008, 2009 Gael Guennebaud, 21 | # Copyright (c) 2009 Benoit Jacob 22 | # Redistribution and use is allowed according to the terms of the 2-clause BSD license. 23 | 24 | if(NOT Eigen3_FIND_VERSION) 25 | if(NOT Eigen3_FIND_VERSION_MAJOR) 26 | set(Eigen3_FIND_VERSION_MAJOR 2) 27 | endif(NOT Eigen3_FIND_VERSION_MAJOR) 28 | if(NOT Eigen3_FIND_VERSION_MINOR) 29 | set(Eigen3_FIND_VERSION_MINOR 91) 30 | endif(NOT Eigen3_FIND_VERSION_MINOR) 31 | if(NOT Eigen3_FIND_VERSION_PATCH) 32 | set(Eigen3_FIND_VERSION_PATCH 0) 33 | endif(NOT Eigen3_FIND_VERSION_PATCH) 34 | 35 | set(Eigen3_FIND_VERSION "${Eigen3_FIND_VERSION_MAJOR}.${Eigen3_FIND_VERSION_MINOR}.${Eigen3_FIND_VERSION_PATCH}") 36 | endif(NOT Eigen3_FIND_VERSION) 37 | 38 | macro(_eigen3_check_version) 39 | file(READ "${EIGEN3_INCLUDE_DIR}/Eigen/src/Core/util/Macros.h" _eigen3_version_header) 40 | 41 | string(REGEX MATCH "define[ \t]+EIGEN_WORLD_VERSION[ \t]+([0-9]+)" _eigen3_world_version_match "${_eigen3_version_header}") 42 | set(EIGEN3_WORLD_VERSION "${CMAKE_MATCH_1}") 43 | string(REGEX MATCH "define[ \t]+EIGEN_MAJOR_VERSION[ \t]+([0-9]+)" _eigen3_major_version_match "${_eigen3_version_header}") 44 | set(EIGEN3_MAJOR_VERSION "${CMAKE_MATCH_1}") 45 | string(REGEX MATCH "define[ \t]+EIGEN_MINOR_VERSION[ \t]+([0-9]+)" _eigen3_minor_version_match "${_eigen3_version_header}") 46 | set(EIGEN3_MINOR_VERSION "${CMAKE_MATCH_1}") 47 | 48 | set(EIGEN3_VERSION ${EIGEN3_WORLD_VERSION}.${EIGEN3_MAJOR_VERSION}.${EIGEN3_MINOR_VERSION}) 49 | if(${EIGEN3_VERSION} VERSION_LESS ${Eigen3_FIND_VERSION}) 50 | set(EIGEN3_VERSION_OK FALSE) 51 | else(${EIGEN3_VERSION} VERSION_LESS ${Eigen3_FIND_VERSION}) 52 | set(EIGEN3_VERSION_OK TRUE) 53 | endif(${EIGEN3_VERSION} VERSION_LESS ${Eigen3_FIND_VERSION}) 54 | 55 | if(NOT EIGEN3_VERSION_OK) 56 | 57 | message(STATUS "Eigen3 version ${EIGEN3_VERSION} found in ${EIGEN3_INCLUDE_DIR}, " 58 | "but at least version ${Eigen3_FIND_VERSION} is required") 59 | endif(NOT EIGEN3_VERSION_OK) 60 | endmacro(_eigen3_check_version) 61 | 62 | if (EIGEN3_INCLUDE_DIR) 63 | 64 | # in cache already 65 | _eigen3_check_version() 66 | set(EIGEN3_FOUND ${EIGEN3_VERSION_OK}) 67 | 68 | else (EIGEN3_INCLUDE_DIR) 69 | 70 | # search first if an Eigen3Config.cmake is available in the system, 71 | # if successful this would set EIGEN3_INCLUDE_DIR and the rest of 72 | # the script will work as usual 73 | find_package(Eigen3 ${Eigen3_FIND_VERSION} NO_MODULE QUIET) 74 | 75 | if(NOT EIGEN3_INCLUDE_DIR) 76 | find_path(EIGEN3_INCLUDE_DIR NAMES signature_of_eigen3_matrix_library 77 | HINTS 78 | ENV EIGEN3_ROOT 79 | ENV EIGEN3_ROOT_DIR 80 | PATHS 81 | ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/eigen3-tf 82 | ${CMAKE_CURRENT_SOURCE_DIR}/third_party/eigen3-tf 83 | ${CMAKE_INSTALL_PREFIX}/include 84 | ${KDE4_INCLUDE_DIR} 85 | /eigen3-tf 86 | PATH_SUFFIXES eigen3 eigen 87 | ) 88 | endif(NOT EIGEN3_INCLUDE_DIR) 89 | 90 | if(EIGEN3_INCLUDE_DIR) 91 | _eigen3_check_version() 92 | endif(EIGEN3_INCLUDE_DIR) 93 | 94 | include(FindPackageHandleStandardArgs) 95 | find_package_handle_standard_args(Eigen3 DEFAULT_MSG EIGEN3_INCLUDE_DIR EIGEN3_VERSION_OK) 96 | 97 | mark_as_advanced(EIGEN3_INCLUDE_DIR) 98 | 99 | endif(EIGEN3_INCLUDE_DIR) 100 | -------------------------------------------------------------------------------- /CMakeModules/FindProtobuf.cmake: -------------------------------------------------------------------------------- 1 | include(FindPackageHandleStandardArgs) 2 | # include(Protobuf_VERSION) 3 | unset(PROTOBUF_FOUND) 4 | 5 | find_path(Protobuf_INCLUDE_DIR 6 | NAMES 7 | protobuf 8 | HINTS 9 | ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf/install/include/google 10 | ${CMAKE_CURRENT_SOURCE_DIR}/third_party/protobuf/install/include/google 11 | /usr/include/google) 12 | 13 | find_library(Protobuf_LIBRARY NAMES protobuf 14 | HINTS 15 | /usr/lib) 16 | 17 | # set Protobuf_FOUND 18 | find_package_handle_standard_args(Protobuf DEFAULT_MSG Protobuf_INCLUDE_DIR Protobuf_LIBRARY) 19 | 20 | # set external variables for usage in CMakeLists.txt 21 | if(PROTOBUF_FOUND) 22 | set(PROTOBUF_LIBRARIES ${Protobuf_LIBRARY}) 23 | set(PROTOBUF_INCLUDE_DIRS ${Protobuf_INCLUDE_DIR}) 24 | message(STATUS "Protobuf found (include: ${PROTOBUF_INCLUDE_DIRS})") 25 | message(STATUS "Protobuf found (library: ${PROTOBUF_LIBRARIES})") 26 | endif() 27 | 28 | # hide locals from GUI 29 | # mark_as_advanced(Protobuf_INCLUDE_DIR) 30 | -------------------------------------------------------------------------------- /CMakeModules/FindTensorFlow.cmake: -------------------------------------------------------------------------------- 1 | # Locates the tensorFlow library and include directories. 2 | 3 | include(FindPackageHandleStandardArgs) 4 | 5 | find_path(TensorFlow_base 6 | NAMES 7 | tensorflow 8 | third_party 9 | external 10 | HINTS 11 | /usr/local/include/google/tensorflow 12 | ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/tensorflow/local/include/google/tensorflow 13 | ${CMAKE_CURRENT_SOURCE_DIR}/third_party/tensorflow/local/include/google/tensorflow) 14 | 15 | list(APPEND TensorFlow_INCLUDE_DIR ${TensorFlow_base}) 16 | list(APPEND TensorFlow_INCLUDE_DIR ${TensorFlow_base}/external/nsync/public) 17 | 18 | find_library(TensorFlow_LIBRARIES NAMES tensorflow_cc 19 | HINTS 20 | /usr/local/lib) 21 | 22 | # set TensorFlow_FOUND 23 | find_package_handle_standard_args(TensorFlow DEFAULT_MSG TensorFlow_INCLUDE_DIR TensorFlow_LIBRARIES) 24 | 25 | # set external variables for usage in CMakeLists.txt 26 | if(TENSORFLOW_FOUND) 27 | set(TensorFlow_LIBRARIES ${TensorFlow_LIBRARIES}) 28 | set(TensorFlow_INCLUDE_DIRS ${TensorFlow_INCLUDE_DIR}) 29 | message(STATUS "TensorFlow found (include: ${TensorFlow_INCLUDE_DIRS})") 30 | message(STATUS "TensorFlow found (lib: ${TensorFlow_LIBRARIES})") 31 | 32 | endif() 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Shing Yan Loo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # monodepth-cpp 2 | Tensorflow C++ implementation for single image depth estimation 3 |

4 | 5 |

6 | 7 | The original work is implemented in Python, [click here](https://github.com/mrharicot/monodepth) to go to their repo. Please cite their work if your find it helpful. 8 | 9 | It is using pointer-to-implementation technique, so that you can use it in your project without worrying the actual implementation. Refer to [src/inference_monodepth.cpp](https://github.com/yan99033/monodepth-cpp/tree/master/src/inference_monodepth.cpp) for more information 10 | 11 | The C++ version is about 28fps, in comparison with Python's 13fps, tested with i7 processor and NVidia 1070 graphics laptop. 12 | 13 | ## Related repo 14 | If you are looking for single-image relative depth prediction, feel free to check out my other repo, [MiDaS-cpp](https://github.com/yan99033/MiDaS-cpp), a PyTorch C++ implementation of MiDaS. 15 | 16 | 17 | ## Requirements 18 | ### Option 1 19 | Run the Docker container 20 | ``` 21 | chmod +x run_tf_docker.sh 22 | ./run_tf_docker.sh 23 | ``` 24 | Note: You may have a better graphics card, and if that's so, go to [tf.Dockerfile](https://github.com/yan99033/monodepth-cpp/tree/master/tf.Dockerfile) line 55 and change the value of `TF_CUDA_COMPUTE_CAPABILITIES` (check out [Nvidia cuda-gpus](https://developer.nvidia.com/cuda-gpus) ). 25 | ### Option 2 26 | Build Tensorflow library from source, [see here](https://github.com/yan99033/monodepth-cpp/tree/master/Tensorflow_build_instructions) 27 | 28 | ### Freeze TF graph 29 | Either way, you will need to [Freeze a Tensorflow graph](https://github.com/yan99033/monodepth-cpp/tree/master/freeze_graph) (with known input and output names), or download a converted graph [here](https://github.com/yan99033/monodepth-cpp/tree/master/model). 30 | 31 | 32 | ## Configure CMake project 33 | * Make sure you have the compatible versions of Eigen, Protobuf, and Tensorflow (Mine: Eigen 3.3.4; Protobuf 2.6.1-1.3; Tensorflow 1.6) 34 | * If you have a set of different paths to the libraries, modify the `.cmake` files in [CMakeModules](https://github.com/yan99033/monodepth-cpp/tree/master/CMakeModules) 35 | 36 | 37 | ## Build (static/shared) library 38 | **NOTE:** If you use the Docker container, this github repo will be mounted in `/monodepth`. Therefore, run `cd monodepth` in the container first. 39 | 40 | ``` 41 | mkdir build && mkdir install 42 | cd build 43 | cmake -DCMAKE_INSTALL_PREFIX=../install .. 44 | make && make install 45 | ``` 46 | 47 | You will be seeing 'include' and 'lib' folders in the 'install' folder, import them in your project 48 | 49 | To test if Monodepth C++ is working properly, 50 | ``` 51 | cd build 52 | ./inference_monodepth 53 | ``` 54 | 55 | **NOTE:** Select either static or shared library in CMakeLists.txt, unless you want both of them 56 | 57 | ## Use (static/shared) library 58 | In your own CMakeLists.txt project, do the following: 59 | 60 | ``` 61 | set(monodepth_INCLUDE_DIRS /path/to/monodepth-cpp/install/include) 62 | INCLUDE_DIRECTORIES( 63 | ... 64 | monodepth_INCLUDE_DIRS 65 | ... 66 | ) 67 | 68 | TARGET_LINK_LIBRARIES(awesome_exe /path/to/tensorflow/library/libtensorflow_cc.so) # Only if you are using the provided instructions 69 | TARGET_LINK_LIBRARIES(awesome_exe /path/to/monodepth-cpp/install/lib/libmonodepth_static.a) # if you are using static library 70 | 71 | ``` 72 | 73 | ## Personal projects that use monodepth-cpp 74 | 1. [CNN-SVO](https://github.com/yan99033/CNN-SVO) 75 |

76 | 77 | 78 |

79 | 80 | 2. Reproducing [Deep virtual stereo odometry](https://vision.in.tum.de/research/vslam/dvso) 81 | 82 |

83 | 84 | 85 |

86 | 87 | **NOTE:** Because DVSO uses both the left disparity and the right disparity outputs (for left-right consistency check), it requires some modifications in the source code to enable the disparities outputs. 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /Tensorflow_build_instructions/README.md: -------------------------------------------------------------------------------- 1 | # Build Tensorflow C++ library from source to local folder 2 | I use Tensorflow 1.6, CUDA 9.0, and cuDNN 7.6. 3 | (Similar instructions can be found in the Dockerfile) 4 | 5 | Most of the libraries should not be installed in system directory as they are dated. 6 | 7 | If you already have the library, feel free to use your own library 8 | 9 | ## Instructions: 10 | 11 | Create a folder named `third_party` 12 | ``` 13 | mkdir third_party 14 | cd third_party 15 | ``` 16 | ### Install dependencies 17 | 18 | ``` 19 | apt install autoconf automake libtool curl make g++ unzip (Protobuf Dependencies) 20 | apt install python-numpy swig python-dev python-wheel (TensorFlow Dependencies) 21 | apt install wget openjdk-8-jdk 22 | ``` 23 | 24 | Bazel (using a different version might result in build error) 25 | ``` 26 | curl -LO "https://github.com/bazelbuild/bazel/releases/download/0.11.1/bazel_0.11.1-linux-x86_64.deb" 27 | dpkg -i bazel_*.deb && rm -rf bazel_*.deb 28 | ``` 29 | 30 | Protobuf 31 | ``` 32 | wget https://github.com/google/protobuf/archive/396336eb961b75f03b25824fe86cf6490fb75e3a.tar.gz 33 | tar -xvf 396336eb961b75f03b25824fe86cf6490fb75e3a.tar.gz 34 | mv protobuf-396336eb961b75f03b25824fe86cf6490fb75e3a protobuf 35 | cd protobuf 36 | mkdir install 37 | ./autogen.sh 38 | ./configure --prefix=install 39 | make install 40 | cd .. 41 | rm -rf 396336eb961b75f03b25824fe86cf6490fb75e3a.tar.gz 42 | ``` 43 | 44 | Eigen 45 | ``` 46 | wget https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/14e1418fcf12.tar.gz 47 | tar -xvf 14e1418fcf12.tar.gz 48 | mv eigen-eigen-14e1418fcf12 eigen3-tf 49 | rm -rf 14e1418fcf12.tar.gz 50 | ``` 51 | 52 | ### Clone Tensorflow source 53 | 54 | ``` 55 | git clone https://github.com/tensorflow/tensorflow 56 | cd tensorflow 57 | git checkout v1.6.0 58 | ``` 59 | 60 | ### Build the shared library (locally) 61 | 62 | ``` 63 | ./configure (I use default settings) 64 | bazel build --config=opt --config=cuda --config=monolithic tensorflow:libtensorflow_cc.so 65 | ``` 66 | 67 | ### Organize Tensorflow library in local folder 68 | ``` 69 | mkdir -p local/lib 70 | mkdir -p local/include/google/tensorflow 71 | cp -r bazel-genfiles/* local/include/google/tensorflow/ 72 | cp -r tensorflow local/include/google/tensorflow/ 73 | find local/include/google/tensorflow -type f ! -name "*.h" -delete 74 | cp -r third_party local/include/google/tensorflow/ 75 | cp bazel-bin/tensorflow/libtensorflow_cc.so local/lib 76 | mkdir -p local/include/google/tensorflow/external/nsync/public 77 | cp -r bazel-tensorflow/external/nsync/public/* local/include/google/tensorflow/external/nsync/public 78 | ``` 79 | 80 | ## Note: 81 | * Before raising any issues, be sure to checkout the **Issues** in the referenced Github repos. You should be able to identify similar issue faced by others. 82 | 83 | ## Reference: 84 | [Official Tensorflow guide](https://www.tensorflow.org/install/install_sources) 85 | 86 | [Tensorflow-cmake](https://github.com/cjweeks/tensorflow-cmake) 87 | 88 | 89 | -------------------------------------------------------------------------------- /freeze_graph/README.md: -------------------------------------------------------------------------------- 1 | # Convert a Tensorflow checkpoint file to a frozen graph 2 | The Monodepth model is saved as a checkpoint file (.ckpt), we need to convert it to a graph file (.pb) so that the pre-trained model can be used 3 | 4 | # Run the freeze_graph.py 5 | python freeze_graph.py --encoder resnet --ckpt_file /path/to/trained/model --output_dir /path/to/output/folder 6 | 7 | Note: 8 | 9 | * `--encoder resnet OR vgg` 10 | * There is no extension (e.g., .ckpt) for the checkpoint file 11 | * change the file name of the output graph using `--graph output.pb` 12 | 13 | # Download the pre-trained frozen graph 14 | [VGG model](https://drive.google.com/open?id=1yzcndbigENP3kQg6Oioerwvkf_hTotZZ) 15 | 16 | [Resnet50 model](https://drive.google.com/open?id=1SFd-FBGWwWHl1n6coIQV_EWhXUDvlWsk) 17 | 18 | Note: 19 | * VGG model: the pre-trained city2kitti model provided by Monodepth author 20 | * Resnet50 model: city2kitti excluding odometry sequence 00-10 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /freeze_graph/freeze_graph.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file is part of monodepth-cpp 3 | 4 | MIT License 5 | 6 | Copyright (c) 2018 Shing Yan Loo (lsyan@ualberta.ca) 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining a copy 9 | of this software and associated documentation files (the "Software"), to deal 10 | in the Software without restriction, including without limitation the rights 11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | copies of the Software, and to permit persons to whom the Software is 13 | furnished to do so, subject to the following conditions: 14 | 15 | The above copyright notice and this permission notice shall be included in all 16 | copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | """ 26 | 27 | ''' Convert a checkpoint file to a frozen grapth 28 | 1. You can either download a pre-trained model (https://github.com/mrharicot/monodepth) or train a model from scratch 29 | 30 | 2. To make sure the frozen graph has the proper output name, define the 'network output' in build_outputs() method in monodepth_model.py 31 | e.g., out = tf.expand_dims(self.disp1[:, :, :, 0], 3, name='output_depth') 32 | ''' 33 | 34 | import tensorflow as tf 35 | from monodepth import * 36 | 37 | # Arguments 38 | args = tf.app.flags 39 | args.DEFINE_integer('batch_size', 2, 'The size of of a sample batch') 40 | args.DEFINE_string('encoder', 'vgg', 'vgg or resnet') 41 | args.DEFINE_integer('img_height', 256, 'Image height') 42 | args.DEFINE_integer('img_width', 512, 'Image width') 43 | args.DEFINE_string('ckpt_file', '/path/to/monodepth/model/model_city2kitti', 'checkpoint file') 44 | args.DEFINE_string('output_dir', '/path/to/frozen/graph/model', 'the folder that stores the graph') 45 | args.DEFINE_string('graph', 'model_city2kitti.pb', 'the graph file') 46 | arg = args.FLAGS 47 | 48 | # Image placeholder 49 | x = tf.placeholder(tf.float32, shape=[arg.batch_size, arg.img_height, arg.img_width, 3], name='input_image') 50 | 51 | # Load model and get output (disparity) 52 | model = monodepth(x, arg.encoder) 53 | y = model.get_output() 54 | 55 | # add pb extension if not present 56 | if not arg.graph.endswith(".pb"): 57 | arg.graph = arg.graph + ".pb" 58 | 59 | # initialise the saver 60 | saver = tf.train.Saver() 61 | 62 | # SESSION 63 | config = tf.ConfigProto(allow_soft_placement=True) 64 | sess = tf.Session(config=config) 65 | 66 | # Initialize variables 67 | sess.run(tf.global_variables_initializer()) 68 | sess.run(tf.local_variables_initializer()) 69 | 70 | # SESSION 71 | config = tf.ConfigProto(allow_soft_placement=True) 72 | sess = tf.Session(config=config) 73 | 74 | # restore all variables from checkpoint 75 | saver.restore(sess, arg.ckpt_file) 76 | 77 | # node that are required output nodes 78 | output_node_names = ["model/output_depth"] 79 | 80 | # We use a built-in TF helper to export variables to constants 81 | output_graph_def = tf.graph_util.convert_variables_to_constants( 82 | sess, 83 | tf.get_default_graph().as_graph_def(), 84 | # The graph_def is used to retrieve the nodes 85 | output_node_names # The output node names are used to select the useful nodes 86 | ) 87 | 88 | # convert variables to constants 89 | output_graph_def = tf.graph_util.remove_training_nodes(output_graph_def) 90 | 91 | # Finally we serialize and dump the output graph to the filesystem 92 | output_graph = arg.output_dir + '/' + arg.graph 93 | with tf.gfile.GFile(output_graph, "wb") as f: 94 | f.write(output_graph_def.SerializeToString()) 95 | 96 | print("Frozen graph file {} created successfully".format(arg.graph)) 97 | -------------------------------------------------------------------------------- /freeze_graph/monodepth.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file is part of monodepth-cpp 3 | 4 | MIT License 5 | 6 | Copyright (c) 2018 Shing Yan Loo (lsyan@ualberta.ca) 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining a copy 9 | of this software and associated documentation files (the "Software"), to deal 10 | in the Software without restriction, including without limitation the rights 11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | copies of the Software, and to permit persons to whom the Software is 13 | furnished to do so, subject to the following conditions: 14 | 15 | The above copyright notice and this permission notice shall be included in all 16 | copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | """ 26 | 27 | 28 | import numpy as np 29 | import tensorflow as tf 30 | import tensorflow.contrib.slim as slim 31 | import math 32 | 33 | 34 | class monodepth(object): 35 | """monodepth model""" 36 | 37 | def __init__(self, x, model): 38 | self.use_deconv = False 39 | 40 | self.output = self.build_model(x, model) 41 | 42 | def get_output(self): 43 | return self.output 44 | 45 | def scale_pyramid(self, img, num_scales): 46 | scaled_imgs = [img] 47 | s = tf.shape(img) # img.get_shape().as_list() 48 | h = s[1] 49 | w = s[2] 50 | for i in range(num_scales - 1): 51 | ratio = 2 ** (i + 1) 52 | nh = h / ratio 53 | nw = w / ratio 54 | scaled_imgs.append(tf.image.resize_area(img, [math.ceil(nh), math.ceil(nw)])) 55 | return scaled_imgs 56 | 57 | def conv(self, x, num_out_layers, kernel_size, stride, activation_fn=tf.nn.elu): 58 | p = np.floor((kernel_size - 1) / 2).astype(np.int32) 59 | p_x = tf.pad(x, [[0, 0], [p, p], [p, p], [0, 0]]) 60 | return slim.conv2d(p_x, num_out_layers, kernel_size, stride, 'VALID', activation_fn=activation_fn) 61 | 62 | def conv_block(self, x, num_out_layers, kernel_size): 63 | conv1 = self.conv(x, num_out_layers, kernel_size, 1) 64 | conv2 = self.conv(conv1, num_out_layers, kernel_size, 2) 65 | return conv2 66 | 67 | def upconv(self, x, num_out_layers, kernel_size, scale): 68 | upsample = self.upsample_nn(x, scale) 69 | conv = self.conv(upsample, num_out_layers, kernel_size, 1) 70 | return conv 71 | 72 | def deconv(self, x, num_out_layers, kernel_size, scale): 73 | p_x = tf.pad(x, [[0, 0], [1, 1], [1, 1], [0, 0]]) 74 | conv = slim.conv2d_transpose(p_x, num_out_layers, kernel_size, scale, 'SAME') 75 | return conv[:,3:-1,3:-1,:] 76 | 77 | def get_disp(self, x): 78 | disp = 0.3 * self.conv(x, 2, 3, 1, tf.nn.sigmoid) 79 | return disp 80 | 81 | def upsample_nn(self, x, ratio): 82 | s = tf.shape(x) 83 | h = s[1] 84 | w = s[2] 85 | return tf.image.resize_nearest_neighbor(x, [h * ratio, w * ratio]) 86 | 87 | def maxpool(self, x, kernel_size): 88 | p = np.floor((kernel_size - 1) / 2).astype(np.int32) 89 | p_x = tf.pad(x, [[0, 0], [p, p], [p, p], [0, 0]]) 90 | return slim.max_pool2d(p_x, kernel_size) 91 | 92 | def resconv(self, x, num_layers, stride): 93 | do_proj = tf.shape(x)[3] != num_layers or stride == 2 94 | shortcut = [] 95 | conv1 = self.conv(x, num_layers, 1, 1) 96 | conv2 = self.conv(conv1, num_layers, 3, stride) 97 | conv3 = self.conv(conv2, 4 * num_layers, 1, 1, None) 98 | if do_proj: 99 | shortcut = self.conv(x, 4 * num_layers, 1, stride, None) 100 | else: 101 | shortcut = x 102 | return tf.nn.elu(conv3 + shortcut) 103 | 104 | def resblock(self, x, num_layers, num_blocks): 105 | out = x 106 | for i in range(num_blocks - 1): 107 | out = self.resconv(out, num_layers, 1) 108 | out = self.resconv(out, num_layers, 2) 109 | return out 110 | 111 | def build_resnet50(self, x): 112 | # set convenience functions 113 | conv = self.conv 114 | if self.use_deconv: 115 | upconv = self.deconv 116 | else: 117 | upconv = self.upconv 118 | 119 | with tf.variable_scope('encoder'): 120 | conv1 = conv(x, 64, 7, 2) # H/2 - 64D 121 | pool1 = self.maxpool(conv1, 3) # H/4 - 64D 122 | conv2 = self.resblock(pool1, 64, 3) # H/8 - 256D 123 | conv3 = self.resblock(conv2, 128, 4) # H/16 - 512D 124 | conv4 = self.resblock(conv3, 256, 6) # H/32 - 1024D 125 | conv5 = self.resblock(conv4, 512, 3) # H/64 - 2048D 126 | 127 | with tf.variable_scope('skips'): 128 | skip1 = conv1 129 | skip2 = pool1 130 | skip3 = conv2 131 | skip4 = conv3 132 | skip5 = conv4 133 | 134 | # DECODING 135 | with tf.variable_scope('decoder'): 136 | upconv6 = upconv(conv5, 512, 3, 2) # H/32 137 | concat6 = tf.concat([upconv6, skip5], 3) 138 | iconv6 = conv(concat6, 512, 3, 1) 139 | 140 | upconv5 = upconv(iconv6, 256, 3, 2) # H/16 141 | concat5 = tf.concat([upconv5, skip4], 3) 142 | iconv5 = conv(concat5, 256, 3, 1) 143 | 144 | upconv4 = upconv(iconv5, 128, 3, 2) # H/8 145 | concat4 = tf.concat([upconv4, skip3], 3) 146 | iconv4 = conv(concat4, 128, 3, 1) 147 | disp4 = self.get_disp(iconv4) 148 | udisp4 = self.upsample_nn(disp4, 2) 149 | 150 | upconv3 = upconv(iconv4, 64, 3, 2) # H/4 151 | concat3 = tf.concat([upconv3, skip2, udisp4], 3) 152 | iconv3 = conv(concat3, 64, 3, 1) 153 | disp3 = self.get_disp(iconv3) 154 | udisp3 = self.upsample_nn(disp3, 2) 155 | 156 | upconv2 = upconv(iconv3, 32, 3, 2) # H/2 157 | concat2 = tf.concat([upconv2, skip1, udisp3], 3) 158 | iconv2 = conv(concat2, 32, 3, 1) 159 | disp2 = self.get_disp(iconv2) 160 | udisp2 = self.upsample_nn(disp2, 2) 161 | 162 | upconv1 = upconv(iconv2, 16, 3, 2) # H 163 | concat1 = tf.concat([upconv1, udisp2], 3) 164 | iconv1 = conv(concat1, 16, 3, 1) 165 | disp1 = self.get_disp(iconv1) 166 | 167 | # Set name for output 168 | out = tf.expand_dims(disp1[:, :, :, 0], 3, name='output_depth') 169 | 170 | return out 171 | 172 | def build_vgg(self, x): 173 | # set convenience functions 174 | conv = self.conv 175 | if self.use_deconv: 176 | upconv = self.deconv 177 | else: 178 | upconv = self.upconv 179 | 180 | with tf.variable_scope('encoder'): 181 | conv1 = self.conv_block(x, 32, 7) # H/2 182 | conv2 = self.conv_block(conv1, 64, 5) # H/4 183 | conv3 = self.conv_block(conv2, 128, 3) # H/8 184 | conv4 = self.conv_block(conv3, 256, 3) # H/16 185 | conv5 = self.conv_block(conv4, 512, 3) # H/32 186 | conv6 = self.conv_block(conv5, 512, 3) # H/64 187 | conv7 = self.conv_block(conv6, 512, 3) # H/128 188 | # conv_block(x, num_out_layers, kernel_size): 189 | 190 | with tf.variable_scope('skips'): 191 | skip1 = conv1 192 | skip2 = conv2 193 | skip3 = conv3 194 | skip4 = conv4 195 | skip5 = conv5 196 | skip6 = conv6 197 | 198 | with tf.variable_scope('decoder'): 199 | upconv7 = upconv(conv7, 512, 3, 2) # H/64 200 | concat7 = tf.concat([upconv7, skip6], 3) 201 | iconv7 = conv(concat7, 512, 3, 1) 202 | 203 | upconv6 = upconv(iconv7, 512, 3, 2) # H/32 204 | concat6 = tf.concat([upconv6, skip5], 3) 205 | iconv6 = conv(concat6, 512, 3, 1) 206 | 207 | upconv5 = upconv(iconv6, 256, 3, 2) # H/16 208 | concat5 = tf.concat([upconv5, skip4], 3) 209 | iconv5 = conv(concat5, 256, 3, 1) 210 | 211 | upconv4 = upconv(iconv5, 128, 3, 2) # H/8 212 | concat4 = tf.concat([upconv4, skip3], 3) 213 | iconv4 = conv(concat4, 128, 3, 1) 214 | disp4 = self.get_disp(iconv4) 215 | udisp4 = self.upsample_nn(disp4, 2) 216 | 217 | upconv3 = upconv(iconv4, 64, 3, 2) # H/4 218 | concat3 = tf.concat([upconv3, skip2, udisp4], 3) 219 | iconv3 = conv(concat3, 64, 3, 1) 220 | disp3 = self.get_disp(iconv3) 221 | udisp3 = self.upsample_nn(disp3, 2) 222 | 223 | upconv2 = upconv(iconv3, 32, 3, 2) # H/2 224 | concat2 = tf.concat([upconv2, skip1, udisp3], 3) 225 | iconv2 = conv(concat2, 32, 3, 1) 226 | disp2 = self.get_disp(iconv2) 227 | udisp2 = self.upsample_nn(disp2, 2) 228 | 229 | upconv1 = upconv(iconv2, 16, 3, 2) # H 230 | concat1 = tf.concat([upconv1, udisp2], 3) 231 | iconv1 = conv(concat1, 16, 3, 1) 232 | disp1 = self.get_disp(iconv1) 233 | # batch_size, height, width, channels = disp1.get_shape().as_list() 234 | 235 | # Set name for output 236 | out = tf.expand_dims(disp1[:, :, :, 0], 3, name='output_depth') 237 | 238 | return out 239 | 240 | def build_model(self, x, model): 241 | with slim.arg_scope([slim.conv2d, slim.conv2d_transpose], activation_fn=tf.nn.elu): 242 | with tf.variable_scope('model', reuse=None): 243 | 244 | #build model 245 | if model == 'vgg': 246 | return self.build_vgg(x) 247 | elif model == 'resnet': 248 | return self.build_resnet50(x) 249 | else: 250 | return None 251 | -------------------------------------------------------------------------------- /include/monodepth/monodepth.h: -------------------------------------------------------------------------------- 1 | /* 2 | This file is part of monodepth-cpp 3 | 4 | MIT License 5 | 6 | Copyright (c) 2018 Shing Yan Loo (lsyan@ualberta.ca) 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining a copy 9 | of this software and associated documentation files (the "Software"), to deal 10 | in the Software without restriction, including without limitation the rights 11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | copies of the Software, and to permit persons to whom the Software is 13 | furnished to do so, subject to the following conditions: 14 | 15 | The above copyright notice and this permission notice shall be included in all 16 | copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | */ 26 | 27 | #ifndef MONODEPTH_H_ 28 | #define MONODEPTH_H_ 29 | 30 | #include 31 | #include 32 | 33 | namespace monodepth { 34 | 35 | class MonoDepth 36 | { 37 | public: 38 | // constructor 39 | MonoDepth(const int img_width, 40 | const int img_height, 41 | const std::string graph_str); 42 | 43 | // destructor 44 | ~MonoDepth(); 45 | 46 | bool inference(const cv::Mat& image, cv::Mat& depth); 47 | 48 | private: 49 | /// forward declare the implementation 50 | class impl; 51 | std::unique_ptr p_impl_; 52 | }; 53 | 54 | } // namespace monodepth 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /include/monodepth/my_utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | This file is part of monodepth-cpp 3 | 4 | MIT License 5 | 6 | Copyright (c) 2018 Shing Yan Loo (lsyan@ualberta.ca) 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining a copy 9 | of this software and associated documentation files (the "Software"), to deal 10 | in the Software without restriction, including without limitation the rights 11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | copies of the Software, and to permit persons to whom the Software is 13 | furnished to do so, subject to the following conditions: 14 | 15 | The above copyright notice and this permission notice shall be included in all 16 | copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | 26 | NOTE: 27 | This is an implementation of std::make_unique, which is not available to my compiler 28 | std::make_unique -> util::make_unique 29 | 30 | From: http://www.cplusplus.com/forum/beginner/154004/ 31 | */ 32 | 33 | #ifndef MAKE_UNIQUE_H_INCLUDED 34 | #define MAKE_UNIQUE_H_INCLUDED 35 | 36 | #include 37 | #include 38 | 39 | namespace util 40 | { 41 | #if __cplusplus == 201402L // C++14 42 | 43 | using std::make_unique ; 44 | 45 | #else // C++11 46 | 47 | template < typename T, typename... CONSTRUCTOR_ARGS > 48 | std::unique_ptr make_unique( CONSTRUCTOR_ARGS&&... constructor_args ) 49 | { return std::unique_ptr( new T( std::forward(constructor_args)... ) ); } 50 | 51 | #endif // __cplusplus == 201402L 52 | } 53 | 54 | 55 | #endif // MAKE_UNIQUE_H_INCLUDED 56 | -------------------------------------------------------------------------------- /model/README.md: -------------------------------------------------------------------------------- 1 | # Monodepth pre-trained model 2 | Because of size limit, you can download the converted graph [here](https://drive.google.com/open?id=1UdGwRp2Qo7Frl9cAjw-CDAYqLwxbk6IH) here to perform a quick test 3 | -------------------------------------------------------------------------------- /preview/dvso_final_map.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yan99033/monodepth-cpp/1ea2fc30b96deba4e4d094088e4dc9f4c9c7980a/preview/dvso_final_map.gif -------------------------------------------------------------------------------- /preview/dvso_kitti_preview.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yan99033/monodepth-cpp/1ea2fc30b96deba4e4d094088e4dc9f4c9c7980a/preview/dvso_kitti_preview.gif -------------------------------------------------------------------------------- /preview/kitti_preview.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yan99033/monodepth-cpp/1ea2fc30b96deba4e4d094088e4dc9f4c9c7980a/preview/kitti_preview.gif -------------------------------------------------------------------------------- /preview/monodepth_preview.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yan99033/monodepth-cpp/1ea2fc30b96deba4e4d094088e4dc9f4c9c7980a/preview/monodepth_preview.gif -------------------------------------------------------------------------------- /preview/robotcar_preview.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yan99033/monodepth-cpp/1ea2fc30b96deba4e4d094088e4dc9f4c9c7980a/preview/robotcar_preview.gif -------------------------------------------------------------------------------- /run_tf_docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker build \ 3 | --file=tf.Dockerfile \ 4 | --tag="tf:0.1" \ 5 | . 6 | 7 | # GUI visualization 8 | # https://answers.ros.org/question/300113/docker-how-to-use-rviz-and-gazebo-from-a-container/ 9 | XAUTH=/tmp/.docker.xauth 10 | 11 | echo "Preparing Xauthority data..." 12 | xauth_list=$(xauth nlist :0 | tail -n 1 | sed -e 's/^..../ffff/') 13 | if [ ! -f $XAUTH ]; then 14 | if [ ! -z "$xauth_list" ]; then 15 | echo $xauth_list | xauth -f $XAUTH nmerge - 16 | else 17 | touch $XAUTH 18 | fi 19 | chmod a+r $XAUTH 20 | fi 21 | 22 | echo "Done." 23 | echo "" 24 | echo "Verifying file contents:" 25 | file $XAUTH 26 | echo "--> It should say \"X11 Xauthority data\"." 27 | echo "" 28 | echo "Permissions:" 29 | ls -FAlh $XAUTH 30 | echo "" 31 | echo "Running docker..." 32 | 33 | # Create a container (to be destroyed upon exit) 34 | docker run \ 35 | --interactive \ 36 | --tty \ 37 | --rm \ 38 | --gpus all \ 39 | --privileged \ 40 | --net=host \ 41 | -e "DISPLAY=$DISPLAY" \ 42 | -v $XSOCK:$XSOCK -v $XAUTH:$XAUTH -e XAUTHORITY=$XAUTH \ 43 | --name=tf-container \ 44 | --mount type=bind,source="$(pwd)",target=/monodepth \ 45 | tf:0.1 46 | -------------------------------------------------------------------------------- /src/inference_monodepth.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | This file is part of monodepth-cpp 3 | 4 | MIT License 5 | 6 | Copyright (c) 2018 Shing Yan Loo (lsyan@ualberta.ca) 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining a copy 9 | of this software and associated documentation files (the "Software"), to deal 10 | in the Software without restriction, including without limitation the rights 11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | copies of the Software, and to permit persons to whom the Software is 13 | furnished to do so, subject to the following conditions: 14 | 15 | The above copyright notice and this permission notice shall be included in all 16 | copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | */ 26 | 27 | #include 28 | 29 | #include 30 | 31 | /* 32 | TODO: 33 | To test if your Monodepth frozen graph is working properly 34 | 1. create a folder with a bunch of images 35 | 2. Set the image height and width 36 | 3. set the proper path to your pre-trained model and images directory 37 | 4. define the number of images (in the beginning of for loop) 38 | */ 39 | 40 | int main() 41 | { 42 | int image_width = 1241; 43 | int image_height = 376; 44 | monodepth::MonoDepth m(image_width, image_height, "../model/model_city2kitti.pb"); 45 | cv::Mat depth; 46 | 47 | std::string img_path = "../data/"; 48 | 49 | int number_of_images = 5; 50 | 51 | for (int i=0; i 28 | #include 29 | 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | 40 | namespace monodepth { 41 | // remove the namespace reference 42 | using tensorflow::Flag; 43 | using tensorflow::Tensor; 44 | using tensorflow::Status; 45 | using tensorflow::string; 46 | using tensorflow::int32; 47 | 48 | class MonoDepth::impl 49 | { 50 | public: 51 | /// Constructor 52 | impl(const int img_width, 53 | const int img_height, 54 | const std::string graph_str); 55 | 56 | ~impl(); 57 | 58 | /// Infer depth from image (implementation) 59 | bool inference(const cv::Mat& image, cv::Mat& depth); 60 | 61 | private: 62 | /// Precompute masks for post processing 63 | void PrecomputeMasks(); 64 | 65 | /// Convert image to input tensor, estimate depthmap, and convert output tensor to depthmap 66 | Status PreprocessImage(const cv::Mat& image); 67 | Status PostprocessImage(const tensorflow::Tensor& depth); 68 | 69 | /// Input placeholder size 70 | static const int input_width_ = 512; 71 | static const int input_height_ = 256; 72 | 73 | int img_width_; /// Width of the input placeholder 74 | int img_height_; /// Height of the input placeholder 75 | cv::Size original_size_; /// Keep the original shape of the image (resize the image if necessary) 76 | tensorflow::Scope root; /// Scope of the variable 77 | cv::Mat mask1_; /// Precomputed mask for depth1 78 | cv::Mat mask2_; /// Precomputed mask for depth2 79 | tensorflow::Tensor images_; /// Stacked image1 and image2 tensor 80 | tensorflow::Tensor image1_; /// Allocated memory for image1 81 | cv::Mat cv_image1_; /// Shared memory of the image1 tensor 82 | float* image1_p_; /// Pointer to image1 tensor 83 | tensorflow::Tensor image2_; /// Allocated memory for image2 84 | cv::Mat cv_image2_; /// Shared memory of the image2 tensor 85 | float* image2_p_; /// Pointer to image2 tensor 86 | std::vector depth_; /// output from monodepth 87 | cv::Mat cv_depth_; /// Postprocessed depth as cv::Mat 88 | std::unique_ptr session_; /// Tensorflow Session 89 | tensorflow::SessionOptions session_options_; /// Tensorflow Session options 90 | tensorflow::GraphDef graph_def_; /// Monodepth computation graph 91 | std::string input_name_; /// Input placeholder 92 | std::string output_name_; /// Output placeholder 93 | }; 94 | 95 | // Constructor of the interface 96 | MonoDepth::MonoDepth(const int img_width, 97 | const int img_height, 98 | const std::string graph_str) : 99 | p_impl_{util::make_unique(img_width, img_height, graph_str)} {} 100 | 101 | // Constructor of the implementation 102 | MonoDepth::impl::impl(const int img_width, 103 | const int img_height, 104 | const std::string graph_str) : 105 | img_width_(img_width), 106 | img_height_(img_height), 107 | root(tensorflow::Scope::NewRootScope()) 108 | { 109 | // Store the original image size 110 | original_size_ = cv::Size(img_width, img_height); 111 | 112 | // Precompute mask 113 | mask1_ = cv::Mat(input_height_, input_width_, CV_32FC1); 114 | mask2_ = cv::Mat(input_height_, input_width_, CV_32FC1); 115 | PrecomputeMasks(); 116 | 117 | // Image placeholders 118 | images_ = tensorflow::Tensor(tensorflow::DT_FLOAT, tensorflow::TensorShape({2, input_height_, input_width_, 3})); 119 | image1_p_ = images_.flat().data(); 120 | image2_p_ = image1_p_ + input_width_ * input_height_ * 3; 121 | cv_image1_ = cv::Mat(input_height_, input_width_, CV_32FC3, image1_p_); 122 | cv_image2_ = cv::Mat(input_height_, input_width_, CV_32FC3, image2_p_); 123 | 124 | // Load graph 125 | TF_CHECK_OK(ReadBinaryProto(tensorflow::Env::Default(), graph_str, &graph_def_)); 126 | 127 | // Initialize session 128 | session_options_.config.mutable_gpu_options()->set_allow_growth(true); 129 | (&session_)->reset(tensorflow::NewSession(session_options_)); 130 | TF_CHECK_OK(session_->Create(graph_def_)); 131 | 132 | // Input and output 133 | input_name_ = "input_image"; 134 | output_name_ = "model/output_depth"; 135 | } 136 | 137 | // Destructor (interface) 138 | MonoDepth::~MonoDepth() = default; 139 | 140 | // Destructor(Implementation) 141 | MonoDepth::impl::~impl() 142 | { 143 | // Kill session 144 | session_->Close(); 145 | } 146 | 147 | //Link interface to implementation 148 | bool MonoDepth::inference(const cv::Mat& image, cv::Mat& depth) 149 | { 150 | return p_impl_->inference(image, depth); 151 | } 152 | 153 | void MonoDepth::impl::PrecomputeMasks() 154 | { 155 | std::vector row; 156 | float val = 0.0f; 157 | float step = 1.0f / (float)(input_width_ - 1); // assume that it is not divided by zero 158 | 159 | // Create a row 160 | for (size_t i=0; i> input = {std::make_pair(input_name_, images_)}; 192 | TF_CHECK_OK(session_->Run(input, {output_name_}, {}, &depth_ )); 193 | 194 | // Postprocess depthmap 195 | TF_CHECK_OK(PostprocessImage(depth_[0])); 196 | 197 | cv_depth_.copyTo(depth); 198 | 199 | return true; 200 | /* 201 | direct copy of image to tensor: https://github.com/tensorflow/tensorflow/issues/8033 202 | */ 203 | } 204 | 205 | Status MonoDepth::impl::PreprocessImage(const cv::Mat& image) 206 | { 207 | // Store the image size 208 | // original_size_ = image.size(); 209 | 210 | // Resize and flip image 211 | cv::Mat resized_img, resized_flipped_img; 212 | cv::resize(image, resized_img, cv::Size(input_width_, input_height_)); 213 | resized_img.convertTo(cv_image1_, CV_32FC3, 1/255.0); 214 | cv::flip(cv_image1_, cv_image2_, 1); 215 | 216 | return Status::OK(); 217 | } 218 | 219 | Status MonoDepth::impl::PostprocessImage(const tensorflow::Tensor& depth) 220 | { 221 | // Combine the two depthmaps (original and flipped) 222 | cv::Mat cv_depth2; 223 | float* depth1_p = depth_[0].flat().data(); 224 | float* depth1_flipped_p = depth1_p + input_width_ * input_height_; 225 | cv::Mat cv_depth1 = cv::Mat(input_height_, input_width_, CV_32FC1, depth1_p); 226 | cv::Mat cv_depth1_flipped = cv::Mat(input_height_, input_width_, CV_32FC1, depth1_flipped_p); 227 | cv::flip(cv_depth1_flipped, cv_depth2, 1); 228 | cv_depth_ = 0.5f * (cv_depth1 + cv_depth2); 229 | cv_depth_ = cv_depth_.mul(1.0f - mask1_ - mask2_) + mask2_.mul(cv_depth1) + mask1_.mul(cv_depth2); 230 | 231 | // Upsize the image to original Shape 232 | cv::resize(cv_depth_, cv_depth_, original_size_); 233 | 234 | return Status::OK(); 235 | } 236 | 237 | } // namespace monodepth 238 | -------------------------------------------------------------------------------- /tf.Dockerfile: -------------------------------------------------------------------------------- 1 | # Note: All of the links below are copied over from the source code. 2 | # The dependencies should be self-contained 3 | 4 | FROM nvidia/cudagl:9.0-devel-ubuntu16.04 5 | LABEL maintainer "NVIDIA CORPORATION " 6 | 7 | ENV CUDNN_VERSION 7.6.4.38 8 | 9 | LABEL com.nvidia.cudnn.version="${CUDNN_VERSION}" 10 | 11 | RUN apt-get update && apt-get install -y --no-install-recommends \ 12 | libcudnn7=$CUDNN_VERSION-1+cuda9.0 \ 13 | libcudnn7-dev=$CUDNN_VERSION-1+cuda9.0 \ 14 | && apt-mark hold libcudnn7 && \ 15 | rm -rf /var/lib/apt/lists/* 16 | 17 | # Add nvidia driver settings 18 | ENV NVIDIA_VISIBLE_DEVICES ${NVIDIA_VISIBLE_DEVICES:-all} 19 | ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES:+$NVIDIA_DRIVER_CAPABILITIES,}graphics 20 | 21 | RUN apt-get update && apt-get install -y --no-install-recommends \ 22 | build-essential curl git cmake unzip autoconf autogen automake \ 23 | libtool mlocate zlib1g-dev g++-6 python python-numpy python-dev \ 24 | python3-pip python3-wheel wget gzip openjdk-8-jdk make swig\ 25 | bash-completion python-wheel libopencv-dev && \ 26 | rm -rf /var/lib/apt/lists/* 27 | RUN updatedb 28 | 29 | # Bazel 30 | RUN curl -LO "https://github.com/bazelbuild/bazel/releases/download/0.11.1/bazel_0.11.1-linux-x86_64.deb" 31 | RUN dpkg -i bazel_*.deb && rm -rf bazel_*.deb 32 | 33 | # Protobuf 34 | RUN wget https://github.com/google/protobuf/archive/396336eb961b75f03b25824fe86cf6490fb75e3a.tar.gz && \ 35 | tar -xvf 396336eb961b75f03b25824fe86cf6490fb75e3a.tar.gz && \ 36 | cd protobuf-396336eb961b75f03b25824fe86cf6490fb75e3a && \ 37 | ./autogen.sh && \ 38 | ./configure --prefix=/usr && \ 39 | make install && \ 40 | cd .. && \ 41 | rm -rf protobuf-* && \ 42 | rm -rf 396336eb961b75f03b25824fe86cf6490fb75e3a.tar.gz 43 | 44 | # Eigen 45 | RUN wget https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/14e1418fcf12.tar.gz && \ 46 | tar -xvf 14e1418fcf12.tar.gz && \ 47 | mv eigen-eigen-14e1418fcf12 eigen3-tf && \ 48 | rm -rf 14e1418fcf12.tar.gz 49 | 50 | ENV CUDA_TOOLKIT_PATH=/usr/local/cuda-9.0 51 | ENV CUDNN_INSTALL_PATH=/usr/lib/x86_64-linux-gnu 52 | ENV TF_NEED_GCP=0 53 | ENV TF_NEED_CUDA=1 54 | ENV TF_CUDA_VERSION=9.0 55 | ENV TF_CUDA_COMPUTE_CAPABILITIES=6.1 56 | ENV TF_CUDNN_VERSION=7.6.4 57 | ENV CC_OPT_FLAGS="-march=native" 58 | ENV GCC_HOST_COMPILER_PATH=/usr/bin/gcc 59 | 60 | # Prompt for settings 61 | RUN git clone https://github.com/tensorflow/tensorflow.git && \ 62 | cd tensorflow && \ 63 | git checkout v1.6.0 && \ 64 | ./configure && \ 65 | bazel build --config=opt --config=cuda --config=monolithic //tensorflow:libtensorflow_cc.so && \ 66 | mkdir -p /usr/local/include/google/tensorflow && \ 67 | cp -r bazel-genfiles/* /usr/local/include/google/tensorflow/ && \ 68 | cp -r tensorflow /usr/local/include/google/tensorflow/ && \ 69 | find /usr/local/include/google/tensorflow -type f ! -name "*.h" -delete && \ 70 | cp -r third_party /usr/local/include/google/tensorflow/ && \ 71 | cp bazel-bin/tensorflow/libtensorflow_cc.so /usr/local/lib && \ 72 | mkdir -p /usr/local/include/google/tensorflow/external/nsync/public && \ 73 | cp -r bazel-tensorflow/external/nsync/public/* /usr/local/include/google/tensorflow/external/nsync/public 74 | 75 | WORKDIR / 76 | RUN rm -rf tensorflow 77 | --------------------------------------------------------------------------------