├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── Makefile ├── README.md ├── cmake ├── FindDataLoaders.cmake └── FindEasyPBR.cmake ├── config ├── eval.cfg ├── evaluate_confidence.cfg ├── explicit2render.cfg └── train.cfg ├── imgs └── teaser.png ├── include └── neural_mvs │ ├── NeuralMVSGUI.h │ ├── PyBridge.h │ ├── SFM.h │ └── TrainParams.h ├── neural_mvs_py ├── __pycache__ │ ├── meta_modules.cpython-36.pyc │ └── modules.cpython-36.pyc ├── callbacks │ ├── __pycache__ │ │ ├── callback.cpython-36.pyc │ │ ├── phase.cpython-36.pyc │ │ ├── scores.cpython-36.pyc │ │ ├── state_callback.cpython-36.pyc │ │ ├── tensorboard_callback.cpython-36.pyc │ │ ├── viewer_callback.cpython-36.pyc │ │ ├── vis.cpython-36.pyc │ │ └── visdom_callback.cpython-36.pyc │ ├── callback.py │ ├── phase.py │ ├── scores.py │ ├── state_callback.py │ ├── tensorboard_callback.py │ ├── viewer_callback.py │ ├── vis.py │ └── visdom_callback.py ├── misc │ ├── __pycache__ │ │ └── gradcheck_custom.cpython-36.pyc │ ├── grad_check.py │ └── gradcheck_custom.py ├── neural_mvs │ ├── __pycache__ │ │ ├── MS_SSIM_L1_loss.cpython-36.pyc │ │ ├── deform_conv.cpython-36.pyc │ │ ├── funcs.cpython-36.pyc │ │ ├── models.cpython-36.pyc │ │ ├── modules.cpython-36.pyc │ │ ├── nerf_utils.cpython-36.pyc │ │ ├── pac.cpython-36.pyc │ │ ├── smooth_loss.cpython-36.pyc │ │ ├── ssim.cpython-36.pyc │ │ ├── unet_model.cpython-36.pyc │ │ ├── unet_parts.cpython-36.pyc │ │ ├── utils.cpython-36.pyc │ │ └── warmup_scheduler.cpython-36.pyc │ ├── models.py │ ├── modules.py │ └── utils.py ├── optimizers │ ├── __pycache__ │ │ ├── adahessian.cpython-36.pyc │ │ └── ranger2020.cpython-36.pyc │ └── over9000 │ │ ├── .gitignore │ │ ├── Imagenette_128_5_epoch-bs64.ipynb │ │ ├── Imagewoof_128_5_epoch-bs64.ipynb │ │ ├── LICENSE │ │ ├── README.md │ │ ├── adamod.py │ │ ├── apollo.py │ │ ├── diffgrad.py │ │ ├── environment.yml │ │ ├── lamb.py │ │ ├── lookahead.py │ │ ├── novograd.py │ │ ├── over9000.py │ │ ├── radam.py │ │ ├── ralamb.py │ │ ├── ranger.py │ │ ├── rangerlars.py │ │ ├── train.py │ │ └── xresnet.py └── train.py ├── setup.py └── src ├── NeuralMVSGUI.cxx ├── PyBridge.cxx ├── SFM.cxx └── TrainParams.cxx /.gitignore: -------------------------------------------------------------------------------- 1 | # wandb 2 | wandb/ 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | share/python-wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .nox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | *.py,cover 53 | .hypothesis/ 54 | .pytest_cache/ 55 | cover/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | db.sqlite3 65 | db.sqlite3-journal 66 | 67 | # Flask stuff: 68 | instance/ 69 | .webassets-cache 70 | 71 | # Scrapy stuff: 72 | .scrapy 73 | 74 | # Sphinx documentation 75 | docs/_build/ 76 | 77 | # PyBuilder 78 | .pybuilder/ 79 | target/ 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # IPython 85 | profile_default/ 86 | ipython_config.py 87 | 88 | # pyenv 89 | # For a library or package, you might want to ignore these files since the code is 90 | # intended to run in multiple environments; otherwise, check them in: 91 | # .python-version 92 | 93 | # pipenv 94 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 95 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 96 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 97 | # install all needed dependencies. 98 | #Pipfile.lock 99 | 100 | # poetry 101 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 102 | # This is especially recommended for binary packages to ensure reproducibility, and is more 103 | # commonly ignored for libraries. 104 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 105 | #poetry.lock 106 | 107 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 108 | __pypackages__/ 109 | 110 | # Celery stuff 111 | celerybeat-schedule 112 | celerybeat.pid 113 | 114 | # SageMath parsed files 115 | *.sage.py 116 | 117 | # Environments 118 | .env 119 | .venv 120 | env/ 121 | venv/ 122 | ENV/ 123 | env.bak/ 124 | venv.bak/ 125 | 126 | # Spyder project settings 127 | .spyderproject 128 | .spyproject 129 | 130 | # Rope project settings 131 | .ropeproject 132 | 133 | # mkdocs documentation 134 | /site 135 | 136 | # mypy 137 | .mypy_cache/ 138 | .dmypy.json 139 | dmypy.json 140 | 141 | # Pyre type checker 142 | .pyre/ 143 | 144 | # pytype static type analyzer 145 | .pytype/ 146 | 147 | # Cython debug symbols 148 | cython_debug/ 149 | 150 | # PyCharm 151 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 152 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 153 | # and can be added to the global gitignore or merged into this file. For a more nuclear 154 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 155 | #.idea/ 156 | 157 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # cmake_minimum_required(VERSION 2.8 FATAL_ERROR) # we need at least 3.11 because we use the FetchContent feature to manage dependencies and at least 3.14 because we use FetchContent_MakeAvailable 2 | cmake_minimum_required(VERSION 2.8...3.22) 3 | 4 | project(neural_mvs) 5 | 6 | 7 | ### VARIABLES ############################################################## 8 | set(CMAKE_BUILD_TYPE RelWithDebInfo) 9 | set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" "${CMAKE_CURRENT_SOURCE_DIR}/cmake") 10 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wall -O3") 11 | # set(PYTORCH_PATH "/opt/pytorch") 12 | # set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/opt/ros/melodic/\;${PYTORCH_PATH}" 13 | # set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "${PYTORCH_PATH}") 14 | set(CMAKE_CXX_STANDARD 17) #we need c++17 because this solves alignment issues with eigen http://eigen.tuxfamily.org/bz/show_bug.cgi?id=1409 15 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 16 | set(CMAKE_CXX_EXTENSIONS OFF) 17 | 18 | # set(CMAKE_VERBOSE_MAKEFILE ON) 19 | 20 | 21 | # add_subdirectory( ${CMAKE_SOURCE_DIR}/deps/pybind11) 22 | 23 | #pybind attempt 3 24 | # pybind11_add_module(DataLoaderTest ${CMAKE_SOURCE_DIR}/src/pycom/PyCom.cxx ) 25 | 26 | 27 | #### GLOBAL OPTIONS ###https://stackoverflow.com/questions/15201064/cmake-conditional-preprocessor-define-on-code 28 | # libigl 29 | # libigl options: choose between header only and compiled static library 30 | # Header-only is preferred for small projects. For larger projects the static build 31 | # considerably reduces the compilation times 32 | option(LIBIGL_USE_STATIC_LIBRARY "Use libigl as static library" OFF) 33 | option(LIBIGL_WITH_ANTTWEAKBAR "Use AntTweakBar" OFF) 34 | option(LIBIGL_WITH_CGAL "Use CGAL" OFF) 35 | option(LIBIGL_WITH_COMISO "Use CoMiso" OFF) 36 | option(LIBIGL_WITH_CORK "Use Cork" OFF) 37 | option(LIBIGL_WITH_EMBREE "Use Embree" OFF) 38 | option(LIBIGL_WITH_LIM "Use LIM" OFF) 39 | option(LIBIGL_WITH_MATLAB "Use Matlab" OFF) 40 | option(LIBIGL_WITH_MOSEK "Use MOSEK" OFF) 41 | option(LIBIGL_WITH_OPENGL "Use OpenGL" OFF) 42 | option(LIBIGL_WITH_OPENGL_GLFW "Use GLFW" OFF) 43 | option(LIBIGL_WITH_OPENGL_GLFW_IMGUI "Use ImGui" OFF) 44 | option(LIBIGL_WITH_PNG "Use PNG" OFF) 45 | option(LIBIGL_WITH_PYTHON "Use Python" OFF) 46 | option(LIBIGL_WITH_TETGEN "Use Tetgen" OFF) 47 | option(LIBIGL_WITH_TRIANGLE "Use Triangle" ON) 48 | option(LIBIGL_WITH_VIEWER "Use OpenGL viewer" OFF) 49 | option(LIBIGL_WITH_XML "Use XML" OFF) 50 | 51 | 52 | 53 | 54 | 55 | 56 | ###### PACKAGES ############################################################ 57 | # find_package(catkin REQUIRED) 58 | find_package(Eigen3 3.3 REQUIRED NO_MODULE) 59 | # find_package(OpenCV REQUIRED COMPONENTS core imgproc highgui imgcodecs features2d ) 60 | find_package(OpenCV REQUIRED COMPONENTS core imgproc highgui) 61 | find_package(EasyPBR REQUIRED) 62 | # find_package(DataLoaders REQUIRED) 63 | # find_package(Torch REQUIRED) 64 | find_package(CUDA REQUIRED) 65 | find_package(Ceres REQUIRED) 66 | #libigl 67 | set(LIBIGL_INCLUDE_DIR "${EASYPBR_SRC_PATH}/deps/libigl/include") 68 | include(FindPackageHandleStandardArgs) 69 | find_package_handle_standard_args(LIBIGL 70 | "\nlibigl not found --- You can add it as a submodule it using:\n\tgit add submodule https://github.com/libigl/libigl.git deps/libigl" 71 | LIBIGL_INCLUDE_DIR) 72 | mark_as_advanced(LIBIGL_INCLUDE_DIR) 73 | list(APPEND CMAKE_MODULE_PATH "${LIBIGL_INCLUDE_DIR}/../cmake") 74 | include(libigl) 75 | # find_package(LIBIGL REQUIRED) 76 | #compile with pytorch 77 | # get and append paths for finding dep 78 | # execute_process( #do it like this https://github.com/facebookresearch/hanabi_SAD/blob/6e4ed590f5912fcb99633f4c224778a3ba78879b/rela/CMakeLists.txt#L10 79 | # COMMAND python -c "import torch; import os; print(os.path.dirname(torch.__file__), end='')" 80 | # OUTPUT_VARIABLE TorchPath 81 | # ) 82 | # list(APPEND CMAKE_PREFIX_PATH ${TorchPath}) 83 | # find_package(Torch REQUIRED) 84 | 85 | # add_subdirectory(${PROJECT_SOURCE_DIR}/deps/pybind11) 86 | # find_package(catkin REQUIRED COMPONENTS roscpp std_msgs cv_bridge pcl_ros image_transport ) 87 | #opencv contrib things 88 | # set(HAVE_EIGEN TRUE) 89 | # add_subdirectory( ${PROJECT_SOURCE_DIR}/deps/opencv_contrib/modules/sfm ) 90 | 91 | add_subdirectory( ${EASYPBR_SRC_PATH}/deps/pybind11 ./../../build/pybind11 EXCLUDE_FROM_ALL) 92 | pybind11_add_module(neuralmvs ${PROJECT_SOURCE_DIR}/src/PyBridge.cxx ) 93 | 94 | ### INCLUDES ######################################################### 95 | # include_directories(${catkin_INCLUDE_DIRS}) 96 | include_directories(${CMAKE_SOURCE_DIR}/include) 97 | include_directories(${CMAKE_SOURCE_DIR}/deps) 98 | include_directories(${EIGEN3_INCLUDE_DIR}) 99 | include_directories(${EASYPBR_INCLUDE_DIR}) 100 | include_directories(${DATALOADERS_INCLUDE_DIR}) 101 | include_directories(${CUDA_INCLUDE_DIRS}) 102 | include_directories(${TORCH_INCLUDE_DIRS}) 103 | include_directories(${CERES_INCLUDE_DIRS}) 104 | include_directories(${LIBIGL_INCLUDE_DIR}) 105 | 106 | # include_directories(${LIBIGL_INCLUDE_DIR}) 107 | # catkin_package( INCLUDE_DIRS include CATKIN_DEPENDS roscpp std_msgs cv_bridge pcl_ros image_transport ) 108 | 109 | 110 | ### SOURCES ################################################################# 111 | set(MY_SRC 112 | # ${PROJECT_SOURCE_DIR}/src/Frame.cxx 113 | # ${PROJECT_SOURCE_DIR}/src/DataLoaderVolRef.cxx 114 | # ${PROJECT_SOURCE_DIR}/src/DataLoaderImgRos.cxx 115 | # ${PROJECT_SOURCE_DIR}/src/DataLoaderScanNet.cxx 116 | # ${PROJECT_SOURCE_DIR}/src/DataLoaderShapeNetPartSeg.cxx 117 | # ${PROJECT_SOURCE_DIR}/src/DataTransformer.cxx 118 | # ${PROJECT_SOURCE_DIR}/src/DataLoaderSemanticKitti.cxx 119 | # ${PROJECT_SOURCE_DIR}/src/DataLoaderStanfordIndoor.cxx 120 | # ${PROJECT_SOURCE_DIR}/src/DataLoaderToyExample.cxx 121 | # ${PROJECT_SOURCE_DIR}/src/DataLoaderCloudRos.cxx 122 | # ${PROJECT_SOURCE_DIR}/src/DataLoaderRueMonge.cxx 123 | # ${PROJECT_SOURCE_DIR}/src/RosBagPlayer.cxx 124 | # ${PROJECT_SOURCE_DIR}/src/DataLoaderPNG.cxx 125 | # ${PROJECT_SOURCE_DIR}/src/DataLoaderModelNet40.cxx 126 | ) 127 | set(DEPS_SRC 128 | # ${PROJECT_SOURCE_DIR}/deps/json11/json11.cpp 129 | ) 130 | 131 | 132 | ### SET ALL THE GLOBAL OPTIONS ######################################### 133 | 134 | 135 | #lib 136 | # add_library( latticenet_cpp SHARED ${MY_SRC} ${DEPS_SRC} ) 137 | 138 | ## CUDA library ###################### 139 | #configure the include paths of jitify so that the kernels can properly include other ones 140 | # configure_file(${PROJECT_SOURCE_DIR}/include/neural_mvs/jitify_helper/jitify_options.hpp.in ${PROJECT_SOURCE_DIR}/include/neural_mvs/jitify_helper/jitify_options.hpp) 141 | # install(FILES "${PROJECT_SOURCE_DIR}/include/neural_mvs/jitify_helper/jitify_options.hpp" DESTINATION ${PROJECT_SOURCE_DIR}/include/neural_mvs/jitify_helper/) 142 | 143 | 144 | 145 | add_library( neuralmvs_lib SHARED 146 | # ${CMAKE_SOURCE_DIR}/src/NeuralMVS.cxx 147 | ${CMAKE_SOURCE_DIR}/src/NeuralMVSGUI.cxx 148 | ${CMAKE_SOURCE_DIR}/src/SFM.cxx 149 | # ${CMAKE_SOURCE_DIR}/include/neural_mvs/kernels/NeuralMVSGPU.cuh 150 | # ${CMAKE_SOURCE_DIR}/src/ModelParams.cxx 151 | ${CMAKE_SOURCE_DIR}/src/TrainParams.cxx 152 | # ${CMAKE_SOURCE_DIR}/src/EvalParams.cxx 153 | # ${CMAKE_SOURCE_DIR}/src/TinyLoader.cxx 154 | ) 155 | 156 | 157 | 158 | 159 | ########### NEW modules 160 | # #png required by libmv 161 | # # find_package(PNG REQUIRED) 162 | # include_directories(${PNG_INCLUDE_DIR}) 163 | # #libmv which is needed by the sfm module in opencv_contrib 164 | # # set(WITH_SYSTEM_CERES TRUE) 165 | # set(BUILD_SHARED_LIBS TRUE) 166 | # add_subdirectory(${PROJECT_SOURCE_DIR}/deps/libmv/src) 167 | # include_directories(${PROJECT_SOURCE_DIR}/deps/libmv/src ) 168 | # set(LIBMV_LIBS autotrack base camera correspondence descriptor detector image multiview numeric reconstruction simple_pipeline tools tracking) 169 | # #Make the SFM module from the opencv_contrib that we added as a submodule in deps 170 | # include_directories(${PROJECT_SOURCE_DIR}/deps/opencv_contrib/modules/sfm/include/ ) 171 | # # message(" including ------------------------------------------------------- " ${PROJECT_SOURCE_DIR}/deps/opencv_contrib/modules/sfm/include/ ) 172 | # FILE(GLOB OPENCV_SFM_SRC ${PROJECT_SOURCE_DIR}/deps/opencv_contrib/modules/sfm/src/*.cpp) 173 | # # FILE(GLOB OPENCV_SFM_HDRS 174 | # # ${PROJECT_SOURCE_DIR}/deps/opencv_contrib/modules/sfm/include/opencv2/sfm.hpp 175 | # # ${PROJECT_SOURCE_DIR}/deps/opencv_contrib/modules/sfm/include/opencv2/sfm/*.hpp) 176 | # add_library ( opencv_sfm SHARED 177 | # ${OPENCV_SFM_SRC} 178 | # ) 179 | # target_link_libraries(opencv_sfm ${LIBMV_LIBS} ) 180 | ################### 181 | 182 | 183 | 184 | 185 | 186 | ##pybind 187 | # pybind11_add_module(dataloaders ${PROJECT_SOURCE_DIR}/src/PyBridge.cxx ) 188 | 189 | ### EXECUTABLE ####################################### 190 | # add_executable(run_data_loaders ${PROJECT_SOURCE_DIR}/src/main.cxx ) 191 | 192 | 193 | ### LIBS ############################################### 194 | #torch 1.5.0 and above mess with pybind and we therefore need to link against libtorch_python.so also 195 | # find_library(TORCH_PYTHON_LIBRARY torch_python PATHS "${TORCH_INSTALL_PREFIX}/lib") 196 | # message(STATUS "TORCH_PYTHON_LIBRARY: ${TORCH_PYTHON_LIBRARY}") 197 | # if(TORCH_PYTHON_LIBRARY) 198 | # message(STATUS "Linking to torch_python_library") 199 | # set(LIBS ${LIBS} ${TORCH_PYTHON_LIBRARY} ) 200 | # endif() 201 | # set(LIBS ${LIBS} ${catkin_LIBRARIES} ${EASYPBR_LIBRARY} ${DATALOADERS_LIBRARY} ${CUDA_LIBRARIES} ${TORCH_LIBRARIES} ${TORCH_LIBRARIES} ${OpenCV_LIBS} opencv_sfm ) 202 | set(LIBS ${LIBS} ${catkin_LIBRARIES} ${EASYPBR_LIBRARY} ${CUDA_LIBRARIES} ${OpenCV_LIBS} ${CERES_LIBRARIES} igl::triangle ) 203 | 204 | 205 | 206 | 207 | 208 | 209 | target_link_libraries(neuralmvs_lib ${LIBS} ) 210 | target_link_libraries(neuralmvs PRIVATE neuralmvs_lib ) 211 | # target_link_libraries(run_data_loaders PRIVATE data_loaders_cpp ) 212 | 213 | 214 | #definitions for cmake variables that are necesarry during runtime 215 | target_compile_definitions(neuralmvs_lib PRIVATE PROJECT_SOURCE_DIR="${PROJECT_SOURCE_DIR}") #point to the cmakelist folder of the easy_pbr 216 | target_compile_definitions(neuralmvs_lib PRIVATE CMAKE_SOURCE_DIR="${CMAKE_SOURCE_DIR}") # points to the CMakeList folder of whichever project included easy_pbr 217 | target_compile_definitions(neuralmvs_lib PUBLIC -DDEFAULT_CONFIG="${EASYPBR_SRC_PATH}/config/default_params.cfg" ) 218 | 219 | 220 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 University of Bonn 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | echo "Building NeuralMVS" 3 | python3 -m pip install -v --user --editable ./ 4 | 5 | clean: 6 | python3 -m pip uninstall neuralmvs 7 | rm -rf build *.egg-info build neuralmvs*.so libneuralmvs_cpp.so 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NeuralMVS 2 | 3 | 4 | 5 | 6 | 7 | 8 | [NeuralMVS: Bridging Multi-View Stereo and Novel View Synthesis](https://arxiv.org/abs/2108.03880) 9 | [Radu Alexandru Rosu](https://www.ais.uni-bonn.de/%7Erosu/) 1, 10 | [Sven Behnke](https://www.ais.uni-bonn.de/behnke/) 1 11 |
12 | 1University of Bonn, Autonomous Intelligent Systems 13 | 14 | 15 |

16 | 17 |

18 | 19 | This is the official PyTorch implementation of [NeuralMVS: Bridging Multi-View Stereo and Novel View Synthesis](https://arxiv.org/abs/2108.03880) 20 | 21 | NeuralMVS can process RGB images of a scene in order to perform novel view rendering and additionally learns the 3D structure of the scene in an unsupervized way. It can render novel high resolution images in real-time together with the depth of the scene and a confidence map. The implementation is written in PyTorch. 22 | 23 | # Getting started 24 | 25 | 26 | ### Dependencies 27 | 28 | For NeuralMVS we use [DataLoaders] for loading the data and interface it using [EasyPBR]. The dependencies of these two packages and also NeuralMVS can be installed with: 29 | 30 | ```sh 31 | $ sudo apt-get install python3-dev python3-pip python3-setuptools 32 | $ sudo apt-get install libglfw3-dev libboost-dev libeigen3-dev libopencv-dev 33 | $ sudo python3 -m pip install piq tensorboard 34 | ``` 35 | 36 | 37 | 38 | ### Install 39 | 40 | The easiest way is to sequentially install each required package using the following: 41 | 42 | ```sh 43 | $ git clone --recursive https://github.com/RaduAlexandru/easy_pbr 44 | $ cd easy_pbr && make && cd .. 45 | $ git clone --recursive https://github.com/RaduAlexandru/data_loaders 46 | $ cd data_loaders && make && cd .. 47 | $ git clone --recursive https://github.com/RaduAlexandru/neural_mvs 48 | $ cd neural_mvs && make && cd .. 49 | ``` 50 | 51 | ### Data 52 | 53 | NeuralMVS uses RGB images together with camera pose for training. The data is loaded with the [DataLoaders] package and interfaced using [EasyPBR]. Here we show how to train on the [DTU], [LLFF] and [NeRF] synthetic dataset.
54 | Download either dataset by clicking on their respective links. 55 | 56 | Next modify the `neural_mvs/config/train.cfg` and for each respective loader modify the `dataset_path` to the folder in which your dataset is located. 57 | 58 | 59 | 60 | # Usage 61 | 62 | ### Train 63 | 64 | NeuralMVS uses config files to configure the dataset used, the training parameters, model architecture and various visualization options.
65 | The config file used to train can be found under "neural_mvs/config/train.cfg".
66 | Running the training script will by default read this config file and start the training. 67 | 68 | 69 | ```sh 70 | $ cd neural_mvs 71 | $ ./neural_mvs_py/train.py 72 | ``` 73 | 74 | By default the data is visualized using tensorboard. Therefore we recommend to run the following lines before starting the training script. 75 | 76 | ```sh 77 | $ cd neural_mvs 78 | $ mkdir tensorboard_logs 79 | $ tensorboard --logdir=tensorboard_logs --port=6006 80 | ``` 81 | 82 | The training can be visualized in the browser under `http://localhost:6006/` 83 | 84 | 85 | ### Configuration options 86 | 87 | Various configuration options can be interesting to check out and modify. We take neural_mvs/config/train.cfg as an example. 88 | 89 | ``` 90 | dataset_name: "nerf" #can be changed to dtu or llff to switch the dataset we use 91 | core: hdpi: false #can be turned on an off to accomodate high DPI displays. If the text and fonts in the visualizer are too big, set this option to false 92 | train: with_viewer: false #setting to true will start a visualizer which displays the currently predicted 3D point cloud and image. May not be usable when running on a headless server 93 | ``` 94 | 95 | ## Citation 96 | 97 | ``` 98 | @inproceedings{rosu2021neuralmvs, 99 | title={NeuralMVS: Bridging Multi-View Stereo and Novel View Synthesis}, 100 | author={Rosu, Radu Alexandru and Behnke, Sven}, 101 | booktitle="International Joint Conference on Neural Networks (IJCNN)", 102 | year={2022} 103 | } 104 | 105 | ``` 106 | 107 | 108 | 109 | 110 | 111 | [EasyPBR]: 112 | [DataLoaders]: 113 | [DTU]: 114 | [LLFF]: 115 | [NeRF]: 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | # License 127 | NeuralMVS is provided under the terms of the MIT license (see LICENSE) 128 | -------------------------------------------------------------------------------- /cmake/FindDataLoaders.cmake: -------------------------------------------------------------------------------- 1 | if(EASYPBR_FOUND) 2 | return() 3 | endif() 4 | 5 | 6 | #get the path of site packages which is where easypbr got installed https://stackoverflow.com/a/31384782 and https://stackoverflow.com/a/40006251 7 | execute_process( 8 | COMMAND "python" -c "if True: 9 | import sys 10 | print( next(p for p in sys.path if 'site-packages' in p))" 11 | OUTPUT_VARIABLE PYTHON_SITE 12 | OUTPUT_STRIP_TRAILING_WHITESPACE) 13 | message("--------------------PYTHON is at ", ${PYTHON_SITE}) 14 | #Get only the first line of the egg_link because that is pointing towards the easypbrsrc 15 | execute_process ( 16 | # COMMAND bash -c "date +'%F %T'" "${PYTHON_SITE}/easypbr.egg-link" 17 | COMMAND bash -c "head -1 $0" "${PYTHON_SITE}/dataloaders.egg-link" 18 | OUTPUT_VARIABLE DATALOADERS_SRC_PATH 19 | OUTPUT_STRIP_TRAILING_WHITESPACE 20 | ) 21 | message("DATALOADERS source is ${DATALOADERS_SRC_PATH}") 22 | 23 | #DEBUG 24 | # set(EASYPBR_SRC_PATH "/media/rosu/Data/phd/c_ws/src/easy_pbr") 25 | 26 | find_path(DATALOADERS_CORE_INCLUDE_DIR NAMES data_loaders/DataLoaderShapeNetPartSeg.h PATHS "${DATALOADERS_SRC_PATH}/include" ) 27 | # find_path(EASYPBR_EASYGL_INCLUDE_DIR NAMES Texture2D.h PATHS "${EASYPBR_SRC_PATH}/deps/easy_gl" ) 28 | # find_path(EASYPBR_LOGURU_INCLUDE_DIR NAMES loguru.hpp PATHS "${EASYPBR_SRC_PATH}/deps/loguru" ) 29 | # find_path(EASYPBR_CONFIGURU_INCLUDE_DIR NAMES configuru.hpp PATHS "${EASYPBR_SRC_PATH}/deps/configuru" ) 30 | # find_path(EASYPBR_UTILS_INCLUDE_DIR NAMES Profiler.h PATHS "${EASYPBR_SRC_PATH}/deps/utils/include" ) 31 | # find_path(EASYPBR_CONCURRENTQUEUE_INCLUDE_DIR NAMES concurrentqueue.h PATHS "${EASYPBR_SRC_PATH}/deps/concurrent_queue" ) 32 | # find_path(EASYPBR_IMGUI_INCLUDE_DIR NAMES imgui.h PATHS "${EASYPBR_SRC_PATH}/deps/imgui" ) 33 | # find_path(EASYPBR_IMGUIZMO_INCLUDE_DIR NAMES ImGuizmo.h PATHS "${EASYPBR_SRC_PATH}/deps/imguizmo" ) 34 | # find_path(EASYPBR_BETTERENUM_INCLUDE_DIR NAMES enum.h PATHS "${EASYPBR_SRC_PATH}/deps/better_enums" ) 35 | 36 | # set( EASYPBR_INCLUDE_DIR ${EASYPBR_SRC_PATH}/extern ${EASYPBR_CORE_INCLUDE_DIR} ${EASYPBR_EASYGL_INCLUDE_DIR} ${EASYPBR_LOGURU_INCLUDE_DIR} ${EASYPBR_CONFIGURU_INCLUDE_DIR} ${EASYPBR_UTILS_INCLUDE_DIR} ${EASYPBR_CONCURRENTQUEUE_INCLUDE_DIR} ${EASYPBR_IMGUI_INCLUDE_DIR} ${EASYPBR_IMGUIZMO_INCLUDE_DIR} ${EASYPBR_BETTERENUM_INCLUDE_DIR} ) 37 | set( DATALOADERS_INCLUDE_DIR ${DATALOADERS_CORE_INCLUDE_DIR} ) 38 | 39 | # message("--------------------EASYPPBR include dir is at ", ${EASYPBR_INCLUDE_DIR}) 40 | 41 | 42 | find_library(DATALOADERS_LIBRARY 43 | NAMES libdataloaders_cpp.so 44 | HINTS ${DATALOADERS_SRC_PATH} 45 | DOC "The DataLoaders lib directory" 46 | NO_DEFAULT_PATH) 47 | message("--------------------DATALOADERS lib dir is at ", ${DATALOADERS_LIBRARY}) 48 | 49 | 50 | -------------------------------------------------------------------------------- /cmake/FindEasyPBR.cmake: -------------------------------------------------------------------------------- 1 | if(EASYPBR_FOUND) 2 | return() 3 | endif() 4 | 5 | 6 | #get the path of site packages which is where easypbr got installed https://stackoverflow.com/a/31384782 and https://stackoverflow.com/a/40006251 7 | execute_process( 8 | COMMAND "python3" -c "if True: 9 | import sys 10 | print( next(p for p in sys.path if 'site-packages' in p))" 11 | OUTPUT_VARIABLE PYTHON_SITE 12 | OUTPUT_STRIP_TRAILING_WHITESPACE) 13 | # message("--------------------PYTHON is at ", ${PYTHON_SITE}) 14 | #Get only the first line of the egg_link because that is pointing towards the easypbrsrc 15 | execute_process ( 16 | # COMMAND bash -c "date +'%F %T'" "${PYTHON_SITE}/easypbr.egg-link" 17 | COMMAND bash -c "head -1 $0" "${PYTHON_SITE}/easypbr.egg-link" 18 | OUTPUT_VARIABLE EASYPBR_SRC_PATH 19 | OUTPUT_STRIP_TRAILING_WHITESPACE 20 | ) 21 | message("EASYPPBR source is ${EASYPBR_SRC_PATH}") 22 | 23 | #DEBUG 24 | # set(EASYPBR_SRC_PATH "/media/rosu/Data/phd/c_ws/src/easy_pbr") 25 | 26 | find_path(EASYPBR_CORE_INCLUDE_DIR NAMES easy_pbr/Viewer.h PATHS "${EASYPBR_SRC_PATH}/include" ) 27 | find_path(EASYPBR_EASYGL_INCLUDE_DIR NAMES easy_gl/Texture2D.h PATHS "${EASYPBR_SRC_PATH}/deps/easy_gl/include" ) 28 | find_path(EASYPBR_LOGURU_INCLUDE_DIR NAMES loguru.hpp PATHS "${EASYPBR_SRC_PATH}/deps/loguru" ) 29 | find_path(EASYPBR_CONFIGURU_INCLUDE_DIR NAMES configuru.hpp PATHS "${EASYPBR_SRC_PATH}/deps/configuru" ) 30 | find_path(EASYPBR_UTILS_INCLUDE_DIR NAMES Profiler.h PATHS "${EASYPBR_SRC_PATH}/deps/utils/include" ) 31 | find_path(EASYPBR_CONCURRENTQUEUE_INCLUDE_DIR NAMES concurrentqueue.h PATHS "${EASYPBR_SRC_PATH}/deps/concurrent_queue" ) 32 | find_path(EASYPBR_IMGUI_INCLUDE_DIR NAMES imgui.h PATHS "${EASYPBR_SRC_PATH}/deps/imgui" ) 33 | find_path(EASYPBR_IMGUIZMO_INCLUDE_DIR NAMES ImGuizmo.h PATHS "${EASYPBR_SRC_PATH}/deps/imguizmo" ) 34 | find_path(EASYPBR_BETTERENUM_INCLUDE_DIR NAMES enum.h PATHS "${EASYPBR_SRC_PATH}/deps/better_enums" ) 35 | find_path(EASYPBR_EASYPYTORCH_INCLUDE_DIR NAMES UtilsPytorch.h PATHS "${EASYPBR_SRC_PATH}/deps/easy_pytorch" ) 36 | find_path(EASYPBR_PYBIND_INCLUDE_DIR NAMES pybind11/pybind11.h PATHS "${EASYPBR_SRC_PATH}/deps/pybind11/include" ) #important because any module that tries to have python modules that interact with easypbr should be built with the same pybind version 37 | find_path(EASYPBR_TINYPLY_INCLUDE_DIR NAMES tinyply.h PATHS "${EASYPBR_SRC_PATH}/deps/tiny_ply/source" ) 38 | 39 | set( EASYPBR_INCLUDE_DIR ${EASYPBR_SRC_PATH}/extern ${EASYPBR_CORE_INCLUDE_DIR} ${EASYPBR_EASYGL_INCLUDE_DIR} ${EASYPBR_LOGURU_INCLUDE_DIR} 40 | ${EASYPBR_CONFIGURU_INCLUDE_DIR} ${EASYPBR_UTILS_INCLUDE_DIR} ${EASYPBR_CONCURRENTQUEUE_INCLUDE_DIR} 41 | ${EASYPBR_IMGUI_INCLUDE_DIR} ${EASYPBR_IMGUIZMO_INCLUDE_DIR} ${EASYPBR_BETTERENUM_INCLUDE_DIR} 42 | ${EASYPBR_EASYPYTORCH_INCLUDE_DIR} ${EASYPBR_PYBIND_INCLUDE_DIR} ${EASYPBR_TINYPLY_INCLUDE_DIR} 43 | ) 44 | 45 | # message("--------------------EASYPPBR include dir is at ", ${EASYPBR_INCLUDE_DIR}) 46 | 47 | 48 | find_library(EASYPBR_LIBRARY 49 | NAMES libeasypbr_cpp.so 50 | HINTS ${EASYPBR_SRC_PATH} 51 | DOC "The EasyPBR lib directory" 52 | NO_DEFAULT_PATH) 53 | # message("--------------------EASYPPBR lib dir is at ", ${EASYPBR_LIBRARY}) 54 | 55 | add_definitions( -DDEFAULT_CONFIG="${EASYPBR_SRC_PATH}/config/default_params.cfg" ) 56 | -------------------------------------------------------------------------------- /config/eval.cfg: -------------------------------------------------------------------------------- 1 | core: { 2 | loguru_verbosity: 3 3 | hidpi: false 4 | debug_with_profiler: true //makes the profiler print when it starts and stops time 5 | } 6 | 7 | lattice_gpu: { 8 | hash_table_capacity: 10000 //good for shapenet which splat at around 1k for sigma 0.03 9 | nr_sigmas: 1 10 | sigma_0: "0.5 3" 11 | 12 | } 13 | 14 | tiny_loader: { 15 | imgs_path: "/media/rosu/Data/phd/c_ws/src/phenorob/neural_mvs/data/phenorob" 16 | pose_file_path: "/media/rosu/Data/phd/c_ws/src/phenorob/neural_mvs/data/phenorob/intrinsics_and_poses_world_raw_optimized.cfg" 17 | subsample_factor: 16 18 | } 19 | 20 | loader_vol_ref: { 21 | // dataset_path: "/media/rosu/Data/data/volumetric_refienement_data/augustus-ps" 22 | // dataset_path: "/media/rosu/Data/data/volumetric_refienement_data/lucy-ps" 23 | // dataset_path: "/media/rosu/Data/data/volumetric_refienement_data/sokrates-ps" 24 | // dataset_path: "/media/rosu/Data/data/volumetric_refienement_data/figure-mvs" 25 | dataset_path: "/media/rosu/Data/data/volumetric_refienement_data/sokrates-mvs" 26 | // dataset_path: "/media/rosu/Data/data/volumetric_refienement_data/vase-mvs" 27 | autostart: false 28 | preload: true //preload the meshes in memory which is usually quite fast if they are small, or continously read them from memory 29 | 30 | nr_samples_to_skip: 0 31 | nr_samples_to_read: -1 32 | shuffle: false 33 | rgb_subsample_factor: 8 34 | depth_subsample_factor: 8 35 | load_rgb_with_valid_depth: true 36 | do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 37 | // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 38 | 39 | scene_translation: [0.1, -0.1, 1.2] 40 | scene_scale_multiplier: 0.7 41 | 42 | 43 | } 44 | 45 | // loader_shapenet_img: { 46 | // dataset_path: "/media/rosu/Data/data/shapenet_images/ShapeNetRendering" 47 | // restrict_to_object: "chair" // you can leave it empty to get all of them or write any of (airplane, bag, cap, car, chair, earphone, guitar, knife, lamp, laptop, motorbike, mug, pistol, rocket, skateboard, table) 48 | // nr_samples_to_skip: 0 49 | // nr_samples_to_read: -1 50 | // // nr_samples_to_read: 10 51 | // subsample_factor: 4 52 | // shuffle: true 53 | // // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 54 | // do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 55 | // } 56 | 57 | loader_shapenet_img: { 58 | // dataset_path: "/media/rosu/Data/data/shapenet_images/ShapeNetRendering" 59 | // dataset_path: "/media/rosu/Data/data/shapenet_images/image" 60 | // dataset_depth_path: "/media/rosu/Data/data/shapenet_images/depth" 61 | 62 | dataset_path: "/media/rosu/Data/data/shapenet_images/renders/image" 63 | dataset_depth_path: "/media/rosu/Data/data/shapenet_images/renders/depth" 64 | 65 | // dataset_path: "/media/rosu/Data/data/shapenet_images/renders_2/image" 66 | // dataset_depth_path: "/media/rosu/Data/data/shapenet_images/renders_2/depth" 67 | 68 | // dataset_path: "/media/rosu/Data/data/shapenet_images/renders_test/image" 69 | // dataset_depth_path: "/media/rosu/Data/data/shapenet_images/renders_test/depth" 70 | 71 | restrict_to_object: "bench" // you can leave it empty to get all of them or write any of (plane, car, bench) 72 | nr_samples_to_skip: 1 73 | nr_samples_to_read: -1 74 | nr_imgs_to_read: -1 //nr of images for a certain scene that we want to read, a -1 means that we read all images which is around 36 75 | subsample_factor: 2 76 | shuffle: true 77 | // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 78 | do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 79 | difficulty: "easy" 80 | load_depth: false 81 | load_as_shell: true 82 | } 83 | 84 | 85 | loader_img: { 86 | autostart: false 87 | nr_cams: 1 88 | // rgb_path_cam_0: "/media/rosu/Data/phd/c_ws/src/phenorob/vae_from_others/VAE/data" 89 | rgb_path_cam_0: "/media/rosu/Data/data/celeba_dataset/data" 90 | imgs_to_skip: 0 91 | nr_images_to_read: -1 92 | // nr_images_to_read: 50 93 | only_rgb: true 94 | rgb_subsample_factor: 1 95 | shuffle: false 96 | sort_by_filename: false 97 | do_overfit: true 98 | // do_overfit: false 99 | } 100 | 101 | loader_nerf: { 102 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_synthetic/nerf_synthetic/chair" 103 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_synthetic/nerf_synthetic/drums" 104 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_synthetic/nerf_synthetic/ficus" 105 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_synthetic/nerf_synthetic/hotdog" 106 | dataset_path: "/media/rosu/Data/data/nerf/nerf_synthetic/nerf_synthetic/lego" 107 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_synthetic/nerf_synthetic/materials" 108 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_synthetic/nerf_synthetic/mic" 109 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_synthetic/nerf_synthetic/ship 110 | // subsample_factor: 4 111 | subsample_factor: 1 112 | autostart: false 113 | shuffle: true 114 | mode: "train" //train, val, test 115 | // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 116 | do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 117 | scene_scale_multiplier: 0.137 118 | } 119 | 120 | loader_colmap: { 121 | // dataset_path: "/media/rosu/Data/data/phenorob/data_from_home/christmas_thing/colmap/dense" 122 | dataset_path: "/media/rosu/Data/data/phenorob/data_from_home/fine_leaves_plant/colmap/dense" 123 | // subsample_factor: 8 124 | subsample_factor: 16 125 | // subsample_factor: 32 126 | // subsample_factor: 64 127 | autostart: false 128 | shuffle: true 129 | // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 130 | do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 131 | // scene_scale_multiplier: 0.121 132 | scene_scale_multiplier: 0.06 133 | load_imgs_with_transparency: false 134 | } 135 | 136 | loader_easypbr: { 137 | dataset_path: "/media/rosu/Data/data/easy_pbr_renders" 138 | object_name: "head" 139 | // object_name: "vase" 140 | subsample_factor: 32 141 | autostart: false 142 | shuffle: true 143 | mode: "train" //train, val, test 144 | // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 145 | do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 146 | // scene_scale_multiplier: 0.3 147 | scene_scale_multiplier: { 148 | head: 0.3 149 | vase: 0.0003 150 | } 151 | } 152 | 153 | loader_srn: { 154 | 155 | dataset_path: "/media/rosu/Data/data/pixel_nerf_data/" 156 | object_name: "car" 157 | // object_name: "chair" 158 | mode: "train" 159 | get_spiral_test_else_split_train: false //the spiral in the test set looks like crap so we rather split the train set in two. Splitting occurs when we set this to false 160 | autostart: false 161 | 162 | 163 | nr_samples_to_skip: 0 164 | nr_samples_to_read: -1 165 | nr_imgs_to_read: -1 //nr of images for a certain scene that we want to read, a -1 means that we read all images which is around 36 166 | subsample_factor: 1 167 | shuffle: true 168 | // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 169 | do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 170 | load_as_shell: true 171 | scene_scale_multiplier: { 172 | car: 0.3 173 | chair: 0.2 174 | } 175 | } 176 | 177 | 178 | loader_dtu: { 179 | 180 | dataset_path: "/media/rosu/Data/data/pixel_nerf_data/dtu_dataset/rs_dtu_4/DTU" 181 | mode: "train" 182 | restrict_to_scan_idx: -1 183 | // restrict_to_scan_idx: 8 184 | autostart: false 185 | read_with_bg_thread: false 186 | 187 | 188 | subsample_factor: 1 189 | // subsample_factor: 4 190 | shuffle: true 191 | // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 192 | do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 193 | load_as_shell: true 194 | scene_scale_multiplier: 0.15 195 | 196 | } 197 | 198 | loader_llff: { 199 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_llff_data/fern" 200 | dataset_path: "/media/rosu/Data/data/nerf/nerf_llff_data/flower" 201 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_llff_data/fortress" 202 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_llff_data/horns" 203 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_llff_data/leaves" 204 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_llff_data/orchids" 205 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_llff_data/room" 206 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_llff_data/trex" 207 | //dataset_path: "/home/user/rosu/data/nerf/nerf_llff_data/leaves" 208 | // dataset_path: "/home/user/rosu/data/nerf/nerf_llff_data/flower" 209 | // subsample_factor: 8 210 | // subsample_factor: 16 211 | subsample_factor: 32 212 | // subsample_factor: 4 213 | autostart: false 214 | shuffle: true 215 | // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 216 | do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 217 | // scene_scale_multiplier: 0.01 218 | // scene_scale_multiplier: 0.001 219 | scene_scale_multiplier: "auto" 220 | } 221 | 222 | 223 | train: { 224 | // dataset_name: "volref" 225 | // dataset_name: "nerf_lego" 226 | // dataset_name: "easypbr" 227 | // dataset_name: "shapenetimg" //currently has no good implementatinm of train and test split 228 | // dataset_name: "srn" 229 | // dataset_name: "dtu" 230 | dataset_name: "llff" 231 | with_viewer: true 232 | with_visdom: false 233 | with_debug_output: false 234 | with_error_checking: false 235 | batch_size: 1 236 | // lr:0.001 //for adam 237 | // lr:0.0003 //for adam 238 | lr:0.0001 //for adam 239 | // lr:0.00003 //for adam 240 | // lr:0.1 // for sgd 241 | // lr:0.03 //for radam 242 | // lr:0.5 // for apollo optimizer 243 | weight_decay: 3e-4 //from the adamWpaper 244 | // weight_decay: 1.0 //from the adamWpaper 245 | // weight_decay: 0.0 //from the adamWpaper 246 | // weight_decay: 0.1 //from the adamWpaper 247 | max_training_epochs: -1 248 | 249 | save_checkpoint: false 250 | // checkpoint_path: "/media/rosu/Data/phd/c_ws/src/phenorob/neural_mvs/saved_models/fine_leaves_home_plant" 251 | // checkpoint_path: "/media/rosu/Data/phd/c_ws/src/phenorob/neural_mvs/saved_models/nerf_lego" 252 | // checkpoint_path: "/media/rosu/Data/phd/c_ws/src/phenorob/neural_mvs/saved_models/head_s16_smooth" 253 | checkpoint_path: "/media/rosu/Data/phd/c_ws/src/phenorob/neural_mvs/saved_models/dtu_sub2_sr_v6" 254 | // checkpoint_path: "" 255 | save_every_x_epoch: 50 256 | } 257 | 258 | 259 | model: { 260 | positions_mode: "uv+rgb" 261 | values_mode: "rgb" 262 | pointnet_layers: [16,32,64] 263 | pointnet_start_nr_channels: 16 264 | nr_downsamples: 1 265 | nr_blocks_down_stage: [2,2,2] 266 | nr_blocks_bottleneck: 3 267 | nr_blocks_up_stage: [2,2,2] 268 | nr_levels_down_with_normal_resnet: 3 269 | nr_levels_up_with_normal_resnet: 2 270 | compression_factor: 1.0 271 | dropout_last_layer: 0.0 272 | 273 | experiment: "none" 274 | } 275 | 276 | 277 | 278 | 279 | visualization: { 280 | show_gui: true 281 | 282 | subsample_factor: 1 283 | enable_culling: false 284 | render_uv_to_gbuffer: true 285 | 286 | cam: { 287 | fov: 60 //can be a float value (fov: 30.0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 288 | near: "auto" //can be a float value (near: 0.01) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 289 | far: "auto" //can be a float value (far: 10,0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 290 | exposure: 1.2 //can be floar or "auto" 291 | } 292 | 293 | scene: { 294 | floor_visible: false 295 | floor_metric: true 296 | } 297 | 298 | 299 | ssao: { 300 | auto_settings: false 301 | enable_ssao: false 302 | ao_downsample: 0 303 | // kernel_radius: "auto" //can be a float value (kernel_radius: 10,0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 304 | kernel_radius: 0.1 //can be a float value (kernel_radius: 10,0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 305 | ao_power: 4 306 | ao_blur_sigma_spacial: 2.0 307 | ao_blur_sigma_depth: 1.0 308 | } 309 | 310 | bloom: { 311 | enable_bloom: false 312 | threshold: 0.85 313 | mip_map_lvl: 1 314 | blur_iters: 3 315 | } 316 | 317 | edl: { 318 | auto_settings: false 319 | enable_edl_lighting: true 320 | edl_strength: 7.0 321 | } 322 | 323 | background:{ 324 | show_background_img: false 325 | background_img_path: "" 326 | } 327 | 328 | ibl: { 329 | enable_ibl: false 330 | show_environment_map: false 331 | // environment_map_path: "/media/rosu/Data/data/sibl/Desert_Highway/Road_to_MonumentValley_Ref.hdr" 332 | // environment_map_path: "/media/rosu/Data/data/sibl/Footprint_Court/Footprint_Court_2k.hdr" 333 | // environment_map_path: "/media/rosu/Data/data/sibl/Circus_Backstage/Circus_Backstage_3k.hdr" 334 | // environment_map_path: "/media/rosu/Data/data/sibl/canary_wharf_4k.hdr" 335 | environment_map_path: "sibl/Barcelona_Rooftops/Barce_Rooftop_C_3k.hdr" 336 | // environment_cubemap_resolution: 2048 337 | environment_cubemap_resolution: 32 338 | irradiance_cubemap_resolution: 32 339 | prefilter_cubemap_resolution: 32 340 | brdf_lut_resolution: 32 341 | } 342 | 343 | lights:{ 344 | nr_spot_lights: 3 345 | spot_light_0: { 346 | power: "auto" //can be a float value (power: 1.0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 347 | color: "auto" //can be a vector of rgb [1.0, 1.0, 0.5] or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 348 | create_shadow: true 349 | shadow_map_resolution: 2048 350 | } 351 | spot_light_1: { 352 | power: "auto" //can be a float value (power: 1.0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 353 | color: "auto" //can be a vector of rgb [1.0, 1.0, 0.5] or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 354 | create_shadow: true 355 | shadow_map_resolution: 1024 356 | } 357 | spot_light_2: { 358 | power: "auto" //can be a float value (power: 1.0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 359 | color: "auto" //can be a vector of rgb [1.0, 1.0, 0.5] or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 360 | create_shadow: true 361 | shadow_map_resolution: 1024 362 | } 363 | } 364 | 365 | } 366 | -------------------------------------------------------------------------------- /config/evaluate_confidence.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/config/evaluate_confidence.cfg -------------------------------------------------------------------------------- /config/explicit2render.cfg: -------------------------------------------------------------------------------- 1 | core: { 2 | loguru_verbosity: 3 3 | hidpi: false 4 | debug_with_profiler: true //makes the profiler print when it starts and stops time 5 | } 6 | 7 | lattice_gpu: { 8 | hash_table_capacity: 10000 //good for shapenet which splat at around 1k for sigma 0.03 9 | nr_sigmas: 1 10 | sigma_0: "0.5 3" 11 | 12 | } 13 | 14 | tiny_loader: { 15 | imgs_path: "/media/rosu/Data/phd/c_ws/src/phenorob/neural_mvs/data/phenorob" 16 | pose_file_path: "/media/rosu/Data/phd/c_ws/src/phenorob/neural_mvs/data/phenorob/intrinsics_and_poses_world_raw_optimized.cfg" 17 | subsample_factor: 16 18 | } 19 | 20 | loader_vol_ref: { 21 | // dataset_path: "/media/rosu/Data/data/volumetric_refienement_data/augustus-ps" 22 | // dataset_path: "/media/rosu/Data/data/volumetric_refienement_data/lucy-ps" 23 | // dataset_path: "/media/rosu/Data/data/volumetric_refienement_data/relief-mvs" 24 | // dataset_path: "/media/rosu/Data/data/volumetric_refienement_data/sokrates-ps" 25 | // dataset_path: "/media/rosu/Data/data/volumetric_refienement_data/sokrates-mvs" 26 | // dataset_path: "/media/rosu/Data/data/volumetric_refienement_data/figure-mvs" 27 | dataset_path: "/media/rosu/Data/data/volumetric_refienement_data/vase-mvs" 28 | autostart: false 29 | rgb_subsample_factor: 20 30 | // depth_subsample_factor: 64 // good for stanford3d 31 | depth_subsample_factor: 20 32 | nr_samples_to_skip: 0 33 | nr_samples_to_read: -1 34 | shuffle: true 35 | do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 36 | // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 37 | } 38 | 39 | // loader_shapenet_img: { 40 | // dataset_path: "/media/rosu/Data/data/shapenet_images/ShapeNetRendering" 41 | // restrict_to_object: "chair" // you can leave it empty to get all of them or write any of (airplane, bag, cap, car, chair, earphone, guitar, knife, lamp, laptop, motorbike, mug, pistol, rocket, skateboard, table) 42 | // nr_samples_to_skip: 0 43 | // nr_samples_to_read: -1 44 | // // nr_samples_to_read: 10 45 | // subsample_factor: 4 46 | // shuffle: true 47 | // // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 48 | // do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 49 | // } 50 | 51 | loader_shapenet_img: { 52 | // dataset_path: "/media/rosu/Data/data/shapenet_images/ShapeNetRendering" 53 | // dataset_path: "/media/rosu/Data/data/shapenet_images/image" 54 | // dataset_depth_path: "/media/rosu/Data/data/shapenet_images/depth" 55 | 56 | dataset_path: "/media/rosu/Data/data/shapenet_images/renders/image" 57 | dataset_depth_path: "/media/rosu/Data/data/shapenet_images/renders/depth" 58 | 59 | // dataset_path: "/media/rosu/Data/data/shapenet_images/renders_2/image" 60 | // dataset_depth_path: "/media/rosu/Data/data/shapenet_images/renders_2/depth" 61 | 62 | // dataset_path: "/media/rosu/Data/data/shapenet_images/renders_test/image" 63 | // dataset_depth_path: "/media/rosu/Data/data/shapenet_images/renders_test/depth" 64 | 65 | restrict_to_object: "bench" // you can leave it empty to get all of them or write any of (plane, car, bench) 66 | nr_samples_to_skip: 1 67 | nr_samples_to_read: -1 68 | nr_imgs_to_read: -1 //nr of images for a certain scene that we want to read, a -1 means that we read all images which is around 36 69 | subsample_factor: 2 70 | shuffle: true 71 | // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 72 | do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 73 | difficulty: "easy" 74 | load_depth: false 75 | load_as_shell: true 76 | } 77 | 78 | 79 | loader_img: { 80 | autostart: false 81 | nr_cams: 1 82 | // rgb_path_cam_0: "/media/rosu/Data/phd/c_ws/src/phenorob/vae_from_others/VAE/data" 83 | rgb_path_cam_0: "/media/rosu/Data/data/celeba_dataset/data" 84 | imgs_to_skip: 0 85 | nr_images_to_read: -1 86 | // nr_images_to_read: 50 87 | only_rgb: true 88 | rgb_subsample_factor: 1 89 | shuffle: false 90 | sort_by_filename: false 91 | do_overfit: true 92 | // do_overfit: false 93 | } 94 | 95 | loader_nerf: { 96 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_example_data/nerf_synthetic/lego" 97 | dataset_path: "/media/rosu/Data/data/nerf/nerf_example_data/nerf_synthetic/lego" 98 | subsample_factor: 2 99 | autostart: false 100 | shuffle: true 101 | mode: "train" //train, val, test 102 | // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 103 | do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 104 | scene_scale_multiplier: 0.137 105 | } 106 | 107 | loader_colmap: { 108 | // dataset_path: "/media/rosu/Data/data/phenorob/data_from_home/christmas_thing/colmap/dense" 109 | dataset_path: "/media/rosu/Data/data/phenorob/data_from_home/fine_leaves_plant/colmap/dense" 110 | // subsample_factor: 8 111 | subsample_factor: 16 112 | // subsample_factor: 32 113 | // subsample_factor: 64 114 | autostart: false 115 | shuffle: true 116 | // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 117 | do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 118 | // scene_scale_multiplier: 0.121 119 | scene_scale_multiplier: 0.06 120 | load_imgs_with_transparency: false 121 | } 122 | 123 | loader_easypbr: { 124 | dataset_path: "/media/rosu/Data/data/easy_pbr_renders" 125 | object_name: "head" 126 | // object_name: "vase" 127 | subsample_factor: 8 128 | autostart: false 129 | shuffle: true 130 | mode: "train" //train, val, test 131 | // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 132 | do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 133 | // scene_scale_multiplier: 0.3 134 | scene_scale_multiplier: { 135 | head: 0.3 136 | vase: 0.00015 137 | } 138 | } 139 | 140 | train: { 141 | // dataset_name: "stanford3d" //volref 142 | // dataset_name: "volref" //volref 143 | // dataset_name: "nerf_lego" 144 | dataset_name: "easypbr" 145 | with_viewer: true 146 | with_visdom: false 147 | with_debug_output: false 148 | with_error_checking: false 149 | batch_size: 1 150 | lr:0.001 //for adam 151 | // lr:0.1 // for sgd 152 | // lr:0.03 //for radam 153 | // lr:0.5 // for apollo optimizer 154 | // weight_decay: 3e-4 //from the adamWpaper 155 | // weight_decay: 1.0 //from the adamWpaper 156 | weight_decay: 0.0 //from the adamWpaper 157 | // weight_decay: 0.1 //from the adamWpaper 158 | max_training_epochs: -1 159 | 160 | save_checkpoint: false 161 | // checkpoint_path: "/media/rosu/Data/phd/c_ws/src/phenorob/neural_mvs/saved_models/fine_leaves_home_plant" 162 | checkpoint_path: "/media/rosu/Data/phd/c_ws/src/phenorob/neural_mvs/saved_models/nerf_lego" 163 | // checkpoint_path: "" 164 | save_every_x_epoch: 50 165 | } 166 | 167 | 168 | model: { 169 | positions_mode: "uv+rgb" 170 | values_mode: "rgb" 171 | pointnet_layers: [16,32,64] 172 | pointnet_start_nr_channels: 16 173 | nr_downsamples: 1 174 | nr_blocks_down_stage: [2,2,2] 175 | nr_blocks_bottleneck: 3 176 | nr_blocks_up_stage: [2,2,2] 177 | nr_levels_down_with_normal_resnet: 3 178 | nr_levels_up_with_normal_resnet: 2 179 | compression_factor: 1.0 180 | dropout_last_layer: 0.0 181 | 182 | experiment: "none" 183 | } 184 | 185 | 186 | 187 | 188 | visualization: { 189 | show_gui: true 190 | 191 | subsample_factor: 1 192 | enable_culling: false 193 | render_uv_to_gbuffer: true 194 | 195 | cam: { 196 | fov: 60 //can be a float value (fov: 30.0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 197 | near: "auto" //can be a float value (near: 0.01) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 198 | far: "auto" //can be a float value (far: 10,0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 199 | exposure: 1.2 //can be floar or "auto" 200 | } 201 | 202 | scene: { 203 | floor_visible: true 204 | floor_metric: true 205 | } 206 | 207 | 208 | ssao: { 209 | auto_settings: false 210 | enable_ssao: false 211 | ao_downsample: 0 212 | // kernel_radius: "auto" //can be a float value (kernel_radius: 10,0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 213 | kernel_radius: 0.1 //can be a float value (kernel_radius: 10,0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 214 | ao_power: 4 215 | ao_blur_sigma_spacial: 2.0 216 | ao_blur_sigma_depth: 1.0 217 | } 218 | 219 | bloom: { 220 | enable_bloom: false 221 | threshold: 0.85 222 | mip_map_lvl: 1 223 | blur_iters: 3 224 | } 225 | 226 | edl: { 227 | auto_settings: false 228 | enable_edl_lighting: true 229 | edl_strength: 7.0 230 | } 231 | 232 | background:{ 233 | show_background_img: false 234 | background_img_path: "" 235 | } 236 | 237 | ibl: { 238 | enable_ibl: false 239 | show_environment_map: false 240 | // environment_map_path: "/media/rosu/Data/data/sibl/Desert_Highway/Road_to_MonumentValley_Ref.hdr" 241 | // environment_map_path: "/media/rosu/Data/data/sibl/Footprint_Court/Footprint_Court_2k.hdr" 242 | // environment_map_path: "/media/rosu/Data/data/sibl/Circus_Backstage/Circus_Backstage_3k.hdr" 243 | // environment_map_path: "/media/rosu/Data/data/sibl/canary_wharf_4k.hdr" 244 | environment_map_path: "sibl/Barcelona_Rooftops/Barce_Rooftop_C_3k.hdr" 245 | // environment_cubemap_resolution: 2048 246 | environment_cubemap_resolution: 32 247 | irradiance_cubemap_resolution: 32 248 | prefilter_cubemap_resolution: 32 249 | brdf_lut_resolution: 32 250 | } 251 | 252 | lights:{ 253 | nr_spot_lights: 3 254 | spot_light_0: { 255 | power: "auto" //can be a float value (power: 1.0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 256 | color: "auto" //can be a vector of rgb [1.0, 1.0, 0.5] or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 257 | create_shadow: true 258 | shadow_map_resolution: 2048 259 | } 260 | spot_light_1: { 261 | power: "auto" //can be a float value (power: 1.0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 262 | color: "auto" //can be a vector of rgb [1.0, 1.0, 0.5] or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 263 | create_shadow: true 264 | shadow_map_resolution: 1024 265 | } 266 | spot_light_2: { 267 | power: "auto" //can be a float value (power: 1.0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 268 | color: "auto" //can be a vector of rgb [1.0, 1.0, 0.5] or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 269 | create_shadow: true 270 | shadow_map_resolution: 1024 271 | } 272 | } 273 | 274 | } -------------------------------------------------------------------------------- /config/train.cfg: -------------------------------------------------------------------------------- 1 | core: { 2 | loguru_verbosity: 3 3 | hidpi: false 4 | debug_with_profiler: true //makes the profiler print when it starts and stops time 5 | } 6 | 7 | 8 | loader_nerf: { 9 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_synthetic/nerf_synthetic/chair" 10 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_synthetic/nerf_synthetic/drums" 11 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_synthetic/nerf_synthetic/ficus" 12 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_synthetic/nerf_synthetic/hotdog" 13 | dataset_path: "/media/rosu/Data/data/nerf/nerf_synthetic/nerf_synthetic/lego" 14 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_synthetic/nerf_synthetic/materials" 15 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_synthetic/nerf_synthetic/mic" 16 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_synthetic/nerf_synthetic/ship" 17 | subsample_factor: 4 18 | autostart: false 19 | shuffle: true 20 | mode: "train" //train, val, test 21 | // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 22 | do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 23 | scene_scale_multiplier: 0.137 24 | } 25 | 26 | loader_dtu: { 27 | 28 | dataset_path: "/media/rosu/Data/data/pixel_nerf_data/dtu_dataset/rs_dtu_4/DTU" 29 | mode: "train" 30 | restrict_to_scan_idx: -1 31 | autostart: false 32 | read_with_bg_thread: false 33 | 34 | 35 | // subsample_factor: 1 36 | subsample_factor: 4 37 | shuffle: true 38 | // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 39 | do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 40 | load_as_shell: true 41 | scene_scale_multiplier: 0.15 42 | 43 | } 44 | 45 | loader_llff: { 46 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_llff_data/fern" 47 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_llff_data/flower" 48 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_llff_data/fortress" 49 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_llff_data/horns" 50 | //dataset_path: "/media/rosu/Data/data/nerf/nerf_llff_data/leaves" 51 | dataset_path: "/media/rosu/Data/data/nerf/nerf_llff_data/orchids" 52 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_llff_data/room" 53 | // dataset_path: "/media/rosu/Data/data/nerf/nerf_llff_data/trex" 54 | // dataset_path: "/home/user/rosu/data/nerf/nerf_llff_data/leaves" 55 | // dataset_path: "/home/user/rosu/data/nerf/nerf_llff_data/flower" 56 | // subsample_factor: 64 57 | // subsample_factor: 8 58 | // subsample_factor: 32 59 | // subsample_factor: 4 60 | subsample_factor: 1 61 | autostart: false 62 | shuffle: true 63 | // do_overfit: true //return only one of the samples the whole time, concretely the first sample in the dataset 64 | do_overfit: false //return only one of the samples the whole time, concretely the first sample in the dataset 65 | // scene_scale_multiplier: 0.01 66 | // scene_scale_multiplier: 0.001 67 | scene_scale_multiplier: "auto" 68 | // scene_scale_multiplier: 1.0 69 | } 70 | 71 | 72 | 73 | train: { 74 | dataset_name: "nerf" 75 | // dataset_name: "dtu" 76 | // dataset_name: "llff" 77 | with_viewer: false 78 | with_visdom: false 79 | with_tensorboard: true 80 | with_debug_output: false 81 | with_error_checking: false 82 | // lr:0.001 //for adam 83 | // lr:0.0003 //for adam 84 | // lr:0.0001 //for adam 85 | lr:3e-4 //for adam 86 | // lr:0.00003 //for adam 87 | // lr:0.1 // for sgd 88 | // lr:0.03 //for radam 89 | // lr:0.5 // for apollo optimizer 90 | // weight_decay: 3e-4 //from the adamWpaper 91 | weight_decay: 3e-4 //from the adamWpaper 92 | // weight_decay: 1.0 //from the adamWpaper 93 | // weight_decay: 0.0 //from the adamWpaper 94 | // weight_decay: 0.1 //from the adamWpaper 95 | max_training_epochs: -1 96 | 97 | save_checkpoint: false 98 | checkpoint_path: "/media/rosu/Data/phd/c_ws/src/phenorob/neural_mvs/saved_models/test_llff3_orchids" 99 | // checkpoint_path: "/media/rosu/Data/phd/c_ws/src/phenorob/neural_mvs/saved_models/test_dtu" 100 | // checkpoint_path: "" 101 | save_every_x_epoch: 1 102 | } 103 | 104 | 105 | 106 | 107 | visualization: { 108 | show_gui: true 109 | 110 | subsample_factor: 1 111 | enable_culling: false 112 | render_uv_to_gbuffer: true 113 | 114 | cam: { 115 | fov: 60 //can be a float value (fov: 30.0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 116 | near: "auto" //can be a float value (near: 0.01) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 117 | far: "auto" //can be a float value (far: 10,0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 118 | exposure: 1.2 //can be floar or "auto" 119 | } 120 | 121 | scene: { 122 | floor_visible: true 123 | floor_metric: true 124 | } 125 | 126 | 127 | ssao: { 128 | auto_settings: false 129 | enable_ssao: false 130 | ao_downsample: 0 131 | // kernel_radius: "auto" //can be a float value (kernel_radius: 10,0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 132 | kernel_radius: 0.1 //can be a float value (kernel_radius: 10,0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 133 | ao_power: 4 134 | ao_blur_sigma_spacial: 2.0 135 | ao_blur_sigma_depth: 1.0 136 | } 137 | 138 | bloom: { 139 | enable_bloom: false 140 | threshold: 0.85 141 | mip_map_lvl: 1 142 | blur_iters: 3 143 | } 144 | 145 | edl: { 146 | auto_settings: false 147 | enable_edl_lighting: true 148 | edl_strength: 7.0 149 | } 150 | 151 | background:{ 152 | show_background_img: false 153 | background_img_path: "" 154 | } 155 | 156 | ibl: { 157 | enable_ibl: false 158 | show_environment_map: false 159 | // environment_map_path: "/media/rosu/Data/data/sibl/Desert_Highway/Road_to_MonumentValley_Ref.hdr" 160 | // environment_map_path: "/media/rosu/Data/data/sibl/Footprint_Court/Footprint_Court_2k.hdr" 161 | // environment_map_path: "/media/rosu/Data/data/sibl/Circus_Backstage/Circus_Backstage_3k.hdr" 162 | // environment_map_path: "/media/rosu/Data/data/sibl/canary_wharf_4k.hdr" 163 | environment_map_path: "sibl/Barcelona_Rooftops/Barce_Rooftop_C_3k.hdr" 164 | // environment_cubemap_resolution: 2048 165 | environment_cubemap_resolution: 32 166 | irradiance_cubemap_resolution: 32 167 | prefilter_cubemap_resolution: 32 168 | brdf_lut_resolution: 32 169 | } 170 | 171 | lights:{ 172 | nr_spot_lights: 3 173 | spot_light_0: { 174 | power: "auto" //can be a float value (power: 1.0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 175 | color: "auto" //can be a vector of rgb [1.0, 1.0, 0.5] or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 176 | create_shadow: true 177 | shadow_map_resolution: 2048 178 | } 179 | spot_light_1: { 180 | power: "auto" //can be a float value (power: 1.0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 181 | color: "auto" //can be a vector of rgb [1.0, 1.0, 0.5] or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 182 | create_shadow: true 183 | shadow_map_resolution: 1024 184 | } 185 | spot_light_2: { 186 | power: "auto" //can be a float value (power: 1.0) or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 187 | color: "auto" //can be a vector of rgb [1.0, 1.0, 0.5] or can be set to "auto" so that it's set automatically when the first mesh is added to the scene 188 | create_shadow: true 189 | shadow_map_resolution: 1024 190 | } 191 | } 192 | 193 | } 194 | -------------------------------------------------------------------------------- /imgs/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/imgs/teaser.png -------------------------------------------------------------------------------- /include/neural_mvs/NeuralMVSGUI.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | //ceres (ADDING THIS first before any loguru stuff because otherwise ceres overwrites the LOG functions) 4 | #include "ceres/ceres.h" 5 | #include "ceres/rotation.h" 6 | 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | 13 | #include 14 | #include "easy_pbr/Frame.h" 15 | 16 | namespace easy_pbr{ 17 | class Mesh; 18 | class Viewer; 19 | } 20 | 21 | 22 | class NeuralMVSGUI : public std::enable_shared_from_this{ 23 | public: 24 | template 25 | static std::shared_ptr create( Args&& ...args ){ 26 | return std::shared_ptr( new NeuralMVSGUI(std::forward(args)...) ); 27 | } 28 | ~NeuralMVSGUI(); 29 | 30 | bool m_show_rgb; 31 | bool m_show_depth; 32 | bool m_show_normal; 33 | float m_min_depth; 34 | float m_max_depth; 35 | bool m_control_secondary_cam; 36 | 37 | private: 38 | NeuralMVSGUI(const std::shared_ptr& view); 39 | 40 | std::shared_ptr m_view; 41 | 42 | 43 | void install_callbacks(const std::shared_ptr& view); //installs some callbacks that will be called by the viewer after it finishes an update 44 | 45 | //post draw callbacks 46 | void post_draw(easy_pbr::Viewer& view); 47 | 48 | 49 | }; 50 | -------------------------------------------------------------------------------- /include/neural_mvs/PyBridge.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | 4 | //pybind so you can read the cloud from python 5 | #include 6 | #include 7 | #include 8 | -------------------------------------------------------------------------------- /include/neural_mvs/SFM.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | //ceres (ADDING THIS first before any loguru stuff because otherwise ceres overwrites the LOG functions) 4 | #include "ceres/ceres.h" 5 | #include "ceres/rotation.h" 6 | 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | 13 | #include 14 | #include "easy_pbr/Frame.h" 15 | 16 | namespace easy_pbr{ 17 | class Mesh; 18 | } 19 | 20 | 21 | class SFM : public std::enable_shared_from_this{ 22 | public: 23 | template 24 | static std::shared_ptr create( Args&& ...args ){ 25 | return std::shared_ptr( new SFM(std::forward(args)...) ); 26 | } 27 | ~SFM(); 28 | 29 | 30 | static std::tuple fit_sphere( const Eigen::MatrixXd& points); 31 | static easy_pbr::MeshSharedPtr compute_triangulation_stegreographic( const Eigen::MatrixXd& points, const Eigen::Vector3d& sphere_center, double sphere_radius ); //return triangulation 32 | static easy_pbr::MeshSharedPtr compute_triangulation_plane( const Eigen::MatrixXd& points ); //return triangulation 33 | static std::tuple compute_closest_triangle( const Eigen::Vector3d& point, const easy_pbr::MeshSharedPtr& triangulated_mesh3d); //returns the closest face and the barycentric coords of the point projected onto that closest triangle 34 | static Eigen::Vector3d compute_barycentric_weights_from_triangle_points( const Eigen::Vector3d& point, const Eigen::Matrix3d& vertices_for_face ); 35 | static Eigen::Vector3d compute_barycentric_weights_from_face_and_mesh_points( const Eigen::Vector3d& point, const Eigen::Vector3i& face, const Eigen::Matrix3d& points_mesh ); //convenience func 36 | static Eigen::Vector3d compute_barycentric_coordinates_of_closest_point_inside_triangle(const Eigen::Matrix3d& vertices_for_face, const Eigen::Vector3d& weights ); //The barycentric coordinates computed for a triangle by the other two function can return the barucentric coordinates corresponding to a point outside of the triangle. This function gives me the barycentric coordiantes of the point that would be closest to the triangle from the query point 37 | 38 | 39 | 40 | private: 41 | SFM(); 42 | 43 | std::pair< std::vector, cv::Mat > compute_keypoints_and_descriptor( const easy_pbr::Frame& frame ); 44 | 45 | 46 | static Eigen::Vector3d stereographic_projection(const Eigen::Vector3d& point3d, const Eigen::Vector3d& sphere_center, const double sphere_radius); 47 | static easy_pbr::MeshSharedPtr create_normalized_sphere(); 48 | 49 | 50 | static easy_pbr::MeshSharedPtr m_sphere_normalized; 51 | 52 | }; 53 | -------------------------------------------------------------------------------- /include/neural_mvs/TrainParams.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | //class used to read some network training parameters from a config file ( things like learnign rate and batch size ) This class is also exposed to python so it can be used in pytorch 7 | 8 | class TrainParams: public std::enable_shared_from_this 9 | { 10 | public: 11 | template 12 | static std::shared_ptr create( Args&& ...args ){ 13 | return std::shared_ptr( new TrainParams(std::forward(args)...) ); 14 | } 15 | 16 | bool with_viewer(); 17 | bool with_visdom(); 18 | bool with_tensorboard(); 19 | bool with_debug_output(); 20 | bool with_error_checking(); 21 | std::string dataset_name(); 22 | float lr(); 23 | float weight_decay(); 24 | int max_training_epochs(); 25 | bool save_checkpoint(); 26 | std::string checkpoint_path(); 27 | int save_every_x_epoch(); 28 | 29 | 30 | private: 31 | TrainParams(const std::string config_file); 32 | void init_params(const std::string config_file); 33 | 34 | std::string m_dataset_name; 35 | bool m_with_viewer; //wether the training script will show in a viewer the gt_cloud and prediciton cloud 36 | bool m_with_visdom; //wether the training script will show in a viewer the gt_cloud and prediciton cloud 37 | bool m_with_tensorboard; 38 | bool m_with_debug_output; //weather the training script should output a bunch of debug stuff 39 | bool m_with_error_checking; //weather the training script should check for erronoues things like how many positions we sliced correctly 40 | float m_lr; 41 | float m_weight_decay; 42 | int m_max_training_epochs; 43 | bool m_save_checkpoint; 44 | std::string m_checkpoint_path; 45 | int m_save_every_x_epoch; 46 | 47 | }; -------------------------------------------------------------------------------- /neural_mvs_py/__pycache__/meta_modules.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/__pycache__/meta_modules.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/__pycache__/modules.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/__pycache__/modules.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/callbacks/__pycache__/callback.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/callbacks/__pycache__/callback.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/callbacks/__pycache__/phase.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/callbacks/__pycache__/phase.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/callbacks/__pycache__/scores.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/callbacks/__pycache__/scores.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/callbacks/__pycache__/state_callback.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/callbacks/__pycache__/state_callback.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/callbacks/__pycache__/tensorboard_callback.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/callbacks/__pycache__/tensorboard_callback.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/callbacks/__pycache__/viewer_callback.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/callbacks/__pycache__/viewer_callback.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/callbacks/__pycache__/vis.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/callbacks/__pycache__/vis.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/callbacks/__pycache__/visdom_callback.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/callbacks/__pycache__/visdom_callback.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/callbacks/callback.py: -------------------------------------------------------------------------------- 1 | #https://github.com/devforfu/pytorch_playground/blob/master/loop.ipynb 2 | 3 | import re 4 | 5 | def to_snake_case(string): 6 | """Converts CamelCase string into snake_case.""" 7 | 8 | s = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', string) 9 | return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s).lower() 10 | 11 | def classname(obj): 12 | return obj.__class__.__name__ 13 | 14 | 15 | class Callback: 16 | """ 17 | The base class inherited by callbacks. 18 | 19 | Provides a lot of hooks invoked on various stages of the training loop 20 | execution. The signature of functions is as broad as possible to allow 21 | flexibility and customization in descendant classes. 22 | """ 23 | def training_started(self, **kwargs): pass 24 | 25 | def training_ended(self, **kwargs): pass 26 | 27 | def epoch_started(self, **kwargs): pass 28 | 29 | def phase_started(self, **kwargs): pass 30 | 31 | def phase_ended(self, **kwargs): pass 32 | 33 | def epoch_ended(self, **kwargs): pass 34 | 35 | def batch_started(self, **kwargs): pass 36 | 37 | def batch_ended(self, **kwargs): pass 38 | 39 | def before_forward_pass(self, **kwargs): pass 40 | 41 | def after_forward_pass(self, **kwargs): pass 42 | 43 | def before_backward_pass(self, **kwargs): pass 44 | 45 | def after_backward_pass(self, **kwargs): pass 46 | 47 | 48 | class CallbacksGroup(Callback): 49 | """ 50 | Groups together several callbacks and delegates training loop 51 | notifications to the encapsulated objects. 52 | """ 53 | def __init__(self, callbacks): 54 | self.callbacks = callbacks 55 | self.named_callbacks = {to_snake_case(classname(cb)): cb for cb in self.callbacks} 56 | 57 | def __getitem__(self, item): 58 | item = to_snake_case(item) 59 | if item in self.named_callbacks: 60 | return self.named_callbacks[item] 61 | raise KeyError(f'callback name is not found: {item}') 62 | 63 | def training_started(self, **kwargs): self.invoke('training_started', **kwargs) 64 | 65 | def training_ended(self, **kwargs): self.invoke('training_ended', **kwargs) 66 | 67 | def epoch_started(self, **kwargs): self.invoke('epoch_started', **kwargs) 68 | 69 | def phase_started(self, **kwargs): self.invoke('phase_started', **kwargs) 70 | 71 | def phase_ended(self, **kwargs): self.invoke('phase_ended', **kwargs) 72 | 73 | def epoch_ended(self, **kwargs): self.invoke('epoch_ended', **kwargs) 74 | 75 | def batch_started(self, **kwargs): self.invoke('batch_started', **kwargs) 76 | 77 | def batch_ended(self, **kwargs): self.invoke('batch_ended', **kwargs) 78 | 79 | def before_forward_pass(self, **kwargs): self.invoke('before_forward_pass', **kwargs) 80 | 81 | def after_forward_pass(self, **kwargs): self.invoke('after_forward_pass', **kwargs) 82 | 83 | def before_backward_pass(self, **kwargs): self.invoke('before_backward_pass', **kwargs) 84 | 85 | def after_backward_pass(self, **kwargs): self.invoke('after_backward_pass', **kwargs) 86 | 87 | def invoke(self, method, **kwargs): 88 | for cb in self.callbacks: 89 | getattr(cb, method)(**kwargs) -------------------------------------------------------------------------------- /neural_mvs_py/callbacks/phase.py: -------------------------------------------------------------------------------- 1 | #https://github.com/devforfu/pytorch_playground/blob/master/loop.ipynbA 2 | 3 | from callbacks.scores import * 4 | 5 | class Phase: 6 | """ 7 | Model training loop phase. 8 | 9 | Each model's training loop iteration could be separated into (at least) two 10 | phases: training and validation. The instances of this class track 11 | metrics and counters, related to the specific phase, and keep the reference 12 | to subset of data, used during phase. 13 | """ 14 | 15 | def __init__(self, name, loader, grad): 16 | self.name = name 17 | self.loader = loader 18 | self.grad = grad 19 | self.iter_nr = 0 20 | self.epoch_nr = 0 21 | self.samples_processed_this_epoch = 0 22 | self.scores= Scores() 23 | self.loss_acum_per_epoch=0.0 24 | # self.batch_loss = None 25 | # self.batch_index = 0 26 | # self.rolling_loss = 0 27 | # self.losses = [] 28 | # self.metrics = OrderedDict() 29 | 30 | # @property 31 | # def last_loss(self): 32 | # return self.losses[-1] if self.losses else None 33 | 34 | # @property 35 | # def last_metrics(self): 36 | # metrics = OrderedDict() 37 | # metrics[f'{self.name}_loss'] = self.last_loss 38 | # for name, values in self.metrics.items(): 39 | # metrics[f'{self.name}_{name}'] = values[-1] 40 | # return metrics 41 | 42 | # @property 43 | # def metrics_history(self): 44 | # metrics = OrderedDict() 45 | # for name, values in self.metrics.items(): 46 | # metrics[f'{self.name}_{name}'] = values 47 | # return metrics 48 | 49 | # def update(self, loss): 50 | # self.losses.append(loss) 51 | 52 | # def update_metric(self, name, value): 53 | # if name not in self.metrics: 54 | # self.metrics[name] = [] 55 | # self.metrics[name].append(value) -------------------------------------------------------------------------------- /neural_mvs_py/callbacks/scores.py: -------------------------------------------------------------------------------- 1 | 2 | import torchnet 3 | import numpy as np 4 | import torch 5 | import csv 6 | 7 | 8 | class Scores(): 9 | def __init__(self): 10 | 11 | self.psnr_acum=0 12 | self.nr_times_accumulated=0 13 | self.cur_epoch_nr=0 14 | #things fro keeping track of the best starts and the best epoch 15 | self.best_avg_psnr=0 16 | self.best_epoch_nr=0 #epoch idx where the best psnr was found 17 | 18 | #attempt 2 19 | self.clear() 20 | 21 | 22 | #adapted from https://github.com/NVlabs/splatnet/blob/f7e8ca1eb16f6e1d528934c3df660bfaaf2d7f3b/splatnet/semseg3d/eval_seg.py 23 | def accumulate_scores(self, psnr): 24 | self.psnr_acum+=psnr 25 | self.nr_times_accumulated+=1 26 | 27 | #compute all the starts that you may need, psnr, ssim, etc . TODO implement ssim and other metrics 28 | def compute_stats(self, print_stats=False): 29 | 30 | if(self.nr_times_accumulated==0): 31 | avg_psnr=0 32 | else: 33 | avg_psnr=self.psnr_acum/self.nr_times_accumulated 34 | 35 | if print_stats: 36 | print("average psnr is", avg_psnr ) 37 | return avg_psnr 38 | 39 | 40 | def avg_psnr(self, print_stats=False): 41 | avg_psnr=self.compute_stats(print_stats=print_stats) 42 | return avg_psnr 43 | 44 | def update_best(self): 45 | avg_psnr=self.compute_stats(print_stats=False) 46 | if avg_psnr>self.best_avg_psnr: 47 | self.best_avg_psnr=avg_psnr 48 | self.best_epoch_nr=self.cur_epoch_nr 49 | 50 | 51 | 52 | 53 | def show(self, epoch_nr): 54 | 55 | self.compute_stats(print_stats=True) 56 | 57 | #clear once, when you want to start completely fresh, with no knowledge of the best stats 58 | def clear(self): 59 | 60 | self.psnr_acum=0 61 | self.nr_times_accumulated=0 62 | self.best_avg_psnr=0 63 | self.best_epoch_nr=0 64 | 65 | #to be called every epoch, to clear the statistics for that epoch but keep in mind the knowledge of the best stats 66 | def start_fresh_eval(self, cur_epoch_nr): 67 | self.psnr_acum=0 68 | self.nr_times_accumulated=0 69 | self.cur_epoch_nr=cur_epoch_nr 70 | 71 | 72 | def write_stats_to_csv(self,filename): 73 | avg_psnr=self.avg_psnr(print_stats=False) 74 | w = csv.writer(open(filename, "w")) 75 | w.writerow(["avg_psnr", avg_psnr]) 76 | w.writerow(["best_avg_psnr", self.best_avg_psnr]) 77 | w.writerow(["best_epoch_nr", self.best_epoch_nr]) 78 | 79 | 80 | -------------------------------------------------------------------------------- /neural_mvs_py/callbacks/state_callback.py: -------------------------------------------------------------------------------- 1 | from callbacks.callback import * 2 | import os 3 | import torch 4 | 5 | 6 | class StateCallback(Callback): 7 | 8 | def __init__(self): 9 | pass 10 | 11 | def after_forward_pass(self, phase, loss, psnr, **kwargs): 12 | phase.iter_nr+=1 13 | phase.samples_processed_this_epoch+=1 14 | phase.loss_acum_per_epoch+=loss 15 | 16 | phase.scores.accumulate_scores(psnr) 17 | 18 | def epoch_started(self, phase, **kwargs): 19 | phase.loss_acum_per_epoch=0.0 20 | phase.scores.start_fresh_eval(phase.epoch_nr) 21 | 22 | def epoch_ended(self, phase, model, save_checkpoint, checkpoint_path, save_every_x_epoch, **kwargs): 23 | phase.scores.update_best() 24 | 25 | message_string="" 26 | message_string+="epoch "+str(phase.epoch_nr)+" phase.grad is " + str(phase.grad) 27 | 28 | #for evaluation phase print the iou 29 | if not phase.grad: 30 | avg_psnr=phase.scores.avg_psnr(print_stats=False) 31 | best_avg_psnr=phase.scores.best_avg_psnr 32 | message_string+=" best_avg_psnr " + str(best_avg_psnr) + " at epoch " + str(phase.scores.best_epoch_nr) + " avg psnr this epoch " + str(avg_psnr) 33 | 34 | #save the checkpoint of the model if we are in testing mode 35 | if not phase.grad: 36 | if save_checkpoint and model is not None and phase.epoch_nr%save_every_x_epoch==0: 37 | model_name="model_e_"+str(phase.epoch_nr)+"_score_"+str(avg_psnr)+".pt" 38 | info_txt_name="model_e_"+str(phase.epoch_nr)+"_info"+".csv" 39 | out_model_path=os.path.join(checkpoint_path, model_name) 40 | out_info_path=os.path.join(checkpoint_path, info_txt_name) 41 | torch.save(model.state_dict(), out_model_path) 42 | phase.scores.write_stats_to_csv(out_info_path) 43 | 44 | print(message_string) 45 | 46 | 47 | 48 | phase.epoch_nr+=1 49 | 50 | def phase_started(self, phase, **kwargs): 51 | phase.samples_processed_this_epoch=0 52 | 53 | def phase_ended(self, phase, **kwargs): 54 | pass 55 | if phase.loader.is_finished(): 56 | print("resetting loader for phase ", phase.name) 57 | phase.loader.reset() 58 | -------------------------------------------------------------------------------- /neural_mvs_py/callbacks/tensorboard_callback.py: -------------------------------------------------------------------------------- 1 | from instant_ngp_2_py.callbacks.callback import * 2 | from torch.utils.tensorboard import SummaryWriter 3 | 4 | from neural_mvs.utils import * 5 | 6 | 7 | class TensorboardCallback(Callback): 8 | 9 | def __init__(self, experiment_name): 10 | self.tensorboard_writer = SummaryWriter("tensorboard_logs/"+experiment_name) 11 | self.experiment_name=experiment_name 12 | 13 | 14 | def after_forward_pass(self, phase, loss, loss_rgb, psnr, lr, rgb_pred, rgb_gt, confidence_map, point3d, **kwargs): 15 | # self.vis.log(phase.iter_nr, loss, "loss_"+phase.name, "loss_"+phase.name+"_"+self.experiment_name, smooth=True, show_every=10, skip_first=10) 16 | # self.vis.log(phase.iter_nr, loss_dice, "loss_dice_"+phase.name, "loss_"+phase.name+"_"+self.experiment_name, smooth=True, show_every=10, skip_first=10) 17 | # if phase.grad: 18 | # self.vis.log(phase.iter_nr, lr, "lr", "loss_"+phase.name+"_"+self.experiment_name, smooth=False, show_every=30) 19 | 20 | # self.tensorboard_writer.add_scalar('instant_ngp_2/' + phase.name + '/lr', lr, phase.iter_nr) 21 | 22 | self.tensorboard_writer.add_scalar('neural_mvs/' + phase.name + '/loss', loss, phase.iter_nr) 23 | if loss_rgb!=0: 24 | self.tensorboard_writer.add_scalar('neural_mvs/' + phase.name + '/loss_rgb', loss_rgb, phase.iter_nr) 25 | if psnr!=0: 26 | self.tensorboard_writer.add_scalar('neural_mvs/' + phase.name + '/psnr', psnr, phase.iter_nr) 27 | 28 | #show imgs 29 | if (phase.iter_nr<=1 or phase.iter_nr%500==0) and rgb_pred is not None: 30 | self.tensorboard_writer.add_image('neural_mvs/' + phase.name + '/rgb_pred', rgb_pred.squeeze(0), phase.iter_nr) 31 | if (phase.iter_nr<=1 or phase.iter_nr%500==0) and rgb_gt is not None: 32 | self.tensorboard_writer.add_image('neural_mvs/' + phase.name + '/rgb_gt', rgb_gt.squeeze(0), phase.iter_nr) 33 | if (phase.iter_nr<=1 or phase.iter_nr%500==0) and confidence_map is not None: 34 | self.tensorboard_writer.add_image('neural_mvs/' + phase.name + '/confidence', confidence_map.squeeze(0), phase.iter_nr) 35 | if (phase.iter_nr<=1 or phase.iter_nr%500==0) and point3d is not None: 36 | normal_img=compute_normal(point3d) 37 | normal_vis=(normal_img+1.0)*0.5 38 | self.tensorboard_writer.add_image('neural_mvs/' + phase.name + '/normal', normal_vis.squeeze(0), phase.iter_nr) 39 | 40 | 41 | 42 | 43 | 44 | 45 | def epoch_ended(self, phase, **kwargs): 46 | pass 47 | # mean_iou=phase.scores.avg_class_iou(print_per_class_iou=False) 48 | # self.tensorboard_writer.add_scalar('instant_ngp_2/' + phase.name + '/mean_iou', mean_iou, phase.epoch_nr) 49 | # self.vis.log(phase.epoch_nr, mean_iou, "iou_"+phase.name, "loss_"+phase.name+"_"+self.experiment_name, smooth=False, show_every=1) -------------------------------------------------------------------------------- /neural_mvs_py/callbacks/viewer_callback.py: -------------------------------------------------------------------------------- 1 | from callbacks.callback import * 2 | from callbacks.vis import * #for the map_range 3 | from easypbr import Scene 4 | from easypbr import Gui 5 | from easypbr import Frame 6 | from easypbr import tensor2mat, mat2tensor 7 | import numpy as np 8 | 9 | class ViewerCallback(Callback): 10 | 11 | def __init__(self): 12 | pass 13 | 14 | 15 | def after_forward_pass(self, **kwargs): 16 | pass 17 | 18 | #show the input 19 | # self.show_rgb(input_frame_color) 20 | # self.show_depth(input_frame_depth) 21 | # self.show_frustum(input_frame_depth) 22 | 23 | #show the rgb valid 24 | # rgb_valid=input_frame_color.rgb_with_valid_depth(input_frame_depth) 25 | # Gui.show(rgb_valid, "rgb_valid") 26 | 27 | #show the output color from the network 28 | # output_color_mat=tensor2mat(output_color) #output color is in bgr 29 | # Gui.show(output_color_mat, "output_color_mat") 30 | 31 | # #show the output 32 | # frame_depth_output= input_frame_depth 33 | # depth_tensor_clamped=map_range(output_tensor, 1.2, 1.7, 0.0, 1.0) 34 | # frame_depth_output.depth=tensor2mat(depth_tensor_clamped) 35 | # Gui.show(frame_depth_output.depth, "depth_output") 36 | 37 | # #show the mesh with the depth predicted by the net 38 | # frame_depth_output.depth=tensor2mat(output_tensor) 39 | # self.show_colored_cloud(input_frame_color, frame_depth_output, "cloud_output") 40 | 41 | 42 | 43 | 44 | def show_rgb(self, frame_color): 45 | Gui.show(frame_color.rgb_32f, "rgb") 46 | 47 | def show_depth(self, frame_depth): 48 | # depth_tensor=frame_depth.depth2tensor() #tensor fo size h,w,1 49 | depth_tensor=mat2tensor(frame_depth.depth, False) 50 | depth_tensor_clamped=map_range(depth_tensor, 1.2, 1.7, 0.0, 1.0) 51 | frame_depth.depth=tensor2mat(depth_tensor) 52 | # frame_depth.tensor2depth(depth_tensor_clamped) 53 | Gui.show(frame_depth.depth, "depth") 54 | frame_depth.depth=tensor2mat(depth_tensor) 55 | # frame_depth.tensor2depth(depth_tensor) 56 | 57 | def show_frustum(self, frame): 58 | frustum_mesh=frame.create_frustum_mesh(0.1) 59 | frustum_mesh.m_vis.m_line_width=3 60 | Scene.show(frustum_mesh, "frustum") 61 | 62 | def show_colored_cloud(self, frame_color, frame_depth, name): 63 | cloud=frame_depth.backproject_depth() 64 | frame_color.assign_color(cloud) #project the cloud into this frame and creates a color matrix for it 65 | Scene.show(cloud, name) 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /neural_mvs_py/callbacks/vis.py: -------------------------------------------------------------------------------- 1 | import torchnet 2 | import numpy as np 3 | import torch 4 | 5 | node_name="lnn" 6 | port=8097 7 | # logger_iou = torchnet.logger.VisdomPlotLogger('line', opts={'title': 'logger_iou'}, port=port, env='train_'+node_name) 8 | 9 | #map a range of values to another range of values for pytorch tensors 10 | def map_range(input, input_start,input_end, output_start, output_end): 11 | input_clamped=torch.clamp(input, input_start, input_end) 12 | return output_start + ((output_end - output_start) / (input_end - input_start)) * (input_clamped - input_start) 13 | 14 | 15 | class Vis(): 16 | def __init__(self, env, port): 17 | self.port=port 18 | self.env=env 19 | # self.win_id="0" 20 | # self.win_id=None 21 | 22 | self.name_dict=dict() #maps from name of the plot to the values that are stored currectly for that plot 23 | self.name2id_dict=dict() #maps from the name of the plot to the windows id 24 | self.logger_dict=dict() 25 | self.exp_alpha=0.03 #the lower the value the smoother the plot is 26 | 27 | def update_val(self, val, name, smooth): 28 | if name not in self.name_dict: 29 | self.name2id_dict[name] = str(len(self.name_dict)) 30 | self.name_dict[name]=val 31 | else: 32 | if smooth: 33 | self.name_dict[name]= self.name_dict[name] + self.exp_alpha*(val-self.name_dict[name]) 34 | else: 35 | self.name_dict[name]=val 36 | 37 | return self.name_dict[name] 38 | 39 | def update_logger(self, x_axis, val, name_window, name_plot): 40 | if name_window not in self.logger_dict: 41 | # self.logger_dict[name_window]=torchnet.logger.VisdomPlotLogger('line', opts={'title': name_window}, port=self.port, env=self.env, win=self.win_id) 42 | self.logger_dict[name_window]=torchnet.logger.VisdomPlotLogger('line', opts={'title': name_window}, port=self.port, env=self.env, win=self.name2id_dict[name_plot] ) 43 | print("started new line plot on win ", self.logger_dict[name_window].win) 44 | 45 | # print("update_logger val is ", val, "name plot is ", name_plot) 46 | self.logger_dict[name_window].log(x_axis, val, name=name_plot) 47 | 48 | def log(self, x_axis, val, name_window, name_plot, smooth, show_every=1): 49 | new_val=self.update_val(val,name_plot, smooth) 50 | if(x_axis%show_every==0): 51 | self.update_logger(x_axis, new_val, name_window, name_plot) 52 | 53 | def show_img_from_tensor(self, tensor, name_window, min_val=0.0, max_val=1.0): 54 | if name_window not in self.logger_dict: 55 | self.logger_dict[name_window]=torchnet.logger.VisdomLogger('image', opts={'title': name_window}, port=self.port, env=self.env, win=self.win_id) 56 | 57 | img=tensor.cpu().squeeze(0).detach() 58 | # print("min max", img.min(), img.max()) 59 | img=map_range(img, min_val, max_val, 0.0, 1.0) 60 | # print("after mapping range min max", img.min(), img.max()) 61 | 62 | # print("img has shape ", img.shape) 63 | 64 | 65 | self.logger_dict[name_window].log(img) 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /neural_mvs_py/callbacks/visdom_callback.py: -------------------------------------------------------------------------------- 1 | from callbacks.callback import * 2 | from callbacks.vis import * 3 | 4 | class VisdomCallback(Callback): 5 | 6 | def __init__(self, experiment_name): 7 | self.vis=Vis("lnn", 8097) 8 | # self.iter_nr=0 9 | self.experiment_name=experiment_name 10 | 11 | def after_forward_pass(self, phase, loss, lr, **kwargs): 12 | pass 13 | 14 | # print("loss and smooth loss ", loss, " ", smooth_loss) 15 | 16 | 17 | 18 | # if phase.show_visdom: 19 | # self.vis.log(phase.iter_nr, loss, "loss_"+phase.name, self.experiment_name, smooth=True, show_every=30) 20 | # self.vis.log(phase.iter_nr, smooth_loss.item(), "smooth_loss_"+phase.name, self.experiment_name, smooth=True, show_every=30) 21 | # if phase.grad: 22 | # self.vis.log(phase.iter_nr, lr, "lr", "lr", smooth=False) 23 | 24 | #show image of the depth 25 | # min_val=1.2 26 | # max_val=1.7 27 | # self.vis.show_img_from_tensor(output_tensor, "output_tensor", min_val, max_val) 28 | # self.vis.show_img_from_tensor(gt_tensor, "gt_tensor", min_val, max_val) 29 | # image_logger = VisdomLogger('image') 30 | 31 | 32 | def epoch_ended(self, phase, **kwargs): 33 | # pass 34 | # mean_iou=phase.scores.avg_class_iou(print_per_class_iou=False) 35 | # self.vis.log(phase.epoch_nr, mean_iou, "iou_"+phase.name, "iou_"+phase.name, smooth=False) 36 | 37 | if phase.show_visdom and phase.samples_processed_this_epoch!=0: 38 | loss_avg_per_eppch = phase.loss_acum_per_epoch/ phase.samples_processed_this_epoch 39 | self.vis.log(phase.epoch_nr, loss_avg_per_eppch, "loss_"+phase.name, self.experiment_name, smooth=False, show_every=1) 40 | -------------------------------------------------------------------------------- /neural_mvs_py/misc/__pycache__/gradcheck_custom.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/misc/__pycache__/gradcheck_custom.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/misc/grad_check.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.6 2 | 3 | import torch 4 | from torch.autograd import Function 5 | # from torch.autograd import gradcheck 6 | from torch import Tensor 7 | 8 | import sys 9 | import os 10 | import numpy as np 11 | # from matplotlib import pyplot as plt 12 | # http://wiki.ros.org/Packages#Client_Library_Support 13 | # import rospkg 14 | # rospack = rospkg.RosPack() 15 | # sf_src_path=rospack.get_path('surfel_renderer') 16 | # sf_build_path=os.path.abspath(sf_src_path + "/../../build/surfel_renderer") 17 | # sys.path.append(sf_build_path) #contains the modules of pycom 18 | 19 | # from DataLoaderTest import * 20 | # from lattice_py import LatticePy 21 | # import visdom 22 | # import torchnet 23 | # from lr_finder import LRFinder 24 | # from scores import Scores 25 | # from model_ctx import ModelCtx 26 | # from lattice_funcs import * 27 | # from lattice_modules import * 28 | # from models import * 29 | from neural_mesh_py.neural_mesh.funcs import * 30 | from gradcheck_custom import * 31 | 32 | # np.set_printoptions(threshold=np.inf) 33 | # torch.set_printoptions(threshold=50000) 34 | # torch.set_printoptions(profile="full") 35 | torch.autograd.set_detect_anomaly(True) 36 | 37 | torch.manual_seed(0) 38 | 39 | # config_file="lnn_grad_check.cfg" 40 | # with_viewer=True 41 | 42 | 43 | def check_splat(positions_tensor, uv_tensor, texture_size): 44 | gradcheck(SplatTexture.apply, (positions_tensor, uv_tensor, texture_size), eps=1e-5) 45 | 46 | def check_slice(texture, uv_tensor): 47 | gradcheck(SliceTexture.apply, (texture, uv_tensor), eps=1e-5) 48 | 49 | 50 | 51 | def run(): 52 | 53 | while True: 54 | 55 | #get positions and values 56 | texture_size= 32 57 | positions_tensor = torch.rand(16,3).to("cuda") 58 | uv_tensor = torch.rand(16,2).to("cuda") 59 | texture = torch.rand(texture_size, texture_size, 4).to("cuda") 60 | uv_tensor=(uv_tensor*2)-1.0 #get it in range -1, 1 61 | uv_tensor*=0.1 #make a it a bit smaller to avoid border effects when accesign the texture 62 | #print 63 | print("positions_tensor is ", positions_tensor) 64 | print("uv_tensor is ", uv_tensor) 65 | #set the reguired grads 66 | # positions_tensor.requires_grad=True 67 | # texture.requires_grad=True 68 | uv_tensor.requires_grad=True 69 | 70 | 71 | 72 | check_splat(positions_tensor, uv_tensor, texture_size) 73 | # check_slice(texture, uv_tensor) 74 | 75 | print("FINISHED GRAD CHECK") 76 | 77 | return 78 | 79 | 80 | def main(): 81 | run() 82 | 83 | 84 | 85 | if __name__ == "__main__": 86 | main() # This is what you would have, but the following is useful: 87 | 88 | # # These are temporary, for debugging, so meh for programming style. 89 | # import sys, trace 90 | 91 | # # If there are segfaults, it's a good idea to always use stderr as it 92 | # # always prints to the screen, so you should get as much output as 93 | # # possible. 94 | # sys.stdout = sys.stderr 95 | 96 | # # Now trace execution: 97 | # tracer = trace.Trace(trace=1, count=0, ignoredirs=["/usr", sys.prefix]) 98 | # tracer.run('main()') -------------------------------------------------------------------------------- /neural_mvs_py/neural_mvs/__pycache__/MS_SSIM_L1_loss.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/neural_mvs/__pycache__/MS_SSIM_L1_loss.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/neural_mvs/__pycache__/deform_conv.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/neural_mvs/__pycache__/deform_conv.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/neural_mvs/__pycache__/funcs.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/neural_mvs/__pycache__/funcs.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/neural_mvs/__pycache__/models.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/neural_mvs/__pycache__/models.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/neural_mvs/__pycache__/modules.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/neural_mvs/__pycache__/modules.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/neural_mvs/__pycache__/nerf_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/neural_mvs/__pycache__/nerf_utils.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/neural_mvs/__pycache__/pac.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/neural_mvs/__pycache__/pac.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/neural_mvs/__pycache__/smooth_loss.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/neural_mvs/__pycache__/smooth_loss.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/neural_mvs/__pycache__/ssim.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/neural_mvs/__pycache__/ssim.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/neural_mvs/__pycache__/unet_model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/neural_mvs/__pycache__/unet_model.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/neural_mvs/__pycache__/unet_parts.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/neural_mvs/__pycache__/unet_parts.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/neural_mvs/__pycache__/utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/neural_mvs/__pycache__/utils.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/neural_mvs/__pycache__/warmup_scheduler.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/neural_mvs/__pycache__/warmup_scheduler.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/neural_mvs/modules.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from easypbr import * 5 | import sys 6 | import math 7 | 8 | 9 | from neural_mvs_py.neural_mvs.utils import * 10 | 11 | 12 | #wraps module, and changes them to become a torchscrip version of them during inference 13 | class TorchScriptTraceWrapper(torch.nn.Module): 14 | def __init__(self, module): 15 | super(TorchScriptTraceWrapper, self).__init__() 16 | 17 | self.module=module 18 | self.module_traced=None 19 | 20 | def forward(self, *args): 21 | args_list=[] 22 | for arg in args: 23 | args_list.append(arg) 24 | if self.module_traced==None: 25 | self.module_traced = torch.jit.trace(self.module, args_list ) 26 | return self.module_traced(*args) 27 | 28 | class FrameWeightComputer(torch.nn.Module): 29 | 30 | def __init__(self ): 31 | super(FrameWeightComputer, self).__init__() 32 | 33 | # self.s_weight = torch.nn.Parameter(torch.randn(1)) #from equaiton 3 here https://arxiv.org/pdf/2010.08888.pdf 34 | # with torch.set_grad_enabled(False): 35 | # # self.s_weight.fill_(0.5) 36 | # self.s_weight.fill_(10.0) 37 | 38 | ######CREATING A NEW parameter for the s_weight for some reason destroys the rest of the network and it doesnt optimize anymore. Either way, it barely changes so we just set it to 10 39 | # self.s_weight=10 40 | self.s_weight=1 41 | 42 | def forward(self, frame, frames_close): 43 | cur_dir=frame.look_dir 44 | exponential_weight_towards_neighbour=[] 45 | for i in range(len(frames_close)): 46 | dir_neighbour=frames_close[i].look_dir 47 | dot= torch.dot( cur_dir.view(-1), dir_neighbour.view(-1) ) 48 | s_dot= self.s_weight*(dot-1) 49 | exp=torch.exp(s_dot) 50 | exponential_weight_towards_neighbour.append(exp.view(1)) 51 | all_exp=torch.cat(exponential_weight_towards_neighbour) 52 | exp_minimum= all_exp.min() 53 | unnormalized_weights=[] 54 | for i in range(len(frames_close)): 55 | cur_exp= exponential_weight_towards_neighbour[i] 56 | exp_sub_min= cur_exp-exp_minimum 57 | unnormalized_weight= torch.relu(exp_sub_min) 58 | unnormalized_weights.append(unnormalized_weight) 59 | # print("unnormalized_weight", unnormalized_weight) 60 | all_unormalized_weights=torch.cat(unnormalized_weights) 61 | weight_sum=all_unormalized_weights.sum() 62 | weights=[] 63 | for i in range(len(frames_close)): 64 | unnormalized_weight= unnormalized_weights[i] 65 | weight= unnormalized_weight/weight_sum 66 | weights.append(weight) 67 | weights=torch.cat(weights) 68 | 69 | # ##attempt 2 by just using barycentric coords 70 | # frames_close_list=[] 71 | # for framepy in frames_close: 72 | # frames_close_list.append(framepy.frame) 73 | # weights_vec=SFM.compute_frame_weights(frame.frame, frames_close_list) 74 | # # print("weigrs vec is ", weights_vec) 75 | # weights=torch.from_numpy( np.array(weights_vec) ).float().to("cuda") 76 | # #clamp them 77 | # weights=torch.clamp(weights,0.0, 1.0) 78 | 79 | 80 | return weights 81 | 82 | 83 | 84 | 85 | 86 | 87 | class WNConvActiv(torch.nn.Module): 88 | def __init__(self, in_channels, out_channels, kernel_size, stride, padding, dilation, bias, with_dropout, transposed, activ=torch.nn.Mish(), init=None, do_norm=False, is_first_layer=False ): 89 | # def __init__(self, out_channels, kernel_size, stride, padding, dilation, bias, with_dropout, transposed, activ=torch.nn.GELU(), init=None ): 90 | # def __init__(self, out_channels, kernel_size, stride, padding, dilation, bias, with_dropout, transposed, activ=torch.sin, init=None ): 91 | # def __init__(self, out_channels, kernel_size, stride, padding, dilation, bias, with_dropout, transposed, activ=torch.nn.ELU(), init=None ): 92 | # def __init__(self, out_channels, kernel_size, stride, padding, dilation, bias, with_dropout, transposed, activ=torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), init=None ): 93 | # def __init__(self, out_channels, kernel_size, stride, padding, dilation, bias, with_dropout, transposed, activ=torch.nn.SELU(inplace=False), init=None ): 94 | super(WNConvActiv, self).__init__() 95 | self.out_channels=out_channels 96 | self.kernel_size=kernel_size 97 | self.stride=stride 98 | self.padding=padding 99 | self.dilation=dilation 100 | self.bias=bias 101 | self.conv= None 102 | self.norm= None 103 | # self.relu=torch.nn.ReLU(inplace=False) 104 | self.activ=activ 105 | self.with_dropout=with_dropout 106 | self.transposed=transposed 107 | # self.cc=ConcatCoord() 108 | self.init=init 109 | self.do_norm=do_norm 110 | self.is_first_layer=is_first_layer 111 | 112 | if with_dropout: 113 | self.drop=torch.nn.Dropout2d(0.2) 114 | 115 | # if do_norm: 116 | # nr_groups=32 117 | # #if the groups is not diivsalbe so for example if we have 80 params 118 | # if in_channels%nr_groups!=0: 119 | # nr_groups= int(in_channels/4) 120 | # if in_channels==32: 121 | # nr_groups= int(in_channels/4) 122 | # # print("nr groups is ", nr_groups, " in channels ", in_channels) 123 | # self.norm = torch.nn.GroupNorm(nr_groups, in_channels).cuda() 124 | # self.norm = torch.nn.GroupNorm(in_channels, in_channels).cuda() 125 | # self.norm = torch.nn.GroupNorm( int(in_channels/4), in_channels).cuda() 126 | # self.norm = torch.nn.GroupNorm(1, in_channels).cuda() 127 | if do_norm: 128 | if self.transposed: 129 | self.conv= ConvTranspose2dWN(in_channels, self.out_channels, kernel_size=self.kernel_size, stride=self.stride, padding=self.padding, dilation=self.dilation, groups=1, bias=self.bias).cuda() 130 | else: 131 | self.conv= Conv2dWN(in_channels, self.out_channels, kernel_size=self.kernel_size, stride=self.stride, padding=self.padding, dilation=self.dilation, groups=1, bias=self.bias).cuda() 132 | # self.conv= DeformConv2d(in_channels, self.out_channels, kernel_size=self.kernel_size, stride=self.stride, padding=self.padding, bias=self.bias).cuda() 133 | else: 134 | if self.transposed: 135 | self.conv=torch.nn.ConvTranspose2d(in_channels, self.out_channels, kernel_size=self.kernel_size, stride=self.stride, padding=self.padding, dilation=self.dilation, groups=1, bias=self.bias).cuda() 136 | else: 137 | self.conv= torch.nn.Conv2d(in_channels, self.out_channels, kernel_size=self.kernel_size, stride=self.stride, padding=self.padding, dilation=self.dilation, groups=1, bias=self.bias).cuda() 138 | 139 | 140 | # self.conv= torch.nn.Conv2d(in_channels, self.out_channels, kernel_size=self.kernel_size, stride=self.stride, padding=self.padding, dilation=self.dilation, groups=1, bias=self.bias).cuda() 141 | 142 | 143 | 144 | # print("initializing with kaiming uniform") 145 | # torch.nn.init.kaiming_uniform_(self.conv.weight, a=math.sqrt(5), mode='fan_in', nonlinearity='relu') 146 | # if self.bias is not False: 147 | # fan_in, _ = torch.nn.init._calculate_fan_in_and_fan_out(self.conv.weight) 148 | # bound = 1 / math.sqrt(fan_in) 149 | # torch.nn.init.uniform_(self.conv.bias, -bound, bound) 150 | 151 | 152 | def forward(self, x): 153 | # if params is None: 154 | # params = OrderedDict(self.named_parameters()) 155 | # print("params is", params) 156 | 157 | # x=self.cc(x) 158 | 159 | in_channels=x.shape[1] 160 | 161 | # if self.do_norm: 162 | # x=self.norm(x) 163 | x = self.conv(x ) 164 | if self.activ !=None: 165 | x=self.activ(x) 166 | 167 | 168 | return x 169 | 170 | 171 | 172 | class TwoBlock2D(torch.nn.Module): 173 | 174 | def __init__(self, out_channels, kernel_size, stride, padding, dilations, biases, with_dropout, do_norm=False, activ=torch.nn.Mish(), is_first_layer=False, block_type=WNConvActiv ): 175 | # def __init__(self, out_channels, kernel_size, stride, padding, dilations, biases, with_dropout, do_norm=False, activ=torch.nn.GELU(), is_first_layer=False ): 176 | # def __init__(self, out_channels, kernel_size, stride, padding, dilations, biases, with_dropout, do_norm=False, activ=torch.nn.GELU(), is_first_layer=False ): 177 | super(TwoBlock2D, self).__init__() 178 | 179 | #again with bn-relu-conv 180 | # self.conv1=GnReluConv(out_channels, kernel_size=3, stride=1, padding=1, dilation=dilations[0], bias=biases[0], with_dropout=False, transposed=False) 181 | # self.conv2=GnReluConv(out_channels, kernel_size=3, stride=1, padding=1, dilation=dilations[0], bias=biases[0], with_dropout=with_dropout, transposed=False) 182 | 183 | # self.conv1=BlockForResnet(in_channels=out_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilations[0], bias=biases[0], with_dropout=False, transposed=False, do_norm=do_norm, activ=activ, is_first_layer=is_first_layer ) 184 | # self.conv2=BlockForResnet(in_channels=out_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilations[0], bias=biases[0], with_dropout=with_dropout, transposed=False, do_norm=do_norm, activ=activ, is_first_layer=False ) 185 | 186 | # self.conv1=BlockPAC(in_channels=out_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilations[0], bias=biases[0], with_dropout=False, transposed=False, do_norm=do_norm, activ=activ, is_first_layer=is_first_layer ) 187 | # self.conv2=BlockPAC(in_channels=out_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilations[0], bias=biases[0], with_dropout=with_dropout, transposed=False, do_norm=do_norm, activ=activ, is_first_layer=False ) 188 | 189 | self.conv1=block_type(in_channels=out_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilations[0], bias=biases[0], with_dropout=False, transposed=False, do_norm=do_norm, activ=activ, is_first_layer=is_first_layer ) 190 | self.conv2=block_type(in_channels=out_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilations[0], bias=biases[0], with_dropout=with_dropout, transposed=False, do_norm=do_norm, activ=activ, is_first_layer=False ) 191 | 192 | def forward(self, x): 193 | identity=x 194 | # x=self.conv1(x, x) 195 | # x=self.conv2(x, x) 196 | x=self.conv1(x) 197 | x=self.conv2(x) 198 | return x 199 | 200 | 201 | 202 | 203 | class ConcatCoord(torch.nn.Module): 204 | def __init__(self): 205 | super(ConcatCoord, self).__init__() 206 | 207 | def forward(self, x): 208 | 209 | #concat the coordinates in x an y as in coordconv https://github.com/Wizaron/coord-conv-pytorch/blob/master/coord_conv.py 210 | image_height=x.shape[2] 211 | image_width=x.shape[3] 212 | x_coords = 2.0 * torch.arange(image_width).unsqueeze( 213 | 0).expand(image_height, image_width) / (image_width - 1.0) - 1.0 214 | y_coords = 2.0 * torch.arange(image_height).unsqueeze( 215 | 1).expand(image_height, image_width) / (image_height - 1.0) - 1.0 216 | coords = torch.stack((x_coords, y_coords), dim=0).float() 217 | coords=coords.unsqueeze(0) 218 | coords=coords.repeat(x.shape[0],1,1,1) 219 | # print("coords have size ", coords.size()) 220 | x_coord = torch.cat((coords.to("cuda"), x), dim=1) 221 | 222 | return x_coord 223 | 224 | class PositionalEncoding(torch.nn.Module): 225 | def __init__(self, in_channels, num_encoding_functions): 226 | super(PositionalEncoding, self).__init__() 227 | self.in_channels=in_channels 228 | self.num_encoding_functions=num_encoding_functions 229 | 230 | out_channels=in_channels*self.num_encoding_functions*2 231 | 232 | self.conv= torch.nn.Linear(in_channels, int(out_channels/2), bias=False).cuda() #in the case we set the weight ourselves 233 | self.init_weights() 234 | 235 | 236 | #we dont train because that causes it to overfit to the input views and not generalize the specular effects to novel views 237 | self.conv.weight.requires_grad = False 238 | 239 | def init_weights(self): 240 | with torch.no_grad(): 241 | num_input = self.in_channels 242 | self.conv.weight.uniform_(-np.sqrt(6 / num_input) , np.sqrt(6 / num_input) ) 243 | # print("weight is ", self.conv.weight.shape) #60x3 244 | 245 | #we make the same as the positonal encoding, which is mutiplying each coordinate with this linespaced frequencies 246 | lin=2.0 ** torch.linspace( 247 | 0.0, 248 | self.num_encoding_functions - 1, 249 | self.num_encoding_functions, 250 | dtype=torch.float32, 251 | device=torch.device("cuda"), 252 | ) 253 | lin_size=lin.shape[0] 254 | weight=torch.zeros([self.in_channels, self.num_encoding_functions*self.in_channels], dtype=torch.float32, device=torch.device("cuda") ) 255 | for i in range(self.in_channels): 256 | weight[i:i+1, i*lin_size:i*lin_size+lin_size ] = lin 257 | 258 | weight=weight.t().contiguous() 259 | 260 | #set the new weights = 261 | self.conv.weight=torch.nn.Parameter(weight) 262 | # self.conv.weight.requires_grad=False 263 | # print("weight is", weight.shape) 264 | # print("bias is", self.conv.bias.shape) 265 | # print("weight is", weight) 266 | 267 | self.weights_initialized=True 268 | 269 | 270 | def forward(self, x): 271 | 272 | with torch.no_grad(): 273 | 274 | x_proj = self.conv(x) 275 | 276 | # if self.only_sin: 277 | # return torch.cat([x, torch.sin(x_proj) ], -1) 278 | # else: 279 | return torch.cat([torch.sin(x_proj), torch.cos(x_proj), x], -1) 280 | 281 | -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/__pycache__/adahessian.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/optimizers/__pycache__/adahessian.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/__pycache__/ranger2020.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AIS-Bonn/neural_mvs/3945b31062d91e004dc4c7bf36f6c21aa218d415/neural_mvs_py/optimizers/__pycache__/ranger2020.cpython-36.pyc -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled source # 2 | ################### 3 | *.com 4 | *.class 5 | *.dll 6 | *.exe 7 | *.o 8 | *.so 9 | 10 | # Packages # 11 | ############ 12 | # it's better to unpack these files and commit the raw source 13 | # git has its own built in compression methods 14 | *.7z 15 | *.dmg 16 | *.gz 17 | *.iso 18 | *.jar 19 | *.rar 20 | *.tar 21 | *.zip 22 | 23 | # Logs and databases # 24 | ###################### 25 | *.log 26 | *.sql 27 | *.sqlite 28 | 29 | # OS generated files # 30 | ###################### 31 | .DS_Store 32 | .DS_Store? 33 | ._* 34 | .Spotlight-V100 35 | .Trashes 36 | ehthumbs.db 37 | Thumbs.db 38 | 39 | *-checkpoint.ipynb 40 | __pycache__/ 41 | .vscode/settings.json 42 | 43 | wandb/* 44 | postgres/* 45 | data/* 46 | runs/* -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/README.md: -------------------------------------------------------------------------------- 1 | # Optimizers and tests 2 | 3 | Every result is avg of 20 runs. 4 | 5 | Dataset | LR Schedule| Imagenette size 128, 5 epoch | Imagewoof size 128, 5 epoch 6 | --- | -- | --- | --- 7 | Adam - baseline |OneCycle| 0.8493 | 0.6125 8 | RangerLars (RAdam + LARS + Lookahead) |Flat and anneal| 0.8732 | 0.6523 9 | Ralamb (RAdam + LARS) |Flat and anneal| 0.8675 | 0.6367 10 | Ranger (RAdam + Lookahead) |Flat and anneal| 0.8594 | 0.5946 11 | Novograd |Flat and anneal| 0.8711 | 0.6126 12 | Radam |Flat and anneal| 0.8444 | 0.537 13 | Lookahead |OneCycle| 0.8578 | 0.6106 14 | Lamb |OneCycle| 0.8400 | 0.5597 15 | DiffGrad |OneCycle| 0.8527 | 0.5912 16 | AdaMod |OneCycle| 0.8473 | 0.6132 17 | -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/adamod.py: -------------------------------------------------------------------------------- 1 | # original repo https://github.com/lancopku/AdaMod/blob/master/adamod/adamod.py 2 | import math 3 | import torch 4 | from torch.optim import Optimizer 5 | 6 | class AdaMod(Optimizer): 7 | """Implements AdaMod algorithm with Decoupled Weight Decay (arxiv.org/abs/1711.05101) 8 | It has been proposed in `Adaptive and Momental Bounds for Adaptive Learning Rate Methods`_. 9 | Arguments: 10 | params (iterable): iterable of parameters to optimize or dicts defining 11 | parameter groups 12 | lr (float, optional): learning rate (default: 1e-3) 13 | betas (Tuple[float, float], optional): coefficients used for computing 14 | running averages of gradient and its square (default: (0.9, 0.999)) 15 | beta3 (float, optional): smoothing coefficient for adaptive learning rates (default: 0.9999) 16 | eps (float, optional): term added to the denominator to improve 17 | numerical stability (default: 1e-8) 18 | weight_decay (float, optional): weight decay (L2 penalty) (default: 0) 19 | """ 20 | 21 | def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), beta3=0.999, 22 | eps=1e-8, weight_decay=0): 23 | if not 0.0 <= lr: 24 | raise ValueError("Invalid learning rate: {}".format(lr)) 25 | if not 0.0 <= eps: 26 | raise ValueError("Invalid epsilon value: {}".format(eps)) 27 | if not 0.0 <= betas[0] < 1.0: 28 | raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) 29 | if not 0.0 <= betas[1] < 1.0: 30 | raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) 31 | if not 0.0 <= beta3 < 1.0: 32 | raise ValueError("Invalid beta3 parameter: {}".format(beta3)) 33 | defaults = dict(lr=lr, betas=betas, beta3=beta3, eps=eps, 34 | weight_decay=weight_decay) 35 | super(AdaMod, self).__init__(params, defaults) 36 | 37 | def __setstate__(self, state): 38 | super(AdaMod, self).__setstate__(state) 39 | 40 | def step(self, closure=None): 41 | """Performs a single optimization step. 42 | Arguments: 43 | closure (callable, optional): A closure that reevaluates the model 44 | and returns the loss. 45 | """ 46 | loss = None 47 | if closure is not None: 48 | loss = closure() 49 | 50 | for group in self.param_groups: 51 | for p in group['params']: 52 | if p.grad is None: 53 | continue 54 | grad = p.grad.data 55 | if grad.is_sparse: 56 | raise RuntimeError( 57 | 'AdaMod does not support sparse gradients') 58 | 59 | state = self.state[p] 60 | 61 | # State initialization 62 | if len(state) == 0: 63 | state['step'] = 0 64 | # Exponential moving average of gradient values 65 | state['exp_avg'] = torch.zeros_like(p.data) 66 | # Exponential moving average of squared gradient values 67 | state['exp_avg_sq'] = torch.zeros_like(p.data) 68 | # Exponential moving average of actual learning rates 69 | state['exp_avg_lr'] = torch.zeros_like(p.data) 70 | 71 | exp_avg, exp_avg_sq, exp_avg_lr = state['exp_avg'], state['exp_avg_sq'], state['exp_avg_lr'] 72 | beta1, beta2 = group['betas'] 73 | 74 | state['step'] += 1 75 | 76 | # Decay the first and second moment running average coefficient 77 | exp_avg.mul_(beta1).add_(1 - beta1, grad) 78 | exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) 79 | 80 | denom = exp_avg_sq.sqrt().add_(group['eps']) 81 | 82 | bias_correction1 = 1 - beta1 ** state['step'] 83 | bias_correction2 = 1 - beta2 ** state['step'] 84 | step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1 85 | 86 | if group['weight_decay'] != 0: 87 | p.data.add_(-group['weight_decay'] * group['lr'], p.data) 88 | 89 | # Applies momental bounds on actual learning rates 90 | step_size = torch.full_like(denom, step_size) 91 | step_size.div_(denom) 92 | exp_avg_lr.mul_(group['beta3']).add_(1 - group['beta3'], step_size) 93 | step_size = torch.min(step_size, exp_avg_lr) 94 | step_size.mul_(exp_avg) 95 | 96 | p.data.add_(-step_size) 97 | 98 | return loss 99 | -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/apollo.py: -------------------------------------------------------------------------------- 1 | # from https://raw.githubusercontent.com/XuezheMax/apollo/master/optim/apollo.py 2 | import torch 3 | from torch.optim.optimizer import Optimizer 4 | 5 | 6 | import numpy as np 7 | import torch 8 | from torch.optim.optimizer import Optimizer 9 | 10 | 11 | class Apollo(Optimizer): 12 | r"""Implements Atom algorithm. 13 | 14 | Arguments: 15 | params (iterable): iterable of parameters to optimize or dicts defining 16 | parameter groups 17 | lr (float): learning rate 18 | beta (float, optional): coefficient used for computing running averages of gradient (default: 0.9) 19 | eps (float, optional): term added to the denominator to improve numerical stability (default: 1e-4) 20 | rebound (str, optional): recified bound for diagonal hessian: 21 | ``'constant'`` | ``'belief'`` (default: None) 22 | warmup (int, optional): number of warmup steps (default: 100) 23 | init_lr (float, optional): initial learning rate for warmup (default: 0.01) 24 | weight_decay (float, optional): weight decay coefficient (default: 0) 25 | weight_decay_type (str, optional): type of weight decay: 26 | ``'L2'`` | ``'decoupled'`` | ``'stable'`` (default: 'L2') 27 | """ 28 | 29 | def __init__(self, params, lr, beta=0.9, eps=1e-4, rebound='constant', warmup=100, init_lr=0.01, weight_decay=0, weight_decay_type=None): 30 | if not 0.0 < lr: 31 | raise ValueError("Invalid learning rate value: {}".format(lr)) 32 | if not 0.0 <= eps: 33 | raise ValueError("Invalid epsilon value: {}".format(eps)) 34 | if not 0.0 <= beta < 1.0: 35 | raise ValueError("Invalid beta parameter at index 0: {}".format(beta)) 36 | if rebound not in ['constant', 'belief']: 37 | raise ValueError("Invalid recitifed bound: {}".format(rebound)) 38 | if not 0.0 <= warmup: 39 | raise ValueError("Invalid warmup updates: {}".format(warmup)) 40 | if not 0.0 <= init_lr <= lr: 41 | raise ValueError("Invalid initial learning rate: {}".format(init_lr)) 42 | if not 0.0 <= weight_decay: 43 | raise ValueError("Invalid weight_decay value: {}".format(weight_decay)) 44 | if weight_decay_type is None: 45 | weight_decay_type = 'L2' if rebound == 'constant' else 'decoupled' 46 | if weight_decay_type not in ['L2', 'decoupled', 'stable']: 47 | raise ValueError("Invalid weight decay type: {}".format(weight_decay_type)) 48 | 49 | defaults = dict(lr=lr, beta=beta, eps=eps, rebound=rebound, 50 | warmup=warmup, init_lr=init_lr, base_lr=lr, 51 | weight_decay=weight_decay, weight_decay_type=weight_decay_type) 52 | super(Apollo, self).__init__(params, defaults) 53 | 54 | def __setstate__(self, state): 55 | super(Apollo, self).__setstate__(state) 56 | 57 | @torch.no_grad() 58 | def step(self, closure=None): 59 | """Performs a single optimization step. 60 | Arguments: 61 | closure (callable, optional): A closure that reevaluates the model 62 | and returns the loss. 63 | """ 64 | loss = None 65 | if closure is not None: 66 | with torch.enable_grad(): 67 | loss = closure() 68 | 69 | for group in self.param_groups: 70 | for p in group['params']: 71 | if p.grad is None: 72 | continue 73 | 74 | state = self.state[p] 75 | 76 | # State initialization 77 | if len(state) == 0: 78 | state['step'] = 0 79 | # Exponential moving average of gradient values 80 | state['exp_avg_grad'] = torch.zeros_like(p, memory_format=torch.preserve_format) 81 | # Exponential moving average of squared gradient values 82 | state['approx_hessian'] = torch.zeros_like(p, memory_format=torch.preserve_format) 83 | # Previous update direction 84 | state['update'] = torch.zeros_like(p, memory_format=torch.preserve_format) 85 | 86 | # Calculate current lr 87 | if state['step'] < group['warmup']: 88 | curr_lr = (group['base_lr'] - group['init_lr']) * state['step'] / group['warmup'] + group['init_lr'] 89 | else: 90 | curr_lr = group['lr'] 91 | 92 | # Perform optimization step 93 | grad = p.grad 94 | if grad.is_sparse: 95 | raise RuntimeError('Atom does not support sparse gradients.') 96 | 97 | # Perform step weight decay 98 | if group['weight_decay'] != 0 and group['weight_decay_type'] == 'L2': 99 | grad = grad.add(p, alpha=group['weight_decay']) 100 | 101 | beta = group['beta'] 102 | exp_avg_grad = state['exp_avg_grad'] 103 | B = state['approx_hessian'] 104 | d_p = state['update'] 105 | 106 | state['step'] += 1 107 | bias_correction = 1 - beta ** state['step'] 108 | alpha = (1 - beta) / bias_correction 109 | 110 | # calc the diff grad 111 | delta_grad = grad - exp_avg_grad 112 | if group['rebound'] == 'belief': 113 | rebound = delta_grad.norm(p=np.inf) 114 | else: 115 | rebound = 1.0 116 | 117 | # Update the running average grad 118 | exp_avg_grad.add_(delta_grad, alpha=alpha) 119 | 120 | denom = d_p.norm(p=4).add(group['eps']) 121 | d_p.div_(denom) 122 | v_sq = d_p.mul(d_p) 123 | delta = delta_grad.div_(denom).mul_(d_p).sum().mul(-alpha) - B.mul(v_sq).sum() 124 | 125 | # Update B 126 | B.addcmul_(v_sq, delta) 127 | 128 | # calc direction of parameter updates 129 | if group['rebound'] == 'belief': 130 | denom = torch.max(B.abs(), rebound).add_(group['eps'] / alpha) 131 | else: 132 | denom = B.abs().clamp_(min=rebound) 133 | 134 | d_p.copy_(exp_avg_grad.div(denom)) 135 | 136 | # Perform step weight decay 137 | if group['weight_decay'] != 0 and group['weight_decay_type'] != 'L2': 138 | if group['weight_decay_type'] == 'stable': 139 | weight_decay = group['weight_decay'] / denom.mean().item() 140 | else: 141 | weight_decay = group['weight_decay'] 142 | d_p.add_(p, alpha=weight_decay) 143 | 144 | p.add_(d_p, alpha=-curr_lr) 145 | 146 | return loss 147 | -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/diffgrad.py: -------------------------------------------------------------------------------- 1 | 2 | import math 3 | import torch 4 | from torch.optim.optimizer import Optimizer 5 | import numpy as np 6 | import torch.nn as nn 7 | 8 | # Original source: https://github.com/shivram1987/diffGrad/blob/master/diffGrad.py 9 | 10 | # modifications: @lessw2020 11 | # https://github.com/lessw2020/Best-Deep-Learning-Optimizers/blob/master/diffgrad/diffgrad.py 12 | 13 | class DiffGrad(Optimizer): 14 | r"""Implements diffGrad algorithm. It is modified from the pytorch implementation of Adam. 15 | It has been proposed in `diffGrad: An Optimization Method for Convolutional Neural Networks`_. 16 | Arguments: 17 | params (iterable): iterable of parameters to optimize or dicts defining 18 | parameter groups 19 | lr (float, optional): learning rate (default: 1e-3) 20 | betas (Tuple[float, float], optional): coefficients used for computing 21 | running averages of gradient and its square (default: (0.9, 0.999)) 22 | eps (float, optional): term added to the denominator to improve 23 | numerical stability (default: 1e-8) 24 | weight_decay (float, optional): weight decay (L2 penalty) (default: 0) 25 | amsgrad (boolean, optional): whether to use the AMSGrad variant of this 26 | algorithm from the paper `On the Convergence of Adam and Beyond`_ 27 | (default: False) 28 | .. _diffGrad: An Optimization Method for Convolutional Neural Networks: 29 | https://arxiv.org/abs/1909.11015 30 | .. _Adam\: A Method for Stochastic Optimization: 31 | https://arxiv.org/abs/1412.6980 32 | .. _On the Convergence of Adam and Beyond: 33 | https://openreview.net/forum?id=ryQu7f-RZ 34 | """ 35 | 36 | def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, version=0, weight_decay=0): 37 | if not 0.0 <= lr: 38 | raise ValueError("Invalid learning rate: {}".format(lr)) 39 | if not 0.0 <= eps: 40 | raise ValueError("Invalid epsilon value: {}".format(eps)) 41 | if not 0.0 <= betas[0] < 1.0: 42 | raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) 43 | if not 0.0 <= betas[1] < 1.0: 44 | raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) 45 | 46 | 47 | defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) 48 | 49 | super().__init__(params, defaults) 50 | 51 | #save version 52 | self.version = version 53 | 54 | def __setstate__(self, state): 55 | super().__setstate__(state) 56 | 57 | def step(self, closure=None): 58 | """Performs a single optimization step. 59 | Arguments: 60 | closure (callable, optional): A closure that reevaluates the model 61 | and returns the loss. 62 | """ 63 | loss = None 64 | if closure is not None: 65 | loss = closure() 66 | 67 | for group in self.param_groups: 68 | for p in group['params']: 69 | if p.grad is None: 70 | continue 71 | grad = p.grad.data 72 | if grad.is_sparse: 73 | raise RuntimeError('diffGrad does not support sparse gradients, please consider SparseAdam instead') 74 | 75 | state = self.state[p] 76 | 77 | # State initialization 78 | if len(state) == 0: 79 | state['step'] = 0 80 | # Exponential moving average of gradient values 81 | state['exp_avg'] = torch.zeros_like(p.data) 82 | # Exponential moving average of squared gradient values 83 | state['exp_avg_sq'] = torch.zeros_like(p.data) 84 | # Previous gradient 85 | state['previous_grad'] = torch.zeros_like(p.data) 86 | 87 | exp_avg, exp_avg_sq, previous_grad = state['exp_avg'], state['exp_avg_sq'], state['previous_grad'] 88 | beta1, beta2 = group['betas'] 89 | 90 | state['step'] += 1 91 | 92 | if group['weight_decay'] != 0: 93 | grad.add_(group['weight_decay'], p.data) 94 | 95 | # Decay the first and second moment running average coefficient 96 | exp_avg.mul_(beta1).add_(1 - beta1, grad) 97 | exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) 98 | denom = exp_avg_sq.sqrt().add_(group['eps']) 99 | 100 | bias_correction1 = 1 - beta1 ** state['step'] 101 | bias_correction2 = 1 - beta2 ** state['step'] 102 | 103 | # compute diffgrad coefficient (dfc) 104 | 105 | 106 | if self.version==0: 107 | diff = abs(previous_grad - grad) 108 | elif self.version ==1: 109 | diff = previous_grad-grad 110 | elif self.version ==2: 111 | diff = .5*abs(previous_grad - grad) 112 | 113 | if self.version==0 or self.version==1: 114 | dfc = 1. / (1. + torch.exp(-diff)) 115 | elif self.version==2: 116 | dfc = 9. / (1. + torch.exp(-diff))-4 #DFC2 = 9/(1+e-(.5/g/)-4 #range .5,5 117 | 118 | state['previous_grad'] = grad 119 | 120 | # update momentum with dfc 121 | exp_avg1 = exp_avg * dfc 122 | 123 | step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1 124 | 125 | p.data.addcdiv_(-step_size, exp_avg1, denom) 126 | 127 | return loss -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/environment.yml: -------------------------------------------------------------------------------- 1 | name: fastai 2 | channels: 3 | - fastai 4 | - pytorch 5 | - conda-forge 6 | - defaults 7 | dependencies: 8 | - ipykernel 9 | - python 10 | - pip 11 | - fastai 12 | - pytorch>=1.2 13 | - future 14 | - torchvision 15 | - cuda100 16 | - cudatoolkit 17 | - absl-py 18 | - statsmodels 19 | - pip: 20 | - wandb 21 | 22 | -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/lamb.py: -------------------------------------------------------------------------------- 1 | """Lamb optimizer.""" 2 | # from https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py 3 | 4 | import collections 5 | import math 6 | 7 | import torch 8 | from torch.optim import Optimizer 9 | 10 | try: 11 | from tensorboardX import SummaryWriter 12 | 13 | def log_lamb_rs(optimizer: Optimizer, event_writer: SummaryWriter, token_count: int): 14 | """Log a histogram of trust ratio scalars in across layers.""" 15 | results = collections.defaultdict(list) 16 | for group in optimizer.param_groups: 17 | for p in group['params']: 18 | state = optimizer.state[p] 19 | for i in ('weight_norm', 'adam_norm', 'trust_ratio'): 20 | if i in state: 21 | results[i].append(state[i]) 22 | 23 | for k, v in results.items(): 24 | event_writer.add_histogram(f'lamb/{k}', torch.tensor(v), token_count) 25 | except ModuleNotFoundError as e: 26 | print("To use this log_lamb_rs, please run 'pip install tensorboardx'. Also you must have Tensorboard running to see results") 27 | 28 | class Lamb(Optimizer): 29 | r"""Implements Lamb algorithm. 30 | 31 | It has been proposed in `Large Batch Optimization for Deep Learning: Training BERT in 76 minutes`_. 32 | 33 | Arguments: 34 | params (iterable): iterable of parameters to optimize or dicts defining 35 | parameter groups 36 | lr (float, optional): learning rate (default: 1e-3) 37 | betas (Tuple[float, float], optional): coefficients used for computing 38 | running averages of gradient and its square (default: (0.9, 0.999)) 39 | eps (float, optional): term added to the denominator to improve 40 | numerical stability (default: 1e-8) 41 | weight_decay (float, optional): weight decay (L2 penalty) (default: 0) 42 | adam (bool, optional): always use trust ratio = 1, which turns this into 43 | Adam. Useful for comparison purposes. 44 | 45 | .. _Large Batch Optimization for Deep Learning: Training BERT in 76 minutes: 46 | https://arxiv.org/abs/1904.00962 47 | """ 48 | 49 | def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-6, 50 | weight_decay=0, adam=False): 51 | if not 0.0 <= lr: 52 | raise ValueError("Invalid learning rate: {}".format(lr)) 53 | if not 0.0 <= eps: 54 | raise ValueError("Invalid epsilon value: {}".format(eps)) 55 | if not 0.0 <= betas[0] < 1.0: 56 | raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) 57 | if not 0.0 <= betas[1] < 1.0: 58 | raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) 59 | defaults = dict(lr=lr, betas=betas, eps=eps, 60 | weight_decay=weight_decay) 61 | self.adam = adam 62 | super(Lamb, self).__init__(params, defaults) 63 | 64 | def step(self, closure=None): 65 | """Performs a single optimization step. 66 | 67 | Arguments: 68 | closure (callable, optional): A closure that reevaluates the model 69 | and returns the loss. 70 | """ 71 | loss = None 72 | if closure is not None: 73 | loss = closure() 74 | 75 | for group in self.param_groups: 76 | for p in group['params']: 77 | if p.grad is None: 78 | continue 79 | grad = p.grad.data 80 | if grad.is_sparse: 81 | raise RuntimeError('Lamb does not support sparse gradients, consider SparseAdam instad.') 82 | 83 | state = self.state[p] 84 | 85 | # State initialization 86 | if len(state) == 0: 87 | state['step'] = 0 88 | # Exponential moving average of gradient values 89 | state['exp_avg'] = torch.zeros_like(p.data) 90 | # Exponential moving average of squared gradient values 91 | state['exp_avg_sq'] = torch.zeros_like(p.data) 92 | 93 | exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] 94 | beta1, beta2 = group['betas'] 95 | 96 | state['step'] += 1 97 | 98 | # Decay the first and second moment running average coefficient 99 | # m_t 100 | exp_avg.mul_(beta1).add_(1 - beta1, grad) 101 | # v_t 102 | exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) 103 | 104 | # Paper v3 does not use debiasing. 105 | # bias_correction1 = 1 - beta1 ** state['step'] 106 | # bias_correction2 = 1 - beta2 ** state['step'] 107 | # Apply bias to lr to avoid broadcast. 108 | step_size = group['lr'] # * math.sqrt(bias_correction2) / bias_correction1 109 | 110 | weight_norm = p.data.pow(2).sum().sqrt().clamp(0, 10) 111 | 112 | adam_step = exp_avg / exp_avg_sq.sqrt().add(group['eps']) 113 | if group['weight_decay'] != 0: 114 | adam_step.add_(group['weight_decay'], p.data) 115 | 116 | adam_norm = adam_step.pow(2).sum().sqrt() 117 | if weight_norm == 0 or adam_norm == 0: 118 | trust_ratio = 1 119 | else: 120 | trust_ratio = weight_norm / adam_norm 121 | state['weight_norm'] = weight_norm 122 | state['adam_norm'] = adam_norm 123 | state['trust_ratio'] = trust_ratio 124 | if self.adam: 125 | trust_ratio = 1 126 | 127 | p.data.add_(-step_size * trust_ratio, adam_step) 128 | 129 | return loss 130 | -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/lookahead.py: -------------------------------------------------------------------------------- 1 | # Lookahead implementation from https://github.com/rwightman/pytorch-image-models/blob/master/timm/optim/lookahead.py 2 | 3 | """ Lookahead Optimizer Wrapper. 4 | Implementation modified from: https://github.com/alphadl/lookahead.pytorch 5 | Paper: `Lookahead Optimizer: k steps forward, 1 step back` - https://arxiv.org/abs/1907.08610 6 | """ 7 | import torch 8 | from torch.optim.optimizer import Optimizer 9 | from collections import defaultdict 10 | 11 | class Lookahead(Optimizer): 12 | def __init__(self, base_optimizer, alpha=0.5, k=6): 13 | if not 0.0 <= alpha <= 1.0: 14 | raise ValueError(f'Invalid slow update rate: {alpha}') 15 | if not 1 <= k: 16 | raise ValueError(f'Invalid lookahead steps: {k}') 17 | defaults = dict(lookahead_alpha=alpha, lookahead_k=k, lookahead_step=0) 18 | self.base_optimizer = base_optimizer 19 | self.param_groups = self.base_optimizer.param_groups 20 | self.defaults = base_optimizer.defaults 21 | self.defaults.update(defaults) 22 | self.state = defaultdict(dict) 23 | # manually add our defaults to the param groups 24 | for name, default in defaults.items(): 25 | for group in self.param_groups: 26 | group.setdefault(name, default) 27 | 28 | def update_slow(self, group): 29 | for fast_p in group["params"]: 30 | if fast_p.grad is None: 31 | continue 32 | param_state = self.state[fast_p] 33 | if 'slow_buffer' not in param_state: 34 | param_state['slow_buffer'] = torch.empty_like(fast_p.data) 35 | param_state['slow_buffer'].copy_(fast_p.data) 36 | slow = param_state['slow_buffer'] 37 | slow.add_(group['lookahead_alpha'], fast_p.data - slow) 38 | fast_p.data.copy_(slow) 39 | 40 | def sync_lookahead(self): 41 | for group in self.param_groups: 42 | self.update_slow(group) 43 | 44 | def step(self, closure=None): 45 | # print(self.k) 46 | #assert id(self.param_groups) == id(self.base_optimizer.param_groups) 47 | loss = self.base_optimizer.step(closure) 48 | for group in self.param_groups: 49 | group['lookahead_step'] += 1 50 | if group['lookahead_step'] % group['lookahead_k'] == 0: 51 | self.update_slow(group) 52 | return loss 53 | 54 | def state_dict(self): 55 | fast_state_dict = self.base_optimizer.state_dict() 56 | slow_state = { 57 | (id(k) if isinstance(k, torch.Tensor) else k): v 58 | for k, v in self.state.items() 59 | } 60 | fast_state = fast_state_dict['state'] 61 | param_groups = fast_state_dict['param_groups'] 62 | return { 63 | 'state': fast_state, 64 | 'slow_state': slow_state, 65 | 'param_groups': param_groups, 66 | } 67 | 68 | def load_state_dict(self, state_dict): 69 | fast_state_dict = { 70 | 'state': state_dict['state'], 71 | 'param_groups': state_dict['param_groups'], 72 | } 73 | self.base_optimizer.load_state_dict(fast_state_dict) 74 | 75 | # We want to restore the slow state, but share param_groups reference 76 | # with base_optimizer. This is a bit redundant but least code 77 | slow_state_new = False 78 | if 'slow_state' not in state_dict: 79 | print('Loading state_dict from optimizer without Lookahead applied.') 80 | state_dict['slow_state'] = defaultdict(dict) 81 | slow_state_new = True 82 | slow_state_dict = { 83 | 'state': state_dict['slow_state'], 84 | 'param_groups': state_dict['param_groups'], # this is pointless but saves code 85 | } 86 | super(Lookahead, self).load_state_dict(slow_state_dict) 87 | self.param_groups = self.base_optimizer.param_groups # make both ref same container 88 | if slow_state_new: 89 | # reapply defaults to catch missing lookahead specific ones 90 | for name, default in self.defaults.items(): 91 | for group in self.param_groups: 92 | group.setdefault(name, default) 93 | 94 | def LookaheadAdam(params, alpha=0.5, k=6, *args, **kwargs): 95 | adam = Adam(params, *args, **kwargs) 96 | return Lookahead(adam, alpha, k) 97 | -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/novograd.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import torch 16 | from torch.optim import Optimizer 17 | import math 18 | 19 | class AdamW(Optimizer): 20 | """Implements AdamW algorithm. 21 | 22 | It has been proposed in `Adam: A Method for Stochastic Optimization`_. 23 | 24 | Arguments: 25 | params (iterable): iterable of parameters to optimize or dicts defining 26 | parameter groups 27 | lr (float, optional): learning rate (default: 1e-3) 28 | betas (Tuple[float, float], optional): coefficients used for computing 29 | running averages of gradient and its square (default: (0.9, 0.999)) 30 | eps (float, optional): term added to the denominator to improve 31 | numerical stability (default: 1e-8) 32 | weight_decay (float, optional): weight decay (L2 penalty) (default: 0) 33 | amsgrad (boolean, optional): whether to use the AMSGrad variant of this 34 | algorithm from the paper `On the Convergence of Adam and Beyond`_ 35 | 36 | Adam: A Method for Stochastic Optimization: 37 | https://arxiv.org/abs/1412.6980 38 | On the Convergence of Adam and Beyond: 39 | https://openreview.net/forum?id=ryQu7f-RZ 40 | """ 41 | 42 | def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, 43 | weight_decay=0, amsgrad=False): 44 | if not 0.0 <= lr: 45 | raise ValueError("Invalid learning rate: {}".format(lr)) 46 | if not 0.0 <= eps: 47 | raise ValueError("Invalid epsilon value: {}".format(eps)) 48 | if not 0.0 <= betas[0] < 1.0: 49 | raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) 50 | if not 0.0 <= betas[1] < 1.0: 51 | raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) 52 | defaults = dict(lr=lr, betas=betas, eps=eps, 53 | weight_decay=weight_decay, amsgrad=amsgrad) 54 | super(AdamW, self).__init__(params, defaults) 55 | 56 | def __setstate__(self, state): 57 | super(AdamW, self).__setstate__(state) 58 | for group in self.param_groups: 59 | group.setdefault('amsgrad', False) 60 | 61 | def step(self, closure=None): 62 | """Performs a single optimization step. 63 | 64 | Arguments: 65 | closure (callable, optional): A closure that reevaluates the model 66 | and returns the loss. 67 | """ 68 | loss = None 69 | if closure is not None: 70 | loss = closure() 71 | 72 | for group in self.param_groups: 73 | for p in group['params']: 74 | if p.grad is None: 75 | continue 76 | grad = p.grad.data 77 | if grad.is_sparse: 78 | raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead') 79 | amsgrad = group['amsgrad'] 80 | 81 | state = self.state[p] 82 | 83 | # State initialization 84 | if len(state) == 0: 85 | state['step'] = 0 86 | # Exponential moving average of gradient values 87 | state['exp_avg'] = torch.zeros_like(p.data) 88 | # Exponential moving average of squared gradient values 89 | state['exp_avg_sq'] = torch.zeros_like(p.data) 90 | if amsgrad: 91 | # Maintains max of all exp. moving avg. of sq. grad. values 92 | state['max_exp_avg_sq'] = torch.zeros_like(p.data) 93 | 94 | exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] 95 | if amsgrad: 96 | max_exp_avg_sq = state['max_exp_avg_sq'] 97 | beta1, beta2 = group['betas'] 98 | 99 | state['step'] += 1 100 | # Decay the first and second moment running average coefficient 101 | exp_avg.mul_(beta1).add_(1 - beta1, grad) 102 | exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) 103 | if amsgrad: 104 | # Maintains the maximum of all 2nd moment running avg. till now 105 | torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq) 106 | # Use the max. for normalizing running avg. of gradient 107 | denom = max_exp_avg_sq.sqrt().add_(group['eps']) 108 | else: 109 | denom = exp_avg_sq.sqrt().add_(group['eps']) 110 | 111 | bias_correction1 = 1 - beta1 ** state['step'] 112 | bias_correction2 = 1 - beta2 ** state['step'] 113 | step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1 114 | p.data.add_(-step_size, torch.mul(p.data, group['weight_decay']).addcdiv_(1, exp_avg, denom) ) 115 | 116 | return loss 117 | 118 | class Novograd(Optimizer): 119 | """ 120 | Implements Novograd algorithm. 121 | 122 | Args: 123 | params (iterable): iterable of parameters to optimize or dicts defining 124 | parameter groups 125 | lr (float, optional): learning rate (default: 1e-3) 126 | betas (Tuple[float, float], optional): coefficients used for computing 127 | running averages of gradient and its square (default: (0.95, 0)) 128 | eps (float, optional): term added to the denominator to improve 129 | numerical stability (default: 1e-8) 130 | weight_decay (float, optional): weight decay (L2 penalty) (default: 0) 131 | grad_averaging: gradient averaging 132 | amsgrad (boolean, optional): whether to use the AMSGrad variant of this 133 | algorithm from the paper `On the Convergence of Adam and Beyond`_ 134 | (default: False) 135 | """ 136 | 137 | def __init__(self, params, lr=1e-3, betas=(0.95, 0), eps=1e-8, 138 | weight_decay=0, grad_averaging=False, amsgrad=False): 139 | if not 0.0 <= lr: 140 | raise ValueError("Invalid learning rate: {}".format(lr)) 141 | if not 0.0 <= eps: 142 | raise ValueError("Invalid epsilon value: {}".format(eps)) 143 | if not 0.0 <= betas[0] < 1.0: 144 | raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) 145 | if not 0.0 <= betas[1] < 1.0: 146 | raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) 147 | defaults = dict(lr=lr, betas=betas, eps=eps, 148 | weight_decay=weight_decay, 149 | grad_averaging=grad_averaging, 150 | amsgrad=amsgrad) 151 | 152 | super(Novograd, self).__init__(params, defaults) 153 | 154 | def __setstate__(self, state): 155 | super(Novograd, self).__setstate__(state) 156 | for group in self.param_groups: 157 | group.setdefault('amsgrad', False) 158 | 159 | def step(self, closure=None): 160 | """Performs a single optimization step. 161 | 162 | Arguments: 163 | closure (callable, optional): A closure that reevaluates the model 164 | and returns the loss. 165 | """ 166 | loss = None 167 | if closure is not None: 168 | loss = closure() 169 | 170 | for group in self.param_groups: 171 | for p in group['params']: 172 | if p.grad is None: 173 | continue 174 | grad = p.grad.data 175 | if grad.is_sparse: 176 | raise RuntimeError('Sparse gradients are not supported.') 177 | amsgrad = group['amsgrad'] 178 | 179 | state = self.state[p] 180 | 181 | # State initialization 182 | if len(state) == 0: 183 | state['step'] = 0 184 | # Exponential moving average of gradient values 185 | state['exp_avg'] = torch.zeros_like(p.data) 186 | # Exponential moving average of squared gradient values 187 | state['exp_avg_sq'] = torch.zeros([]).to(state['exp_avg'].device) 188 | if amsgrad: 189 | # Maintains max of all exp. moving avg. of sq. grad. values 190 | state['max_exp_avg_sq'] = torch.zeros([]).to(state['exp_avg'].device) 191 | 192 | exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] 193 | if amsgrad: 194 | max_exp_avg_sq = state['max_exp_avg_sq'] 195 | beta1, beta2 = group['betas'] 196 | 197 | state['step'] += 1 198 | 199 | norm = torch.sum(torch.pow(grad, 2)) 200 | 201 | if exp_avg_sq == 0: 202 | exp_avg_sq.copy_(norm) 203 | else: 204 | exp_avg_sq.mul_(beta2).add_(1 - beta2, norm) 205 | 206 | if amsgrad: 207 | # Maintains the maximum of all 2nd moment running avg. till now 208 | torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq) 209 | # Use the max. for normalizing running avg. of gradient 210 | denom = max_exp_avg_sq.sqrt().add_(group['eps']) 211 | else: 212 | denom = exp_avg_sq.sqrt().add_(group['eps']) 213 | 214 | grad.div_(denom) 215 | if group['weight_decay'] != 0: 216 | grad.add_(group['weight_decay'], p.data) 217 | if group['grad_averaging']: 218 | grad.mul_(1 - beta1) 219 | exp_avg.mul_(beta1).add_(grad) 220 | 221 | p.data.add_(-group['lr'], exp_avg) 222 | 223 | return loss -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/over9000.py: -------------------------------------------------------------------------------- 1 | import torch, math 2 | from torch.optim.optimizer import Optimizer 3 | import itertools as it 4 | from lookahead import * 5 | from ralamb import * 6 | 7 | # RAdam + LARS + LookAHead 8 | 9 | # Lookahead implementation from https://github.com/lonePatient/lookahead_pytorch/blob/master/optimizer.py 10 | # RAdam + LARS implementation from https://gist.github.com/redknightlois/c4023d393eb8f92bb44b2ab582d7ec20 11 | 12 | def Over9000(params, alpha=0.5, k=6, *args, **kwargs): 13 | ralamb = Ralamb(params, *args, **kwargs) 14 | return Lookahead(ralamb, alpha, k) 15 | 16 | RangerLars = Over9000 17 | -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/radam.py: -------------------------------------------------------------------------------- 1 | # from https://github.com/LiyuanLucasLiu/RAdam/blob/master/radam.py 2 | 3 | import math 4 | import torch 5 | from torch.optim.optimizer import Optimizer, required 6 | 7 | class RAdam(Optimizer): 8 | 9 | def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0): 10 | defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) 11 | self.buffer = [[None, None, None] for ind in range(10)] 12 | super(RAdam, self).__init__(params, defaults) 13 | 14 | def __setstate__(self, state): 15 | super(RAdam, self).__setstate__(state) 16 | 17 | def step(self, closure=None): 18 | 19 | loss = None 20 | if closure is not None: 21 | loss = closure() 22 | 23 | for group in self.param_groups: 24 | 25 | for p in group['params']: 26 | if p.grad is None: 27 | continue 28 | grad = p.grad.data.float() 29 | if grad.is_sparse: 30 | raise RuntimeError('RAdam does not support sparse gradients') 31 | 32 | p_data_fp32 = p.data.float() 33 | 34 | state = self.state[p] 35 | 36 | if len(state) == 0: 37 | state['step'] = 0 38 | state['exp_avg'] = torch.zeros_like(p_data_fp32) 39 | state['exp_avg_sq'] = torch.zeros_like(p_data_fp32) 40 | else: 41 | state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32) 42 | state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32) 43 | 44 | exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] 45 | beta1, beta2 = group['betas'] 46 | 47 | exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) 48 | exp_avg.mul_(beta1).add_(1 - beta1, grad) 49 | 50 | state['step'] += 1 51 | buffered = self.buffer[int(state['step'] % 10)] 52 | if state['step'] == buffered[0]: 53 | N_sma, step_size = buffered[1], buffered[2] 54 | else: 55 | buffered[0] = state['step'] 56 | beta2_t = beta2 ** state['step'] 57 | N_sma_max = 2 / (1 - beta2) - 1 58 | N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t) 59 | buffered[1] = N_sma 60 | 61 | # more conservative since it's an approximated value 62 | if N_sma >= 5: 63 | step_size = math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step']) 64 | else: 65 | step_size = 1.0 / (1 - beta1 ** state['step']) 66 | buffered[2] = step_size 67 | 68 | if group['weight_decay'] != 0: 69 | p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32) 70 | 71 | # more conservative since it's an approximated value 72 | if N_sma >= 5: 73 | denom = exp_avg_sq.sqrt().add_(group['eps']) 74 | p_data_fp32.addcdiv_(-step_size * group['lr'], exp_avg, denom) 75 | else: 76 | p_data_fp32.add_(-step_size * group['lr'], exp_avg) 77 | 78 | p.data.copy_(p_data_fp32) 79 | 80 | return loss 81 | 82 | class PlainRAdam(Optimizer): 83 | 84 | def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0): 85 | defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) 86 | 87 | super(PlainRAdam, self).__init__(params, defaults) 88 | 89 | def __setstate__(self, state): 90 | super(PlainRAdam, self).__setstate__(state) 91 | 92 | def step(self, closure=None): 93 | 94 | loss = None 95 | if closure is not None: 96 | loss = closure() 97 | 98 | for group in self.param_groups: 99 | 100 | for p in group['params']: 101 | if p.grad is None: 102 | continue 103 | grad = p.grad.data.float() 104 | if grad.is_sparse: 105 | raise RuntimeError('RAdam does not support sparse gradients') 106 | 107 | p_data_fp32 = p.data.float() 108 | 109 | state = self.state[p] 110 | 111 | if len(state) == 0: 112 | state['step'] = 0 113 | state['exp_avg'] = torch.zeros_like(p_data_fp32) 114 | state['exp_avg_sq'] = torch.zeros_like(p_data_fp32) 115 | else: 116 | state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32) 117 | state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32) 118 | 119 | exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] 120 | beta1, beta2 = group['betas'] 121 | 122 | exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) 123 | exp_avg.mul_(beta1).add_(1 - beta1, grad) 124 | 125 | state['step'] += 1 126 | beta2_t = beta2 ** state['step'] 127 | N_sma_max = 2 / (1 - beta2) - 1 128 | N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t) 129 | 130 | if group['weight_decay'] != 0: 131 | p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32) 132 | 133 | # more conservative since it's an approximated value 134 | if N_sma >= 5: 135 | step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step']) 136 | denom = exp_avg_sq.sqrt().add_(group['eps']) 137 | p_data_fp32.addcdiv_(-step_size, exp_avg, denom) 138 | else: 139 | step_size = group['lr'] / (1 - beta1 ** state['step']) 140 | p_data_fp32.add_(-step_size, exp_avg) 141 | 142 | p.data.copy_(p_data_fp32) 143 | 144 | return loss 145 | 146 | 147 | class AdamW(Optimizer): 148 | 149 | def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0, warmup = 0): 150 | defaults = dict(lr=lr, betas=betas, eps=eps, 151 | weight_decay=weight_decay, warmup = warmup) 152 | super(AdamW, self).__init__(params, defaults) 153 | 154 | def __setstate__(self, state): 155 | super(AdamW, self).__setstate__(state) 156 | 157 | def step(self, closure=None): 158 | loss = None 159 | if closure is not None: 160 | loss = closure() 161 | 162 | for group in self.param_groups: 163 | 164 | for p in group['params']: 165 | if p.grad is None: 166 | continue 167 | grad = p.grad.data.float() 168 | if grad.is_sparse: 169 | raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead') 170 | 171 | p_data_fp32 = p.data.float() 172 | 173 | state = self.state[p] 174 | 175 | if len(state) == 0: 176 | state['step'] = 0 177 | state['exp_avg'] = torch.zeros_like(p_data_fp32) 178 | state['exp_avg_sq'] = torch.zeros_like(p_data_fp32) 179 | else: 180 | state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32) 181 | state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32) 182 | 183 | exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] 184 | beta1, beta2 = group['betas'] 185 | 186 | state['step'] += 1 187 | 188 | exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) 189 | exp_avg.mul_(beta1).add_(1 - beta1, grad) 190 | 191 | denom = exp_avg_sq.sqrt().add_(group['eps']) 192 | bias_correction1 = 1 - beta1 ** state['step'] 193 | bias_correction2 = 1 - beta2 ** state['step'] 194 | 195 | if group['warmup'] > state['step']: 196 | scheduled_lr = 1e-8 + state['step'] * group['lr'] / group['warmup'] 197 | else: 198 | scheduled_lr = group['lr'] 199 | 200 | step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1 201 | 202 | if group['weight_decay'] != 0: 203 | p_data_fp32.add_(-group['weight_decay'] * scheduled_lr, p_data_fp32) 204 | 205 | p_data_fp32.addcdiv_(-step_size, exp_avg, denom) 206 | 207 | p.data.copy_(p_data_fp32) 208 | 209 | return loss 210 | -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/ralamb.py: -------------------------------------------------------------------------------- 1 | import torch, math 2 | from torch.optim.optimizer import Optimizer 3 | 4 | # RAdam + LARS 5 | class Ralamb(Optimizer): 6 | 7 | def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0): 8 | defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) 9 | self.buffer = [[None, None, None] for ind in range(10)] 10 | super(Ralamb, self).__init__(params, defaults) 11 | 12 | def __setstate__(self, state): 13 | super(Ralamb, self).__setstate__(state) 14 | 15 | def step(self, closure=None): 16 | 17 | loss = None 18 | if closure is not None: 19 | loss = closure() 20 | 21 | for group in self.param_groups: 22 | 23 | for p in group['params']: 24 | if p.grad is None: 25 | continue 26 | grad = p.grad.data.float() 27 | if grad.is_sparse: 28 | raise RuntimeError('Ralamb does not support sparse gradients') 29 | 30 | p_data_fp32 = p.data.float() 31 | 32 | state = self.state[p] 33 | 34 | if len(state) == 0: 35 | state['step'] = 0 36 | state['exp_avg'] = torch.zeros_like(p_data_fp32) 37 | state['exp_avg_sq'] = torch.zeros_like(p_data_fp32) 38 | else: 39 | state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32) 40 | state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32) 41 | 42 | exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] 43 | beta1, beta2 = group['betas'] 44 | 45 | # Decay the first and second moment running average coefficient 46 | # m_t 47 | exp_avg.mul_(beta1).add_(1 - beta1, grad) 48 | # v_t 49 | exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) 50 | 51 | state['step'] += 1 52 | buffered = self.buffer[int(state['step'] % 10)] 53 | 54 | if state['step'] == buffered[0]: 55 | N_sma, radam_step_size = buffered[1], buffered[2] 56 | else: 57 | buffered[0] = state['step'] 58 | beta2_t = beta2 ** state['step'] 59 | N_sma_max = 2 / (1 - beta2) - 1 60 | N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t) 61 | buffered[1] = N_sma 62 | 63 | # more conservative since it's an approximated value 64 | if N_sma >= 5: 65 | radam_step_size = math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step']) 66 | else: 67 | radam_step_size = 1.0 / (1 - beta1 ** state['step']) 68 | buffered[2] = radam_step_size 69 | 70 | if group['weight_decay'] != 0: 71 | p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32) 72 | 73 | # more conservative since it's an approximated value 74 | radam_step = p_data_fp32.clone() 75 | if N_sma >= 5: 76 | denom = exp_avg_sq.sqrt().add_(group['eps']) 77 | radam_step.addcdiv_(-radam_step_size * group['lr'], exp_avg, denom) 78 | else: 79 | radam_step.add_(-radam_step_size * group['lr'], exp_avg) 80 | 81 | radam_norm = radam_step.pow(2).sum().sqrt() 82 | weight_norm = p.data.pow(2).sum().sqrt().clamp(0, 10) 83 | if weight_norm == 0 or radam_norm == 0: 84 | trust_ratio = 1 85 | else: 86 | trust_ratio = weight_norm / radam_norm 87 | 88 | state['weight_norm'] = weight_norm 89 | state['adam_norm'] = radam_norm 90 | state['trust_ratio'] = trust_ratio 91 | 92 | if N_sma >= 5: 93 | p_data_fp32.addcdiv_(-radam_step_size * group['lr'] * trust_ratio, exp_avg, denom) 94 | else: 95 | p_data_fp32.add_(-radam_step_size * group['lr'] * trust_ratio, exp_avg) 96 | 97 | p.data.copy_(p_data_fp32) 98 | 99 | return loss 100 | -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/ranger.py: -------------------------------------------------------------------------------- 1 | 2 | import math 3 | import torch 4 | from torch.optim.optimizer import Optimizer, required 5 | import itertools as it 6 | from optimizers.over9000.lookahead import * 7 | from optimizers.over9000.radam import * 8 | 9 | def Ranger(params, alpha=0.5, k=6, *args, **kwargs): 10 | radam = RAdam(params, *args, **kwargs) 11 | return Lookahead(radam, alpha, k) 12 | 13 | -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/rangerlars.py: -------------------------------------------------------------------------------- 1 | import torch, math 2 | from torch.optim.optimizer import Optimizer 3 | import itertools as it 4 | from optimizers.over9000.lookahead import * 5 | from optimizers.over9000.ralamb import * 6 | 7 | # RAdam + LARS + LookAHead 8 | 9 | # Lookahead implementation from https://github.com/lonePatient/lookahead_pytorch/blob/master/optimizer.py 10 | # RAdam + LARS implementation from https://gist.github.com/redknightlois/c4023d393eb8f92bb44b2ab582d7ec20 11 | 12 | def RangerLars(params, alpha=0.5, k=6, *args, **kwargs): 13 | ralamb = Ralamb(params, *args, **kwargs) 14 | return Lookahead(ralamb, alpha, k) 15 | -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/train.py: -------------------------------------------------------------------------------- 1 | # the code mostly from https://github.com/sdoria/SimpleSelfAttention 2 | 3 | # adapted from https://github.com/fastai/fastai/blob/master/examples/train_imagenette.py 4 | # added self attention parameter 5 | # changed per gpu bs for bs_rat 6 | 7 | 8 | from fastai.script import * 9 | from fastai.vision import * 10 | from fastai.callbacks import * 11 | from fastai.distributed import * 12 | from fastprogress import fastprogress 13 | from torchvision.models import * 14 | #from fastai.vision.models.xresnet import * 15 | #from fastai.vision.models.xresnet2 import * 16 | #from fastai.vision.models.presnet import * 17 | from xresnet import * 18 | from functools import partial 19 | import statsmodels.stats.api as sms 20 | 21 | torch.backends.cudnn.benchmark = True 22 | fastprogress.MAX_COLS = 80 23 | 24 | def get_data(size, woof, bs, workers=None): 25 | if size<=128: path = URLs.IMAGEWOOF_160 if woof else URLs.IMAGENETTE_160 26 | elif size<=224: path = URLs.IMAGEWOOF_320 if woof else URLs.IMAGENETTE_320 27 | else : path = URLs.IMAGEWOOF if woof else URLs.IMAGENETTE 28 | path = untar_data(path) 29 | 30 | n_gpus = num_distrib() or 1 31 | if workers is None: workers = min(8, num_cpus()//n_gpus) 32 | 33 | return (ImageList.from_folder(path).split_by_folder(valid='val') 34 | .label_from_folder().transform(([flip_lr(p=0.5)], []), size=size) 35 | .databunch(bs=bs, num_workers=workers) 36 | .presize(size, scale=(0.35,1)) 37 | .normalize(imagenet_stats)) 38 | 39 | from radam import * 40 | from novograd import * 41 | from ranger import * 42 | from ralamb import * 43 | from over9000 import * 44 | from lookahead import * 45 | from lamb import * 46 | from diffgrad import DiffGrad 47 | from adamod import AdaMod 48 | 49 | def fit_with_annealing(learn:Learner, num_epoch:int, lr:float=defaults.lr, annealing_start:float=0.7)->None: 50 | n = len(learn.data.train_dl) 51 | anneal_start = int(n*num_epoch*annealing_start) 52 | phase0 = TrainingPhase(anneal_start).schedule_hp('lr', lr) 53 | phase1 = TrainingPhase(n*num_epoch - anneal_start).schedule_hp('lr', lr, anneal=annealing_cos) 54 | phases = [phase0, phase1] 55 | sched = GeneralScheduler(learn, phases) 56 | learn.callbacks.append(sched) 57 | learn.fit(num_epoch) 58 | 59 | def train( 60 | gpu:Param("GPU to run on", str)=None, 61 | woof: Param("Use imagewoof (otherwise imagenette)", int)=0, 62 | lr: Param("Learning rate", float)=1e-3, 63 | size: Param("Size (px: 128,192,224)", int)=128, 64 | alpha: Param("Alpha", float)=0.99, 65 | mom: Param("Momentum", float)=0.9, 66 | eps: Param("epsilon", float)=1e-6, 67 | epochs: Param("Number of epochs", int)=5, 68 | bs: Param("Batch size", int)=256, 69 | mixup: Param("Mixup", float)=0., 70 | opt: Param("Optimizer (adam,rms,sgd)", str)='adam', 71 | arch: Param("Architecture (xresnet34, xresnet50)", str)='xresnet50', 72 | sa: Param("Self-attention", int)=0, 73 | sym: Param("Symmetry for self-attention", int)=0, 74 | dump: Param("Print model; don't train", int)=0, 75 | lrfinder: Param("Run learning rate finder; don't train", int)=0, 76 | log: Param("Log file name", str)='log', 77 | sched_type: Param("LR schedule type", str)='one_cycle', 78 | ann_start: Param("Mixup", float)=-1.0, 79 | ): 80 | "Distributed training of Imagenette." 81 | 82 | bs_one_gpu = bs 83 | gpu = setup_distrib(gpu) 84 | if gpu is None: bs *= torch.cuda.device_count() 85 | if opt=='adam' : opt_func = partial(optim.Adam, betas=(mom,alpha), eps=eps) 86 | elif opt=='radam' : opt_func = partial(RAdam, betas=(mom,alpha), eps=eps) 87 | elif opt=='novograd' : opt_func = partial(Novograd, betas=(mom,alpha), eps=eps) 88 | elif opt=='rms' : opt_func = partial(optim.RMSprop, alpha=alpha, eps=eps) 89 | elif opt=='sgd' : opt_func = partial(optim.SGD, momentum=mom) 90 | elif opt=='ranger' : opt_func = partial(Ranger, betas=(mom,alpha), eps=eps) 91 | elif opt=='ralamb' : opt_func = partial(Ralamb, betas=(mom,alpha), eps=eps) 92 | elif opt=='over9000' : opt_func = partial(Over9000, betas=(mom,alpha), eps=eps) 93 | elif opt=='lookahead' : opt_func = partial(LookaheadAdam, betas=(mom,alpha), eps=eps) 94 | elif opt=='lamb' : opt_func = partial(Lamb, betas=(mom,alpha), eps=eps) 95 | elif opt=='diffgrad' : opt_func = partial(DiffGrad, version=1, betas=(mom,alpha),eps=eps) 96 | elif opt=='adamod' : opt_func = partial(AdaMod, betas=(mom,alpha), eps=eps, beta3=0.999) 97 | 98 | data = get_data(size, woof, bs) 99 | bs_rat = bs/bs_one_gpu #originally bs/256 100 | if gpu is not None: bs_rat *= max(num_distrib(), 1) 101 | if not gpu: print(f'lr: {lr}; eff_lr: {lr*bs_rat}; size: {size}; alpha: {alpha}; mom: {mom}; eps: {eps}') 102 | lr *= bs_rat 103 | 104 | m = globals()[arch] 105 | 106 | log_cb = partial(CSVLogger,filename=log) 107 | 108 | learn = (Learner(data, m(c_out=10, sa=sa, sym=sym), wd=1e-2, opt_func=opt_func, 109 | metrics=[accuracy,top_k_accuracy], 110 | bn_wd=False, true_wd=True, 111 | loss_func = LabelSmoothingCrossEntropy(), 112 | callback_fns=[log_cb]) 113 | ) 114 | print(learn.path) 115 | 116 | if dump: print(learn.model); exit() 117 | if mixup: learn = learn.mixup(alpha=mixup) 118 | learn = learn.to_fp16(dynamic=True) 119 | if gpu is None: learn.to_parallel() 120 | elif num_distrib()>1: learn.to_distributed(gpu) # Requires `-m fastai.launch` 121 | 122 | if lrfinder: 123 | # run learning rate finder 124 | IN_NOTEBOOK = 1 125 | learn.lr_find(wd=1e-2) 126 | learn.recorder.plot() 127 | else: 128 | if sched_type == 'one_cycle': 129 | learn.fit_one_cycle(epochs, lr, div_factor=10, pct_start=0.3) 130 | elif sched_type == 'flat_and_anneal': 131 | fit_with_annealing(learn, epochs, lr, ann_start) 132 | 133 | return learn.recorder.metrics[-1][0] 134 | 135 | @call_parse 136 | def main( 137 | run: Param("Number of run", int)=20, 138 | gpu:Param("GPU to run on", str)=None, 139 | woof: Param("Use imagewoof (otherwise imagenette)", int)=0, 140 | lr: Param("Learning rate", float)=1e-3, 141 | size: Param("Size (px: 128,192,224)", int)=128, 142 | alpha: Param("Alpha", float)=0.99, 143 | mom: Param("Momentum", float)=0.9, 144 | eps: Param("epsilon", float)=1e-6, 145 | epochs: Param("Number of epochs", int)=5, 146 | bs: Param("Batch size", int)=256, 147 | mixup: Param("Mixup", float)=0., 148 | opt: Param("Optimizer (adam,rms,sgd)", str)='adam', 149 | arch: Param("Architecture (xresnet34, xresnet50)", str)='xresnet50', 150 | sa: Param("Self-attention", int)=0, 151 | sym: Param("Symmetry for self-attention", int)=0, 152 | dump: Param("Print model; don't train", int)=0, 153 | lrfinder: Param("Run learning rate finder; don't train", int)=0, 154 | log: Param("Log file name", str)='log', 155 | sched_type: Param("LR schedule type", str)='one_cycle', 156 | ann_start: Param("Mixup", float)=-1.0, 157 | ): 158 | 159 | acc = np.array( 160 | [train(gpu,woof,lr,size,alpha,mom,eps,epochs,bs,mixup,opt,arch,sa,sym,dump,lrfinder,log,sched_type,ann_start) 161 | for i in range(run)]) 162 | 163 | print(acc) 164 | print(f'mean = {np.mean(acc)}, std = {np.std(acc)}, ci-95 = {sms.DescrStatsW(acc).tconfint_mean()}') 165 | 166 | -------------------------------------------------------------------------------- /neural_mvs_py/optimizers/over9000/xresnet.py: -------------------------------------------------------------------------------- 1 | # adapted from https://github.com/fastai/fastai/blob/master/fastai/vision/models/xresnet.py 2 | # added simple self attention layer, conv1d 3 | # ideally conv1d should have been available from fast.ai's layers.py/ or SSA layer should be in layers.py 4 | # added sa option to XResNet class 5 | # added sa to ResBlock class, xresnet function 6 | # added import fastai.torch_core for spectral norm and tensor 7 | 8 | from fastai.torch_core import * 9 | 10 | import torch.nn as nn 11 | import torch,math,sys 12 | import torch.utils.model_zoo as model_zoo 13 | from functools import partial 14 | 15 | __all__ = ['XResNet', 'xresnet18', 'xresnet34', 'xresnet50', 'xresnet101', 'xresnet152'] 16 | 17 | # or: ELU+init (a=0.54; gain=1.55) 18 | act_fn = nn.ReLU(inplace=True) 19 | 20 | 21 | 22 | #Unmodified from https://github.com/fastai/fastai/blob/5c51f9eabf76853a89a9bc5741804d2ed4407e49/fastai/layers.py 23 | def conv1d(ni:int, no:int, ks:int=1, stride:int=1, padding:int=0, bias:bool=False): 24 | "Create and initialize a `nn.Conv1d` layer with spectral normalization." 25 | conv = nn.Conv1d(ni, no, ks, stride=stride, padding=padding, bias=bias) 26 | nn.init.kaiming_normal_(conv.weight) 27 | if bias: conv.bias.data.zero_() 28 | return spectral_norm(conv) 29 | 30 | 31 | 32 | # Adapted from SelfAttention layer at https://github.com/fastai/fastai/blob/5c51f9eabf76853a89a9bc5741804d2ed4407e49/fastai/layers.py 33 | # Inspired by https://arxiv.org/pdf/1805.08318.pdf 34 | class SimpleSelfAttention(nn.Module): 35 | 36 | def __init__(self, n_in:int, ks=1, sym=False):#, n_out:int): 37 | super().__init__() 38 | 39 | self.conv = conv1d(n_in, n_in, ks, padding=ks//2, bias=False) 40 | 41 | self.gamma = nn.Parameter(tensor([0.])) 42 | 43 | self.sym = sym 44 | self.n_in = n_in 45 | 46 | def forward(self,x): 47 | 48 | 49 | if self.sym: 50 | # symmetry hack by https://github.com/mgrankin 51 | c = self.conv.weight.view(self.n_in,self.n_in) 52 | c = (c + c.t())/2 53 | self.conv.weight = c.view(self.n_in,self.n_in,1) 54 | 55 | size = x.size() 56 | x = x.view(*size[:2],-1) # (C,N) 57 | 58 | # changed the order of mutiplication to avoid O(N^2) complexity 59 | # (x*xT)*(W*x) instead of (x*(xT*(W*x))) 60 | 61 | convx = self.conv(x) # (C,C) * (C,N) = (C,N) => O(NC^2) 62 | xxT = torch.bmm(x,x.permute(0,2,1).contiguous()) # (C,N) * (N,C) = (C,C) => O(NC^2) 63 | 64 | o = torch.bmm(xxT, convx) # (C,C) * (C,N) = (C,N) => O(NC^2) 65 | 66 | o = self.gamma * o + x 67 | 68 | 69 | return o.view(*size).contiguous() 70 | 71 | 72 | 73 | 74 | class Flatten(nn.Module): 75 | def forward(self, x): return x.view(x.size(0), -1) 76 | 77 | def init_cnn(m): 78 | if getattr(m, 'bias', None) is not None: nn.init.constant_(m.bias, 0) 79 | if isinstance(m, (nn.Conv2d,nn.Linear)): nn.init.kaiming_normal_(m.weight) 80 | for l in m.children(): init_cnn(l) 81 | 82 | def conv(ni, nf, ks=3, stride=1, bias=False): 83 | return nn.Conv2d(ni, nf, kernel_size=ks, stride=stride, padding=ks//2, bias=bias) 84 | 85 | def noop(x): return x 86 | 87 | def conv_layer(ni, nf, ks=3, stride=1, zero_bn=False, act=True): 88 | bn = nn.BatchNorm2d(nf) 89 | nn.init.constant_(bn.weight, 0. if zero_bn else 1.) 90 | layers = [conv(ni, nf, ks, stride=stride), bn] 91 | if act: layers.append(act_fn) 92 | return nn.Sequential(*layers) 93 | 94 | class ResBlock(nn.Module): 95 | def __init__(self, expansion, ni, nh, stride=1,sa=False, sym=False): 96 | super().__init__() 97 | nf,ni = nh*expansion,ni*expansion 98 | layers = [conv_layer(ni, nh, 3, stride=stride), 99 | conv_layer(nh, nf, 3, zero_bn=True, act=False) 100 | ] if expansion == 1 else [ 101 | conv_layer(ni, nh, 1), 102 | conv_layer(nh, nh, 3, stride=stride), 103 | conv_layer(nh, nf, 1, zero_bn=True, act=False) 104 | ] 105 | 106 | self.sa = SimpleSelfAttention(nf,ks=1,sym=sym) if sa else noop 107 | 108 | self.convs = nn.Sequential(*layers) 109 | # TODO: check whether act=True works better 110 | self.idconv = noop if ni==nf else conv_layer(ni, nf, 1, act=False) 111 | self.pool = noop if stride==1 else nn.AvgPool2d(2, ceil_mode=True) 112 | 113 | def forward(self, x): return act_fn(self.sa(self.convs(x)) + self.idconv(self.pool(x))) 114 | 115 | 116 | 117 | 118 | def filt_sz(recep): return min(64, 2**math.floor(math.log2(recep*0.75))) 119 | 120 | 121 | 122 | 123 | class XResNet(nn.Sequential): 124 | def __init__(self, expansion, layers, c_in=3, c_out=1000, sa = False, sym= False): 125 | 126 | stem = [] 127 | sizes = [c_in,32,32,64] 128 | for i in range(3): 129 | stem.append(conv_layer(sizes[i], sizes[i+1], stride=2 if i==0 else 1)) 130 | #nf = filt_sz(c_in*9) 131 | #stem.append(conv_layer(c_in, nf, stride=2 if i==1 else 1)) 132 | #c_in = nf 133 | 134 | block_szs = [64//expansion,64,128,256,512] 135 | blocks = [self._make_layer(expansion, block_szs[i], block_szs[i+1], l, 1 if i==0 else 2, sa = sa if i in[len(layers)-4] else False, sym=sym) 136 | for i,l in enumerate(layers)] 137 | super().__init__( 138 | *stem, 139 | nn.MaxPool2d(kernel_size=3, stride=2, padding=1), 140 | *blocks, 141 | nn.AdaptiveAvgPool2d(1), Flatten(), 142 | nn.Linear(block_szs[-1]*expansion, c_out), 143 | ) 144 | init_cnn(self) 145 | 146 | def _make_layer(self, expansion, ni, nf, blocks, stride, sa=False, sym=False): 147 | return nn.Sequential( 148 | *[ResBlock(expansion, ni if i==0 else nf, nf, stride if i==0 else 1, sa if i in [blocks -1] else False,sym) 149 | for i in range(blocks)]) 150 | 151 | def xresnet(expansion, n_layers, name, pretrained=False, **kwargs): 152 | model = XResNet(expansion, n_layers, **kwargs) 153 | if pretrained: model.load_state_dict(model_zoo.load_url(model_urls[name])) 154 | return model 155 | 156 | me = sys.modules[__name__] 157 | for n,e,l in [ 158 | [ 18 , 1, [2,2,2 ,2] ], 159 | [ 34 , 1, [3,4,6 ,3] ], 160 | [ 50 , 4, [3,4,6 ,3] ], 161 | [ 101, 4, [3,4,23,3] ], 162 | [ 152, 4, [3,8,36,3] ], 163 | ]: 164 | name = f'xresnet{n}' 165 | setattr(me, name, partial(xresnet, expansion=e, n_layers=l, name=name)) -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | import platform 5 | import subprocess 6 | import glob 7 | 8 | from setuptools import setup, Extension 9 | from setuptools.command.build_ext import build_ext 10 | from distutils.version import LooseVersion 11 | from distutils.command.install_headers import install_headers as install_headers_orig 12 | from setuptools import setup 13 | from distutils.sysconfig import get_python_inc 14 | import site #section 2.7 in https://docs.python.org/3/distutils/setupscript.html 15 | # import catkin.workspace 16 | 17 | 18 | class CMakeExtension(Extension): 19 | def __init__(self, name, sourcedir=''): 20 | Extension.__init__(self, name, sources=[]) 21 | self.sourcedir = os.path.abspath(sourcedir) 22 | 23 | 24 | class CMakeBuild(build_ext): 25 | def run(self): 26 | try: 27 | out = subprocess.check_output(['cmake', '--version']) 28 | except OSError: 29 | raise RuntimeError("CMake must be installed to build the following extensions: " + 30 | ", ".join(e.name for e in self.extensions)) 31 | 32 | if platform.system() == "Windows": 33 | cmake_version = LooseVersion(re.search(r'version\s*([\d.]+)', out.decode()).group(1)) 34 | if cmake_version < '3.1.0': 35 | raise RuntimeError("CMake >= 3.1.0 is required on Windows") 36 | 37 | for ext in self.extensions: 38 | self.build_extension(ext) 39 | 40 | def build_extension(self, ext): 41 | extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) 42 | cmake_args = [ 43 | '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir, 44 | '-DPYTHON_EXECUTABLE=' + sys.executable 45 | # '-GNinja' 46 | ] 47 | 48 | # cfg = 'Debug' if self.debug else 'Release' 49 | # build_args = ['--config', cfg] 50 | build_args = [] 51 | 52 | if platform.system() == "Windows": 53 | # cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(cfg.upper(), extdir)] 54 | if sys.maxsize > 2**32: 55 | cmake_args += ['-A', 'x64'] 56 | build_args += ['--', '/m'] 57 | else: 58 | # cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg] 59 | build_args += ['--', '-j4'] 60 | 61 | env = os.environ.copy() 62 | env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(env.get('CXXFLAGS', ''), 63 | self.distribution.get_version()) 64 | if not os.path.exists(self.build_temp): 65 | os.makedirs(self.build_temp) 66 | # print ("build temp is ", self.build_temp) 67 | 68 | #find out where do the header file and the shader files get copied into https://stackoverflow.com/questions/14375222/find-python-header-path-from-within-python 69 | # print("PRINTING-------------------------------------------------") 70 | # print( get_python_inc() ) #this gives the include dir 71 | # print( site.USER_BASE ) # data files that are added through data_files=[] get added here for --user instaltions https://docs.python.org/3/distutils/setupscript.html 72 | # pkg_dir, dist = self.create_dist(headers="dummy_header") 73 | # dummy_install_headers=install_headers_orig(dist=dist) 74 | # help(install_headers_orig) 75 | # dummy_install_headers=install_headers_orig(self.distribution) 76 | # print( dummy_install_headers.install_dir ) #this gives the include dir 77 | # cmake_args+=['-DEASYPBR_SHADERS_PATH=' + get_python_inc()+"/easypbr"] 78 | # cmake_args+=['-DEASYPBR_SHADERS_PATH=' + site.USER_BASE] 79 | # cmake_args+=['-DDATA_DIR=' + site.USER_BASE] 80 | # cmake_args+=['-DCATKIN_PACKAGE_LIB_DESTINATION=' + "./"] 81 | 82 | 83 | subprocess.check_call(['cmake', ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env) 84 | subprocess.check_call(['cmake', '--build', '.'] + build_args, cwd=self.build_temp) 85 | # subprocess.check_call(['make', 'install'], cwd=self.build_temp) 86 | # subprocess.check_call(['catkin build', ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env) 87 | # subprocess.check_call(['catkin', 'build' ,'--this'] + build_args + cmake_args, cwd=self.build_temp, env=env) 88 | 89 | # ##get the workspace path depending on the current path and catkin.workspace.get_workspaces 90 | # # print("current path") 91 | # cur_dir=os.path.dirname(os.path.abspath(__file__)) 92 | # workspaces=catkin.workspace.get_workspaces() 93 | # # current_workspace=[x if cur_dir in else '' for x in row] 94 | # current_workspace="" 95 | # for path in workspaces: 96 | # last_part=os.path.basename(os.path.normpath(path)) 97 | # if(last_part=="devel"): 98 | # potential_workspace=os.path.dirname(path) #gets rid of /devel 99 | # if(potential_workspace in cur_dir): 100 | # current_workspace=potential_workspace 101 | # break 102 | # print("current workspace is ", current_workspace) 103 | 104 | # #simlink the libraries from the devel space towards the current dir so that the egg link can find them 105 | # catkin_lib_dir=os.path.join(current_workspace,"devel/lib/") 106 | # libs = [f for f in os.listdir(catkin_lib_dir) if os.path.isfile(os.path.join(catkin_lib_dir, f))] 107 | # print(libs) 108 | # for lib in libs: 109 | # if "dataloaders" in lib: 110 | # print ("linking ", lib) 111 | # print("cmd", ['ln', '-sf'] + [ os.path.join(catkin_lib_dir,lib) + " " + os.path.join(cur_dir, lib ) ] ) 112 | # subprocess.check_call(['ln', '-sf'] + [ os.path.join(catkin_lib_dir,lib) ] + [ os.path.join(cur_dir, lib ) ] ) 113 | # # subprocess.check_call(['cp'] + [ os.path.join(catkin_lib_dir,lib) + " " + os.path.join(cur_dir, lib )] ) 114 | 115 | 116 | 117 | with open("README.md", "r") as fh: 118 | long_description = fh.read() 119 | 120 | #install headers while retaining the structure of the tree folder https://stackoverflow.com/a/50114715 121 | class install_headers(install_headers_orig): 122 | 123 | def run(self): 124 | headers = self.distribution.headers or [] 125 | for header in headers: 126 | dst = os.path.join(self.install_dir, os.path.dirname(header)) 127 | print("----------------copying in ", dst) 128 | # dst = os.path.join(get_python_inc(), os.path.dirname(header)) 129 | self.mkpath(dst) 130 | (out, _) = self.copy_file(header, dst) 131 | self.outfiles.append(out) 132 | 133 | def has_any_extension(filename, extensions): 134 | for each in extensions: 135 | if filename.endswith(each): 136 | return True 137 | return False 138 | 139 | # https://stackoverflow.com/a/41031566 140 | def find_files(directory, strip, extensions): 141 | """ 142 | Using glob patterns in ``package_data`` that matches a directory can 143 | result in setuptools trying to install that directory as a file and 144 | the installation to fail. 145 | 146 | This function walks over the contents of *directory* and returns a list 147 | of only filenames found. The filenames will be stripped of the *strip* 148 | directory part. 149 | 150 | It only takes file that have a certain extension 151 | """ 152 | 153 | result = [] 154 | for root, dirs, files in os.walk(directory): 155 | for filename in files: 156 | # if filename.endswith('.h') or filename.endswith('.hpp') or filename.endswith('.cuh'): 157 | if has_any_extension(filename, extensions): 158 | # print("filename", filename) 159 | filename = os.path.join(root, filename) 160 | result.append(os.path.relpath(filename, strip)) 161 | 162 | # print("result of find_files is ", result) 163 | 164 | return result 165 | # return 'include/easy_pbr/LabelMngr.h' 166 | 167 | setup( 168 | name='neuralmvs', 169 | version='1.0.0', 170 | author="Radu Alexandru Rosu", 171 | author_email="rosu@ais.uni-bonn.de", 172 | description="NeuralMVS", 173 | long_description=long_description, 174 | ext_modules=[CMakeExtension('neuralmvs')], 175 | cmdclass={ 'build_ext':CMakeBuild, 176 | 'install_headers': install_headers, 177 | }, 178 | zip_safe=False, 179 | ) 180 | 181 | 182 | 183 | -------------------------------------------------------------------------------- /src/NeuralMVSGUI.cxx: -------------------------------------------------------------------------------- 1 | #include "neural_mvs/NeuralMVSGUI.h" 2 | 3 | 4 | 5 | //c++ 6 | #include 7 | 8 | //easypbr 9 | #include "easy_pbr/Gui.h" 10 | #include "easy_pbr/Scene.h" 11 | 12 | 13 | //opencv 14 | #include "opencv2/highgui/highgui.hpp" 15 | #include "opencv2/features2d/features2d.hpp" 16 | // #include "opencv2/sfm/triangulation.hpp" 17 | #include 18 | 19 | 20 | //loguru 21 | #define LOGURU_REPLACE_GLOG 1 22 | #include //needs to be added after torch.h otherwise loguru stops printing for some reason 23 | 24 | //configuru 25 | #define CONFIGURU_WITH_EIGEN 1 26 | #define CONFIGURU_IMPLICIT_CONVERSIONS 1 27 | #include 28 | using namespace configuru; 29 | //Add this header after we add all cuda stuff because we need the profiler to have cudaDeviceSyncronize defined 30 | #define ENABLE_CUDA_PROFILING 1 31 | #include "Profiler.h" 32 | 33 | 34 | #include "string_utils.h" 35 | #include "numerical_utils.h" 36 | #include "eigen_utils.h" 37 | #include "easy_pbr/Mesh.h" 38 | #include "easy_pbr/Gui.h" 39 | #include "easy_pbr/Scene.h" 40 | #include "easy_pbr/Viewer.h" 41 | 42 | #include 43 | 44 | 45 | 46 | 47 | 48 | //boost 49 | #include 50 | namespace fs = boost::filesystem; 51 | 52 | 53 | 54 | 55 | NeuralMVSGUI::NeuralMVSGUI(const std::shared_ptr& view): 56 | m_show_rgb(true), 57 | m_show_depth(false), 58 | m_show_normal(false), 59 | m_min_depth(0.0), 60 | m_max_depth(1.0), 61 | m_control_secondary_cam(false), 62 | m_view(view) 63 | { 64 | 65 | install_callbacks(view); 66 | 67 | } 68 | 69 | NeuralMVSGUI::~NeuralMVSGUI(){ 70 | LOG(WARNING) << "Deleting NeuralMVSGUI"; 71 | } 72 | 73 | void NeuralMVSGUI::install_callbacks(const std::shared_ptr& view){ 74 | //pre draw functions (can install multiple functions and they will be called in order) 75 | 76 | //post draw functions 77 | view->add_callback_post_draw( [this]( easy_pbr::Viewer& v ) -> void{ this->post_draw(v); } ); 78 | } 79 | 80 | 81 | void NeuralMVSGUI::post_draw(easy_pbr::Viewer& view){ 82 | //get the final render as a opencv mat 83 | // cv::Mat mat = view.rendered_tex_no_gui(false).download_to_cv_mat(); 84 | 85 | // //the opencv mat can now be written to disk or even rendered in the GUI as a texture 86 | // cv::flip(mat, mat, 0); 87 | // cv::cvtColor(mat, mat, cv::COLOR_BGR2RGB); 88 | // Gui::show(mat, "mat"); 89 | 90 | //draw some gui elements 91 | ImGuiWindowFlags window_flags = 0; 92 | ImGui::Begin("NeuralMVSGUI", nullptr, window_flags); 93 | 94 | // if (ImGui::Button("My new button")){ 95 | // VLOG(1) << "Clicked button"; 96 | // } 97 | 98 | ImGui::Checkbox("Show rgb", &m_show_rgb); 99 | ImGui::Checkbox("Show depth", &m_show_depth); 100 | ImGui::Checkbox("Show normal", &m_show_normal); 101 | ImGui::Checkbox("Control secondary cam", &m_control_secondary_cam); 102 | ImGui::SliderFloat("Min Depth", &m_min_depth, 0.0f, 5.0f); 103 | ImGui::SliderFloat("Max Depth", &m_max_depth, 0.0f, 5.0f); 104 | 105 | ImGui::End(); 106 | 107 | // m_iter++; 108 | } 109 | 110 | 111 | 112 | -------------------------------------------------------------------------------- /src/PyBridge.cxx: -------------------------------------------------------------------------------- 1 | #include "neural_mvs/PyBridge.h" 2 | 3 | 4 | //my stuff 5 | #include "easy_pbr/Mesh.h" 6 | #include "easy_pbr/Viewer.h" 7 | #include "neural_mvs/NeuralMVSGUI.h" 8 | #include "neural_mvs/SFM.h" 9 | #include "neural_mvs/TrainParams.h" 10 | 11 | 12 | 13 | namespace py = pybind11; 14 | 15 | 16 | 17 | 18 | PYBIND11_MODULE(neuralmvs, m) { 19 | 20 | 21 | //TrainParams 22 | py::class_ > (m, "TrainParams") 23 | .def_static("create", &TrainParams::create ) //for templated methods like this one we need to explicitly instantiate one of the arguments 24 | .def("dataset_name", &TrainParams::dataset_name ) 25 | .def("with_viewer", &TrainParams::with_viewer ) 26 | .def("with_visdom", &TrainParams::with_visdom ) 27 | .def("with_tensorboard", &TrainParams::with_tensorboard ) 28 | .def("with_debug_output", &TrainParams::with_debug_output ) 29 | .def("with_error_checking", &TrainParams::with_error_checking ) 30 | .def("lr", &TrainParams::lr ) 31 | .def("weight_decay", &TrainParams::weight_decay ) 32 | .def("max_training_epochs", &TrainParams::max_training_epochs ) 33 | .def("save_checkpoint", &TrainParams::save_checkpoint ) 34 | .def("checkpoint_path", &TrainParams::checkpoint_path ) 35 | .def("save_every_x_epoch", &TrainParams::save_every_x_epoch ) 36 | ; 37 | 38 | 39 | //NeuralMVSGUI 40 | py::class_ > (m, "NeuralMVSGUI") 41 | .def_static("create", &NeuralMVSGUI::create& > ) //for templated methods like this one we need to explicitly instantiate one of the arguments 42 | .def_readwrite("m_show_rgb", &NeuralMVSGUI::m_show_rgb ) 43 | .def_readwrite("m_show_depth", &NeuralMVSGUI::m_show_depth ) 44 | .def_readwrite("m_show_normal", &NeuralMVSGUI::m_show_normal ) 45 | .def_readwrite("m_min_depth", &NeuralMVSGUI::m_min_depth ) 46 | .def_readwrite("m_max_depth", &NeuralMVSGUI::m_max_depth ) 47 | .def_readwrite("m_control_secondary_cam", &NeuralMVSGUI::m_control_secondary_cam ) 48 | ; 49 | 50 | 51 | 52 | //SFM 53 | py::class_ > (m, "SFM") 54 | .def_static("create", &SFM::create<> ) //for templated methods like this one we need to explicitly instantiate one of the arguments 55 | .def_static("fit_sphere", &SFM::fit_sphere) 56 | .def_static("compute_triangulation_stegreographic", &SFM::compute_triangulation_stegreographic) 57 | .def_static("compute_triangulation_plane", &SFM::compute_triangulation_plane) 58 | .def_static("compute_closest_triangle", &SFM::compute_closest_triangle) 59 | ; 60 | 61 | 62 | 63 | } -------------------------------------------------------------------------------- /src/TrainParams.cxx: -------------------------------------------------------------------------------- 1 | #include "neural_mvs/TrainParams.h" 2 | 3 | //loguru 4 | #define LOGURU_REPLACE_GLOG 1 5 | #include 6 | 7 | //configuru 8 | #define CONFIGURU_WITH_EIGEN 1 9 | #define CONFIGURU_IMPLICIT_CONVERSIONS 1 10 | #include 11 | using namespace configuru; 12 | 13 | //boost 14 | #include 15 | namespace fs = boost::filesystem; 16 | 17 | 18 | TrainParams::TrainParams(const std::string config_file){ 19 | 20 | init_params(config_file); 21 | } 22 | 23 | void TrainParams::init_params(const std::string config_file){ 24 | 25 | //read all the parameters 26 | Config cfg = configuru::parse_file(std::string(CMAKE_SOURCE_DIR)+"/config/"+config_file, CFG); 27 | Config train_config=cfg["train"]; 28 | m_dataset_name=(std::string)train_config["dataset_name"]; 29 | m_with_viewer = train_config["with_viewer"]; 30 | m_with_visdom = train_config["with_visdom"]; 31 | m_with_tensorboard = train_config["with_tensorboard"]; 32 | m_with_debug_output = train_config["with_debug_output"]; 33 | m_with_error_checking = train_config["with_error_checking"]; 34 | m_lr = train_config["lr"]; 35 | m_weight_decay = train_config["weight_decay"]; 36 | m_max_training_epochs = train_config["max_training_epochs"]; 37 | m_save_checkpoint=train_config["save_checkpoint"]; 38 | m_checkpoint_path=(std::string)train_config["checkpoint_path"]; 39 | m_save_every_x_epoch=train_config["save_every_x_epoch"]; 40 | 41 | if(m_save_checkpoint && !fs::is_directory(m_checkpoint_path)) { 42 | LOG(FATAL) << "The directory for saving checkpoint was not created under " << m_checkpoint_path << ". Maybe you need to create it or maybe you are on the wrong machine."; 43 | } 44 | 45 | } 46 | 47 | std::string TrainParams::dataset_name(){ 48 | return m_dataset_name; 49 | } 50 | bool TrainParams::with_viewer(){ 51 | return m_with_viewer; 52 | } 53 | bool TrainParams::with_visdom(){ 54 | return m_with_visdom; 55 | } 56 | bool TrainParams::with_tensorboard(){ 57 | return m_with_tensorboard; 58 | } 59 | bool TrainParams::with_debug_output(){ 60 | return m_with_debug_output; 61 | } 62 | bool TrainParams::with_error_checking(){ 63 | return m_with_error_checking; 64 | } 65 | float TrainParams::lr(){ 66 | return m_lr; 67 | } 68 | float TrainParams::weight_decay(){ 69 | return m_weight_decay; 70 | } 71 | int TrainParams::max_training_epochs(){ 72 | return m_max_training_epochs; 73 | } 74 | bool TrainParams::save_checkpoint(){ 75 | return m_save_checkpoint; 76 | } 77 | std::string TrainParams::checkpoint_path(){ 78 | return m_checkpoint_path; 79 | } 80 | int TrainParams::save_every_x_epoch(){ 81 | return m_save_every_x_epoch; 82 | } 83 | 84 | 85 | 86 | --------------------------------------------------------------------------------