├── .gitignore ├── CMakeLists.txt ├── INSTALL.md ├── LICENSE ├── README.md ├── docker └── Dockerfile ├── include ├── common.h ├── config.h.in ├── dense_flow.h └── utils.h ├── src ├── common.cpp ├── denseflow_gpu.cpp └── utils.cpp └── tools └── denseflow.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | .idea/ 3 | build/ 4 | .vscode/ 5 | out/ 6 | imgs/ 7 | test/ 8 | extract* 9 | log/ 10 | logs/ 11 | *.mp4 12 | *.h5 13 | data 14 | *.log 15 | *.txt 16 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.12) 2 | project(DenseFlow) 3 | 4 | option(USE_HDF5 "Whether to build hdf5 for optical flow image saving" OFF) 5 | option(USE_NVFLOW "Whether to use nvidia hardware optical flow" OFF) 6 | 7 | include(CheckCXXCompilerFlag) 8 | CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11) 9 | CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X) 10 | if(COMPILER_SUPPORTS_CXX11) 11 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC") 12 | elseif(COMPILER_SUPPORTS_CXX0X) 13 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") 14 | else() 15 | message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.") 16 | endif() 17 | 18 | find_package(CUDA REQUIRED) 19 | find_package(OpenCV REQUIRED) 20 | 21 | set(DenseFlow_LIB zzdenseflow) 22 | 23 | message(STATUS "OpenCV library status:") 24 | message(STATUS " config: ${OpenCV_DIR}") 25 | message(STATUS " version: ${OpenCV_VERSION}") 26 | message(STATUS " libraries: ${OpenCV_LIBS}") 27 | message(STATUS " include path: ${OpenCV_INCLUDE_DIRS}") 28 | 29 | find_package(Boost REQUIRED COMPONENTS date_time filesystem iostreams) 30 | message(STATUS "Boost library status:") 31 | message(STATUS " version: ${Boost_VERSION}") 32 | message(STATUS " libraries: ${Boost_LIBRARIES}") 33 | message(STATUS " include path: ${Boost_INCLUDE_DIRS}") 34 | 35 | add_library(${DenseFlow_LIB} src/common.cpp src/utils.cpp src/denseflow_gpu.cpp) 36 | include_directories(${OpenCV_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} include ${PROJECT_BINARY_DIR}) 37 | link_directories(${OpenCV_LIB_DIRS}) 38 | target_link_libraries(${DenseFlow_LIB} ${OpenCV_LIBS} Boost::filesystem ${Boost_LIBRARIES} pthread) 39 | 40 | if (USE_HDF5) 41 | find_package(HDF5 COMPONENTS HL REQUIRED) 42 | message(STATUS "HDF5 library status:") 43 | message(STATUS " version: ${HDF5_VERSION}") 44 | message(STATUS " libraries: ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES}") 45 | message(STATUS " include path: ${HDF5_INCLUDE_DIRS} ${HDF5_HL_INCLUDE_DIR}") 46 | set(HDF5_DEFINITION 1) 47 | include_directories(${HDF5_INCLUDE_DIRS} ${HDF5_HL_INCLUDE_DIR}) 48 | target_link_libraries(${DenseFlow_LIB} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES}) 49 | else() 50 | set(HDF5_DEFINITION 0) 51 | endif() 52 | 53 | if (USE_NVFLOW) 54 | set(NVFLOW_DEFINITION 1) 55 | else() 56 | set(NVFLOW_DEFINITION 0) 57 | endif() 58 | 59 | configure_file( 60 | "${PROJECT_SOURCE_DIR}/include/config.h.in" 61 | "${PROJECT_BINARY_DIR}/config.h" 62 | ) 63 | 64 | add_executable(denseflow tools/denseflow.cpp) 65 | target_link_libraries(denseflow ${DenseFlow_LIB}) 66 | 67 | install ( 68 | TARGETS denseflow ${DenseFlow_LIB} 69 | RUNTIME DESTINATION bin 70 | ARCHIVE DESTINATION lib 71 | LIBRARY DESTINATION lib 72 | ) 73 | -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | # Install Guide 2 | 3 | ## Dependencies: 4 | 5 | - CUDA (driver version > 400) 6 | - OpenCV v3/v4 7 | - Boost 8 | - HDF5 (Optional) 9 | 10 | ### 1. Install CUDA 11 | 12 | Make sure that the driver version is >400. 13 | Otherwise, the speed would be painfully slow, although it compiles and runs. 14 | 15 | ### 2. Install OpenCV with CUDA support 16 | 17 | We need OpenCV with CUDA enabled. 18 | An install script can be found [Here](https://github.com/innerlee/setup/blob/master/zzopencv.sh). 19 | Since OpenCV itself has many dependencies, 20 | you can follow the example below to compile. 21 | 22 | ```bash 23 | # ZZROOT is the root dir of all the installation 24 | # you may put these lines into your `.bashrc` or `.zshrc`. 25 | export ZZROOT=$HOME/app 26 | export PATH=$ZZROOT/bin:$PATH 27 | export LD_LIBRARY_PATH=$ZZROOT/lib:$ZZROOT/lib64:$LD_LIBRARY_PATH 28 | 29 | # fetch install scripts 30 | git clone https://github.com/innerlee/setup.git 31 | cd setup 32 | 33 | # opencv depends on ffmpeg for video decoding 34 | # ffmpeg depends on nasm, yasm, libx264, libx265, libvpx 35 | ./zznasm.sh 36 | ./zzyasm.sh 37 | ./zzlibx264.sh 38 | ./zzlibx265.sh 39 | ./zzlibvpx.sh 40 | # finally install ffmpeg 41 | ./zzffmpeg.sh 42 | 43 | # install opencv 4.3.0 44 | ./zzopencv.sh 45 | # you may put this line into your .bashrc 46 | export OpenCV_DIR=$ZZROOT 47 | 48 | ``` 49 | 50 | ### 3. Install Boost Library 51 | 52 | ```bash 53 | # install boost 54 | ./zzboost.sh 55 | # you may put this line into your .bashrc 56 | export BOOST_ROOT=$ZZROOT 57 | ``` 58 | 59 | ### 4. Install HDF5 Library (Optional) 60 | 61 | ```bash 62 | # install hdf5 63 | ./zzhdf5.sh 64 | ``` 65 | 66 | ### 5. Install denseflow 67 | 68 | ```bash 69 | # finally, install denseflow 70 | ./zzdenseflow.sh 71 | ``` 72 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Multimedia Laboratory, The Chinese University of Hong Kong. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Denseflow 2 | 3 | Extracting dense flow field given a video. 4 | 5 | ## Features 6 | 7 | - support multiple optical flow algorithms, including Nvidia hardware optical flow 8 | - support single video (or a frame folder) / a list of videos (or a list of frame folders) as input 9 | - support multiple output types (image, hdf5) 10 | - faster, 40% faster (by parallelize IO & computation) 11 | - record the progress when extract a list of videos, and resume by simply running the same command again (idempotent) 12 | 13 | ## Install 14 | 15 | ### Dependencies: 16 | 17 | - CUDA (driver version > 400) 18 | - OpenCV (with CUDA support): 19 | [opencv3](https://www.learnopencv.com/install-opencv3-on-ubuntu/) | 20 | [opencv4](https://www.learnopencv.com/install-opencv-4-on-ubuntu-16-04/) 21 | - Boost 22 | - HDF5 (Optional) 23 | 24 | ```bash 25 | git clone https://github.com/open-mmlab/denseflow.git 26 | cd denseflow && mkdir build && cd build 27 | cmake -DCMAKE_INSTALL_PREFIX=$HOME/app -DUSE_HDF5=no -DUSE_NVFLOW=no .. 28 | make -j 29 | make install 30 | ``` 31 | 32 | If you have trouble setting up building environments, scripts in [INSTALL](INSTALL.md) might be helpful. 33 | 34 | ## Usage 35 | 36 | ### Extract optical flow of a single video 37 | 38 | ```bash 39 | denseflow test.avi -b=20 -a=tvl1 -s=1 -v 40 | ``` 41 | 42 | - `test.avi`: input video 43 | - `-b=20` bound set to 20 44 | - `-a=tvl1` algorithm is tvl1 45 | - `-s=1` step is 1, ie flow of adjacent frames 46 | - `-v`: verbose 47 | 48 | ### Extract optical flow of a list of videos 49 | 50 | ```bash 51 | denseflow videolist.txt -b=20 -a=tvl1 -s=1 -v 52 | ``` 53 | 54 | - `videolist.txt`: a list of video paths 55 | - `-b=20` bound set to 20 56 | - `-a=tvl1` algorithm is tvl1 57 | - `-s=1` step is 1, ie flow of adjacent frames 58 | - `-v`: verbose 59 | 60 | ### Extract optical flow of a list of videos, each video is under a class folder 61 | 62 | ```bash 63 | denseflow videolist.txt -b=20 -a=tvl1 -s=1 -cf -v 64 | ``` 65 | 66 | - `videolist.txt`: a list of video paths 67 | - `-b=20` bound set to 20 68 | - `-a=tvl1` algorithm is tvl1 69 | - `-s=1` step is 1, ie flow of adjacent frames 70 | - `-cf` this switch means that parent folder of the video is a class name 71 | - `-v`: verbose 72 | 73 | ### Extract optical flow of a folder of frame images 74 | 75 | ```bash 76 | denseflow test -b=20 -a=tvl1 -s=1 -if -v 77 | ``` 78 | 79 | - `test`: folder of the frame images 80 | - `-b=20` bound set to 20 81 | - `-a=tvl1` algorithm is tvl1 82 | - `-s=1` step is 1, ie flow of adjacent frames 83 | - `-if` indicates that inputs are frames 84 | - `-v`: verbose 85 | 86 | ### Extract frames of a single video 87 | 88 | ```bash 89 | denseflow test.avi -s=0 -v 90 | ``` 91 | 92 | - `test.avi`: input video 93 | - `-s=0` step 0 is reserved for extracting frames 94 | - `-v`: verbose 95 | 96 | ### Extract frames of a list of videos 97 | 98 | ```bash 99 | denseflow videolist.txt -s=0 -v 100 | ``` 101 | 102 | - `videolist.txt`: a list of video paths 103 | - `-s=1` step is 1, ie flow of adjacent frames 104 | - `-s=0` step 0 is reserved for extracting frames 105 | - `-v`: verbose 106 | 107 | ## Documentation 108 | 109 | ```bash 110 | $ denseflow -h 111 | GPU optical flow extraction. 112 | Usage: denseflow [params] input 113 | 114 | -a, --algorithm (value:tvl1) 115 | optical flow algorithm (nv/tvl1/farn/brox) 116 | -b, --bound (value:32) 117 | maximum of optical flow 118 | --cf, --classFolder 119 | outputDir/class/video/flow.jpg 120 | -f, --force 121 | regardless of the marked .done file 122 | -h, --help (value:true) 123 | print help message 124 | --if, --inputFrames 125 | inputs are frames 126 | --newHeight, --nh (value:0) 127 | new height 128 | --newShort, --ns (value:0) 129 | short side length 130 | --newWidth, --nw (value:0) 131 | new width 132 | -o, --outputDir (value:.) 133 | root dir of output 134 | -s, --step (value:0) 135 | right - left (0 for img, non-0 for flow) 136 | --saveType, --st (value:jpg) 137 | save format type (png/h5/jpg) 138 | -v, --verbose 139 | verbose 140 | 141 | input 142 | filename of video or folder of frames or a list.txt of those 143 | ``` 144 | 145 | ## Citation 146 | 147 | If you use this tool in your research, please cite this project. 148 | 149 | ``` 150 | @misc{denseflow, 151 | author = {Wang, Shiguang* and Li, Zhizhong* and Zhao, Yue and Xiong, Yuanjun and Wang, Limin and Lin, Dahua}, 152 | title = {{denseflow}}, 153 | howpublished = {\url{https://github.com/open-mmlab/denseflow}}, 154 | year = {2020} 155 | } 156 | ``` 157 | 158 | ## Acknowledgement 159 | 160 | Rewritten based on [yuanjun's fork of dense_flow](https://github.com/yjxiong/dense_flow). 161 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG CUDA="11.1.1" 2 | ARG CUDNN="8" 3 | 4 | FROM nvidia/cuda:${CUDA}-cudnn${CUDNN}-devel-ubuntu18.04 5 | 6 | ARG OPENCV="4.5.2" 7 | # "Pascal" "Volta" "Turing" "Ampere" 8 | ARG CUDA_GENERATION="Pascal" 9 | 10 | RUN apt update \ 11 | && apt install -y git cmake wget software-properties-common nasm yasm libx264-dev libx265-dev libvpx-dev libboost-all-dev ffmpeg libavcodec-dev libavformat-dev libavutil-dev libblas-dev liblapack-dev libswscale-dev libtiff-dev libdc1394-22-dev libpng-dev libavresample-dev ccache libgflags-dev libhdf5-dev liblapack-dev libeigen3-dev libgoogle-glog-dev libfreetype6-dev \ 12 | && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null \ 13 | && apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' \ 14 | && apt update \ 15 | && apt install cmake -y \ 16 | && apt-get clean \ 17 | && rm -rf /var/lib/apt/lists/* /var/cache/apt /tmp/* \ 18 | && wget https://github.com/opencv/opencv/archive/${OPENCV}.tar.gz -O opencv.tar.gz \ 19 | && wget https://github.com/opencv/opencv_contrib/archive/${OPENCV}.tar.gz -O opencv_contrib.tar.gz \ 20 | && mkdir opencv \ 21 | && mkdir opencv_contrib \ 22 | && tar xf opencv.tar.gz -C opencv/ --strip-components 1 \ 23 | && tar xf opencv_contrib.tar.gz -C opencv_contrib/ --strip-components 1 \ 24 | && cd opencv \ 25 | && mkdir build \ 26 | && cd build \ 27 | && cmake \ 28 | -DBUILD_EXAMPLES=OFF \ 29 | -DWITH_QT=OFF \ 30 | -DCUDA_GENERATION=${CUDA_GENERATION} \ 31 | -DOpenGL_GL_PREFERENCE=GLVND \ 32 | -DBUILD_opencv_hdf=OFF \ 33 | -DBUILD_PERF_TESTS=OFF \ 34 | -DBUILD_TESTS=OFF \ 35 | -DCMAKE_BUILD_TYPE=RELEASE \ 36 | -DBUILD_opencv_cnn_3dobj=OFF \ 37 | -DBUILD_opencv_dnn=OFF \ 38 | -DBUILD_opencv_datasets=OFF \ 39 | -DBUILD_opencv_aruco=OFF \ 40 | -DBUILD_opencv_tracking=OFF \ 41 | -DBUILD_opencv_text=OFF \ 42 | -DBUILD_opencv_stereo=OFF \ 43 | -DBUILD_opencv_saliency=OFF \ 44 | -DBUILD_opencv_rgbd=OFF \ 45 | -DBUILD_opencv_reg=OFF \ 46 | -DBUILD_opencv_ovis=OFF \ 47 | -DBUILD_opencv_matlab=OFF \ 48 | -DBUILD_opencv_freetype=OFF \ 49 | -DBUILD_opencv_dpm=OFF \ 50 | -DBUILD_opencv_face=OFF \ 51 | -DBUILD_opencv_dnn_superres=OFF \ 52 | -DBUILD_opencv_dnn_objdetect=OFF \ 53 | -DBUILD_opencv_bgsegm=OFF \ 54 | -DBUILD_opencv_cvv=OFF \ 55 | -DBUILD_opencv_ccalib=OFF \ 56 | -DBUILD_opencv_bioinspired=OFF \ 57 | -DBUILD_opencv_dnn_modern=OFF \ 58 | -DBUILD_opencv_dnns_easily_fooled=OFF \ 59 | -DBUILD_JAVA=OFF \ 60 | -DBUILD_opencv_python2=OFF \ 61 | -DBUILD_NEW_PYTHON_SUPPORT=ON \ 62 | -DBUILD_opencv_python3=OFF \ 63 | -DHAVE_opencv_python3=OFF \ 64 | -DPYTHON_DEFAULT_EXECUTABLE="$(which python)" \ 65 | -DWITH_OPENGL=ON \ 66 | -DWITH_VTK=OFF \ 67 | -DFORCE_VTK=OFF \ 68 | -DWITH_TBB=ON \ 69 | -DWITH_GDAL=ON \ 70 | -DCUDA_FAST_MATH=ON \ 71 | -DWITH_CUBLAS=ON \ 72 | -DWITH_MKL=ON \ 73 | -DMKL_USE_MULTITHREAD=ON \ 74 | -DOPENCV_ENABLE_NONFREE=ON \ 75 | -DWITH_CUDA=ON \ 76 | -DNVCC_FLAGS_EXTRA="--default-stream per-thread" \ 77 | -DWITH_NVCUVID=OFF \ 78 | -DBUILD_opencv_cudacodec=OFF \ 79 | -DMKL_WITH_TBB=ON \ 80 | -DWITH_FFMPEG=ON \ 81 | -DMKL_WITH_OPENMP=ON \ 82 | -DWITH_XINE=ON \ 83 | -DENABLE_PRECOMPILED_HEADERS=OFF \ 84 | -DOPENCV_GENERATE_PKGCONFIG=ON \ 85 | -DOPENCV_EXTRA_MODULES_PATH=../../opencv_contrib/modules \ 86 | -DCMAKE_INSTALL_PREFIX=/usr \ 87 | .. \ 88 | && make -j"$(nproc)" \ 89 | && make install \ 90 | && cd ../../ \ 91 | && git clone https://github.com/open-mmlab/denseflow \ 92 | && cd denseflow \ 93 | && mkdir build \ 94 | && cd build \ 95 | && cmake -DCMAKE_INSTALL_PREFIX=/usr -DUSE_HDF5=yes .. \ 96 | && make -j"$(nproc)" \ 97 | && make install \ 98 | && cd ../../ \ 99 | && rm -rf *gz open* dense* 100 | 101 | ENTRYPOINT ["/usr/bin/denseflow"] 102 | -------------------------------------------------------------------------------- /include/common.h: -------------------------------------------------------------------------------- 1 | #ifndef DENSEFLOW_COMMON_H_H 2 | #define DENSEFLOW_COMMON_H_H 3 | #include "config.h" 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #if (USE_HDF5) 10 | #include "hdf5.h" 11 | #include "hdf5_hl.h" 12 | #endif 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | using namespace cv; 27 | using namespace cv::cuda; 28 | using boost::filesystem::create_directories; 29 | using boost::filesystem::directory_iterator; 30 | using boost::filesystem::exists; 31 | using boost::filesystem::is_directory; 32 | using boost::filesystem::is_regular_file; 33 | using boost::filesystem::path; 34 | using std::condition_variable; 35 | using std::cout; 36 | using std::endl; 37 | using std::mutex; 38 | using std::queue; 39 | using std::string; 40 | using std::thread; 41 | using std::unique_lock; 42 | using std::vector; 43 | 44 | void convertFlowToImage(const Mat &flow_x, const Mat &flow_y, Mat &img_x, Mat &img_y, double lowerBound, 45 | double higherBound); 46 | 47 | void encodeFlowMap(const Mat &flow_map_x, const Mat &flow_map_y, vector &encoded_x, vector &encoded_y, 48 | int bound, bool to_jpg = true); 49 | void encodeFlowMapPng(const Mat &flow_map_x, const Mat &flow_map_y, vector &encoded); 50 | 51 | void writeImages(vector> images, string name_prefix, const int start = 0); 52 | 53 | void writeFlowImages(vector> images, string name_prefix, const int step = 1, const int start = 0); 54 | void writeFlowImagesPng(vector> images, string name_prefix, const int step, const int start); 55 | 56 | #if (USE_HDF5) 57 | void writeHDF5(const vector &images, string name_prefix, string phase, const int step = 1, const int start = 0); 58 | #endif 59 | #endif // DENSEFLOW_COMMON_H_H 60 | -------------------------------------------------------------------------------- /include/config.h.in: -------------------------------------------------------------------------------- 1 | #define USE_HDF5 @HDF5_DEFINITION@ 2 | #define USE_NVFLOW @NVFLOW_DEFINITION@ 3 | -------------------------------------------------------------------------------- /include/dense_flow.h: -------------------------------------------------------------------------------- 1 | #ifndef DENSEFLOW_DENSE_FLOW_H 2 | #define DENSEFLOW_DENSE_FLOW_H 3 | 4 | #include "common.h" 5 | 6 | void calcDenseFlowVideoGPU(vector video_paths, vector output_dirs, string algorithm, int step, int bound, 7 | int new_width, int new_height, int new_short, bool has_class, bool use_frames, 8 | string save_type, bool is_record, bool verbose); 9 | 10 | class FlowBuffer { 11 | public: 12 | vector item_data; 13 | path output_dir; 14 | int base_start; 15 | bool last_buffer; 16 | FlowBuffer(vector item_data, path output_dir, int base_start, bool last_buffer) 17 | : item_data(item_data), output_dir(output_dir), base_start(base_start), last_buffer(last_buffer) {} 18 | }; 19 | 20 | class DenseFlow { 21 | private: 22 | vector video_paths; 23 | vector output_dirs; 24 | string algorithm; 25 | string save_type; 26 | int step; 27 | int bound; 28 | int new_width; 29 | int new_height; 30 | int new_short; 31 | bool has_class; 32 | bool is_record; 33 | Stream stream; 34 | 35 | mutex frames_gray_mtx, flows_mtx; 36 | condition_variable cond_frames_gray_produce, cond_frames_gray_consume; 37 | condition_variable cond_flows_produce, cond_flows_consume; 38 | int frames_gray_maxsize; 39 | int flows_maxsize; 40 | int batch_maxsize; 41 | bool ready_to_exit1; 42 | bool ready_to_exit2; 43 | bool ready_to_exit3; 44 | 45 | queue frames_gray_queue; 46 | queue flows_queue; 47 | unsigned long total_frames; 48 | unsigned long total_flows; 49 | 50 | // meber functions 51 | bool check_param(); 52 | bool get_new_size(const VideoCapture &video_stream, const vector &frames_path, bool use_frames, 53 | Size &new_size, int &frames_num); 54 | bool load_frames_batch(VideoCapture &video_stream, const vector &frames_path, bool use_frames, 55 | vector &frames_gray, bool do_resize, const Size &size, bool to_gray); 56 | int load_frames_video(VideoCapture &video_stream, vector &frames_path, bool use_frames, bool do_resize, 57 | const Size &size, path output_dir, bool is_last, bool verbose); 58 | void calc_optflows_imp(const FlowBuffer &frames_gray, const string &algorithm, int step, bool verbose, 59 | Stream &stream = Stream::Null()); 60 | void load_frames(bool use_frames, string save_type, bool verbose = true); 61 | void calc_optflows(bool verbose = true); 62 | void encode_save(string save_type, bool verbose = true); 63 | int extract_frames_video(VideoCapture &video_stream, vector &frames_path, bool use_frames, bool do_resize, 64 | const Size &size, path output_dir, bool verbose); 65 | 66 | public: 67 | static void load_frames_wrap(void *arg, bool use_frames, string save_type, bool verbose) { 68 | return static_cast(arg)->load_frames(use_frames, save_type, verbose); 69 | } 70 | static void calc_optflows_wrap(void *arg, bool verbose) { 71 | return static_cast(arg)->calc_optflows(verbose); 72 | } 73 | static void encode_save_wrap(void *arg, string save_type, bool verbose) { 74 | return static_cast(arg)->encode_save(save_type, verbose); 75 | } 76 | void launch(bool use_frames, string save_type, bool verbose) { 77 | thread thread_load_frames(load_frames_wrap, this, use_frames, save_type, verbose); 78 | thread thread_calc_optflow(calc_optflows_wrap, this, false); 79 | thread thread_encode_save(encode_save_wrap, this, save_type, false); 80 | thread_load_frames.join(); 81 | thread_calc_optflow.join(); 82 | thread_encode_save.join(); 83 | } 84 | void extract_frames_only(bool use_frames, bool verbose); 85 | unsigned long get_processed_total_frames() { return total_frames; } 86 | unsigned long get_processed_total_flows() { return total_flows; } 87 | 88 | DenseFlow(vector video_paths, vector output_dirs, string algorithm, int step, int bound, int new_width, 89 | int new_height, int new_short, bool has_class, bool is_record, string save_type) 90 | : video_paths(video_paths), output_dirs(output_dirs), algorithm(algorithm), step(step), bound(bound), 91 | new_width(new_width), new_height(new_height), new_short(new_short), has_class(has_class), 92 | is_record(is_record), save_type(save_type) { 93 | if (!check_param()) 94 | throw std::runtime_error("check init param error."); 95 | batch_maxsize = 512; 96 | frames_gray_maxsize = flows_maxsize = 3; 97 | ready_to_exit1 = ready_to_exit2 = ready_to_exit3 = false; 98 | total_frames = 0; 99 | total_flows = 0; 100 | } 101 | }; 102 | 103 | #endif // DENSEFLOW_DENSE_FLOW_H 104 | -------------------------------------------------------------------------------- /include/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef DENSEFLOW_UTILS_H 2 | #define DENSEFLOW_UTILS_H 3 | 4 | #include "common.h" 5 | 6 | void SplitString(const std::string &s, std::vector &v, const std::string &c); 7 | 8 | double CurrentSeconds(); 9 | 10 | void createFile(const path &ph); 11 | 12 | inline bool fileExists(const string &name) { 13 | struct stat info; 14 | return stat(name.c_str(), &info) == 0; 15 | } 16 | 17 | inline bool dirExists(const string &path) { 18 | struct stat info; 19 | return stat(path.c_str(), &info) == 0 && (info.st_mode & S_IFDIR); 20 | } 21 | #if (USE_HDF5) 22 | void hdf5_save_nd_dataset(const hid_t file_id, const string &dataset_name, const Mat &mat); 23 | #endif 24 | #endif // DENSEFLOW_UTILS_H 25 | -------------------------------------------------------------------------------- /src/common.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | #include "utils.h" 3 | 4 | void convertFlowToImage(const Mat &flow_x, const Mat &flow_y, Mat &img_x, Mat &img_y, double lowerBound, 5 | double higherBound) { 6 | #define CAST(v, L, H) ((v) > (H) ? 255 : (v) < (L) ? 0 : cvRound(255 * ((v) - (L)) / ((H) - (L)))) 7 | for (int i = 0; i < flow_x.rows; ++i) { 8 | for (int j = 0; j < flow_y.cols; ++j) { 9 | float x = flow_x.at(i, j); 10 | float y = flow_y.at(i, j); 11 | img_x.at(i, j) = CAST(x, lowerBound, higherBound); 12 | img_y.at(i, j) = CAST(y, lowerBound, higherBound); 13 | } 14 | } 15 | #undef CAST 16 | } 17 | 18 | void convertFlowToPngImage(const Mat &flow_x, const Mat &flow_y, Mat &img_bgr) { 19 | double base = 1. / 128.; 20 | double h = flow_x.rows; 21 | double w = flow_x.cols; 22 | double min_v, max_v; 23 | minMaxLoc(flow_x, &min_v, &max_v); 24 | auto bound_x = min(255. * 4, ceil((min(w, max(abs(min_v), abs(max_v))) * 128. / 127.) / 4) * 4); 25 | minMaxLoc(flow_y, &min_v, &max_v); 26 | auto bound_y = min(255. * 4, ceil((min(h, max(abs(min_v), abs(max_v))) * 128. / 127.) / 4) * 4); 27 | if (int(bound_x) % 8 == 0){ 28 | bound_x += 4; 29 | } 30 | if (int(bound_y) % 8 == 0){ 31 | bound_y += 4; 32 | } 33 | float eps_x_inv = 1. / (base * bound_x); 34 | float eps_y_inv = 1. / (base * bound_y); 35 | Mat x(flow_x.size(), CV_8UC1); 36 | Mat y(flow_x.size(), CV_8UC1); 37 | Mat b(flow_x.size(), CV_8UC1); 38 | flow_x.convertTo(x, CV_8UC1, eps_x_inv, 128.); 39 | flow_y.convertTo(y, CV_8UC1, eps_y_inv, 128.); 40 | auto half_h = h / 2; 41 | rectangle(b, Point(0, 0), Point(w - 1, half_h), bound_x / 4, FILLED); 42 | rectangle(b, Point(0, half_h + 1), Point(w - 1, h - 1), bound_y / 4, FILLED); 43 | Mat source[] = {x, y, b}; 44 | int from_to[] = {0, 0, 1, 1, 2, 2}; 45 | mixChannels(source, 3, &img_bgr, 1, from_to, 3); 46 | } 47 | 48 | void encodeFlowMap(const Mat &flow_map_x, const Mat &flow_map_y, vector &encoded_x, vector &encoded_y, 49 | int bound, bool to_jpg) { 50 | Mat flow_img_x(flow_map_x.size(), CV_8UC1); 51 | Mat flow_img_y(flow_map_y.size(), CV_8UC1); 52 | 53 | convertFlowToImage(flow_map_x, flow_map_y, flow_img_x, flow_img_y, -bound, bound); 54 | 55 | if (to_jpg) { 56 | imencode(".jpg", flow_img_x, encoded_x); 57 | imencode(".jpg", flow_img_y, encoded_y); 58 | } else { 59 | encoded_x.resize(flow_img_x.total()); 60 | encoded_y.resize(flow_img_y.total()); 61 | memcpy(encoded_x.data(), flow_img_x.data, flow_img_x.total()); 62 | memcpy(encoded_y.data(), flow_img_y.data, flow_img_y.total()); 63 | } 64 | } 65 | 66 | void encodeFlowMapPng(const Mat &flow_map_x, const Mat &flow_map_y, vector &encoded) { 67 | Mat flow_img_bgr(flow_map_x.size(), CV_8UC3); 68 | convertFlowToPngImage(flow_map_x, flow_map_y, flow_img_bgr); 69 | vector compression_params; 70 | imencode(".png", flow_img_bgr, encoded); 71 | } 72 | 73 | void writeImages(vector> images, string name_prefix, const int start) { 74 | for (int i = 0; i < images.size(); ++i) { 75 | char tmp[1024]; 76 | sprintf(tmp, "_%05d.jpg", start + i); 77 | FILE *fp; 78 | fp = fopen((name_prefix + tmp).c_str(), "wb"); 79 | fwrite(images[i].data(), 1, images[i].size(), fp); 80 | fclose(fp); 81 | } 82 | } 83 | 84 | void writeFlowImages(vector> images, string name_prefix, const int step, const int start) { 85 | int base = step > 0 ? 0 : -step; 86 | for (int i = 0; i < images.size(); ++i) { 87 | char tmp[1024]; 88 | if (step > 1) { 89 | sprintf(tmp, "_p%d_%05d.jpg", step, start + i + base); 90 | } else if (step < 0) { 91 | sprintf(tmp, "_m%d_%05d.jpg", -step, start + i + base); 92 | } else { 93 | sprintf(tmp, "_%05d.jpg", start + i + base); 94 | } 95 | FILE *fp; 96 | fp = fopen((name_prefix + tmp).c_str(), "wb"); 97 | fwrite(images[i].data(), 1, images[i].size(), fp); 98 | fclose(fp); 99 | } 100 | } 101 | 102 | void writeFlowImagesPng(vector> images, string name_prefix, const int step, const int start) { 103 | int base = step > 0 ? 0 : -step; 104 | for (int i = 0; i < images.size(); ++i) { 105 | char tmp[1024]; 106 | if (step > 1) { 107 | sprintf(tmp, "_p%d_%05d.png", step, start + i + base); 108 | } else if (step < 0) { 109 | sprintf(tmp, "_m%d_%05d.png", -step, start + i + base); 110 | } else { 111 | sprintf(tmp, "_%05d.png", start + i + base); 112 | } 113 | FILE *fp; 114 | fp = fopen((name_prefix + tmp).c_str(), "wb"); 115 | fwrite(images[i].data(), 1, images[i].size(), fp); 116 | fclose(fp); 117 | } 118 | } 119 | 120 | #if (USE_HDF5) 121 | void writeHDF5(const vector &images, string name_prefix, string phase, const int step, const int start) { 122 | char h5_ext[1024]; 123 | if (step > 1) { 124 | sprintf(h5_ext, "_p%d.h5", step); 125 | } else if (step < 0) { 126 | sprintf(h5_ext, "_m%d.h5", -step); 127 | } else { 128 | sprintf(h5_ext, ".h5"); 129 | } 130 | string h5_file = name_prefix + h5_ext; 131 | hid_t file_id = H5Fopen(h5_file.c_str(), H5F_ACC_RDWR, H5P_DEFAULT); 132 | int base = step > 0 ? 0 : -step; 133 | for (int i = 0; i < images.size(); ++i) { 134 | char tmp[1024]; 135 | if (step > 1) { 136 | sprintf(tmp, "_p%d_%05d", step, start + i + base); 137 | } else if (step < 0) { 138 | sprintf(tmp, "_m%d_%05d", -step, start + i + base); 139 | } else { 140 | sprintf(tmp, "_%05d", start + i + base); 141 | } 142 | string flow_dataset = "/" + phase + tmp; 143 | // no group 144 | hdf5_save_nd_dataset(file_id, flow_dataset, images[i]); 145 | } 146 | herr_t status = H5Fclose(file_id); 147 | if (status < 0) 148 | throw std::runtime_error("Failed to save hdf5 file: " + h5_file); 149 | } 150 | #endif 151 | -------------------------------------------------------------------------------- /src/denseflow_gpu.cpp: -------------------------------------------------------------------------------- 1 | #include "dense_flow.h" 2 | #include "opencv2/cudaarithm.hpp" 3 | #include "opencv2/cudaoptflow.hpp" 4 | #include "utils.h" 5 | #include 6 | #include 7 | #include 8 | 9 | bool DenseFlow::check_param() { 10 | for (int i = 0; i < video_paths.size(); i++) { 11 | if (!exists(video_paths[i])) { 12 | cout << video_paths[i] << " does not exist!" << endl; 13 | return false; 14 | } 15 | if (!is_directory(output_dirs[i])) { 16 | cout << output_dirs[i] << " is not a valid dir!" << endl; 17 | return false; 18 | } 19 | } 20 | if (algorithm != "nv" && algorithm != "tvl1" && algorithm != "farn" && algorithm != "brox") { 21 | cout << algorithm << " not supported!" << endl; 22 | return false; 23 | } 24 | if (bound <= 0) { 25 | cout << "bound should > 0!" << endl; 26 | return false; 27 | } 28 | if (new_height < 0 || new_width < 0 || new_short < 0) { 29 | cout << "height and width cannot < 0!" << endl; 30 | return false; 31 | } 32 | if (new_short > 0 && new_height + new_width != 0) { 33 | cout << "do not set height and width when set short!" << endl; 34 | return false; 35 | } 36 | if (save_type != "jpg" && save_type != "png" && save_type != "h5") { 37 | cout << "only jpg/png/h5 are supported (no " << save_type << ") for output" << endl; 38 | return false; 39 | } 40 | 41 | return true; 42 | } 43 | 44 | bool DenseFlow::get_new_size(const VideoCapture &video_stream, const vector &frames_path, bool use_frames, 45 | Size &new_size, int &frames_num) { 46 | int width, height; 47 | if (use_frames) { 48 | Mat src = imread(frames_path[0].string()); 49 | width = src.size().width; 50 | height = src.size().height; 51 | frames_num = frames_path.size(); 52 | } else { 53 | width = video_stream.get(cv::CAP_PROP_FRAME_WIDTH); 54 | height = video_stream.get(cv::CAP_PROP_FRAME_HEIGHT); 55 | frames_num = video_stream.get(cv::CAP_PROP_FRAME_COUNT); 56 | } 57 | // check resize 58 | bool do_resize = true; 59 | if (new_width > 0 && new_height > 0) { 60 | new_size.width = new_width; 61 | new_size.height = new_height; 62 | } else if (new_width > 0 && new_height == 0) { 63 | new_size.width = new_width; 64 | new_size.height = (int)round(height * 1.0 / width * new_width); 65 | } else if (new_width == 0 && new_height > 0) { 66 | new_size.width = (int)round(width * 1.0 / height * new_height); 67 | new_size.height = new_height; 68 | } else if (new_short > 0 && min(width, height) > new_short) { 69 | if (width < height) { 70 | new_size.width = new_short; 71 | new_size.height = (int)round(height * 1.0 / width * new_short); 72 | } else { 73 | new_size.width = (int)round(width * 1.0 / height * new_short); 74 | new_size.height = new_short; 75 | } 76 | } else { 77 | do_resize = false; 78 | } 79 | return do_resize; 80 | } 81 | 82 | int DenseFlow::extract_frames_video(VideoCapture &video_stream, vector &frames_path, bool use_frames, 83 | bool do_resize, const Size &size, path output_dir, bool verbose) { 84 | int video_frame_idx = 0; 85 | while (true) { 86 | vector frames_color; 87 | bool is_open = load_frames_batch(video_stream, frames_path, use_frames, frames_color, do_resize, size, false); 88 | vector> output_img; 89 | int N = frames_color.size(); 90 | for (size_t i = 0; i < N; i++) { 91 | vector str_img; 92 | imencode(".jpg", frames_color[i], str_img); 93 | output_img.push_back(str_img); 94 | } 95 | writeImages(output_img, (output_dir / "img").c_str(), video_frame_idx); 96 | video_frame_idx += N; 97 | if (!is_open) { 98 | break; 99 | } 100 | if (use_frames) { 101 | frames_path.erase(frames_path.begin(), frames_path.begin() + N); 102 | } 103 | } 104 | return video_frame_idx; 105 | } 106 | 107 | void DenseFlow::extract_frames_only(bool use_frames, bool verbose) { 108 | for (size_t i = 0; i < video_paths.size(); i++) { 109 | path video_path = video_paths[i]; 110 | path output_dir = output_dirs[i]; 111 | VideoCapture video_stream; 112 | vector frames_path; 113 | if (use_frames) { 114 | directory_iterator end_itr; 115 | for (directory_iterator itr(video_path); itr != end_itr; ++itr) { 116 | if (!is_regular_file(itr->status()) || itr->path().extension() != ".jpg") 117 | continue; 118 | frames_path.push_back(itr->path()); 119 | } 120 | if (frames_path.size() == 0) { 121 | if (verbose) 122 | cout << video_path << " is empty!" << endl; 123 | continue; 124 | } 125 | sort(frames_path.begin(), frames_path.end()); 126 | } else { 127 | video_stream.open(video_path.c_str()); 128 | if (!video_stream.isOpened()) 129 | throw std::runtime_error("cannot open video_path stream:" + video_path.string()); 130 | } 131 | 132 | Size size; 133 | int frames_num; 134 | bool do_resize = get_new_size(video_stream, frames_path, use_frames, size, frames_num); 135 | if (verbose) 136 | cout << video_path << ", frames ≈ " << frames_num << endl; 137 | frames_num = extract_frames_video(video_stream, frames_path, use_frames, do_resize, size, output_dir, verbose); 138 | total_frames += frames_num; // exact frame count 139 | if (!use_frames) 140 | video_stream.release(); 141 | if (verbose) 142 | cout << "extracted frames of video " << video_path << ", " << frames_num << " frames" << endl; 143 | } 144 | } 145 | 146 | bool DenseFlow::load_frames_batch(VideoCapture &video_stream, const vector &frames_path, bool use_frames, 147 | vector &frames_gray, bool do_resize, const Size &size, bool to_gray) { 148 | Mat capture_frame; 149 | int cnt = 0; 150 | while (cnt < batch_maxsize) { 151 | if (use_frames) { 152 | if (cnt == frames_path.size()) 153 | return false; 154 | capture_frame = imread(frames_path[cnt].string(), IMREAD_COLOR); 155 | } else { 156 | video_stream >> capture_frame; 157 | if (capture_frame.empty()) 158 | return false; 159 | } 160 | 161 | Mat frame_gray; 162 | if (to_gray) 163 | cvtColor(capture_frame, frame_gray, COLOR_BGR2GRAY); 164 | else 165 | frame_gray = capture_frame.clone(); 166 | if (do_resize) { 167 | Mat resized_frame_gray; 168 | resized_frame_gray.create(size, CV_8UC1); 169 | cv::resize(frame_gray, resized_frame_gray, size); 170 | frames_gray.push_back(resized_frame_gray); 171 | } else { 172 | frames_gray.push_back(frame_gray); 173 | } 174 | cnt++; 175 | } 176 | return true; 177 | } 178 | 179 | int DenseFlow::load_frames_video(VideoCapture &video_stream, vector &frames_path, bool use_frames, bool do_resize, 180 | const Size &size, path output_dir, bool is_last, bool verbose) { 181 | int video_flow_idx = 0; 182 | vector frames_gray_padding; 183 | while (true) { 184 | vector frames_gray; 185 | bool is_open = load_frames_batch(video_stream, frames_path, use_frames, frames_gray, do_resize, size, true); 186 | vector padded_frames_gray(frames_gray_padding.size() + frames_gray.size()); 187 | copy(frames_gray_padding.begin(), frames_gray_padding.end(), padded_frames_gray.begin()); 188 | copy(frames_gray.begin(), frames_gray.end(), padded_frames_gray.begin() + frames_gray_padding.size()); 189 | FlowBuffer frames_gray_item(padded_frames_gray, output_dir, video_flow_idx, !is_open); 190 | unique_lock lock(frames_gray_mtx); 191 | while (frames_gray_queue.size() == frames_gray_maxsize) { 192 | if (verbose) 193 | cout << "frames_gray_queue full, waiting..." << endl; 194 | cond_frames_gray_produce.wait(lock); 195 | } 196 | frames_gray_queue.push(frames_gray_item); 197 | if (verbose) 198 | cout << "push frames gray, video_flow_idx " << video_flow_idx << ", batch_size " << frames_gray.size() 199 | << endl; 200 | cond_frames_gray_consume.notify_all(); 201 | if (is_last && !is_open) 202 | ready_to_exit1 = true; 203 | lock.unlock(); 204 | frames_gray_padding.resize(abs(step)); 205 | copy(padded_frames_gray.end() - abs(step), padded_frames_gray.end(), frames_gray_padding.begin()); 206 | 207 | int M = padded_frames_gray.size() - abs(step); 208 | video_flow_idx += M; 209 | // read done a video 210 | if (!is_open) 211 | break; 212 | if (use_frames) { 213 | frames_path.erase(frames_path.begin(), frames_path.begin() + M); 214 | } 215 | } 216 | return video_flow_idx + abs(step); 217 | } 218 | 219 | void DenseFlow::load_frames(bool use_frames, string save_type, bool verbose) { 220 | for (size_t i = 0; i < video_paths.size(); i++) { 221 | path video_path = video_paths[i]; 222 | path output_dir = output_dirs[i]; 223 | if (save_type == "h5") { 224 | #if (USE_HDF5) 225 | // create h5 226 | char h5_ext[1024]; 227 | if (step > 1) { 228 | sprintf(h5_ext, "_p%d.h5", step); 229 | } else if (step < 0) { 230 | sprintf(h5_ext, "_m%d.h5", -step); 231 | } else { 232 | sprintf(h5_ext, ".h5"); 233 | } 234 | string h5_file = output_dir.string() + h5_ext; 235 | // overwrite if exists 236 | hid_t file_id = H5Fcreate(h5_file.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); 237 | herr_t status = H5Fclose(file_id); 238 | if (status < 0) 239 | throw std::runtime_error("Failed to save hdf5 file " + h5_file); 240 | #else 241 | throw std::runtime_error("HDF5 support is not enabled, pls recompile"); 242 | #endif 243 | } 244 | VideoCapture video_stream; 245 | vector frames_path; 246 | if (use_frames) { 247 | directory_iterator end_itr; 248 | for (directory_iterator itr(video_path); itr != end_itr; ++itr) { 249 | if (!is_regular_file(itr->status()) || itr->path().extension() != ".jpg") 250 | continue; 251 | frames_path.push_back(itr->path()); 252 | } 253 | if (frames_path.size() == 0) { 254 | if (verbose) 255 | cout << video_path << " is empty!" << endl; 256 | continue; 257 | } 258 | sort(frames_path.begin(), frames_path.end()); 259 | } else { 260 | video_stream.open(video_path.c_str()); 261 | if (!video_stream.isOpened()) 262 | throw std::runtime_error("cannot open video_path stream:" + video_path.string()); 263 | } 264 | 265 | Size size; 266 | int frames_num; 267 | bool do_resize = get_new_size(video_stream, frames_path, use_frames, size, frames_num); 268 | if (verbose) 269 | cout << video_path << ", frames ≈ " << frames_num << endl; 270 | frames_num = load_frames_video(video_stream, frames_path, use_frames, do_resize, size, output_dir, 271 | i == video_paths.size() - 1, verbose); 272 | total_frames += frames_num; // exact frame count 273 | if (!use_frames) 274 | video_stream.release(); 275 | if (verbose) 276 | cout << "loaded video " << video_path << ", " << frames_num << " frames" << endl; 277 | } 278 | if (verbose) 279 | cout << "load frames exit." << endl; 280 | } 281 | 282 | void DenseFlow::calc_optflows_imp(const FlowBuffer &frames_gray, const string &algorithm, int step, bool verbose, 283 | Stream &stream) { 284 | // create 285 | Ptr alg_farn; 286 | Ptr alg_tvl1; 287 | Ptr alg_brox; 288 | #if (USE_NVFLOW) 289 | Ptr alg_nv; 290 | #endif 291 | if (algorithm == "nv") { 292 | #if (USE_NVFLOW) 293 | auto size = frames_gray.item_data[0].size(); 294 | alg_nv = NvidiaOpticalFlow_1_0::create(size.width, size.height); 295 | #else 296 | throw std::runtime_error("NV hardware flow not enabled, pls recompile"); 297 | #endif 298 | } else if (algorithm == "tvl1") { 299 | alg_tvl1 = cuda::OpticalFlowDual_TVL1::create(); 300 | } else if (algorithm == "farn") { 301 | alg_farn = cuda::FarnebackOpticalFlow::create(); 302 | } else if (algorithm == "brox") { 303 | alg_brox = cuda::BroxOpticalFlow::create(0.197f, 50.0f, 0.8f, 10, 77, 10); 304 | } 305 | 306 | // compute 307 | int N = frames_gray.item_data.size(); 308 | int M = max(N - abs(step), 0); 309 | vector flows(M); 310 | if (M > 0) { 311 | GpuMat flow_gpu; 312 | GpuMat gray_a, gray_b; 313 | for (size_t i = 0; i < M; ++i) { 314 | Mat flow; 315 | int a = step > 0 ? i : i - step; 316 | int b = step > 0 ? i + step : i; 317 | gray_a.upload(frames_gray.item_data[a], stream); 318 | gray_b.upload(frames_gray.item_data[b], stream); 319 | if (algorithm == "nv") { 320 | #if (USE_NVFLOW) 321 | auto size = frames_gray.item_data[0].size(); 322 | alg_nv->calc(gray_a, gray_b, flow, stream); 323 | alg_nv->upSampler(flow, size.width, size.height, alg_nv->getGridSize(), flows[i]); 324 | #endif 325 | } else { 326 | if (algorithm == "tvl1") { 327 | alg_tvl1->calc(gray_a, gray_b, flow_gpu, stream); 328 | } else if (algorithm == "farn") { 329 | alg_farn->calc(gray_a, gray_b, flow_gpu, stream); 330 | } else if (algorithm == "brox") { 331 | GpuMat d_buf_0, d_buf_1; 332 | gray_a.convertTo(d_buf_0, CV_32F, 1.0 / 255.0, stream); 333 | gray_b.convertTo(d_buf_1, CV_32F, 1.0 / 255.0, stream); 334 | alg_brox->calc(d_buf_0, d_buf_1, flow_gpu, stream); 335 | } else { 336 | throw std::runtime_error("unknown optical algorithm " + algorithm); 337 | return; 338 | } 339 | flow_gpu.download(flows[i]); 340 | total_flows += 1; 341 | } 342 | } 343 | 344 | // release 345 | if (algorithm == "nv") { 346 | #if (USE_NVFLOW) 347 | alg_nv.release(); 348 | #endif 349 | } else if (algorithm == "tvl1") { 350 | alg_tvl1.release(); 351 | } else if (algorithm == "farn") { 352 | alg_farn.release(); 353 | } else if (algorithm == "brox") { 354 | alg_brox.release(); 355 | } 356 | } 357 | 358 | FlowBuffer flow_buffer(flows, frames_gray.output_dir, frames_gray.base_start, frames_gray.last_buffer); 359 | unique_lock lock(flows_mtx); 360 | while (flows_queue.size() == flows_maxsize) { 361 | if (verbose) 362 | cout << "flows queue is full, waiting..." << endl; 363 | cond_flows_produce.wait(lock); 364 | } 365 | flows_queue.push(flow_buffer); 366 | if (verbose) 367 | cout << "flows queue push a item, size " << flows_queue.size() << endl; 368 | cond_flows_consume.notify_all(); 369 | lock.unlock(); 370 | } 371 | 372 | void DenseFlow::calc_optflows(bool verbose) { 373 | while (true) { 374 | unique_lock lock(frames_gray_mtx); 375 | while (frames_gray_queue.size() == 0) { 376 | if (verbose) 377 | cout << "frames_gray_queue empty, waiting..." << endl; 378 | cond_frames_gray_consume.wait(lock); 379 | } 380 | FlowBuffer frames_gray = frames_gray_queue.front(); 381 | frames_gray_queue.pop(); 382 | if (ready_to_exit1 && frames_gray_queue.size() == 0) 383 | ready_to_exit2 = true; 384 | cond_frames_gray_produce.notify_all(); 385 | lock.unlock(); 386 | calc_optflows_imp(frames_gray, algorithm, step, false, stream); 387 | if (ready_to_exit2) 388 | break; 389 | else 390 | std::this_thread::yield(); 391 | } 392 | if (verbose) 393 | cout << "calc optflows exit." << endl; 394 | } 395 | 396 | void DenseFlow::encode_save(string save_type, bool verbose) { 397 | while (true) { 398 | unique_lock lock(flows_mtx); 399 | while (flows_queue.size() == 0) { 400 | if (verbose) 401 | cout << "flows queue empty, waiting..." << endl; 402 | cond_flows_consume.wait(lock); 403 | } 404 | FlowBuffer flow_buffer = flows_queue.front(); 405 | flows_queue.pop(); 406 | if (ready_to_exit2 && flows_queue.size() == 0) 407 | ready_to_exit3 = true; 408 | if (verbose) 409 | cout << "flows queue get a item, size " << flows_queue.size() << endl; 410 | cond_flows_produce.notify_all(); 411 | lock.unlock(); 412 | // encode & save 413 | int M = flow_buffer.item_data.size(); 414 | if (save_type == "jpg") { 415 | vector> output_x, output_y; 416 | Mat planes[2]; 417 | for (int i = 0; i < M; ++i) { 418 | split(flow_buffer.item_data[i], planes); 419 | Mat flow_x(planes[0]); 420 | Mat flow_y(planes[1]); 421 | vector str_x, str_y; 422 | encodeFlowMap(flow_x, flow_y, str_x, str_y, bound); 423 | output_x.push_back(str_x); 424 | output_y.push_back(str_y); 425 | } 426 | writeFlowImages(output_x, (flow_buffer.output_dir / "flow_x").c_str(), step, flow_buffer.base_start); 427 | writeFlowImages(output_y, (flow_buffer.output_dir / "flow_y").c_str(), step, flow_buffer.base_start); 428 | } else if (save_type == "h5") { 429 | #if (USE_HDF5) 430 | vector output_h5_x, output_h5_y; 431 | for (int i = 0; i < M; ++i) { 432 | Mat planes[2]; 433 | split(flow_buffer.item_data[i], planes); 434 | Mat flow_x(planes[0]); 435 | Mat flow_y(planes[1]); 436 | output_h5_x.push_back(flow_x); 437 | output_h5_y.push_back(flow_y); 438 | } 439 | writeHDF5(output_h5_x, flow_buffer.output_dir.c_str(), "flow_x", step, flow_buffer.base_start); 440 | writeHDF5(output_h5_y, flow_buffer.output_dir.c_str(), "flow_y", step, flow_buffer.base_start); 441 | #endif 442 | } else if (save_type == "png") { 443 | vector> output; 444 | for (int i = 0; i < M; ++i) { 445 | Mat planes[2]; 446 | split(flow_buffer.item_data[i], planes); 447 | Mat flow_x(planes[0]); 448 | Mat flow_y(planes[1]); 449 | vector str; 450 | encodeFlowMapPng(flow_x, flow_y, str); 451 | output.push_back(str); 452 | } 453 | writeFlowImagesPng(output, (flow_buffer.output_dir / "flow").c_str(), step, flow_buffer.base_start); 454 | } 455 | // mark done for the last batch of the video 456 | if (is_record && flow_buffer.last_buffer) { 457 | path donedir; 458 | path title; 459 | if (has_class) { 460 | donedir = flow_buffer.output_dir.parent_path().parent_path() / ".done" / 461 | flow_buffer.output_dir.parent_path().filename(); 462 | title = flow_buffer.output_dir.parent_path().filename() / flow_buffer.output_dir.filename(); 463 | } else { 464 | donedir = flow_buffer.output_dir.parent_path() / ".done"; 465 | title = flow_buffer.output_dir.filename(); 466 | } 467 | path donefile = donedir / flow_buffer.output_dir.stem().string(); 468 | createFile(donefile); 469 | cout << "done video " << title << endl; 470 | } 471 | 472 | if (ready_to_exit3) 473 | break; 474 | } 475 | if (verbose) 476 | cout << "post process exit." << endl; 477 | } 478 | 479 | void calcDenseFlowVideoGPU(vector video_paths, vector output_dirs, string algorithm, int step, int bound, 480 | int new_width, int new_height, int new_short, bool has_class, bool use_frames, 481 | string save_type, bool is_record, bool verbose) { 482 | setDevice(0); 483 | DenseFlow flow_video_gpu(video_paths, output_dirs, algorithm, step, bound, new_width, new_height, new_short, 484 | has_class, is_record, save_type); 485 | double start_t = CurrentSeconds(); 486 | if (step == 0) { 487 | flow_video_gpu.extract_frames_only(use_frames, verbose); 488 | } else { 489 | flow_video_gpu.launch(use_frames, save_type, verbose); 490 | } 491 | double end_t = CurrentSeconds(); 492 | unsigned long N = flow_video_gpu.get_processed_total_frames(); 493 | unsigned long F = flow_video_gpu.get_processed_total_flows(); 494 | cout << video_paths.size() << " videos (" << N << " frames, " << F << " " << algorithm 495 | << " flows) processed, using " << end_t - start_t << "s, decoding speed " << N / (end_t - start_t) 496 | << "fps, flow speed " << F / (end_t - start_t) << "fps" << endl; 497 | } 498 | -------------------------------------------------------------------------------- /src/utils.cpp: -------------------------------------------------------------------------------- 1 | #include "utils.h" 2 | 3 | void SplitString(const std::string &s, std::vector &v, const std::string &c) { 4 | std::string::size_type pos1, pos2; 5 | pos2 = s.find(c); 6 | pos1 = 0; 7 | while (std::string::npos != pos2) { 8 | v.push_back(s.substr(pos1, pos2 - pos1)); 9 | 10 | pos1 = pos2 + c.size(); 11 | pos2 = s.find(c, pos1); 12 | } 13 | if (pos1 != s.length()) 14 | v.push_back(s.substr(pos1)); 15 | } 16 | 17 | void createFile(const path &ph) { 18 | std::ofstream f(ph.string()); 19 | f.close(); 20 | } 21 | 22 | double CurrentSeconds() { 23 | return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) 24 | .count() / 25 | 1000.0; 26 | } 27 | #if (USE_HDF5) 28 | void hdf5_save_nd_dataset(const hid_t file_id, const string &dataset_name, const cv::Mat &mat) { 29 | int num_axes = mat.channels() == 1 ? 2 : mat.channels(); 30 | hsize_t *dims = new hsize_t[num_axes]; 31 | dims[0] = mat.rows; 32 | dims[1] = mat.cols; 33 | if (num_axes == 3) 34 | dims[2] = 3; 35 | herr_t status = H5LTmake_dataset_float(file_id, dataset_name.c_str(), num_axes, dims, mat.ptr()); 36 | if (status < 0) { 37 | throw "Failed to make float dataset " + dataset_name; 38 | } 39 | delete[] dims; 40 | } 41 | #endif 42 | -------------------------------------------------------------------------------- /tools/denseflow.cpp: -------------------------------------------------------------------------------- 1 | #include "dense_flow.h" 2 | #include "opencv2/opencv.hpp" 3 | #include "utils.h" 4 | #include 5 | 6 | int main(int argc, char **argv) { 7 | try { 8 | const char *keys = {"{ h help | | print help message }" 9 | "{ @input | | filename of video or folder of frames or a list.txt of those }" 10 | "{ o outputDir | . | root dir of output }" 11 | "{ a algorithm | tvl1 | optical flow algorithm (nv/tvl1/farn/brox) }" 12 | "{ s step | 0 | right - left (0 for img, non-0 for flow) }" 13 | "{ b bound | 32 | maximum of optical flow }" 14 | "{ nw newWidth | 0 | new width }" 15 | "{ nh newHeight | 0 | new height }" 16 | "{ ns newShort | 0 | short side length }" 17 | "{ cf classFolder | | outputDir/class/video/flow.jpg }" 18 | "{ if inputFrames | | inputs are frames }" 19 | "{ st saveType | jpg | save format type (png/h5/jpg) }" 20 | "{ f force | | regardless of the marked .done file }" 21 | "{ v verbose | | verbose }"}; 22 | 23 | CommandLineParser cmd(argc, argv, keys); 24 | 25 | cmd.about("GPU optical flow extraction."); 26 | if (cmd.get("@input") == "" || cmd.has("help")) { 27 | cmd.printMessage(); 28 | return 0; 29 | } 30 | if (!cmd.check()) { 31 | cmd.printErrors(); 32 | return 0; 33 | } 34 | 35 | path video_path(cmd.get("@input")); 36 | path output_dir(cmd.get("outputDir")); 37 | string algorithm = cmd.get("algorithm"); 38 | int step = cmd.get("step"); 39 | int bound = cmd.get("bound"); 40 | int new_width = cmd.get("newWidth"); 41 | int new_height = cmd.get("newHeight"); 42 | int new_short = cmd.get("newShort"); 43 | bool has_class = cmd.has("classFolder"); 44 | bool use_frames = cmd.has("inputFrames"); 45 | bool force = cmd.has("force"); 46 | string save_type = cmd.get("saveType"); 47 | bool verbose = cmd.has("verbose"); 48 | 49 | Mat::setDefaultAllocator(HostMem::getAllocator(HostMem::AllocType::PAGE_LOCKED)); 50 | 51 | vector video_paths; 52 | vector output_dirs; 53 | bool is_record = false; 54 | if (video_path.extension() == ".txt") { 55 | is_record = true; 56 | std::ifstream ifs(video_path.string()); 57 | string line; 58 | while (getline(ifs, line)) { 59 | path vidfile(line); 60 | path outdir; 61 | path donedir; 62 | path donefile; 63 | if (has_class) { 64 | outdir = output_dir / vidfile.parent_path().filename() / vidfile.stem(); 65 | donedir = output_dir / ".done" / vidfile.parent_path().filename(); 66 | } else { 67 | outdir = output_dir / vidfile.stem(); 68 | donedir = output_dir / ".done"; 69 | } 70 | donefile = donedir / vidfile.stem(); 71 | if (!force && is_regular_file(donefile)) { 72 | if (verbose) { 73 | cout << "skip " << vidfile.parent_path().filename() / vidfile.stem() << endl; 74 | } 75 | continue; 76 | } 77 | create_directories(outdir); 78 | create_directories(donedir); 79 | video_paths.push_back(vidfile); 80 | output_dirs.push_back(outdir); 81 | } 82 | } else { 83 | path outdir = output_dir / video_path.stem(); 84 | create_directories(outdir); 85 | video_paths.push_back(video_path); 86 | output_dirs.push_back(outdir); 87 | } 88 | if (video_paths.size() > 0) { 89 | calcDenseFlowVideoGPU(video_paths, output_dirs, algorithm, step, bound, new_width, new_height, new_short, 90 | has_class, use_frames, save_type, is_record, verbose); 91 | } 92 | 93 | } catch (const std::exception &ex) { 94 | cout << ex.what() << endl; 95 | return 1; 96 | } 97 | return 0; 98 | } 99 | --------------------------------------------------------------------------------