├── .gitignore ├── CMakeLists.txt ├── README.md ├── cmake ├── face_video_segment-config-version.cmake.in └── face_video_segment-config.cmake.in ├── doc ├── CMakeLists.txt └── Doxyfile.in ├── face_video_segment ├── CMakeLists.txt ├── face_regions.cpp ├── face_regions.h ├── face_segmentation_unit.cpp ├── face_segmentation_unit.h ├── face_video_segment.proto ├── keyframe_unit.cpp ├── keyframe_unit.h ├── keyframer.cpp ├── keyframer.h ├── landmarks_unit.cpp ├── landmarks_unit.h ├── utilities.cpp ├── utilities.h ├── video_reader_unit2.cpp ├── video_reader_unit2.h ├── video_writer_unit2.cpp └── video_writer_unit2.h ├── fvs_editor ├── CMakeLists.txt ├── editor.cpp ├── editor.h ├── editor_ui.cpp ├── fvs_editor.cpp ├── fvs_editor.qrc ├── fvs_editor.rc └── images │ ├── borders.png │ ├── contours.png │ ├── fvs_editor.ico │ ├── iterations.png │ ├── new.png │ ├── open.png │ ├── postprocess.png │ ├── postprocess_disconnected.png │ ├── postprocess_holes.png │ ├── postprocess_smooth.png │ ├── radius.png │ ├── save.png │ └── segmentation.png ├── fvs_editor_v2 ├── CMakeLists.txt ├── fvs_editor.cpp ├── fvs_editor.h ├── fvs_editor.qrc ├── fvs_editor.rc ├── fvs_editor.ui ├── fvs_editor_main.cpp ├── fvs_editor_states.cpp ├── fvs_editor_states.h └── images │ ├── backward.png │ ├── borders.png │ ├── close.png │ ├── contours.png │ ├── exit.png │ ├── face.png │ ├── forward.png │ ├── fvs_editor.ico │ ├── hierarchy.png │ ├── iterations.png │ ├── key.png │ ├── new.png │ ├── open.png │ ├── pause.png │ ├── play.png │ ├── postprocess.png │ ├── postprocess_disconnected.png │ ├── postprocess_holes.png │ ├── postprocess_smooth.png │ ├── radius.png │ ├── save-as.png │ ├── save.png │ └── segmentation.png ├── fvs_find_regions ├── CMakeLists.txt └── fvs_find_regions.cpp ├── fvs_segment ├── CMakeLists.txt └── fvs_segment.cpp ├── fvs_write_keyframes ├── CMakeLists.txt └── fvs_write_keyframes.cpp └── interfaces └── matlab ├── CMakeLists.txt ├── add_pascal_voc_db.m ├── face_fcn_test.m ├── face_fcn_train.m ├── face_pascal_voc_setup.m ├── face_video_seg_batch.m ├── fvs_extract_list.m ├── fvs_find_regions.m ├── fvs_init_trainval.m ├── fvs_segment.m ├── fvs_write_keyframes.m ├── images_per_video.m ├── produce_pascal_voc_db.m ├── seg_tree.m └── seg_viewer.m /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.txt.user 3 | .* 4 | !.gitignore 5 | build 6 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0.2) 2 | project(face_video_segment) 3 | 4 | # Version 5 | # =================================================== 6 | set(FACE_VIDEO_SEGMENT_MAJOR_VERSION 0) 7 | set(FACE_VIDEO_SEGMENT_MINOR_VERSION 8) 8 | set(FACE_VIDEO_SEGMENT_VERSION ${FACE_VIDEO_SEGMENT_MAJOR_VERSION}.${FACE_VIDEO_SEGMENT_MINOR_VERSION}) 9 | 10 | # Global configurations 11 | # =================================================== 12 | if(WIN32) 13 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNOMINMAX") 14 | set(DEF_INSTALL_CMAKE_DIR cmake) 15 | else() 16 | #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 17 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14") 18 | set(DEF_INSTALL_CMAKE_DIR lib/cmake/${PROJECT_NAME}) 19 | endif() 20 | set(INSTALL_CMAKE_DIR ${DEF_INSTALL_CMAKE_DIR} CACHE PATH 21 | "Installation directory for CMake files") 22 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake") 23 | set_property(GLOBAL PROPERTY USE_FOLDERS ON) 24 | 25 | # Optional 3rd party components 26 | # =================================================== 27 | option(WITH_BOOST_STATIC "Boost static libraries" ON) 28 | 29 | # Build components 30 | # =================================================== 31 | option(BUILD_DOCS "Build documentation using Doxygen" ON) 32 | option(BUILD_INTERFACE_MATLAB "Build interface for Matlab" ON) 33 | 34 | # Find dependencies 35 | # =================================================== 36 | # Boost 37 | set(Boost_USE_STATIC_LIBS ${WITH_BOOST_STATIC}) 38 | set(BOOST_ALL_DYN_LINK NOT ${WITH_BOOST_STATIC}) 39 | if(WIN32) 40 | if(${WITH_BOOST_STATIC}) 41 | add_definitions(-DBOOST_ALL_NO_LIB) 42 | else() 43 | add_definitions(-DBOOST_ALL_DYN_LINK) 44 | endif() 45 | endif() 46 | find_package(Boost REQUIRED filesystem program_options) 47 | 48 | find_package(OpenCV REQUIRED highgui imgproc imgcodecs) 49 | find_package(dlib REQUIRED) 50 | find_package(glog REQUIRED) 51 | find_package(protobuf REQUIRED) 52 | find_package(Qt5Widgets REQUIRED) 53 | find_package(video_segment REQUIRED) 54 | find_package(find_face_landmarks REQUIRED) 55 | 56 | # OpenMP 57 | find_package(OpenMP) 58 | if (OPENMP_FOUND) 59 | set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") 60 | set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") 61 | endif() 62 | 63 | # Doxygen 64 | find_package(Doxygen) 65 | 66 | # Process subdirectories 67 | # =================================================== 68 | add_subdirectory(face_video_segment) 69 | add_subdirectory(fvs_find_regions) 70 | add_subdirectory(fvs_segment) 71 | add_subdirectory(fvs_write_keyframes) 72 | add_subdirectory(fvs_editor) 73 | 74 | # interfaces 75 | if(BUILD_INTERFACE_MATLAB) 76 | add_subdirectory(interfaces/matlab) 77 | endif() 78 | 79 | # Documentation 80 | if(BUILD_DOCS) 81 | add_subdirectory(doc) 82 | endif() 83 | 84 | # Export configuration 85 | # =================================================== 86 | 87 | # Add all targets to the build-tree export set 88 | export(TARGETS face_video_segment 89 | FILE "${PROJECT_BINARY_DIR}/face_video_segment-targets.cmake") 90 | 91 | # Export the package for use from the build-tree 92 | # (this registers the build-tree with a global CMake-registry) 93 | export(PACKAGE face_video_segment) 94 | 95 | # Create config files 96 | configure_file(cmake/face_video_segment-config.cmake.in 97 | "${PROJECT_BINARY_DIR}/face_video_segment-config.cmake" @ONLY) 98 | configure_file(cmake/face_video_segment-config-version.cmake.in 99 | "${PROJECT_BINARY_DIR}/face_video_segment-config-version.cmake" @ONLY) 100 | 101 | # Install config files 102 | install(FILES 103 | "${PROJECT_BINARY_DIR}/face_video_segment-config.cmake" 104 | "${PROJECT_BINARY_DIR}/face_video_segment-config-version.cmake" 105 | DESTINATION "cmake" COMPONENT dev) 106 | 107 | # Install the export set for use with the install-tree 108 | install(EXPORT face_video_segment-targets DESTINATION cmake COMPONENT dev) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Face Video Segmentation - Face segmentation ground truth from videos 2 | ![alt text](https://yuvalnirkin.github.io/assets/img/projects/face_video_segment_teaser.jpg "Snapshot") 3 | Snapshot from the Face Video Segmentation Editor. 4 | 5 | [Yuval Nirkin](http://www.nirkin.com/), [Iacopo Masi](http://www-bcf.usc.edu/~iacopoma/), [Anh Tuan Tran](https://sites.google.com/site/anhttranusc/), [Tal Hassner](http://www.openu.ac.il/home/hassner/), and [Gerard Medioni](http://iris.usc.edu/people/medioni/index.html). 6 | 7 | ## Overview 8 | This project contains a collection of tools for semi-supervised gathering of ground truth face segmentation data from videos. A stable hierarchy of regions with temporal coherence is computed from dense optical flow using the method of [2]. Facial landmarks, extended to also include the forehead, are then used to extract the face contour. Regions are classified as belonging to the face segment according to their overlap with the face contour. The regions can then be further processed using a simple interface which allows browsing the entire video and manually classifying the regions using simple mouse clicks. 9 | 10 | 11 | If you find this code useful, please make sure to cite our paper in your work [1]: 12 | 13 | Yuval Nirkin, Iacopo Masi, Anh Tuan Tran, Tal Hassner, Gerard Medioni, "[On Face Segmentation, Face Swapping, and Face Perception](https://arxiv.org/abs/1704.06729)", IEEE Conference on Automatic Face and Gesture Recognition (FG), Xi'an, China on May 2018 14 | 15 | Please see [project page](http://www.openu.ac.il/home/hassner/projects/faceswap/) for more details, more resources and updates on this project. 16 | 17 | ## Dependencies 18 | | Library | Minimum Version | Notes | 19 | |--------------------------------------------------------------------|-----------------|------------------------------------------| 20 | | [Boost](http://www.boost.org/) | 1.47 | | 21 | | [OpenCV](http://opencv.org/) | 3.0 | | 22 | | [find_face_landmarks](https://github.com/YuvalNirkin/find_face_landmarks) | 1.1 | | 23 | | [video_segment](https://github.com/YuvalNirkin/video_segment) | 1.0 | | 24 | | [protobuf](https://github.com/google/protobuf) | 3.0.0 | | 25 | | [Qt](https://www.qt.io/) | 5.4.0 | | 26 | 27 | ## Installation 28 | - Use CMake and your favorite compiler to build and install the library. 29 | - Add face_video_segment/bin to path. 30 | - Add face_video_segment/interfaces/matlab to Matlab's path. 31 | - Download the [landmarks model file](http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2). 32 | 33 | ## Usage 34 | For using the library's C++ interface, please take a look at the [Doxygen generated documentation](https://yuvalnirkin.github.io/docs/face_video_segment/). 35 | 36 | Before running the pipeline please prepare an input directory with all the face videos for processing, and an output directory that will contain all the experiment data. 37 | To execute the entire automatic pipeline use the following Matlab code: 38 | ```Matlab 39 | inDir = 'videos'; % Input videos directory 40 | outDir = 'fvs_expr'; % Output directory 41 | landmarksFile = 'shape_predictor_68_face_landmarks.dat'; % Landmarks model file 42 | face_video_seg_batch(inDir, outDir, landmarksFile); 43 | ``` 44 | 45 | Now we have 4 directories in "fvs_expr": 46 | - seg_trees - Containing the all the video segmentations hierarchies for each video. 47 | - landmarks - Containing all the landmarks for each video. 48 | - fvs_files - Containing all the classified regions for each video. 49 | - output - Containing all the keyframe images and segmentations for each video in a separate directory. 50 | 51 | For additional manual processing go to the "fvs_files" directory and edit the appropiate file with the fvs_editor. 52 | ```DOS .bat 53 | fvs_editor video_name.fvs 54 | ``` 55 | On Windows you can right click the .fvs file, select open with... and point to the fvs_editor. 56 | 57 | To regenerate the output images and segmentations you can either use fvs_write_keyframes.m: 58 | ```Matlab 59 | fvsFile = 'fvs_expr/fvs_files/video_name.fvs'; % Edited fvs file 60 | outDir = 'fvs_expr/output/video_name'; % Output directory for the specific video 61 | face_video_seg_batch(fvsFile, outDir); 62 | ``` 63 | Or you can delete the corresponding video directories in "fvs_expr/output" and re-run the automatic pipeline (existing files will not be overwritten). 64 | 65 | To convert the generated ground truth to a dataset in PASCAL VOC format, do the following: 66 | Create an empty trainval.csv file: 67 | ```Matlab 68 | fvs_init_trainval('fvs_expr/output', 'fvs_expr/output/trainval.csv'); 69 | ``` 70 | Fill the "target" column in the trainval.csv file with "train" for training, "val" for valuation, or leave empty for ignoring the video. 71 | Then to produce the dataset: 72 | ```Matlab 73 | produce_pascal_voc_db('fvs_expr/output', 'fvs_expr/pascal_voc_db'); 74 | ``` 75 | Use "add_pascal_voc_db.m" to add additional images and segmentations to the dataset. 76 | 77 | ## Citation 78 | 79 | Please cite our paper with the following bibtex if you use our tool: 80 | 81 | ``` latex 82 | @inproceedings{nirkin2018_faceswap, 83 | title={On Face Segmentation, Face Swapping, and Face Perception}, 84 | booktitle = {IEEE Conference on Automatic Face and Gesture Recognition}, 85 | author={Nirkin, Yuval and Masi, Iacopo and Tran, Anh Tuan and Hassner, Tal and Medioni, and G\'{e}rard Medioni}, 86 | year={2018}, 87 | } 88 | ``` 89 | 90 | ## Bibliography 91 | [1] Yuval Nirkin, Iacopo Masi, Anh Tuan Tran, Tal Hassner, Gerard Medioni, [On Face Segmentation, Face Swapping, and Face Perception](https://arxiv.org/pdf/1704.06729.pdf), IEEE Conference on Automatic Face and Gesture Recognition, 2018. 92 | [2] Grundmann, Matthias and Kwatra, Vivek and Han, Mei and Essa, Irfan, [Efficient hierarchical graph-based video segmentation](https://smartech.gatech.edu/bitstream/handle/1853/38305/cvpr2010_videosegmentation.pdf), In Computer Vision and Pattern Recognition (CVPR), 2010 IEEE Conference on, pp. 2141-2148. IEEE, 2010. 93 | 94 | ## Related projects 95 | - [End-to-end, automatic face swapping pipeline](https://github.com/YuvalNirkin/face_swap), example application using out face segmentation method. 96 | - [Deep face segmentation](https://github.com/YuvalNirkin/face_segmentation), used to segment face regions in the face swapping pipeline. 97 | - [CNN3DMM](http://www.openu.ac.il/home/hassner/projects/CNN3DMM/), used to estimate 3D face shapes from single images. 98 | - [ResFace101](http://www.openu.ac.il/home/hassner/projects/augmented_faces/), deep face recognition used in the paper to test face swapping capabilities. 99 | 100 | ## Copyright 101 | Copyright 2017, Yuval Nirkin, Iacopo Masi, Anh Tuan Tran, Tal Hassner, and Gerard Medioni 102 | 103 | The SOFTWARE provided in this page is provided "as is", without any guarantee made as to its suitability or fitness for any particular use. It may contain bugs, so use of this tool is at your own risk. We take no responsibility for any damage of any sort that may unintentionally be caused through its use. 104 | -------------------------------------------------------------------------------- /cmake/face_video_segment-config-version.cmake.in: -------------------------------------------------------------------------------- 1 | set(PACKAGE_VERSION "@FACE_VIDEO_SEGMENT_VERSION@") 2 | 3 | # Check whether the requested PACKAGE_FIND_VERSION is compatible 4 | if("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}") 5 | set(PACKAGE_VERSION_COMPATIBLE FALSE) 6 | else() 7 | set(PACKAGE_VERSION_COMPATIBLE TRUE) 8 | if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}") 9 | set(PACKAGE_VERSION_EXACT TRUE) 10 | endif() 11 | endif() -------------------------------------------------------------------------------- /cmake/face_video_segment-config.cmake.in: -------------------------------------------------------------------------------- 1 | # - Config file for the face_video_segment package 2 | # It defines the following variables 3 | # FACE_VIDEO_SEGMENT_INCLUDE_DIRS - include directories for face_video_segment 4 | # FACE_VIDEO_SEGMENT_LIBRARIES - libraries to link against 5 | # FACE_VIDEO_SEGMENT_EXECUTABLE - the face_video_segment executable 6 | 7 | # Compute paths 8 | get_filename_component(FACE_VIDEO_SEGMENT_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" DIRECTORY) 9 | set(FACE_VIDEO_SEGMENT_INCLUDE_DIRS ${FACE_VIDEO_SEGMENT_CMAKE_DIR}/include) 10 | 11 | # Our library dependencies (contains definitions for IMPORTED targets) 12 | if(NOT TARGET face_video_segment AND NOT face_video_segment_BINARY_DIR) 13 | include("${FACE_VIDEO_SEGMENT_CMAKE_DIR}/face_video_segment-targets.cmake") 14 | endif() 15 | 16 | # These are IMPORTED targets created by face_video_segment-targets.cmake 17 | set(FACE_VIDEO_SEGMENT_LIBRARIES face_video_segment) 18 | if(TARGET face_video_segment) 19 | set(FACE_VIDEO_SEGMENT_EXECUTABLE face_video_segment) 20 | endif() 21 | 22 | if(face_video_segment_FOUND) 23 | message(STATUS "Found face_video_segment!") 24 | endif() -------------------------------------------------------------------------------- /doc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # additional config 2 | set(doxyfile "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile") 3 | set(DOXYGEN_INPUT_PATH "\"${CMAKE_SOURCE_DIR}/face_video_segment\"") 4 | set(DOXYGEN_OUTPUT_PATH "\"${CMAKE_CURRENT_BINARY_DIR}/doxygen\"") 5 | 6 | # Write doxygen configuration file 7 | configure_file(Doxyfile.in ${doxyfile} @ONLY) 8 | 9 | add_custom_target(doxygen 10 | COMMAND ${CMAKE_COMMAND} -E echo "Building API Documentation..." 11 | COMMAND ${DOXYGEN_EXECUTABLE} ${doxyfile} 12 | COMMAND ${CMAKE_COMMAND} -E echo "Done." 13 | DEPENDS ${doxyfile} 14 | ) 15 | 16 | install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/doxygen/html 17 | DESTINATION doc 18 | COMPONENT "docs" OPTIONAL 19 | ) -------------------------------------------------------------------------------- /face_video_segment/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Source 2 | set(FVS_SRC 3 | landmarks_unit.cpp 4 | video_reader_unit2.cpp 5 | video_writer_unit2.cpp 6 | utilities.cpp 7 | face_segmentation_unit.cpp 8 | face_regions.cpp 9 | keyframer.cpp 10 | keyframe_unit.cpp 11 | ) 12 | set(FVS_INCLUDE 13 | landmarks_unit.h 14 | video_reader_unit2.h 15 | video_writer_unit2.h 16 | utilities.h 17 | face_segmentation_unit.h 18 | face_regions.h 19 | keyframer.h 20 | keyframe_unit.h 21 | ) 22 | set(PROTO_FILES face_video_segment.proto) 23 | protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${PROTO_FILES}) 24 | set(FVS_SRC ${FVS_SRC} ${PROTO_SRCS} ${PROTO_FILES}) 25 | set(FVS_INCLUDE ${FVS_INCLUDE} ${PROTO_HDRS}) 26 | 27 | # Target 28 | add_library(face_video_segment ${FVS_SRC} ${FVS_INCLUDE}) 29 | target_include_directories(face_video_segment PRIVATE 30 | ${CMAKE_BINARY_DIR}/face_video_segment 31 | ) 32 | target_include_directories(face_video_segment PUBLIC 33 | $ 34 | $ 35 | ${VIDEO_SEGMENT_INCLUDE_DIRS} 36 | ${gflags_INCLUDE_DIRS} 37 | ${dlib_INCLUDE_DIRS} 38 | ${FIND_FACE_LANDMARKS_INCLUDE_DIRS} 39 | ${Boost_INCLUDE_DIRS} 40 | ) 41 | target_link_libraries(face_video_segment PUBLIC 42 | ${VIDEO_SEGMENT_LIBRARIES} 43 | ${gflags_LIBRARIES} 44 | ${dlib_LIBRARIES} 45 | ${FIND_FACE_LANDMARKS_LIBRARIES} 46 | ${Boost_LIBRARIES} 47 | ) 48 | 49 | # Installation 50 | install(TARGETS face_video_segment 51 | EXPORT face_video_segment-targets 52 | RUNTIME DESTINATION bin COMPONENT dev 53 | LIBRARY DESTINATION lib COMPONENT dev 54 | ARCHIVE DESTINATION lib COMPONENT dev) 55 | -------------------------------------------------------------------------------- /face_video_segment/face_regions.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #include "face_regions.h" 30 | #include "utilities.h" 31 | 32 | // std 33 | #include // debug 34 | 35 | // sfl 36 | #include 37 | 38 | // OpenCV 39 | #include 40 | 41 | using namespace segmentation; 42 | 43 | namespace fvs 44 | { 45 | FaceRegions::FaceRegions() 46 | { 47 | } 48 | 49 | void FaceRegions::addFrame(const SegmentationDesc& seg_desc, 50 | const sfl::Frame& sfl_frame, Frame& fvs_frame) 51 | { 52 | // If it is the first frame, save hierarchy. 53 | if (m_seg_hier == nullptr) m_seg_hier.reset(new SegmentationDesc(seg_desc)); 54 | else if (seg_desc.hierarchy_size() > 0) // Update hierarchy when one present. 55 | *m_seg_hier = seg_desc; 56 | 57 | // For each sfl face 58 | for (auto& sfl_face : sfl_frame.faces) 59 | addFace(seg_desc, *sfl_face, fvs_frame); 60 | } 61 | 62 | void FaceRegions::addFace(const segmentation::SegmentationDesc& seg_desc, 63 | const sfl::Face& sfl_face, Frame& fvs_frame) 64 | { 65 | auto& faces = *fvs_frame.mutable_faces(); 66 | Face& fvs_face = faces[sfl_face.id]; 67 | fvs_face.set_id(sfl_face.id); 68 | auto& regions = *fvs_face.mutable_regions(); 69 | const VectorMesh& mesh = seg_desc.vector_mesh(); 70 | cv::Mat face_map = createFaceMap( 71 | cv::Size(fvs_frame.width(), fvs_frame.height()), sfl_face.landmarks); 72 | 73 | // Calculate total face area and jaw area 74 | unsigned char *face_map_data = face_map.data; 75 | unsigned int total_face_area = 0, total_jaw_area = 0; 76 | for (size_t i = 0; i < face_map.total(); ++i, ++face_map_data) 77 | { 78 | if (*face_map_data == 255) ++total_face_area; 79 | else if (*face_map_data == 128) ++total_jaw_area; 80 | } 81 | 82 | // For each region 83 | cv::Mat poly_map = cv::Mat::zeros(face_map.size(), CV_8U); 84 | for (const auto& r : seg_desc.region()) 85 | { 86 | if (r.vectorization().polygon().empty()) continue; 87 | std::vector poly_types; 88 | poly_types.reserve(r.vectorization().polygon_size()); 89 | 90 | // Find holes 91 | std::vector> holes; 92 | for (const auto& poly : r.vectorization().polygon()) 93 | { 94 | if (!poly.hole()) continue; 95 | if (poly.coord_idx_size() == 0) continue; 96 | createContours(mesh, poly, holes); 97 | } 98 | 99 | // For each polygon 100 | PolygonType poly_type; 101 | for (const auto& poly : r.vectorization().polygon()) 102 | { 103 | if (poly.hole()) continue; 104 | if (poly.coord_idx_size() == 0) continue; 105 | 106 | poly_type = EMPTY; 107 | std::vector> contours; 108 | createContours(mesh, poly, contours); 109 | 110 | if (!contours.empty()) 111 | { 112 | // Render polygon 113 | cv::drawContours(poly_map, contours, 0, cv::Scalar(255, 255, 255), CV_FILLED); 114 | 115 | // Remove holes 116 | cv::drawContours(poly_map, holes, 0, cv::Scalar(0, 0, 0), CV_FILLED); 117 | 118 | // Compare maps 119 | unsigned char *face_map_data = face_map.data, *poly_map_data = poly_map.data; 120 | unsigned char fp; 121 | unsigned int face_area = 0, total_area = 0, jaw_area = 0; 122 | int pr, pc; 123 | float avg_out_r = 0, avg_out_c = 0; 124 | for (pr = 0; pr < face_map.rows; ++pr) 125 | { 126 | for (pc = 0; pc < face_map.cols; ++pc) 127 | { 128 | if (*poly_map_data++ > 0) 129 | { 130 | ++total_area; 131 | fp = *face_map_data++; 132 | if (fp == 255) ++face_area; 133 | else // Outside face 134 | { 135 | if (fp == 128) ++jaw_area; // Jaw outline 136 | avg_out_r += (float)pr; 137 | avg_out_c += (float)pc; 138 | } 139 | 140 | } 141 | else ++face_map_data; 142 | } 143 | } 144 | unsigned int out_area = total_area - face_area; 145 | float cos_a = 0; 146 | if (out_area > 0) 147 | { 148 | avg_out_r /= out_area; 149 | avg_out_c /= out_area; 150 | cv::Point2f poly_center(std::round(avg_out_c), std::round(avg_out_r)); 151 | 152 | cv::Point2f face_dir = sfl_face.landmarks[8] - sfl_face.landmarks[27]; 153 | cv::Point2f poly_dir = poly_center - cv::Point2f(sfl_face.landmarks[27]); 154 | face_dir /= cv::norm(face_dir); 155 | poly_dir /= cv::norm(poly_dir); 156 | cos_a = face_dir.dot(poly_dir); 157 | } 158 | 159 | // Test against threshold 160 | if (total_area > 0) 161 | { 162 | float in_ratio = float(face_area) / total_area; 163 | float out_ratio = float(out_area) / total_area; 164 | float jaw_poly_ratio = float(jaw_area) / total_area; 165 | float jaw_ratio = float(jaw_area) / total_jaw_area; 166 | float in_face_ratio = float(face_area) / total_face_area; 167 | float out_face_ratio = float(out_area) / total_face_area; 168 | 169 | if (in_face_ratio > 0.01f && out_face_ratio > 0.01f && (jaw_ratio > 0.05f && jaw_poly_ratio > 0.02f && jaw_poly_ratio < 0.5f)) 170 | { 171 | // Found neck region 172 | poly_type = INTERSECTION; 173 | } 174 | else if (in_ratio > 0.5f) poly_type = FULL; 175 | } 176 | 177 | // Clear map 178 | cv::drawContours(poly_map, contours, 0, cv::Scalar(0, 0, 0), CV_FILLED); 179 | } 180 | 181 | // Add polygon type 182 | poly_types.push_back(poly_type); 183 | } 184 | 185 | // Check whether there is at least one non empty polygon 186 | bool add_region = false; 187 | for (PolygonType type : poly_types) 188 | if (type != EMPTY) add_region = true; 189 | 190 | // Add new face region 191 | if (add_region) 192 | { 193 | Region& fvs_region = regions[r.id()]; 194 | fvs_region.set_id(r.id()); 195 | for (PolygonType type : poly_types) 196 | fvs_region.add_polygons(type); 197 | } 198 | } 199 | } 200 | 201 | cv::Mat FaceRegions::createFaceMap(const cv::Size& size, 202 | const std::vector& landmarks) 203 | { 204 | std::vector> face(1); 205 | sfl::createFullFace(landmarks, face.back()); 206 | cv::Mat face_map = cv::Mat::zeros(size, CV_8U); 207 | 208 | // Fill jaw 209 | float jaw_width = cv::norm(landmarks[2] - landmarks[14]); 210 | float side = getFaceDominantSide(landmarks); 211 | const float min_side = 0.3f, max_side = 0.7f; 212 | side = std::max(std::min(side, max_side), min_side); 213 | side = (side - min_side) / (max_side - min_side); 214 | float right_ratio = std::max(side - 0.5f, 0.0f)*2.0f; 215 | float left_ratio = std::min(side, 0.5f)*2.0f; 216 | int right_ind = 3 + (int)std::round(right_ratio * 5); 217 | int left_ind = 8 + (int)std::round(left_ratio * 5); 218 | int jaw_thickness = (int)std::round(0.1f*jaw_width); 219 | for (size_t i = right_ind + 1; i <= left_ind; ++i) 220 | cv::line(face_map, landmarks[i], landmarks[i - 1], 221 | cv::Scalar(128, 128, 128), jaw_thickness); 222 | 223 | // Fill face 224 | cv::drawContours(face_map, face, 0, cv::Scalar(255, 255, 255), CV_FILLED); 225 | 226 | /// Debug /// 227 | //std::cout << "side = " << side << std::endl; 228 | //std::cout << "right_ratio = " << right_ratio << std::endl; 229 | //std::cout << "left_ratio = " << left_ratio << std::endl; 230 | ///////////// 231 | 232 | return face_map; 233 | } 234 | 235 | } // namespace fvs 236 | -------------------------------------------------------------------------------- /face_video_segment/face_regions.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #ifndef FVS_FACE_REGIONS_H__ 30 | #define FVS_FACE_REGIONS_H__ 31 | 32 | 33 | #include "face_video_segment.pb.h" 34 | 35 | // std 36 | #include 37 | 38 | // sfl 39 | #include 40 | 41 | // segmentation 42 | #include 43 | 44 | // OpenCV 45 | #include 46 | 47 | namespace fvs { 48 | 49 | /** @brief Classifies segmented regions into face region types. 50 | The face region types are: 51 | - Empty - The Region does not participate in the face segmentation. 52 | - Full - The entire region is taken as part of the face segmentation. 53 | - Intersection - Only the intersection of the region with the face's 54 | contours is taken as part of the face segmentation. 55 | */ 56 | class FaceRegions 57 | { 58 | public: 59 | /** @brief Constructor. 60 | */ 61 | FaceRegions(); 62 | 63 | /** @brief Add a new frame. 64 | @param[in] seg_desc Segmentation descriptor. 65 | @param[in] sfl_frame Landmarks frame. 66 | @param[in,out] fvs_frame Regions frame. 67 | */ 68 | void addFrame(const segmentation::SegmentationDesc& seg_desc, 69 | const sfl::Frame& sfl_frame, Frame& fvs_frame); 70 | 71 | private: 72 | /** @brief Add a new face. 73 | @param[in] sfl_face Landmarks face. 74 | @param[in,out] fvs_frame Regions frame. 75 | */ 76 | void addFace(const segmentation::SegmentationDesc& seg_desc, 77 | const sfl::Face& sfl_face, Frame& fvs_frame); 78 | 79 | /** @brief Fill the inside face contours to create a face mask. 80 | */ 81 | cv::Mat createFaceMap(const cv::Size& size, const std::vector& landmarks); 82 | 83 | // Holds the segmentation for the current chunk. 84 | std::unique_ptr m_seg_hier; 85 | }; 86 | 87 | } // namespace fvs 88 | 89 | #endif // FVS_FACE_REGIONS_H__ 90 | -------------------------------------------------------------------------------- /face_video_segment/face_segmentation_unit.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #include "face_segmentation_unit.h" 30 | #include "utilities.h" 31 | 32 | #include 33 | #include 34 | #include 35 | 36 | #include "base/base_impl.h" 37 | #include 38 | #include 39 | #include 40 | 41 | using namespace video_framework; 42 | using namespace segmentation; 43 | 44 | namespace fvs 45 | { 46 | FaceRegionsReaderUnit::FaceRegionsReaderUnit(const FaceRegionsReaderOptions& options, 47 | const std::string& fvs_path) 48 | : options_(options) 49 | { 50 | fvs_sequence_.reset(new Sequence()); 51 | std::ifstream input(fvs_path, std::ifstream::binary); 52 | fvs_sequence_->ParseFromIstream(&input); 53 | } 54 | 55 | FaceRegionsReaderUnit::~FaceRegionsReaderUnit() { 56 | } 57 | 58 | bool FaceRegionsReaderUnit::OpenStreams(StreamSet* set) 59 | { 60 | // Add stream. 61 | set->push_back(shared_ptr(new DataStream(options_.stream_name))); 62 | 63 | return true; 64 | } 65 | 66 | void FaceRegionsReaderUnit::ProcessFrame(FrameSetPtr input, std::list* output) 67 | { 68 | // Get fvs frame from sequence 69 | Frame* fvs_frame = fvs_sequence_->mutable_frames(frame_number_); 70 | 71 | // Forward input 72 | input->push_back(std::shared_ptr>( 73 | new ValueFrame(fvs_frame))); 74 | 75 | output->push_back(input); 76 | ++frame_number_; 77 | } 78 | 79 | bool FaceRegionsReaderUnit::PostProcess(list* append) 80 | { 81 | return false; 82 | } 83 | 84 | FaceRegionsUnit::FaceRegionsUnit(const FaceRegionsOptions& options) 85 | : options_(options) 86 | { 87 | face_regions_.reset(new FaceRegions()); 88 | m_fvs_sequence.reset(new Sequence()); 89 | m_fvs_sequence->set_video_path(options.video_path); 90 | m_fvs_sequence->set_seg_path(options.seg_path); 91 | m_fvs_sequence->set_landmarks_path(options.landmarks_path); 92 | } 93 | 94 | FaceRegionsUnit::~FaceRegionsUnit() { 95 | } 96 | 97 | bool FaceRegionsUnit::OpenStreams(StreamSet* set) 98 | { 99 | // Find video stream idx. 100 | video_stream_idx_ = FindStreamIdx(options_.video_stream_name, set); 101 | 102 | if (video_stream_idx_ < 0) { 103 | LOG(ERROR) << "Could not find Video stream!\n"; 104 | return false; 105 | } 106 | 107 | const VideoStream& vid_stream = set->at(video_stream_idx_)->As(); 108 | 109 | frame_width_ = vid_stream.frame_width(); 110 | frame_height_ = vid_stream.frame_height(); 111 | 112 | // Get segmentation stream. 113 | seg_stream_idx_ = FindStreamIdx(options_.segment_stream_name, set); 114 | if (seg_stream_idx_ < 0) { 115 | LOG(ERROR) << "SegmentationRenderUnit::OpenStreams: " 116 | << "Could not find Segmentation stream!\n"; 117 | return false; 118 | } 119 | 120 | // Get landmarks stream 121 | landmarks_stream_idx_ = FindStreamIdx(options_.landmarks_stream_name, set); 122 | if (landmarks_stream_idx_ < 0) { 123 | LOG(ERROR) << "SegmentationRenderUnit::OpenStreams: " 124 | << "Could not find landmarks stream!\n"; 125 | return false; 126 | } 127 | 128 | // Add stream. 129 | set->push_back(shared_ptr(new DataStream(options_.stream_name))); 130 | 131 | return true; 132 | } 133 | 134 | void FaceRegionsUnit::ProcessFrame(FrameSetPtr input, std::list* output) 135 | { 136 | // Retrieve video frame 137 | const VideoFrame* vid_frame = input->at(video_stream_idx_)->AsPtr(); 138 | cv::Mat frame; 139 | vid_frame->MatView(&frame); 140 | 141 | // Retrieve Segmentation. 142 | const PointerFrame& seg_frame = 143 | input->at(seg_stream_idx_)->As>(); 144 | const SegmentationDesc& seg_desc = seg_frame.Ref(); 145 | 146 | // Retrieve landmarks 147 | const ValueFrame& landmarks_frame = 148 | input->at(landmarks_stream_idx_)->As>(); 149 | const sfl::Frame* sfl_frame = landmarks_frame.Value(); 150 | // const PointerFrame>& landmarks_frame = 151 | // input->at(landmarks_stream_idx_)->As>>(); 152 | // const std::vector& landmarks = landmarks_frame.Ref(); 153 | 154 | // Add a new frame to the fvs sequence 155 | Frame* fvs_frame = nullptr; 156 | if (sfl_frame != nullptr) 157 | { 158 | fvs_frame = m_fvs_sequence->add_frames(); 159 | fvs_frame->set_id(frame_number_); 160 | fvs_frame->set_width(frame_width_); 161 | fvs_frame->set_height(frame_height_); 162 | 163 | // Calculate face regions 164 | face_regions_->addFrame(seg_desc, *sfl_frame, *fvs_frame); 165 | } 166 | 167 | // Forward input 168 | input->push_back(std::shared_ptr>( 169 | new ValueFrame(fvs_frame))); 170 | 171 | output->push_back(input); 172 | ++frame_number_; 173 | } 174 | 175 | bool FaceRegionsUnit::PostProcess(list* append) 176 | { 177 | return false; 178 | } 179 | 180 | void FaceRegionsUnit::save(const std::string & filePath) const 181 | { 182 | if (m_fvs_sequence->frames_size() == 0) return; 183 | std::ofstream output(filePath, std::fstream::trunc | std::fstream::binary); 184 | m_fvs_sequence->SerializeToOstream(&output); 185 | } 186 | 187 | } // namespace fvs 188 | -------------------------------------------------------------------------------- /face_video_segment/face_segmentation_unit.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #ifndef FACE_VIDEO_SEGMENT_FACE_SEGMENTATION_UNIT_H__ 30 | #define FACE_VIDEO_SEGMENT_FACE_SEGMENTATION_UNIT_H__ 31 | 32 | #include "base/base.h" 33 | #include "video_framework/video_unit.h" 34 | #include "segment_util/segmentation_util.h" 35 | #include "face_regions.h" 36 | 37 | namespace fvs 38 | { 39 | using namespace segmentation; 40 | 41 | struct FaceRegionsReaderOptions { 42 | std::string stream_name = "FaceRegionsStream"; 43 | }; 44 | 45 | /** @brief Reads face regions from file into stream. 46 | */ 47 | class FaceRegionsReaderUnit : public video_framework::VideoUnit 48 | { 49 | public: 50 | FaceRegionsReaderUnit(const FaceRegionsReaderOptions& options, 51 | const std::string& fvs_path); 52 | ~FaceRegionsReaderUnit(); 53 | 54 | FaceRegionsReaderUnit(const FaceRegionsReaderUnit&) = delete; 55 | FaceRegionsReaderUnit& operator=(const FaceRegionsReaderUnit&) = delete; 56 | 57 | virtual bool OpenStreams(video_framework::StreamSet* set); 58 | virtual void ProcessFrame(video_framework::FrameSetPtr input, std::list* output); 59 | virtual bool PostProcess(std::list* append); 60 | 61 | private: 62 | FaceRegionsReaderOptions options_; 63 | std::unique_ptr fvs_sequence_; 64 | int frame_number_ = 0; 65 | }; 66 | 67 | struct FaceRegionsOptions { 68 | std::string stream_name = "FaceRegionsStream"; 69 | std::string video_stream_name = "VideoStream"; 70 | std::string segment_stream_name = "SegmentationStream"; 71 | std::string landmarks_stream_name = "LandmarksStream"; 72 | std::string video_path = ""; 73 | std::string seg_path = ""; 74 | std::string landmarks_path = ""; 75 | }; 76 | 77 | /** @brief Classifies face regions from stream. 78 | */ 79 | class FaceRegionsUnit : public video_framework::VideoUnit 80 | { 81 | public: 82 | FaceRegionsUnit(const FaceRegionsOptions& options); 83 | ~FaceRegionsUnit(); 84 | 85 | FaceRegionsUnit(const FaceRegionsUnit&) = delete; 86 | FaceRegionsUnit& operator=(const FaceRegionsUnit&) = delete; 87 | 88 | virtual bool OpenStreams(video_framework::StreamSet* set); 89 | virtual void ProcessFrame(video_framework::FrameSetPtr input, std::list* output); 90 | virtual bool PostProcess(std::list* append); 91 | 92 | /** @brief Save current sequence of face segmentations to file. 93 | */ 94 | virtual void save(const std::string& filePath) const; 95 | 96 | private: 97 | FaceRegionsOptions options_; 98 | int video_stream_idx_; 99 | int landmarks_stream_idx_; 100 | int seg_stream_idx_; 101 | 102 | int frame_width_; 103 | int frame_height_; 104 | int frame_number_ = 0; 105 | 106 | std::unique_ptr face_regions_; 107 | std::unique_ptr m_fvs_sequence; 108 | }; 109 | 110 | } // namespace fvs 111 | 112 | #endif // FACE_VIDEO_SEGMENT_FACE_SEGMENTATION_UNIT_H__ 113 | -------------------------------------------------------------------------------- /face_video_segment/face_video_segment.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package fvs; 4 | 5 | message Sequence { 6 | string video_path = 1; 7 | string seg_path = 2; 8 | string landmarks_path = 3; 9 | repeated Frame frames = 4; 10 | } 11 | 12 | message Frame { 13 | uint32 id = 1; 14 | uint32 width = 2; 15 | uint32 height = 3; 16 | map faces = 4; 17 | } 18 | 19 | message Face { 20 | uint32 id = 1; 21 | bool keyframe = 2; 22 | map regions = 3; 23 | Postprocessing postprocessing = 4; 24 | repeated Point silhouette = 5; 25 | } 26 | 27 | message Region { 28 | uint32 id = 1; 29 | repeated PolygonType polygons = 3; 30 | } 31 | 32 | enum PolygonType { 33 | EMPTY = 0; 34 | FULL = 1; 35 | INTERSECTION = 2; 36 | } 37 | 38 | message Postprocessing { 39 | bool disconnected = 1; 40 | bool holes = 2; 41 | bool smooth = 3; 42 | uint32 smooth_iterations = 4; 43 | uint32 smooth_kernel_radius = 5; 44 | } 45 | 46 | message Point { 47 | int32 x = 1; 48 | int32 y = 2; 49 | } -------------------------------------------------------------------------------- /face_video_segment/keyframe_unit.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #include "keyframe_unit.h" 30 | #include "face_segmentation_unit.h" 31 | #include "utilities.h" 32 | 33 | #include "base/base_impl.h" 34 | 35 | #include 36 | 37 | #include 38 | #include 39 | 40 | #include 41 | 42 | #include // 43 | 44 | using namespace video_framework; 45 | 46 | namespace fvs 47 | { 48 | KeyframeDetectionUnit::KeyframeDetectionUnit(const KeyframeDetectionOptions& options) 49 | : options_(options) 50 | { 51 | // keyframer_ = std::make_unique( 52 | // options.start_frame, options.stability_range); 53 | keyframer_ = std::unique_ptr(new Keyframer( 54 | options.start_frame, options.stability_range)); 55 | } 56 | 57 | KeyframeDetectionUnit::~KeyframeDetectionUnit() { 58 | } 59 | 60 | bool KeyframeDetectionUnit::OpenStreams(StreamSet* set) { 61 | // Find video stream idx. 62 | video_stream_idx_ = FindStreamIdx(options_.video_stream_name, set); 63 | 64 | if (video_stream_idx_ < 0) { 65 | LOG(ERROR) << "KeyframeDetectionUnit::OpenStreams: " 66 | << "Could not find video stream!\n"; 67 | return false; 68 | } 69 | 70 | const VideoStream& vid_stream = set->at(video_stream_idx_)->As(); 71 | 72 | const int frame_width = vid_stream.frame_width(); 73 | const int frame_height = vid_stream.frame_height(); 74 | const float fps = vid_stream.fps(); 75 | 76 | // Get landmarks stream 77 | landmarks_stream_idx_ = FindStreamIdx(options_.landmarks_stream_name, set); 78 | if (landmarks_stream_idx_ < 0) { 79 | LOG(ERROR) << "KeyframeDetectionUnit::OpenStreams: " 80 | << "Could not find landmarks stream!\n"; 81 | return false; 82 | } 83 | 84 | // Get face regions stream. 85 | face_regions_stream_idx_ = FindStreamIdx(options_.face_regions_stream_name, set); 86 | if (face_regions_stream_idx_ < 0) { 87 | LOG(ERROR) << "KeyframeDetectionUnit::OpenStreams: " 88 | << "Could not find face regions stream!\n"; 89 | return false; 90 | } 91 | 92 | return true; 93 | } 94 | 95 | void KeyframeDetectionUnit::ProcessFrame(FrameSetPtr input, std::list* output) 96 | { 97 | // Retrieve video frame 98 | const VideoFrame* vid_frame = input->at(video_stream_idx_)->AsPtr(); 99 | cv::Mat frame; 100 | vid_frame->MatView(&frame); 101 | 102 | // Retrieve landmarks 103 | const ValueFrame& landmarks_value_frame = 104 | input->at(landmarks_stream_idx_)->As>(); 105 | const sfl::Frame* sfl_frame = landmarks_value_frame.Value(); 106 | 107 | // Retrieve face regions data 108 | const ValueFrame& fvs_value_frame = 109 | input->at(face_regions_stream_idx_)->As>(); 110 | Frame* fvs_frame = fvs_value_frame.Value(); 111 | 112 | // Process frame 113 | if (sfl_frame != nullptr && fvs_frame != nullptr) 114 | keyframer_->addFrame(*sfl_frame, *fvs_frame); 115 | 116 | // Forward input 117 | output->push_back(input); 118 | ++frame_number_; 119 | } 120 | 121 | bool KeyframeDetectionUnit::PostProcess(list* append) 122 | { 123 | return false; 124 | } 125 | 126 | KeyframeWriterUnit::KeyframeWriterUnit(const KeyframeWriterOptions& options, 127 | const std::string& output_dir, const std::string& src_name) 128 | : options_(options), output_dir_(output_dir), src_name_(src_name) 129 | { 130 | } 131 | 132 | KeyframeWriterUnit::~KeyframeWriterUnit() { 133 | } 134 | 135 | bool KeyframeWriterUnit::OpenStreams(StreamSet* set) 136 | { 137 | // Find video stream idx. 138 | video_stream_idx_ = FindStreamIdx(options_.video_stream_name, set); 139 | 140 | if (video_stream_idx_ < 0) { 141 | LOG(ERROR) << "KeyframeWriterUnit::OpenStreams: " 142 | << "Could not find video stream!\n"; 143 | return false; 144 | } 145 | 146 | const VideoStream& vid_stream = set->at(video_stream_idx_)->As(); 147 | 148 | const int frame_width = vid_stream.frame_width(); 149 | const int frame_height = vid_stream.frame_height(); 150 | const float fps = vid_stream.fps(); 151 | 152 | // Get segmentation stream. 153 | seg_stream_idx_ = FindStreamIdx(options_.segment_stream_name, set); 154 | 155 | if (seg_stream_idx_ < 0) { 156 | LOG(ERROR) << "KeyframeWriterUnit::OpenStreams: " 157 | << "Could not find Segmentation stream!\n"; 158 | return false; 159 | } 160 | 161 | // Get landmarks stream 162 | landmarks_stream_idx_ = FindStreamIdx(options_.landmarks_stream_name, set); 163 | if (landmarks_stream_idx_ < 0) { 164 | LOG(ERROR) << "KeyframeWriterUnit::OpenStreams: " 165 | << "Could not find landmarks stream!\n"; 166 | return false; 167 | } 168 | 169 | // Get face regions stream. 170 | face_regions_stream_idx_ = FindStreamIdx(options_.face_regions_stream_name, set); 171 | if (face_regions_stream_idx_ < 0) { 172 | LOG(ERROR) << "KeyframeWriterUnit::OpenStreams: " 173 | << "Could not find face regions stream!\n"; 174 | return false; 175 | } 176 | 177 | return true; 178 | } 179 | 180 | void KeyframeWriterUnit::ProcessFrame(FrameSetPtr input, std::list* output) 181 | { 182 | // Retrieve video frame 183 | const VideoFrame* vid_frame = input->at(video_stream_idx_)->AsPtr(); 184 | cv::Mat frame; 185 | vid_frame->MatView(&frame); 186 | 187 | // Retrieve Segmentation. 188 | const PointerFrame& seg_frame = 189 | input->at(seg_stream_idx_)->As>(); 190 | const SegmentationDesc& seg_desc = seg_frame.Ref(); 191 | 192 | // Retrieve landmarks 193 | const ValueFrame& landmarks_value_frame = 194 | input->at(landmarks_stream_idx_)->As>(); 195 | const sfl::Frame* sfl_frame = landmarks_value_frame.Value(); 196 | 197 | // Retrieve face regions data 198 | const ValueFrame& fvs_value_frame = 199 | input->at(face_regions_stream_idx_)->As>(); 200 | Frame* fvs_frame = fvs_value_frame.Value(); 201 | 202 | // Process frame 203 | if (sfl_frame != nullptr && fvs_frame != nullptr) 204 | { 205 | // For each face 206 | for (auto& fvs_face : fvs_frame->faces()) 207 | { 208 | if (!fvs_face.second.keyframe()) continue; 209 | const sfl::Face* sfl_face = sfl_frame->getFace(fvs_face.second.id()); 210 | 211 | // Create segmentation 212 | cv::Mat face_map, seg; 213 | if (sfl_face != nullptr) 214 | { 215 | face_map = cv::Mat::zeros(frame.size(), CV_8U); 216 | std::vector> face_boundary(1); 217 | sfl::createFullFace(sfl_face->landmarks, face_boundary.back()); 218 | cv::drawContours(face_map, face_boundary, 0, cv::Scalar(255, 255, 255), CV_FILLED); 219 | seg = calcSegmentation(face_map, fvs_face.second.regions(), seg_desc); 220 | } 221 | else seg = calcSegmentation(frame.size(), fvs_face.second.regions(), seg_desc); 222 | 223 | // Postprocess segmentation 224 | if (fvs_face.second.has_postprocessing()) 225 | { 226 | const Postprocessing& post = fvs_face.second.postprocessing(); 227 | postprocessSegmentation(seg, post.disconnected(), post.holes(), 228 | post.smooth(), post.smooth_iterations(), post.smooth_kernel_radius()); 229 | } 230 | else postprocessSegmentation(seg); 231 | 232 | // Render segmentation 233 | cv::Mat seg_render, seg_debug; 234 | renderSegmentation(seg_render, seg, 15); 235 | if (options_.debug) 236 | { 237 | seg_debug = frame.clone(); 238 | cv::Scalar red(0, 0, 255); 239 | renderSegmentationBlend(seg_debug, seg, 0.5f, red, red); 240 | } 241 | 242 | // Crop frame and segmentation 243 | cv::Rect bbox; 244 | if (sfl_face != nullptr) 245 | bbox = sfl::getFaceBBoxFromLandmarks(sfl_face->landmarks, frame.size(), true); 246 | else bbox = getFaceBBoxFromSegmentation(seg, true); 247 | 248 | cv::Mat frame_cropped = frame(bbox); 249 | cv::Mat seg_render_cropped = seg_render(bbox); 250 | cv::Mat seg_debug_cropped; 251 | if (options_.debug) seg_debug_cropped = seg_debug(bbox); 252 | 253 | // Limit resolution 254 | cv::MatSize size = frame_cropped.size; 255 | int max_index = std::distance(size.p, std::max_element(size.p, size.p + 2)); 256 | if (options_.upscale || size[max_index] > (int)options_.max_scale) 257 | { 258 | float scale = (float)options_.max_scale / (float)size[max_index]; 259 | int w = (int)std::round(frame_cropped.cols * scale); 260 | int h = (int)std::round(frame_cropped.rows * scale); 261 | cv::resize(frame_cropped, frame_cropped, cv::Size(w, h), 262 | 0.0, 0.0, cv::INTER_CUBIC); 263 | cv::resize(seg_render_cropped, seg_render_cropped, cv::Size(w, h), 264 | 0.0, 0.0, cv::INTER_NEAREST); 265 | if (options_.debug) cv::resize(seg_debug_cropped, seg_debug_cropped, 266 | cv::Size(w, h), 0.0, 0.0, cv::INTER_NEAREST); 267 | } 268 | 269 | // Output frame and segmentation 270 | cv::imwrite(str(boost::format("%s\\%s_frame_%04d_face_%04d.jpg") % 271 | output_dir_ % src_name_ % frame_number_ % fvs_face.second.id()), frame_cropped); 272 | cv::imwrite(str(boost::format("%s\\%s_seg_%04d_face_%04d.png") % 273 | output_dir_ % src_name_ % frame_number_ % fvs_face.second.id()), seg_render_cropped); 274 | if (options_.debug) 275 | cv::imwrite(str(boost::format("%s\\%s_debug_%04d_face_%04d.jpg") % 276 | output_dir_ % src_name_ % frame_number_ % fvs_face.second.id()), seg_debug_cropped); 277 | 278 | /*/// Debug /// 279 | if (options_.debug) 280 | { 281 | std::ofstream output(str(boost::format("%s\\%s_debug_%04d_face_%04d.txt") % 282 | output_dir_ % src_name_ % frame_number_ % fvs_face.second.id())); 283 | output << fvs_face.second.DebugString(); 284 | } 285 | /////////////*/ 286 | } 287 | } 288 | 289 | // Forward input 290 | output->push_back(input); 291 | ++frame_number_; 292 | } 293 | 294 | bool KeyframeWriterUnit::PostProcess(list* append) 295 | { 296 | return false; 297 | } 298 | 299 | } // namespace fvs. 300 | -------------------------------------------------------------------------------- /face_video_segment/keyframe_unit.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #ifndef FVS_KEYFRAME_UNIT_H__ 30 | #define FVS_KEYFRAME_UNIT_H__ 31 | 32 | #include "base/base.h" 33 | #include "video_framework/video_unit.h" 34 | #include "keyframer.h" 35 | 36 | #include 37 | 38 | namespace fvs { 39 | 40 | struct KeyframeDetectionOptions 41 | { 42 | std::string video_stream_name = "VideoStream"; 43 | std::string segment_stream_name = "SegmentationStream"; 44 | std::string landmarks_stream_name = "LandmarksStream"; 45 | std::string face_regions_stream_name = "FaceRegionsStream"; 46 | int start_frame = 10; 47 | int stability_range = 5; 48 | bool debug = false; 49 | }; 50 | 51 | /** @brief Detects key frames from stream. 52 | */ 53 | class KeyframeDetectionUnit : public video_framework::VideoUnit 54 | { 55 | public: 56 | KeyframeDetectionUnit(const KeyframeDetectionOptions& options); 57 | ~KeyframeDetectionUnit(); 58 | 59 | KeyframeDetectionUnit(const KeyframeDetectionUnit&) = delete; 60 | KeyframeDetectionUnit& operator=(const KeyframeDetectionUnit&) = delete; 61 | 62 | virtual bool OpenStreams(video_framework::StreamSet* set); 63 | virtual void ProcessFrame(video_framework::FrameSetPtr input, std::list* output); 64 | virtual bool PostProcess(std::list* append); 65 | 66 | private: 67 | KeyframeDetectionOptions options_; 68 | std::string output_dir_; 69 | std::string src_name_; 70 | 71 | int video_stream_idx_; 72 | int landmarks_stream_idx_; 73 | int face_regions_stream_idx_; 74 | 75 | int frame_number_ = 0; 76 | std::unique_ptr keyframer_; 77 | }; 78 | 79 | struct KeyframeWriterOptions 80 | { 81 | std::string video_stream_name = "VideoStream"; 82 | std::string segment_stream_name = "SegmentationStream"; 83 | std::string landmarks_stream_name = "LandmarksStream"; 84 | std::string face_regions_stream_name = "FaceRegionsStream"; 85 | unsigned int max_scale = 500; 86 | bool upscale = false; 87 | bool debug = false; 88 | }; 89 | 90 | /** @brief Writes keyframe from stream to file. 91 | */ 92 | class KeyframeWriterUnit : public video_framework::VideoUnit 93 | { 94 | public: 95 | KeyframeWriterUnit(const KeyframeWriterOptions& options, 96 | const std::string& output_dir, const std::string& src_name); 97 | ~KeyframeWriterUnit(); 98 | 99 | KeyframeWriterUnit(const KeyframeWriterUnit&) = delete; 100 | KeyframeWriterUnit& operator=(const KeyframeWriterUnit&) = delete; 101 | 102 | virtual bool OpenStreams(video_framework::StreamSet* set); 103 | virtual void ProcessFrame(video_framework::FrameSetPtr input, std::list* output); 104 | virtual bool PostProcess(std::list* append); 105 | 106 | private: 107 | KeyframeWriterOptions options_; 108 | std::string output_dir_; 109 | std::string src_name_; 110 | 111 | int video_stream_idx_; 112 | int seg_stream_idx_; 113 | int landmarks_stream_idx_; 114 | int face_regions_stream_idx_; 115 | int frame_number_ = 0; 116 | }; 117 | 118 | } // namespace fvs 119 | 120 | #endif // FACE_VIDEO_SEGMENT_KEYFRAME_WRITER_UNIT_H__ 121 | -------------------------------------------------------------------------------- /face_video_segment/keyframer.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #include "keyframer.h" 30 | #include 31 | 32 | const float MAX_DIST = 0.5f*sqrt(3.0f)*CV_PI; 33 | const float MIN_DIST = MAX_DIST / 20.0f; 34 | 35 | namespace fvs 36 | { 37 | Keyframer::Keyframer(int start_frame, int stability_range) : 38 | m_start_frame(start_frame), 39 | m_stability_range(stability_range), 40 | m_frame_counter(0) 41 | { 42 | } 43 | 44 | void Keyframer::addFrame(const sfl::Frame& sfl_frame, Frame& fvs_frame) 45 | { 46 | auto& fvs_face_map = *fvs_frame.mutable_faces(); 47 | 48 | // For each sfl face 49 | for (auto& sfl_face : sfl_frame.faces) 50 | { 51 | // Check to find a corresponding fvs face 52 | auto fvs_face = fvs_face_map.find(sfl_face->id); 53 | if (fvs_face == fvs_face_map.end()) continue; 54 | 55 | addFace(*sfl_face, fvs_face->second); 56 | } 57 | 58 | // Clear history for faces that were not found 59 | for (auto& face_data : m_face_data_map) 60 | { 61 | if (face_data.second.frame_updated_ind < m_frame_counter) 62 | face_data.second.history.clear(); 63 | } 64 | 65 | ++m_frame_counter; 66 | } 67 | 68 | bool Keyframer::addFace(const sfl::Face& sfl_face, Face& fvs_face) 69 | { 70 | FaceData& face_data = m_face_data_map[sfl_face.id]; 71 | face_data.frame_updated_ind = m_frame_counter; 72 | 73 | // Check if we the landmarks are not empty 74 | if (sfl_face.landmarks.empty()) 75 | { 76 | face_data.history.clear(); 77 | return false; 78 | } 79 | 80 | // Update history 81 | face_data.history.push_back(sfl_face.landmarks); 82 | if (face_data.history.size() > m_stability_range) 83 | face_data.history.pop_front(); 84 | else return false; 85 | 86 | // Check start frame 87 | if (m_frame_counter < m_start_frame) 88 | return false; 89 | 90 | // Compare against previous keyframes 91 | cv::Point3f face_euler = sfl::getFaceApproxEulerAngles(sfl_face.landmarks); 92 | for (Keyframe& kf : face_data.keyframes) 93 | { 94 | float d = cv::norm(face_euler - kf.euler_angles); 95 | if (d < MIN_DIST) return false; 96 | } 97 | 98 | // Add new keyframe 99 | face_data.keyframes.push_front({ m_frame_counter , face_euler }); 100 | fvs_face.set_keyframe(true); 101 | return true; 102 | } 103 | 104 | } // namespace fvs 105 | -------------------------------------------------------------------------------- /face_video_segment/keyframer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #ifndef FVS_KEYFRAMER_H__ 30 | #define FVS_KEYFRAMER_H__ 31 | 32 | #include "face_video_segment.pb.h" 33 | 34 | // std 35 | #include 36 | 37 | // sfl 38 | #include 39 | 40 | // OpenCV 41 | #include 42 | 43 | namespace fvs { 44 | 45 | /** @brief Divides a sequence of frames containing faces into keyframes. 46 | Each keyframe is associated with it's corresponding face orientation. 47 | A new frame will only become a keyframe if the euclidean distance between 48 | it's face orientation (in euler angles) to the orientation of any other keyframe 49 | is large enough. 50 | */ 51 | class Keyframer 52 | { 53 | public: 54 | 55 | struct Keyframe 56 | { 57 | int id; ///< Frame sequence id. 58 | cv::Point3f euler_angles; ///< Face orientation in euler angles. 59 | }; 60 | 61 | struct FaceData 62 | { 63 | std::list keyframes; 64 | std::list> history; 65 | int frame_updated_ind; 66 | }; 67 | 68 | /** @brief Constructor. 69 | @param[in] start_frame The frame index to start adding keyframes from. 70 | @param[in] stability_range Specify how many frames in a row must have 71 | landmarks before a new keyframe can be added. 72 | */ 73 | Keyframer(int start_frame = 10, int stability_range = 5); 74 | 75 | /** @brief Add a new frame. 76 | @param[in] sfl_frame Landmarks frame. 77 | @param[in,out] fvs_frame Regions frame. 78 | */ 79 | void addFrame(const sfl::Frame& sfl_frame, Frame& fvs_frame); 80 | 81 | private: 82 | /** @brief Add a new face. 83 | @param[in] sfl_face Landmarks face. 84 | @param[in,out] fvs_frame Regions frame. 85 | */ 86 | bool addFace(const sfl::Face& sfl_face, Face& fvs_face); 87 | 88 | private: 89 | //std::list m_keyframes; 90 | //std::list> m_history; 91 | std::map m_face_data_map; 92 | int m_start_frame; 93 | int m_stability_range; 94 | int m_frame_counter; 95 | }; 96 | 97 | } // namespace fvs 98 | 99 | #endif // FVS_KEYFRAMER_H__ 100 | -------------------------------------------------------------------------------- /face_video_segment/landmarks_unit.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #include "landmarks_unit.h" 30 | #include "face_segmentation_unit.h" 31 | #include 32 | 33 | #include 34 | #include 35 | #include 36 | #include 37 | 38 | #include "base/base_impl.h" 39 | #include 40 | /* 41 | // dlib 42 | #include 43 | #include 44 | #include 45 | */ 46 | 47 | using namespace video_framework; 48 | 49 | namespace segmentation 50 | { 51 | LandmarksUnit::LandmarksUnit(const LandmarksOptions& options) 52 | : options_(options), main_face_id_(0) 53 | { 54 | sfl_ = sfl::SequenceFaceLandmarks::create(options.landmarks_path, 55 | options.frame_scale, options.tracking); 56 | if (sfl_->size() > 0) 57 | { 58 | const std::list>& sequence = sfl_->getSequence(); 59 | sequence_it = sequence.begin(); 60 | main_face_id_ = sfl::getMainFaceID(sequence); 61 | } 62 | } 63 | 64 | LandmarksUnit::~LandmarksUnit() { 65 | } 66 | 67 | bool LandmarksUnit::OpenStreams(StreamSet* set) { 68 | // Find video stream idx. 69 | video_stream_idx_ = FindStreamIdx(options_.video_stream_name, set); 70 | 71 | if (video_stream_idx_ < 0) { 72 | LOG(ERROR) << "LandmarksUnit::OpenStreams: Could not find Video stream!\n"; 73 | return false; 74 | } 75 | 76 | const VideoStream& vid_stream = set->at(video_stream_idx_)->As(); 77 | frame_width_ = vid_stream.frame_width(); 78 | frame_height_ = vid_stream.frame_height(); 79 | 80 | // Add stream. 81 | DataStream* landmarks_stream = new DataStream(options_.stream_name); 82 | set->push_back(shared_ptr(landmarks_stream)); 83 | 84 | return true; 85 | } 86 | 87 | void LandmarksUnit::ProcessFrame(FrameSetPtr input, std::list* output) 88 | { 89 | // Retrieve video frame 90 | const VideoFrame* frame = input->at(video_stream_idx_)->AsPtr(); 91 | cv::Mat image; 92 | frame->MatView(&image); 93 | 94 | const sfl::Frame* sfl_frame = nullptr; 95 | if (sequence_it != sfl_->getSequence().end()) 96 | { 97 | // Calculate landmarks 98 | if (sfl_->getModel().empty()) 99 | sfl_frame = (*sequence_it++).get(); 100 | else sfl_frame = &sfl_->addFrame(image); 101 | } 102 | 103 | // Forward input 104 | input->push_back(std::shared_ptr>( 105 | new ValueFrame(sfl_frame))); 106 | // input->push_back(std::shared_ptr>>( 107 | // new PointerFrame>(std::move(landmarks_out)))); 108 | 109 | output->push_back(input); 110 | } 111 | 112 | bool LandmarksUnit::PostProcess(list* append) 113 | { 114 | return false; 115 | } 116 | 117 | LandmarksRendererUnit::LandmarksRendererUnit(const LandmarksRendererOptions& options) 118 | : options_(options) 119 | { 120 | } 121 | 122 | LandmarksRendererUnit::~LandmarksRendererUnit() { 123 | } 124 | 125 | bool LandmarksRendererUnit::OpenStreams(StreamSet* set) 126 | { 127 | // Find video stream idx. 128 | video_stream_idx_ = FindStreamIdx(options_.video_stream_name, set); 129 | 130 | if (video_stream_idx_ < 0) { 131 | LOG(ERROR) << "LandmarksRendererUnit::OpenStreams: " 132 | << "Could not find Video stream!\n"; 133 | return false; 134 | } 135 | 136 | const VideoStream& vid_stream = set->at(video_stream_idx_)->As(); 137 | 138 | frame_width = vid_stream.frame_width(); 139 | frame_height = vid_stream.frame_height(); 140 | 141 | // Get landmarks stream 142 | landmarks_stream_idx_ = FindStreamIdx(options_.landmarks_stream_name, set); 143 | if (landmarks_stream_idx_ < 0) { 144 | LOG(ERROR) << "LandmarksRendererUnit::OpenStreams: " 145 | << "Could not find landmarks stream!\n"; 146 | return false; 147 | } 148 | 149 | // Add stream. 150 | //DataStream* landmarks_stream = new DataStream(options_.stream_name); 151 | //set->push_back(shared_ptr(landmarks_stream)); 152 | 153 | return true; 154 | } 155 | 156 | void LandmarksRendererUnit::ProcessFrame(FrameSetPtr input, std::list* output) 157 | { 158 | // Retrieve video frame 159 | ++frame_num_; 160 | const VideoFrame* frame = input->at(video_stream_idx_)->AsPtr(); 161 | cv::Mat image; 162 | frame->MatView(&image); 163 | 164 | // Retrieve landmarks 165 | const PointerFrame>& landmarks_frame = 166 | input->at(landmarks_stream_idx_)->As>>(); 167 | 168 | const std::vector& landmarks = landmarks_frame.Ref(); 169 | 170 | // Render 171 | sfl::render(image, landmarks); 172 | 173 | // Show overlay 174 | std::string msg = "Frame count: " + std::to_string(frame_num_); 175 | cv::putText(image, msg, cv::Point(15, 15), 176 | cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 102, 255), 1, CV_AA); 177 | 178 | // Forward input 179 | /*input->push_back(std::shared_ptr>( 180 | new PointerFrame(std::move(landmarks_ring))));*/ 181 | 182 | output->push_back(input); 183 | } 184 | 185 | bool LandmarksRendererUnit::PostProcess(list* append) 186 | { 187 | return false; 188 | } 189 | 190 | } // namespace video_framework. 191 | -------------------------------------------------------------------------------- /face_video_segment/landmarks_unit.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #ifndef FACE_VIDEO_SEGMENT_VIDEO_LANDMARKS_UNIT_H__ 30 | #define FACE_VIDEO_SEGMENT_VIDEO_LANDMARKS_UNIT_H__ 31 | 32 | #include "base/base.h" 33 | #include "video_framework/video_unit.h" 34 | #include 35 | #include 36 | #include 37 | 38 | namespace segmentation { 39 | 40 | struct LandmarksOptions { 41 | std::string stream_name = "LandmarksStream"; 42 | std::string video_stream_name = "VideoStream"; 43 | std::string landmarks_path = ""; 44 | float frame_scale = 1.0f; 45 | sfl::FaceTrackingType tracking = sfl::TRACKING_BRISK; 46 | }; 47 | 48 | /** @brief Calculates face landmarks in video frames from stream. 49 | */ 50 | class LandmarksUnit : public video_framework::VideoUnit { 51 | public: 52 | LandmarksUnit(const LandmarksOptions& options); 53 | ~LandmarksUnit(); 54 | 55 | LandmarksUnit(const LandmarksUnit&) = delete; 56 | LandmarksUnit& operator=(const LandmarksUnit&) = delete; 57 | 58 | virtual bool OpenStreams(video_framework::StreamSet* set); 59 | virtual void ProcessFrame(video_framework::FrameSetPtr input, std::list* output); 60 | virtual bool PostProcess(std::list* append); 61 | 62 | private: 63 | LandmarksOptions options_; 64 | std::shared_ptr sfl_; 65 | std::list>::const_iterator sequence_it; 66 | 67 | int video_stream_idx_; 68 | int frame_width_; 69 | int frame_height_; 70 | int main_face_id_; 71 | }; 72 | 73 | struct LandmarksRendererOptions { 74 | //std::string stream_name = "LandmarksRendererStream"; 75 | std::string video_stream_name = "VideoStream"; 76 | std::string landmarks_stream_name = "LandmarksStream"; 77 | }; 78 | 79 | /** @brief Renders face landmarks from stream. 80 | */ 81 | class LandmarksRendererUnit : public video_framework::VideoUnit 82 | { 83 | public: 84 | LandmarksRendererUnit(const LandmarksRendererOptions& options); 85 | ~LandmarksRendererUnit(); 86 | 87 | LandmarksRendererUnit(const LandmarksRendererUnit&) = delete; 88 | LandmarksRendererUnit& operator=(const LandmarksRendererUnit&) = delete; 89 | 90 | virtual bool OpenStreams(video_framework::StreamSet* set); 91 | virtual void ProcessFrame(video_framework::FrameSetPtr input, std::list* output); 92 | virtual bool PostProcess(std::list* append); 93 | 94 | private: 95 | LandmarksRendererOptions options_; 96 | int video_stream_idx_; 97 | int landmarks_stream_idx_; 98 | 99 | int frame_width; 100 | int frame_height; 101 | int frame_num_ = 0; 102 | 103 | }; 104 | 105 | } // namespace segmentation 106 | 107 | #endif // FACE_VIDEO_SEGMENT_VIDEO_LANDMARKS_UNIT_H__ 108 | -------------------------------------------------------------------------------- /face_video_segment/utilities.cpp: -------------------------------------------------------------------------------- 1 | #include "utilities.h" 2 | 3 | #include 4 | 5 | // OpenCV 6 | #include 7 | #include // debug 8 | 9 | using std::runtime_error; 10 | using namespace segmentation; 11 | 12 | namespace fvs 13 | { 14 | void createContours(const VectorMesh& mesh, 15 | const SegmentationDesc_Polygon& poly, 16 | std::vector>& contours) 17 | { 18 | contours.emplace_back(); 19 | std::vector& contour = contours.back(); 20 | 21 | // For each coordinate 22 | contour.resize(poly.coord_idx_size() - 1); 23 | for (int c = 0; c < contour.size(); ++c) 24 | { 25 | int idx = poly.coord_idx(c); 26 | contour[c] = cv::Point(mesh.coord(idx), mesh.coord(idx + 1)); 27 | } 28 | } 29 | 30 | cv::Mat calcSegmentation(cv::Size size, 31 | const google::protobuf::Map& regions, 32 | const segmentation::SegmentationDesc & seg_desc) 33 | { 34 | cv::Mat face_map = cv::Mat::zeros(size, CV_8U); 35 | return calcSegmentation(face_map, regions, seg_desc); 36 | } 37 | 38 | cv::Mat calcSegmentation(const cv::Mat& face_map, 39 | const google::protobuf::Map& regions, 40 | const segmentation::SegmentationDesc& seg_desc) 41 | { 42 | cv::Mat seg = cv::Mat::zeros(face_map.size(), CV_8U); 43 | const VectorMesh& mesh = seg_desc.vector_mesh(); 44 | 45 | // For each region 46 | cv::Mat poly_map = cv::Mat::zeros(face_map.size(), CV_8U); 47 | cv::Scalar poly_color; 48 | for (const auto& r : seg_desc.region()) 49 | { 50 | if (r.vectorization().polygon().empty()) continue; 51 | auto face_region = regions.find((unsigned int)r.id()); 52 | if (face_region == regions.end()) continue; 53 | 54 | // Find holes 55 | std::vector> holes; 56 | for (const auto& poly : r.vectorization().polygon()) 57 | { 58 | if (!poly.hole()) continue; 59 | if (poly.coord_idx_size() == 0) continue; 60 | createContours(mesh, poly, holes); 61 | } 62 | 63 | // For each polygon 64 | int poly_ind = 0; 65 | for (const auto& poly : r.vectorization().polygon()) 66 | { 67 | if (poly.hole()) continue; 68 | if (poly.coord_idx_size() == 0) continue; 69 | PolygonType type = face_region->second.polygons(poly_ind++); 70 | if (type == EMPTY) continue; 71 | std::vector> contours; 72 | createContours(mesh, poly, contours); 73 | 74 | if (!contours.empty()) 75 | { 76 | // Set polygon color 77 | if (type == FULL) poly_color = cv::Scalar(255, 255, 255); 78 | else poly_color = cv::Scalar(128, 128, 128); 79 | 80 | // Render polygon 81 | cv::drawContours(poly_map, contours, 0, poly_color, CV_FILLED); 82 | 83 | // Remove holes 84 | cv::drawContours(poly_map, holes, 0, cv::Scalar(0, 0, 0), CV_FILLED); 85 | 86 | // Add to segmentation 87 | unsigned char *seg_data = seg.data, *poly_map_data = poly_map.data; 88 | unsigned char *face_map_data = face_map.data; 89 | unsigned char pv = 0; 90 | for (size_t i = 0; i < seg.total(); ++i) 91 | { 92 | pv = *poly_map_data++; 93 | if (pv == 255 || (pv == 128 && *face_map_data > 0)) 94 | *seg_data = pv; 95 | ++seg_data; 96 | ++face_map_data; 97 | } 98 | 99 | // Clear map 100 | cv::drawContours(poly_map, contours, 0, cv::Scalar(0, 0, 0), CV_FILLED); 101 | } 102 | } 103 | } 104 | 105 | return seg; 106 | } 107 | 108 | void removeSmallerComponents(cv::Mat& seg) 109 | { 110 | cv::Mat labels; 111 | cv::Mat stats, centroids; 112 | cv::connectedComponentsWithStats(seg, labels, stats, centroids); 113 | if (stats.rows <= 2) return; 114 | 115 | // Find the label of the connected component with maximum area 116 | cv::Mat areas = stats.colRange(4, 5).clone(); 117 | int* areas_data = (int*)areas.data; 118 | int max_label = std::distance(areas_data, 119 | std::max_element(areas_data + 1, areas_data + stats.rows)); 120 | 121 | // Clear smaller components 122 | unsigned char* seg_data = seg.data; 123 | int* labels_data = (int*)labels.data; 124 | for (size_t i = 0; i < seg.total(); ++i, ++seg_data) 125 | if (*labels_data++ != max_label) *seg_data = 0; 126 | } 127 | 128 | void smoothFlaws(cv::Mat& seg, int smooth_iterations = 1, int smooth_kernel_radius = 2) 129 | { 130 | int kernel_size = smooth_kernel_radius * 2 + 1; 131 | cv::Mat kernel = cv::getStructuringElement( 132 | cv::MorphShapes::MORPH_ELLIPSE, cv::Size(kernel_size, kernel_size)); 133 | // for (int i = 0; i < smooth_iterations; ++i) 134 | { 135 | cv::morphologyEx(seg, seg, cv::MORPH_OPEN, kernel, cv::Point(-1, -1), smooth_iterations); 136 | cv::morphologyEx(seg, seg, cv::MORPH_CLOSE, kernel, cv::Point(-1, -1), smooth_iterations); 137 | } 138 | } 139 | 140 | void fillHoles(cv::Mat& seg) 141 | { 142 | double min_val, max_val; 143 | cv::minMaxLoc(seg, &min_val, &max_val); 144 | cv::Mat holes = seg.clone(); 145 | cv::floodFill(holes, cv::Point2i(0, 0), cv::Scalar(max_val)); 146 | for (size_t i = 0; i < seg.total(); ++i) 147 | { 148 | if (holes.data[i] == 0) 149 | seg.data[i] = (unsigned char)max_val; 150 | } 151 | } 152 | 153 | void postprocessSegmentation(cv::Mat & seg, bool disconnected, 154 | bool holes, bool smooth, int smooth_iterations, int smooth_kernel_radius) 155 | { 156 | if(disconnected) removeSmallerComponents(seg); 157 | if(holes) fillHoles(seg); 158 | if(smooth) smoothFlaws(seg, smooth_iterations, smooth_kernel_radius); 159 | if(disconnected) removeSmallerComponents(seg); 160 | if(holes) fillHoles(seg); 161 | } 162 | 163 | void renderBoundaries(cv::Mat& img, int hierarchy_level, 164 | const SegmentationDesc& desc, 165 | const Hierarchy* seg_hier, 166 | const cv::Scalar& color) 167 | { 168 | cv::Point3_ bgr((uchar)color[0], (uchar)color[1], (uchar)color[2]); 169 | cv::Mat tmp(img.size(), img.type()); 170 | RenderRegionsRandomColor(hierarchy_level, false, false, desc, seg_hier, &tmp); 171 | //cv::imshow("img", img); 172 | //cv::imshow("temp", tmp); 173 | //cv::waitKey(0); 174 | 175 | // Edge highlight post-process. 176 | const int height = img.rows; 177 | const int width = img.cols; 178 | const int width_step = img.step[0]; 179 | const int channels = img.channels(); 180 | for (int i = 0; i < height - 1; ++i) { 181 | uint8_t* row_ptr = tmp.ptr(i); 182 | uint8_t* out_row_ptr = img.ptr(i); 183 | for (int j = 0; j < width - 1; ++j, row_ptr += channels, out_row_ptr += channels) 184 | { 185 | if (ColorDiff_L1(row_ptr, row_ptr + channels) != 0 || 186 | ColorDiff_L1(row_ptr, row_ptr + width_step) != 0) 187 | { 188 | out_row_ptr[0] = bgr.x; 189 | out_row_ptr[1] = bgr.y; 190 | out_row_ptr[2] = bgr.z; 191 | } 192 | } 193 | 194 | // Last column. 195 | if (ColorDiff_L1(row_ptr, row_ptr + width_step) != 0) 196 | { 197 | out_row_ptr[0] = bgr.x; 198 | out_row_ptr[1] = bgr.y; 199 | out_row_ptr[2] = bgr.z; 200 | } 201 | } 202 | 203 | // Last row. 204 | uint8_t* row_ptr = tmp.ptr(height - 1); 205 | uint8_t* out_row_ptr = img.ptr(height - 1); 206 | for (int j = 0; j < width - 1; ++j, row_ptr += channels) { 207 | if (ColorDiff_L1(row_ptr, row_ptr + channels) != 0) 208 | { 209 | out_row_ptr[0] = bgr.x; 210 | out_row_ptr[1] = bgr.y; 211 | out_row_ptr[2] = bgr.z; 212 | } 213 | } 214 | } 215 | 216 | void renderSegmentation(cv::Mat& img, const cv::Mat& seg, uchar color) 217 | { 218 | if (img.size() != seg.size()) img.create(seg.size(), seg.type()); 219 | 220 | int r, c; 221 | unsigned char* img_data = img.data; 222 | const unsigned char* seg_data = seg.data; 223 | for (r = 0; r < img.rows; ++r) 224 | { 225 | seg_data = seg.ptr(r); 226 | img_data = img.ptr(r); 227 | for (c = 0; c < img.cols; ++c) 228 | { 229 | if (*seg_data++ > 0) *img_data++ = color; 230 | else *img_data++ = 0; 231 | } 232 | } 233 | } 234 | 235 | void renderSegmentationBlend(cv::Mat& img, const cv::Mat& seg, float alpha, 236 | const cv::Scalar& full_color, const cv::Scalar& intersection_color) 237 | { 238 | cv::Point3_ full_bgr((uchar)full_color[0], (uchar)full_color[1], 239 | (uchar)full_color[2]); 240 | cv::Point3_ intersection_bgr((uchar)intersection_color[0], 241 | (uchar)intersection_color[1], (uchar)intersection_color[2]); 242 | 243 | int r, c; 244 | cv::Point3_* img_data = (cv::Point3_*)img.data; 245 | unsigned char* seg_data = seg.data; 246 | unsigned char s; 247 | for (r = 0; r < img.rows; ++r) 248 | { 249 | for (c = 0; c < img.cols; ++c) 250 | { 251 | s = *seg_data++; 252 | if (s == 255) 253 | { 254 | img_data->x = (unsigned char)std::round(full_bgr.x * alpha + img_data->x*(1 - alpha)); 255 | img_data->y = (unsigned char)std::round(full_bgr.y * alpha + img_data->y*(1 - alpha)); 256 | img_data->z = (unsigned char)std::round(full_bgr.z * alpha + img_data->z*(1 - alpha)); 257 | } 258 | else if (s == 128) 259 | { 260 | img_data->x = (unsigned char)std::round(intersection_bgr.x * alpha + img_data->x*(1 - alpha)); 261 | img_data->y = (unsigned char)std::round(intersection_bgr.y * alpha + img_data->y*(1 - alpha)); 262 | img_data->z = (unsigned char)std::round(intersection_bgr.z * alpha + img_data->z*(1 - alpha)); 263 | } 264 | ++img_data; 265 | } 266 | } 267 | } 268 | 269 | float getFaceDominantSide(const std::vector& landmarks) 270 | { 271 | if (landmarks.size() != 68) return 0; 272 | 273 | const cv::Point& center = landmarks[27]; 274 | const cv::Point& left_eye = landmarks[42]; 275 | const cv::Point& right_eye = landmarks[39]; 276 | float left_dist = cv::norm(center - left_eye); 277 | float right_dist = cv::norm(center - right_eye); 278 | 279 | return left_dist / (left_dist + right_dist); 280 | } 281 | 282 | cv::Rect getFaceBBoxFromSegmentation(const cv::Mat& seg, bool square) 283 | { 284 | int xmin(std::numeric_limits::max()), ymin(std::numeric_limits::max()), 285 | xmax(-1), ymax(-1), sumx(0), sumy(0); 286 | int r, c, foreground_count = 0; 287 | 288 | // For each foreground point 289 | unsigned char* seg_data = seg.data; 290 | for (r = 0; r < seg.rows; ++r) 291 | { 292 | for (c = 0; c < seg.cols; ++c) 293 | { 294 | if (*seg_data++ == 0) continue; 295 | xmin = std::min(xmin, c); 296 | ymin = std::min(ymin, r); 297 | xmax = std::max(xmax, c); 298 | ymax = std::max(ymax, r); 299 | sumx += c; 300 | sumy += r; 301 | ++foreground_count; 302 | } 303 | } 304 | 305 | int width = xmax - xmin + 1; 306 | int height = ymax - ymin + 1; 307 | int centerx = (xmin + xmax) / 2; 308 | int centery = (ymin + ymax) / 2; 309 | int avgx = (int)std::round(sumx / foreground_count); 310 | int avgy = (int)std::round(sumy / foreground_count); 311 | int devx = centerx - avgx; 312 | int devy = centery - avgy; 313 | int dleft = (int)std::round(0.1*width) + abs(devx < 0 ? devx : 0); 314 | int dtop = (int)std::round(height*(std::max(float(width) / height, 1.0f) * 1.5f - 1)) + abs(devy < 0 ? devy : 0); 315 | int dright = (int)std::round(0.1*width) + abs(devx > 0 ? devx : 0); 316 | int dbottom = (int)std::round(0.1*height) + abs(devy > 0 ? devy : 0); 317 | 318 | // Limit to frame boundaries 319 | xmin = std::max(0, xmin - dleft); 320 | ymin = std::max(0, ymin - dtop); 321 | xmax = std::min(seg.cols - 1, xmax + dright); 322 | ymax = std::min(seg.rows - 1, ymax + dbottom); 323 | 324 | // Make square 325 | if (square) 326 | { 327 | int sq_width = std::max(xmax - xmin + 1, ymax - ymin + 1); 328 | centerx = (xmin + xmax) / 2; 329 | centery = (ymin + ymax) / 2; 330 | xmin = centerx - ((sq_width - 1) / 2); 331 | ymin = centery - ((sq_width - 1) / 2); 332 | xmax = xmin + sq_width - 1; 333 | ymax = ymin + sq_width - 1; 334 | 335 | // Limit to frame boundaries 336 | xmin = std::max(0, xmin); 337 | ymin = std::max(0, ymin); 338 | xmax = std::min(seg.cols - 1, xmax); 339 | ymax = std::min(seg.rows - 1, ymax); 340 | } 341 | 342 | return cv::Rect(cv::Point(xmin, ymin), cv::Point(xmax, ymax)); 343 | } 344 | 345 | } // namespace fvs 346 | 347 | -------------------------------------------------------------------------------- /face_video_segment/utilities.h: -------------------------------------------------------------------------------- 1 | /** @file 2 | @brief Face video segmentation utility functions. 3 | */ 4 | 5 | #ifndef __FVG_UTILITIES__ 6 | #define __FVG_UTILITIES__ 7 | 8 | #include 9 | #include 10 | 11 | namespace fvs 12 | { 13 | void createContours(const segmentation::VectorMesh& mesh, 14 | const segmentation::SegmentationDesc_Polygon& poly, 15 | std::vector>& contours); 16 | 17 | cv::Mat calcSegmentation(cv::Size size, 18 | const google::protobuf::Map& regions, 19 | const segmentation::SegmentationDesc& seg_desc); 20 | 21 | cv::Mat calcSegmentation(const cv::Mat& face_map, 22 | const google::protobuf::Map& regions, 23 | const segmentation::SegmentationDesc& seg_desc); 24 | 25 | void postprocessSegmentation(cv::Mat& seg, bool disconnected = true, 26 | bool holes = true, bool smooth = true, int smooth_iterations = 1, 27 | int smooth_kernel_radius = 2); 28 | 29 | // void createFullFace(const std::vector& landmarks, 30 | // std::vector& full_face); 31 | 32 | /** Render the boundaries of the segmentation regions. 33 | @param img The image that boundaries will be rendered on. 34 | @param hierarchy_level The level of the hierarchy. 35 | 0 denotes the over-segmentation. >= 1 : denotes a hierarchical level. 36 | @param desc The segmentation descriptor which contains the segmentations data. 37 | @param seg_hier The hierarchy information. 38 | @param color Boundaries color. 39 | */ 40 | void renderBoundaries(cv::Mat& img, int hierarchy_level, 41 | const segmentation::SegmentationDesc& desc, 42 | const segmentation::Hierarchy* seg_hier, 43 | const cv::Scalar& color = cv::Scalar(255, 255, 255)); 44 | 45 | void renderSegmentation(cv::Mat& img, const cv::Mat& seg, uchar color); 46 | 47 | /** Render segmentation blended with image 48 | @param img The image that the segmentation will be blended with. 49 | @param seg The segmentation as an 8-bit image. 50 | Values of 255 denote full regions and values of 128 denote intersection regions. 51 | @param alpha Blending weight [0, 1]. 52 | 0 means completely transparent and 1 means completely opaque. 53 | @param full_color The color to be used for full regions. 54 | @param intersection_color The color to be used for intersection regions. 55 | */ 56 | void renderSegmentationBlend(cv::Mat& img, const cv::Mat& seg, float alpha = 0.5f, 57 | const cv::Scalar& full_color = cv::Scalar(0, 0, 255), 58 | const cv::Scalar& intersection_color = cv::Scalar(255, 0, 0)); 59 | 60 | float getFaceDominantSide(const std::vector& landmarks); 61 | 62 | /** @brief Get face bounding box from segmentation. 63 | @param seg The segmentation. 64 | @param square Make the bounding box square (limited to segmentation boundaries). 65 | */ 66 | cv::Rect getFaceBBoxFromSegmentation(const cv::Mat& seg, bool square); 67 | 68 | } // namespace fvs 69 | 70 | #endif // __FVG_UTILITIES__ 71 | -------------------------------------------------------------------------------- /face_video_segment/video_reader_unit2.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #include "video_reader_unit2.h" 30 | 31 | #include 32 | 33 | #include "base/base_impl.h" 34 | 35 | using namespace video_framework; 36 | 37 | namespace fvs 38 | { 39 | 40 | VideoReaderUnit2::VideoReaderUnit2(const VideoReader2Options& options, 41 | const std::string& video_file) 42 | : options_(options), 43 | video_file_(video_file) 44 | { 45 | } 46 | 47 | VideoReaderUnit2::~VideoReaderUnit2() 48 | { 49 | } 50 | 51 | bool VideoReaderUnit2::OpenStreams(StreamSet* set) 52 | { 53 | // Initialize video capture 54 | m_cap.reset(new cv::VideoCapture()); 55 | if (!m_cap->open(video_file_)) 56 | LOG(ERROR) << "Could not open file: " << video_file_; 57 | 58 | frame_width_ = (int)m_cap->get(cv::CAP_PROP_FRAME_WIDTH); 59 | frame_height_ = (int)m_cap->get(cv::CAP_PROP_FRAME_HEIGHT); 60 | fps_ = m_cap->get(cv::CAP_PROP_FPS); 61 | frame_width_step_ = frame_width_ * 3; 62 | 63 | // Add video output stream. 64 | VideoStream* rendered_stream = new VideoStream(frame_width_, 65 | frame_height_, 66 | frame_width_step_, 67 | fps_, 68 | PIXEL_FORMAT_BGR24, 69 | options_.stream_name); 70 | 71 | set->push_back(shared_ptr(rendered_stream)); 72 | frame_num_ = 0; 73 | return true; 74 | } 75 | 76 | void VideoReaderUnit2::ProcessFrame(FrameSetPtr input, list* output) { 77 | if (!used_as_root_) { 78 | input->push_back(shared_ptr(ReadNextFrame())); 79 | output->push_back(input); 80 | ++frame_num_; 81 | } 82 | } 83 | 84 | bool VideoReaderUnit2::PostProcess(list* append) { 85 | if (used_as_root_) { 86 | VideoFrame* next_frame = ReadNextFrame(); 87 | if (next_frame != nullptr) { 88 | // Make new frameset and push VideoFrame. 89 | FrameSetPtr frame_set (new FrameSet()); 90 | frame_set->push_back(shared_ptr(next_frame)); 91 | append->push_back(frame_set); 92 | ++frame_num_; 93 | return true; 94 | } else { 95 | return false; 96 | } 97 | } 98 | return false; 99 | } 100 | 101 | VideoFrame* VideoReaderUnit2::ReadNextFrame() 102 | { 103 | if (!m_cap->read(m_frame)) return nullptr; 104 | 105 | VideoFrame* curr_frame = new VideoFrame(frame_width_, 106 | frame_height_, 107 | 3); 108 | 109 | memcpy(curr_frame->mutable_data(), m_frame.data, 110 | frame_width_ * frame_height_ * 3); 111 | 112 | return curr_frame; 113 | } 114 | 115 | } // namespace fvs 116 | -------------------------------------------------------------------------------- /face_video_segment/video_reader_unit2.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #ifndef VFS_VIDEO_READER_UNIT2_H__ 30 | #define VFS_VIDEO_READER_UNIT2_H__ 31 | 32 | #include "base/base.h" 33 | #include "video_framework/video_unit.h" 34 | 35 | #include 36 | 37 | struct AVCodec; 38 | struct AVCodecContext; 39 | struct AVFormatContext; 40 | struct AVFrame; 41 | struct AVPacket; 42 | struct SwsContext; 43 | 44 | namespace fvs { 45 | 46 | struct VideoReader2Options { 47 | int trim_frames = 0; 48 | std::string stream_name = "VideoStream"; 49 | video_framework::VideoPixelFormat pixel_format = 50 | video_framework::PIXEL_FORMAT_BGR24; 51 | 52 | // For settings below only downscale will be performed, ie. never upscaling. 53 | enum DOWNSCALE { 54 | DOWNSCALE_NONE, 55 | DOWNSCALE_BY_FACTOR, // Resizes each dimension by downscale_factor. 56 | 57 | DOWNSCALE_TO_MIN_SIZE, // Resizes minimum dimension to downscale_size. 58 | DOWNSCALE_TO_MAX_SIZE, // Resizes maximum dimension to downscale_size. 59 | }; 60 | 61 | DOWNSCALE downscale = DOWNSCALE_NONE; 62 | 63 | float downscale_factor = 1.0f; 64 | int downscale_size = 0; 65 | }; 66 | 67 | /** @brief Reads video frames from file into stream. 68 | */ 69 | class VideoReaderUnit2 : public video_framework::VideoUnit { 70 | public: 71 | VideoReaderUnit2(const VideoReader2Options& options, 72 | const std::string& video_file); 73 | ~VideoReaderUnit2(); 74 | 75 | virtual bool OpenStreams(video_framework::StreamSet* set); 76 | virtual void ProcessFrame(video_framework::FrameSetPtr input, 77 | std::list* output); 78 | virtual bool PostProcess(std::list* append); 79 | /* 80 | // Experimental (might not seek to correct locations). 81 | virtual bool SeekImpl(int64_t pts); 82 | bool PrevFrame(); 83 | 84 | // Can be positive or negative. 85 | bool SkipFrames(int frame_offset); 86 | */ 87 | 88 | private: 89 | // Returns allocated VideoFrame (ownership passed to caller). 90 | // Returns NULL if end of file is reached. 91 | video_framework::VideoFrame* ReadNextFrame(); 92 | 93 | private: 94 | VideoReader2Options options_; 95 | std::string video_file_; 96 | 97 | int video_stream_idx_; 98 | int frame_num_ = 0; 99 | 100 | int frame_width_ = 0; 101 | int frame_height_ = 0; 102 | int frame_width_step_ = 0; 103 | double fps_; 104 | 105 | bool used_as_root_ = true; 106 | 107 | std::unique_ptr m_cap; 108 | cv::Mat m_frame; 109 | }; 110 | 111 | } // namespace fvs. 112 | 113 | #endif // VFS_VIDEO_READER_UNIT2_H__ 114 | -------------------------------------------------------------------------------- /face_video_segment/video_writer_unit2.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #include "video_writer_unit2.h" 30 | 31 | #include "base/base_impl.h" 32 | 33 | #include 34 | #include 35 | #include 36 | 37 | using namespace video_framework; 38 | 39 | namespace fvs 40 | { 41 | VideoWriterUnit2::VideoWriterUnit2(const VideoWriter2Options& options, 42 | const std::string& video_file) 43 | : options_(options), video_file_(video_file) 44 | { 45 | } 46 | 47 | VideoWriterUnit2::~VideoWriterUnit2() { 48 | } 49 | 50 | bool VideoWriterUnit2::OpenStreams(StreamSet* set) { 51 | // Find video stream idx. 52 | video_stream_idx_ = FindStreamIdx(options_.stream_name, set); 53 | 54 | if (video_stream_idx_ < 0) { 55 | LOG(ERROR) << "Could not find Video stream!\n"; 56 | return false; 57 | } 58 | 59 | const VideoStream& vid_stream = set->at(video_stream_idx_)->As(); 60 | 61 | const int frame_width = vid_stream.frame_width(); 62 | const int frame_height = vid_stream.frame_height(); 63 | const float fps = vid_stream.fps(); 64 | 65 | //writer_.open(video_file_, CV_FOURCC('F', 'M', 'P', '4'), fps, cv::Size(frame_width, frame_height)); 66 | writer_.open(video_file_, CV_FOURCC('X', '2', '6', '4'), fps, cv::Size(frame_width, frame_height)); 67 | 68 | if(!writer_.isOpened()) 69 | LOG(ERROR) << "Could not open video writer!\n"; 70 | 71 | // Add stream. 72 | //DataStream* landmarks_stream = new DataStream(options_.stream_name); 73 | //set->push_back(shared_ptr(landmarks_stream)); 74 | 75 | return true; 76 | } 77 | 78 | void VideoWriterUnit2::ProcessFrame(FrameSetPtr input, std::list* output) 79 | { 80 | // Retrieve video frame 81 | const VideoFrame* frame = input->at(video_stream_idx_)->AsPtr(); 82 | cv::Mat image; 83 | frame->MatView(&image); 84 | 85 | writer_.write(image); 86 | 87 | // Forward input 88 | output->push_back(input); 89 | } 90 | 91 | bool VideoWriterUnit2::PostProcess(list* append) 92 | { 93 | return false; 94 | } 95 | 96 | } // namespace video_framework. 97 | -------------------------------------------------------------------------------- /face_video_segment/video_writer_unit2.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #ifndef FACE_VIDEO_SEGMENT_VIDEO_WRITER_UNIT_2_H__ 30 | #define FACE_VIDEO_SEGMENT_VIDEO_WRITER_UNIT_2_H__ 31 | 32 | #include "base/base.h" 33 | #include "video_framework/video_unit.h" 34 | 35 | #include 36 | 37 | namespace fvs 38 | { 39 | 40 | struct VideoWriter2Options { 41 | // Codec settings. 42 | // Bit-rate default is 2000 kbps 43 | int bit_rate = 2000000; 44 | 45 | // Use video streams fps as default. 46 | // TODO: will be ignored, fix! 47 | float fps = 0; 48 | 49 | // Scale factor of output. Overrides all other scale settings. 50 | float scale = 1.0f; 51 | 52 | // width and height will be rounded to multiple of fraction. 53 | int fraction = 4; 54 | 55 | // If > 0, maximum dimension is scaled to fit that value. Can not both be set. 56 | int scale_max_dim = 0; 57 | int scale_min_dim = 0; 58 | 59 | // Guess format from format std::string as default. 60 | // Call "ffmpeg -formats" to get a std::list of valid formats. 61 | std::string output_format; 62 | 63 | // Output stream name. 64 | std::string stream_name = "VideoStream"; 65 | }; 66 | 67 | /** @brief Writes video frames from stream to file. 68 | */ 69 | class VideoWriterUnit2 : public video_framework::VideoUnit { 70 | public: 71 | public: 72 | VideoWriterUnit2(const VideoWriter2Options& options, 73 | const std::string& video_file); 74 | ~VideoWriterUnit2(); 75 | 76 | VideoWriterUnit2(const VideoWriterUnit2&) = delete; 77 | VideoWriterUnit2& operator=(const VideoWriterUnit2&) = delete; 78 | 79 | virtual bool OpenStreams(video_framework::StreamSet* set); 80 | virtual void ProcessFrame(video_framework::FrameSetPtr input, std::list* output); 81 | virtual bool PostProcess(std::list* append); 82 | 83 | private: 84 | VideoWriter2Options options_; 85 | std::string video_file_; 86 | 87 | int video_stream_idx_; 88 | 89 | cv::VideoWriter writer_; 90 | }; 91 | 92 | } // namespace segmentation 93 | 94 | #endif // FACE_VIDEO_SEGMENT_VIDEO_WRITER_UNIT_2_H__ 95 | -------------------------------------------------------------------------------- /fvs_editor/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(FVS_EDITOR_SRC fvs_editor.cpp editor.cpp editor_ui.cpp) 2 | set(FVS_EDITOR_HDR editor.h) 3 | qt5_wrap_cpp(FVS_EDITOR_HDR_MOC ${FVS_EDITOR_HDR}) 4 | qt5_add_resources(FVS_EDITOR_QRC fvs_editor.qrc) 5 | if(MSVC) 6 | set(FVS_EDITOR_SRC ${FVS_EDITOR_SRC} fvs_editor.rc) 7 | endif() 8 | 9 | add_executable(fvs_editor ${FVS_EDITOR_SRC} ${FVS_EDITOR_HDR} 10 | ${FVS_EDITOR_HDR_MOC} ${FVS_EDITOR_QRC}) 11 | target_include_directories(fvs_editor PRIVATE 12 | ${CMAKE_SOURCE_DIR}/face_video_segment 13 | ${CMAKE_BINARY_DIR}/face_video_segment) 14 | target_link_libraries(fvs_editor PRIVATE face_video_segment) 15 | 16 | # Installation 17 | install(TARGETS fvs_editor 18 | EXPORT face_video_segment-targets 19 | RUNTIME DESTINATION bin COMPONENT dev 20 | LIBRARY DESTINATION lib COMPONENT dev 21 | ARCHIVE DESTINATION lib COMPONENT dev) 22 | -------------------------------------------------------------------------------- /fvs_editor/editor.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #ifndef __FVS_EDITOR_H__ 30 | #define __FVS_EDITOR_H__ 31 | 32 | // std 33 | #include 34 | #include 35 | 36 | // Qt 37 | #include 38 | 39 | // OpenCV 40 | #include 41 | 42 | // Forward declarations 43 | class QLabel; 44 | class QImage; 45 | class QSlider; 46 | class QMouseEvent; 47 | class QKeyEvent; 48 | class QComboBox; 49 | class QToolButton; 50 | class QString; 51 | class QCheckBox; 52 | class QSpinBox; 53 | class QAction; 54 | 55 | namespace cv 56 | { 57 | class Mat; 58 | //class Point; 59 | class VideoCapture; 60 | } 61 | 62 | namespace sfl 63 | { 64 | class SequenceFaceLandmarks; 65 | class Frame; 66 | } 67 | 68 | namespace segmentation 69 | { 70 | class SegmentationReader; 71 | class SegmentationDesc; 72 | } 73 | 74 | namespace google 75 | { 76 | namespace protobuf 77 | { 78 | template 79 | class Map; 80 | } 81 | } 82 | 83 | namespace fvs 84 | { 85 | class Sequence; 86 | class Frame; 87 | class Face; 88 | class Region; 89 | //enum PolygonType; 90 | class Keyframer; 91 | 92 | class Editor : public QMainWindow 93 | { 94 | Q_OBJECT 95 | 96 | public: 97 | explicit Editor(const std::string& fvs_path, const std::string& output_dir = "", 98 | const std::string& video_file = "", const std::string& seg_file = "", 99 | const std::string& landmarks_file = "", bool debug = false); 100 | ~Editor(); 101 | 102 | private: 103 | // UI 104 | void createMenu(); 105 | 106 | protected: 107 | bool event(QEvent *event) Q_DECL_OVERRIDE; 108 | bool eventFilter(QObject * object, QEvent * event) Q_DECL_OVERRIDE; 109 | void keyPressEvent(QKeyEvent* event) Q_DECL_OVERRIDE; 110 | 111 | private: 112 | void update(); 113 | void updateLater(); 114 | void seek(int index); 115 | void pause(bool pause); 116 | void render(cv::Mat& frame); 117 | void regionSelected(QMouseEvent* event); 118 | Face& getFaceForEditing(); 119 | Frame* getNearestEditedFrame(int frame_id); 120 | Face* getNearestEditedFace(int frame_id, int face_id); 121 | Face* getExistingEditedFace(int frame_id, int face_id); 122 | Face* getExistingInputFace(int frame_id, int face_id); 123 | void getMergedRegions(int frame_id, int face_id, 124 | google::protobuf::Map& region_map); 125 | void getCurrMergedRegions( 126 | google::protobuf::Map& region_map); 127 | bool isKeyframe(int frame_id, int face_id); 128 | bool isInputKeyframe(int frame_id, int face_id); 129 | bool saveFile(const std::string& filename); 130 | void initPostprocessing(Face& face); 131 | void updatePostprocessing(int frame_id, int face_id); 132 | 133 | public slots: 134 | void frameIndexChanged(int); 135 | void frameSliderPress(); 136 | void frameSliderRelease(); 137 | void hierarchyLevelChanged(int); 138 | void playButtonClicked(); 139 | void previousKeyFrameButtonClicked(); 140 | void nextKeyFrameButtonClicked(); 141 | void currFaceIdChanged(const QString& text); 142 | void toggleKeyframe(bool checked); 143 | void newFile(); 144 | void open(); 145 | bool save(); 146 | bool saveAs(); 147 | void about(); 148 | void toggleContours(bool toggled); 149 | void toggleBorders(bool toggled); 150 | void toggleSegmentation(bool toggled); 151 | void togglePostprocess(bool toggled); 152 | void alphaChanged(int n); 153 | void toggleDisconnected(bool toggled); 154 | void toggleHoles(bool toggled); 155 | void toggleSmooth(bool toggled); 156 | void smoothIterationsChanged(int i); 157 | void smoothKernelRadiusChanged(int r); 158 | 159 | private: 160 | QLabel* m_main_widget; 161 | QLabel* m_display_widget; 162 | QSlider* m_frame_slider; 163 | QSlider* m_hierarchy_slider; 164 | QLabel* m_frame_label; 165 | QLabel* m_curr_frame_label; 166 | QLabel* m_max_frame_label; 167 | QLabel* m_hierarchy_label; 168 | QLabel* m_curr_hierarchy_label; 169 | QLabel* m_max_hierarchy_label; 170 | QLabel* m_face_id_label; 171 | QComboBox* m_face_id_combobox; 172 | QToolButton* m_play_button; 173 | QToolButton* m_previous_keyframe_button; 174 | QToolButton* m_next_keyframe_button; 175 | QCheckBox* m_toggle_keyframe_checkbox; 176 | QLabel* m_keyframe_label; 177 | QAction* m_disconnectedAct; 178 | QAction* m_holesAct; 179 | QAction* m_smoothAct; 180 | QLabel* m_smooth_iterations_label; 181 | QLabel* m_smooth_kernel_radius_label; 182 | QSpinBox* m_smooth_iterations_spinbox; 183 | QSpinBox* m_smooth_kernel_radius_spinbox; 184 | 185 | bool m_loop; 186 | bool m_refresh; 187 | bool m_slider_pause; 188 | bool m_update_pending; 189 | bool m_update_frame; 190 | bool m_update_face; 191 | bool m_debug; 192 | 193 | int m_curr_frame_ind; 194 | int m_next_frame_ind; 195 | int m_curr_hierarchy_level; 196 | int m_max_hierarchy_level; 197 | 198 | std::string m_curr_file; 199 | std::string m_output_dir; 200 | std::string m_video_file; 201 | std::string m_seg_file; 202 | std::string m_landmarks_file; 203 | 204 | // video 205 | std::unique_ptr m_cap; 206 | std::unique_ptr m_scaled_frame; 207 | std::unique_ptr m_render_frame; 208 | std::unique_ptr m_render_image; 209 | 210 | // View 211 | bool m_render_contours; 212 | bool m_render_borders; 213 | bool m_render_seg; 214 | bool m_postprocess; 215 | float m_alpha; 216 | 217 | // sfl 218 | std::shared_ptr m_sfl; 219 | std::vector m_sfl_frames; 220 | int m_main_face_id; 221 | 222 | // segmentation 223 | std::unique_ptr m_seg_reader; 224 | std::unique_ptr m_seg_desc; 225 | std::unique_ptr m_seg_hierarchy; 226 | int m_hierarchy_pos; 227 | 228 | // Face segmentation 229 | std::unique_ptr m_input_regions; 230 | std::unique_ptr m_edited_regions; 231 | //std::unique_ptr m_inserted_regions; 232 | //std::unique_ptr m_removed_regions; 233 | int m_edit_index; 234 | std::unique_ptr>> m_face_boundary; 235 | std::unique_ptr m_face_map; 236 | std::unique_ptr m_keyframer; 237 | //std::vector m_keyframes; 238 | int m_curr_face_id; 239 | std::set m_face_ids; 240 | 241 | int m_frame_width, m_frame_height; 242 | double m_fps; 243 | size_t m_total_frames; 244 | }; 245 | 246 | } // namespace fvs 247 | 248 | #endif // __FVS_EDITOR_H__ -------------------------------------------------------------------------------- /fvs_editor/fvs_editor.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #include "editor.h" 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | 42 | #include 43 | 44 | using std::string; 45 | using std::cout; 46 | using std::endl; 47 | using std::cerr; 48 | using namespace boost::program_options; 49 | using namespace boost::filesystem; 50 | using namespace video_framework; 51 | using namespace segmentation; 52 | 53 | int main(int argc, char* argv[]) 54 | { 55 | // Parse command line arguments 56 | string fvsPath, outputDir, segPath, landmarksPath, videoPath; 57 | unsigned int debug; 58 | try { 59 | options_description desc("Allowed options"); 60 | desc.add_options() 61 | ("help", "display the help message") 62 | ("input,i", value(&fvsPath)->required(), 63 | "path to face video segmentation (.fvs)") 64 | ("output,o", value(&outputDir), "output directory") 65 | ("segmentation,s", value(&segPath), "input segmentation protobuffer (.pb)") 66 | ("landmarks,l", value(&landmarksPath), "path to landmarks cache (.pb)") 67 | ("video,v", value(&videoPath), "path to video file") 68 | ("debug,d", value(&debug)->default_value(0), "output debug information") 69 | ; 70 | variables_map vm; 71 | store(command_line_parser(argc, argv).options(desc). 72 | positional(positional_options_description().add("input", -1)).run(), vm); 73 | if (vm.count("help")) { 74 | cout << "Usage: face_video_segment [options]" << endl; 75 | cout << desc << endl; 76 | exit(0); 77 | } 78 | notify(vm); 79 | if (!is_regular_file(fvsPath)) throw error("input must be a path to a file!"); 80 | if (vm.count("output") && !is_directory(outputDir)) 81 | throw error("output must be a path to a directory!"); 82 | /* 83 | if (!is_regular_file(segPath)) throw error("segmentation must be a path to a file!"); 84 | if (!is_regular_file(landmarksPath)) 85 | { 86 | path input = path(inputPath); 87 | landmarksPath = 88 | (input.parent_path() / (input.stem() += "_landmarks.pb")).string(); 89 | if (!is_regular_file(landmarksPath)) 90 | throw error("Couldn't find landmarks model or cache file!"); 91 | } 92 | */ 93 | } 94 | catch (const error& e) { 95 | cout << "Error while parsing command-line arguments: " << e.what() << endl; 96 | cout << "Use --help to display a list of options." << endl; 97 | exit(1); 98 | } 99 | 100 | //try 101 | { 102 | Q_INIT_RESOURCE(fvs_editor); 103 | 104 | QApplication app(argc, argv); 105 | fvs::Editor editor(fvsPath, outputDir, videoPath, segPath, landmarksPath, debug > 0); 106 | editor.show(); 107 | return app.exec(); 108 | } 109 | /* 110 | catch (std::exception& e) 111 | { 112 | cerr << e.what() << endl; 113 | return 1; 114 | }*/ 115 | 116 | return 0; 117 | } -------------------------------------------------------------------------------- /fvs_editor/fvs_editor.qrc: -------------------------------------------------------------------------------- 1 | 2 | 3 | images/new.png 4 | images/open.png 5 | images/save.png 6 | images/contours.png 7 | images/borders.png 8 | images/segmentation.png 9 | images/postprocess.png 10 | images/postprocess_disconnected.png 11 | images/postprocess_holes.png 12 | images/postprocess_smooth.png 13 | images/iterations.png 14 | images/radius.png 15 | 16 | 17 | -------------------------------------------------------------------------------- /fvs_editor/fvs_editor.rc: -------------------------------------------------------------------------------- 1 | IDI_ICON1 ICON DISCARDABLE "images/fvs_editor.ico" -------------------------------------------------------------------------------- /fvs_editor/images/borders.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor/images/borders.png -------------------------------------------------------------------------------- /fvs_editor/images/contours.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor/images/contours.png -------------------------------------------------------------------------------- /fvs_editor/images/fvs_editor.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor/images/fvs_editor.ico -------------------------------------------------------------------------------- /fvs_editor/images/iterations.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor/images/iterations.png -------------------------------------------------------------------------------- /fvs_editor/images/new.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor/images/new.png -------------------------------------------------------------------------------- /fvs_editor/images/open.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor/images/open.png -------------------------------------------------------------------------------- /fvs_editor/images/postprocess.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor/images/postprocess.png -------------------------------------------------------------------------------- /fvs_editor/images/postprocess_disconnected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor/images/postprocess_disconnected.png -------------------------------------------------------------------------------- /fvs_editor/images/postprocess_holes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor/images/postprocess_holes.png -------------------------------------------------------------------------------- /fvs_editor/images/postprocess_smooth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor/images/postprocess_smooth.png -------------------------------------------------------------------------------- /fvs_editor/images/radius.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor/images/radius.png -------------------------------------------------------------------------------- /fvs_editor/images/save.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor/images/save.png -------------------------------------------------------------------------------- /fvs_editor/images/segmentation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor/images/segmentation.png -------------------------------------------------------------------------------- /fvs_editor_v2/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Source 2 | set(FVS_EDITOR_SRC fvs_editor_main.cpp fvs_editor.cpp 3 | fvs_editor_states.h fvs_editor_states.cpp) 4 | set(FVS_EDITOR_HDR fvs_editor.h) 5 | qt5_wrap_cpp(FVS_EDITOR_HDR_MOC ${FVS_EDITOR_HDR}) 6 | qt5_wrap_ui(FVS_EDITOR_UI_MOC fvs_editor.ui) 7 | qt5_add_resources(FVS_EDITOR_QRC fvs_editor.qrc) 8 | 9 | if(MSVC) 10 | set(FVS_EDITOR_SRC ${FVS_EDITOR_SRC} fvs_editor.rc) 11 | endif() 12 | 13 | # Target 14 | add_executable(fvs_editor ${FVS_EDITOR_SRC} ${FVS_EDITOR_HDR} 15 | ${FVS_EDITOR_HDR_MOC} ${FVS_EDITOR_UI_MOC} ${FVS_EDITOR_QRC}) 16 | target_include_directories(fvs_editor PRIVATE 17 | ${CMAKE_SOURCE_DIR}/face_video_segment 18 | ${CMAKE_BINARY_DIR}/face_video_segment 19 | ${CMAKE_SOURCE_DIR}/fvs_editor 20 | ${CMAKE_BINARY_DIR}/fvs_editor) 21 | target_link_libraries(fvs_editor PRIVATE face_video_segment) 22 | 23 | # Installation 24 | install(TARGETS fvs_editor 25 | EXPORT face_video_segment-targets 26 | RUNTIME DESTINATION bin COMPONENT dev 27 | LIBRARY DESTINATION lib COMPONENT dev 28 | ARCHIVE DESTINATION lib COMPONENT dev) -------------------------------------------------------------------------------- /fvs_editor_v2/fvs_editor.cpp: -------------------------------------------------------------------------------- 1 | #include "fvs_editor.h" 2 | //#include 3 | #include 4 | 5 | #include 6 | #include // 7 | 8 | // Boost 9 | #include 10 | 11 | // OpenCV 12 | #include 13 | #include 14 | 15 | // Qt 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include // 23 | 24 | using namespace boost::filesystem; 25 | 26 | namespace fvs 27 | { 28 | Editor::Editor() : 29 | sm(this) 30 | { 31 | setupUi(this); 32 | setupBl(); 33 | } 34 | 35 | void Editor::setupBl() 36 | { 37 | // Initialize state machine 38 | sm.initiate(); 39 | 40 | // Connect actions 41 | connect(actionOpen, &QAction::triggered, this, &Editor::open); 42 | connect(actionClose, &QAction::triggered, this, &Editor::close); 43 | connect(actionPlay, &QAction::triggered, this, &Editor::playPause); 44 | connect(actionBackward, &QAction::triggered, this, &Editor::backward); 45 | connect(actionForward, &QAction::triggered, this, &Editor::forward); 46 | connect(frame_slider, SIGNAL(valueChanged(int)), this, SLOT(frameSliderChanged(int))); 47 | connect(actionContours, SIGNAL(toggled(bool)), this, SLOT(toggleRenderParams(bool))); 48 | connect(actionBorders, SIGNAL(toggled(bool)), this, SLOT(toggleRenderParams(bool))); 49 | connect(actionSegmentation, SIGNAL(toggled(bool)), this, SLOT(toggleRenderParams(bool))); 50 | connect(actionPostprocessing, SIGNAL(toggled(bool)), this, SLOT(toggleRenderParams(bool))); 51 | 52 | play_pause_btn->setDefaultAction(actionPlay); 53 | backward_btn->setDefaultAction(actionBackward); 54 | forward_btn->setDefaultAction(actionForward); 55 | keyframe_btn->setDefaultAction(actionKeyframe); 56 | 57 | // Add scroll bar to toolbar 58 | QSlider* alpha_slider = new QSlider(Qt::Horizontal, this); 59 | alpha_slider->setMinimum(0); 60 | alpha_slider->setMaximum(100); 61 | alpha_slider->setValue(25); 62 | //m_alpha = alpha_slider->value() / 100.0f; 63 | alpha_slider->setMaximumWidth(64); 64 | alpha_slider->setStatusTip("Segmentation Opacity"); 65 | //alpha_slider->setFocusPolicy(Qt::NoFocus); 66 | //connect(alpha_slider, SIGNAL(valueChanged(int)), this, SLOT(alphaChanged(int))); 67 | viewToolBar->insertWidget(actionPostprocessing, alpha_slider); 68 | 69 | // Add smooth iterations 70 | QPixmap iterationsPixmap(":/images/iterations.png"); 71 | QLabel* smooth_iterations_label = new QLabel(this); 72 | smooth_iterations_label->setPixmap(iterationsPixmap); 73 | postToolBar->addWidget(smooth_iterations_label); 74 | smooth_iterations_spinbox = new QSpinBox(this); 75 | smooth_iterations_spinbox->setMinimum(1); 76 | smooth_iterations_spinbox->setMaximum(10); 77 | smooth_iterations_spinbox->setStatusTip("Smooth iterations"); 78 | //smooth_iterations_spinbox->setFocusPolicy(Qt::NoFocus); 79 | connect(smooth_iterations_spinbox, SIGNAL(valueChanged(int)), this, SLOT(smoothIterationsChanged(int))); 80 | postToolBar->addWidget(smooth_iterations_spinbox); 81 | postToolBar->addSeparator(); 82 | 83 | // Add smooth kernel radius 84 | QPixmap radiusPixmap(":/images/radius.png"); 85 | QLabel* smooth_kernel_radius_label = new QLabel(this); 86 | smooth_kernel_radius_label->setPixmap(radiusPixmap); 87 | postToolBar->addWidget(smooth_kernel_radius_label); 88 | smooth_kernel_radius_spinbox = new QSpinBox(this); 89 | smooth_kernel_radius_spinbox->setMinimum(1); 90 | smooth_kernel_radius_spinbox->setMaximum(10); 91 | smooth_kernel_radius_spinbox->setValue(2); 92 | smooth_kernel_radius_spinbox->setStatusTip("Smooth kernel radius"); 93 | //smooth_kernel_radius_spinbox->setFocusPolicy(Qt::NoFocus); 94 | connect(smooth_kernel_radius_spinbox, SIGNAL(valueChanged(int)), this, SLOT(smoothKernelRadiusChanged(int))); 95 | postToolBar->addWidget(smooth_kernel_radius_spinbox); 96 | 97 | // Adjust window size 98 | adjustSize(); 99 | } 100 | 101 | void Editor::setInputPath(const std::string & input_path) 102 | { 103 | if (path(input_path).extension() == ".lms") 104 | initLandmarks(input_path); 105 | else initVideoSource(input_path); 106 | } 107 | 108 | void Editor::initLandmarks(const std::string & _landmarks_path) 109 | { 110 | } 111 | 112 | void Editor::initVideoSource(const std::string & _sequence_path) 113 | { 114 | } 115 | 116 | void Editor::resizeEvent(QResizeEvent* event) 117 | { 118 | QMainWindow::resizeEvent(event); 119 | 120 | QSize displaySize = display->size(); 121 | render_frame = cv::Mat::zeros(displaySize.height(), displaySize.width(), CV_8UC3); 122 | 123 | // Make Qt image. 124 | render_image.reset(new QImage((const uint8_t*)render_frame.data, 125 | render_frame.cols, 126 | render_frame.rows, 127 | render_frame.step[0], 128 | QImage::Format_RGB888)); 129 | 130 | sm.process_event(EvUpdate()); 131 | } 132 | 133 | void Editor::timerEvent(QTimerEvent *event) 134 | { 135 | sm.process_event(EvTimerTick()); 136 | } 137 | 138 | void Editor::open() 139 | { 140 | QString file = QFileDialog::getOpenFileName( 141 | this, 142 | "Select one or more files to open", 143 | QString(), 144 | "Landmarks (*.lms);;Videos (*.mp4 *.mkv *.avi *wmv);;All files (*.*)", 145 | nullptr); 146 | setInputPath(file.toStdString()); 147 | } 148 | 149 | void Editor::close() 150 | { 151 | QMainWindow::close(); 152 | } 153 | 154 | void Editor::playPause() 155 | { 156 | sm.process_event(EvPlayPause()); 157 | } 158 | 159 | void Editor::backward() 160 | { 161 | sm.process_event(EvSeek(curr_frame_pos - 1)); 162 | } 163 | 164 | void Editor::forward() 165 | { 166 | sm.process_event(EvSeek(curr_frame_pos + 1)); 167 | } 168 | 169 | void Editor::frameSliderChanged(int i) 170 | { 171 | sm.process_event(EvSeek(i)); 172 | } 173 | 174 | void Editor::toggleRenderParams(bool toggled) 175 | { 176 | sm.process_event(EvUpdate()); 177 | } 178 | 179 | void Editor::render() 180 | { 181 | } 182 | 183 | } // namespace fvs 184 | 185 | -------------------------------------------------------------------------------- /fvs_editor_v2/fvs_editor.h: -------------------------------------------------------------------------------- 1 | #ifndef __SFL_VIEWER_H__ 2 | #define __SFL_VIEWER_H__ 3 | 4 | #include "ui_fvs_editor.h" 5 | #include "fvs_editor_states.h" 6 | 7 | //#include 8 | #include 9 | 10 | #include 11 | 12 | // Qt 13 | 14 | namespace fvs 15 | { 16 | class Editor : public QMainWindow, public Ui::Editor 17 | { 18 | Q_OBJECT 19 | 20 | public: 21 | Editor(); 22 | void setupBl(); 23 | 24 | void setInputPath(const std::string& input_path); 25 | void initLandmarks(const std::string& _landmarks_path); 26 | void initVideoSource(const std::string& _sequence_path); 27 | 28 | protected: 29 | void resizeEvent(QResizeEvent* event) Q_DECL_OVERRIDE; 30 | void timerEvent(QTimerEvent *event) Q_DECL_OVERRIDE; 31 | 32 | public slots: 33 | void open(); 34 | void close(); 35 | void playPause(); 36 | void backward(); 37 | void forward(); 38 | void frameSliderChanged(int i); 39 | void toggleRenderParams(bool toggled); 40 | void render(); 41 | 42 | public: 43 | EditorSM sm; 44 | std::string sequence_path; 45 | std::string landmarks_path; 46 | 47 | // Extra UI components 48 | QSpinBox* smooth_iterations_spinbox; 49 | QSpinBox* smooth_kernel_radius_spinbox; 50 | 51 | // Video 52 | //std::unique_ptr vs; 53 | cv::Mat frame, resized_frame, landmarks_render_frame; 54 | cv::Mat render_frame; 55 | std::unique_ptr render_image; 56 | int curr_frame_pos = 0; 57 | int total_frames = 0; 58 | double fps = 0.0; 59 | 60 | // sfl 61 | std::shared_ptr sfl; 62 | std::vector sfl_frames; 63 | cv::Scalar landmarks_color = cv::Scalar(0, 255, 0); 64 | cv::Scalar bbox_color = cv::Scalar(0, 0, 255); 65 | 66 | int timer_id = 0; 67 | }; 68 | 69 | } // namespace sfl 70 | 71 | #endif // __SFL_VIEWER_H__ -------------------------------------------------------------------------------- /fvs_editor_v2/fvs_editor.qrc: -------------------------------------------------------------------------------- 1 | 2 | 3 | images/new.png 4 | images/open.png 5 | images/save.png 6 | images/save-as.png 7 | images/close.png 8 | images/exit.png 9 | images/contours.png 10 | images/borders.png 11 | images/segmentation.png 12 | images/postprocess.png 13 | images/postprocess_disconnected.png 14 | images/postprocess_holes.png 15 | images/postprocess_smooth.png 16 | images/iterations.png 17 | images/radius.png 18 | images/face.png 19 | images/backward.png 20 | images/forward.png 21 | images/pause.png 22 | images/play.png 23 | images/key.png 24 | images/hierarchy.png 25 | 26 | 27 | -------------------------------------------------------------------------------- /fvs_editor_v2/fvs_editor.rc: -------------------------------------------------------------------------------- 1 | IDI_ICON1 ICON DISCARDABLE "images/fvs_editor.ico" -------------------------------------------------------------------------------- /fvs_editor_v2/fvs_editor_main.cpp: -------------------------------------------------------------------------------- 1 | #include "fvs_editor.h" 2 | 3 | // std 4 | #include 5 | #include 6 | 7 | // Boost 8 | #include 9 | #include 10 | 11 | // Qt 12 | #include 13 | 14 | using std::cout; 15 | using std::endl; 16 | using std::cerr; 17 | using std::string; 18 | using std::runtime_error; 19 | using namespace boost::program_options; 20 | using namespace boost::filesystem; 21 | 22 | 23 | int main(int argc, char* argv[]) 24 | { 25 | /// Parse command line arguments 26 | string fvsPath, outputDir, segPath, landmarksPath, videoPath; 27 | unsigned int debug; 28 | try { 29 | options_description desc("Allowed options"); 30 | desc.add_options() 31 | ("help", "display the help message") 32 | ("input,i", value(&fvsPath)->required(), 33 | "path to face video segmentation (.fvs)") 34 | ("output,o", value(&outputDir), "output directory") 35 | ("segmentation,s", value(&segPath), "input segmentation protobuffer (.pb)") 36 | ("landmarks,l", value(&landmarksPath), "path to landmarks cache (.pb)") 37 | ("video,v", value(&videoPath), "path to video file") 38 | ("debug,d", value(&debug)->default_value(0), "output debug information") 39 | ; 40 | variables_map vm; 41 | store(command_line_parser(argc, argv).options(desc). 42 | positional(positional_options_description().add("input", -1)).run(), vm); 43 | if (vm.count("help")) { 44 | cout << "Usage: face_video_segment [options]" << endl; 45 | cout << desc << endl; 46 | exit(0); 47 | } 48 | notify(vm); 49 | if (!is_regular_file(fvsPath)) throw error("input must be a path to a file!"); 50 | if (vm.count("output") && !is_directory(outputDir)) 51 | throw error("output must be a path to a directory!"); 52 | } 53 | catch (const error& e) { 54 | cout << "Error while parsing command-line arguments: " << e.what() << endl; 55 | cout << "Use --help to display a list of options." << endl; 56 | exit(1); 57 | } 58 | 59 | try 60 | { 61 | QApplication a(argc, argv); 62 | fvs::Editor editor; 63 | editor.show(); 64 | 65 | return a.exec(); 66 | } 67 | catch (std::exception& e) 68 | { 69 | cerr << e.what() << endl; 70 | return 1; 71 | } 72 | 73 | return 0; 74 | } -------------------------------------------------------------------------------- /fvs_editor_v2/fvs_editor_states.cpp: -------------------------------------------------------------------------------- 1 | #include "fvs_editor_states.h" 2 | #include "fvs_editor.h" 3 | 4 | #include // 5 | 6 | // Boost 7 | #include 8 | 9 | // Qt 10 | #include 11 | #include 12 | 13 | using namespace boost::filesystem; 14 | 15 | namespace fvs 16 | { 17 | EditorSM::EditorSM(Editor * _editor) : editor(_editor) 18 | { 19 | } 20 | 21 | Inactive::Inactive(my_context ctx) : my_base(ctx), editor(nullptr) 22 | { 23 | editor = context().editor; 24 | } 25 | 26 | void Inactive::onUpdate(const EvUpdate & event) 27 | { 28 | } 29 | 30 | sc::result Inactive::react(const EvStart &) 31 | { 32 | return transit< Active >();// 33 | } 34 | 35 | Active::Active(my_context ctx) : my_base(ctx), editor(nullptr) 36 | { 37 | editor = context().editor; 38 | post_event(EvStart()); 39 | } 40 | 41 | void Active::onSeek(const EvSeek & event) 42 | { 43 | } 44 | 45 | void Active::onStart(const EvStart & event) 46 | { 47 | } 48 | 49 | Paused::Paused(my_context ctx) : my_base(ctx), editor(nullptr) 50 | { 51 | editor = context().editor; 52 | } 53 | 54 | void Paused::onUpdate(const EvUpdate& event) 55 | { 56 | } 57 | 58 | Playing::Playing(my_context ctx) : my_base(ctx), editor(nullptr) 59 | { 60 | editor = context().editor; 61 | } 62 | 63 | Playing::~Playing() 64 | { 65 | } 66 | 67 | void Playing::onUpdate(const EvUpdate& event) 68 | { 69 | } 70 | 71 | void Playing::onTimerTick(const EvTimerTick& event) 72 | { 73 | } 74 | } // namespace sfl 75 | 76 | -------------------------------------------------------------------------------- /fvs_editor_v2/fvs_editor_states.h: -------------------------------------------------------------------------------- 1 | #ifndef __FVS_EDITOR_STATES_H__ 2 | #define __FVS_EDITOR_STATES_H__ 3 | 4 | // Boost 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | // Qt 14 | #include 15 | 16 | 17 | namespace sc = boost::statechart; 18 | namespace mpl = boost::mpl; 19 | 20 | namespace fvs 21 | { 22 | // Forward declarations 23 | struct Inactive; 24 | struct Active; 25 | struct Paused; 26 | struct Playing; 27 | class Editor; 28 | 29 | // State Machine 30 | struct EditorSM : sc::state_machine< EditorSM, Inactive > 31 | { 32 | EditorSM(Editor* _editor); 33 | Editor* editor; 34 | }; 35 | 36 | // Events 37 | 38 | struct EvPlayPause : sc::event< EvPlayPause > {}; 39 | struct EvUpdate : sc::event< EvUpdate > {}; 40 | struct EvStart : sc::event< EvStart > {}; 41 | struct EvReset : sc::event< EvReset > {}; 42 | struct EvTimerTick : sc::event< EvTimerTick > {}; 43 | struct EvSeek : sc::event< EvSeek > 44 | { 45 | EvSeek(int _i) : i(_i) {} 46 | int i; 47 | }; 48 | 49 | // States 50 | 51 | struct Inactive : sc::state 52 | { 53 | Inactive(my_context ctx); 54 | 55 | void onUpdate(const EvUpdate& event); 56 | sc::result react(const EvStart&); 57 | 58 | typedef mpl::list< 59 | sc::in_state_reaction, 60 | sc::custom_reaction< EvStart > > reactions; 61 | 62 | Editor* editor; 63 | }; 64 | 65 | struct Active : sc::state 66 | { 67 | Active(my_context ctx); 68 | 69 | void onSeek(const EvSeek& event); 70 | void onStart(const EvStart& event); 71 | 72 | typedef mpl::list< 73 | sc::in_state_reaction, 74 | sc::in_state_reaction, 75 | sc::transition< EvReset, Inactive > > reactions; 76 | 77 | Editor* editor; 78 | }; 79 | 80 | struct Paused : sc::state 81 | { 82 | Paused(my_context ctx); 83 | 84 | void onUpdate(const EvUpdate& event); 85 | 86 | typedef mpl::list< 87 | sc::in_state_reaction, 88 | sc::transition< EvPlayPause, Playing > > reactions; 89 | 90 | Editor* editor; 91 | }; 92 | 93 | // States 94 | struct Playing : sc::state 95 | { 96 | Playing(my_context ctx); 97 | ~Playing(); 98 | 99 | void onUpdate(const EvUpdate& event); 100 | void onTimerTick(const EvTimerTick& event); 101 | 102 | typedef mpl::list< 103 | sc::in_state_reaction, 104 | sc::in_state_reaction, 105 | sc::transition< EvPlayPause, Paused > > reactions; 106 | 107 | Editor* editor; 108 | int timer_id = 0; 109 | }; 110 | } // namespace fvs 111 | 112 | 113 | 114 | 115 | 116 | #endif // __FVS_EDITOR_STATES_H__ -------------------------------------------------------------------------------- /fvs_editor_v2/images/backward.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/backward.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/borders.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/borders.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/close.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/close.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/contours.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/contours.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/exit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/exit.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/face.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/face.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/forward.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/forward.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/fvs_editor.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/fvs_editor.ico -------------------------------------------------------------------------------- /fvs_editor_v2/images/hierarchy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/hierarchy.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/iterations.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/iterations.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/key.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/key.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/new.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/new.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/open.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/open.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/pause.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/pause.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/play.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/play.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/postprocess.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/postprocess.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/postprocess_disconnected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/postprocess_disconnected.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/postprocess_holes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/postprocess_holes.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/postprocess_smooth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/postprocess_smooth.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/radius.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/radius.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/save-as.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/save-as.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/save.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/save.png -------------------------------------------------------------------------------- /fvs_editor_v2/images/segmentation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YuvalNirkin/face_video_segment/05e455756bea685b2c14f99d36acb899de1f8f8e/fvs_editor_v2/images/segmentation.png -------------------------------------------------------------------------------- /fvs_find_regions/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(fvs_find_regions fvs_find_regions.cpp) 2 | target_include_directories(fvs_find_regions PRIVATE 3 | ${CMAKE_SOURCE_DIR}/face_video_segment 4 | ${CMAKE_BINARY_DIR}/face_video_segment) 5 | target_link_libraries(fvs_find_regions PRIVATE face_video_segment) 6 | 7 | # Installation 8 | install(TARGETS fvs_find_regions 9 | EXPORT face_video_segment-targets 10 | RUNTIME DESTINATION bin COMPONENT dev 11 | LIBRARY DESTINATION lib COMPONENT dev 12 | ARCHIVE DESTINATION lib COMPONENT dev) 13 | -------------------------------------------------------------------------------- /fvs_find_regions/fvs_find_regions.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #include "landmarks_unit.h" 30 | #include "face_segmentation_unit.h" 31 | #include "video_reader_unit2.h" 32 | #include "video_writer_unit2.h" 33 | #include "keyframe_unit.h" 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | 45 | using std::string; 46 | using std::cout; 47 | using std::endl; 48 | using std::cerr; 49 | using namespace boost::program_options; 50 | using namespace boost::filesystem; 51 | using namespace fvs; 52 | 53 | int main(int argc, char* argv[]) 54 | { 55 | // Parse command line arguments 56 | string videoPath, outputDir, segPath, landmarksPath; 57 | unsigned int verbose; 58 | try { 59 | options_description desc("Allowed options"); 60 | desc.add_options() 61 | ("help", "display the help message") 62 | ("input,i", value(&videoPath)->required(), "path to video file") 63 | ("output,o", value(&outputDir)->required(), "output directory") 64 | ("segmentation,s", value(&segPath), "input segmentation protobuffer (.pb)") 65 | ("landmarks,l", value(&landmarksPath), "path to landmarks cache (.lms)") 66 | ("verbose,v", value(&verbose)->default_value(0), "output debug information") 67 | ; 68 | variables_map vm; 69 | store(command_line_parser(argc, argv).options(desc). 70 | positional(positional_options_description().add("input", -1)).run(), vm); 71 | if (vm.count("help")) { 72 | cout << "Usage: fvs_find_regions [options]" << endl; 73 | cout << desc << endl; 74 | exit(0); 75 | } 76 | notify(vm); 77 | 78 | if (!is_regular_file(videoPath)) throw error("input must be a path to a video file!"); 79 | if (vm.count("output") && !is_directory(outputDir)) 80 | throw error("output must be a path to a directory!"); 81 | if (!is_regular_file(segPath)) throw error("segmentation must be a path to a file!"); 82 | if (!is_regular_file(landmarksPath)) 83 | { 84 | path input = path(videoPath); 85 | landmarksPath = 86 | (input.parent_path() / (input.stem() += ".lms")).string(); 87 | if (!is_regular_file(landmarksPath)) 88 | throw error("Couldn't find landmarks model or cache file!"); 89 | } 90 | } 91 | catch (const error& e) { 92 | cout << "Error while parsing command-line arguments: " << e.what() << endl; 93 | cout << "Use --help to display a list of options." << endl; 94 | exit(1); 95 | } 96 | 97 | try 98 | { 99 | // Video Reader Unit 100 | //VideoReaderUnit reader(VideoReaderOptions(), videoPath); 101 | VideoReaderUnit2 reader(VideoReader2Options(), videoPath); 102 | 103 | // Segmentation Reader Unit 104 | SegmentationReaderUnitOptions segOptions; 105 | segOptions.filename = segPath; 106 | SegmentationReaderUnit seg_reader(segOptions); 107 | seg_reader.AttachTo(&reader); 108 | 109 | // Landmarks Unit 110 | LandmarksOptions landmarks_options; 111 | landmarks_options.landmarks_path = landmarksPath; 112 | LandmarksUnit landmarks_unit(landmarks_options); 113 | landmarks_unit.AttachTo(&seg_reader); 114 | 115 | // Face Regions Unit 116 | FaceRegionsOptions face_regions_options; 117 | face_regions_options.video_path = videoPath; 118 | face_regions_options.seg_path = segPath; 119 | face_regions_options.landmarks_path = landmarksPath; 120 | FaceRegionsUnit face_regions_unit(face_regions_options); 121 | face_regions_unit.AttachTo(&landmarks_unit); 122 | 123 | // Keyframe Detection Unit 124 | KeyframeDetectionOptions keyframe_detection_options; 125 | KeyframeDetectionUnit keyframe_detection_unit(keyframe_detection_options); 126 | keyframe_detection_unit.AttachTo(&face_regions_unit); 127 | 128 | // Prepare processing 129 | if (!reader.PrepareProcessing()) 130 | throw std::runtime_error("Video framework setup failed."); 131 | 132 | // This call will block and return when the whole has been displayed. 133 | if (!reader.Run()) 134 | throw std::runtime_error("Could not process video file."); 135 | 136 | // Write output to file 137 | boost::filesystem::path orig = videoPath; 138 | std::string fvs_out_path = (boost::filesystem::path(outputDir) / 139 | (orig.stem() += ".fvs")).string(); 140 | std::cout << "Writing face video segmentation to \"" << 141 | fvs_out_path << "\"." << std::endl; 142 | face_regions_unit.save(fvs_out_path); 143 | } 144 | catch (std::exception& e) 145 | { 146 | cerr << e.what() << endl; 147 | return 1; 148 | } 149 | 150 | return 0; 151 | } -------------------------------------------------------------------------------- /fvs_segment/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Target 2 | add_executable(fvs_segment fvs_segment.cpp) 3 | target_link_libraries(fvs_segment PRIVATE face_video_segment) 4 | target_compile_definitions(fvs_segment PRIVATE 5 | -DWITH_QT 6 | -DWITH_OPENCV_VIDEOIO 7 | ) 8 | 9 | # Installation 10 | install(TARGETS fvs_segment 11 | EXPORT face_video_segment-targets 12 | RUNTIME DESTINATION bin COMPONENT dev 13 | LIBRARY DESTINATION lib COMPONENT dev 14 | ARCHIVE DESTINATION lib COMPONENT dev) 15 | -------------------------------------------------------------------------------- /fvs_write_keyframes/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(fvs_write_keyframes fvs_write_keyframes.cpp) 2 | #target_include_directories(fvs_write_keyframes PRIVATE 3 | # ${CMAKE_SOURCE_DIR}/face_video_segment 4 | # ${CMAKE_BINARY_DIR}/face_video_segment) 5 | target_link_libraries(fvs_write_keyframes PRIVATE face_video_segment) 6 | 7 | # Installation 8 | install(TARGETS fvs_write_keyframes 9 | EXPORT face_video_segment-targets 10 | RUNTIME DESTINATION bin COMPONENT dev 11 | LIBRARY DESTINATION lib COMPONENT dev 12 | ARCHIVE DESTINATION lib COMPONENT dev) 13 | -------------------------------------------------------------------------------- /fvs_write_keyframes/fvs_write_keyframes.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2010-2014, The Video Segmentation Project 2 | // All rights reserved. 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // * Neither the name of the The Video Segmentation Project nor the 12 | // names of its contributors may be used to endorse or promote products 13 | // derived from this software without specific prior written permission. 14 | 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | // 27 | // --- 28 | 29 | #include "landmarks_unit.h" 30 | #include "face_segmentation_unit.h" 31 | #include "video_reader_unit2.h" 32 | #include "video_writer_unit2.h" 33 | #include "keyframe_unit.h" 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | 45 | using std::string; 46 | using std::cout; 47 | using std::endl; 48 | using std::cerr; 49 | using namespace boost::program_options; 50 | using namespace boost::filesystem; 51 | using namespace fvs; 52 | 53 | int main(int argc, char* argv[]) 54 | { 55 | // Parse command line arguments 56 | string fvsPath, outputDir, segPath, landmarksPath, videoPath; 57 | unsigned int max_scale, debug; 58 | bool upscale; 59 | try { 60 | options_description desc("Allowed options"); 61 | desc.add_options() 62 | ("help", "display the help message") 63 | ("input,i", value(&fvsPath)->required(), "path to face video segmentation (.fvs)") 64 | ("output,o", value(&outputDir)->required(), "output directory") 65 | ("segmentation,s", value(&segPath), "input segmentation protobuffer (.pb)") 66 | ("landmarks,l", value(&landmarksPath), "path to landmarks cache (.lms)") 67 | ("video,v", value(&videoPath), "path to video file") 68 | ("max_scale,m", value(&max_scale)->default_value(500), 69 | "max keyframe scale [pixels]") 70 | ("upscale,u", value(&upscale)->default_value(false)->implicit_value(true), 71 | "upscale to max scale") 72 | ("debug,d", value(&debug)->default_value(0), "output debug information") 73 | ; 74 | variables_map vm; 75 | store(command_line_parser(argc, argv).options(desc). 76 | positional(positional_options_description().add("input", -1)).run(), vm); 77 | if (vm.count("help")) { 78 | cout << "Usage: fvs_write_keyframes [options]" << endl; 79 | cout << desc << endl; 80 | exit(0); 81 | } 82 | notify(vm); 83 | 84 | // Read fvs file to extract the stored paths 85 | fvs::Sequence sequence; 86 | std::ifstream input(fvsPath, std::ifstream::binary); 87 | sequence.ParseFromIstream(&input); 88 | 89 | path inputPath = path(fvsPath); 90 | if (!is_regular_file(fvsPath)) 91 | throw error("input must be a path to a face video segmentation file!"); 92 | if (outputDir.empty() || !is_directory(outputDir)) 93 | throw error("output must be a path to a directory!"); 94 | if (videoPath.empty()) videoPath = sequence.video_path(); 95 | if (!is_regular_file(videoPath)) 96 | throw error("Couldn't find video file!"); 97 | if (segPath.empty()) segPath = sequence.seg_path(); 98 | if (!is_regular_file(segPath)) 99 | { 100 | segPath = (inputPath.parent_path() / (inputPath.filename() += ".pb")).string(); 101 | if (!is_regular_file(segPath)) 102 | throw error("Couldn't find segmentation file!"); 103 | } 104 | if (landmarksPath.empty()) landmarksPath = sequence.landmarks_path(); 105 | if (!is_regular_file(landmarksPath)) 106 | { 107 | landmarksPath = 108 | (inputPath.parent_path() / (inputPath.stem() += ".lms")).string(); 109 | if (!is_regular_file(landmarksPath)) 110 | throw error("Couldn't find landmarks cache file!"); 111 | } 112 | } 113 | catch (const error& e) { 114 | cout << "Error while parsing command-line arguments: " << e.what() << endl; 115 | cout << "Use --help to display a list of options." << endl; 116 | exit(1); 117 | } 118 | 119 | try 120 | { 121 | // Video Reader Unit 122 | //VideoReaderUnit reader(VideoReaderOptions(), videoPath); 123 | VideoReaderUnit2 reader(VideoReader2Options(), videoPath); 124 | 125 | // Segmentation Reader Unit 126 | SegmentationReaderUnitOptions segOptions; 127 | segOptions.filename = segPath; 128 | SegmentationReaderUnit seg_reader(segOptions); 129 | seg_reader.AttachTo(&reader); 130 | 131 | // Landmarks Unit 132 | LandmarksOptions landmarks_options; 133 | landmarks_options.landmarks_path = landmarksPath; 134 | LandmarksUnit landmarks_unit(landmarks_options); 135 | landmarks_unit.AttachTo(&seg_reader); 136 | 137 | // Face Regions Unit 138 | FaceRegionsReaderOptions face_regions_options; 139 | FaceRegionsReaderUnit face_regions_unit(face_regions_options, fvsPath); 140 | face_regions_unit.AttachTo(&landmarks_unit); 141 | /* 142 | FaceRegionsOptions face_regions_options; 143 | FaceRegionsUnit face_regions_unit(face_regions_options); 144 | face_regions_unit.AttachTo(&landmarks_unit); 145 | 146 | // Keyframe Detection Unit 147 | KeyframeDetectionOptions keyframe_detection_options; 148 | KeyframeDetectionUnit keyframe_detection_unit(keyframe_detection_options); 149 | keyframe_detection_unit.AttachTo(&face_regions_unit); 150 | */ 151 | 152 | // Keyframe Writer Unit 153 | KeyframeWriterOptions keyframe_writer_options; 154 | keyframe_writer_options.max_scale = max_scale; 155 | keyframe_writer_options.upscale = upscale; 156 | keyframe_writer_options.debug = debug > 0; 157 | KeyframeWriterUnit keyframe_writer_unit(keyframe_writer_options, outputDir, 158 | path(videoPath).stem().string()); 159 | keyframe_writer_unit.AttachTo(&face_regions_unit); 160 | 161 | // Prepare processing 162 | if (!reader.PrepareProcessing()) 163 | throw std::runtime_error("Video framework setup failed."); 164 | 165 | // This call will block and return when the whole has been displayed. 166 | if (!reader.Run()) 167 | throw std::runtime_error("Could not process video file."); 168 | } 169 | catch (std::exception& e) 170 | { 171 | cerr << e.what() << endl; 172 | return 1; 173 | } 174 | 175 | return 0; 176 | } -------------------------------------------------------------------------------- /interfaces/matlab/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Installation 2 | file(GLOB MATLAB_SCRIPTS "*.m") 3 | install(FILES ${MATLAB_SCRIPTS} 4 | DESTINATION interfaces/matlab) 5 | #[[ 6 | install(DIRECTORY ${CMAKE_SOURCE_DIR}/interfaces/matlab/utilities 7 | DESTINATION interfaces/matlab 8 | FILES_MATCHING PATTERN "*.m") 9 | ]] -------------------------------------------------------------------------------- /interfaces/matlab/add_pascal_voc_db.m: -------------------------------------------------------------------------------- 1 | function add_pascal_voc_db(varargin) 2 | %ADD_PASCAL_VOC_DB Summary of this function goes here 3 | % Detailed explanation goes here 4 | 5 | %% Parse input arguments 6 | p = inputParser; 7 | addRequired(p, 'imgDir', @ischar); 8 | addRequired(p, 'segDir', @ischar); 9 | addRequired(p, 'trainvalDir', @ischar); 10 | addRequired(p, 'dbDir', @ischar); 11 | parse(p,varargin{:}); 12 | 13 | %% Calculate paths 14 | inTrainPath = fullfile(p.Results.trainvalDir, 'train.txt'); 15 | inValPath = fullfile(p.Results.trainvalDir, 'val.txt'); 16 | outImgPath = fullfile(p.Results.dbDir, 'JPEGImages'); 17 | outSegPath = fullfile(p.Results.dbDir, 'SegmentationClass'); 18 | outTrainPath = fullfile(p.Results.dbDir, 'ImageSets', 'Segmentation', 'train.txt'); 19 | outValPath = fullfile(p.Results.dbDir, 'ImageSets', 'Segmentation', 'val.txt'); 20 | outMainTrainPath = fullfile(p.Results.dbDir, 'ImageSets', 'Main', 'train.txt'); 21 | outMainValPath = fullfile(p.Results.dbDir, 'ImageSets', 'Main', 'val.txt'); 22 | outPersonTrainPath = fullfile(p.Results.dbDir, 'ImageSets', 'Main', 'person_train.txt'); 23 | outPersonValPath = fullfile(p.Results.dbDir, 'ImageSets', 'Main', 'person_val.txt'); 24 | 25 | %% Modify trainval files 26 | train = readTextFile(inTrainPath); 27 | val = readTextFile(inValPath); 28 | writeTextFile(outTrainPath, train); 29 | writeTextFile(outValPath, val); 30 | copyfile(outTrainPath, outMainTrainPath, 'f'); 31 | copyfile(outValPath, outMainValPath, 'f'); 32 | copyfile(outTrainPath, outPersonTrainPath, 'f'); 33 | copyfile(outValPath, outPersonValPath, 'f'); 34 | 35 | %% Parse image directory 36 | filt = '.*(png|jpg)'; 37 | imgDescs = dir(p.Results.imgDir); 38 | %imgNames = {imgDescs.name}; 39 | imgNames = {imgDescs(~cellfun(@isempty,regexpi({imgDescs.name},filt))).name}; 40 | 41 | %% Parse segmentation directory 42 | filt = '.*(png|jpg)'; 43 | segDescs = dir(p.Results.segDir); 44 | %segNames = {segDescs.name}; 45 | segNames = {segDescs(~cellfun(@isempty,regexpi({segDescs.name},filt))).name}; 46 | 47 | %% Copy input images 48 | for i = 1:numel(imgNames) 49 | inFilePath = fullfile(p.Results.imgDir, imgNames{i}); 50 | outFilePath = fullfile(outImgPath, imgNames{i}); 51 | copyfile(inFilePath, outFilePath); 52 | end 53 | 54 | %% Copy segmentation images 55 | for i = 1:numel(segNames) 56 | inFilePath = fullfile(p.Results.segDir, segNames{i}); 57 | outFilePath = fullfile(outSegPath, segNames{i}); 58 | copyfile(inFilePath, outFilePath); 59 | end 60 | 61 | end 62 | 63 | function names = readTextFile(textFile) 64 | fileID = fopen(textFile); 65 | names = textscan(fileID,'%s'); 66 | names = names{1}; 67 | fclose(fileID); 68 | end 69 | 70 | function writeTextFile(textFile, set) 71 | fid = fopen(textFile,'a'); 72 | for i = 1:length(set) 73 | fprintf(fid, '%s\n', set{i}); 74 | end 75 | fclose(fid); 76 | end -------------------------------------------------------------------------------- /interfaces/matlab/face_fcn_test.m: -------------------------------------------------------------------------------- 1 | function info = face_fcn_test(varargin) 2 | 3 | %% Initialize options 4 | matconvnetPath = 'D:\Dev\Matlab\Shared\matconvnet'; 5 | run(fullfile(matconvnetPath, 'matlab/vl_setupnn')); 6 | addpath(fullfile(matconvnetPath, 'examples')); 7 | addpath 'D:\Dev\Matlab\Shared\matconvnet-fcn' 8 | 9 | % experiment and data paths 10 | opts.expDir = 'D:\Dev\Matlab\Shared\matconvnet-fcn\data\face_seg\fcn32s' ; 11 | %opts.dataDir = 'D:\Dev\Matlab\Shared\matconvnet-fcn\data\voc11'; 12 | opts.dataDir = 'D:\Dev\Matlab\Shared\matconvnet-fcn\data\face_seg\face_seg_db'; 13 | opts.modelPath = 'D:\Dev\Matlab\Shared\matconvnet-fcn\data\face_seg\fcn32s\net-epoch-20.mat' 14 | opts.modelFamily = 'matconvnet' ; 15 | [opts, varargin] = vl_argparse(opts, varargin) ; 16 | 17 | % experiment setup 18 | opts.imdbPath = fullfile(opts.expDir, 'imdb.mat') ; 19 | opts.vocEdition = '11' ; 20 | opts.vocAdditionalSegmentations = false ; 21 | opts.vocAdditionalSegmentationsMergeMode = 2 ; 22 | %opts.gpus = [] ; 23 | opts.gpus = 1;% 24 | opts = vl_argparse(opts, varargin) ; 25 | 26 | resPath = fullfile(opts.expDir, 'results.mat') ; 27 | if exist(resPath) 28 | info = load(resPath) ; 29 | return ; 30 | end 31 | 32 | if ~isempty(opts.gpus) 33 | gpuDevice(opts.gpus(1)) 34 | end 35 | 36 | % ------------------------------------------------------------------------- 37 | % Setup data 38 | % ------------------------------------------------------------------------- 39 | 40 | % Get PASCAL VOC 11/12 segmentation dataset plus Berkeley's additional 41 | % segmentations 42 | if exist(opts.imdbPath) 43 | imdb = load(opts.imdbPath) ; 44 | else 45 | imdb = face_pascal_voc_setup('dataDir', opts.dataDir, ... 46 | 'edition', opts.vocEdition, ... 47 | 'includeTest', false, ... 48 | 'includeSegmentation', true, ... 49 | 'includeDetection', false) ; 50 | if opts.vocAdditionalSegmentations 51 | imdb = vocSetupAdditionalSegmentations(... 52 | imdb, ... 53 | 'dataDir', opts.dataDir, ... 54 | 'mergeMode', opts.vocAdditionalSegmentationsMergeMode) ; 55 | end 56 | mkdir(opts.expDir) ; 57 | save(opts.imdbPath, '-struct', 'imdb') ; 58 | end 59 | 60 | % Get validation subset 61 | val = find(imdb.images.set == 2 & imdb.images.segmentation) ; 62 | 63 | % Compare the validation set to the one used in the FCN paper 64 | % valNames = sort(imdb.images.name(val)') ; 65 | % valNames = textread('data/seg11valid.txt', '%s') ; 66 | % valNames_ = textread('data/seg12valid-tvg.txt', '%s') ; 67 | % assert(isequal(valNames, valNames_)) ; 68 | 69 | %imdb.paths.image = strrep(imdb.paths.image,'\','/');% 70 | %imdb.paths.classSegmentation = strrep(imdb.paths.classSegmentation,'\','/');% 71 | 72 | % ------------------------------------------------------------------------- 73 | % Setup model 74 | % ------------------------------------------------------------------------- 75 | 76 | switch opts.modelFamily 77 | case 'matconvnet' 78 | net = load(opts.modelPath) ; 79 | net = dagnn.DagNN.loadobj(net.net) ; 80 | net.mode = 'test' ; 81 | for name = {'objective', 'accuracy'} 82 | net.removeLayer(name) ; 83 | end 84 | net.meta.normalization.averageImage = reshape(net.meta.normalization.rgbMean,1,1,3) ; 85 | predVar = net.getVarIndex('prediction') ; 86 | inputVar = 'input' ; 87 | imageNeedsToBeMultiple = true ; 88 | 89 | case 'ModelZoo' 90 | net = dagnn.DagNN.loadobj(load(opts.modelPath)) ; 91 | net.mode = 'test' ; 92 | predVar = net.getVarIndex('upscore') ; 93 | inputVar = 'data' ; 94 | imageNeedsToBeMultiple = false ; 95 | 96 | case 'TVG' 97 | net = dagnn.DagNN.loadobj(load(opts.modelPath)) ; 98 | net.mode = 'test' ; 99 | predVar = net.getVarIndex('coarse') ; 100 | inputVar = 'data' ; 101 | imageNeedsToBeMultiple = false ; 102 | end 103 | 104 | if ~isempty(opts.gpus) 105 | gpuDevice(opts.gpus(1)) ; 106 | net.move('gpu') ; 107 | end 108 | net.mode = 'test' ; 109 | 110 | % ------------------------------------------------------------------------- 111 | % Train 112 | % ------------------------------------------------------------------------- 113 | 114 | numGpus = 0 ; 115 | confusion = zeros(21) ; 116 | 117 | for i = 1:numel(val) 118 | imId = val(i) ; 119 | name = imdb.images.name{imId} ; 120 | rgbPath = sprintf(imdb.paths.image, name) ; 121 | labelsPath = sprintf(imdb.paths.classSegmentation, name) ; 122 | 123 | % Load an image and gt segmentation 124 | rgb = vl_imreadjpeg({rgbPath}) ; 125 | rgb = rgb{1} ; 126 | anno = imread(labelsPath) ; 127 | lb = single(anno) ; 128 | lb = mod(lb + 1, 256) ; % 0 = ignore, 1 = bkg 129 | 130 | % Subtract the mean (color) 131 | im = bsxfun(@minus, single(rgb), net.meta.normalization.averageImage) ; 132 | 133 | % Soome networks requires the image to be a multiple of 32 pixels 134 | if imageNeedsToBeMultiple 135 | sz = [size(im,1), size(im,2)] ; 136 | sz_ = round(sz / 32)*32 ; 137 | im_ = imresize(im, sz_) ; 138 | else 139 | im_ = im ; 140 | end 141 | 142 | if ~isempty(opts.gpus) 143 | im_ = gpuArray(im_) ; 144 | end 145 | 146 | net.eval({inputVar, im_}) ; 147 | scores_ = gather(net.vars(predVar).value) ; 148 | [~,pred_] = max(scores_,[],3) ; 149 | 150 | if imageNeedsToBeMultiple 151 | pred = imresize(pred_, sz, 'method', 'nearest') ; 152 | else 153 | pred = pred_ ; 154 | end 155 | 156 | % Accumulate errors 157 | ok = lb > 0 ; 158 | confusion = confusion + accumarray([lb(ok),pred(ok)],1,[21 21]) ; 159 | 160 | % Plots 161 | % if mod(i - 1,30) == 0 || i == numel(val) 162 | if mod(i - 1,1) == 0 || i == numel(val) 163 | clear info ; 164 | [info.iu, info.miu, info.pacc, info.macc] = getAccuracies(confusion) ; 165 | fprintf('IU ') ; 166 | fprintf('%4.1f ', 100 * info.iu) ; 167 | fprintf('\n meanIU: %5.2f pixelAcc: %5.2f, meanAcc: %5.2f\n', ... 168 | 100*info.miu, 100*info.pacc, 100*info.macc) ; 169 | 170 | figure(1) ; clf; 171 | imagesc(normalizeConfusion(confusion)) ; 172 | axis image ; set(gca,'ydir','normal') ; 173 | colormap(jet) ; 174 | drawnow ; 175 | 176 | % Print segmentation 177 | figure(100) ;clf ; 178 | displayImage(rgb/255, lb, pred) ; 179 | drawnow ; 180 | 181 | % Save segmentation 182 | imPath = fullfile(opts.expDir, [name '.png']) ; 183 | imwrite(pred,labelColors(),imPath,'png'); 184 | end 185 | end 186 | 187 | % Save results 188 | save(resPath, '-struct', 'info') ; 189 | 190 | % ------------------------------------------------------------------------- 191 | function nconfusion = normalizeConfusion(confusion) 192 | % ------------------------------------------------------------------------- 193 | % normalize confusion by row (each row contains a gt label) 194 | nconfusion = bsxfun(@rdivide, double(confusion), double(sum(confusion,2))) ; 195 | 196 | % ------------------------------------------------------------------------- 197 | function [IU, meanIU, pixelAccuracy, meanAccuracy] = getAccuracies(confusion) 198 | % ------------------------------------------------------------------------- 199 | pos = sum(confusion,2) ; 200 | res = sum(confusion,1)' ; 201 | tp = diag(confusion) ; 202 | IU = tp ./ max(1, pos + res - tp) ; 203 | meanIU = mean(IU) ; 204 | pixelAccuracy = sum(tp) / max(1,sum(confusion(:))) ; 205 | meanAccuracy = mean(tp ./ max(1, pos)) ; 206 | 207 | % ------------------------------------------------------------------------- 208 | function displayImage(im, lb, pred) 209 | % ------------------------------------------------------------------------- 210 | subplot(2,2,1) ; 211 | image(im) ; 212 | axis image ; 213 | title('source image') ; 214 | 215 | subplot(2,2,2) ; 216 | image(uint8(lb-1)) ; 217 | axis image ; 218 | title('ground truth') 219 | 220 | cmap = labelColors() ; 221 | subplot(2,2,3) ; 222 | image(uint8(pred-1)) ; 223 | axis image ; 224 | title('predicted') ; 225 | 226 | colormap(cmap) ; 227 | 228 | % ------------------------------------------------------------------------- 229 | function cmap = labelColors() 230 | % ------------------------------------------------------------------------- 231 | N=21; 232 | cmap = zeros(N,3); 233 | for i=1:N 234 | id = i-1; r=0;g=0;b=0; 235 | for j=0:7 236 | r = bitor(r, bitshift(bitget(id,1),7 - j)); 237 | g = bitor(g, bitshift(bitget(id,2),7 - j)); 238 | b = bitor(b, bitshift(bitget(id,3),7 - j)); 239 | id = bitshift(id,-3); 240 | end 241 | cmap(i,1)=r; cmap(i,2)=g; cmap(i,3)=b; 242 | end 243 | cmap = cmap / 255; 244 | -------------------------------------------------------------------------------- /interfaces/matlab/face_fcn_train.m: -------------------------------------------------------------------------------- 1 | function face_fcn_train(varargin) 2 | %FACE_FCN_TRAIN Train FCN model using MatConvNet 3 | % Detailed explanation goes here 4 | 5 | %% Initialize options 6 | matconvnetPath = 'D:\Dev\Matlab\Shared\matconvnet'; 7 | run(fullfile(matconvnetPath, 'matlab/vl_setupnn')); 8 | addpath(fullfile(matconvnetPath, 'examples')); 9 | addpath 'D:\Dev\Matlab\Shared\matconvnet-fcn' 10 | 11 | % experiment and data paths 12 | opts.expDir = 'D:\Dev\Matlab\Shared\matconvnet-fcn\data\face_seg\fcn32s' ; 13 | opts.dataDir = 'D:\Dev\Matlab\Shared\matconvnet-fcn\data\face_seg\face_seg_db' ; 14 | opts.modelType = 'fcn32s' ; 15 | opts.sourceModelPath = 'D:\Dev\Matlab\Shared\matconvnet-fcn\data\models\imagenet-vgg-verydeep-16.mat' ; 16 | [opts, varargin] = vl_argparse(opts, varargin); 17 | 18 | % experiment setup 19 | opts.imdbPath = fullfile(opts.expDir, 'imdb.mat') ; 20 | opts.imdbStatsPath = fullfile(opts.expDir, 'imdbStats.mat') ; 21 | opts.vocEdition = '11' ; 22 | opts.vocAdditionalSegmentations = false ; 23 | opts.numFetchThreads = 1 ; % not used yet 24 | 25 | % training options (SGD) 26 | %opts.train = struct([]) ; 27 | opts.train.gpus = 1;% 28 | [opts, varargin] = vl_argparse(opts, varargin) ; 29 | 30 | trainOpts.batchSize = 20 ; 31 | trainOpts.numSubBatches = 10 ; 32 | trainOpts.continue = true ; 33 | trainOpts.gpus = [] ; 34 | trainOpts.prefetch = true ; 35 | trainOpts.expDir = opts.expDir ; 36 | trainOpts.learningRate = 0.0001 * ones(1,50) ; 37 | trainOpts.numEpochs = numel(trainOpts.learningRate) ; 38 | 39 | %% Setup data 40 | 41 | % Get PASCAL VOC 12 segmentation dataset plus Berkeley's additional 42 | % segmentations 43 | if exist(opts.imdbPath) 44 | imdb = load(opts.imdbPath) ; 45 | else 46 | imdb = face_pascal_voc_setup('dataDir', opts.dataDir, ... 47 | 'edition', opts.vocEdition, ... 48 | 'includeTest', false, ... 49 | 'includeSegmentation', true, ... 50 | 'includeDetection', false) ; 51 | if opts.vocAdditionalSegmentations 52 | imdb = vocSetupAdditionalSegmentations(imdb, 'dataDir', opts.dataDir) ; 53 | end 54 | mkdir(opts.expDir) ; 55 | save(opts.imdbPath, '-struct', 'imdb') ; 56 | end 57 | %imdb.paths.objectSegmentation = strrep(imdb.paths.objectSegmentation,'\','/');% 58 | %imdb.paths.classSegmentation = strrep(imdb.paths.classSegmentation,'\','/');% 59 | 60 | % Get training and test/validation subsets 61 | train = find(imdb.images.set == 1 & imdb.images.segmentation) ; 62 | val = find(imdb.images.set == 2 & imdb.images.segmentation) ; 63 | 64 | % Get dataset statistics 65 | if exist(opts.imdbStatsPath) 66 | stats = load(opts.imdbStatsPath) ; 67 | else 68 | stats = getDatasetStatistics(imdb) ; 69 | save(opts.imdbStatsPath, '-struct', 'stats') ; 70 | end 71 | 72 | %% Setup model 73 | % Get initial model from VGG-VD-16 74 | net = fcnInitializeModel('sourceModelPath', opts.sourceModelPath) ; 75 | if any(strcmp(opts.modelType, {'fcn16s', 'fcn8s'})) 76 | % upgrade model to FCN16s 77 | net = fcnInitializeModel16s(net) ; 78 | end 79 | if strcmp(opts.modelType, 'fcn8s') 80 | % upgrade model fto FCN8s 81 | net = fcnInitializeModel8s(net) ; 82 | end 83 | net.meta.normalization.rgbMean = stats.rgbMean ; 84 | net.meta.classes = imdb.classes.name ; 85 | 86 | %% Train 87 | % Setup data fetching options 88 | bopts.numThreads = opts.numFetchThreads ; 89 | bopts.labelStride = 1 ; 90 | bopts.labelOffset = 1 ; 91 | bopts.classWeights = ones(1,21,'single') ; 92 | bopts.rgbMean = stats.rgbMean ; 93 | bopts.useGpu = numel(opts.train.gpus) > 0 ; 94 | 95 | % Launch SGD 96 | info = cnn_train_dag(net, imdb, getBatchWrapper(bopts), ... 97 | trainOpts, .... 98 | 'train', train, ... 99 | 'val', val, ... 100 | opts.train) ; 101 | 102 | end 103 | 104 | function fn = getBatchWrapper(opts) 105 | fn = @(imdb,batch) getBatch(imdb,batch,opts,'prefetch',nargout==0); 106 | end 107 | 108 | -------------------------------------------------------------------------------- /interfaces/matlab/face_pascal_voc_setup.m: -------------------------------------------------------------------------------- 1 | function imdb = face_pascal_voc_setup(varargin) 2 | %FACE_PASCAL_VOC_SETUP Summary of this function goes here 3 | % Detailed explanation goes here 4 | opts.edition = '07' ; 5 | opts.dataDir = fullfile('data','voc07') ; 6 | opts.archiveDir = fullfile('data','archives') ; 7 | opts.includeDetection = false ; 8 | opts.includeSegmentation = false ; 9 | opts.includeTest = false ; 10 | opts = vl_argparse(opts, varargin) ; 11 | 12 | %% Source images and classes 13 | imdb.paths.image = esc(fullfile(opts.dataDir, 'JPEGImages', '%s.jpg')) ; 14 | imdb.sets.id = uint8([1 2 3]) ; 15 | imdb.sets.name = {'train', 'val', 'test'} ; 16 | imdb.classes.id = uint8(1:20) ; 17 | imdb.classes.name = {... 18 | 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', ... 19 | 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', ... 20 | 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor'} ; 21 | imdb.classes.images = cell(1,20) ; 22 | imdb.images.id = [] ; 23 | imdb.images.name = {} ; 24 | imdb.images.set = [] ; 25 | index = containers.Map() ; 26 | [imdb, index] = addImageSet(opts, imdb, index, 'train', 1) ; 27 | [imdb, index] = addImageSet(opts, imdb, index, 'val', 2) ; 28 | if opts.includeTest, [imdb, index] = addImageSet(opts, imdb, index, 'test', 3) ; end 29 | 30 | %% Source segmentations 31 | if opts.includeSegmentation 32 | n = numel(imdb.images.id) ; 33 | imdb.paths.objectSegmentation = esc(fullfile(opts.dataDir, 'SegmentationObject', '%s.png')) ; 34 | imdb.paths.classSegmentation = esc(fullfile(opts.dataDir, 'SegmentationClass', '%s.png')) ; 35 | imdb.images.segmentation = false(1, n) ; 36 | [imdb, index] = addSegmentationSet(opts, imdb, index, 'train', 1) ; 37 | [imdb, index] = addSegmentationSet(opts, imdb, index, 'val', 2) ; 38 | if opts.includeTest, [imdb, index] = addSegmentationSet(opts, imdb, index, 'test', 3) ; end 39 | end 40 | 41 | %% Compress data types 42 | imdb.images.id = uint32(imdb.images.id) ; 43 | imdb.images.set = uint8(imdb.images.set) ; 44 | for i=1:20 45 | imdb.classes.images{i} = uint32(imdb.classes.images{i}) ; 46 | end 47 | 48 | %% Check images on disk and get their size 49 | imdb = getImageSizes(imdb) ; 50 | 51 | end 52 | 53 | function [imdb, index] = addImageSet(opts, imdb, index, setName, setCode) 54 | j = length(imdb.images.id) ; 55 | for ci = 1:length(imdb.classes.name) 56 | className = imdb.classes.name{ci} ; 57 | annoPath = fullfile(opts.dataDir, 'ImageSets', 'Main', ... 58 | [className '_' setName '.txt']) ; 59 | fprintf('%s: reading %s\n', mfilename, annoPath) ; 60 | [names,labels] = textread(annoPath, '%s %f') ; 61 | for i=1:length(names) 62 | if ~index.isKey(names{i}) 63 | j = j + 1 ; 64 | index(names{i}) = j ; 65 | imdb.images.id(j) = j ; 66 | imdb.images.set(j) = setCode ; 67 | imdb.images.name{j} = names{i} ; 68 | imdb.images.classification(j) = true ; 69 | else 70 | j = index(names{i}) ; 71 | end 72 | if labels(i) > 0, imdb.classes.images{ci}(end+1) = j ; end 73 | end 74 | end 75 | end 76 | 77 | function [imdb, index] = addSegmentationSet(opts, imdb, index, setName, setCode) 78 | segAnnoPath = fullfile(opts.dataDir, 'ImageSets', 'Segmentation', [setName '.txt']) ; 79 | fprintf('%s: reading %s\n', mfilename, segAnnoPath) ; 80 | segNames = textread(segAnnoPath, '%s') ; 81 | j = numel(imdb.images.id) ; 82 | for i=1:length(segNames) 83 | if index.isKey(segNames{i}) 84 | k = index(segNames{i}) ; 85 | imdb.images.segmentation(k) = true ; 86 | imdb.images.set(k) = setCode ; 87 | else 88 | j = j + 1 ; 89 | index(segNames{i}) = j ; 90 | imdb.images.id(j) = j ; 91 | imdb.images.set(j) = setCode ; 92 | imdb.images.name{j} = segNames{i} ; 93 | imdb.images.classification(j) = false ; 94 | imdb.images.segmentation(j) = true ; 95 | end 96 | end 97 | end 98 | 99 | function imdb = getImageSizes(imdb) 100 | for j=1:numel(imdb.images.id) 101 | info = imfinfo(sprintf(imdb.paths.image, imdb.images.name{j})) ; 102 | imdb.images.size(:,j) = uint16([info.Width ; info.Height]) ; 103 | fprintf('%s: checked image %s [%d x %d]\n', mfilename, imdb.images.name{j}, info.Height, info.Width) ; 104 | end 105 | end 106 | 107 | function str=esc(str) 108 | str = strrep(str, '\', '/') ; 109 | end 110 | -------------------------------------------------------------------------------- /interfaces/matlab/face_video_seg_batch.m: -------------------------------------------------------------------------------- 1 | function face_video_seg_batch(varargin) 2 | %FACE_VIDEO_SEG_BATCH Face Video Segmentation automatic pipeline. 3 | % Input: 4 | % inDir - Path to input video directory 5 | % outDir - Path to output directory 6 | % landmarks - Path to the landmarks model file 7 | % 8 | % Optional: 9 | % indices (=[]) - Video indices to process 10 | % minWidth (=0) - The minimum width of videos to process 11 | % minHeight (=0) - The minimum height of videos to process 12 | % maxScaleWidth (=0) - The maximum width scale to detect faces 13 | % maxScaleHeight (=0) - The maximum height scale to detect faces 14 | % track (=2) - Tracker type [0=NONE|1=BRISK|2=LBP] 15 | % verbose (=0) - output debug information level (0 means no debug) 16 | % 17 | % Output: 18 | % The output directory will contain 4 directories: 19 | % seg_trees - Containing the all the video segmentations hierarchies 20 | % for each video. 21 | % landmarks - Containing all the landmarks for each video. 22 | % fvs_files - Containing all the classified regions for each video. 23 | % output - Containing all the keyframe images and segmentations for 24 | % each video in a separate directory. 25 | 26 | %% Parse input arguments 27 | p = inputParser; 28 | addRequired(p, 'inDir', @ischar); 29 | addRequired(p, 'outDir', @ischar); 30 | addRequired(p, 'landmarks', @ischar); 31 | addParameter(p, 'indices', [], @isvector); 32 | addParameter(p, 'minWidth', 0, @isscalar); 33 | addParameter(p, 'minHeight', 0, @isscalar); 34 | addParameter(p, 'maxScaleWidth', 0, @isscalar); 35 | addParameter(p, 'maxScaleHeight', 0, @isscalar); 36 | addParameter(p, 'track', 2, @isscalar); 37 | addParameter(p, 'verbose', 0, @isscalar); 38 | parse(p,varargin{:}); 39 | indices = p.Results.indices; 40 | 41 | %% Create output directory structure 42 | outputPath = fullfile(p.Results.outDir, 'output'); 43 | segmentationsPath = fullfile(p.Results.outDir, 'face_segmentations'); 44 | segTreesPath = fullfile(p.Results.outDir, 'seg_trees'); 45 | landmarksPath = fullfile(p.Results.outDir, 'landmarks'); 46 | fvsPath = fullfile(p.Results.outDir, 'fvs_files'); 47 | if(~exist(outputPath, 'dir')) 48 | mkdir(outputPath); 49 | mkdir(segmentationsPath); 50 | mkdir(segTreesPath); 51 | mkdir(landmarksPath); 52 | mkdir(fvsPath); 53 | end 54 | 55 | %% Parse input directory 56 | filt = '.*(avi|mp4|mkv)'; 57 | fileDescs = dir(p.Results.inDir); 58 | fileNames = {fileDescs(~cellfun(@is_video,{fileDescs.name})).name}; 59 | if(isempty(indices)) 60 | indices = 1:length(fileNames); 61 | elseif(max(indices) > length(fileNames) || min(indices) < 1) 62 | error(['indices must be from 1 to ' num2str(length(fileNames))]); 63 | end 64 | 65 | %% For each video file 66 | for i = indices 67 | vidFile = fullfile(p.Results.inDir, fileNames{i}); 68 | [vidPath,vidName, vidExt] = fileparts(vidFile); 69 | disp(['Processing "', [vidName vidExt], '"']); 70 | 71 | %% Check resolution 72 | vid = VideoReader(vidFile); 73 | if(p.Results.minWidth > 0 && p.Results.minHeight > 0) 74 | %vid = VideoReader(vidFile); 75 | if(vid.Width < p.Results.minWidth || vid.Height < p.Results.minHeight) 76 | disp(['Skipping "', [vidName vidExt], '" because of low resolution']); 77 | continue; 78 | end 79 | end 80 | 81 | %% Segmentation tree 82 | dstSegTreeFile = [vidName vidExt '.pb']; 83 | dstSegTreePath = fullfile(segTreesPath, dstSegTreeFile); 84 | dstSegTreePath2 = fullfile(vidPath, dstSegTreeFile); 85 | if(exist(dstSegTreePath, 'file') == 2) 86 | disp(['"' dstSegTreeFile '" already exists. Skipping segmentation tree creation.']); 87 | elseif(exist(dstSegTreePath2, 'file') == 2) 88 | disp(['"' dstSegTreeFile '" already exists. Skipping segmentation tree creation.']); 89 | copyfile(dstSegTreePath2, segTreesPath); 90 | else 91 | disp(['Creating segmentation tree "' dstSegTreeFile '".']); 92 | %seg_tree(vidFile, segTreesPath, 'verbose', 0); 93 | fvs_segment(vidFile, 'outFile', dstSegTreePath); 94 | end 95 | 96 | %% Landmarks cache 97 | dstLandmarksFile = [vidName '.lms']; 98 | dstLandmarksPath = fullfile(landmarksPath, dstLandmarksFile); 99 | dstLandmarksPath2 = fullfile(vidPath, dstLandmarksFile); 100 | if(exist(dstLandmarksPath, 'file') == 2) 101 | disp(['"' dstLandmarksFile '" already exists. Skipping caching landmarks.']); 102 | elseif(exist(dstLandmarksPath2, 'file') == 2) 103 | disp(['"' dstLandmarksFile '" already exists. Skipping caching landmarks.']); 104 | copyfile(dstLandmarksPath2, dstLandmarksPath); 105 | else 106 | disp(['Creating landmarks cache "' dstLandmarksFile '".']); 107 | if((p.Results.maxScaleWidth > 0 && (vid.Width*2) > p.Results.maxScaleWidth) ||... 108 | (p.Results.maxScaleHeight > 0 && (vid.Height*2) > p.Results.maxScaleHeight)) 109 | scales = 1; 110 | else 111 | scales = 1:2; 112 | end 113 | sfl_cache(vidFile, p.Results.landmarks, 'output', dstLandmarksPath, 'scales', scales, 'track', p.Results.track); 114 | end 115 | 116 | %% Find regions 117 | dstFvsFile = [vidName '.fvs']; 118 | dstFvsPath = fullfile(fvsPath, dstFvsFile); 119 | if(exist(dstFvsPath, 'file') == 2) 120 | disp(['"' dstFvsFile '" already exists. Skipping finding regions.']); 121 | else 122 | disp(['Creating face video segmentation file "' dstFvsFile '".']); 123 | fvs_find_regions(vidFile, fvsPath, dstLandmarksPath, dstSegTreePath, 'verbose', p.Results.verbose); 124 | end 125 | 126 | %% Write keyframes 127 | vidOutDir = fullfile(outputPath, vidName); 128 | if(exist(vidOutDir, 'dir') == 7) 129 | disp(['"' vidName '" directory already exists. Skipping writing keyframes.']); 130 | else 131 | disp(['Writing keyframes to the directory "' vidName '".']); 132 | mkdir(vidOutDir); 133 | fvs_write_keyframes(dstFvsPath, vidOutDir, vidFile, dstSegTreePath, dstLandmarksPath, 'debug', p.Results.verbose); 134 | end 135 | end 136 | 137 | function b = is_video(file) 138 | [filePath, fileName, fileExt] = fileparts(file); 139 | b = isempty(regexpi(fileExt, filt)); 140 | end 141 | end 142 | 143 | -------------------------------------------------------------------------------- /interfaces/matlab/fvs_extract_list.m: -------------------------------------------------------------------------------- 1 | function fvs_extract_list(inDir, outDir, listFile) 2 | %FVS_EXTRACT_LIST Summary of this function goes here 3 | % Detailed explanation goes here 4 | 5 | %% Read list 6 | include = readList(listFile); 7 | 8 | %% Extract trainval file 9 | trainvalFile = fullfile(inDir, 'trainval.csv'); 10 | if(exist(trainvalFile) == 2) 11 | T = readtable(trainvalFile,'Delimiter',';','Format', '%s%s%s%s'); 12 | names = table2cell(T(:,1)); 13 | [~,table_ind,~] = intersect(names,include); 14 | 15 | % Write intersection table 16 | outTrainvalFile = fullfile(outDir, 'trainval.csv'); 17 | writetable(T(table_ind, :), outTrainvalFile, 'Delimiter', ';'); 18 | end 19 | 20 | %% Parse input directory 21 | fileDescs = dir(inDir); 22 | dirNames = {fileDescs([fileDescs.isdir]).name}; 23 | dirNames = dirNames(3:end); 24 | 25 | %% Copy intersection directories 26 | [~,dir_ind,~] = intersect(dirNames,include); 27 | for i = dir_ind' 28 | disp(['Copying ' dirNames{i}]); 29 | inDirPath = fullfile(inDir, dirNames{i}); 30 | outDirPath = fullfile(outDir, dirNames{i}); 31 | copyfile(inDirPath, outDirPath); 32 | end 33 | 34 | %% Parse video files 35 | filt = '^.*\.(avi|mp4|mkv)$'; 36 | vidNames = {fileDescs(~cellfun(@isempty,regexpi({fileDescs.name},filt))).name}; 37 | vidNamesNoExt = cellfun(@(x) x(1:end-4), vidNames, 'un', 0); 38 | 39 | %% Copy intersection videos 40 | [~,vid_ind,~] = intersect(vidNamesNoExt,include); 41 | for i = vid_ind' 42 | disp(['Copying ' vidNames{i}]); 43 | inDirPath = fullfile(inDir, vidNames{i}); 44 | outDirPath = fullfile(outDir, vidNames{i}); 45 | copyfile(inDirPath, outDirPath); 46 | end 47 | 48 | end 49 | 50 | function names = readList(filename) 51 | fileID = fopen(filename); 52 | names = textscan(fileID,'%s'); 53 | names = names{1}; 54 | fclose(fileID); 55 | end 56 | -------------------------------------------------------------------------------- /interfaces/matlab/fvs_find_regions.m: -------------------------------------------------------------------------------- 1 | function fvs_find_regions(varargin) 2 | %FVS_FIND_REGIONS(videoFile, outDir, landmarksFile, segmentationFile, 3 | % 'verbose', verbose): 4 | % videoFile - Path to input video file 5 | % outDir - Path to output directory 6 | % landmarksFile - Path to the landmarks model file or landmarks cache 7 | % file (.pb) 8 | % segmentationFile - Path to the segmentation protobuffer file (.pb) 9 | % verbose (=0) - output debug information level (0 means no debug) 10 | 11 | %% Parse input arguments 12 | p = inputParser; 13 | addRequired(p, 'videoFile', @ischar); 14 | addRequired(p, 'outDir', @ischar); 15 | addRequired(p, 'landmarksFile', @ischar); 16 | addRequired(p, 'segmentationFile', @ischar); 17 | addParameter(p, 'verbose', 0, @isscalar); 18 | parse(p,varargin{:}); 19 | 20 | %% Execute local face video segmentation 21 | exeName = 'fvs_find_regions'; 22 | [status, cmdout] = system([exeName ' "' p.Results.videoFile... 23 | '" -o "' p.Results.outDir '" -l "' p.Results.landmarksFile... 24 | '" -s "' p.Results.segmentationFile... 25 | '" -v ' num2str(p.Results.verbose)]); 26 | if(status ~= 0) 27 | error(cmdout); 28 | end 29 | 30 | end 31 | 32 | 33 | -------------------------------------------------------------------------------- /interfaces/matlab/fvs_init_trainval.m: -------------------------------------------------------------------------------- 1 | function fvs_init_trainval(inDir, outFile) 2 | %FVS_INIT_EMPTY_TRAINVAL Summary of this function goes here 3 | % Detailed explanation goes here 4 | 5 | 6 | %% Parse input directory 7 | fileDescs = dir(inDir); 8 | dirNames = {fileDescs([fileDescs.isdir]).name}; 9 | dirNames = dirNames(3:end); 10 | 11 | %% Initialize sheet cell array 12 | M = cell(length(dirNames), 3); 13 | M(:,1) = dirNames; 14 | 15 | %% Write to file 16 | T = cell2table(M, 'VariableNames',{'Name', 'Target', 'Notes'}); 17 | writetable(T, outFile); -------------------------------------------------------------------------------- /interfaces/matlab/fvs_segment.m: -------------------------------------------------------------------------------- 1 | function fvs_segment(varargin) 2 | %FVS_SEGMENT Create video segmentation tree 3 | % FVS_SEGMENT(videoFile, 'outFile', outFile, 'preview', preview): 4 | % videoFile - Path to input video file 5 | % outFile - Path to output directory 6 | % preview (=-1) - If >= 0, display segmentation at the specified level 7 | % [0, 1] 8 | 9 | %% Parse input arguments 10 | p = inputParser; 11 | addRequired(p, 'videoFile', @ischar); 12 | addParameter(p, 'outFile', '', @ischar); 13 | addParameter(p, 'preview', -1, @isscalar); 14 | parse(p,varargin{:}); 15 | 16 | %% Execute segmentation 17 | cmd = [mfilename ' "' p.Results.videoFile '"'... 18 | ' -p ' num2str(p.Results.preview)]; 19 | if(~isempty(p.Results.outFile)) 20 | cmd = [cmd ' -o "' p.Results.outFile '"']; 21 | end 22 | [status, cmdout] = system(cmd); 23 | if(status ~= 0) 24 | error(sprintf(['error in seg_tree_sample executable:\n' cmdout])); 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /interfaces/matlab/fvs_write_keyframes.m: -------------------------------------------------------------------------------- 1 | function fvs_write_keyframes(varargin) 2 | %FVS_WRITE_KEYFRAMES(fvsFile, outDir, 'max_scale', max_scale, 3 | % 'upscale', upscale, 'verbose', verbose): 4 | % fvsFile - Path to the face video segmentation file (.fvs). The paths 5 | % for video, segmentation and landmarks will be taken from this file. 6 | % outDir - Path to output directory 7 | % max_scale (=0) - Maximum width or height of the keyframes [pixels]. 8 | % upscale (=0) - If true all keyframes will be upscaled up to max_scale. 9 | % debug (=0) - output debug information level (0 means no debug) 10 | % 11 | %FVS_WRITE_KEYFRAMES(fvsFile, outDir, videoFile, segmentationFile, 12 | % landmarksFile): 13 | % videoFile - Path to input video file 14 | % segmentationFile - Path to the segmentation protobuffer file (.pb) 15 | % landmarksFile - Path to the landmarks model file or landmarks cache 16 | % file (.pb) 17 | 18 | %% Parse input arguments 19 | p = inputParser; 20 | addRequired(p, 'fvsFile', @ischar); 21 | addRequired(p, 'outDir', @ischar); 22 | if(mod(nargin, 2) == 1) 23 | addRequired(p, 'videoFile', @ischar); 24 | addRequired(p, 'segmentationFile', @ischar); 25 | addRequired(p, 'landmarksFile', @ischar); 26 | end 27 | addParameter(p, 'max_scale', 0, @isscalar); 28 | addParameter(p, 'upscale', 0, @isscalar); 29 | addParameter(p, 'debug', 0, @isscalar); 30 | parse(p,varargin{:}); 31 | 32 | %% Execute face video segmentation write keyframes 33 | exeName = 'fvs_write_keyframes'; 34 | cmd = [exeName ' "' p.Results.fvsFile '"'... 35 | ' -o "' p.Results.outDir '"'... 36 | ' -d ' num2str(p.Results.debug)]; 37 | if(mod(nargin, 2) == 1) 38 | cmd = [cmd ' -v "' p.Results.videoFile '"'... 39 | ' -s "' p.Results.segmentationFile '"'... 40 | ' -l "' p.Results.landmarksFile '"']; 41 | end 42 | if(p.Results.max_scale > 0) 43 | cmd = [cmd ' -m ' num2str(p.Results.max_scale)]; 44 | end 45 | if(p.Results.upscale > 0) 46 | cmd = [cmd ' -u']; 47 | end 48 | [status, cmdout] = system(cmd); 49 | if(status ~= 0) 50 | error(cmdout); 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /interfaces/matlab/images_per_video.m: -------------------------------------------------------------------------------- 1 | function vi = images_per_video(imgListFile) 2 | %GET_IMAGES_PER_VIDEO(imgListFile) Count the number of images per video 3 | %from a specified image list file. 4 | D = dlmread(imgListFile,'_'); 5 | [imageCount videos] = hist(D(:,1), unique(D(:,1))); 6 | [imageCount, indices] = sort(imageCount, 2, 'descend'); 7 | videos = videos(indices); 8 | vi = [videos imageCount']; 9 | 10 | -------------------------------------------------------------------------------- /interfaces/matlab/produce_pascal_voc_db.m: -------------------------------------------------------------------------------- 1 | function produce_pascal_voc_db(varargin) 2 | %PRODUCE_PASCAL_VOC_DB Convert the generated ground truth to a dataset in 3 | %PASCAL VOC format. 4 | % Input: 5 | % inDir - Path to genrated ground truth directory 6 | % outDir - Path to output directory 7 | % 8 | % Optional: 9 | % indices (=[]) - Video indices to process 10 | % max_scale (=0) - The maximum allowed image resolution [pixels] 11 | % upscale (=0) - Upscale all images to max_scale resolution 12 | % copyImages (=1) - Toggle whether to copy the images to the dataset. If 13 | % disabled then only the meta-data will be written. This is useful to 14 | % quickly change selected images for training and valuation 15 | 16 | %% Parse input arguments 17 | p = inputParser; 18 | addRequired(p, 'inDir', @ischar); 19 | addRequired(p, 'outDir', @ischar); 20 | addParameter(p, 'indices', [], @isvector); 21 | addParameter(p, 'max_scale', 0, @isscalar); 22 | addParameter(p, 'upscale', 0, @isscalar); 23 | addParameter(p, 'copyImages', 1, @isscalar); 24 | parse(p,varargin{:}); 25 | indices = p.Results.indices; 26 | 27 | %% Create output directory structure 28 | if(~exist(p.Results.outDir, 'dir')) 29 | mkdir(p.Results.outDir) 30 | end 31 | framesPath = fullfile(p.Results.outDir, 'JPEGImages'); 32 | segmentationsPath = fullfile(p.Results.outDir, 'SegmentationClass'); 33 | imageSetsMainPath = fullfile(p.Results.outDir, 'ImageSets', 'Main'); 34 | imageSetsSegPath = fullfile(p.Results.outDir, 'ImageSets', 'Segmentation'); 35 | if(~exist(framesPath, 'dir')) 36 | mkdir(framesPath); 37 | end 38 | if(~exist(segmentationsPath, 'dir')) 39 | mkdir(segmentationsPath); 40 | end 41 | if(~exist(imageSetsMainPath, 'dir')) 42 | mkdir(imageSetsMainPath); 43 | end 44 | if(~exist(imageSetsSegPath, 'dir')) 45 | mkdir(imageSetsSegPath); 46 | end 47 | 48 | %% Parse input directory 49 | fileDescs = dir(p.Results.inDir); 50 | dirNames = {fileDescs([fileDescs.isdir]).name}; 51 | dirNames = dirNames(3:end); 52 | if(isempty(indices)) 53 | indices = 1:length(dirNames); 54 | elseif(max(indices) > length(dirNames) || min(indices) < 1) 55 | error(['indices must be from 1 to ' num2str(length(dirNames))]); 56 | end 57 | 58 | %% Read trainval file 59 | trainvalFile = fullfile(p.Results.inDir, 'trainval.csv'); 60 | T = readtable(trainvalFile,'Delimiter',';','Format', '%s%s%s%s'); 61 | names = table2cell(T(:,1)); 62 | targets = table2cell(T(:,2)); 63 | if(length(dirNames) ~= length(names)) 64 | error(['There is a mismatch between the trainval file '... 65 | 'and number of directories in "' p.Results.inDir '".']); 66 | end 67 | 68 | %% Initialize training and valuation set 69 | train = []; 70 | val = []; 71 | 72 | %% For each directory 73 | cmap = labelColors(); 74 | for i = indices 75 | dirPath = fullfile(p.Results.inDir, dirNames{i}); 76 | if(~strcmp(dirNames{i}, names{i})) 77 | error(['There is a mismatch between the trainval file '... 78 | 'and the directories!']); 79 | end 80 | if(~any(strcmp(targets{i}, {'train','val'}))) 81 | disp(['Skipping "' dirNames{i} '"']); 82 | continue; 83 | end 84 | disp(['Processing "' dirNames{i} '"']); 85 | 86 | %% Parse current directory 87 | fileDescs = dir(fullfile(dirPath, '*frame*')); 88 | srcFrames = {fileDescs.name}; 89 | fileDescs = dir(fullfile(dirPath, '*seg*')); 90 | srcSegmentations = {fileDescs.name}; 91 | 92 | %% Copy files 93 | dstNames = calc_dst_names(srcFrames); 94 | if(p.Results.copyImages) 95 | process_files(srcFrames, dstNames, dirPath, framesPath, 'bicubic'); 96 | process_files(srcSegmentations, dstNames, dirPath,... 97 | segmentationsPath, 'nearest'); 98 | end 99 | 100 | %% Add images to training and valuation sets 101 | if(strcmp(targets{i}, 'train')) 102 | train = [train; dstNames]; 103 | else % val 104 | val = [val; dstNames]; 105 | end 106 | end 107 | 108 | %% Write training and valuation sets to files 109 | write_to_file(fullfile(imageSetsMainPath, 'train.txt'), train); 110 | write_to_file(fullfile(imageSetsSegPath, 'train.txt'), train); 111 | write_to_file(fullfile(imageSetsMainPath, 'val.txt'), val); 112 | write_to_file(fullfile(imageSetsSegPath, 'val.txt'), val); 113 | 114 | classes = {... 115 | 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', ... 116 | 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', ... 117 | 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor'}; 118 | for i = 1:length(classes) 119 | trainPath = fullfile(imageSetsMainPath, [classes{i} '_train.txt']); 120 | valPath = fullfile(imageSetsMainPath, [classes{i} '_val.txt']); 121 | if(strcmp(classes{i}, 'person')) 122 | write_to_file(trainPath, train); 123 | write_to_file(valPath, val); 124 | else 125 | write_to_file(trainPath, []); 126 | write_to_file(valPath, []); 127 | end 128 | end 129 | 130 | %% Helper functions 131 | 132 | function process_files(srcFiles, dstNames, inDir, outDir,... 133 | inter_method) 134 | for j = 1:length(srcFiles) 135 | [~,~,ext] = fileparts(srcFiles{j}); 136 | dstFile = [dstNames{j} ext]; 137 | srcPath = fullfile(inDir, srcFiles{j}); 138 | dstPath = fullfile(outDir, dstFile); 139 | I = imread(srcPath); 140 | if(p.Results.max_scale > 0 && ... 141 | (max(size(I)) > p.Results.max_scale || ... 142 | (max(size(I)) < p.Results.max_scale && p.Results.upscale))) 143 | scale = p.Results.max_scale / max(size(I)); 144 | outputSize = round([size(I,1),size(I,2)]*scale); 145 | I = imresize(I, outputSize, inter_method); 146 | end 147 | imwrite(I,cmap,dstPath,'png'); 148 | end 149 | end 150 | 151 | function write_to_file(fileName, stringArray) 152 | fid = fopen(fileName,'wt'); 153 | for j = 1:length(stringArray) 154 | fprintf(fid, '%s\n', stringArray{j}); 155 | end 156 | fclose(fid); 157 | end 158 | end 159 | 160 | function dstNames = calc_dst_names(srcFiles) 161 | dstNames = cell(length(srcFiles),1); 162 | for i = 1:length(srcFiles) 163 | C = strsplit(srcFiles{i}, {'_', '.'}); 164 | dstNames{i} = [C{1} '_' C{3} '_' C{5}]; 165 | end 166 | end 167 | 168 | function cmap = labelColors() 169 | N=21; 170 | cmap = zeros(N,3); 171 | for i=1:N 172 | id = i-1; r=0;g=0;b=0; 173 | for j=0:7 174 | r = bitor(r, bitshift(bitget(id,1),7 - j)); 175 | g = bitor(g, bitshift(bitget(id,2),7 - j)); 176 | b = bitor(b, bitshift(bitget(id,3),7 - j)); 177 | id = bitshift(id,-3); 178 | end 179 | cmap(i,1)=r; cmap(i,2)=g; cmap(i,3)=b; 180 | end 181 | cmap = cmap / 255; 182 | end 183 | 184 | -------------------------------------------------------------------------------- /interfaces/matlab/seg_tree.m: -------------------------------------------------------------------------------- 1 | function seg_tree(varargin) 2 | %SEG_TREE Create segmentation tree 3 | % SEG_TREE(videoFile, outDir, 'verbose', verbose): 4 | % videoFile - Path to input video file 5 | % outDir - Path to output directory 6 | % verbose (=-1) - If >= 0, display segmentation at the specified level 7 | % [0, 1] 8 | 9 | %% Parse input arguments 10 | p = inputParser; 11 | addRequired(p, 'videoFile', @ischar); 12 | addRequired(p, 'outDir', @ischar); 13 | addParameter(p, 'verbose', -1, @isscalar); 14 | parse(p,varargin{:}); 15 | 16 | %% Execute face motion segmentation 17 | exeName = 'seg_tree_sample'; 18 | [status, cmdout] = system([exeName ' --input_file="' p.Results.videoFile... 19 | '" --display="' num2str(p.Results.verbose)... 20 | '" --write_to_file --use_pipeline']); 21 | if(status ~= 0) 22 | %error(cmdout); 23 | error(sprintf(['error in seg_tree_sample executable:\n' cmdout])); 24 | end 25 | 26 | seg_path = [p.Results.videoFile '.pb']; 27 | if(~exist(seg_path, 'file') == 2) 28 | error(['Failed to create segmentation tree for "' p.Results.videoFile '"']); 29 | end 30 | 31 | %% Copy segmentation to output directory 32 | movefile(seg_path, p.Results.outDir); 33 | 34 | end 35 | -------------------------------------------------------------------------------- /interfaces/matlab/seg_viewer.m: -------------------------------------------------------------------------------- 1 | function seg_viewer(segFile) 2 | %SEG_VIEWER View segmentation files 3 | % SEG_VIEWER(segFile) will view the segmentation file, segFile. 4 | 5 | %% Execute segmentation viewer 6 | exeName = 'segment_viewer'; 7 | [status, cmdout] = system([exeName ' --input="' segFile... 8 | '" --window_name=seg_viewer']); 9 | if(status ~= 0) 10 | error(cmdout); 11 | end 12 | 13 | end 14 | 15 | --------------------------------------------------------------------------------