├── .clang-format ├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── README.md ├── include ├── base │ ├── camera_manager.h │ ├── exceptions │ │ └── lock_timeout.h │ ├── logging.h │ ├── scoped_timer.h │ ├── server │ │ ├── camera_control.h │ │ ├── packet_stream.h │ │ └── websocket_server.h │ └── video │ │ ├── frame_map.h │ │ ├── frame_queue.h │ │ ├── render_text.h │ │ ├── rendered_frame.h │ │ └── type_managers.h ├── encode.h └── server.h ├── proto ├── CMakeLists.txt └── nes.proto └── src ├── base ├── camera_manager.cc ├── server │ ├── camera_control.cc │ ├── packet_stream.cc │ └── websocket_server.cc └── video │ ├── frame_map.cc │ ├── frame_queue.cc │ ├── render_text.cc │ ├── rendered_frame.cc │ └── type_managers.cc ├── encode.cpp ├── main.cpp └── server.cpp /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | BasedOnStyle: Google 4 | AccessModifierOffset: -1 5 | AlignAfterOpenBracket: Align 6 | AlignConsecutiveMacros: false 7 | AlignConsecutiveAssignments: false 8 | AlignConsecutiveDeclarations: false 9 | AlignEscapedNewlines: Left 10 | AlignOperands: true 11 | AlignTrailingComments: true 12 | AllowAllArgumentsOnNextLine: true 13 | AllowAllConstructorInitializersOnNextLine: true 14 | AllowAllParametersOfDeclarationOnNextLine: true 15 | AllowShortBlocksOnASingleLine: Never 16 | AllowShortCaseLabelsOnASingleLine: false 17 | AllowShortFunctionsOnASingleLine: All 18 | AllowShortLambdasOnASingleLine: All 19 | AllowShortIfStatementsOnASingleLine: WithoutElse 20 | AllowShortLoopsOnASingleLine: true 21 | AlwaysBreakAfterDefinitionReturnType: None 22 | AlwaysBreakAfterReturnType: None 23 | AlwaysBreakBeforeMultilineStrings: true 24 | AlwaysBreakTemplateDeclarations: Yes 25 | BinPackArguments: true 26 | BinPackParameters: true 27 | BraceWrapping: 28 | AfterCaseLabel: false 29 | AfterClass: false 30 | AfterControlStatement: false 31 | AfterEnum: false 32 | AfterFunction: false 33 | AfterNamespace: false 34 | AfterObjCDeclaration: false 35 | AfterStruct: false 36 | AfterUnion: false 37 | AfterExternBlock: false 38 | BeforeCatch: false 39 | BeforeElse: false 40 | IndentBraces: false 41 | SplitEmptyFunction: true 42 | SplitEmptyRecord: true 43 | SplitEmptyNamespace: true 44 | BreakBeforeBinaryOperators: None 45 | BreakBeforeBraces: Attach 46 | BreakBeforeInheritanceComma: false 47 | BreakInheritanceList: BeforeColon 48 | BreakBeforeTernaryOperators: true 49 | BreakConstructorInitializersBeforeComma: false 50 | BreakConstructorInitializers: BeforeColon 51 | BreakAfterJavaFieldAnnotations: false 52 | BreakStringLiterals: true 53 | ColumnLimit: 80 54 | CommentPragmas: '^ IWYU pragma:' 55 | CompactNamespaces: false 56 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 57 | ConstructorInitializerIndentWidth: 4 58 | ContinuationIndentWidth: 4 59 | Cpp11BracedListStyle: true 60 | DeriveLineEnding: true 61 | DerivePointerAlignment: true 62 | DisableFormat: false 63 | ExperimentalAutoDetectBinPacking: false 64 | FixNamespaceComments: true 65 | ForEachMacros: 66 | - foreach 67 | - Q_FOREACH 68 | - BOOST_FOREACH 69 | IncludeBlocks: Regroup 70 | IncludeCategories: 71 | - Regex: '^' 72 | Priority: 2 73 | SortPriority: 0 74 | - Regex: '^<.*\.h>' 75 | Priority: 1 76 | SortPriority: 0 77 | - Regex: '^<.*' 78 | Priority: 2 79 | SortPriority: 0 80 | - Regex: '.*' 81 | Priority: 3 82 | SortPriority: 0 83 | IncludeIsMainRegex: '([-_](test|unittest))?$' 84 | IncludeIsMainSourceRegex: '' 85 | IndentCaseLabels: true 86 | IndentGotoLabels: true 87 | IndentPPDirectives: None 88 | IndentWidth: 2 89 | IndentWrappedFunctionNames: false 90 | JavaScriptQuotes: Leave 91 | JavaScriptWrapImports: true 92 | KeepEmptyLinesAtTheStartOfBlocks: false 93 | MacroBlockBegin: '' 94 | MacroBlockEnd: '' 95 | MaxEmptyLinesToKeep: 1 96 | NamespaceIndentation: None 97 | ObjCBinPackProtocolList: Never 98 | ObjCBlockIndentWidth: 2 99 | ObjCSpaceAfterProperty: false 100 | ObjCSpaceBeforeProtocolList: true 101 | PenaltyBreakAssignment: 2 102 | PenaltyBreakBeforeFirstCallParameter: 1 103 | PenaltyBreakComment: 300 104 | PenaltyBreakFirstLessLess: 120 105 | PenaltyBreakString: 1000 106 | PenaltyBreakTemplateDeclaration: 10 107 | PenaltyExcessCharacter: 1000000 108 | PenaltyReturnTypeOnItsOwnLine: 200 109 | PointerAlignment: Left 110 | RawStringFormats: 111 | - Language: Cpp 112 | Delimiters: 113 | - cc 114 | - CC 115 | - cpp 116 | - Cpp 117 | - CPP 118 | - 'c++' 119 | - 'C++' 120 | CanonicalDelimiter: '' 121 | BasedOnStyle: google 122 | - Language: TextProto 123 | Delimiters: 124 | - pb 125 | - PB 126 | - proto 127 | - PROTO 128 | EnclosingFunctions: 129 | - EqualsProto 130 | - EquivToProto 131 | - PARSE_PARTIAL_TEXT_PROTO 132 | - PARSE_TEST_PROTO 133 | - PARSE_TEXT_PROTO 134 | - ParseTextOrDie 135 | - ParseTextProtoOrDie 136 | CanonicalDelimiter: '' 137 | BasedOnStyle: google 138 | ReflowComments: true 139 | SortIncludes: true 140 | SortUsingDeclarations: true 141 | SpaceAfterCStyleCast: false 142 | SpaceAfterLogicalNot: false 143 | SpaceAfterTemplateKeyword: true 144 | SpaceBeforeAssignmentOperators: true 145 | SpaceBeforeCpp11BracedList: false 146 | SpaceBeforeCtorInitializerColon: true 147 | SpaceBeforeInheritanceColon: true 148 | SpaceBeforeParens: ControlStatements 149 | SpaceBeforeRangeBasedForLoopColon: true 150 | SpaceInEmptyBlock: false 151 | SpaceInEmptyParentheses: false 152 | SpacesBeforeTrailingComments: 2 153 | SpacesInAngles: false 154 | SpacesInConditionalStatement: false 155 | SpacesInContainerLiterals: true 156 | SpacesInCStyleCastParentheses: false 157 | SpacesInParentheses: false 158 | SpacesInSquareBrackets: false 159 | SpaceBeforeSquareBrackets: false 160 | Standard: Auto 161 | StatementMacros: 162 | - Q_UNUSED 163 | - QT_REQUIRE_VERSION 164 | TabWidth: 8 165 | UseCRLF: false 166 | UseTab: Never 167 | ... 168 | 169 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | .vscode/ 3 | *.mp4 4 | massif.out.* 5 | valgrind* 6 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "dependencies/tinylogger"] 2 | path = dependencies/tinylogger 3 | url = https://github.com/moonsikpark/tinylogger.git 4 | [submodule "dependencies/args"] 5 | path = dependencies/args 6 | url = https://github.com/Taywee/args.git 7 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Moonsik Park. All rights reserved. 3 | # 4 | # @file CMakeLists.txt 5 | # @author Moonsik Park, Korea Institute of Science and Technology 6 | # 7 | 8 | cmake_minimum_required(VERSION 3.19) 9 | 10 | PROJECT(ngp-encode-server 11 | VERSION 1.0 12 | DESCRIPTION "NGP encode server" 13 | LANGUAGES C CXX 14 | ) 15 | 16 | set(NES_VERSION "${CMAKE_PROJECT_VERSION}") 17 | 18 | if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) 19 | message(STATUS "No release type specified. Setting to 'Release'.") 20 | set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE) 21 | set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "RelWithDebInfo") 22 | endif() 23 | 24 | set(CMAKE_CXX_STANDARD 20) 25 | set(CMAKE_CXX_EXTENSIONS OFF) 26 | SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread") 27 | 28 | include_directories("include") 29 | include_directories("dependencies") 30 | include_directories("dependencies/tinylogger") 31 | 32 | find_path(AVCODEC_INCLUDE_DIR libavcodec/avcodec.h) 33 | find_library(AVCODEC_LIBRARY avcodec) 34 | 35 | find_path(AVFORMAT_INCLUDE_DIR libavformat/avformat.h) 36 | find_library(AVFORMAT_LIBRARY avformat) 37 | 38 | find_path(AVUTIL_INCLUDE_DIR libavutil/avutil.h) 39 | find_library(AVUTIL_LIBRARY avutil) 40 | 41 | find_path(AVFILTER_INCLUDE_DIR libavfilter/avfilter.h) 42 | find_library(AVFILTER_LIBRARY avfilter) 43 | 44 | find_path(SWSCALE_INCLUDE_DIR libswscale/swscale.h) 45 | find_library(SWSCALE_LIBRARY swscale) 46 | 47 | find_package(Freetype REQUIRED) 48 | find_package(websocketpp REQUIRED) 49 | find_package(OpenSSL REQUIRED) 50 | find_package(Protobuf REQUIRED) 51 | 52 | add_subdirectory(proto) 53 | 54 | include_directories(${AVCODEC_INCLUDE_DIR}) 55 | include_directories(${AVFORMAT_INCLUDE_DIR}) 56 | include_directories(${AVUTIL_INCLUDE_DIR}) 57 | include_directories(${SWSCALE_INCLUDE_DIR}) 58 | include_directories(${FREETYPE_INCLUDE_DIRS}) 59 | include_directories(${WEBSOCKETPP_INCLUDE_DIR}) 60 | include_directories(${CMAKE_BINARY_DIR}/proto) 61 | 62 | set(SOURCES 63 | src/encode.cpp 64 | src/server.cpp 65 | src/main.cpp 66 | src/base/camera_manager.cc 67 | src/base/server/camera_control.cc 68 | src/base/server/packet_stream.cc 69 | src/base/server/websocket_server.cc 70 | src/base/video/frame_queue.cc 71 | src/base/video/frame_map.cc 72 | src/base/video/type_managers.cc 73 | src/base/video/render_text.cc 74 | src/base/video/rendered_frame.cc 75 | ) 76 | 77 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) 78 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR}) 79 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELWITHDEBINFO ${CMAKE_BINARY_DIR}) 80 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_MINSIZEREL ${CMAKE_BINARY_DIR}) 81 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_BINARY_DIR}) 82 | 83 | add_executable(neserver src/main.cpp ${SOURCES}) 84 | target_include_directories(neserver PRIVATE ${AVCODEC_INCLUDE_DIR} ${AVFORMAT_INCLUDE_DIR} ${AVFILTER_INCLUDE_DIR} ${AVUTIL_INCLUDE_DIR} ${SWSCALE_INCLUDE_DIR} ${FREETYPE_INCLUDE_DIRS} ${WEBSOCKETPP_INCLUDE_DIR} ${OPENSSL_INCLUDE_DIR}) 85 | target_link_libraries(neserver PRIVATE proto ${AVCODEC_LIBRARY} ${AVFORMAT_LIBRARY} ${AVFILTER_LIBRARY} ${AVUTIL_LIBRARY} ${SWSCALE_LIBRARY} ${FREETYPE_LIBRARIES} OpenSSL::SSL OpenSSL::Crypto) 86 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ngp-encode-server 2 | 3 | **ngp-encode-server** receives VR user's head position and field of view, and renders the view corresponding to user position using [instant-ngp](https://github.com/NVlabs/instant-ngp). The goal of this program is to study whether **Neural Radiance Field** technology is suitable for background generation of immersive VR environments. 4 | 5 | To elaborate, **ngp-encode-server** receives the user's field of view and head position from [ngp-client](https://github.com/moonsikpark/ngp-client). The program requests to render a frame containing the scene and the depth data generated by NeRF, corresponding to the received user position, to multiple instances of [instant-ngp-renderer](https://github.com/moonsikpark/instant-ngp-renderer), which is a customized version of instant-ngp. The views are encoded to a video using H.264 codec and streamed to the [ngp-client](https://github.com/moonsikpark/ngp-client) via WebSocket. [ngp-client](https://github.com/moonsikpark/ngp-client) decodes the packet using WebCodecs and plays it to the user's VR device. The client only uses web standards-compliant technologies, and does not have to rely on external programs or plug-ins. 6 | 7 | ## Requirements 8 | 9 | - A C++20 compatable compiler. (GCC 8 or later) 10 | - Linux distribution of your choice. Distributions other than Ubuntu 20.04 has not been tested. 11 | 12 | ## Dependencies 13 | 14 | - libavcodec 15 | - libavformat 16 | - libavutil 17 | - libswscale 18 | - freetype2 19 | - libwebsocketpp 20 | - OpenSSL 21 | - libprotobuf 22 | 23 | If you are using Ubuntu 20.04, install the following packages; 24 | ```sh 25 | sudo apt install build-essential \ 26 | cmake \ 27 | git \ 28 | gcc-10 \ 29 | libavcodec-dev \ 30 | libavformat-dev \ 31 | libavutil-dev \ 32 | libswscale-dev \ 33 | libfreetype-dev \ 34 | libwebsocketpp-dev \ 35 | libssl-dev \ 36 | libprotobuf-dev 37 | 38 | ``` 39 | 40 | ## Compilation 41 | 42 | Begin by cloning this repository and all its submodules using the following command: 43 | ```sh 44 | $ git clone --recursive https://github.com/moonsikpark/ngp-encode-server 45 | $ cd ngp-encode-server 46 | ``` 47 | 48 | Then, use CMake to build the project: 49 | ```sh 50 | ngp-encode-server$ cmake . -B build 51 | ngp-encode-server$ cmake --build build --config RelWithDebInfo -j $(nproc) 52 | ``` 53 | 54 | If the build succeeds, you can now run the code via the `build/neserver` executable. 55 | 56 | ## Author 57 | 58 | Moonsik Park, Korea Instutute of Science and Tecnhology - moonsik.park@kist.re.kr 59 | 60 | ## Copyright 61 | 62 | Copyright (c) 2022 Moonsik Park. All rights reserved. 63 | -------------------------------------------------------------------------------- /include/base/camera_manager.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #ifndef NES_BASE_CAMERA_MANAGER_ 4 | #define NES_BASE_CAMERA_MANAGER_ 5 | 6 | #include 7 | 8 | #include "base/video/type_managers.h" 9 | #include "nes.pb.h" 10 | 11 | // CameraManager handles camera matrix used for rendering a frame. It accepts a 12 | // user position converted to a camera matrix, stores it internally, and 13 | // provides it when a FrameRequest is generated. 14 | class CameraManager { 15 | public: 16 | // Initial camera matrix set to the initial coordinate (0, 0, 0) and field of 17 | // view. 18 | static constexpr float kInitialCameraMatrix[] = { 19 | 1.0f, 0.0f, 0.0f, 0.5f, 0.0f, -1.0f, 0.0f, 0.5f, 0.0f, 0.0f, -1.0f, 0.5f}; 20 | 21 | // Initialize Camera with kInitialCameraMatrix and provided default 22 | // dimensions. 23 | CameraManager(std::shared_ptr codec_scene_left, 24 | std::shared_ptr codec_depth_left, 25 | std::shared_ptr codec_scene_right, 26 | std::shared_ptr codec_depth_right, 27 | uint32_t default_width, uint32_t default_height); 28 | 29 | // Replace camera with the provided camera data. If the resolution has 30 | // changed, reinitialize the encoder. 31 | void set_camera_left(nesproto::Camera camera); 32 | void set_camera_right(nesproto::Camera camera); 33 | 34 | inline nesproto::Camera get_camera_left() const { return m_camera_left; } 35 | inline nesproto::Camera get_camera_right() const { return m_camera_right; } 36 | 37 | private: 38 | nesproto::Camera m_camera_left; 39 | nesproto::Camera m_camera_right; 40 | std::shared_ptr m_codec_scene_left; 41 | std::shared_ptr m_codec_depth_left; 42 | std::shared_ptr m_codec_scene_right; 43 | std::shared_ptr m_codec_depth_right; 44 | }; 45 | 46 | #endif // NES_BASE_CAMERA_MANAGER_ 47 | -------------------------------------------------------------------------------- /include/base/exceptions/lock_timeout.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #ifndef NES_BASE_EXCEPTIONS_LOCK_TIMEOUT_ 4 | #define NES_BASE_EXCEPTIONS_LOCK_TIMEOUT_ 5 | 6 | #include 7 | 8 | // Timeout reached while waiting for a lock. 9 | class LockTimeout : public std::exception { 10 | virtual const char *what() const throw() { 11 | return "Waiting for lock timed out."; 12 | } 13 | }; 14 | 15 | #endif // NES_BASE_EXCEPTIONS_LOCK_TIMEOUT_ 16 | -------------------------------------------------------------------------------- /include/base/logging.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #ifndef NES_BASE_LOGGING_ 4 | #define NES_BASE_LOGGING_ 5 | 6 | #include "tinylogger/tinylogger.h" 7 | 8 | using namespace tlog; 9 | 10 | #endif // NES_BASE_LOGGING_ 11 | -------------------------------------------------------------------------------- /include/base/scoped_timer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #ifndef NES_BASE_SCOPED_TIMER_ 4 | #define NES_BASE_SCOPED_TIMER_ 5 | 6 | #include 7 | 8 | // A timer operating inside a scope. 9 | class ScopedTimer { 10 | public: 11 | using clock = std::chrono::steady_clock; 12 | using time_format = std::chrono::milliseconds; 13 | // Start the timer in a scope. 14 | ScopedTimer() : _start(clock::now()) {} 15 | 16 | // Returns the time elapsed since the timer start. 17 | inline time_format elapsed() { 18 | return std::chrono::duration_cast(clock::now() - _start); 19 | } 20 | 21 | private: 22 | std::chrono::time_point _start; 23 | }; 24 | 25 | #endif // NES_BASE_SCOPED_TIMER_ 26 | -------------------------------------------------------------------------------- /include/base/server/camera_control.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #ifndef NES_BASE_SERVER_CAMERA_CONTROL_ 4 | #define NES_BASE_SERVER_CAMERA_CONTROL_ 5 | 6 | #include 7 | 8 | #include "base/camera_manager.h" 9 | #include "base/logging.h" 10 | #include "base/server/websocket_server.h" 11 | 12 | // A Websocket server that receives nesproto::Camera. 13 | class CameraControlServer : public WebSocketServer { 14 | public: 15 | // Interval of logging the receive event. 16 | static constexpr unsigned kReceivedLoggingInterval = 1000; 17 | 18 | CameraControlServer(std::shared_ptr cameramgr, 19 | uint16_t bind_port); 20 | 21 | void message_handler(websocketpp::connection_hdl hdl, message_ptr msg); 22 | 23 | private: 24 | std::shared_ptr m_camera_manager; 25 | uint64_t m_message_count = 0; 26 | }; 27 | 28 | #endif // NES_BASE_SERVER_CAMERA_CONTROL_ 29 | -------------------------------------------------------------------------------- /include/base/server/packet_stream.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #ifndef NES_BASE_SERVER_PACKET_STREAM_ 4 | #define NES_BASE_SERVER_PACKET_STREAM_ 5 | 6 | #include 7 | 8 | #include "base/logging.h" 9 | #include "base/server/websocket_server.h" 10 | 11 | extern "C" { 12 | #include "libavcodec/avcodec.h" // AVPacket, AV_PKT_FLAG_KEY 13 | } 14 | 15 | class PacketStreamServer : public WebSocketServer { 16 | public: 17 | PacketStreamServer(uint16_t bind_port, std::string server_name) 18 | : WebSocketServer(server_name, bind_port) {} 19 | 20 | inline void message_handler(websocketpp::connection_hdl hdl, 21 | message_ptr msg) {} 22 | 23 | void consume_packet(AVPacket *pkt); 24 | }; 25 | 26 | #endif // NES_BASE_SERVER_PACKET_STREAM_SERVER_ 27 | -------------------------------------------------------------------------------- /include/base/server/websocket_server.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #ifndef NES_BASE_SERVER_WEBSOCKET_SERVER_ 4 | #define NES_BASE_SERVER_WEBSOCKET_SERVER_ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "base/logging.h" 13 | 14 | typedef websocketpp::server server_notls; 15 | 16 | using websocketpp::lib::bind; 17 | using websocketpp::lib::placeholders::_1; 18 | using websocketpp::lib::placeholders::_2; 19 | using websocketpp::log::alevel; 20 | 21 | typedef websocketpp::config::asio::message_type::ptr message_ptr; 22 | typedef websocketpp::lib::shared_ptr 23 | context_ptr; 24 | typedef std::set> 26 | con_list; 27 | 28 | class WebSocketServer { 29 | private: 30 | server_notls m_server; 31 | con_list m_connections; 32 | websocketpp::lib::shared_ptr m_thread; 33 | std::string m_server_name; 34 | uint16_t m_bind_port; 35 | bool m_running = false; 36 | 37 | static void run_server(server_notls *s) { s->run(); } 38 | 39 | public: 40 | virtual void message_handler(websocketpp::connection_hdl hdl, 41 | message_ptr msg) = 0; 42 | 43 | WebSocketServer(std::string server_name, uint16_t bind_port); 44 | void start(); 45 | void stop(); 46 | void send_to_all(const char *data, size_t size); 47 | }; 48 | 49 | #endif // NES_BASE_SERVER_WEBSOCKET_SERVER_ 50 | -------------------------------------------------------------------------------- /include/base/video/frame_map.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #ifndef NES_BASE_VIDEO_FRAME_MAP_ 4 | #define NES_BASE_VIDEO_FRAME_MAP_ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "base/video/rendered_frame.h" 14 | 15 | // Thread-safe Map implementation based on std::map used to store unique_ptr of 16 | // RenderedFrame. 17 | class FrameMap { 18 | public: 19 | // Max size of FrameMap. 20 | static constexpr std::size_t kFrameMapMaxSize = 100; 21 | 22 | // Timeout waiting for insert/get of FrameMap. 23 | // This timeout value is important to skip frames that are taking too long 24 | // to render or occured an error while rendering. 25 | static constexpr std::chrono::milliseconds kFrameMapLockTimeout{1000}; 26 | 27 | // Interval of cleaning up unused Frames (i.e. frames older than the 28 | // requested frame). 29 | static constexpr unsigned kFrameMapDropFramesInterval = 1000; 30 | 31 | using element = std::unique_ptr; 32 | using keytype = std::uint64_t; 33 | 34 | void insert(keytype index, element &&el); 35 | element get_delete(keytype index); 36 | 37 | private: 38 | std::map m_map; 39 | std::condition_variable m_getter, m_inserter; 40 | std::mutex m_mutex; 41 | using unique_lock = std::unique_lock; 42 | }; 43 | 44 | #endif // NES_BASE_FRAME_MAP_ 45 | -------------------------------------------------------------------------------- /include/base/video/frame_queue.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #ifndef NES_BASE_VIDEO_FRAME_QUEUE_ 4 | #define NES_BASE_VIDEO_FRAME_QUEUE_ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "base/video/rendered_frame.h" 14 | 15 | // Thread-safe queue implementation based on std::queue used to store unique_ptr 16 | // of RenderedFrame. 17 | class FrameQueue { 18 | public: 19 | // Max size of FrameQueue. 20 | static constexpr std::size_t kFrameQueueMaxSize = 100; 21 | 22 | // Timeout of push/pop operation. 23 | static constexpr std::chrono::milliseconds kFrameQueueLockTimeout{1000}; 24 | 25 | using element = std::unique_ptr; 26 | 27 | void push(element &&el); 28 | element pop(); 29 | 30 | private: 31 | std::queue m_queue; 32 | std::condition_variable m_pusher, m_popper; 33 | std::mutex m_mutex; 34 | using unique_lock = std::unique_lock; 35 | }; 36 | 37 | #endif // NES_BASE_FRAME_QUEUE_ 38 | -------------------------------------------------------------------------------- /include/base/video/render_text.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #ifndef NES_BASE_VIDEO_RENDER_TEXT_ 4 | #define NES_BASE_VIDEO_RENDER_TEXT_ 5 | 6 | #include 7 | 8 | #include "base/video/type_managers.h" 9 | 10 | extern "C" { 11 | #include "freetype2/ft2build.h" 12 | #include FT_FREETYPE_H 13 | } 14 | 15 | class RenderTextContext { 16 | public: 17 | enum RenderPosition { 18 | RENDER_POSITION_LEFT_TOP, 19 | RENDER_POSITION_LEFT_BOTTOM, 20 | RENDER_POSITION_RIGHT_TOP, 21 | RENDER_POSITION_RIGHT_BOTTOM, 22 | RENDER_POSITION_CENTER 23 | }; 24 | 25 | RenderTextContext(std::string font_location); 26 | 27 | void render_string_to_frame(types::FrameManager &frame, 28 | RenderTextContext::RenderPosition opt, 29 | std::string content); 30 | 31 | ~RenderTextContext(); 32 | 33 | private: 34 | FT_Library _library; 35 | FT_Face _face; 36 | std::mutex m_mutex; 37 | std::condition_variable m_wait; 38 | using unique_lock = std::unique_lock; 39 | }; 40 | 41 | #endif // NES_BASE_VIDEO_RENDER_TEXT_ 42 | -------------------------------------------------------------------------------- /include/base/video/rendered_frame.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #ifndef NES_BASE_RENDERED_FRAME_ 4 | #define NES_BASE_RENDERED_FRAME_ 5 | 6 | #include "base/video/type_managers.h" 7 | #include "nes.pb.h" 8 | 9 | // RenderedFrame stores all information related to an uncompressed frame. It is 10 | // created with a raw RGB image stored in m_source_avframe. The RGB image buffer 11 | // should be visible to other programs to make modifications such as overlaying 12 | // texts. It converts the RGB image to YUV image using swscale and stores it in 13 | // m_converted_avframe_scene. After the image is ready, the program provides the 14 | // converted image to the encoder. 15 | class RenderedFrame { 16 | public: 17 | RenderedFrame(nesproto::RenderedFrame frame, AVPixelFormat pix_fmt_scene, 18 | AVPixelFormat pix_fmt_depth, 19 | std::shared_ptr ctxmgr_scene, 20 | std::shared_ptr ctxmgr_depth); 21 | 22 | // Convert frame stored in m_source_avframe from RGB to YUV and store it in 23 | // m_converted_avframe_scene. 24 | inline void convert_frame() { 25 | if (m_converted) { 26 | throw std::runtime_error{"Tried to convert a converted RenderedFrame."}; 27 | } 28 | types::SwsContextManager sws_context_scene(m_source_avframe_scene, 29 | m_converted_avframe_scene); 30 | types::SwsContextManager sws_context_depth(m_source_avframe_depth, 31 | m_converted_avframe_depth); 32 | m_converted = true; 33 | } 34 | 35 | // Index of the frame. 36 | const inline uint64_t index() const { return this->m_frame_response.index(); } 37 | 38 | const inline bool is_left() const { return this->m_frame_response.is_left(); } 39 | 40 | // Camera FOV and coordinate of the frame. 41 | const inline nesproto::Camera &get_cam() const { 42 | return this->m_frame_response.camera(); 43 | } 44 | 45 | // Raw RGB frame. 46 | inline types::FrameManager &source_frame_scene() { 47 | return m_source_avframe_scene; 48 | } 49 | 50 | // Converted YUV frame. 51 | inline types::FrameManager &converted_frame_scene() { 52 | return m_converted_avframe_scene; 53 | } 54 | 55 | // Converted YUV depth frame. 56 | inline types::FrameManager &converted_frame_depth() { 57 | return m_converted_avframe_depth; 58 | } 59 | 60 | private: 61 | nesproto::RenderedFrame m_frame_response; 62 | types::FrameManager m_source_avframe_scene; 63 | types::FrameManager m_converted_avframe_scene; 64 | AVPixelFormat m_pix_fmt_scene; 65 | types::FrameManager m_source_avframe_depth; 66 | types::FrameManager m_converted_avframe_depth; 67 | AVPixelFormat m_pix_fmt_depth; 68 | bool m_converted; 69 | }; 70 | 71 | #endif // NES_BASE_RENDERED_FRAME_ 72 | -------------------------------------------------------------------------------- /include/base/video/type_managers.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #ifndef NES_BASE_VIDEO_TYPE_MANAGERS_ 4 | #define NES_BASE_VIDEO_TYPE_MANAGERS_ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "base/logging.h" 13 | 14 | extern "C" { 15 | #include "libavcodec/avcodec.h" // AVPacket, AVCodecContext 16 | #include "libavutil/imgutils.h" // av_image_alloc(), av_image_fill_pointers() 17 | #include "libavutil/opt.h" // av_opt_set() 18 | #include "libswscale/swscale.h" // SwsContext 19 | } 20 | 21 | namespace types { 22 | 23 | // Turn AVError errnum to a human-readable error string. 24 | std::string averror_explain(int errnum); 25 | 26 | // AVPacketManager manages creation and deletion of AVPacket. 27 | class AVPacketManager { 28 | public: 29 | // Allocate AVPacket using av_packet_alloc() and check return value. 30 | AVPacketManager(); 31 | 32 | // Free AVPacket using av_packet_free(). 33 | ~AVPacketManager(); 34 | 35 | // Access the stored AVPacket. 36 | inline AVPacket *operator()() { return m_packet; } 37 | 38 | private: 39 | AVPacket *m_packet; 40 | }; 41 | 42 | // AVDictionaryManager manages creation and deletion of AVDictionary. 43 | class AVDictionaryManager { 44 | public: 45 | // Free AVDictionary using av_dict_free(). 46 | ~AVDictionaryManager(); 47 | 48 | // Access the stored AVDictionary. 49 | inline AVDictionary *operator()() { return m_dict; } 50 | 51 | private: 52 | // AVDictionary will be automatically allocated by av_dict_set(), no need for 53 | // a ctor. 54 | AVDictionary *m_dict = nullptr; 55 | }; 56 | 57 | // AVCodecContextManager manages the lifecycle of a codec. It stores 58 | // configuration state of the encoder in CodecInitInfo, and provides access to 59 | // the information with CodecInfoProvider. It also stores the AVCodecContext, 60 | // which is a gateway to the opened encoder. Access of CodecInitInfo is 61 | // protected with a shared mutex m_codec_info_mutex. Multiple reads 62 | // are allowed but when a thread wants to write, it has to wait for all readers 63 | // to unlock the mutex. The AVCodecContext is protected with a normal mutex 64 | // m_codec_context_mutex to prevent concurrent operations. When the encoder 65 | // needs to be reinitalized (e.g. due to resolution change), the manager 66 | // acquires both the write lock for m_codec_info_mutex and m_codec_context_mutex 67 | // to prevent anyone from accessing the codec. Then the value of CodecInitInfo 68 | // is changed and the encoder is reinitalized according to it. 69 | class AVCodecContextManager { 70 | public: 71 | // CodecInitInfo stores the configuration data for the encoder. This 72 | // information is used to provide current configuration state of the 73 | // encoder to the program and reinitalizing the encoder. 74 | struct CodecInitInfo { 75 | CodecInitInfo(AVCodecID codec_id, AVPixelFormat pix_fmt, 76 | std::string x264_encode_preset, std::string x264_encode_tune, 77 | unsigned width, unsigned height, unsigned bit_rate, 78 | unsigned fps, unsigned keyframe_interval) 79 | : codec_id(codec_id), 80 | pix_fmt(pix_fmt), 81 | x264_encode_preset(x264_encode_preset), 82 | x264_encode_tune(x264_encode_tune), 83 | width(width), 84 | height(height), 85 | bit_rate(bit_rate), 86 | fps(fps), 87 | keyframe_interval(keyframe_interval) {} 88 | AVCodecID codec_id; 89 | AVPixelFormat pix_fmt; 90 | std::string x264_encode_preset; 91 | std::string x264_encode_tune; 92 | unsigned width; 93 | unsigned height; 94 | unsigned bit_rate; 95 | unsigned fps; 96 | unsigned keyframe_interval; 97 | }; 98 | 99 | // CodecInfoProvider encapsulates CodecInitInfo. The reader of CodecInitInfo 100 | // should acquire a shared "read" lock to prevent it from being changed 101 | // because if CodecInitInfo changes (and the encoder reinitializes) while the 102 | // reader processes the data, the reader might feed the encoder with 103 | // incorrectly configured data. This class automates holding and releasing the 104 | // lock. 105 | class CodecInfoProvider { 106 | public: 107 | CodecInfoProvider(CodecInitInfo &info, std::shared_mutex &mutex) 108 | : m_info(std::make_shared(info)), m_lock(mutex) {} 109 | 110 | // Access CodecInitInfo pointer. 111 | inline CodecInitInfo *operator->() { return m_info.get(); } 112 | 113 | private: 114 | std::shared_ptr m_info; 115 | std::shared_lock m_lock; 116 | }; 117 | 118 | AVCodecContextManager(CodecInitInfo info); 119 | 120 | inline CodecInfoProvider get_codec_info() { 121 | return CodecInfoProvider{m_info, m_codec_info_mutex}; 122 | } 123 | 124 | // Locks both m_codec_info_mutex and m_codec_context_mutex 125 | // to ensure no thread is doing any operation while reinitialization and 126 | // updates CodecInitInfo with the requested width and height and calls 127 | // codec_ctx_init(). 128 | void change_resolution(unsigned width, unsigned height); 129 | 130 | // Thread safe wrapper for avcodec_send_frame(). 131 | int send_frame(AVFrame *frm); 132 | 133 | // Thread safe wrapper for avcodec_receive_packet(). 134 | int receive_packet(AVPacket *pkt); 135 | 136 | ~AVCodecContextManager(); 137 | 138 | private: 139 | AVCodecContext *m_ctx; 140 | mutable std::shared_mutex m_codec_info_mutex; 141 | std::mutex m_codec_context_mutex; 142 | std::condition_variable m_codec_context_waiter; 143 | CodecInitInfo m_info; 144 | bool m_opened = false; 145 | using unique_lock = std::unique_lock; 146 | 147 | // Initializes or reinitializes AVCodecContext. 148 | void codec_ctx_init(); 149 | }; 150 | 151 | // FrameManager manages an individual frame in FrameData struct. A frame could 152 | // be a raw RGB frame, a converted YUV frame, or could be empty, waiting to be 153 | // filled. The manager stores FrameContext along with the frame, which specifies 154 | // the properties of the current frame (or the frame that is to be filled 155 | // later). FrameManager can export the frame as libavcodec's AVFrame struct 156 | // using AVFrameWrapper. The wrapper is necessary because we can't have two or 157 | // more AVFrames at the same time. 158 | // XXX: Find whether we could reuse AVFrames. 159 | class FrameManager { 160 | public: 161 | // This value allows the encoder to align the buffer to use fast/aligned SIMD 162 | // routines for data access. An optimal value is 32 (256 bits) which is the 163 | // size of the instruction. 164 | static constexpr unsigned kBufferSizeAlignValueBytes = 32; 165 | 166 | // Stores a frame. 167 | struct FrameData { 168 | uint8_t *data[AV_NUM_DATA_POINTERS] = {0}; 169 | int linesize[AV_NUM_DATA_POINTERS] = {0}; 170 | }; 171 | 172 | // Stores the properties of a frame. 173 | struct FrameContext { 174 | FrameContext(unsigned width, unsigned height, AVPixelFormat pix_fmt) 175 | : width(width), height(height), pix_fmt(pix_fmt) {} 176 | FrameContext(types::AVCodecContextManager::CodecInfoProvider codecinfo) 177 | : width(codecinfo->width), 178 | height(codecinfo->height), 179 | pix_fmt(codecinfo->pix_fmt) {} 180 | unsigned width; 181 | unsigned height; 182 | AVPixelFormat pix_fmt; 183 | }; 184 | 185 | // A wrapper for libavcodec's AVFrame that supports creation using 186 | // FrameData and proper destruction. 187 | class AVFrameWrapper { 188 | public: 189 | // Allocate an AVFrame using FrameData and FrameContext 190 | AVFrameWrapper(FrameData &data, FrameContext &context) { 191 | m_avframe = av_frame_alloc(); 192 | 193 | if (m_avframe == nullptr) { 194 | throw std::runtime_error{"Failed to allocate AVFrame."}; 195 | } 196 | 197 | m_avframe->format = context.pix_fmt; 198 | m_avframe->width = context.width; 199 | m_avframe->height = context.height; 200 | 201 | if (int ret = av_image_alloc(m_avframe->data, m_avframe->linesize, 202 | context.width, context.height, 203 | context.pix_fmt, kBufferSizeAlignValueBytes); 204 | ret < 0) { 205 | throw std::runtime_error{ 206 | std::string("AVFrameWrapper: Failed to allocate AVFrame data: ") + 207 | averror_explain(ret)}; 208 | } 209 | 210 | if (int ret = av_image_fill_pointers(m_avframe->data, context.pix_fmt, 211 | context.height, data.data[0], 212 | data.linesize); 213 | ret < 0) { 214 | throw std::runtime_error{ 215 | std::string("AVFrameWrapper: Failed to fill pointer: ") + 216 | averror_explain(ret)}; 217 | } 218 | } 219 | 220 | // Returns the stored AVFrame. 221 | inline AVFrame *get() { return m_avframe; } 222 | 223 | ~AVFrameWrapper() { 224 | av_frame_unref(m_avframe); 225 | av_freep(&m_avframe->data[0]); 226 | av_frame_free(&m_avframe); 227 | } 228 | 229 | private: 230 | AVFrame *m_avframe; 231 | }; 232 | 233 | FrameManager(FrameContext context, uint8_t *buffer = nullptr); 234 | inline FrameContext &context() { return m_context; } 235 | inline FrameData &data() { return m_data; } 236 | 237 | inline AVFrameWrapper to_avframe() { 238 | return AVFrameWrapper(m_data, m_context); 239 | } 240 | 241 | ~FrameManager(); 242 | 243 | private: 244 | FrameData m_data; 245 | FrameContext m_context; 246 | bool m_should_free_buffer = true; 247 | }; 248 | 249 | // SwsContextManager manages the lifecycle of libswscale. The manager 250 | // initializes a sws context using the values from FrameContext in both source 251 | // and destination. Then it initiates the conversion and after it's done 252 | // destroys the context. 253 | class SwsContextManager { 254 | public: 255 | // Initialize a sws context using the values from FrameContext in both 256 | // frames and initiate the conversion. 257 | SwsContextManager(FrameManager &source, FrameManager &dest); 258 | 259 | // Destroy the sws context. 260 | ~SwsContextManager(); 261 | 262 | private: 263 | struct SwsContext *m_sws_ctx; 264 | }; 265 | 266 | } // namespace types 267 | 268 | #endif // NES_BASE_VIDEO_TYPE_MANAGERS_ 269 | -------------------------------------------------------------------------------- /include/encode.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Moonsik Park. All rights reserved. 3 | * 4 | * @file encode.h 5 | * @author Moonsik Park, Korea Institute of Science and Technology 6 | **/ 7 | 8 | #ifndef _ENCODE_H_ 9 | #define _ENCODE_H_ 10 | 11 | #include "base/video/frame_map.h" 12 | #include "base/video/frame_queue.h" 13 | #include "base/video/render_text.h" 14 | #include "base/video/type_managers.h" 15 | 16 | void process_frame_thread(std::shared_ptr ctxmgr, 17 | std::shared_ptr frame_queue, 18 | std::shared_ptr encode_queue, 19 | std::shared_ptr etctx, 20 | std::atomic &shutdown_requested); 21 | 22 | void send_frame_thread( 23 | std::shared_ptr scene_codecctx, 24 | std::shared_ptr depth_codecctx, 25 | std::shared_ptr encode_queue, 26 | std::atomic &shutdown_requested); 27 | 28 | void receive_packet_thread(std::shared_ptr ctxmgr, 29 | std::shared_ptr mctx, 30 | std::atomic &shutdown_requested); 31 | 32 | void encode_stats_thread(std::atomic &frame_index_left, 33 | std::atomic &frame_index_right, 34 | std::atomic &shutdown_requested); 35 | 36 | #endif // _ENCODE_H_ 37 | -------------------------------------------------------------------------------- /include/server.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Moonsik Park. All rights reserved. 3 | * 4 | * @file server.h 5 | * @author Moonsik Park, Korea Institute of Science and Technology 6 | **/ 7 | 8 | #ifndef _SERVER_H_ 9 | #define _SERVER_H_ 10 | 11 | #include 12 | 13 | #include 14 | 15 | #include "base/camera_manager.h" 16 | #include "base/video/frame_queue.h" 17 | 18 | void socket_main_thread( 19 | std::vector renderers, 20 | std::shared_ptr frame_queue_left, 21 | std::shared_ptr frame_queue_right, 22 | std::atomic &frame_index_left, 23 | std::atomic &frame_index_right, std::atomic &is_left, 24 | std::shared_ptr cameramgr, 25 | std::shared_ptr ctxmgr_scene, 26 | std::shared_ptr ctxmgr_depth, 27 | std::atomic &shutdown_requested); 28 | 29 | void socket_client_thread( 30 | int targetfd, std::shared_ptr frame_queue_left, 31 | std::shared_ptr frame_queue_right, 32 | std::atomic &frame_index_left, 33 | std::atomic &frame_index_right, std::atomic &is_left, 34 | std::shared_ptr cameramgr, 35 | std::shared_ptr ctxmgr_scene, 36 | std::shared_ptr ctxmgr_depth, 37 | std::atomic &shutdown_requested); 38 | 39 | #endif // _SERVER_H_ 40 | -------------------------------------------------------------------------------- /proto/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories(${Protobuf_INCLUDE_DIRS}) 2 | PROTOBUF_GENERATE_CPP(ProtoSources ProtoHeaders nes.proto) 3 | add_library(proto ${ProtoSources} ${ProtoSources}) 4 | target_link_libraries(proto INTERFACE ${Protobuf_LIBRARIES}) 5 | -------------------------------------------------------------------------------- /proto/nes.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package nesproto; 4 | 5 | message Camera { 6 | bool is_left = 2; 7 | uint32 width = 3; 8 | uint32 height = 4; 9 | repeated float matrix = 12; 10 | } 11 | 12 | message FrameRequest { 13 | uint64 index = 1; 14 | Camera camera = 2; 15 | bool is_left = 3; 16 | } 17 | 18 | message RenderedFrame { 19 | uint64 index = 1; 20 | Camera camera = 2; 21 | bool is_left = 3; 22 | 23 | bytes frame = 6; 24 | bytes depth = 7; 25 | } 26 | -------------------------------------------------------------------------------- /src/base/camera_manager.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #include "base/camera_manager.h" 4 | 5 | #include 6 | #include 7 | 8 | #include "base/logging.h" 9 | #include "base/video/type_managers.h" 10 | #include "nes.pb.h" 11 | 12 | CameraManager::CameraManager( 13 | std::shared_ptr codec_scene_left, 14 | std::shared_ptr codec_depth_left, 15 | std::shared_ptr codec_scene_right, 16 | std::shared_ptr codec_depth_right, 17 | uint32_t default_width, uint32_t default_height) 18 | : m_codec_scene_left(codec_scene_left), 19 | m_codec_depth_left(codec_depth_left), 20 | m_codec_scene_right(codec_scene_right), 21 | m_codec_depth_right(codec_depth_right) { 22 | *m_camera_left.mutable_matrix() = {kInitialCameraMatrix, 23 | kInitialCameraMatrix + 12}; 24 | m_camera_left.set_width(default_width); 25 | m_camera_left.set_height(default_height); 26 | 27 | *m_camera_right.mutable_matrix() = {kInitialCameraMatrix, 28 | kInitialCameraMatrix + 12}; 29 | m_camera_right.set_width(default_width); 30 | m_camera_right.set_height(default_height); 31 | } 32 | 33 | void CameraManager::set_camera_left(nesproto::Camera camera) { 34 | if (m_camera_left.width() != camera.width() || 35 | m_camera_left.height() != camera.height()) { 36 | // Resolution changed. Reinitialize the encoder. 37 | 38 | // Resolution must be divisible by 2. 39 | if (camera.width() % 2 != 0) { 40 | camera.set_width(camera.width() - 1); 41 | } 42 | if (camera.height() % 2 != 0) { 43 | camera.set_height(camera.height() - 1); 44 | } 45 | 46 | m_codec_scene_left->change_resolution(camera.width(), camera.height()); 47 | m_codec_depth_left->change_resolution(camera.width(), camera.height()); 48 | } 49 | m_camera_left = camera; 50 | } 51 | 52 | void CameraManager::set_camera_right(nesproto::Camera camera) { 53 | if (m_camera_right.width() != camera.width() || 54 | m_camera_right.height() != camera.height()) { 55 | // Resolution changed. Reinitialize the encoder. 56 | 57 | // Resolution must be divisible by 2. 58 | if (camera.width() % 2 != 0) { 59 | camera.set_width(camera.width() - 1); 60 | } 61 | if (camera.height() % 2 != 0) { 62 | camera.set_height(camera.height() - 1); 63 | } 64 | 65 | m_codec_scene_right->change_resolution(camera.width(), camera.height()); 66 | m_codec_depth_right->change_resolution(camera.width(), camera.height()); 67 | } 68 | m_camera_right = camera; 69 | } 70 | -------------------------------------------------------------------------------- /src/base/server/camera_control.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #include "base/server/camera_control.h" 4 | 5 | #include "base/logging.h" 6 | #include "nes.pb.h" 7 | 8 | CameraControlServer::CameraControlServer( 9 | std::shared_ptr cameramgr, uint16_t bind_port) 10 | : WebSocketServer(std::string("CameraControlServer"), bind_port), 11 | m_camera_manager(cameramgr) {} 12 | 13 | void CameraControlServer::message_handler(websocketpp::connection_hdl hdl, 14 | message_ptr msg) { 15 | nesproto::Camera cam; 16 | 17 | if (cam.ParseFromString(msg->get_raw_payload())) { 18 | if (cam.is_left()) { 19 | m_camera_manager->set_camera_left(cam); 20 | } else { 21 | m_camera_manager->set_camera_right(cam); 22 | } 23 | if (m_message_count % kReceivedLoggingInterval == 0) { 24 | tlog::success() << "CameraControlServer: Receiving camera matrix..."; 25 | } 26 | } else { 27 | tlog::error() << "CameraControlServer: Failed to set camera matrix."; 28 | } 29 | m_message_count++; 30 | } 31 | -------------------------------------------------------------------------------- /src/base/server/packet_stream.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #include "base/server/packet_stream.h" 4 | 5 | #include "base/logging.h" 6 | 7 | extern "C" { 8 | #include "libavcodec/avcodec.h" // AVPacket, AV_PKT_FLAG_KEY 9 | } 10 | 11 | void PacketStreamServer::consume_packet(AVPacket *pkt) { 12 | // TODO: Find better way to indicate key frame. 13 | pkt->data[0] = pkt->flags == AV_PKT_FLAG_KEY ? 0 : 1; 14 | send_to_all((const char *)pkt->data, pkt->size); 15 | // tlog::debug() << "PacketStreamServer: sent packet."; 16 | } 17 | -------------------------------------------------------------------------------- /src/base/server/websocket_server.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #include "base/server/websocket_server.h" 4 | 5 | #include 6 | #include 7 | 8 | #include "base/logging.h" 9 | 10 | WebSocketServer::WebSocketServer(std::string server_name, uint16_t bind_port) 11 | : m_server_name(server_name), m_bind_port(bind_port) { 12 | m_server.clear_access_channels(alevel::all); 13 | m_server.init_asio(); 14 | m_server.set_reuse_addr(true); 15 | 16 | m_server.set_open_handler([&](websocketpp::connection_hdl hdl) { 17 | m_connections.insert(hdl); 18 | tlog::success() << m_server_name << "(" << m_bind_port 19 | << "): Accepted client connection."; 20 | }); 21 | 22 | m_server.set_close_handler([&](websocketpp::connection_hdl hdl) { 23 | m_connections.erase(hdl); 24 | tlog::warning() << m_server_name << "(" << m_bind_port 25 | << "): Client connection closed."; 26 | }); 27 | m_server.set_message_handler( 28 | [&](websocketpp::connection_hdl hdl, message_ptr msg) { 29 | message_handler(hdl, msg); 30 | }); 31 | } 32 | 33 | void WebSocketServer::start() { 34 | if (m_running) { 35 | throw std::runtime_error{ 36 | m_server_name + std::string(" websocket server is already running.")}; 37 | } 38 | m_server.listen(m_bind_port); 39 | m_server.start_accept(); 40 | 41 | m_thread = websocketpp::lib::make_shared(run_server, 42 | &m_server); 43 | m_running = true; 44 | tlog::success() << m_server_name << "(" << m_bind_port 45 | << "): Successfully initialized websocket server."; 46 | } 47 | 48 | void WebSocketServer::stop() { 49 | if (!m_running) { 50 | throw std::runtime_error{m_server_name + 51 | std::string(" websocket server is not running.")}; 52 | } 53 | m_server.stop_listening(); 54 | for (auto it : m_connections) { 55 | m_server.close(it, websocketpp::close::status::going_away, ""); 56 | } 57 | m_thread->join(); 58 | m_running = false; 59 | tlog::info() << m_server_name << "(" << m_bind_port 60 | << "): Successfully closed websocket server."; 61 | } 62 | 63 | void WebSocketServer::send_to_all(const char *data, size_t size) { 64 | if (!m_running) { 65 | throw std::runtime_error{m_server_name + 66 | std::string(" websocket server is not running.")}; 67 | } 68 | for (auto it : m_connections) { 69 | m_server.send(it, data, size, websocketpp::frame::opcode::binary); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/base/video/frame_map.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #include "base/video/frame_map.h" 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #include "base/exceptions/lock_timeout.h" 10 | #include "base/logging.h" 11 | 12 | void FrameMap::insert(FrameMap::keytype index, FrameMap::element&& item) { 13 | unique_lock lock(m_mutex); 14 | // Acquire the lock when the mutex is released and the map is not full. 15 | // If lock timeout is reached, throw LockTimeout exception. 16 | if (m_inserter.wait_for(lock, kFrameMapLockTimeout, 17 | [&] { return m_map.size() < kFrameMapMaxSize; })) { 18 | m_map.insert({index, std::forward(item)}); 19 | // Notify one of the threads waiting to get from the map. 20 | m_getter.notify_all(); 21 | } else { 22 | throw LockTimeout{}; 23 | } 24 | } 25 | 26 | FrameMap::element FrameMap::get_delete(FrameMap::keytype index) { 27 | unique_lock lock(m_mutex); 28 | // Acquire a lock when the mutex is released and there is a frame of requested 29 | // index. If lock timeout is reached, throw LockTimeout exception. 30 | if (m_getter.wait_for(lock, kFrameMapLockTimeout, 31 | [&] { return m_map.contains(index); })) { 32 | element elem = std::move(m_map.at(index)); 33 | m_map.erase(index); 34 | // When interval is reached, delete frames where frame_index < index to 35 | // clean up unused frames. 36 | if (index % kFrameMapDropFramesInterval == 0) { 37 | const auto count = std::erase_if(m_map, [&index](const auto& item) { 38 | auto const& [key, value] = item; 39 | return key < index; 40 | }); 41 | 42 | if (count) { 43 | tlog::error() << "FrameMap: " << count 44 | << " frame(s) dropped; current index=" << index; 45 | } 46 | } 47 | // Notify one of the threads waiting to insert to the map. 48 | m_inserter.notify_one(); 49 | return elem; 50 | } else { 51 | throw LockTimeout{}; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/base/video/frame_queue.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #include "base/video/frame_queue.h" 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #include "base/exceptions/lock_timeout.h" 10 | 11 | void FrameQueue::push(element &&el) { 12 | unique_lock lock(m_mutex); 13 | // Acquire the lock when the mutex is released and the queue is not full. 14 | // If lock timeout is reached, throw LockTimeout exception. 15 | if (m_pusher.wait_for(lock, kFrameQueueLockTimeout, 16 | [&] { return m_queue.size() < kFrameQueueMaxSize; })) { 17 | m_queue.push(std::forward(el)); 18 | // Notify one of the threads waiting to pop from the queue. 19 | m_popper.notify_one(); 20 | } else { 21 | throw LockTimeout{}; 22 | } 23 | } 24 | 25 | FrameQueue::element FrameQueue::pop() { 26 | unique_lock lock(m_mutex); 27 | // Acquire a lock when the mutex is released and the queue is not empty. 28 | // If lock timeout is reached, throw LockTimeout exception. 29 | if (m_popper.wait_for(lock, kFrameQueueLockTimeout, 30 | [&] { return m_queue.size() > 0; })) { 31 | element item = std::move(m_queue.front()); 32 | m_queue.pop(); 33 | // Notify one of the threads waiting to push to the queue. 34 | m_pusher.notify_one(); 35 | return item; 36 | } else { 37 | throw LockTimeout{}; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/base/video/render_text.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #include "base/video/render_text.h" 4 | 5 | extern "C" { 6 | #include "freetype2/ft2build.h" 7 | #include FT_FREETYPE_H 8 | } 9 | 10 | RenderTextContext::RenderTextContext(std::string font_location) { 11 | FT_Error ret; 12 | if ((ret = FT_Init_FreeType(&_library)) < 0) { 13 | throw std::runtime_error{ 14 | std::string("EncodeTextContext: Failed to init freetype: ") + 15 | std::string(FT_Error_String(ret))}; 16 | } 17 | 18 | if ((ret = FT_New_Face(_library, font_location.c_str(), 0, &_face)) < 0) { 19 | throw std::runtime_error{ 20 | std::string("EncodeTextContext: Failed to init font face: ") + 21 | std::string(FT_Error_String(ret))}; 22 | } 23 | if ((ret = FT_Set_Char_Size(_face, /* handle to face object */ 24 | 0, /* char_width in 1/64th of points */ 25 | 20 * 64, /* char_height in 1/64th of points */ 26 | 0, /* horizontal device resolution */ 27 | 0) /* vertical device resolution */ 28 | ) < 0) { 29 | throw std::runtime_error{ 30 | std::string("EncodeTextContext: Failed to set character size: ") + 31 | std::string(FT_Error_String(ret))}; 32 | } 33 | } 34 | 35 | void RenderTextContext::render_string_to_frame( 36 | types::FrameManager &frame, RenderTextContext::RenderPosition opt, 37 | std::string content) { 38 | unique_lock lock(m_mutex); 39 | m_wait.wait(lock, [] { return true; }); 40 | uint8_t *surface = frame.data().data[0]; 41 | uint32_t width = frame.context().width; 42 | uint32_t height = frame.context().height; 43 | FT_Error ret; 44 | FT_GlyphSlot slot = _face->glyph; 45 | FT_UInt glyph_index; 46 | 47 | int pen_x, pen_y, n; 48 | int x_box = 300; 49 | int y_box = 100; 50 | int margin = 50; 51 | 52 | switch (opt) { 53 | case RenderPosition::RENDER_POSITION_LEFT_TOP: 54 | pen_x = margin; 55 | pen_y = margin; 56 | break; 57 | case RenderPosition::RENDER_POSITION_LEFT_BOTTOM: 58 | pen_x = margin; 59 | pen_y = height - y_box + margin; 60 | break; 61 | case RenderPosition::RENDER_POSITION_RIGHT_TOP: 62 | pen_x = width - x_box + margin; 63 | pen_y = margin; 64 | break; 65 | case RenderPosition::RENDER_POSITION_RIGHT_BOTTOM: 66 | pen_x = width - x_box + margin; 67 | pen_y = height - y_box + margin; 68 | break; 69 | case RenderPosition::RENDER_POSITION_CENTER: 70 | pen_x = width / 2 - x_box; 71 | pen_y = height / 2 - y_box; 72 | break; 73 | default: 74 | pen_x = margin; 75 | pen_y = margin; 76 | break; 77 | } 78 | 79 | int orig_pen_x = pen_x; 80 | 81 | for (auto &ch : content) { 82 | if (ch == '\n') { 83 | pen_x = orig_pen_x; 84 | pen_y = pen_y + 20; 85 | continue; 86 | } 87 | /* load glyph image into the slot (erase previous one) */ 88 | ret = FT_Load_Char(_face, ch, FT_LOAD_RENDER); 89 | if (ret) { 90 | tlog::info() << "Error while rendering character=" << ch 91 | << " error=" << ret; 92 | } 93 | 94 | FT_Int i, j, p, q; 95 | FT_Int x_max = pen_x + slot->bitmap_left + slot->bitmap.width; 96 | FT_Int y_max = pen_y - slot->bitmap_top + slot->bitmap.rows; 97 | for (j = pen_y - slot->bitmap_top, q = 0; j < y_max; j++, q++) { 98 | for (i = pen_x + slot->bitmap_left, p = 0; i < x_max; i++, p++) { 99 | if (i < 0 || j < 0 || i >= width || j >= height) continue; 100 | if (slot->bitmap.buffer[q * slot->bitmap.width + p]) { 101 | surface[(j * width + i) * 3] = 255; 102 | surface[(j * width + i) * 3 + 1] = 255; 103 | surface[(j * width + i) * 3 + 2] = 255; 104 | } 105 | } 106 | } 107 | 108 | /* increment pen position */ 109 | pen_x += slot->advance.x >> 6; 110 | } 111 | } 112 | 113 | RenderTextContext::~RenderTextContext() { FT_Done_FreeType(_library); } 114 | -------------------------------------------------------------------------------- /src/base/video/rendered_frame.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #include "base/video/rendered_frame.h" 4 | 5 | RenderedFrame::RenderedFrame( 6 | nesproto::RenderedFrame frame, AVPixelFormat pix_fmt_scene, 7 | AVPixelFormat pix_fmt_depth, 8 | std::shared_ptr ctxmgr_scene, 9 | std::shared_ptr ctxmgr_depth) 10 | : m_frame_response(frame), 11 | m_pix_fmt_scene(pix_fmt_scene), 12 | m_pix_fmt_depth(pix_fmt_depth), 13 | m_converted(false), 14 | m_source_avframe_scene( 15 | types::FrameManager::FrameContext(m_frame_response.camera().width(), 16 | m_frame_response.camera().height(), 17 | pix_fmt_scene), 18 | (uint8_t *)m_frame_response.frame().data()), 19 | m_converted_avframe_scene( 20 | types::FrameManager::FrameContext(ctxmgr_scene->get_codec_info())), 21 | m_source_avframe_depth( 22 | types::FrameManager::FrameContext(m_frame_response.camera().width(), 23 | m_frame_response.camera().height(), 24 | pix_fmt_depth), 25 | (uint8_t *)m_frame_response.depth().data()), 26 | m_converted_avframe_depth( 27 | types::FrameManager::FrameContext(ctxmgr_depth->get_codec_info())) {} 28 | -------------------------------------------------------------------------------- /src/base/video/type_managers.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Moonsik Park. 2 | 3 | #include "base/video/type_managers.h" 4 | 5 | extern "C" { 6 | #include "libavcodec/avcodec.h" 7 | // avcodec_free_context(), avcodec_find_encoder(), avcodec_alloc_context3(), 8 | // avcodec_open2(), avcodec_send_frame(), avcodec_receive_packet(), 9 | // avcodec_free_context() 10 | #include "libavutil/dict.h" // av_dict_set() 11 | #include "libavutil/error.h" // av_strerror() 12 | #include "libavutil/imgutils.h" // av_image_alloc(), av_image_get_linesize() 13 | #include "libavutil/mem.h" // av_freep() 14 | #include "libavutil/opt.h" // av_opt_set() 15 | #include "libswscale/swscale.h" // sws_getContext(), sws_scale() 16 | } 17 | 18 | namespace types { 19 | 20 | constexpr int kAVErrorExplainBufferLength = 200; 21 | 22 | std::string averror_explain(int errnum) { 23 | char errbuf[kAVErrorExplainBufferLength]; 24 | if (av_strerror(errnum, errbuf, kAVErrorExplainBufferLength) < 0) { 25 | return std::string{""}; 26 | } 27 | return std::string(errbuf); 28 | } 29 | 30 | AVPacketManager::AVPacketManager() { 31 | m_packet = av_packet_alloc(); 32 | if (!m_packet) { 33 | throw std::runtime_error{"Failed to allocate AVPacket."}; 34 | } 35 | } 36 | 37 | AVPacketManager::~AVPacketManager() { av_packet_free(&m_packet); } 38 | 39 | AVDictionaryManager::~AVDictionaryManager() { av_dict_free(&m_dict); } 40 | 41 | AVCodecContextManager::AVCodecContextManager( 42 | AVCodecContextManager::CodecInitInfo info) 43 | : m_info(info) { 44 | this->codec_ctx_init(); 45 | } 46 | 47 | void AVCodecContextManager::codec_ctx_init() { 48 | if (m_opened) { 49 | avcodec_free_context(&m_ctx); 50 | } 51 | 52 | AVCodec *codec = avcodec_find_encoder(m_info.codec_id); 53 | if (codec == nullptr) { 54 | throw std::runtime_error{"Failed to find encoder."}; 55 | } 56 | 57 | m_ctx = avcodec_alloc_context3(codec); 58 | if (m_ctx == nullptr) { 59 | throw std::runtime_error{"Failed to allocate codec context."}; 60 | } 61 | 62 | m_ctx->bit_rate = m_info.bit_rate; 63 | m_ctx->width = m_info.width; 64 | m_ctx->height = m_info.height; 65 | m_ctx->time_base = (AVRational){1, (int)m_info.fps}; 66 | m_ctx->pix_fmt = m_info.pix_fmt; 67 | 68 | av_opt_set(m_ctx->priv_data, "x264opts", "keyint", m_info.keyframe_interval); 69 | 70 | { 71 | AVDictionaryManager dict; 72 | AVDictionary *options = dict(); 73 | av_dict_set(&options, "preset", m_info.x264_encode_preset.c_str(), 0); 74 | av_dict_set(&options, "tune", m_info.x264_encode_tune.c_str(), 0); 75 | 76 | if (int ret = avcodec_open2(m_ctx, (const AVCodec *)codec, &options); 77 | ret < 0) { 78 | throw std::runtime_error{std::string{"Failed to open codec: "} + 79 | averror_explain(ret)}; 80 | } 81 | } // Context for AVDictionaryManager 82 | m_opened = true; 83 | tlog::debug() << "codec_ctx_init() success; width=" << m_info.width 84 | << " height=" << m_info.height 85 | << " bit_rate=" << m_info.bit_rate << " fps=" << m_info.fps 86 | << " keyframe_interval=" << m_info.keyframe_interval; 87 | } 88 | 89 | void AVCodecContextManager::change_resolution(unsigned width, unsigned height) { 90 | std::scoped_lock lock{m_codec_info_mutex, m_codec_context_mutex}; 91 | m_info.width = width; 92 | m_info.height = height; 93 | this->codec_ctx_init(); 94 | } 95 | 96 | int AVCodecContextManager::send_frame(AVFrame *frm) { 97 | unique_lock lock{m_codec_context_mutex}; 98 | m_codec_context_waiter.wait(lock, [] { return true; }); 99 | int ret = avcodec_send_frame(m_ctx, frm); 100 | m_codec_context_waiter.notify_one(); 101 | return ret; 102 | } 103 | 104 | int AVCodecContextManager::receive_packet(AVPacket *pkt) { 105 | unique_lock lock{m_codec_context_mutex}; 106 | m_codec_context_waiter.wait(lock, [] { return true; }); 107 | int ret = avcodec_receive_packet(m_ctx, pkt); 108 | m_codec_context_waiter.notify_one(); 109 | return ret; 110 | } 111 | 112 | AVCodecContextManager::~AVCodecContextManager() { 113 | avcodec_free_context(&m_ctx); 114 | } 115 | 116 | FrameManager::FrameManager(FrameContext context, uint8_t *buffer) 117 | : m_context(context) { 118 | if (buffer == nullptr) { 119 | if (int ret = av_image_alloc(m_data.data, m_data.linesize, context.width, 120 | context.height, context.pix_fmt, 121 | kBufferSizeAlignValueBytes); 122 | ret < 0) { 123 | std::runtime_error{std::string("Failed to allocate frame data: ") + 124 | averror_explain(ret)}; 125 | } 126 | } else { 127 | m_should_free_buffer = false; 128 | const AVPixFmtDescriptor *in_pixfmt = av_pix_fmt_desc_get(context.pix_fmt); 129 | for (int plane = 0; plane < in_pixfmt->nb_components; plane++) { 130 | m_data.linesize[plane] = 131 | av_image_get_linesize(context.pix_fmt, context.width, plane); 132 | } 133 | m_data.data[0] = {(uint8_t *)buffer}; 134 | } 135 | } 136 | 137 | FrameManager::~FrameManager() { 138 | if (m_should_free_buffer) { 139 | av_freep(&m_data.data[0]); 140 | } 141 | } 142 | 143 | SwsContextManager::SwsContextManager(FrameManager &source, FrameManager &dest) { 144 | m_sws_ctx = 145 | sws_getContext(source.context().width, source.context().height, 146 | source.context().pix_fmt, dest.context().width, 147 | dest.context().height, dest.context().pix_fmt, 0, 0, 0, 0); 148 | if (!m_sws_ctx) { 149 | throw std::runtime_error{"Failed to allocate sws_context."}; 150 | } 151 | sws_scale(m_sws_ctx, source.data().data, source.data().linesize, 0, 152 | source.context().height, dest.data().data, dest.data().linesize); 153 | } 154 | 155 | SwsContextManager::~SwsContextManager() { sws_freeContext(m_sws_ctx); } 156 | } // namespace types 157 | -------------------------------------------------------------------------------- /src/encode.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Moonsik Park. All rights reserved. 3 | * 4 | * @file encode.cpp 5 | * @author Moonsik Park, Korea Institute of Science and Technology 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "base/camera_manager.h" 15 | #include "base/exceptions/lock_timeout.h" 16 | #include "base/scoped_timer.h" 17 | #include "base/server/packet_stream.h" 18 | #include "base/video/frame_map.h" 19 | #include "base/video/frame_queue.h" 20 | #include "base/video/render_text.h" 21 | #include "base/video/type_managers.h" 22 | 23 | std::string timestamp() { 24 | using namespace std::chrono; 25 | using clock = system_clock; 26 | 27 | const auto current_time_point{clock::now()}; 28 | const auto current_time{clock::to_time_t(current_time_point)}; 29 | const auto current_localtime{*std::localtime(¤t_time)}; 30 | const auto current_time_since_epoch{current_time_point.time_since_epoch()}; 31 | const auto current_milliseconds{ 32 | duration_cast(current_time_since_epoch).count() % 1000}; 33 | 34 | std::ostringstream stream; 35 | stream << std::put_time(¤t_localtime, "%T") << "." << std::setw(3) 36 | << std::setfill('0') << current_milliseconds; 37 | return stream.str(); 38 | } 39 | 40 | static constexpr unsigned kLogStatsIntervalFrame = 100; 41 | 42 | void process_frame_thread(std::shared_ptr ctxmgr, 43 | std::shared_ptr frame_queue, 44 | std::shared_ptr encode_queue, 45 | std::shared_ptr etctx, 46 | std::atomic &shutdown_requested) { 47 | // set_thread_name("process_frame"); 48 | int ret; 49 | unsigned index = 0; 50 | uint64_t elapsed = 0; 51 | while (!shutdown_requested) { 52 | try { 53 | std::unique_ptr frame = frame_queue->pop(); 54 | { 55 | ScopedTimer timer; 56 | uint64_t frame_index = frame->index(); 57 | std::stringstream cam_matrix; 58 | int idx = 0; 59 | for (auto it : frame->get_cam().matrix()) { 60 | idx++; 61 | cam_matrix << std::fixed << std::showpos << std::setw(7) 62 | << std::setprecision(5) << std::setfill('0') << it << ' '; 63 | if (idx % 4 == 0) { 64 | cam_matrix << '\n'; 65 | } 66 | } 67 | cam_matrix << std::fixed << std::showpos << std::setw(7) 68 | << std::setprecision(5) << std::setfill('0') << 0.f << ' ' 69 | << std::fixed << std::showpos << std::setw(7) 70 | << std::setprecision(5) << std::setfill('0') << 0.f << ' ' 71 | << std::fixed << std::showpos << std::setw(7) 72 | << std::setprecision(5) << std::setfill('0') << 0.f << ' ' 73 | << std::fixed << std::showpos << std::setw(7) 74 | << std::setprecision(5) << std::setfill('0') << 1.f << ' '; 75 | 76 | etctx->render_string_to_frame( 77 | frame->source_frame_scene(), 78 | RenderTextContext::RenderPosition::RENDER_POSITION_CENTER, 79 | cam_matrix.str()); 80 | 81 | etctx->render_string_to_frame( 82 | frame->source_frame_scene(), 83 | RenderTextContext::RenderPosition::RENDER_POSITION_LEFT_BOTTOM, 84 | std::string("index=") + std::to_string(frame->index())); 85 | 86 | etctx->render_string_to_frame( 87 | frame->source_frame_scene(), 88 | RenderTextContext::RenderPosition::RENDER_POSITION_LEFT_TOP, 89 | timestamp()); 90 | 91 | std::string direction = 92 | frame->is_left() ? "direction=left" : "direction=right"; 93 | 94 | etctx->render_string_to_frame( 95 | frame->source_frame_scene(), 96 | RenderTextContext::RenderPosition::RENDER_POSITION_RIGHT_TOP, 97 | direction); 98 | frame->convert_frame(); 99 | 100 | encode_queue->insert(frame_index, std::move(frame)); 101 | index++; 102 | elapsed += timer.elapsed().count(); 103 | 104 | if (index == kLogStatsIntervalFrame) { 105 | tlog::info() 106 | << "process_frame_thread: frame processing average time of " 107 | << kLogStatsIntervalFrame 108 | << " frames: " << elapsed / kLogStatsIntervalFrame << " msec."; 109 | index = 0; 110 | elapsed = 0; 111 | } 112 | } 113 | } catch (const LockTimeout &) { 114 | continue; 115 | } 116 | } 117 | 118 | tlog::info() << "process_frame_thread: Exiting thread."; 119 | } 120 | 121 | void send_frame_thread( 122 | std::shared_ptr scene_codecctx, 123 | std::shared_ptr depth_codecctx, 124 | std::shared_ptr encode_queue, 125 | std::atomic &shutdown_requested) { 126 | // set_thread_name("send_frame"); 127 | uint64_t frame_index = 0; 128 | unsigned index = 0; 129 | uint64_t elapsed = 0; 130 | while (!shutdown_requested) { 131 | try { 132 | ScopedTimer timer; 133 | std::unique_ptr processed_frame = 134 | encode_queue->get_delete(frame_index); 135 | 136 | switch (scene_codecctx->send_frame( 137 | processed_frame->converted_frame_scene().to_avframe().get())) { 138 | case AVERROR(EINVAL): 139 | // Codec not opened, it is a decoder, or requires flush. 140 | std::runtime_error{ 141 | "Codec not opened, it is a decoder, or requires flush."}; 142 | break; 143 | case AVERROR(ENOMEM): 144 | // Failed to add packet to internal queue, or similar other errors: 145 | // legitimate encoding errors. 146 | std::runtime_error{ 147 | "Failed to add packet to internal queue, or other."}; 148 | break; 149 | case AVERROR_EOF: 150 | // The encoder has been flushed, and no new frames can be sent to it. 151 | std::runtime_error{ 152 | "The encoder has been flushed, and no new frames can be sent to " 153 | "it."}; 154 | break; 155 | case AVERROR(EAGAIN): 156 | // Input is not accepted in the current state - user must read output 157 | // with avcodec_receive_packet() (once all output is read, the packet 158 | // should be resent, and the call will not fail with EAGAIN). 159 | default: 160 | // Success. 161 | break; 162 | } 163 | 164 | switch (depth_codecctx->send_frame( 165 | processed_frame->converted_frame_depth().to_avframe().get())) { 166 | case AVERROR(EINVAL): 167 | // Codec not opened, it is a decoder, or requires flush. 168 | std::runtime_error{ 169 | "Codec not opened, it is a decoder, or requires flush."}; 170 | break; 171 | case AVERROR(ENOMEM): 172 | // Failed to add packet to internal queue, or similar other errors: 173 | // legitimate encoding errors. 174 | std::runtime_error{ 175 | "Failed to add packet to internal queue, or other."}; 176 | break; 177 | case AVERROR_EOF: 178 | // The encoder has been flushed, and no new frames can be sent to it. 179 | std::runtime_error{ 180 | "The encoder has been flushed, and no new frames can be sent to " 181 | "it."}; 182 | break; 183 | case AVERROR(EAGAIN): 184 | // Input is not accepted in the current state - user must read output 185 | // with avcodec_receive_packet() (once all output is read, the packet 186 | // should be resent, and the call will not fail with EAGAIN). 187 | default: 188 | // Success. 189 | break; 190 | } 191 | index++; 192 | elapsed += timer.elapsed().count(); 193 | 194 | if (index == kLogStatsIntervalFrame) { 195 | tlog::info() << "send_frame_thread: average time of sending frame to " 196 | "encoder of " 197 | << kLogStatsIntervalFrame 198 | << " frames: " << elapsed / kLogStatsIntervalFrame 199 | << " msec."; 200 | index = 0; 201 | elapsed = 0; 202 | } 203 | } catch (const LockTimeout &) { 204 | // If the frame is not located until timeout, go to next frame. 205 | tlog::error() << "send_frame_thread (index=" << frame_index 206 | << "): Timeout reached while waiting for frame. Skipping."; 207 | } 208 | frame_index++; 209 | } 210 | 211 | tlog::info() << "send_frame_thread: Exiting thread."; 212 | } 213 | 214 | int receive_packet_handler(std::shared_ptr ctxmgr, 215 | AVPacket *pkt, 216 | std::shared_ptr mctx, 217 | std::atomic &shutdown_requested) { 218 | int ret; 219 | 220 | while (!shutdown_requested) { 221 | ret = ctxmgr->receive_packet(pkt); 222 | 223 | switch (ret) { 224 | case AVERROR(EAGAIN): // output is not available in the current state - 225 | // user must try to send input 226 | // We must sleep here so that other threads can acquire 227 | // AVCodecContext. 228 | std::this_thread::sleep_for(std::chrono::milliseconds(1)); 229 | break; 230 | case 0: 231 | mctx->consume_packet(pkt); 232 | return 0; 233 | case AVERROR(EINVAL): // codec not opened, or it is a decoder other 234 | // errors: legitimate encoding errors 235 | default: 236 | tlog::error() << "receive_packet_handler: Failed to receive " 237 | "packet: " 238 | /* << averror_explain(ret)*/; 239 | case AVERROR_EOF: // the encoder has been fully flushed, and there will 240 | // be no more output packets 241 | return -1; 242 | } 243 | } 244 | 245 | return -1; 246 | } 247 | 248 | void receive_packet_thread(std::shared_ptr ctxmgr, 249 | std::shared_ptr mctx, 250 | std::atomic &shutdown_requested) { 251 | // set_thread_name("receive_packet"); 252 | while (!shutdown_requested) { 253 | types::AVPacketManager pkt; 254 | try { 255 | if (receive_packet_handler(ctxmgr, pkt(), mctx, 256 | std::ref(shutdown_requested)) < 0) { 257 | shutdown_requested = true; 258 | } 259 | } catch (const LockTimeout &) { 260 | tlog::info() << "receive_packet_thread: lock_timeout while acquiring " 261 | "resource lock for AVCodecContext."; 262 | break; 263 | } 264 | std::this_thread::sleep_for(std::chrono::milliseconds(1)); 265 | } 266 | 267 | tlog::info() << "receive_packet_thread: Exiting thread."; 268 | } 269 | 270 | static constexpr unsigned kEncodeStatsLogIntervalSeconds = 10; 271 | 272 | void encode_stats_thread(std::atomic &frame_index_left, 273 | std::atomic &frame_index_right, 274 | std::atomic &shutdown_requested) { 275 | // set_thread_name("encode_stats"); 276 | uint64_t previous_index = 0; 277 | uint64_t seconds = 0; 278 | while (!shutdown_requested) { 279 | seconds++; 280 | uint64_t current_index = frame_index_left.load() + frame_index_right.load(); 281 | if (seconds == kEncodeStatsLogIntervalSeconds) { 282 | tlog::info() << "encode_stats_thread: Average frame rate of the last " 283 | << kEncodeStatsLogIntervalSeconds 284 | << " seconds: " << (current_index - previous_index) / seconds 285 | << " fps."; 286 | previous_index = current_index; 287 | seconds = 0; 288 | } 289 | std::this_thread::sleep_for(std::chrono::seconds(1)); 290 | } 291 | tlog::info() << "encode_stats_thread: Exiting thread."; 292 | } 293 | -------------------------------------------------------------------------------- /src/main.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Moonsik Park. All rights reserved. 3 | * 4 | * @file main.cpp 5 | * @author Moonsik Park, Korea Institute of Science and Technology 6 | **/ 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "base/camera_manager.h" 15 | #include "base/server/camera_control.h" 16 | #include "base/server/packet_stream.h" 17 | #include "base/video/frame_queue.h" 18 | #include "base/video/render_text.h" 19 | #include "base/video/type_managers.h" 20 | #include "encode.h" 21 | #include "server.h" 22 | 23 | using namespace args; 24 | 25 | namespace { 26 | 27 | volatile std::sig_atomic_t signal_status = 0; 28 | 29 | static_assert(std::atomic::is_always_lock_free); 30 | 31 | std::atomic shutdown_requested{false}; 32 | 33 | } // namespace 34 | 35 | void signal_handler(int signum) { 36 | signal_status = signum; 37 | shutdown_requested.store(true); 38 | } 39 | 40 | void set_thread_name(std::string name) { prctl(PR_SET_NAME, name.c_str()); } 41 | 42 | int main(int argc, char **argv) { 43 | // set_thread_name("main"); 44 | GOOGLE_PROTOBUF_VERIFY_VERSION; 45 | std::signal(SIGINT, signal_handler); 46 | 47 | try { 48 | ArgumentParser parser{ 49 | "ngp encode server\n" 50 | "version 1.0" 51 | "", 52 | }; 53 | 54 | HelpFlag help_flag{ 55 | parser, 56 | "HELP", 57 | "Display help.", 58 | {'h', "help"}, 59 | }; 60 | 61 | Flag version_flag{ 62 | parser, 63 | "VERSION", 64 | "Display the version of ngp encode server.", 65 | {'v', "version"}, 66 | }; 67 | 68 | ValueFlagList renderer_addr_flag{ 69 | parser, 70 | "RENDERER_ADDR", 71 | "Address(es) of the renderers.", 72 | {"r", "renderer"}}; 73 | 74 | ValueFlag address_flag{ 75 | parser, "BIND_ADDRESS", "Address to bind to.", 76 | {'a', "address"}, "0.0.0.0", 77 | }; 78 | 79 | ValueFlag port_flag{ 80 | parser, "BIND_PORT", "Port to bind to.", {"p", "port"}, 9991, 81 | }; 82 | 83 | ValueFlag encode_preset_flag{ 84 | parser, 85 | "ENCODE_PRESET", 86 | "Encode preset {ultrafast, superfast, veryfast, faster, fast, medium, " 87 | "slow, slower, veryslow (default), placebo}", 88 | {"encode_preset"}, 89 | "ultrafast", 90 | }; 91 | 92 | ValueFlag encode_tune_flag{ 93 | parser, 94 | "ENCODE_TUNE", 95 | "Encode tune {film, animation, grain, stillimage, fastdecode, " 96 | "zerolatency, psnr, ssim}. default: stillimage,zerolatency", 97 | {'t', "encode_tune"}, 98 | "stillimage,zerolatency", 99 | }; 100 | 101 | ValueFlag width_flag{ 102 | parser, "WIDTH", "Width of requesting image.", {"width"}, 1280, 103 | }; 104 | 105 | ValueFlag height_flag{ 106 | parser, "HEIGHT", "Height of requesting image.", {"height"}, 720, 107 | }; 108 | 109 | ValueFlag bitrate_flag{ 110 | parser, "BITRATE", "Bitrate of output stream.", {"bitrate"}, 400000, 111 | }; 112 | 113 | ValueFlag fps_flag{ 114 | parser, 115 | "FPS", 116 | "Frame per second of output stream. This does not guarantee that n " 117 | "frames will be present.", 118 | {"fps"}, 119 | 30, 120 | }; 121 | ValueFlag keyint_flag{ 122 | parser, "KEYINT", "Group of picture (GOP) size", {"keyint"}, 250, 123 | }; 124 | 125 | ValueFlag font_flag{ 126 | parser, 127 | "FONT", 128 | "Location of a font file used to render texts.", 129 | {"font"}, 130 | "/usr/share/fonts/truetype/noto/NotoMono-Regular.ttf", 131 | }; 132 | 133 | ValueFlag camera_control_server_port{ 134 | parser, 135 | "CAMERA_CONTROL_SERVER_PORT", 136 | "Port the camera control websocket server should bind to.", 137 | {"camera_control_server_port"}, 138 | 9998, 139 | }; 140 | 141 | ValueFlag server_packet_stream_scene_left_port{ 142 | parser, 143 | "SCENE_PACKET_STREAM_SERVER_PORT", 144 | "Port the scene packet stream websocket server should bind to.", 145 | {"server_packet_stream_scene_left_port"}, 146 | 9999, 147 | }; 148 | 149 | ValueFlag server_packet_stream_depth_left_port{ 150 | parser, 151 | "DEPTH_PACKET_STREAM_SERVER_PORT", 152 | "Port the depth packet stream websocket server should bind to.", 153 | {"server_packet_stream_depth_left_port"}, 154 | 10000, 155 | }; 156 | 157 | ValueFlag server_packet_stream_scene_right_port{ 158 | parser, 159 | "SCENE_PACKET_STREAM_SERVER_PORT", 160 | "Port the scene packet stream websocket server should bind to.", 161 | {"server_packet_stream_scene_right_port"}, 162 | 10001, 163 | }; 164 | 165 | ValueFlag server_packet_stream_depth_right_port{ 166 | parser, 167 | "DEPTH_PACKET_STREAM_SERVER_PORT", 168 | "Port the depth packet stream websocket server should bind to.", 169 | {"server_packet_stream_depth_right_port"}, 170 | 10002, 171 | }; 172 | 173 | try { 174 | parser.ParseCLI(argc, argv); 175 | } catch (const Help &) { 176 | std::cout << parser; 177 | return 0; 178 | } catch (const ParseError &e) { 179 | std::cerr << e.what() << std::endl; 180 | std::cerr << parser; 181 | return -1; 182 | } catch (const ValidationError &e) { 183 | std::cerr << e.what() << std::endl; 184 | std::cerr << parser; 185 | return -2; 186 | } 187 | 188 | if (version_flag) { 189 | tlog::info() << "ngp encode server version 1.0"; 190 | return 0; 191 | } 192 | 193 | tlog::info() << "Initalizing encoder."; 194 | 195 | auto codec_scene_left = std::make_shared( 196 | types::AVCodecContextManager::CodecInitInfo( 197 | AV_CODEC_ID_H264, AV_PIX_FMT_YUV420P, get(encode_preset_flag), 198 | get(encode_tune_flag), get(width_flag), get(height_flag), 199 | get(bitrate_flag), get(fps_flag), get(keyint_flag))); 200 | 201 | auto codec_depth_left = std::make_shared( 202 | types::AVCodecContextManager::CodecInitInfo( 203 | AV_CODEC_ID_H264, AV_PIX_FMT_YUV420P, get(encode_preset_flag), 204 | get(encode_tune_flag), get(width_flag), get(height_flag), 205 | get(bitrate_flag), get(fps_flag), get(keyint_flag))); 206 | 207 | auto codec_scene_right = std::make_shared( 208 | types::AVCodecContextManager::CodecInitInfo( 209 | AV_CODEC_ID_H264, AV_PIX_FMT_YUV420P, get(encode_preset_flag), 210 | get(encode_tune_flag), get(width_flag), get(height_flag), 211 | get(bitrate_flag), get(fps_flag), get(keyint_flag))); 212 | 213 | auto codec_depth_right = std::make_shared( 214 | types::AVCodecContextManager::CodecInitInfo( 215 | AV_CODEC_ID_H264, AV_PIX_FMT_YUV420P, get(encode_preset_flag), 216 | get(encode_tune_flag), get(width_flag), get(height_flag), 217 | get(bitrate_flag), get(fps_flag), get(keyint_flag))); 218 | 219 | auto etctx = std::make_shared(get(font_flag)); 220 | tlog::info() << "Initialized text renderer."; 221 | 222 | auto server_packet_stream_scene_left = std::make_shared( 223 | get(server_packet_stream_scene_left_port), 224 | std::string("server_packet_stream_scene_left")); 225 | 226 | auto server_packet_stream_depth_left = std::make_shared( 227 | get(server_packet_stream_depth_left_port), 228 | std::string("server_packet_stream_depth_left")); 229 | 230 | auto server_packet_stream_scene_right = 231 | std::make_shared( 232 | get(server_packet_stream_scene_right_port), 233 | std::string("server_packet_stream_scene_right")); 234 | 235 | auto server_packet_stream_depth_right = 236 | std::make_shared( 237 | get(server_packet_stream_depth_right_port), 238 | std::string("server_packet_stream_depth_right")); 239 | 240 | server_packet_stream_scene_left->start(); 241 | server_packet_stream_depth_left->start(); 242 | server_packet_stream_scene_right->start(); 243 | server_packet_stream_depth_right->start(); 244 | 245 | tlog::info() << "Initalizing queue."; 246 | auto frame_queue_left = std::make_shared(); 247 | auto frame_map_left = std::make_shared(); 248 | auto frame_queue_right = std::make_shared(); 249 | auto frame_map_right = std::make_shared(); 250 | auto cameramgr = std::make_shared( 251 | codec_scene_left, codec_depth_left, codec_scene_right, 252 | codec_depth_right, get(width_flag), get(height_flag)); 253 | 254 | tlog::info() << "Initalizing camera control server."; 255 | auto ccsvr = std::make_shared( 256 | cameramgr, get(camera_control_server_port)); 257 | ccsvr->start(); 258 | 259 | std::atomic frame_index_left = 0; 260 | std::atomic frame_index_right = 0; 261 | std::atomic is_left{0}; 262 | 263 | tlog::info() << "Done bootstrapping."; 264 | 265 | std::vector threads; 266 | 267 | std::thread _socket_main_thread( 268 | socket_main_thread, get(renderer_addr_flag), frame_queue_left, 269 | frame_queue_right, std::ref(frame_index_left), 270 | std::ref(frame_index_right), std::ref(is_left), cameramgr, 271 | codec_scene_left, codec_depth_left, std::ref(shutdown_requested)); 272 | threads.push_back(std::move(_socket_main_thread)); 273 | 274 | std::thread _process_frame_thread_left( 275 | process_frame_thread, codec_scene_left, frame_queue_left, 276 | frame_map_left, etctx, std::ref(shutdown_requested)); 277 | threads.push_back(std::move(_process_frame_thread_left)); 278 | 279 | std::thread _process_frame_thread_right( 280 | process_frame_thread, codec_scene_right, frame_queue_right, 281 | frame_map_right, etctx, std::ref(shutdown_requested)); 282 | threads.push_back(std::move(_process_frame_thread_right)); 283 | 284 | std::thread _receive_packet_thread_scene_left( 285 | receive_packet_thread, codec_scene_left, 286 | server_packet_stream_scene_left, std::ref(shutdown_requested)); 287 | threads.push_back(std::move(_receive_packet_thread_scene_left)); 288 | 289 | std::thread _receive_packet_thread_depth_left( 290 | receive_packet_thread, codec_depth_left, 291 | server_packet_stream_depth_left, std::ref(shutdown_requested)); 292 | threads.push_back(std::move(_receive_packet_thread_depth_left)); 293 | 294 | std::thread _receive_packet_thread_scene_right( 295 | receive_packet_thread, codec_scene_right, 296 | server_packet_stream_scene_right, std::ref(shutdown_requested)); 297 | threads.push_back(std::move(_receive_packet_thread_scene_right)); 298 | 299 | std::thread _receive_packet_thread_depth_right( 300 | receive_packet_thread, codec_depth_right, 301 | server_packet_stream_depth_right, std::ref(shutdown_requested)); 302 | threads.push_back(std::move(_receive_packet_thread_depth_right)); 303 | 304 | std::thread _send_frame_thread_left(send_frame_thread, codec_scene_left, 305 | codec_depth_left, frame_map_left, 306 | std::ref(shutdown_requested)); 307 | threads.push_back(std::move(_send_frame_thread_left)); 308 | 309 | std::thread _send_frame_thread_right(send_frame_thread, codec_scene_right, 310 | codec_depth_right, frame_map_right, 311 | std::ref(shutdown_requested)); 312 | threads.push_back(std::move(_send_frame_thread_right)); 313 | 314 | std::thread _encode_stats_thread( 315 | encode_stats_thread, std::ref(frame_index_left), 316 | std::ref(frame_index_right), std::ref(shutdown_requested)); 317 | threads.push_back(std::move(_encode_stats_thread)); 318 | 319 | for (auto &th : threads) { 320 | th.join(); 321 | } 322 | 323 | server_packet_stream_scene_left->stop(); 324 | server_packet_stream_depth_left->stop(); 325 | server_packet_stream_scene_right->stop(); 326 | server_packet_stream_depth_right->stop(); 327 | ccsvr->stop(); 328 | 329 | tlog::info() << "All threads are terminated. Shutting down."; 330 | } catch (const std::exception &e) { 331 | tlog::error() << "Uncaught exception: " << e.what(); 332 | } 333 | 334 | return 0; 335 | } 336 | -------------------------------------------------------------------------------- /src/server.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Moonsik Park. All rights reserved. 3 | * 4 | * @file server.cpp 5 | * @author Moonsik Park, Korea Institute of Science and Technology 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | #include "base/camera_manager.h" 19 | #include "base/exceptions/lock_timeout.h" 20 | #include "base/scoped_timer.h" 21 | #include "base/video/frame_queue.h" 22 | 23 | int socket_send_blocking(int targetfd, uint8_t *buf, size_t size) { 24 | ssize_t ret; 25 | ssize_t sent = 0; 26 | 27 | while (sent < size) { 28 | ret = send(targetfd, buf + sent, size - sent, MSG_NOSIGNAL); 29 | if (ret < 0) { 30 | // Buffer is full. Try again. 31 | if (errno == EAGAIN) { 32 | continue; 33 | } 34 | // Misc error. Terminate the socket. 35 | tlog::error() << "socket_send_blocking: " 36 | << std::string(std::strerror(errno)); 37 | return -errno; 38 | } 39 | sent += ret; 40 | } 41 | 42 | return 0; 43 | } 44 | 45 | // Send message with length prefix framing. 46 | int socket_send_blocking_lpf(int targetfd, uint8_t *buf, size_t size) { 47 | int ret; 48 | // hack: not very platform portable 49 | // but then, the program isn't. 50 | if ((ret = socket_send_blocking(targetfd, (uint8_t *)&size, sizeof(size))) < 51 | 0) { 52 | return ret; 53 | } 54 | 55 | if ((ret = socket_send_blocking(targetfd, buf, size)) < 0) { 56 | return ret; 57 | } 58 | 59 | return ret; 60 | } 61 | 62 | int socket_receive_blocking(int targetfd, uint8_t *buf, size_t size) { 63 | ssize_t ret; 64 | ssize_t recv = 0; 65 | 66 | while (recv < size) { 67 | ret = read(targetfd, buf + recv, size - recv); 68 | if (ret < 0) { 69 | // Buffer is full. Try again. 70 | if (errno == EAGAIN) { 71 | continue; 72 | } 73 | // Misc error. Terminate the socket. 74 | tlog::error() << "socket_receive_blocking: " 75 | << std::string(std::strerror(errno)); 76 | return -errno; 77 | } 78 | if (ret == 0 && recv < size) { 79 | // Client disconnected while sending data. Terminate the socket. 80 | tlog::error() 81 | << "socket_receive_blocking: Received EOF when transfer is not done."; 82 | return -1; 83 | } 84 | recv += ret; 85 | } 86 | 87 | return 0; 88 | } 89 | 90 | // Receive message with length prefix framing. 91 | std::string socket_receive_blocking_lpf(int targetfd) { 92 | int ret; 93 | size_t size; 94 | // hack: not very platform portable 95 | // todo: silently fail, do not wail error. 96 | if ((ret = socket_receive_blocking(targetfd, (uint8_t *)&size, 97 | sizeof(size))) < 0) { 98 | throw std::runtime_error{ 99 | "socket_receive_blocking_lpf: Error while " 100 | "receiving data size from socket."}; 101 | } 102 | 103 | auto buffer = std::make_unique(size); 104 | 105 | if ((ret = socket_receive_blocking(targetfd, (uint8_t *)buffer.get(), size)) < 106 | 0) { 107 | throw std::runtime_error{ 108 | "socket_receive_blocking_lpf: Error while receiving data from socket."}; 109 | } 110 | 111 | return std::string(buffer.get(), buffer.get() + size); 112 | } 113 | 114 | static constexpr unsigned kLogStatsIntervalFrame = 100; 115 | 116 | void socket_client_thread( 117 | int targetfd, std::shared_ptr frame_queue_left, 118 | std::shared_ptr frame_queue_right, 119 | std::atomic &frame_index_left, 120 | std::atomic &frame_index_right, std::atomic &is_left, 121 | std::shared_ptr cameramgr, 122 | std::shared_ptr ctxmgr_scene, 123 | std::shared_ptr ctxmgr_depth, 124 | std::atomic &shutdown_requested) { 125 | // set_thread_name(std::string("socket_client=") + std::to_string(targetfd)); 126 | int ret = 0; 127 | tlog::info() << "socket_client_thread (fd=" << targetfd << "): Spawned."; 128 | 129 | uint64_t count = 0; 130 | uint64_t elapsed = 0; 131 | 132 | while (!shutdown_requested) { 133 | if (ret < 0) { 134 | // If there were errors, exit the loop. 135 | tlog::error() << "socket_client_thread (fd=" << targetfd 136 | << "): Error occured. Exiting."; 137 | break; 138 | } 139 | 140 | nesproto::FrameRequest req; 141 | // is_left xor true op has same effect as not op 142 | // t xor t = f (not t) 143 | // f xor t = t (not f) 144 | bool is_left_val = is_left.fetch_xor(true); 145 | req.set_is_left(is_left_val); 146 | 147 | uint64_t frame_index; 148 | if (is_left_val) { 149 | frame_index = frame_index_left.fetch_add(1); 150 | } else { 151 | frame_index = frame_index_right.fetch_add(1); 152 | } 153 | req.set_index(frame_index); 154 | // set_allocated_* destroys the object. Use mutable_*()->CopyFrom(). 155 | if (is_left_val) { 156 | req.mutable_camera()->CopyFrom(cameramgr->get_camera_left()); 157 | } else { 158 | req.mutable_camera()->CopyFrom(cameramgr->get_camera_right()); 159 | } 160 | 161 | std::string req_serialized = req.SerializeAsString(); 162 | 163 | // Send request from request queue. 164 | if ((ret = socket_send_blocking_lpf(targetfd, 165 | (uint8_t *)req_serialized.data(), 166 | req_serialized.size())) < 0) { 167 | continue; 168 | } 169 | 170 | nesproto::RenderedFrame frame; 171 | { 172 | ScopedTimer timer; 173 | 174 | try { 175 | if (!frame.ParseFromString(socket_receive_blocking_lpf(targetfd))) { 176 | continue; 177 | } 178 | } catch (const std::runtime_error &) { 179 | continue; 180 | } 181 | count++; 182 | elapsed += timer.elapsed().count(); 183 | if (count == kLogStatsIntervalFrame) { 184 | tlog::debug() << "socket_client_thread (fd=" << targetfd 185 | << "): Frame receiving average time of " 186 | << kLogStatsIntervalFrame 187 | << " frames: " << elapsed / count << " msec."; 188 | count = 0; 189 | elapsed = 0; 190 | } 191 | } 192 | 193 | std::unique_ptr frame_o = std::make_unique( 194 | frame, AV_PIX_FMT_RGB24, AV_PIX_FMT_GRAY8, ctxmgr_scene, ctxmgr_depth); 195 | 196 | try { 197 | // Push the frame to the frame queue. 198 | if (frame_o->is_left()) { 199 | frame_queue_left->push(std::move(frame_o)); 200 | } else { 201 | frame_queue_right->push(std::move(frame_o)); 202 | } 203 | } catch (const LockTimeout &) { 204 | // It takes too much time to acquire a lock of frame_queue. Drop the 205 | // frame. BUG: If we drop the frame, the program will hang and look for 206 | // the frame. 207 | continue; 208 | } 209 | } 210 | // Cleanup: close the client socket. 211 | close(targetfd); 212 | tlog::info() << "socket_client_thread (fd=" << targetfd 213 | << "): Exiting thread."; 214 | } 215 | 216 | void socket_manage_thread( 217 | std::string renderer, std::shared_ptr frame_queue_left, 218 | std::shared_ptr frame_queue_right, 219 | std::atomic &frame_index_left, 220 | std::atomic &frame_index_right, std::atomic &is_left, 221 | std::shared_ptr cameramgr, 222 | std::shared_ptr ctxmgr_scene, 223 | std::shared_ptr ctxmgr_depth, 224 | std::atomic &shutdown_requested) { 225 | // set_thread_name(std::string("socket_manage=") + renderer); 226 | int error_times = 0; 227 | while (!shutdown_requested) { 228 | std::stringstream renderer_parsed(renderer); 229 | 230 | std::string ip; 231 | std::string port_str; 232 | 233 | std::getline(renderer_parsed, ip, ':'); 234 | std::getline(renderer_parsed, port_str, ':'); 235 | 236 | uint16_t port((uint16_t)std::stoi(port_str)); 237 | 238 | int fd; 239 | 240 | struct sockaddr_in addr; 241 | 242 | if ((fd = socket(AF_INET, SOCK_STREAM, 0)) < 0) { 243 | tlog::error() << "socket_client_thread_factory(" << renderer 244 | << "): Failed to create socket : " << std::strerror(errno) 245 | << "; Retrying."; 246 | std::this_thread::sleep_for(std::chrono::milliseconds(1000)); 247 | continue; 248 | } 249 | 250 | memset(&addr, 0, sizeof(addr)); 251 | addr.sin_family = AF_INET; 252 | addr.sin_addr.s_addr = inet_addr(ip.c_str()); 253 | addr.sin_port = htons(port); 254 | 255 | if (connect(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { 256 | std::this_thread::sleep_for(std::chrono::milliseconds(1000)); 257 | error_times++; 258 | if (error_times > 30) { 259 | tlog::error() << "socket_client_thread_factory(" << renderer 260 | << "): Failed to connect : " << std::strerror(errno) 261 | << "; Retrying."; 262 | error_times = 0; 263 | } 264 | continue; 265 | } 266 | 267 | tlog::success() << "socket_client_thread_factory(" << renderer 268 | << "): Connected to " << renderer; 269 | 270 | std::thread _socket_client_thread( 271 | socket_client_thread, fd, frame_queue_left, frame_queue_right, 272 | std::ref(frame_index_left), std::ref(frame_index_right), 273 | std::ref(is_left), cameramgr, ctxmgr_scene, ctxmgr_depth, 274 | std::ref(shutdown_requested)); 275 | 276 | _socket_client_thread.join(); 277 | 278 | error_times = 0; 279 | 280 | tlog::error() << "socket_manage_thread (" << renderer 281 | << "): Connection is dead. Trying to reconnect."; 282 | } 283 | } 284 | 285 | void socket_main_thread( 286 | std::vector renderers, 287 | std::shared_ptr frame_queue_left, 288 | std::shared_ptr frame_queue_right, 289 | std::atomic &frame_index_left, 290 | std::atomic &frame_index_right, std::atomic &is_left, 291 | std::shared_ptr cameramgr, 292 | std::shared_ptr ctxmgr_scene, 293 | std::shared_ptr ctxmgr_depth, 294 | std::atomic &shutdown_requested) { 295 | // set_thread_name("socket_main"); 296 | std::vector threads; 297 | 298 | tlog::info() << "socket_main_thread: Connecting to renderers."; 299 | 300 | for (const auto renderer : renderers) { 301 | threads.push_back( 302 | std::thread(socket_manage_thread, renderer, frame_queue_left, 303 | frame_queue_right, std::ref(frame_index_left), 304 | std::ref(frame_index_right), std::ref(is_left), cameramgr, 305 | ctxmgr_scene, ctxmgr_depth, std::ref(shutdown_requested))); 306 | } 307 | 308 | tlog::info() << "socket_main_thread: Connectd to all renderers."; 309 | 310 | for (auto &thread : threads) { 311 | if (thread.joinable()) { 312 | thread.join(); 313 | } 314 | } 315 | 316 | tlog::info() << "socket_main_thread: Closed all connections. Exiting thread."; 317 | } 318 | --------------------------------------------------------------------------------