├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── README.md ├── bgr2rgb.s ├── glsl-test.cpp ├── image_gpu.cpp ├── image_gpu.h ├── image_gpu_private.h ├── mux.sh ├── omxcv-config.h.in ├── omxcv-impl.h ├── omxcv-test.cpp ├── omxcv.cpp ├── omxcv.h ├── omxcv_jpeg.cpp ├── simplefragshader.glsl └── simplevertshader.glsl /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dylib 14 | *.dll 15 | 16 | # Fortran module files 17 | *.mod 18 | 19 | # Compiled Static libraries 20 | *.lai 21 | *.la 22 | *.a 23 | *.lib 24 | 25 | # Executables 26 | *.exe 27 | *.out 28 | *.app 29 | 30 | CMakeCache.txt 31 | CMakeFiles 32 | cmake_install.cmake 33 | Makefile 34 | 35 | omxcv 36 | omxcv-test 37 | omxcv-config.h 38 | save.mkv 39 | save.mp4 40 | timecodes.txt 41 | glsl-test 42 | save.jpg 43 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #omxcv cmake configuration file 2 | 3 | cmake_minimum_required (VERSION 2.8) 4 | project (omxcv) 5 | 6 | include (FindPkgConfig) 7 | 8 | #Versioning 9 | execute_process (COMMAND git rev-parse --short HEAD 10 | OUTPUT_STRIP_TRAILING_WHITESPACE 11 | OUTPUT_VARIABLE OMXCV_VERSION 12 | ) 13 | execute_process (COMMAND git show -s --format=%ci HEAD 14 | OUTPUT_STRIP_TRAILING_WHITESPACE 15 | OUTPUT_VARIABLE OMXCV_DATE 16 | ) 17 | 18 | #Required libraries 19 | find_package (OpenCV REQUIRED) 20 | include_directories (${OpenCV_INCLUDE_DIR}) 21 | find_package (Threads REQUIRED) # For pthreads 22 | 23 | pkg_check_modules (LIBAVFORMAT libavformat REQUIRED) 24 | pkg_check_modules (LIBAVCODEC libavcodec REQUIRED) 25 | pkg_check_modules (LIBAVUTIL libavutil REQUIRED) 26 | 27 | include_directories (${LIBAVFORMAT_INCLUDE_DIRS}) 28 | link_directories (${LIBAVFORMAT_LIBRARY_DIRS}) 29 | include_directories (${LIBAVCODEC_INCLUDE_DIRS}) 30 | link_directories (${LIBAVCODEC_LIBRARY_DIRS}) 31 | include_directories (${LIBAVUTIL_INCLUDE_DIRS}) 32 | link_directories (${LIBAVUTIL_LIBRARY_DIRS}) 33 | 34 | #Optional libraries 35 | CHECK_LIBRARY_EXISTS(tcmalloc malloc "" HAVE_LIBTCMALLOC) 36 | CHECK_LIBRARY_EXISTS(profiler malloc "" HAVE_LIBPROFILER) 37 | 38 | #Raspberry Pi specific includes. 39 | #Requires the installation of the userland package. 40 | include_directories (/opt/vc/include /opt/vc/src/hello_pi/libs/ilclient) 41 | include_directories (/opt/vc/include/interface/vcos/pthreads) 42 | include_directories (/opt/vc/include/IL /opt/vc/include/interface/vmcs_host/linux) 43 | link_directories (/opt/vc/lib /opt/vc/src/hello_pi/libs/ilclient) 44 | 45 | #Compile options 46 | if (CMAKE_COMPILER_IS_GNUCXX) 47 | set (CMAKE_CXX_FLAGS "-Wall -Wno-variadic-macros -std=c++11 -pedantic -g") 48 | endif (CMAKE_COMPILER_IS_GNUCXX) 49 | 50 | #Source files 51 | set (HEADERS 52 | omxcv-config.h 53 | omxcv.h 54 | image_gpu.h 55 | ) 56 | 57 | set (SOURCE 58 | image_gpu.cpp 59 | omxcv.cpp 60 | omxcv_jpeg.cpp 61 | ) 62 | 63 | #Enable NEON extensions 64 | if (ENABLE_NEON) 65 | set (SOURCE ${SOURCE} bgr2rgb.s) 66 | set_property (SOURCE bgr2rgb.s PROPERTY LANGUAGE C) 67 | set_source_files_properties (bgr2rgb.s PROPERTIES COMPILE_FLAGS "-mfpu=neon -mcpu=cortex-a7 -g") 68 | endif (ENABLE_NEON) 69 | 70 | #Generate config header 71 | configure_file (omxcv-config.h.in omxcv-config.h) 72 | 73 | #Library creation 74 | add_library (omxcv ${HEADERS} ${SOURCE}) 75 | target_link_libraries (omxcv LINK_PRIVATE EGL GLESv2) 76 | target_link_libraries (omxcv LINK_PRIVATE ${LIBAVFORMAT_LIBRARIES} ${LIBAVCODEC_LIBRARIES} ${LIBAVUTIL_LIBRARIES} ${OpenCV_LIBS}) 77 | target_link_libraries (omxcv LINK_PRIVATE bcm_host openmaxil bcm_host vcos vchiq_arm ilclient) 78 | target_link_libraries (omxcv LINK_PRIVATE ${CMAKE_THREAD_LIBS_INIT}) 79 | 80 | if (HAVE_LIBTCMALLOC) 81 | target_link_libraries (omxcv LINK_PUBLIC tcmalloc) 82 | endif (HAVE_LIBTCMALLOC) 83 | 84 | if (HAVE_LIBPROFILER) 85 | target_link_libraries (omxcv LINK_PUBLIC profiler) 86 | endif (HAVE_LIBPROFILER) 87 | 88 | #Test executable 89 | add_executable (omxcv-test omxcv-test.cpp) 90 | add_executable (glsl-test glsl-test.cpp) 91 | target_link_libraries (omxcv-test LINK_PUBLIC omxcv) 92 | target_link_libraries (glsl-test LINK_PUBLIC omxcv) 93 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # omxcv 2 | GPU assisted (hardware) H.264 encoder for OpenCV on the Raspberry Pi using OpenMAX. 3 | 4 | Allows OpenCV Mat data to be encoded easily in real-time using the GPU encoder on the Raspberry Pi. On the Raspberry Pi 2, resolutions up to 640x480 can be saved in real-time (i.e. maintaining 30FPS). 5 | 6 | omxcv maintains an internal clock, so the time at which the frame is sent to omxcv is when it will be displayed on playback. In addition, omxcv **will drop** frames if it detects that it is falling behind. 7 | 8 | Restrictions: 9 | * Width should be mod8 10 | * Input `Mat`s should be of type CV_8UC3 BGR images. 11 | * Output format should either be MKV or MP4. 12 | -------------------------------------------------------------------------------- /bgr2rgb.s: -------------------------------------------------------------------------------- 1 | .global omxcv_bgr2rgb_neon 2 | 3 | @void omxcv_bgr2rgb_neon32(const uint8_t *src, uint8_t *dst, int pixels) 4 | omxcv_bgr2rgb_neon32: 5 | mov r2, r2, lsr #5 @Width/32 (Number of runs needed) 6 | vpush {q4-q5} @q4-q11 must be saved by us 7 | loop3: 8 | @pld [r0, #512] @Preload for reading 9 | vld3.8 {d0-d2}, [r0]! @Read in 8 pixels 10 | vld3.8 {d3-d5}, [r0]! @Read in 8 more pixels 11 | vld3.8 {d6-d8}, [r0]! @Read in 8 more pixels 12 | vld3.8 {d9-d11}, [r0]! @Read in 8 more pixels 13 | vswp d0, d2 @Swap R/B 14 | vswp d3, d5 @Swap R/B 15 | vswp d6, d8 16 | vswp d9, d11 17 | subs r2, r2, #1 @Decrement counter 18 | vst3.8 {d0-d2}, [r1]! @Store 8 pixels 19 | vst3.8 {d3-d5}, [r1]! @Store 8 more 20 | vst3.8 {d6-d8}, [r1]! @Store 8 more pixels 21 | vst3.8 {d9-d11}, [r1]! @Store 8 more pixels 22 | bgt loop3 @Loop 23 | vpop {q4-q5} @Restore q4-q5 24 | bx lr @Return 25 | 26 | @void omxcv_bgr2rgb_neon(const uint8_t *src, uint8_t *dst, int pixels) 27 | omxcv_bgr2rgb_neon: 28 | mov r2, r2, lsr #4 @Width/16 (Number of runs needed) 29 | loop2: 30 | pld [r0, #384] @Preload for reading 31 | vld3.8 {d0-d2}, [r0]! @Read in 8 pixels 32 | vld3.8 {d3-d5}, [r0]! @Read in 8 more pixels 33 | vswp d0, d2 @Swap R/B 34 | vswp d3, d5 @Swap R/B 35 | subs r2, r2, #1 @Decrement counter 36 | vst3.8 {d0-d2}, [r1]! @Store 8 pixels 37 | vst3.8 {d3-d5}, [r1]! @Store 8 more pixels 38 | bgt loop2 @Loop 39 | bx lr @Return 40 | 41 | @void omxcv_bgr2rgb_neon1(const uint8_t *src, uint8_t *dst, int pixels) 42 | omxcv_bgr2rgb_neon1: 43 | mov r2, r2, lsr #3 @Width/8 (Number of runs needed) 44 | loop: 45 | pld [r0, #192] @Preload for reading 46 | vld3.8 {d0-d2}, [r0]! @Read in 8 pixels 47 | vswp d0, d2 @Swap R/B 48 | subs r2, r2, #1 @Decrement counter 49 | vst3.8 {d0-d2}, [r1]! @Store 8 pixels 50 | bgt loop2 @Loop 51 | bx lr @Return 52 | -------------------------------------------------------------------------------- /glsl-test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file glsl-test.cpp 3 | * @brief Simple testing application for GLSL. 4 | */ 5 | 6 | #include "image_gpu.h" 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #define TIMEDIFF(start) (duration_cast(steady_clock::now() - start).count()) 15 | 16 | using std::this_thread::sleep_for; 17 | using std::chrono::microseconds; 18 | using std::chrono::milliseconds; 19 | using std::chrono::steady_clock; 20 | using std::chrono::duration_cast; 21 | 22 | using namespace picopter; 23 | 24 | int main(int argc, char *argv[]) { 25 | cv::VideoCapture capture(-1); 26 | 27 | int width = 640, height = 480, framecount = 200, processed = 0; 28 | 29 | if (argc >= 3) { 30 | width = atoi(argv[1]); 31 | height = atoi(argv[2]); 32 | } 33 | 34 | if (argc >= 4) { 35 | framecount = atoi(argv[3]); 36 | } 37 | 38 | //Open the camera (testing only) 39 | if (!capture.isOpened()) { 40 | printf("Cannot open camera\n"); 41 | return 1; 42 | } 43 | //We can try to set these, but the camera may ignore this anyway... 44 | #if (CV_MAJOR_VERSION < 3) 45 | capture.set(CV_CAP_PROP_FRAME_WIDTH, width); 46 | capture.set(CV_CAP_PROP_FRAME_HEIGHT, height); 47 | capture.set(CV_CAP_PROP_FPS, 30); 48 | #else 49 | capture.set(cv::CAP_PROP_FRAME_WIDTH, width); 50 | capture.set(cv::CAP_PROP_FRAME_HEIGHT, height); 51 | capture.set(cv::CAP_PROP_FPS, 30); 52 | #endif // (CV_MAJOR_VERSION < 3) 53 | 54 | auto totstart = steady_clock::now(); 55 | cv::Mat image, out; 56 | //FILE *fp = fopen("log.txt", "w"); 57 | 58 | #if (CV_MAJOR_VERSION < 3) 59 | GLThreshold t(capture.get(CV_CAP_PROP_FRAME_WIDTH), capture.get(CV_CAP_PROP_FRAME_HEIGHT)); 60 | #else 61 | GLThreshold t(capture.get(cv::CAP_PROP_FRAME_WIDTH), capture.get(cv::CAP_PROP_FRAME_HEIGHT)); 62 | #endif // (CV_MAJOR_VERSION < 3) 63 | 64 | for(int i = 0; i < framecount; i++) { 65 | capture >> image; 66 | auto start = steady_clock::now(); 67 | t.Threshold(image, out); 68 | printf("Processed frame %4d (%4d ms)\r", i+1, (int)TIMEDIFF(start)/1000); 69 | fflush(stdout); 70 | } 71 | } -------------------------------------------------------------------------------- /image_gpu.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file image_gpu.cpp 3 | * @brief GPU processing image library 4 | */ 5 | 6 | #include "image_gpu.h" 7 | #include "image_gpu_private.h" 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | using std::chrono::milliseconds; 20 | using std::chrono::steady_clock; 21 | using std::chrono::duration_cast; 22 | 23 | #define CHECKED(c, v) if ((c)) throw std::invalid_argument(v) 24 | #define GLCHECKED(c, v) if ((c) || glGetError() != 0) throw std::invalid_argument(v) 25 | #define TIMEDIFF(start) (duration_cast(steady_clock::now() - start).count()) 26 | 27 | #define check() assert(glGetError() == 0) 28 | 29 | using namespace picopter; 30 | 31 | GLThreshold::GLThreshold(Options *opts, int width, int height) 32 | : m_width(width) 33 | , m_height(height) 34 | , m_thresh_min{} 35 | , m_thresh_max{} 36 | { 37 | EGLBoolean result; 38 | EGLint num_config; 39 | 40 | static const EGLint attribute_list[] = 41 | { 42 | EGL_RED_SIZE, 8, 43 | EGL_GREEN_SIZE, 8, 44 | EGL_BLUE_SIZE, 8, 45 | EGL_ALPHA_SIZE, 8, 46 | EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, 47 | EGL_NONE 48 | }; 49 | 50 | static const EGLint context_attributes[] = 51 | { 52 | EGL_CONTEXT_CLIENT_VERSION, 2, 53 | EGL_NONE 54 | }; 55 | EGLConfig config; 56 | 57 | //Get a display 58 | m_display = eglGetDisplay(EGL_DEFAULT_DISPLAY); 59 | GLCHECKED(m_display == EGL_NO_DISPLAY, "Cannot get EGL display."); 60 | 61 | //Initialise the EGL display connection 62 | result = eglInitialize(m_display, NULL, NULL); 63 | GLCHECKED(result == EGL_FALSE, "Cannot initialise display connection."); 64 | 65 | //Get an appropriate EGL frame buffer configuration 66 | result = eglChooseConfig(m_display, attribute_list, &config, 1, &num_config); 67 | GLCHECKED(result == EGL_FALSE, "Cannot get buffer configuration."); 68 | 69 | //Bind to the right EGL API. 70 | result = eglBindAPI(EGL_OPENGL_ES_API); 71 | GLCHECKED(result == EGL_FALSE, "Could not bind EGL API."); 72 | 73 | //Create an EGL rendering context 74 | EGLContext context = eglCreateContext(m_display, config, EGL_NO_CONTEXT, context_attributes); 75 | GLCHECKED(context == EGL_NO_CONTEXT, "Could not create EGL context."); 76 | 77 | //Create an offscreen rendering surface 78 | static const EGLint rendering_attributes[] = 79 | { 80 | EGL_WIDTH, width, 81 | EGL_HEIGHT, height, 82 | EGL_NONE 83 | }; 84 | m_surface = eglCreatePbufferSurface(m_display, config, rendering_attributes); 85 | GLCHECKED(m_surface == EGL_NO_SURFACE, "Could not create PBuffer surface."); 86 | 87 | //Bind the context to the current thread 88 | result = eglMakeCurrent(m_display, m_surface, m_surface, context); 89 | GLCHECKED(result == EGL_FALSE, "Failed to bind context."); 90 | 91 | //xyzw 92 | static const GLfloat quad_vertex_positions[] = { 93 | 0.0f, 0.0f, 1.0f, 1.0f, 94 | 1.0f, 0.0f, 1.0f, 1.0f, 95 | 0.0f, 1.0f, 1.0f, 1.0f, 96 | 1.0f, 1.0f, 1.0f, 1.0f 97 | }; 98 | 99 | glGenBuffers(1, &m_quad_buffer); 100 | glBindBuffer(GL_ARRAY_BUFFER, m_quad_buffer); 101 | glBufferData(GL_ARRAY_BUFFER, sizeof(quad_vertex_positions), quad_vertex_positions, GL_STATIC_DRAW); 102 | glBindBuffer(GL_ARRAY_BUFFER, 0); 103 | 104 | //Setup the shaders and texture buffer. 105 | m_program = new GLProgram("simplevertshader.glsl", "simplefragshader.glsl"); 106 | m_texture = new GLTexture(width, height, GL_RGB); 107 | } 108 | 109 | GLThreshold::GLThreshold(int width, int height) 110 | : GLThreshold(NULL, width, height) {} 111 | 112 | GLThreshold::~GLThreshold() { 113 | eglDestroySurface(m_display, m_surface); 114 | delete m_program; 115 | delete m_texture; 116 | } 117 | 118 | void GLThreshold::Threshold(const cv::Mat &in, cv::Mat &out) { 119 | //Load the data into a texture. 120 | m_texture->SetData(in.data); 121 | 122 | //Blank the display 123 | glBindFramebuffer(GL_FRAMEBUFFER, m_texture->GetFramebufferId()); 124 | glViewport(0, 0, m_texture->GetWidth(), m_texture->GetHeight()); 125 | check(); 126 | glClear(GL_COLOR_BUFFER_BIT); 127 | check(); 128 | 129 | glUseProgram(*m_program); 130 | check(); 131 | 132 | //Load in the texture and thresholding parameters. 133 | glUniform1i(glGetUniformLocation(*m_program,"tex"), 0); 134 | glUniform4f(glGetUniformLocation(*m_program, "threshLow"),0,167/255.0, 86/255.0,0); 135 | glUniform4f(glGetUniformLocation(*m_program, "threshHigh"),255/255.0,255/255.0, 141/255.0,1); 136 | check(); 137 | 138 | glBindBuffer(GL_ARRAY_BUFFER, m_quad_buffer); check(); 139 | glBindTexture(GL_TEXTURE_2D, *m_texture); check(); 140 | 141 | //Initialize the vertex position attribute from the vertex shader 142 | GLuint loc = glGetAttribLocation(*m_program, "vPosition"); 143 | glVertexAttribPointer(loc, 4, GL_FLOAT, GL_FALSE, 0, 0); check(); 144 | glEnableVertexAttribArray(loc); check(); 145 | 146 | glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); check(); 147 | 148 | glBindBuffer(GL_ARRAY_BUFFER, 0); 149 | glBindTexture(GL_TEXTURE_2D, 0); 150 | glBindFramebuffer(GL_FRAMEBUFFER,0); 151 | 152 | //glFinish(); check(); 153 | //glFlush(); check(); 154 | 155 | //RENDER 156 | eglSwapBuffers(m_display, m_surface); 157 | //check(); 158 | 159 | //glFinish(); 160 | 161 | /*void *out = malloc(3 * m_texture->GetWidth() * m_texture->GetHeight()); 162 | m_texture->GetRenderedData(out); 163 | FILE *fp = fopen("TEMP.RGB", "wb"); 164 | fwrite(out, 3 * m_texture->GetWidth() * m_texture->GetHeight(), 1, fp); 165 | fclose(fp);*/ 166 | } 167 | 168 | GLProgram::GLProgram(const char *vertex_file, const char *fragment_file) { 169 | GLint status; 170 | m_program_id = glCreateProgram(); 171 | 172 | m_vertex_id = LoadShader(GL_VERTEX_SHADER, vertex_file); 173 | m_fragment_id = LoadShader(GL_FRAGMENT_SHADER, fragment_file); 174 | glAttachShader(m_program_id, m_vertex_id); 175 | glAttachShader(m_program_id, m_fragment_id); 176 | 177 | glLinkProgram(m_program_id); 178 | glGetProgramiv(m_program_id, GL_LINK_STATUS, &status); 179 | if (!status) { 180 | GLint msg_len; 181 | char *msg; 182 | std::stringstream s; 183 | 184 | glGetProgramiv(m_program_id, GL_INFO_LOG_LENGTH, &msg_len); 185 | msg = new char[msg_len]; 186 | glGetProgramInfoLog(m_program_id, msg_len, NULL, msg); 187 | 188 | s << "Failed to link shaders: " << msg; 189 | delete [] msg; 190 | throw std::invalid_argument(s.str()); 191 | } 192 | } 193 | 194 | GLProgram::~GLProgram() { 195 | glDeleteShader(m_fragment_id); 196 | glDeleteShader(m_vertex_id); 197 | glDeleteProgram(m_program_id); 198 | } 199 | 200 | GLuint GLProgram::GetId() { 201 | return m_program_id; 202 | } 203 | 204 | GLuint GLProgram::LoadShader(GLenum shader_type, const char *source_file) { 205 | GLint status; 206 | GLuint shader_id; 207 | char *shader_source = ReadFile(source_file); 208 | 209 | if (!shader_source) { 210 | std::stringstream s; 211 | const char *error = std::strerror(errno); 212 | s << "Could not load " << source_file << ": " << error; 213 | throw std::invalid_argument(s.str()); 214 | } 215 | 216 | shader_id = glCreateShader(shader_type); 217 | glShaderSource(shader_id, 1, (const GLchar**)&shader_source, NULL); 218 | glCompileShader(shader_id); 219 | delete [] shader_source; 220 | 221 | glGetShaderiv(shader_id, GL_COMPILE_STATUS, &status); 222 | if (!status) { 223 | GLint msg_len; 224 | char *msg; 225 | std::stringstream s; 226 | 227 | glGetShaderiv(shader_id, GL_INFO_LOG_LENGTH, &msg_len); 228 | msg = new char[msg_len]; 229 | glGetShaderInfoLog(shader_id, msg_len, NULL, msg); 230 | 231 | s << "Failed to compile " << source_file << ": " << msg; 232 | delete [] msg; 233 | throw std::invalid_argument(s.str()); 234 | } 235 | 236 | return shader_id; 237 | } 238 | 239 | char* GLProgram::ReadFile(const char *file) { 240 | std::FILE *fp = std::fopen(file, "rb"); 241 | char *ret = NULL; 242 | size_t length; 243 | 244 | if (fp) { 245 | std::fseek(fp, 0, SEEK_END); 246 | length = std::ftell(fp); 247 | std::fseek(fp, 0, SEEK_SET); 248 | 249 | ret = new char[length + 1]; 250 | length = std::fread(ret, 1, length, fp); 251 | ret[length] = '\0'; 252 | 253 | std::fclose(fp); 254 | } 255 | 256 | return ret; 257 | } 258 | 259 | GLTexture::GLTexture(GLsizei width, GLsizei height, GLint type) 260 | : m_width(width), m_height(height), m_type(type) 261 | { 262 | if (width % 4 || height % 4) { 263 | throw std::invalid_argument("Width/height is not a multiple of 4."); 264 | } 265 | 266 | //Allocate the texture buffer 267 | glGenTextures(1, &m_texture_id); 268 | glBindTexture(GL_TEXTURE_2D, m_texture_id); 269 | glTexImage2D(GL_TEXTURE_2D, 0, type, width, height, 0, type, GL_UNSIGNED_BYTE, NULL); 270 | if (glGetError() != GL_NO_ERROR) { 271 | throw std::invalid_argument("glTexImage2D failed. Could not allocate texture buffer."); 272 | } 273 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); 274 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); 275 | glBindTexture(GL_TEXTURE_2D, 0); 276 | 277 | //Allocate the frame buffer 278 | glGenFramebuffers(1, &m_framebuffer_id); 279 | glBindFramebuffer(GL_FRAMEBUFFER, m_framebuffer_id); 280 | glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, m_texture_id, 0); 281 | if (glGetError() != GL_NO_ERROR) { 282 | throw std::invalid_argument("glFramebufferTexture2D failed. Could not allocate framebuffer."); 283 | } 284 | glBindFramebuffer(GL_FRAMEBUFFER,0); 285 | } 286 | 287 | GLTexture::~GLTexture() { 288 | glDeleteFramebuffers(1, &m_framebuffer_id); 289 | glDeleteTextures(1, &m_texture_id); 290 | } 291 | 292 | GLsizei GLTexture::GetWidth() { 293 | return m_width; 294 | } 295 | 296 | GLsizei GLTexture::GetHeight() { 297 | return m_height; 298 | } 299 | 300 | void GLTexture::SetData(void *data) { 301 | glBindTexture(GL_TEXTURE_2D, m_texture_id); 302 | glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, m_width, m_height, m_type, GL_UNSIGNED_BYTE, data); 303 | glBindTexture(GL_TEXTURE_2D, 0); 304 | } 305 | 306 | void GLTexture::GetRenderedData(void *buffer) { 307 | glBindFramebuffer(GL_FRAMEBUFFER,m_framebuffer_id); 308 | glReadPixels(0, 0, m_width, m_height, m_type, GL_UNSIGNED_BYTE, buffer); 309 | glBindFramebuffer(GL_FRAMEBUFFER,0); 310 | } 311 | 312 | GLuint GLTexture::GetTextureId() { 313 | return m_texture_id; 314 | } 315 | 316 | GLuint GLTexture::GetFramebufferId() { 317 | return m_framebuffer_id; 318 | } 319 | -------------------------------------------------------------------------------- /image_gpu.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file image_gpu.h 3 | * @brief GPU functions for image processing. 4 | */ 5 | 6 | #ifndef _PICOPTERX_IMAGE_GPU_H 7 | #define _PICOPTERX_IMAGE_GPU_H 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | namespace picopter { 14 | /* Forward declaration of the options class */ 15 | class Options; 16 | /* Forward declaration of the GLProgram class */ 17 | class GLProgram; 18 | /* Forward declaration of the GLTexture class */ 19 | class GLTexture; 20 | 21 | typedef struct ThresholdSet { 22 | int x; 23 | int y; 24 | int z; 25 | } ThresholdSet; 26 | 27 | /** 28 | * Class to perform colour thresholding using OpenGL. 29 | */ 30 | class GLThreshold { 31 | public: 32 | GLThreshold(Options *opts, int width, int height); 33 | GLThreshold(int width, int height); 34 | virtual ~GLThreshold(); 35 | 36 | void SetThresholds(ThresholdSet min, ThresholdSet max); 37 | void Threshold(const cv::Mat &in, cv::Mat &out); 38 | private: 39 | int m_width, m_height; 40 | ThresholdSet m_thresh_min, m_thresh_max; 41 | GLProgram *m_program; 42 | GLTexture *m_texture; 43 | 44 | EGLDisplay m_display; 45 | EGLSurface m_surface; 46 | GLuint m_quad_buffer; 47 | 48 | /** Copy constructor (disabled) **/ 49 | GLThreshold(const GLThreshold &other); 50 | /** Assignment operator (disabled) **/ 51 | GLThreshold& operator= (const GLThreshold &other); 52 | }; 53 | 54 | } 55 | 56 | #endif -------------------------------------------------------------------------------- /image_gpu_private.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file image_gpu_private.h 3 | * @brief Private GPU header. GPU functions for image processing. 4 | */ 5 | 6 | #ifndef _PICOPTERX_IMAGE_GPU_PRIVATE_H 7 | #define _PICOPTERX_IMAGE_GPU_PRIVATE_H 8 | 9 | #include 10 | #include 11 | 12 | namespace picopter { 13 | class GLProgram { 14 | public: 15 | GLProgram(const char *vertex_file, const char *fragment_file); 16 | virtual ~GLProgram(); 17 | 18 | GLuint GetId(); 19 | operator GLuint() { return m_program_id; }; 20 | private: 21 | GLuint m_vertex_id, m_fragment_id, m_program_id; 22 | 23 | GLuint LoadShader(GLenum shader_type, const char *source_file); 24 | char* ReadFile(const char *file); 25 | }; 26 | 27 | class GLTexture { 28 | public: 29 | GLTexture(GLsizei width, GLsizei height, GLint type); 30 | virtual ~GLTexture(); 31 | 32 | GLsizei GetWidth(); 33 | GLsizei GetHeight(); 34 | 35 | void SetData(void *data); 36 | void GetRenderedData(void *buffer); 37 | GLuint GetTextureId(); 38 | GLuint GetFramebufferId(); 39 | operator GLuint() { return m_texture_id; }; 40 | private: 41 | GLsizei m_width, m_height; 42 | GLint m_type; 43 | GLuint m_texture_id, m_framebuffer_id; 44 | }; 45 | 46 | extern void DoMoreShit(EGLDisplay display, EGLSurface surface, GLuint buffer, picopter::GLProgram &program, picopter::GLTexture &texture); 47 | extern void DisplayShit(int width, int height, void *data); 48 | } 49 | 50 | #endif // _PICOPTERX_IMAGE_GPU_PRIVATE_H -------------------------------------------------------------------------------- /mux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mkvmerge --timecodes 0:timecodes.txt save.mkv --output mux.mkv 4 | -------------------------------------------------------------------------------- /omxcv-config.h.in: -------------------------------------------------------------------------------- 1 | /** 2 | * @file config.h 3 | * @brief Compile-time configuration options. 4 | */ 5 | 6 | #ifndef __OMXCV_CONFIG_H 7 | #define __OMXCV_CONFIG_H 8 | 9 | /* Short git hash */ 10 | #define OMXCV_VERSION "${OMXCV_VERSION}" 11 | /* Date of git hash */ 12 | #define OMXCV_DATE "${OMXCV_DATE}" 13 | 14 | /* Enable Pi 2 (armv7/NEON) stuff */ 15 | #cmakedefine ENABLE_NEON 16 | 17 | #endif /* __OMXCV_CONFIG_H */ 18 | -------------------------------------------------------------------------------- /omxcv-impl.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file omxcv-impl.cpp 3 | * @brief Actual implementation class 4 | */ 5 | 6 | #ifndef __OMXCV_IMPL_H 7 | #define __OMXCV_IMPL_H 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | 19 | //Sigh 20 | extern "C" { 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | //Without the right struct packing the buffers will be screwed... 29 | //nFlags won't be set correctly... 30 | #pragma pack(4) 31 | #include 32 | #include 33 | //For OMX_IndexParamNalStreamFormatSelect 34 | #include 35 | #pragma pack() 36 | } 37 | 38 | //Determine what frame allocation routine to use 39 | #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55,28,1) 40 | #define OMXCV_AV_FRAME_ALLOC av_frame_alloc 41 | #define OMXCV_AV_FRAME_FREE av_frame_free 42 | #else 43 | #define OMXCV_AV_FRAME_ALLOC avcodec_alloc_frame 44 | #define OMXCV_AV_FRAME_FREE avcodec_free_frame 45 | #endif 46 | 47 | #define OMX_ENCODE_PORT_IN 200 48 | #define OMX_ENCODE_PORT_OUT 201 49 | 50 | #define OMX_JPEG_PORT_IN 340 51 | #define OMX_JPEG_PORT_OUT 341 52 | 53 | //The maximum size of a NALU. We'll just assume 512 KB. 54 | #define MAX_NALU_SIZE (512*1024) 55 | 56 | #define CHECKED(c, v) if ((c)) throw std::invalid_argument(v) 57 | 58 | extern void BGR2RGB(const cv::Mat &src, uint8_t *dst, int stride); 59 | 60 | namespace omxcv { 61 | /** 62 | * Our implementation class of the encoder. 63 | */ 64 | class OmxCvImpl { 65 | public: 66 | OmxCvImpl(const char *name, int width, int height, int bitrate, int fpsnum=-1, int fpsden=-1); 67 | virtual ~OmxCvImpl(); 68 | 69 | bool process(const cv::Mat &mat); 70 | private: 71 | int m_width, m_height, m_stride, m_bitrate, m_fpsnum, m_fpsden; 72 | uint8_t *m_sps, *m_pps; 73 | uint16_t m_sps_length, m_pps_length; 74 | uint32_t m_nalu_filled, m_nalu_required; 75 | uint8_t *m_nalu_buffer; 76 | bool m_initted_header; 77 | 78 | std::string m_filename; 79 | std::condition_variable m_input_signaller; 80 | std::deque> m_input_queue; 81 | std::thread m_input_worker; 82 | std::mutex m_input_mutex; 83 | std::atomic m_stop; 84 | 85 | /** The OpenMAX IL client **/ 86 | ILCLIENT_T *m_ilclient; 87 | COMPONENT_T *m_encoder_component; 88 | 89 | /** Writing out stuff **/ 90 | AVFormatContext *m_mux_ctx; 91 | AVStream *m_video_stream; 92 | 93 | std::chrono::steady_clock::time_point m_frame_start; 94 | int m_frame_count; 95 | 96 | bool lav_init(); 97 | bool dump_codec_private(); 98 | void input_worker(); 99 | bool write_data(OMX_BUFFERHEADERTYPE *out, int64_t timestamp); 100 | }; 101 | 102 | class OmxCvJpegImpl { 103 | public: 104 | OmxCvJpegImpl(int width, int height, int quality=90); 105 | virtual ~OmxCvJpegImpl(); 106 | 107 | bool process(const char *filename, const cv::Mat &mat); 108 | private: 109 | int m_width, m_height, m_stride, m_quality; 110 | 111 | std::condition_variable m_input_signaller; 112 | std::deque> m_input_queue; 113 | std::thread m_input_worker; 114 | std::mutex m_input_mutex; 115 | std::atomic m_stop; 116 | 117 | ILCLIENT_T *m_ilclient; 118 | COMPONENT_T *m_encoder_component; 119 | 120 | void input_worker(); 121 | }; 122 | } 123 | 124 | #endif // __OMXCV_IMPL_H 125 | -------------------------------------------------------------------------------- /omxcv-test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file omxcv-test.cpp 3 | * @brief Simple testing application for omxcv. 4 | */ 5 | #include "omxcv.h" 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #define TIMEDIFF(start) (duration_cast(steady_clock::now() - start).count()) 14 | 15 | using omxcv::OmxCv; 16 | using omxcv::OmxCvJpeg; 17 | using std::this_thread::sleep_for; 18 | using std::chrono::microseconds; 19 | using std::chrono::milliseconds; 20 | using std::chrono::steady_clock; 21 | using std::chrono::duration_cast; 22 | 23 | int main(int argc, char *argv[]) { 24 | cv::VideoCapture capture(-1); 25 | 26 | int width = 640, height = 480, framecount = 200, processed = 0; 27 | 28 | if (argc >= 3) { 29 | width = atoi(argv[1]); 30 | height = atoi(argv[2]); 31 | } 32 | 33 | if (argc >= 4) { 34 | framecount = atoi(argv[3]); 35 | } 36 | 37 | //Open the camera (testing only) 38 | if (!capture.isOpened()) { 39 | printf("Cannot open camera\n"); 40 | return 1; 41 | } 42 | //We can try to set these, but the camera may ignore this anyway... 43 | #if (CV_MAJOR_VERSION < 3) 44 | capture.set(CV_CAP_PROP_FRAME_WIDTH, width); 45 | capture.set(CV_CAP_PROP_FRAME_HEIGHT, height); 46 | capture.set(CV_CAP_PROP_FPS, 30); 47 | 48 | OmxCv *e = new OmxCv((const char*)"save.mkv", (int)capture.get(CV_CAP_PROP_FRAME_WIDTH),(int)capture.get(CV_CAP_PROP_FRAME_HEIGHT), 4000); 49 | //OmxCvJpeg *j = new OmxCvJpeg((int)capture.get(CV_CAP_PROP_FRAME_WIDTH),(int)capture.get(CV_CAP_PROP_FRAME_HEIGHT), 50); 50 | #else 51 | capture.set(cv::CAP_PROP_FRAME_WIDTH, width); 52 | capture.set(cv::CAP_PROP_FRAME_HEIGHT, height); 53 | capture.set(cv::CAP_PROP_FPS, 30); 54 | 55 | OmxCv *e = new OmxCv((const char*)"save.mkv", (int)capture.get(cv::CAP_PROP_FRAME_WIDTH),(int)capture.get(cv::CAP_PROP_FRAME_HEIGHT), 4000); 56 | //OmxCvJpeg *j = new OmxCvJpeg((int)capture.get(cv::CAP_PROP_FRAME_WIDTH),(int)capture.get(cv::CAP_PROP_FRAME_HEIGHT), 50); 57 | #endif // (CV_MAJOR_VERSION < 3) 58 | 59 | auto totstart = steady_clock::now(); 60 | cv::Mat image; 61 | //FILE *fp = fopen("log.txt", "w"); 62 | 63 | for(int i = 0; i < framecount; i++) { 64 | capture >> image; 65 | //auto start = steady_clock::now(); 66 | //if (j->Encode("save.jpg", image, true)) { 67 | if (e->Encode(image)) { 68 | //printf("Processed frame %4d (%4d ms)\r", i+1, (int)TIMEDIFF(start)/1000); 69 | //fflush(stdout); 70 | //fprintf(fp, "%d\n", (int)TIMEDIFF(start)); 71 | processed++; 72 | } 73 | } 74 | //fclose(fp); 75 | 76 | //FILE *fp = fopen("Orig.rgb", "wb"); 77 | //fwrite(image.data, 3 * image.cols * image.rows, 1, fp); 78 | //fclose(fp); 79 | //picopter::DisplayShit(image.cols, image.rows, image.data); 80 | 81 | delete e; 82 | //delete j; 83 | 84 | printf("Average FPS: %.2f\n", (processed * 1000000) / (float)TIMEDIFF(totstart)); 85 | printf("Processed: %d, Dropped: %d\n", processed, framecount-processed); 86 | printf("DEPTH: %d, WIDTH: %d, HEIGHT: %d, IW: %d\n", image.depth(), image.cols, image.rows, static_cast(image.step)); 87 | //sleep_for(milliseconds(300)); 88 | 89 | //bcm_host_deinit(); 90 | } 91 | -------------------------------------------------------------------------------- /omxcv.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file omxcv.cpp 3 | * @brief Routines to perform hardware accelerated H.264 encoding on the RPi. 4 | */ 5 | 6 | #include "omxcv-config.h" 7 | #include "omxcv.h" 8 | #include "omxcv-impl.h" 9 | using namespace omxcv; 10 | 11 | using std::this_thread::sleep_for; 12 | using std::chrono::milliseconds; 13 | using std::chrono::steady_clock; 14 | using std::chrono::duration_cast; 15 | 16 | #include 17 | #include 18 | #include 19 | 20 | #define TIMEDIFF(start) (duration_cast(steady_clock::now() - start).count()) 21 | 22 | #ifdef ENABLE_NEON 23 | extern "C" void omxcv_bgr2rgb_neon(const unsigned char *src, unsigned char *dst, int n); 24 | #endif 25 | 26 | /** 27 | * Perform the BGR2RGB conversion. 28 | * @param [in] src The source buffer. 29 | * @param [in] dst The destination buffer. 30 | * @param [in] stride The stride of the image. 31 | */ 32 | void BGR2RGB(const cv::Mat &src, uint8_t *dst, int stride) { 33 | #ifdef ENABLE_NEON 34 | for (int i = 0; i < src.rows; i++) { 35 | const uint8_t *buffer = src.ptr(i); 36 | omxcv_bgr2rgb_neon(buffer, dst+stride*i, src.cols); 37 | } 38 | #else 39 | cv::Mat omat(src.rows, src.cols, CV_8UC3, dst, stride); 40 | #if (CV_MAJOR_VERSION < 3) 41 | cv::cvtColor(src, omat, CV_BGR2RGB); 42 | #else 43 | cv::cvtColor(src, omat, cv::COLOR_BGR2RGB); 44 | #endif // (CV_MAJOR_VERSION < 3) 45 | #endif 46 | } 47 | 48 | /** 49 | * Initialise LibAVCodec output muxer. 50 | * @return true iff initialised. 51 | */ 52 | bool OmxCvImpl::lav_init() { 53 | av_register_all(); 54 | av_log_set_level(AV_LOG_INFO); 55 | char buf[BUFSIZ]; 56 | 57 | AVOutputFormat *fmt = av_guess_format(NULL, m_filename.c_str(), NULL); 58 | if (fmt == NULL) { 59 | return false; 60 | } 61 | 62 | m_mux_ctx = avformat_alloc_context(); 63 | if (m_mux_ctx == NULL) { 64 | return false; 65 | } 66 | m_mux_ctx->debug = 1; 67 | m_mux_ctx->start_time_realtime = time(NULL); //NOW 68 | m_mux_ctx->start_time = AV_NOPTS_VALUE; 69 | m_mux_ctx->oformat = fmt; 70 | snprintf(m_mux_ctx->filename, sizeof(m_mux_ctx->filename), "%s", m_filename.c_str()); 71 | 72 | m_video_stream = avformat_new_stream(m_mux_ctx, NULL); 73 | if (m_video_stream == NULL) { 74 | avformat_free_context(m_mux_ctx); 75 | return false; 76 | } 77 | 78 | snprintf(buf, sizeof(buf), "Created at %sBitrate: %d Kbps", 79 | ctime((time_t*)&m_mux_ctx->start_time_realtime), m_bitrate); 80 | 81 | av_dict_set(&m_video_stream->metadata, "encoder", "omxcv " OMXCV_VERSION " " OMXCV_DATE, 0); 82 | av_dict_set(&m_video_stream->metadata, "settings", buf, 0); 83 | m_video_stream->codec->width = m_width; 84 | m_video_stream->codec->height = m_height; 85 | m_video_stream->codec->codec_id = AV_CODEC_ID_H264; 86 | m_video_stream->codec->codec_type = AVMEDIA_TYPE_VIDEO; 87 | m_video_stream->codec->bit_rate = m_bitrate; 88 | m_video_stream->codec->time_base.num = 1; 89 | m_video_stream->codec->time_base.den = 1000; 90 | m_video_stream->codec->pix_fmt = AV_PIX_FMT_YUV420P; 91 | 92 | m_video_stream->time_base.num = 1; 93 | m_video_stream->time_base.den = 1000; 94 | 95 | m_video_stream->start_time = AV_NOPTS_VALUE; 96 | 97 | m_video_stream->codec->sample_aspect_ratio.num = m_video_stream->sample_aspect_ratio.num; 98 | m_video_stream->codec->sample_aspect_ratio.den = m_video_stream->sample_aspect_ratio.den; 99 | 100 | if (avio_open(&m_mux_ctx->pb, m_filename.c_str(), AVIO_FLAG_WRITE) < 0) { 101 | avcodec_close(m_video_stream->codec); 102 | avformat_free_context(m_mux_ctx); 103 | return 0; 104 | } 105 | 106 | if (m_mux_ctx->oformat->flags & AVFMT_GLOBALHEADER) { 107 | #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57,24,102) 108 | m_video_stream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; 109 | #else 110 | m_video_stream->codec->flags |= CODEC_FLAG_GLOBAL_HEADER; 111 | #endif 112 | } 113 | 114 | return true; 115 | } 116 | 117 | /** 118 | * Dumps the CodecPrivate data for the Matroska/MP4 container. 119 | * Note this is the same format as required for MP4 containers - in other words, 120 | * the AVCC format (AVCDecoderConfigurationRecord structure). Some values have 121 | * been hard-coded for simplicity's sake. 122 | * @return true iff CodecPrivate (extradata) was updated. 123 | */ 124 | bool OmxCvImpl::dump_codec_private() { 125 | //Example CodecPrivate: 01 64 00 1e ff e1 00 0e 27 64 00 1e ac 2b 40 50 1e d0 0f 12 26 a0 01 00 05 28 ee 02 5c b0 126 | //See also: http://lists.matroska.org/pipermail/matroska-devel/2012-April/004196.html 127 | //And: http://stackoverflow.com/questions/24884827/possible-locations-for-sequence-picture-parameter-sets-for-h-264-stream 128 | //Note: lengthSizeMinusOne is normally 3 (4 bytes to specify NALU length) 129 | //But: Annex B NALU signature is either 3 or 4 bytes long. 130 | //However: Luckily for us, the RPi OMX encoder appears to always return 131 | //NALUs with 4 byte headers. So we're OK! If we get one with a 3 byte header 132 | //then we have to increase the buffer size by 1 to accomodate the size. 133 | assert(m_sps_length > 2); 134 | uint8_t adcr[] = { 135 | 1, //configurationVersion 136 | m_sps[1], //AVCProfileIndication 137 | m_sps[2], //profile_compatibility 138 | m_sps[3], //AVCLevelIndication 139 | 0xfc | 3, //lengthSizeMinusOne; 140 | 0xe0 | 1 //numOfSequenceParameterSets; 0xe0 for reserved bits. We have one SPS. 141 | }; 142 | 143 | AVCodecContext *c = m_video_stream->codec; 144 | av_free(c->extradata); 145 | 146 | c->profile = m_sps[1]; //Ususally 'High' profile 147 | c->level = m_sps[3]; //Usually Level 3.0 148 | 149 | c->extradata_size = sizeof(adcr) + 2 + m_sps_length + 1 + 2 + m_pps_length; 150 | c->extradata = static_cast(av_malloc(c->extradata_size)); 151 | if (c->extradata) { 152 | memcpy(c->extradata, &adcr, sizeof(adcr)); 153 | c->extradata[sizeof(adcr)] = m_sps_length >> 8; 154 | c->extradata[sizeof(adcr)+1] = m_sps_length & 0xFF; 155 | memcpy(c->extradata+sizeof(adcr)+2, m_sps, m_sps_length); 156 | c->extradata[sizeof(adcr)+2+m_sps_length] = 1; //PPS count 157 | c->extradata[sizeof(adcr)+2+m_sps_length+1] = m_pps_length >> 8; 158 | c->extradata[sizeof(adcr)+2+m_sps_length+2] = m_pps_length & 0xFF; 159 | memcpy(c->extradata+sizeof(adcr)+2+m_sps_length+3, m_pps, m_pps_length); 160 | } else { 161 | perror("Extradata allocation failed"); 162 | return false; 163 | } 164 | 165 | //Write the file header. 166 | avformat_write_header(m_mux_ctx, NULL); 167 | //Print info about this format 168 | av_dump_format(m_mux_ctx, 0, m_filename.c_str(), 1); 169 | return true; 170 | } 171 | 172 | /** 173 | * Constructor. 174 | * @param [in] name The file to save to. 175 | * @param [in] width The video width. 176 | * @param [in] height The video height. 177 | * @param [in] bitrate The bitrate, in Kbps. 178 | * @param [in] fpsnum The FPS numerator. 179 | * @param [in] fpsden The FPS denominator. 180 | */ 181 | OmxCvImpl::OmxCvImpl(const char *name, int width, int height, int bitrate, int fpsnum, int fpsden) 182 | : m_width(width) 183 | , m_height(height) 184 | , m_stride(((width + 31) & ~31) * 3) 185 | , m_bitrate(bitrate) 186 | , m_sps(nullptr) 187 | , m_pps(nullptr) 188 | , m_sps_length(0) 189 | , m_pps_length(0) 190 | , m_nalu_filled(0) 191 | , m_nalu_required(0) 192 | , m_initted_header(false) 193 | , m_filename(name) 194 | , m_stop{false} 195 | { 196 | int ret; 197 | bcm_host_init(); 198 | 199 | if (fpsden <= 0 || fpsnum <= 0) { 200 | fpsden = 1; 201 | fpsnum = 25; 202 | } 203 | m_fpsnum = fpsnum; 204 | m_fpsden = fpsden; 205 | //Initialise the scaler and output file 206 | CHECKED(!lav_init(), "Failed to initialise LibAVFormat."); 207 | 208 | //Initialise OpenMAX and the IL client. 209 | CHECKED(OMX_Init() != OMX_ErrorNone, "OMX_Init failed."); 210 | m_ilclient = ilclient_init(); 211 | CHECKED(m_ilclient == NULL, "ILClient initialisation failed."); 212 | 213 | ret = ilclient_create_component(m_ilclient, &m_encoder_component, 214 | (char*)"video_encode", 215 | (ILCLIENT_CREATE_FLAGS_T)(ILCLIENT_DISABLE_ALL_PORTS | 216 | ILCLIENT_ENABLE_INPUT_BUFFERS | ILCLIENT_ENABLE_OUTPUT_BUFFERS)); 217 | CHECKED(ret != 0, "ILCient video_encode component creation failed."); 218 | 219 | //Set input definition to the encoder 220 | OMX_PARAM_PORTDEFINITIONTYPE def = {0}; 221 | def.nSize = sizeof(OMX_PARAM_PORTDEFINITIONTYPE); 222 | def.nVersion.nVersion = OMX_VERSION; 223 | def.nPortIndex = OMX_ENCODE_PORT_IN; 224 | ret = OMX_GetParameter(ILC_GET_HANDLE(m_encoder_component), 225 | OMX_IndexParamPortDefinition, &def); 226 | CHECKED(ret != OMX_ErrorNone, "OMX_GetParameter failed for encode port in."); 227 | 228 | def.format.video.nFrameWidth = m_width; 229 | def.format.video.nFrameHeight = m_height; 230 | def.format.video.xFramerate = 30 << 16; 231 | //Must be a multiple of 16 232 | def.format.video.nSliceHeight = (m_height + 15) & ~15; 233 | //Must be a multiple of 32 234 | def.format.video.nStride = m_stride; 235 | def.format.video.eColorFormat = OMX_COLOR_Format24bitBGR888; //OMX_COLOR_Format32bitABGR8888;//OMX_COLOR_FormatYUV420PackedPlanar; 236 | //Must be manually defined to ensure sufficient size if stride needs to be rounded up to multiple of 32. 237 | def.nBufferSize = def.format.video.nStride * def.format.video.nSliceHeight; 238 | //We allocate 6 input buffers. 239 | def.nBufferCountActual = 6; 240 | 241 | ret = OMX_SetParameter(ILC_GET_HANDLE(m_encoder_component), 242 | OMX_IndexParamPortDefinition, &def); 243 | CHECKED(ret != OMX_ErrorNone, "OMX_SetParameter failed for input format definition."); 244 | 245 | //Set the output format of the encoder 246 | OMX_VIDEO_PARAM_PORTFORMATTYPE format = {0}; 247 | format.nSize = sizeof(OMX_VIDEO_PARAM_PORTFORMATTYPE); 248 | format.nVersion.nVersion = OMX_VERSION; 249 | format.nPortIndex = OMX_ENCODE_PORT_OUT; 250 | format.eCompressionFormat = OMX_VIDEO_CodingAVC; 251 | 252 | ret = OMX_SetParameter(ILC_GET_HANDLE(m_encoder_component), 253 | OMX_IndexParamVideoPortFormat, &format); 254 | CHECKED(ret != OMX_ErrorNone, "OMX_SetParameter failed for setting encoder output format."); 255 | 256 | //Set the encoding bitrate 257 | OMX_VIDEO_PARAM_BITRATETYPE bitrate_type = {0}; 258 | bitrate_type.nSize = sizeof(OMX_VIDEO_PARAM_BITRATETYPE); 259 | bitrate_type.nVersion.nVersion = OMX_VERSION; 260 | bitrate_type.eControlRate = OMX_Video_ControlRateVariable; 261 | bitrate_type.nTargetBitrate = bitrate * 1000; 262 | bitrate_type.nPortIndex = OMX_ENCODE_PORT_OUT; 263 | ret = OMX_SetParameter(ILC_GET_HANDLE(m_encoder_component), 264 | OMX_IndexParamVideoBitrate, &bitrate_type); 265 | CHECKED(ret != OMX_ErrorNone, "OMX_SetParameter failed for setting encoder bitrate."); 266 | 267 | //I think this decreases the chance of NALUs being split across buffers. 268 | /* 269 | OMX_CONFIG_BOOLEANTYPE frg = {0}; 270 | frg.nSize = sizeof(OMX_CONFIG_BOOLEANTYPE); 271 | frg.nVersion.nVersion = OMX_VERSION; 272 | frg.bEnabled = OMX_TRUE; 273 | ret = OMX_SetParameter(ILC_GET_HANDLE(m_encoder_component), 274 | OMX_IndexConfigMinimiseFragmentation, &frg); 275 | CHECKED(ret != 0, "OMX_SetParameter failed for setting fragmentation minimisation."); 276 | */ 277 | 278 | //We want at most one NAL per output buffer that we receive. 279 | OMX_CONFIG_BOOLEANTYPE nal = {0}; 280 | nal.nSize = sizeof(OMX_CONFIG_BOOLEANTYPE); 281 | nal.nVersion.nVersion = OMX_VERSION; 282 | nal.bEnabled = OMX_TRUE; 283 | ret = OMX_SetParameter(ILC_GET_HANDLE(m_encoder_component), 284 | OMX_IndexParamBrcmNALSSeparate, &nal); 285 | CHECKED(ret != 0, "OMX_SetParameter failed for setting separate NALUs."); 286 | 287 | //We want the encoder to write the NALU length instead start codes. 288 | OMX_NALSTREAMFORMATTYPE nal2 = {0}; 289 | nal2.nSize = sizeof(OMX_NALSTREAMFORMATTYPE); 290 | nal2.nVersion.nVersion = OMX_VERSION; 291 | nal2.nPortIndex = OMX_ENCODE_PORT_OUT; 292 | nal2.eNaluFormat = OMX_NaluFormatFourByteInterleaveLength; 293 | ret = OMX_SetParameter(ILC_GET_HANDLE(m_encoder_component), 294 | (OMX_INDEXTYPE)OMX_IndexParamNalStreamFormatSelect, &nal2); 295 | CHECKED(ret != 0, "OMX_SetParameter failed for setting NALU format."); 296 | 297 | ret = ilclient_change_component_state(m_encoder_component, OMX_StateIdle); 298 | CHECKED(ret != 0, "ILClient failed to change encoder to idle state."); 299 | ret = ilclient_enable_port_buffers(m_encoder_component, OMX_ENCODE_PORT_IN, NULL, NULL, NULL); 300 | CHECKED(ret != 0, "ILClient failed to enable input buffers."); 301 | ret = ilclient_enable_port_buffers(m_encoder_component, OMX_ENCODE_PORT_OUT, NULL, NULL, NULL); 302 | CHECKED(ret != 0, "ILClient failed to enable output buffers."); 303 | ret = ilclient_change_component_state(m_encoder_component, OMX_StateExecuting); 304 | CHECKED(ret != 0, "ILClient failed to change encoder to executing stage."); 305 | 306 | //Initialise NALU buffer 307 | m_nalu_buffer = new uint8_t[MAX_NALU_SIZE]; 308 | 309 | //Start the worker thread for dumping the encoded data 310 | m_input_worker = std::thread(&OmxCvImpl::input_worker, this); 311 | } 312 | 313 | /** 314 | * Destructor. 315 | * @return Return_Description 316 | */ 317 | OmxCvImpl::~OmxCvImpl() { 318 | m_stop = true; 319 | m_input_signaller.notify_one(); 320 | m_input_worker.join(); 321 | 322 | //Free the SPS and PPS headers. 323 | delete [] m_sps; 324 | delete [] m_pps; 325 | //Free the NALU buffer. 326 | delete [] m_nalu_buffer; 327 | 328 | //Teardown similar to hello_encode 329 | ilclient_change_component_state(m_encoder_component, OMX_StateIdle); 330 | ilclient_disable_port_buffers(m_encoder_component, OMX_ENCODE_PORT_IN, NULL, NULL, NULL); 331 | ilclient_disable_port_buffers(m_encoder_component, OMX_ENCODE_PORT_OUT, NULL, NULL, NULL); 332 | 333 | //ilclient_change_component_state(m_encoder_component, OMX_StateIdle); 334 | ilclient_change_component_state(m_encoder_component, OMX_StateLoaded); 335 | 336 | COMPONENT_T *list[] = {m_encoder_component, NULL}; 337 | ilclient_cleanup_components(list); 338 | ilclient_destroy(m_ilclient); 339 | 340 | //Close the output file 341 | av_write_trailer(m_mux_ctx); 342 | avcodec_close(m_video_stream->codec); 343 | avio_close(m_mux_ctx->pb); 344 | avformat_free_context(m_mux_ctx); 345 | } 346 | 347 | /** 348 | * Input encoding routine. 349 | */ 350 | void OmxCvImpl::input_worker() { 351 | std::unique_lock lock(m_input_mutex); 352 | OMX_BUFFERHEADERTYPE *out = ilclient_get_output_buffer(m_encoder_component, OMX_ENCODE_PORT_OUT, 1); 353 | 354 | while (true) { 355 | m_input_signaller.wait(lock, [this]{return m_stop || m_input_queue.size() > 0;}); 356 | if (m_stop) { 357 | if (m_input_queue.size() > 0) { 358 | printf("Stop acknowledged but need to flush the input buffer (%d)...\n", m_input_queue.size()); 359 | } else { 360 | break; 361 | } 362 | } 363 | 364 | //auto proc_start = steady_clock::now(); 365 | std::pair frame = m_input_queue.front(); 366 | m_input_queue.pop_front(); 367 | lock.unlock(); 368 | 369 | //auto conv_start = steady_clock::now(); 370 | //static int framecounter = 0; 371 | 372 | OMX_EmptyThisBuffer(ILC_GET_HANDLE(m_encoder_component), frame.first); 373 | //fflush(stdout); 374 | //printf("Encoding time (ms): %d [%d]\r", (int)TIMEDIFF(conv_start), ++framecounter); 375 | do { 376 | out->nFilledLen = 0; //I don't think this is necessary, but whatever. 377 | OMX_FillThisBuffer(ILC_GET_HANDLE(m_encoder_component), out); 378 | out = ilclient_get_output_buffer(m_encoder_component, OMX_ENCODE_PORT_OUT, 1); 379 | } while (!write_data(out, frame.second)); 380 | 381 | lock.lock(); 382 | //printf("Total processing time (ms): %d\n", (int)TIMEDIFF(proc_start)); 383 | } 384 | 385 | //Needed because we call ilclient_get_output_buffer last. 386 | //Otherwise ilclient waits forever for the buffer to be filled. 387 | OMX_FillThisBuffer(ILC_GET_HANDLE(m_encoder_component), out); 388 | } 389 | 390 | /** 391 | * Output muxing routine. 392 | * @param [in] out Buffer to be saved. 393 | * @param [in] timestamp Timestamp of this buffer. 394 | * @return true if buffer was saved. 395 | */ 396 | bool OmxCvImpl::write_data(OMX_BUFFERHEADERTYPE *out, int64_t timestamp) { 397 | uint8_t *buffer = out->pBuffer; 398 | 399 | //Do we already have a partial NALU? 400 | if (m_nalu_required > 0) { 401 | assert((out->nFilledLen+m_nalu_filled) <= m_nalu_required); 402 | memcpy(m_nalu_buffer+m_nalu_filled, out->pBuffer, out->nFilledLen); 403 | m_nalu_filled += out->nFilledLen; 404 | 405 | //Have we got a complete NALU now? 406 | if (m_nalu_filled == m_nalu_required) { 407 | printf("Fulfilled NALU of size %d.\n", m_nalu_filled); 408 | buffer = m_nalu_buffer; 409 | out->nFilledLen = m_nalu_filled; 410 | m_nalu_filled = m_nalu_required = 0; 411 | } else { 412 | return false; 413 | } 414 | } 415 | 416 | //Check for an SPS/PPS header 417 | //First 4 bytes are NALU length (Big endian) 418 | assert(out->nFilledLen > 4); 419 | uint32_t nallength = (buffer[0] << 24) | (buffer[1] << 16) | 420 | (buffer[2] << 8) | (buffer[3]); 421 | if (nallength != out->nFilledLen-4) { 422 | printf("Indicated NAL length of %d is different to filled buffer size (%d)!\n", nallength, out->nFilledLen); 423 | assert(nallength > (out->nFilledLen-4) && nallength <= MAX_NALU_SIZE); 424 | 425 | m_nalu_required = nallength+4; //Size of NALU+4 bytes for length 426 | m_nalu_filled = out->nFilledLen; //How much we've got so far 427 | memcpy(m_nalu_buffer, out->pBuffer, out->nFilledLen); 428 | return false; 429 | } 430 | 431 | uint8_t naltype = buffer[4] & 0x1f; 432 | const uint8_t sig[4] = {0x00, 0x00, 0x00, 0x01}; 433 | 434 | if (!m_initted_header) { 435 | if (naltype == 7) { //SPS 436 | uint8_t *ptr = (uint8_t*)memmem(buffer, out->nFilledLen, sig, 4); 437 | if (ptr) { 438 | m_sps_length = (ptr-buffer)-4; 439 | m_sps = new uint8_t[m_sps_length]; 440 | memcpy(m_sps, buffer+4, m_sps_length); 441 | 442 | m_pps_length = out->nFilledLen-8-m_sps_length; 443 | m_pps = new uint8_t[m_pps_length]; 444 | memcpy(m_pps, ptr+4, m_pps_length); 445 | } else { 446 | m_sps_length = out->nFilledLen - 4; 447 | m_sps = new uint8_t[m_sps_length]; 448 | //Strip the NAL header. 449 | memcpy(m_sps, buffer+4, m_sps_length); 450 | } 451 | } else if (naltype == 8) { 452 | m_pps_length = out->nFilledLen - 4; 453 | m_pps = new uint8_t[m_pps_length]; 454 | //Strip the NAL header. 455 | memcpy(m_pps, buffer+4, m_pps_length); 456 | } else { 457 | printf("Warning: Got NALU (id %d) but no SPS/PPS set received!\n", 458 | naltype); 459 | return true; 460 | } 461 | 462 | if (m_sps && m_pps) { 463 | if (dump_codec_private()) { 464 | m_initted_header = true; 465 | } 466 | } 467 | } else if (naltype != 7 && naltype != 8){ 468 | //We only reach here if we have a full NALU to write out. 469 | AVPacket pkt; 470 | av_init_packet(&pkt); 471 | //uint8_t *ptr = (uint8_t*)memmem(buffer, out->nFilledLen, sig, 4); 472 | //assert(ptr == NULL); 473 | 474 | pkt.stream_index = m_video_stream->index; 475 | pkt.pts = timestamp; 476 | pkt.data = buffer; 477 | pkt.size = out->nFilledLen; 478 | 479 | //Check for IDR frames (keyframes) 480 | if (naltype == 5) { 481 | pkt.flags |= AV_PKT_FLAG_KEY; 482 | } 483 | 484 | av_write_frame(m_mux_ctx, &pkt); 485 | return true; 486 | } 487 | return false; 488 | } 489 | 490 | /** 491 | * Enqueue video to be encoded. 492 | * @param [in] mat The mat to be encoded. 493 | * @return true iff enqueued. 494 | */ 495 | bool OmxCvImpl::process(const cv::Mat &mat) { 496 | OMX_BUFFERHEADERTYPE *in = ilclient_get_input_buffer( 497 | m_encoder_component, OMX_ENCODE_PORT_IN, 0); 498 | if (in == NULL) { 499 | printf("No free buffer; dropping frame!\n"); 500 | return false; 501 | } 502 | 503 | assert(mat.cols == m_width && mat.rows == m_height); 504 | auto now = steady_clock::now(); 505 | BGR2RGB(mat, in->pBuffer, m_stride); 506 | in->nFilledLen = in->nAllocLen; 507 | 508 | std::unique_lock lock(m_input_mutex); 509 | if (m_frame_count++ == 0) { 510 | m_frame_start = now; 511 | } 512 | m_input_queue.push_back(std::pair( 513 | in, duration_cast(now-m_frame_start).count())); 514 | lock.unlock(); 515 | m_input_signaller.notify_one(); 516 | return true; 517 | } 518 | 519 | /** 520 | * Constructor for our wrapper. 521 | * @param [in] name The file to save to. 522 | * @param [in] width The video width. 523 | * @param [in] height The video height. 524 | * @param [in] bitrate The bitrate, in Kbps. 525 | * @param [in] fpsnum The FPS numerator. 526 | * @param [in] fpsden The FPS denominator. 527 | */ 528 | OmxCv::OmxCv(const char *name, int width, int height, int bitrate, int fpsnum, int fpsden) { 529 | m_impl = new OmxCvImpl(name, width, height, bitrate, fpsnum, fpsden); 530 | } 531 | 532 | /** 533 | * Wrapper destructor. 534 | */ 535 | OmxCv::~OmxCv() { 536 | delete m_impl; 537 | } 538 | 539 | /** 540 | * Encode image. 541 | * @param [in] in Image to be encoded. 542 | * @return true iff the image was encoded. 543 | */ 544 | bool OmxCv::Encode(const cv::Mat &in) { 545 | return m_impl->process(in); 546 | } 547 | -------------------------------------------------------------------------------- /omxcv.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file omxcv.cpp 3 | * @brief Header file for the library 4 | */ 5 | 6 | #ifndef __OMXCV_H 7 | #define __OMXCV_H 8 | 9 | #include 10 | 11 | namespace omxcv { 12 | /* Forward declaration of our H.264 implementation. */ 13 | class OmxCvImpl; 14 | /* Forward delaration of our JPEG implementation. */ 15 | class OmxCvJpegImpl; 16 | 17 | /** 18 | * Real-time OpenMAX H.264 encoder for the Raspberry Pi/OpenCV. 19 | */ 20 | class OmxCv { 21 | public: 22 | OmxCv(const char *name, int width, int height, int bitrate=3000, int fpsnum=25, int fpsden=1); 23 | bool Encode(const cv::Mat &in); 24 | virtual ~OmxCv(); 25 | private: 26 | OmxCvImpl *m_impl; 27 | }; 28 | 29 | /** 30 | * Real-time OpenMAX JPEG encoder for the Raspberry Pi/OpenCV. 31 | */ 32 | class OmxCvJpeg { 33 | public: 34 | OmxCvJpeg(int width, int height, int quality=90); 35 | bool Encode(const char *filename, const cv::Mat &in, bool fallback=false); 36 | virtual ~OmxCvJpeg(); 37 | private: 38 | OmxCvJpegImpl *m_impl; 39 | int m_width, m_height, m_quality; 40 | }; 41 | } 42 | 43 | #endif /* _OMXCV_H */ 44 | -------------------------------------------------------------------------------- /omxcv_jpeg.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file omxcv_jpeg.cpp 3 | * @brief OpenMAX JPEG encoder for OpenCV. 4 | */ 5 | 6 | #include "omxcv.h" 7 | #include "omxcv-impl.h" 8 | #include 9 | 10 | using namespace omxcv; 11 | 12 | /** 13 | * Constructor. 14 | * @param [in] width The width of the image to encode. 15 | * @param [in] height The height of the image to encode. 16 | * @param [in] quality The JPEG quality factor (1-100). 100 is best quality. 17 | * @throws std::invalid_argument on error. 18 | */ 19 | OmxCvJpegImpl::OmxCvJpegImpl(int width, int height, int quality) 20 | : m_width(width) 21 | , m_height(height) 22 | , m_stride(((width + 31) & ~31) * 3) 23 | , m_quality(quality) 24 | , m_stop{false} 25 | { 26 | int ret; 27 | bcm_host_init(); 28 | 29 | //Initialise OpenMAX and the IL client. 30 | CHECKED(OMX_Init() != OMX_ErrorNone, "OMX_Init failed."); 31 | m_ilclient = ilclient_init(); 32 | CHECKED(m_ilclient == NULL, "ILClient initialisation failed."); 33 | 34 | ret = ilclient_create_component(m_ilclient, &m_encoder_component, 35 | (char*)"image_encode", 36 | (ILCLIENT_CREATE_FLAGS_T)(ILCLIENT_DISABLE_ALL_PORTS | 37 | ILCLIENT_ENABLE_INPUT_BUFFERS | ILCLIENT_ENABLE_OUTPUT_BUFFERS)); 38 | CHECKED(ret != 0, "ILCient image_encode component creation failed."); 39 | 40 | //Set input definition to the encoder 41 | OMX_PARAM_PORTDEFINITIONTYPE def = {0}; 42 | def.nSize = sizeof(OMX_PARAM_PORTDEFINITIONTYPE); 43 | def.nVersion.nVersion = OMX_VERSION; 44 | def.nPortIndex = OMX_JPEG_PORT_IN; 45 | ret = OMX_GetParameter(ILC_GET_HANDLE(m_encoder_component), 46 | OMX_IndexParamPortDefinition, &def); 47 | CHECKED(ret != OMX_ErrorNone, "OMX_GetParameter failed for encode port in."); 48 | 49 | //We allocate 3 input buffers. 50 | def.nBufferCountActual = 3; 51 | def.format.image.nFrameWidth = m_width; 52 | def.format.image.nFrameHeight = m_height; 53 | //16 byte alignment. I don't know if these also hold for image encoding. 54 | def.format.image.nSliceHeight = (m_height + 15) & ~15; 55 | //Must be manually defined to ensure sufficient size if stride needs to be rounded up to multiple of 32. 56 | def.nBufferSize = def.format.image.nStride * def.format.image.nSliceHeight; 57 | def.format.image.nStride = m_stride; 58 | def.format.image.bFlagErrorConcealment = OMX_FALSE; 59 | def.format.image.eColorFormat = OMX_COLOR_Format24bitBGR888; //OMX_COLOR_Format32bitABGR8888;//OMX_COLOR_FormatYUV420PackedPlanar; 60 | 61 | ret = OMX_SetParameter(ILC_GET_HANDLE(m_encoder_component), 62 | OMX_IndexParamPortDefinition, &def); 63 | CHECKED(ret != OMX_ErrorNone, "OMX_SetParameter failed for input format definition."); 64 | 65 | //Set the output format of the encoder 66 | OMX_IMAGE_PARAM_PORTFORMATTYPE format = {0}; 67 | format.nSize = sizeof(OMX_IMAGE_PARAM_PORTFORMATTYPE); 68 | format.nVersion.nVersion = OMX_VERSION; 69 | format.nPortIndex = OMX_JPEG_PORT_OUT; 70 | format.eCompressionFormat = OMX_IMAGE_CodingJPEG; 71 | 72 | ret = OMX_SetParameter(ILC_GET_HANDLE(m_encoder_component), 73 | OMX_IndexParamImagePortFormat, &format); 74 | CHECKED(ret != OMX_ErrorNone, "OMX_SetParameter failed for setting encoder output format."); 75 | 76 | //Set the encoder quality 77 | OMX_IMAGE_PARAM_QFACTORTYPE qfactor = {0}; 78 | qfactor.nSize = sizeof(OMX_IMAGE_PARAM_QFACTORTYPE); 79 | qfactor.nVersion.nVersion = OMX_VERSION; 80 | qfactor.nPortIndex = OMX_JPEG_PORT_OUT; 81 | qfactor.nQFactor = m_quality; 82 | 83 | ret = OMX_SetParameter(ILC_GET_HANDLE(m_encoder_component), 84 | OMX_IndexParamQFactor, &qfactor); 85 | CHECKED(ret != OMX_ErrorNone, "OMX_SetParameter failed for setting encoder quality."); 86 | 87 | ret = ilclient_change_component_state(m_encoder_component, OMX_StateIdle); 88 | CHECKED(ret != 0, "ILClient failed to change encoder to idle state."); 89 | ret = ilclient_enable_port_buffers(m_encoder_component, OMX_JPEG_PORT_IN, NULL, NULL, NULL); 90 | CHECKED(ret != 0, "ILClient failed to enable input buffers."); 91 | ret = ilclient_enable_port_buffers(m_encoder_component, OMX_JPEG_PORT_OUT, NULL, NULL, NULL); 92 | CHECKED(ret != 0, "ILClient failed to enable output buffers."); 93 | ret = ilclient_change_component_state(m_encoder_component, OMX_StateExecuting); 94 | CHECKED(ret != 0, "ILClient failed to change encoder to executing stage."); 95 | 96 | //Start the worker thread for dumping the encoded data 97 | m_input_worker = std::thread(&OmxCvJpegImpl::input_worker, this); 98 | } 99 | 100 | /** 101 | * Destructor. 102 | */ 103 | OmxCvJpegImpl::~OmxCvJpegImpl() { 104 | m_stop = true; 105 | m_input_signaller.notify_one(); 106 | m_input_worker.join(); 107 | 108 | //Teardown similar to hello_encode 109 | ilclient_change_component_state(m_encoder_component, OMX_StateIdle); 110 | ilclient_disable_port_buffers(m_encoder_component, OMX_JPEG_PORT_IN, NULL, NULL, NULL); 111 | ilclient_disable_port_buffers(m_encoder_component, OMX_JPEG_PORT_OUT, NULL, NULL, NULL); 112 | 113 | //ilclient_change_component_state(m_encoder_component, OMX_StateIdle); 114 | ilclient_change_component_state(m_encoder_component, OMX_StateLoaded); 115 | 116 | COMPONENT_T *list[] = {m_encoder_component, NULL}; 117 | ilclient_cleanup_components(list); 118 | ilclient_destroy(m_ilclient); 119 | } 120 | 121 | /** 122 | * Worker thread. Encodes the images. 123 | */ 124 | void OmxCvJpegImpl::input_worker() { 125 | std::unique_lock lock(m_input_mutex); 126 | OMX_BUFFERHEADERTYPE *out = ilclient_get_output_buffer(m_encoder_component, OMX_JPEG_PORT_OUT, 1); 127 | 128 | while (true) { 129 | m_input_signaller.wait(lock, [this]{return m_stop || m_input_queue.size() > 0;}); 130 | if (m_stop) { 131 | if (m_input_queue.size() > 0) { 132 | printf("Stop acknowledged but need to flush the input buffer (%d)...\n", m_input_queue.size()); 133 | } else { 134 | break; 135 | } 136 | } 137 | 138 | std::pair frame = m_input_queue.front(); 139 | m_input_queue.pop_front(); 140 | lock.unlock(); 141 | 142 | FILE *fp = fopen(frame.second.c_str(), "wb"); 143 | if (!fp) { 144 | perror(frame.second.c_str()); 145 | } 146 | 147 | OMX_EmptyThisBuffer(ILC_GET_HANDLE(m_encoder_component), frame.first); 148 | do { 149 | OMX_FillThisBuffer(ILC_GET_HANDLE(m_encoder_component), out); 150 | out = ilclient_get_output_buffer(m_encoder_component, OMX_JPEG_PORT_OUT, 1); 151 | if (fp) { 152 | fwrite(out->pBuffer, 1, out->nFilledLen, fp); 153 | } 154 | } while (!(out->nFlags & OMX_BUFFERFLAG_ENDOFFRAME)); 155 | 156 | if (fp) { 157 | fclose(fp); 158 | } 159 | lock.lock(); 160 | } 161 | 162 | //Needed because we call ilclient_get_output_buffer last. 163 | //Otherwise ilclient waits forever for the buffer to be filled. 164 | OMX_FillThisBuffer(ILC_GET_HANDLE(m_encoder_component), out); 165 | } 166 | 167 | /** 168 | * Process a frame. 169 | * @param [in] filename The filename to save to. 170 | * @param [in] mat The image data to save. 171 | * @return true iff the image will be saved. Will return false if there's no 172 | * free input buffer. 173 | */ 174 | bool OmxCvJpegImpl::process(const char *filename, const cv::Mat &mat) { 175 | //static const std::vector saveparams = {CV_IMWRITE_JPEG_QUALITY, 75}; 176 | //cv::imwrite(filename, mat, saveparams); 177 | OMX_BUFFERHEADERTYPE *in = ilclient_get_input_buffer( 178 | m_encoder_component, OMX_JPEG_PORT_IN, 0); 179 | if (in == NULL) { //No free buffer. 180 | return false; 181 | } 182 | 183 | assert(mat.cols == m_width && mat.rows == m_height); 184 | BGR2RGB(mat, in->pBuffer, m_stride); 185 | in->nFilledLen = in->nAllocLen; 186 | 187 | std::unique_lock lock(m_input_mutex); 188 | m_input_queue.push_back(std::pair( 189 | in, std::string(filename))); 190 | lock.unlock(); 191 | m_input_signaller.notify_one(); 192 | return true; 193 | } 194 | 195 | 196 | /** 197 | * Constructor for our wrapper. 198 | * @param [in] width The width of the image to encode. 199 | * @param [in] height The height of the image to encode. 200 | * @param [in] quality The JPEG quality factor (1-100). 100 is best quality. 201 | */ 202 | OmxCvJpeg::OmxCvJpeg(int width, int height, int quality) 203 | : m_width(width) 204 | , m_height(height) 205 | , m_quality(quality) 206 | { 207 | m_impl = new OmxCvJpegImpl(width, height, quality); 208 | } 209 | 210 | /** 211 | * Wrapper destructor. 212 | */ 213 | OmxCvJpeg::~OmxCvJpeg() { 214 | delete m_impl; 215 | } 216 | 217 | /** 218 | * Encode image. 219 | * @param [in] filename The path to save the image to. 220 | * @param [in] in Image to be encoded. If the width and height of this 221 | * image does not match what was set in the constructor, 222 | * this function will fallback to using OpenCV's imwrite. 223 | * @param [in] fallback If set to true and there is no available buffer for 224 | * encoding, fallback to using OpenCV to write out the image. 225 | * @return true iff the file was encoded. 226 | */ 227 | bool OmxCvJpeg::Encode(const char *filename, const cv::Mat &in, bool fallback) { 228 | if (in.cols != m_width || in.rows != m_height || in.type() != CV_8UC3) { 229 | std::vector params {cv::IMWRITE_JPEG_QUALITY, m_quality}; 230 | return cv::imwrite(filename, in, params); 231 | } 232 | 233 | bool ret = m_impl->process(filename, in); 234 | if (!ret && fallback) { 235 | std::vector params {cv::IMWRITE_JPEG_QUALITY, m_quality}; 236 | return cv::imwrite(filename, in, params); 237 | } 238 | return ret; 239 | } 240 | 241 | -------------------------------------------------------------------------------- /simplefragshader.glsl: -------------------------------------------------------------------------------- 1 | varying vec2 tcoord; 2 | uniform sampler2D tex; 3 | uniform vec4 threshLow, threshHigh; 4 | 5 | void main(void) 6 | { 7 | mat4 RGBtoYUV = mat4(0.257, 0.439, -0.148, 0.0, 8 | 0.504, -0.368, -0.291, 0.0, 9 | 0.098, -0.071, 0.439, 0.0, 10 | 0.0625, 0.500, 0.500, 1.0 ); 11 | 12 | //Y'CrCb 13 | vec4 yuv = RGBtoYUV * texture2D(tex,tcoord).bgra; 14 | bvec4 l = greaterThanEqual(yuv, threshLow); 15 | bvec4 h = lessThanEqual(yuv, threshHigh); 16 | 17 | if (all(l) && all(h)) { 18 | gl_FragColor = vec4(1,1,1,1); 19 | } else { 20 | gl_FragColor = vec4(0,0,0,1); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /simplevertshader.glsl: -------------------------------------------------------------------------------- 1 | attribute vec4 vPosition; 2 | varying vec2 tcoord; 3 | 4 | void main(void) 5 | { 6 | vec4 pos = vPosition; 7 | tcoord = pos.xy; 8 | pos.xy = pos.xy*vec2(2,2) + vec2(-1,-1); 9 | gl_Position = pos; 10 | } 11 | --------------------------------------------------------------------------------