├── src
    ├── manifest.cpp
    ├── tdiff.cpp
    ├── ffmpeg_subscriber.cpp
    ├── ffmpeg_publisher.cpp
    ├── ffmpeg_encoder.cpp
    └── ffmpeg_decoder.cpp
├── ffmpeg_plugins.xml
├── include
    └── ffmpeg_image_transport
    │   ├── tdiff.h
    │   ├── ffmpeg_subscriber.h
    │   ├── ffmpeg_publisher.h
    │   ├── ffmpeg_decoder.h
    │   └── ffmpeg_encoder.h
├── package.xml
├── CONTRIBUTING.md
├── CMakeLists.txt
├── cfg
    └── EncoderDyn.cfg
├── cmake
    └── FindFFMPEG.cmake
├── docs
    └── compile_ffmpeg.md
├── README.md
└── LICENSE


/src/manifest.cpp:
--------------------------------------------------------------------------------
1 | #include <pluginlib/class_list_macros.h>
2 | #include "ffmpeg_image_transport/ffmpeg_publisher.h"
3 | #include "ffmpeg_image_transport/ffmpeg_subscriber.h"
4 | 
5 | PLUGINLIB_EXPORT_CLASS(ffmpeg_image_transport::FFMPEGPublisher,  image_transport::PublisherPlugin)
6 | PLUGINLIB_EXPORT_CLASS(ffmpeg_image_transport::FFMPEGSubscriber, image_transport::SubscriberPlugin)
7 | 


--------------------------------------------------------------------------------
/src/tdiff.cpp:
--------------------------------------------------------------------------------
 1 | /* -*-c++-*--------------------------------------------------------------------
 2 |  * 2018 Bernd Pfrommer bernd.pfrommer@gmail.com
 3 |  */
 4 | 
 5 | #include "ffmpeg_image_transport/tdiff.h"
 6 | #include <iomanip>
 7 | 
 8 | namespace ffmpeg_image_transport {
 9 |   std::ostream &operator<<(std::ostream &os, const TDiff &td) {
10 |     os << std::fixed << std::setprecision(4) <<
11 |       td.duration_ * (td.cnt_ > 0 ? 1.0 / (double)td.cnt_ : 0);
12 |     return os;
13 |   }
14 |  
15 | }  // namespace
16 | 


--------------------------------------------------------------------------------
/ffmpeg_plugins.xml:
--------------------------------------------------------------------------------
 1 | <library path="lib/libffmpeg_image_transport">
 2 |   <class name="image_transport/ffmpeg_pub" type="ffmpeg_image_transport::FFMPEGPublisher" base_class_type="image_transport::PublisherPlugin">
 3 |     <description>
 4 |       This plugin encodes frame into ffmpeg compressed packets
 5 |     </description>
 6 |   </class>
 7 | 
 8 |   <class name="image_transport/ffmpeg_sub" type="ffmpeg_image_transport::FFMPEGSubscriber" base_class_type="image_transport::SubscriberPlugin">
 9 |     <description>
10 |       This plugin decodes frame from ffmpeg compressed packets
11 |     </description>
12 |   </class>
13 | </library>
14 | 		  
15 | 


--------------------------------------------------------------------------------
/include/ffmpeg_image_transport/tdiff.h:
--------------------------------------------------------------------------------
 1 | /* -*-c++-*--------------------------------------------------------------------
 2 |  * 2018 Bernd Pfrommer bernd.pfrommer@gmail.com
 3 |  */
 4 | 
 5 | #pragma once
 6 | #include <iostream>
 7 | 
 8 | namespace ffmpeg_image_transport {
 9 | 
10 |   class TDiff {
11 |   public:
12 |     friend std::ostream &operator<<(std::ostream &os, const TDiff &td);
13 |     inline void update(double dt) {
14 |       duration_ += dt;
15 |       cnt_++;
16 |     }
17 |     inline void reset() {
18 |       duration_ = 0;
19 |       cnt_ = 0;
20 |     }
21 |   private:
22 |     int64_t cnt_{0};
23 |     double  duration_{0};
24 |   };
25 |   std::ostream &operator<<(std::ostream &os, const TDiff &td);
26 | 
27 | }
28 | 
29 | 


--------------------------------------------------------------------------------
/package.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0"?>
 2 | <package format="2">
 3 |   <name>ffmpeg_image_transport</name>
 4 |   <version>1.0.0</version>
 5 |   <description>ffmpeg compressed image transport for ros</description>
 6 | 
 7 |   <maintainer email="bernd.pfrommer@gmail.com">Bernd Pfrommer</maintainer>
 8 |   <license>Apache</license>
 9 |   <buildtool_depend>catkin</buildtool_depend>
10 | 
11 |   <depend>roscpp</depend>
12 |   <depend>image_transport</depend>
13 |   <depend>sensor_msgs</depend>
14 |   <depend>ffmpeg_image_transport_msgs</depend>
15 |   <depend>opencv</depend>
16 |   <depend>cv_bridge</depend>
17 |   <depend>dynamic_reconfigure</depend>
18 |   <export>
19 |     <image_transport plugin="${prefix}/ffmpeg_plugins.xml"/>
20 |   </export>
21 | 
22 | </package>
23 | 


--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
 1 | Any contribution that you make to this repository will
 2 | be under the Apache 2 License, as dictated by that
 3 | [license](http://www.apache.org/licenses/LICENSE-2.0.html):
 4 | 
 5 | ~~~
 6 | 5. Submission of Contributions. Unless You explicitly state otherwise,
 7 |    any Contribution intentionally submitted for inclusion in the Work
 8 |    by You to the Licensor shall be under the terms and conditions of
 9 |    this License, without any additional terms or conditions.
10 |    Notwithstanding the above, nothing herein shall supersede or modify
11 |    the terms of any separate license agreement you may have executed
12 |    with Licensor regarding such Contributions.
13 | ~~~
14 | 
15 | Contributors must sign-off each commit by adding a `Signed-off-by: ...`
16 | line to commit messages to certify that they have the right to submit
17 | the code they are contributing to the project according to the
18 | [Developer Certificate of Origin (DCO)](https://developercertificate.org/).
19 | 


--------------------------------------------------------------------------------
/include/ffmpeg_image_transport/ffmpeg_subscriber.h:
--------------------------------------------------------------------------------
 1 | /* -*-c++-*--------------------------------------------------------------------
 2 |  * 2018 Bernd Pfrommer bernd.pfrommer@gmail.com
 3 |  */
 4 | #pragma once
 5 | 
 6 | #include "ffmpeg_image_transport_msgs/FFMPEGPacket.h"
 7 | #include "ffmpeg_image_transport/ffmpeg_decoder.h"
 8 | #include <image_transport/simple_subscriber_plugin.h>
 9 | #include <sensor_msgs/Image.h>
10 | #include <string>
11 | 
12 | namespace ffmpeg_image_transport {
13 |   using Image         = sensor_msgs::Image;
14 |   using ImagePtr      = sensor_msgs::ImagePtr;
15 |   using ImageConstPtr = sensor_msgs::ImageConstPtr;
16 |   
17 |   typedef image_transport::SimpleSubscriberPlugin<
18 |     ffmpeg_image_transport_msgs::FFMPEGPacket>  FFMPEGSubscriberPlugin;
19 |   class FFMPEGSubscriber: public FFMPEGSubscriberPlugin  {
20 |   public:
21 |     virtual ~FFMPEGSubscriber() {}
22 | 
23 |     virtual std::string getTransportName() const {
24 |       return "ffmpeg";
25 |     }
26 | 
27 |   protected:
28 |     virtual void
29 |     internalCallback(const typename FFMPEGPacket::ConstPtr& message,
30 |                      const Callback& user_cb) override;
31 |     virtual void
32 |     subscribeImpl(ros::NodeHandle &nh, const std::string &base_topic,
33 |                   uint32_t queue_size, const Callback &callback,
34 |                   const ros::VoidPtr &tracked_object,
35 |                   const image_transport::TransportHints &transport_hints)
36 |       override;
37 |   private:
38 |     void frameReady(const ImageConstPtr &img, bool isKeyFrame) const;
39 |     FFMPEGDecoder decoder_;
40 |     std::string   decoderType_;
41 |     const Callback *userCallback_;
42 |   };
43 | }
44 | 


--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required(VERSION 2.8)
 2 | project(ffmpeg_image_transport)
 3 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -g -Wall")
 4 | 
 5 | find_package(catkin REQUIRED COMPONENTS roscpp cv_bridge image_transport ffmpeg_image_transport_msgs sensor_msgs dynamic_reconfigure)
 6 | 
 7 | set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
 8 | find_package(FFMPEG REQUIRED)
 9 | 
10 | generate_dynamic_reconfigure_options(cfg/EncoderDyn.cfg)
11 | 
12 | catkin_package(
13 | INCLUDE_DIRS include
14 | LIBRARIES ${PROJECT_NAME}
15 | DEPENDS FFMPEG
16 | CATKIN_DEPENDS roscpp cv_bridge image_transport
17 | ffmpeg_image_transport_msgs sensor_msgs dynamic_reconfigure)
18 | 
19 | find_package(OpenCV)
20 | 
21 | include_directories(include
22 |   ${FFMPEG_INCLUDE_DIR}
23 |   ${catkin_INCLUDE_DIRS}
24 |   ${OpenCV_INCLUDE_DIRS})
25 | 
26 | # add the plugin
27 | add_library(${PROJECT_NAME} src/manifest.cpp src/ffmpeg_publisher.cpp
28 | src/ffmpeg_subscriber.cpp src/ffmpeg_encoder.cpp
29 | src/ffmpeg_decoder.cpp src/tdiff.cpp ${FFMPEG_LIBRARIES})
30 | 
31 | add_dependencies(${PROJECT_NAME} ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS})
32 | target_link_libraries(${PROJECT_NAME} ${catkin_LIBRARIES} ${OpenCV_LIBRARIES} ${FFMPEG_LIBRARIES})
33 | 
34 | install(DIRECTORY include/${PROJECT_NAME}/
35 |         DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION}
36 | 	FILES_MATCHING PATTERN "*.h"
37 | 	)
38 | 
39 | install(TARGETS ${PROJECT_NAME}
40 |   ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}
41 |   LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}
42 |   RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
43 | )
44 | 
45 | install(FILES
46 |   ffmpeg_plugins.xml
47 |   DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}
48 | )
49 | 


--------------------------------------------------------------------------------
/src/ffmpeg_subscriber.cpp:
--------------------------------------------------------------------------------
 1 | /* -*-c++-*--------------------------------------------------------------------
 2 |  * 2018 Bernd Pfrommer bernd.pfrommer@gmail.com
 3 |  */
 4 | 
 5 | #include "ffmpeg_image_transport/ffmpeg_subscriber.h"
 6 | #include <cv_bridge/cv_bridge.h>
 7 | #include <opencv2/imgproc/imgproc.hpp>
 8 | 
 9 | namespace ffmpeg_image_transport {
10 | 
11 |   void FFMPEGSubscriber::frameReady(const ImageConstPtr &img,
12 |                                     bool isKeyFrame) const {
13 |     (*userCallback_)(img);
14 |   }
15 | 
16 |   void FFMPEGSubscriber::subscribeImpl(
17 |     ros::NodeHandle &nh, const std::string &base_topic,
18 |     uint32_t queue_size, const Callback &callback,
19 |     const ros::VoidPtr &tracked_object,
20 |     const image_transport::TransportHints &transport_hints) {
21 |     const std::string pname = ros::this_node::getName() + "/ffmpeg/decoder_type";
22 |     nh.param<std::string>(pname, decoderType_, "");
23 |     // bump queue size a bit to avoid lost packets
24 |     queue_size = std::max((int)queue_size, 20);
25 |     FFMPEGSubscriberPlugin::subscribeImpl(nh, base_topic,
26 |                                           queue_size, callback,
27 |                                           tracked_object,
28 |                                           transport_hints);
29 |   }
30 | 
31 |   void FFMPEGSubscriber::internalCallback(const FFMPEGPacket::ConstPtr& msg,
32 |                                         const Callback& user_cb) {
33 |     if (!decoder_.isInitialized()) {
34 |       if (msg->flags == 0) {
35 |         return; // wait for key frame!
36 |       }
37 |       userCallback_ = &user_cb;
38 |       if (!decoder_.initialize(
39 |             msg, boost::bind(&FFMPEGSubscriber::frameReady, this, boost::placeholders::_1, boost::placeholders::_2),
40 |             decoderType_)) {
41 |         ROS_ERROR_STREAM("cannot initialize decoder!");
42 |         return;
43 |       }
44 |     }
45 |     decoder_.decodePacket(msg);
46 |   }
47 | }
48 | 


--------------------------------------------------------------------------------
/cfg/EncoderDyn.cfg:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python2
 2 | PACKAGE = "ffmpeg_image_transport"
 3 | 
 4 | from dynamic_reconfigure.parameter_generator_catkin import *
 5 | 
 6 | gen = ParameterGenerator()
 7 | 
 8 | #
 9 | # encoder options
10 | #
11 | encode_enum = gen.enum([gen.const("h264_nvenc",       str_t, "h264_nvenc", "h264_nvenc"),
12 |                         gen.const("hevc_nvenc",       str_t, "hevc_nvenc", "hevc_nvenc"),
13 |                         gen.const("libx264",       str_t, "libx264", "libx264")],
14 |                        "An enum to set the encoder")
15 | gen.add("encoder", str_t, 0,  "encoding method", "hevc_nvenc", edit_method=encode_enum)
16 | profile_enum = gen.enum([gen.const("main",       str_t, "main",   "main"),
17 |                          gen.const("main10",     str_t, "main10", "main10"),
18 |                          gen.const("rext",       str_t, "rext",   "rext"),
19 |                          gen.const("high",       str_t, "high",   "high")],
20 |                         "An enum to set the profile")
21 | gen.add("profile", str_t, 0,  "profile", "main", edit_method=profile_enum)
22 | preset_enum = gen.enum([gen.const("slow",       str_t, "slow", "slow"),
23 |                         gen.const("medium",     str_t, "medium", "medium"),
24 |                         gen.const("fast",       str_t, "fast", "fast"),
25 |                         gen.const("hp",         str_t, "hp", "hp"),
26 |                         gen.const("hq",         str_t, "hq", "hq"),
27 |                         gen.const("bd",         str_t, "bd", "bd"),
28 |                         gen.const("ll",         str_t, "ll", "low latency"),
29 |                         gen.const("llhq",       str_t, "llhq", "low latency hq"),
30 |                         gen.const("llhp",       str_t, "llhp", "low latency hp"),
31 |                         gen.const("lossless",   str_t, "lossless", "lossless"),
32 |                         gen.const("losslesshp", str_t, "losslesshp", "lossless hp")],
33 |                         "An enum to set the preset")
34 | gen.add("preset", str_t, 0,  "preset", "slow", edit_method=preset_enum)
35 | 
36 | 
37 | gen.add("qmax",  int_t, 0, "maximum allowed quantization", 10, 0, 63)
38 | gen.add("bit_rate",  int_t, 0, "bit_rate [in bits_sec]", 8242880, 10, 2000000000)
39 | gen.add("gop_size",  int_t, 0, "gop_size [frames]", 15, 1, 128)
40 | gen.add("measure_performance",  bool_t, 0, "measure performance", False)
41 | gen.add("performance_interval",  int_t, 0, "interval between perf printout [frames]", 175, 1, 100000)
42 | 
43 | exit(gen.generate(PACKAGE, "ffmpeg_image_transport", "EncoderDyn"))
44 | 


--------------------------------------------------------------------------------
/include/ffmpeg_image_transport/ffmpeg_publisher.h:
--------------------------------------------------------------------------------
 1 | /* -*-c++-*--------------------------------------------------------------------
 2 |  * 2018 Bernd Pfrommer bernd.pfrommer@gmail.com
 3 |  */
 4 | 
 5 | #pragma once
 6 | 
 7 | #include "ffmpeg_image_transport_msgs/FFMPEGPacket.h"
 8 | #include "ffmpeg_image_transport/ffmpeg_encoder.h"
 9 | #include "ffmpeg_image_transport/EncoderDynConfig.h"
10 | 
11 | #include <image_transport/simple_publisher_plugin.h>
12 | #include <dynamic_reconfigure/server.h>
13 | 
14 | #include <mutex>
15 | #include <memory>
16 | 
17 | namespace ffmpeg_image_transport {
18 |   typedef image_transport::SimplePublisherPlugin<
19 |     ffmpeg_image_transport_msgs::FFMPEGPacket>  FFMPEGPublisherPlugin;
20 |   class FFMPEGPublisher : public FFMPEGPublisherPlugin {
21 |     typedef std::unique_lock<std::recursive_mutex> Lock;
22 |     using FFMPEGPacketConstPtr = ffmpeg_image_transport_msgs::FFMPEGPacketConstPtr;
23 |   public:
24 |     virtual std::string getTransportName() const override {
25 |       return "ffmpeg";
26 |     }
27 |     void configure(EncoderDynConfig& config, int level);
28 | 
29 |   protected:
30 |     // override to set up reconfigure server
31 |     virtual void advertiseImpl(ros::NodeHandle &nh,
32 |                                const std::string &base_topic,
33 |                                uint32_t queue_size,
34 |                                const image_transport::SubscriberStatusCallback
35 |                                &user_connect_cb,
36 |                                const image_transport::SubscriberStatusCallback
37 |                                &user_disconnect_cb,
38 |                                const ros::VoidPtr &tracked_object,
39 |                                bool latch) override;
40 |     void publish(const sensor_msgs::Image& message,
41 |                  const PublishFn& publish_fn) const override;
42 |     void connectCallback(const ros::SingleSubscriberPublisher &pub) override;
43 |     void disconnectCallback(const ros::SingleSubscriberPublisher &pub) override;
44 | 
45 |   private:
46 |     void packetReady(const FFMPEGPacketConstPtr &pkt);
47 |     void setCodecFromConfig(const EncoderDynConfig &cfg);
48 |     void initConfigServer();
49 |     // variables ---------
50 |     typedef dynamic_reconfigure::Server<EncoderDynConfig> ConfigServer;
51 |     std::shared_ptr<ros::NodeHandle>   nh_;
52 |     std::shared_ptr<ConfigServer> configServer_;
53 |     const PublishFn              *publishFunction_{NULL};
54 |     FFMPEGEncoder                 encoder_;
55 |     unsigned int                  frameCounter_{0};
56 |     EncoderDynConfig              config_;
57 |     std::recursive_mutex          configMutex_;
58 |   };
59 | }
60 | 


--------------------------------------------------------------------------------
/include/ffmpeg_image_transport/ffmpeg_decoder.h:
--------------------------------------------------------------------------------
 1 | /* -*-c++-*--------------------------------------------------------------------
 2 |  * 2018 Bernd Pfrommer bernd.pfrommer@gmail.com
 3 |  */
 4 | 
 5 | #pragma once
 6 | 
 7 | #include "ffmpeg_image_transport_msgs/FFMPEGPacket.h"
 8 | #include "ffmpeg_image_transport/tdiff.h"
 9 | 
10 | #include <ros/ros.h>
11 | #include <sensor_msgs/Image.h>
12 | 
13 | #include <memory>
14 | #include <unordered_map>
15 | 
16 | extern "C" {
17 | #include <libavcodec/avcodec.h>
18 | #include <libavformat/avio.h>
19 | #include <libavformat/avformat.h>
20 | #include <libswscale/swscale.h>
21 | #include <libavutil/opt.h>
22 | #include <libavutil/imgutils.h>
23 | #include <libavutil/samplefmt.h>
24 | }
25 | 
26 | 
27 | namespace ffmpeg_image_transport {
28 |   using Image = sensor_msgs::Image;
29 |   using ImagePtr = sensor_msgs::ImagePtr;
30 |   using ImageConstPtr = sensor_msgs::ImageConstPtr;
31 |   using FFMPEGPacket = ffmpeg_image_transport_msgs::FFMPEGPacket;
32 |   typedef std::unordered_map<int64_t, ros::Time> PTSMap;
33 | 
34 |   class FFMPEGDecoder {
35 |   public:
36 |     typedef boost::function<void(const ImageConstPtr &img,
37 |                                  bool isKeyFrame)> Callback;
38 |     FFMPEGDecoder();
39 |     ~FFMPEGDecoder();
40 |     bool isInitialized() const { return (codecContext_ != NULL); }
41 |     // Initialize decoder upon first packet received,
42 |     // providing callback to be called when frame is complete.
43 |     // You must still call decodePacket(msg) afterward!
44 |     bool initialize(const FFMPEGPacket::ConstPtr& msg, Callback callback,
45 |                     const std::string &codec=std::string());
46 | 
47 |     // clears all state, but leaves config intact
48 |     void reset();
49 |     // decode packet (may result in frame callback!)
50 |     bool decodePacket(const FFMPEGPacket::ConstPtr &msg);
51 |     void setMeasurePerformance(bool p) {
52 |       measurePerformance_ = p;
53 |     }
54 |     void printTimers(const std::string &prefix) const;
55 |     void resetTimers();
56 | 
57 |   private:
58 |     bool initDecoder(int width, int height,
59 |                      const std::string &codecName,
60 |                      const std::vector<std::string> &codec);
61 |     // --------- variables
62 |     Callback          callback_;
63 |     // mapping of header
64 |     PTSMap            ptsToStamp_;
65 |     // performance analysis
66 |     bool              measurePerformance_{false};
67 |     TDiff             tdiffTotal_;
68 |     // libav stuff
69 |     std::string       encoding_;
70 |     AVCodecContext   *codecContext_{NULL};
71 |     AVFrame          *decodedFrame_{NULL};
72 |     AVFrame          *cpuFrame_{NULL};
73 |     AVFrame          *colorFrame_{NULL};
74 |     SwsContext       *swsContext_{NULL};
75 |     std::unordered_map<std::string, std::vector<std::string>> codecMap_;
76 |     enum AVPixelFormat hwPixFormat_;
77 |     AVPacket           packet_;
78 |     AVBufferRef        *hwDeviceContext_{NULL};
79 |   };
80 | }
81 | 


--------------------------------------------------------------------------------
/cmake/FindFFMPEG.cmake:
--------------------------------------------------------------------------------
  1 | # - Try to find ffmpeg libraries (libavcodec, libavformat and libavutil)
  2 | # Once done this will define
  3 | #
  4 | #  FFMPEG_FOUND - system has ffmpeg or libav
  5 | #  FFMPEG_INCLUDE_DIR - the ffmpeg include directory
  6 | #  FFMPEG_LIBRARIES - Link these to use ffmpeg
  7 | #  FFMPEG_LIBAVCODEC
  8 | #  FFMPEG_LIBAVFORMAT
  9 | #  FFMPEG_LIBAVUTIL
 10 | #  FFMPEG_LIBSWSCALE
 11 | #  FFMPEG_LIBSWRESAMPLE
 12 | #
 13 | #  Copyright (c) 2008 Andreas Schneider <mail@cynapses.org>
 14 | #  Modified for other libraries by Lasse Kärkkäinen <tronic>
 15 | #  Modified for Hedgewars by Stepik777
 16 | #
 17 | #  Redistribution and use is allowed according to the terms of the New
 18 | #  BSD license.
 19 | #
 20 | 
 21 | if (FFMPEG_LIBRARIES AND FFMPEG_INCLUDE_DIR)
 22 |   # in cache already
 23 |   set(FFMPEG_FOUND TRUE)
 24 | else (FFMPEG_LIBRARIES AND FFMPEG_INCLUDE_DIR)
 25 |   # use pkg-config to get the directories and then use these values
 26 |   # in the FIND_PATH() and FIND_LIBRARY() calls
 27 |   find_package(PkgConfig)
 28 |   set(FFMPEG_EXTRA_LIB_DIRS /usr/lib/x86_64-linux-gnu /usr/lib /usr/local/lib /opt/local/lib /sw/lib)
 29 |   set(FFMPEG_EXTRA_LIB_DIRS ${FFMPEG_LIB} /usr/lib /usr/local/lib /opt/local/lib /sw/lib)
 30 |   if (PKG_CONFIG_FOUND)
 31 |     pkg_check_modules(_FFMPEG_AVCODEC libavcodec)
 32 |     pkg_check_modules(_FFMPEG_AVFORMAT libavformat)
 33 |     pkg_check_modules(_FFMPEG_AVUTIL libavutil)
 34 |     pkg_check_modules(_FFMPEG_SWSCALE libswscale)
 35 |     pkg_check_modules(_FFMPEG_SWRESAMPLE libswresample)
 36 |   endif (PKG_CONFIG_FOUND)
 37 | 
 38 |   find_path(FFMPEG_AVCODEC_INCLUDE_DIR
 39 |     NAMES libavcodec/avcodec.h
 40 |     HINTS ${FFMPEG_INC}
 41 |     PATH_SUFFIXES ffmpeg libav
 42 |   )
 43 | #    NO_DEFAULT_PATH
 44 | 
 45 |   find_library(FFMPEG_LIBAVCODEC
 46 |     NAMES avcodec
 47 |     HINTS ${FFMPEG_LIB}
 48 | #    NO_DEFAULT_PATH
 49 |   )
 50 | 
 51 |   find_library(FFMPEG_LIBAVFORMAT
 52 |     NAMES avformat
 53 |     HINTS ${FFMPEG_LIB}
 54 | #    NO_DEFAULT_PATH
 55 |   )
 56 | 
 57 |   find_library(FFMPEG_LIBAVUTIL
 58 |     NAMES avutil
 59 |     HINTS ${FFMPEG_LIB}
 60 | #    NO_DEFAULT_PATH
 61 |   )
 62 | 
 63 |   find_library(FFMPEG_LIBSWSCALE
 64 |     NAMES swscale
 65 |     HINTS ${FFMPEG_LIB}
 66 | #    NO_DEFAULT_PATH
 67 |   )
 68 | 
 69 |   find_library(FFMPEG_LIBSWRESAMPLE
 70 |     NAMES swresample
 71 |     HINTS ${FFMPEG_LIB}
 72 | #    NO_DEFAULT_PATH
 73 |   )
 74 | 
 75 |   if (FFMPEG_LIBAVCODEC AND FFMPEG_LIBAVFORMAT)
 76 |     set(FFMPEG_FOUND TRUE)
 77 |   endif()
 78 | 
 79 |   if (FFMPEG_FOUND)
 80 |     set(FFMPEG_INCLUDE_DIR ${FFMPEG_AVCODEC_INCLUDE_DIR})
 81 | 
 82 |     set(FFMPEG_LIBRARIES
 83 |       ${FFMPEG_LIBSWSCALE}
 84 |       ${FFMPEG_LIBSWRESAMPLE}
 85 |       ${FFMPEG_LIBAVCODEC}
 86 |       ${FFMPEG_LIBAVFORMAT}
 87 |       ${FFMPEG_LIBAVUTIL}
 88 |     )
 89 | 
 90 |   endif (FFMPEG_FOUND)
 91 | 
 92 |   if (FFMPEG_FOUND)
 93 |     if (NOT FFMPEG_FIND_QUIETLY)
 94 |       message(STATUS "Found FFMPEG or Libav: ${FFMPEG_LIBRARIES}, ${FFMPEG_INCLUDE_DIR}")
 95 |     endif (NOT FFMPEG_FIND_QUIETLY)
 96 |   else (FFMPEG_FOUND)
 97 |     if (FFMPEG_FIND_REQUIRED)
 98 |       message(FATAL_ERROR "Could not find libavcodec or libavformat or libavutil")
 99 |     endif (FFMPEG_FIND_REQUIRED)
100 |   endif (FFMPEG_FOUND)
101 | 
102 | endif (FFMPEG_LIBRARIES AND FFMPEG_INCLUDE_DIR)
103 | 
104 | 


--------------------------------------------------------------------------------
/docs/compile_ffmpeg.md:
--------------------------------------------------------------------------------
  1 | # How to compile FFmpeg from scratch for your project
  2 | 
  3 | ## Prepare for FFmpeg installation
  4 | 
  5 | Install ffmpeg build dependencies according to
  6 | [this website](https://trac.ffmpeg.org/wiki/CompilationGuide/Ubuntu): 
  7 | 
  8 |     sudo apt-get update -qq && sudo apt-get -y install \
  9 |     autoconf \
 10 |     automake \
 11 |     build-essential \
 12 |     cmake \
 13 |     git-core \
 14 |     libass-dev \
 15 |     libfreetype6-dev \
 16 |     libsdl2-dev \
 17 |     libtool \
 18 |     libva-dev \
 19 |     libvdpau-dev \
 20 |     libvorbis-dev \
 21 |     libxcb1-dev \
 22 |     libxcb-shm0-dev \
 23 |     libxcb-xfixes0-dev \
 24 |     pkg-config \
 25 |     texinfo \
 26 |     wget \
 27 |     zlib1g-dev \
 28 | 	libx264-dev \
 29 | 	libx265-dev
 30 | 
 31 | Make your own ffmpeg directory:
 32 | 
 33 |     ffmpeg_dir=<absolute_path_to_empty_directory_for_ffmpeg_stuff>
 34 |     mkdir -p $ffmpeg_dir/build $ffmpeg_dir/bin
 35 | 
 36 | Build yasm:
 37 | 
 38 |     cd $ffmpeg_dir
 39 |     wget -O yasm-1.3.0.tar.gz https://www.tortall.net/projects/yasm/releases/yasm-1.3.0.tar.gz && \
 40 |     tar xzvf yasm-1.3.0.tar.gz && \
 41 |     cd yasm-1.3.0 && \
 42 |     ./configure --prefix="$ffmpeg_dir/build" --bindir="$ffmpeg_dir/bin" && \
 43 |     make && \
 44 |     make install
 45 | 
 46 | Build nasm:
 47 | 
 48 | 
 49 |     cd $ffmpeg_dir
 50 |     wget https://www.nasm.us/pub/nasm/releasebuilds/2.13.03/nasm-2.13.03.tar.bz2 && \
 51 |     tar xjvf nasm-2.13.03.tar.bz2 && \
 52 |     cd nasm-2.13.03 && \
 53 |     ./autogen.sh && \
 54 |     PATH="${ffmpeg_dir}/bin:$PATH" ./configure --prefix="${ffmpeg_dir}/build" --bindir="${ffmpeg_dir}/bin" && \
 55 |     make && \
 56 |     make install
 57 | 
 58 | 
 59 | Get nvcodec headers:
 60 | 
 61 | 
 62 |     cd $ffmpeg_dir
 63 |     git clone https://git.videolan.org/git/ffmpeg/nv-codec-headers.git
 64 |     cd nv-codec-headers
 65 | 
 66 | At this point you will have to pick the right version of the headers. You need the one that matches
 67 | your driver. For instance:
 68 | 
 69 |     git checkout n8.2.15.8
 70 | 
 71 | will get you a version that works with with Linux 410.48 or newer (see README file). If you mess
 72 | up here, you will later get an error that looks like this:
 73 | 
 74 |     [hevc_nvenc @ 0x7fc67c03a580] Cannot load libnvidia-encode.so.1
 75 |     [hevc_nvenc @ 0x7fc67c03a580] The minimum required Nvidia driver for nvenc is 418.30 or newer
 76 | 
 77 | Now create a modified Makefile that points to the right location:
 78 | 
 79 |     tmp_var="${ffmpeg_dir//"/"/"\/"}"
 80 |     sed "s/\/usr\/local/${tmp_var}\/build/g" Makefile > Makefile.tmp
 81 |     make -f Makefile.tmp install
 82 | 
 83 | 
 84 | # Compile FFMpeg
 85 | 
 86 | Clone and configure ffmpeg to your needs. The following configuration gives you
 87 | cuda and the hardware accelerated nvidia (nvenc) encoding for x264/x265.
 88 | This builds an ffmpeg that has more components than you need.
 89 | 
 90 |     cd $ffmpeg_dir
 91 |     git clone https://github.com/FFmpeg/FFmpeg.git	
 92 |     cd $ffmpeg_dir/FFmpeg
 93 | 
 94 | To build full NVidia encode/decode support, do this:
 95 | 
 96 |     PATH="$ffmpeg_dir/bin:$PATH" PKG_CONFIG_PATH="$ffmpeg_dir/build/lib/pkgconfig" ./configure --prefix=${ffmpeg_dir}/build --extra-cflags=-I${ffmpeg_dir}/build/include --extra-ldflags=-L${ffmpeg_dir}/build/lib --bindir=${ffmpeg_dir}/bin --enable-cuda-nvcc --enable-cuvid --enable-libnpp --extra-cflags=-I/usr/local/cuda/include/ --extra-ldflags=-L/usr/local/cuda/lib64/ --enable-gpl --enable-nvenc --enable-libx264 --enable-libx265 --enable-nonfree --enable-shared
 97 | 
 98 | If you don't have an NVidia card, do this:
 99 | 
100 |     PATH="$ffmpeg_dir/bin:$PATH" PKG_CONFIG_PATH="$ffmpeg_dir/build/lib/pkgconfig" ./configure --prefix=${ffmpeg_dir}/build --extra-cflags=-I${ffmpeg_dir}/build/include --extra-ldflags=-L${ffmpeg_dir}/build/lib --bindir=${ffmpeg_dir}/bin --enable-gpl --enable-libx264 --enable-libx265 --enable-nonfree --enable-shared
101 | 
102 | Now build and install (runs for a few minutes!):
103 | 
104 |     PATH="$ffmpeg_dir/bin:${PATH}:/usr/local/cuda/bin" make && make install && hash -r
105 | 
106 | 
107 | 


--------------------------------------------------------------------------------
/src/ffmpeg_publisher.cpp:
--------------------------------------------------------------------------------
  1 | /* -*-c++-*--------------------------------------------------------------------
  2 |  * 2018 Bernd Pfrommer bernd.pfrommer@gmail.com
  3 |  */
  4 | #include "ffmpeg_image_transport/ffmpeg_publisher.h"
  5 | #include <boost/algorithm/string/classification.hpp>
  6 | #include <boost/algorithm/string/split.hpp>
  7 | 
  8 | namespace ffmpeg_image_transport {
  9 | 
 10 | 
 11 |   void FFMPEGPublisher::packetReady(const FFMPEGPacketConstPtr &pkt) {
 12 |     (*publishFunction_)(*pkt);
 13 |   }
 14 | 
 15 |   static bool is_equal(const EncoderDynConfig &a,
 16 |                        const EncoderDynConfig &b) {
 17 |     return (a.encoder  == b.encoder &&
 18 |             a.profile  == b.profile &&
 19 |             a.qmax     == b.qmax &&
 20 |             a.bit_rate == b.bit_rate &&
 21 |             a.gop_size == b.gop_size &&
 22 |             a.measure_performance == b.measure_performance);
 23 |   }
 24 |   
 25 |   void
 26 |   FFMPEGPublisher::configure(EncoderDynConfig& config, int level) {
 27 |     if (!is_equal(config_, config)) {
 28 |       config_ = config;
 29 |       setCodecFromConfig(config);
 30 |       encoder_.reset(); // will be opened on next image
 31 |     }
 32 |   }
 33 | 
 34 |   void FFMPEGPublisher::advertiseImpl(
 35 |     ros::NodeHandle &nh,
 36 |     const std::string &base_topic,
 37 |     uint32_t queue_size,
 38 |     const image_transport::SubscriberStatusCallback  &conn_cb,
 39 |     const image_transport::SubscriberStatusCallback  &disconn_cb,
 40 |     const ros::VoidPtr &tracked_object, bool latch) {
 41 |     const std::string transportTopic = getTopicToAdvertise(base_topic);
 42 |     nh_.reset(new ros::NodeHandle(transportTopic));
 43 |     initConfigServer();
 44 |     // make the queue twice the size between keyframes.
 45 |     queue_size = std::max((int)queue_size, 2 * config_.gop_size);
 46 |     FFMPEGPublisherPlugin::advertiseImpl(nh, base_topic, queue_size,
 47 |                                          conn_cb, disconn_cb, tracked_object, latch);
 48 |   }
 49 | 
 50 |   void FFMPEGPublisher::setCodecFromConfig(const EncoderDynConfig &config) {
 51 |     encoder_.setCodec(config.encoder);
 52 |     encoder_.setProfile(config.profile);
 53 |     encoder_.setPreset(config.preset);
 54 |     encoder_.setQMax(config.qmax);
 55 |     encoder_.setBitRate(config.bit_rate);
 56 |     encoder_.setGOPSize(config.gop_size);
 57 |     encoder_.setMeasurePerformance(config.measure_performance);
 58 |     ROS_DEBUG_STREAM("FFMPEGPublisher codec: " << config.encoder <<
 59 |                     ", profile: " << config.profile <<
 60 |                     ", preset: " << config.preset <<
 61 |                     ", bit rate: " << config.bit_rate <<
 62 |                     ", qmax: " << config.qmax);
 63 |   }
 64 | 
 65 | 
 66 |   void
 67 |   FFMPEGPublisher::publish(const sensor_msgs::Image& message,
 68 |                            const PublishFn &publish_fn) const {
 69 |     FFMPEGPublisher *me = const_cast<FFMPEGPublisher *>(this);
 70 |     if (!me->encoder_.isInitialized()) {
 71 |       me->initConfigServer();
 72 |       me->publishFunction_ = &publish_fn;
 73 |       if (!me->encoder_.initialize(message.width, message.height,
 74 |               boost::bind(&FFMPEGPublisher::packetReady, me, boost::placeholders::_1))) {
 75 |         ROS_ERROR_STREAM("cannot initialize encoder!");
 76 |         return;
 77 |       }
 78 |     }
 79 |     me->encoder_.encodeImage(message); // may trigger packetReady() callback(s) from encoder!
 80 |     Lock lock(me->configMutex_);
 81 |     if (me->config_.measure_performance) {
 82 |       if (++me->frameCounter_ > (unsigned int)me->config_.performance_interval) {
 83 |         me->encoder_.printTimers(nh_->getNamespace());
 84 |         me->encoder_.resetTimers();
 85 |         me->frameCounter_ = 0;
 86 |       }
 87 |     }
 88 |   }
 89 | 
 90 |   void FFMPEGPublisher::initConfigServer() {
 91 |     Lock lock(configMutex_);
 92 |     if (!configServer_) {
 93 |       configServer_.reset(new ConfigServer(*nh_));
 94 |       // this will trigger an immediate callback!
 95 |       configServer_->setCallback(boost::bind(&FFMPEGPublisher::configure, this,
 96 |             boost::placeholders::_1, boost::placeholders::_2));
 97 |     }
 98 |   }
 99 |   
100 |   void FFMPEGPublisher::connectCallback(
101 |     const ros::SingleSubscriberPublisher &pub) {
102 |     ROS_DEBUG_STREAM("FFMPEGPublisher: connect() now has subscribers: "
103 |                      << getNumSubscribers());
104 |     initConfigServer();
105 |   }
106 | 
107 |   void FFMPEGPublisher::disconnectCallback(
108 |     const ros::SingleSubscriberPublisher &pub) {
109 |     ROS_DEBUG_STREAM("FFMPEGPublisher: disconnect() subscribers left: "
110 |                      << getNumSubscribers());
111 |   }
112 | }
113 | 


--------------------------------------------------------------------------------
/include/ffmpeg_image_transport/ffmpeg_encoder.h:
--------------------------------------------------------------------------------
  1 | /* -*-c++-*--------------------------------------------------------------------
  2 |  * 2018 Bernd Pfrommer bernd.pfrommer@gmail.com
  3 |  */
  4 | 
  5 | #pragma once
  6 | 
  7 | #include "ffmpeg_image_transport_msgs/FFMPEGPacket.h"
  8 | #include "ffmpeg_image_transport/tdiff.h"
  9 | 
 10 | #include <ros/ros.h>
 11 | #include <sensor_msgs/Image.h>
 12 | #include <opencv2/core/core.hpp>
 13 | 
 14 | #include <memory>
 15 | #include <list>
 16 | #include <iostream>
 17 | #include <unordered_map>
 18 | #include <mutex>
 19 | 
 20 | extern "C" {
 21 | #include <libavcodec/avcodec.h>
 22 | #include <libavformat/avio.h>
 23 | #include <libavformat/avformat.h>
 24 | #include <libswscale/swscale.h>
 25 | #include <libavutil/opt.h>
 26 | #include <libavutil/imgutils.h>
 27 | #include <libavutil/samplefmt.h>
 28 | }
 29 | 
 30 | 
 31 | namespace ffmpeg_image_transport {
 32 |   using Image = sensor_msgs::Image;
 33 |   using ImagePtr = sensor_msgs::ImagePtr;
 34 |   using ImageConstPtr = sensor_msgs::ImageConstPtr;
 35 |   typedef std::unordered_map<int64_t, ros::Time> PTSMap;
 36 | 
 37 |   class FFMPEGEncoder {
 38 |     using FFMPEGPacket = ffmpeg_image_transport_msgs::FFMPEGPacket;
 39 |     using FFMPEGPacketConstPtr =
 40 |       ffmpeg_image_transport_msgs::FFMPEGPacketConstPtr;
 41 |     typedef std::unique_lock<std::recursive_mutex> Lock;
 42 |     typedef boost::function<void(const FFMPEGPacketConstPtr &pkt)> Callback;
 43 |   public:
 44 |     FFMPEGEncoder();
 45 |     ~FFMPEGEncoder();
 46 |     // ------- various encoding settings
 47 |     void setCodec(const std::string &n) {
 48 |       Lock lock(mutex_);
 49 |       codecName_ = n;
 50 |     }
 51 |     void setProfile(const std::string &p) {
 52 |       Lock lock(mutex_);
 53 |       profile_ = p;
 54 |     }
 55 |     void setPreset(const std::string &p) {
 56 |       Lock lock(mutex_);
 57 |       preset_ = p;
 58 |     }
 59 |     void setQMax(int q)    {
 60 |       Lock lock(mutex_);
 61 |       qmax_  = q;
 62 |     }
 63 |     void setBitRate(int r) {
 64 |       Lock lock(mutex_);
 65 |       bitRate_ = r;
 66 |     }
 67 |     void setGOPSize(int g) {
 68 |       Lock lock(mutex_);
 69 |       GOPSize_ = g;
 70 |     }
 71 |     void setFrameRate(int frames, int second) {
 72 |       Lock lock(mutex_);
 73 |       frameRate_.num = frames;
 74 |       frameRate_.den = second;
 75 |       timeBase_.num  = second;
 76 |       timeBase_.den  = frames;
 77 |     }
 78 |     void setMeasurePerformance(bool p) {
 79 |       Lock lock(mutex_);
 80 |       measurePerformance_ = p;
 81 |     }
 82 |     // ------- teardown and startup
 83 |     bool isInitialized() const {
 84 |       Lock lock(mutex_);
 85 |       return (codecContext_ != NULL);
 86 |     }
 87 |     bool initialize(int width, int height, Callback callback);
 88 | 
 89 |     void reset();
 90 |     // encode image
 91 |     void encodeImage(const cv::Mat &img,
 92 |                      const std_msgs::Header &header, const ros::WallTime &t0);
 93 |     void encodeImage(const sensor_msgs::Image &msg);
 94 |     // ------- performance statistics
 95 |     void printTimers(const std::string &prefix) const;
 96 |     void resetTimers();
 97 |   private:
 98 |     bool openCodec(int width, int height);
 99 |     void closeCodec();
100 |     int  drainPacket(const std_msgs::Header &hdr, int width, int height);
101 |     // --------- variables
102 |     mutable std::recursive_mutex mutex_;
103 |     boost::function<void(const FFMPEGPacketConstPtr &pkt)> callback_;
104 |     // config
105 |     std::string       codecName_;
106 |     std::string       preset_;
107 |     std::string       profile_;
108 |     AVPixelFormat     pixFormat_{AV_PIX_FMT_YUV420P};
109 |     AVRational        timeBase_{1, 40};
110 |     AVRational        frameRate_{40, 1};
111 |     int               GOPSize_{15};
112 |     int64_t           bitRate_{1000000};
113 |     int               qmax_{0};
114 |     // libav state
115 |     AVCodecContext    *codecContext_{NULL};
116 |     AVFrame           *frame_{NULL};
117 |     AVPacket          packet_;
118 |     int64_t           pts_{0};
119 |     PTSMap            ptsToStamp_;
120 |     // performance analysis
121 |     bool              measurePerformance_{true};
122 |     int64_t           totalInBytes_{0};
123 |     int64_t           totalOutBytes_{0};
124 |     unsigned int      frameCnt_{0};
125 |     TDiff             tdiffUncompress_;
126 |     TDiff             tdiffEncode_;
127 |     TDiff             tdiffDebayer_;
128 |     TDiff             tdiffFrameCopy_;
129 |     TDiff             tdiffSendFrame_;
130 |     TDiff             tdiffReceivePacket_;
131 |     TDiff             tdiffCopyOut_;
132 |     TDiff             tdiffPublish_;
133 |     TDiff             tdiffTotal_;
134 |   };
135 | }
136 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # ROS image transport for FFmpeg encoding
  2 | 
  3 | 
  4 | NOTE: This repo has reached End-Of-Life. All future development will
  5 | be on the [ROS2 repository here](https://github.com/berndpfrommer/ffmpeg_image_transport/).
  6 | 
  7 | This ROS image transport supports encoding/decoding with the FFMpeg
  8 | library. With this transport, you can encode h264 and h265, using
  9 | nvidia hardware acceleration when available.
 10 | 
 11 | Also have a look at [this repo tools to help with decoding](https://github.com/daniilidis-group/ffmpeg_image_transport_tools).
 12 | 
 13 | ## Downloading
 14 | 
 15 | Create a catkin workspace (if not already there) and download the
 16 | ffmpeg image transport and other required packages:
 17 | 
 18 |     mkdir -p catkin_ws/src
 19 |     cd catkin_ws/src
 20 |     git clone https://github.com/daniilidis-group/ffmpeg_image_transport_msgs.git
 21 |     git clone https://github.com/daniilidis-group/ffmpeg_image_transport.git
 22 | 
 23 | Optionally get the additional set of ffmpeg tools:
 24 | 
 25 |     git clone git@github.com:daniilidis-group/ffmpeg_image_transport_tools.git
 26 | 
 27 | 
 28 | ## Requirements: FFmpeg v4.0 or later, and a "reasonable" resolution
 29 | 
 30 | At some point this software worked on the following systems:
 31 | 
 32 | - Ubuntu 16.04 + ROS kinetic (not tested in a long while, maybe broken
 33 |   by now)
 34 | - Ubuntu 18.04 + ROS melodic
 35 | - Ubuntu 20.04 + ROS noetic (recently tested, should work with stock ffmpeg)
 36 | 
 37 | If you have ffmpeg 4.0 or later you may be able to compile against default header files:
 38 | 
 39 | 	sudo apt install ffmpeg
 40 | 
 41 | If you don't have ffmpeg 4.0 or later, or you hit compile errors
 42 | (supposedly even happens now on Ubuntu 18.04), 
 43 | [follow these instructions to compile ffmpeg from scratch](docs/compile_ffmpeg.md), and point the
 44 | transport to the right place: 
 45 | 
 46 |     ffmpeg_dir=<here the ffmpeg_dir>
 47 |     catkin config -DCMAKE_BUILD_TYPE=Release -DFFMPEG_LIB=${ffmpeg_dir}/build/lib -DFFMPEG_INC=${ffmpeg_dir}/build/include
 48 | 
 49 | Note that not all resolutions are supported in hardware accelerated
 50 | mode for h264 encoding/decoding, so you may get strange results if you
 51 | have cameras with resolutions other than those supported by
 52 | e.g. nvenc. Plain 1920x1200 works, probably also VGA, but other
 53 | resolutions need to be verified. Experiments indicate that the line
 54 | width must be a multiple of 32!
 55 | 
 56 | ## Compiling
 57 | 
 58 | This should be easy as running the following inside your catkin workspace
 59 | 
 60 |     catkin config -DCMAKE_BUILD_TYPE=Release
 61 | 
 62 | or, if you have your own version of ffmpeg installaed under ``${ffmpeg_dir`` (see above)
 63 | 
 64 |     catkin config -DCMAKE_BUILD_TYPE=Release -DFFMPEG_LIB=${ffmpeg_dir}/build/lib -DFFMPEG_INC=${ffmpeg_dir}/build/include
 65 | 
 66 | then compile should be as easy as this:
 67 | 
 68 |     catkin build ffmpeg_image_transport
 69 | 
 70 | ## How to use
 71 | 
 72 | Usage should be transparent, but the ffmpeg image transport plugin
 73 | needs to be available on both the publishing and the subscribing
 74 | node. You also *need to add the location to your custom built ffmpeg
 75 | library* to ``LD_LIBRARY_PATH``, like so (change path to match yours):
 76 | ```
 77 | export LD_LIBRARY_PATH=$HOME/catkin_ws/ffmpeg/build/lib:$LD_LIBRARY_PATH
 78 | ```
 79 | Once done, you should get the following output when you run
 80 | ``list_transports``:
 81 | ```
 82 | rosrun image_transport list_transports 
 83 | Declared transports:
 84 | ... some stuff ...
 85 | image_transport/ffmpeg
 86 | ... some stuff ...
 87 | 
 88 | Details:
 89 | ----------
 90 | ... some stuff ...
 91 | "image_transport/ffmpeg"
 92 |  - Provided by package: ffmpeg_image_transport
 93 |  - Publisher: 
 94 |       This plugin encodes frame into ffmpeg compressed packets
 95 |     
 96 |  - Subscriber: 
 97 |       This plugin decodes frame from ffmpeg compressed packets
 98 | ```
 99 | 
100 | If it says something about "plugins not built", that means the
101 | ``LD_LIBRARY_PATH`` is not set correctly.
102 | 
103 | ### republishing
104 | Most ROS nodes that publish images use an image transport for that. If
105 | not you can use a republish node to convert images into an ffmpeg
106 | stream. The following line will start a republish node in ffmpeg format:
107 | 
108 | ```
109 | rosrun image_transport republish raw in:=/webcam/image_raw ffmpeg out:=/webcam/image_raw/ffmpeg _/webcam/image_raw/ffmpeg/encoding:=x264
110 | ```
111 | 
112 | ### encoder parameters
113 | 
114 | The image transport has various parameters that you can configure
115 | dynamically via ``rqt_reconfigure``, or specify at startup by
116 | prepending the topic name. For instance to use the unaccelerated
117 | ``libx264`` encoding, start a republish node like this:
118 | ```
119 | rosrun image_transport republish raw in:=/webcam/image_raw ffmpeg __name:=repub out:=~/image_raw  _/image_raw/ffmpeg/encoder:=libx264
120 | ```
121 | Note: I had to ``__name`` the node to get the ros parameter show up under the topic name.
122 | 
123 | Parameters (see ffmpeg h264 documentation for explanation):
124 | 
125 | - ``encoder``: ffmpeg encoding to use. Allowed values: ``h265_nvenc``,
126 |   ``hevc_nvenc``, ``libx264`` (no GPU). Default: ``hevc_nvenc``.
127 | - ``profile``: ffmpeg profile. Allowed values: ``main``, ``main10``,
128 |   ``rext``, ``high``. Default: ``main``.
129 | - ``qmax``: maximum allowed quantization, controls quality. Range 0-63, default: 10.
130 | - ``bit_rate``: bit rate in bits/sec. Range 10-2000000000, default: 8242880.
131 | - ``gop_size``: gop size (number of frames): Range 1-128, default: 15
132 | - ``measure_performance``: switch on performance debugging output. Default: false.
133 | - ``performance_interval``: number of frames between performance printouts. Default: 175.
134 | 
135 | ### encoder parameters
136 | 
137 | Usually the decoder infers from the received encoding type what decoding scheme to use.
138 | You can overwrite it with a node-level parameter though:
139 | 
140 | - ``decoder_type``:
141 |   - ``h264``: used if encoding is ``libx264`` or ``h264_nvenc``.
142 |   - ``hevc_cuvid``: first tried if encoding is ``hevc_nvenc``.
143 |   - ``hevc``: second tried if encoding is ``hevc_nvenc``.
144 | 
145 | For example to force ``hevc`` decoding, run a node like this:
146 | ```
147 | rosrun image_transport republish ffmpeg in:=/repub/image_raw raw out:=~/image_raw _/ffmpeg/decoder_type:=hevc
148 | ```
149 | 
150 | ## Trouble shooting:
151 | 
152 | 
153 | On e.g. Ubuntu 16.04, you need a newer version of ffmpeg. If you get an error like this one,
154 | you need a newer version of ffmpeg:
155 | 
156 |     In member function ‘bool ffmpeg_image_transport::FFMPEGDecoder::decodePacket(const ConstPtr&)’:
157 |     /home/pfrommer/Documents/birds/src/ffmpeg_image_transport/src/ffmpeg_decoder.cpp:104:47:
158 |     error: ‘avcodec_send_packet’ was not declared in this scope
159 |     int ret = avcodec_send_packet(ctx, &packet);
160 | 
161 | If the build still fails, make sure you start from scratch:
162 | 
163 |     catkin clean ffmpeg_image_transport
164 | 
165 | ## License
166 | 
167 | This software is issued under the Apache License Version 2.0.
168 | 


--------------------------------------------------------------------------------
/src/ffmpeg_encoder.cpp:
--------------------------------------------------------------------------------
  1 | /* -*-c++-*--------------------------------------------------------------------
  2 |  * 2018 Bernd Pfrommer bernd.pfrommer@gmail.com
  3 |  */
  4 | 
  5 | #include "ffmpeg_image_transport/ffmpeg_encoder.h"
  6 | #include "ffmpeg_image_transport_msgs/FFMPEGPacket.h"
  7 | #include <sensor_msgs/Image.h>
  8 | #include <cv_bridge/cv_bridge.h>
  9 | #include <opencv2/imgproc/imgproc.hpp>
 10 | #include <opencv2/imgcodecs.hpp>
 11 | #include <fstream>
 12 | #include <iomanip>
 13 | 
 14 | namespace ffmpeg_image_transport {
 15 | 
 16 |   FFMPEGEncoder::FFMPEGEncoder() {
 17 |     // must init the packet and set the pointers to zero
 18 |     // in case a closeCodec() happens right away,
 19 |     // and av_packet_unref() is called.
 20 |     av_init_packet(&packet_);
 21 |     packet_.data = NULL;    // packet data will be allocated by the encoder
 22 |     packet_.size = 0;
 23 |   }
 24 | 
 25 |   FFMPEGEncoder::~FFMPEGEncoder() {
 26 |     Lock lock(mutex_);
 27 |     closeCodec();
 28 |   }
 29 | 
 30 |   void FFMPEGEncoder::reset() {
 31 |     Lock lock(mutex_);
 32 |     closeCodec();
 33 |   }
 34 | 
 35 |   void
 36 |   FFMPEGEncoder::closeCodec() {
 37 |     if (codecContext_) {
 38 |       avcodec_close(codecContext_);
 39 |       codecContext_ = NULL;
 40 |     }
 41 |     if (frame_) {
 42 |       av_free(frame_);
 43 |       frame_ = 0;
 44 |     }
 45 |     if (packet_.data != NULL) {
 46 |       av_packet_unref(&packet_); // free packet allocated by encoder
 47 |       packet_.data = NULL;
 48 |       packet_.size = 0;
 49 |     }
 50 |   }
 51 | 
 52 |   bool FFMPEGEncoder::initialize(int width, int height,
 53 |                                  Callback callback) {
 54 |     Lock lock(mutex_);
 55 |     callback_ = callback;
 56 |     return (openCodec(width, height));
 57 |   }
 58 | 
 59 | 	bool FFMPEGEncoder::openCodec(int width, int height) {
 60 |     codecContext_ = NULL;
 61 |     try {
 62 |       if (codecName_.empty()) {
 63 |         throw (std::runtime_error("no codec set!"));
 64 |       }
 65 |       if ((width % 32) != 0) {
 66 |         throw (std::runtime_error("image line width must be "
 67 |                                   "multiple of 32 but is: " + std::to_string(width)));
 68 |       }
 69 |       // find codec
 70 |       AVCodec *codec = avcodec_find_encoder_by_name(codecName_.c_str());
 71 |       if (!codec) {
 72 |         throw (std::runtime_error("cannot find codec: " + codecName_));
 73 |       }
 74 | 	    // allocate codec context
 75 |       codecContext_ = avcodec_alloc_context3(codec);
 76 |       if (!codecContext_) {
 77 |         throw (std::runtime_error("cannot allocate codec context!"));
 78 |       }
 79 |       codecContext_->bit_rate  = bitRate_;
 80 |       codecContext_->qmax      = qmax_;// 0: highest, 63: worst quality bound
 81 |       codecContext_->width     = width;
 82 |       codecContext_->height    = height;
 83 |       codecContext_->time_base = timeBase_;
 84 |       codecContext_->framerate = frameRate_;
 85 | 
 86 |       // gop size is number of frames between keyframes
 87 |       // small gop -> higher bandwidth, lower cpu consumption
 88 |       codecContext_->gop_size = GOPSize_;
 89 |       // number of bidirectional frames (per group?).
 90 |       // NVenc can only handle zero!
 91 |       codecContext_->max_b_frames = 0;
 92 |    
 93 |       // encoded pixel format. Must be supported by encoder
 94 |       // check with e.g.: ffmpeg -h encoder=h264_nvenc -pix_fmts
 95 | 
 96 |       codecContext_->pix_fmt = pixFormat_;
 97 | 
 98 |       if (av_opt_set(codecContext_->priv_data, "profile", profile_.c_str(),
 99 |                      AV_OPT_SEARCH_CHILDREN) != 0) {
100 |         ROS_ERROR_STREAM("cannot set profile: " << profile_);
101 |       }
102 | 
103 |       if (av_opt_set(codecContext_->priv_data, "preset", preset_.c_str(),
104 |                      AV_OPT_SEARCH_CHILDREN) != 0) {
105 |         ROS_ERROR_STREAM("cannot set preset: " << preset_);
106 |       }
107 |       ROS_DEBUG("codec: %10s, profile: %10s, preset: %10s,"
108 |                 " bit_rate: %10ld qmax: %2d",
109 |                 codecName_.c_str(), profile_.c_str(),
110 |                 preset_.c_str(), bitRate_, qmax_);
111 |       /* other optimization options for nvenc
112 |          if (av_opt_set_int(codecContext_->priv_data, "surfaces",
113 |          0, AV_OPT_SEARCH_CHILDREN) != 0) {
114 |          ROS_ERROR_STREAM("cannot set surfaces!");
115 |          }
116 |       */
117 |       if (avcodec_open2(codecContext_, codec, NULL) < 0)  {
118 |         throw (std::runtime_error("cannot open codec!"));
119 |       }
120 |       ROS_DEBUG_STREAM("opened codec: " << codecName_);
121 |       frame_ = av_frame_alloc();
122 |       if (!frame_)  {
123 |         throw (std::runtime_error("cannot alloc frame!"));
124 |       }
125 |       frame_->width  = width;
126 |       frame_->height = height;
127 |       frame_->format = codecContext_->pix_fmt;
128 |       // allocate image for frame
129 |       if (av_image_alloc(frame_->data, frame_->linesize, width, height,
130 |                          (AVPixelFormat)frame_->format, 32) < 0) {
131 |         throw (std::runtime_error("cannot alloc image!"));
132 |       }
133 |       //Initialize packet
134 |       av_init_packet(&packet_);
135 |       packet_.data = NULL;    // packet data will be allocated by the encoder
136 |       packet_.size = 0;
137 |     } catch (const std::runtime_error &e) {
138 |       ROS_ERROR_STREAM(e.what());
139 |       if (codecContext_) {
140 |         avcodec_close(codecContext_);
141 |         codecContext_ = NULL;
142 |       }
143 |       if (frame_) {
144 |         av_free(frame_);
145 |         frame_ = 0;
146 |       }
147 |       return (false);
148 |     }
149 |     ROS_DEBUG_STREAM("intialized codec " << codecName_ << " for image: "
150 |                      << width << "x" << height);
151 |     return (true);
152 | 	}
153 | 
154 |   void FFMPEGEncoder::encodeImage(const sensor_msgs::Image &msg) {
155 |     ros::WallTime t0;
156 |     if (measurePerformance_) { t0 = ros::WallTime::now(); }
157 |     cv::Mat img =
158 |       cv_bridge::toCvCopy(msg, sensor_msgs::image_encodings::BGR8)->image;
159 |     encodeImage(img, msg.header, t0);
160 |     if (measurePerformance_) {
161 |       const auto t1 = ros::WallTime::now();
162 |       tdiffDebayer_.update((t1-t0).toSec());
163 |     }
164 |   }
165 | 
166 |   void FFMPEGEncoder::encodeImage(const cv::Mat &img,
167 |                                   const std_msgs::Header &header,
168 |                                   const ros::WallTime &t0) {
169 |     Lock lock(mutex_);
170 |     ros::WallTime t1, t2, t3;
171 |     if (measurePerformance_) {
172 |       frameCnt_++;
173 |       t1 = ros::WallTime::now();
174 |       totalInBytes_ += img.cols * img.rows; // raw size!
175 |     }
176 | 
177 |     const uint8_t *p = img.data;
178 |     const int width  = img.cols;
179 |     const int height = img.rows;
180 |     const AVPixelFormat targetFmt = codecContext_->pix_fmt;
181 |     if (targetFmt == AV_PIX_FMT_BGR0) {
182 |       memcpy(frame_->data[0], p, width * height * 3);
183 |     } else if (targetFmt == AV_PIX_FMT_YUV420P) {
184 |       cv::Mat yuv;
185 |       cv::cvtColor(img, yuv, cv::COLOR_BGR2YUV_I420);
186 |       const uint8_t *p = yuv.data;
187 |       memcpy(frame_->data[0], p,  width*height);
188 |       memcpy(frame_->data[1], p + width*height, width*height / 4);
189 |       memcpy(frame_->data[2], p + width*(height + height/4), (width*height)/4);
190 |     } else {
191 |       ROS_ERROR_STREAM("cannot convert format bgr8 -> "
192 |                        << (int)codecContext_->pix_fmt);
193 |       return;
194 |     }
195 |     if (measurePerformance_) {
196 |       t2 = ros::WallTime::now();
197 |       tdiffFrameCopy_.update((t2 - t1).toSec());
198 |     }
199 | 
200 |     frame_->pts = pts_++; //
201 |     ptsToStamp_.insert(PTSMap::value_type(frame_->pts, header.stamp));
202 | 
203 |     int ret = avcodec_send_frame(codecContext_, frame_);
204 |     if (measurePerformance_) {
205 |       t3 = ros::WallTime::now();
206 |       tdiffSendFrame_.update((t3 - t2).toSec());
207 |     }
208 |     // now drain all packets
209 |     while (ret == 0) {
210 |       ret = drainPacket(header, width, height);
211 |     }
212 |     if (measurePerformance_) {
213 |       const ros::WallTime t4 = ros::WallTime::now();
214 |       tdiffTotal_.update((t4-t0).toSec());
215 |     }
216 |   }
217 | 
218 |   int FFMPEGEncoder::drainPacket(const std_msgs::Header &header,
219 |                                  int width, int height) {
220 |     ros::WallTime t0, t1, t2;
221 |     if (measurePerformance_) {
222 |       t0 = ros::WallTime::now();
223 |     }
224 |     int ret = avcodec_receive_packet(codecContext_, &packet_);
225 |     if (measurePerformance_) {
226 |       t1 = ros::WallTime::now();
227 |       tdiffReceivePacket_.update((t1 - t0).toSec());
228 |     }
229 |     const AVPacket &pk = packet_;
230 |     if (ret == 0 && packet_.size > 0) {
231 |       FFMPEGPacket *packet = new FFMPEGPacket;
232 |       FFMPEGPacketConstPtr pptr(packet);
233 |       packet->data.resize(packet_.size);
234 |       packet->img_width =  width; 
235 |       packet->img_height = height;
236 |       packet->pts        = pk.pts;
237 |       packet->flags      = pk.flags;
238 |       memcpy(&(packet->data[0]), packet_.data, packet_.size);
239 |       if (measurePerformance_) {
240 |         t2 = ros::WallTime::now();
241 |         totalOutBytes_ += packet_.size;
242 |         tdiffCopyOut_.update((t2 - t1).toSec());
243 |       }
244 |       packet->header = header;
245 |       auto it = ptsToStamp_.find(pk.pts);
246 |       if (it != ptsToStamp_.end()) {
247 |         packet->header.stamp = it->second;
248 |         packet->encoding = codecName_ ;
249 |         callback_(pptr);  // deliver packet callback
250 |         if (measurePerformance_) {
251 |           const ros::WallTime t3 = ros::WallTime::now();
252 |           tdiffPublish_.update((t3 - t2).toSec());
253 |         }
254 |         ptsToStamp_.erase(it);
255 |       } else {
256 |         ROS_ERROR_STREAM("pts " << pk.pts << " has no time stamp!");
257 |       }
258 |       av_packet_unref(&packet_); // free packet allocated by encoder
259 |       av_init_packet(&packet_); // prepare next one
260 |     }
261 |     return (ret);
262 |   }
263 | 
264 |   void FFMPEGEncoder::printTimers(const std::string &prefix) const {
265 |     Lock lock(mutex_);
266 |     ROS_INFO_STREAM(prefix
267 |                     << " pktsz: " << totalOutBytes_ / frameCnt_
268 |                     << " compr: " << totalInBytes_ / (double)totalOutBytes_
269 |                     << " debay: " << tdiffDebayer_
270 |                     << " fmcp: " << tdiffFrameCopy_
271 |                     << " send: " << tdiffSendFrame_
272 |                     << " recv: " << tdiffReceivePacket_
273 |                     << " cout: " << tdiffCopyOut_
274 |                     << " publ: " << tdiffPublish_
275 |                     << " tot: " << tdiffTotal_
276 |       );
277 |   }
278 |   void FFMPEGEncoder::resetTimers() {
279 |     Lock lock(mutex_);
280 |     tdiffDebayer_.reset();
281 |     tdiffFrameCopy_.reset();
282 |     tdiffSendFrame_.reset();
283 |     tdiffReceivePacket_.reset();
284 |     tdiffCopyOut_.reset();
285 |     tdiffPublish_.reset();
286 |     tdiffTotal_.reset();
287 |     frameCnt_ = 0;
288 |     totalOutBytes_ = 0;
289 |     totalInBytes_ = 0;
290 |   }
291 | }  // namespace
292 | 


--------------------------------------------------------------------------------
/src/ffmpeg_decoder.cpp:
--------------------------------------------------------------------------------
  1 | /* -*-c++-*--------------------------------------------------------------------
  2 |  * 2018 Bernd Pfrommer bernd.pfrommer@gmail.com
  3 |  */
  4 | 
  5 | #include "ffmpeg_image_transport/ffmpeg_decoder.h"
  6 | #include <sensor_msgs/Image.h>
  7 | #include <cv_bridge/cv_bridge.h>
  8 | #include <opencv2/imgproc/imgproc.hpp>
  9 | #include <opencv2/imgcodecs.hpp>
 10 | #include <fstream>
 11 | #include <iomanip>
 12 | #include <unordered_map>
 13 | 
 14 | namespace ffmpeg_image_transport {
 15 | 
 16 |   FFMPEGDecoder::FFMPEGDecoder() {
 17 |     codecMap_["h264_nvenc"] = {"h264"};
 18 |     codecMap_["libx264"]    = {"h264"};
 19 |     codecMap_["hevc_nvenc"] = {"hevc_cuvid", "hevc"};
 20 |   }
 21 | 
 22 |   FFMPEGDecoder::~FFMPEGDecoder() {
 23 |     reset();
 24 |   }
 25 | 
 26 |   void FFMPEGDecoder::reset() {
 27 |     if (codecContext_) {
 28 |       avcodec_close(codecContext_);
 29 |       av_free(codecContext_);
 30 |       codecContext_ = NULL;
 31 |     }
 32 |     if (swsContext_) {
 33 |       sws_freeContext(swsContext_);
 34 |       swsContext_ = NULL;
 35 |     }
 36 |     if (hwDeviceContext_) {
 37 |       av_buffer_unref(&hwDeviceContext_);
 38 |     }
 39 |     av_free(decodedFrame_);
 40 |     decodedFrame_ = NULL;
 41 |     av_free(cpuFrame_);
 42 |     cpuFrame_ = NULL;
 43 |     av_free(colorFrame_);
 44 |     colorFrame_ = NULL;
 45 |   }
 46 | 
 47 |   bool FFMPEGDecoder::initialize(const FFMPEGPacket::ConstPtr& msg,
 48 |                                  Callback callback,
 49 |                                  const std::string &codecName) {
 50 |     callback_ = callback;
 51 |     std::string cname = codecName;
 52 |     std::vector<std::string> codecs;
 53 |     if (cname.empty()) {
 54 |       // try and find the right codec from the map
 55 |       const auto it = codecMap_.find(msg->encoding);
 56 |       if (it == codecMap_.end()) {
 57 |         ROS_ERROR_STREAM("message has unknown encoding: " << msg->encoding);
 58 |         return (false);
 59 |       }
 60 |       cname  = msg->encoding;
 61 |       codecs = it->second;
 62 |     } else {
 63 |       codecs.push_back(codecName);
 64 |     }
 65 |     encoding_ = msg->encoding;
 66 |     return (initDecoder(msg->img_width, msg->img_height, cname, codecs));
 67 |   }
 68 | 
 69 |   static enum AVHWDeviceType get_hw_type(const std::string &name) {
 70 |     enum AVHWDeviceType type = av_hwdevice_find_type_by_name(name.c_str());
 71 |     if (type == AV_HWDEVICE_TYPE_NONE) {
 72 |       ROS_WARN_STREAM("hw accel device is not supported: " << name);
 73 |       ROS_INFO_STREAM("available devices:");
 74 |       while((type = av_hwdevice_iterate_types(type)) != AV_HWDEVICE_TYPE_NONE)
 75 |         ROS_INFO_STREAM(av_hwdevice_get_type_name(type));
 76 |       return (type);
 77 |     }
 78 |     return (type);
 79 |   }
 80 | 
 81 |   static AVBufferRef *hw_decoder_init(AVBufferRef **hwDeviceContext,
 82 |                                       const enum AVHWDeviceType hwType) {
 83 |     int rc = av_hwdevice_ctx_create(hwDeviceContext, hwType, NULL, NULL, 0);
 84 |     if (rc < 0) {
 85 |       ROS_ERROR_STREAM("failed to create context for HW device: " << hwType);
 86 |       return (NULL);
 87 |     }
 88 |     return (av_buffer_ref(*hwDeviceContext));
 89 |   }
 90 | 
 91 |   static std::unordered_map<AVCodecContext *, AVPixelFormat> pix_format_map;
 92 |   
 93 |   static enum AVPixelFormat
 94 |   get_hw_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts) {
 95 |     enum AVPixelFormat pf = pix_format_map[ctx];
 96 |     const enum AVPixelFormat *p;
 97 |     for (p = pix_fmts; *p != -1; p++) {
 98 |       if (*p == pf) {
 99 |         //ROS_INFO_STREAM("found hw pix format: " << pf);
100 |         return *p;
101 |       }
102 |     }
103 |     ROS_ERROR_STREAM("Failed to get HW surface format.");
104 |     return AV_PIX_FMT_NONE;
105 |   }
106 | 
107 |   static enum AVPixelFormat find_pix_format(
108 |     const std::string &codecName, enum AVHWDeviceType hwDevType,
109 |     const AVCodec *codec, const std::string &hwAcc) {
110 | 
111 |     for (int i = 0; ; i++) {
112 |       const AVCodecHWConfig *config = avcodec_get_hw_config(codec, i);
113 |       if (!config) {
114 |         ROS_WARN_STREAM("decoder " << codecName <<
115 |                         " does not support hw accel: " << hwAcc);
116 |         return (AV_PIX_FMT_NONE);
117 |       }
118 |       if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX &&
119 |           config->device_type == hwDevType) {
120 |         return (config->pix_fmt);
121 |       }
122 |     }
123 |     return (AV_PIX_FMT_NONE);
124 |   }
125 |                                                   
126 |   bool FFMPEGDecoder::initDecoder(int width, int height,
127 |                                   const std::string &codecName,
128 |                                   const std::vector<std::string> &codecs) {
129 |     std::string codecUsed = "NO_CODEC_FOUND";
130 |     try {
131 |       const AVCodec *codec = NULL;
132 |       for (const auto &c: codecs) {
133 |         codec = avcodec_find_decoder_by_name(c.c_str());
134 |         if (!codec) {
135 |           ROS_WARN_STREAM("no codec " << c << " found!");
136 |           continue;
137 |         }
138 |         codecContext_ = avcodec_alloc_context3(codec);
139 |         if (!codecContext_) {
140 |           ROS_WARN_STREAM("alloc context failed for " + codecName);
141 |           codec = NULL;
142 |           continue;
143 |         }
144 |         av_opt_set_int(codecContext_, "refcounted_frames", 1, 0);  
145 |         const std::string hwAcc("cuda");
146 |         enum AVHWDeviceType hwDevType = get_hw_type(hwAcc);
147 | 
148 |         if (hwDevType != AV_HWDEVICE_TYPE_NONE) {
149 |           codecContext_->hw_device_ctx = hw_decoder_init(&hwDeviceContext_,
150 |                                                          hwDevType);
151 |           hwPixFormat_ = find_pix_format(codecName, hwDevType, codec, hwAcc);
152 |           // must put in global hash for the callback function
153 |           pix_format_map[codecContext_] = hwPixFormat_;
154 |           codecContext_->get_format = get_hw_format;
155 |         } else {
156 |           hwPixFormat_ = AV_PIX_FMT_NONE;
157 |         }
158 |         codecContext_->width  = width;
159 |         codecContext_->height = height;
160 | 
161 |         if (avcodec_open2(codecContext_, codec, NULL) < 0) {
162 |           ROS_WARN_STREAM("open context failed for " + codecName);
163 |           av_free(codecContext_);
164 |           codecContext_ = NULL;
165 |           codec = NULL;
166 |           continue;
167 |         }
168 |         codecUsed = c;
169 |         break;
170 |       }
171 |       if (!codec)
172 |         throw (std::runtime_error("cannot open codec " + codecName));
173 |       
174 |       decodedFrame_       = av_frame_alloc();
175 |       cpuFrame_  = (hwPixFormat_ == AV_PIX_FMT_NONE) ? NULL : av_frame_alloc();
176 |       colorFrame_         = av_frame_alloc();
177 |       colorFrame_->width  = width;
178 |       colorFrame_->height = height;
179 |       colorFrame_->format = AV_PIX_FMT_BGR24;
180 | 
181 | 
182 |     } catch (const std::runtime_error &e) {
183 |       ROS_ERROR_STREAM(e.what());
184 |       reset();
185 |       return (false);
186 |     }
187 |     if (codecName != codecUsed) {
188 |       ROS_INFO_STREAM("message encoded with " <<
189 |                       codecName << " decoded with " << codecUsed);
190 |     } else {
191 |       ROS_INFO_STREAM("decoding with " << codecUsed);
192 |     }
193 |     return (true);
194 | 	}
195 | 
196 |   bool FFMPEGDecoder::decodePacket(const FFMPEGPacket::ConstPtr &msg) {
197 |     ros::WallTime t0;
198 |     if (measurePerformance_) {
199 |       t0 = ros::WallTime::now();
200 |     }
201 |     if (msg->encoding != encoding_) {
202 |       ROS_ERROR_STREAM("cannot change encoding on the fly!!!");
203 |       return (false);
204 |     }
205 |     AVCodecContext *ctx = codecContext_;
206 |     AVPacket packet;
207 |     av_init_packet(&packet);
208 |     av_new_packet(&packet, msg->data.size()); // will add some padding!
209 |     memcpy(packet.data, &msg->data[0], msg->data.size());
210 |     packet.pts = msg->pts;
211 |     packet.dts = packet.pts;
212 |     ptsToStamp_[packet.pts] = msg->header.stamp;
213 |     int ret = avcodec_send_packet(ctx, &packet);
214 |     if (ret != 0) {
215 |       ROS_WARN_STREAM("send_packet failed for pts: " <<  msg->pts);
216 |       av_packet_unref(&packet);
217 |       return (false);
218 |     }
219 |     ret = avcodec_receive_frame(ctx, decodedFrame_);
220 |     const bool isAcc = (ret == 0) && (decodedFrame_->format == hwPixFormat_);
221 |     if (isAcc) {
222 |       ret = av_hwframe_transfer_data(cpuFrame_, decodedFrame_, 0);
223 |       if (ret < 0) {
224 |         ROS_WARN_STREAM("failed to transfer data from GPU->CPU");
225 |         av_packet_unref(&packet);
226 |         return  (false);
227 |       }
228 |     }
229 |     AVFrame *frame = isAcc ? cpuFrame_ : decodedFrame_;
230 |         
231 |     if (ret == 0 && frame->width != 0) {
232 |       // convert image to something palatable
233 |       if (!swsContext_) {
234 |         swsContext_ = sws_getContext(
235 |           ctx->width, ctx->height, (AVPixelFormat)frame->format, //src
236 |           ctx->width, ctx->height, (AVPixelFormat)colorFrame_->format, // dest
237 |           SWS_FAST_BILINEAR, NULL, NULL, NULL);
238 |         if (!swsContext_) {
239 |           ROS_ERROR("cannot allocate sws context!!!!");
240 |           ros::shutdown();
241 |           return (false);
242 |         }
243 |       }
244 |       // prepare the decoded message
245 |       ImagePtr image(new sensor_msgs::Image());
246 |       image->height = frame->height;
247 |       image->width  = frame->width;
248 |       image->step   = image->width * 3; // 3 bytes per pixel
249 |       image->encoding = sensor_msgs::image_encodings::BGR8;
250 |       image->data.resize(image->step * image->height);
251 | 
252 |       // bend the memory pointers in colorFrame to the right locations 
253 |       av_image_fill_arrays(colorFrame_->data,  colorFrame_->linesize,
254 |                            &(image->data[0]),
255 |                            (AVPixelFormat)colorFrame_->format,
256 |                            colorFrame_->width, colorFrame_->height, 1);
257 |       sws_scale(swsContext_,
258 |                 frame->data,  frame->linesize, 0, // src
259 |                 ctx->height, colorFrame_->data, colorFrame_->linesize); // dest
260 |       auto it = ptsToStamp_.find(decodedFrame_->pts);
261 |       if (it == ptsToStamp_.end()) {
262 |         ROS_ERROR_STREAM("cannot find pts that matches "
263 |                          << decodedFrame_->pts);
264 |       } else {
265 |         image->header = msg->header;
266 |         image->header.stamp = it->second;
267 |         ptsToStamp_.erase(it);
268 |         callback_(image, decodedFrame_->key_frame == 1); // deliver callback
269 |       }
270 |     }
271 |     av_packet_unref(&packet);
272 |     if (measurePerformance_) {
273 |       ros::WallTime t1 = ros::WallTime::now();
274 |       double dt = (t1-t0).toSec();
275 |       tdiffTotal_.update(dt);
276 |     }
277 |     return (true);
278 |   }
279 | 
280 |   void FFMPEGDecoder::resetTimers() {
281 |     tdiffTotal_.reset();
282 |   }
283 | 
284 |   void FFMPEGDecoder::printTimers(const std::string &prefix) const {
285 |     ROS_INFO_STREAM(prefix << " total decode: " << tdiffTotal_);
286 |   }
287 |  
288 | }  // namespace
289 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 | 
  2 |                                  Apache License
  3 |                            Version 2.0, January 2004
  4 |                         http://www.apache.org/licenses/
  5 | 
  6 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  7 | 
  8 |    1. Definitions.
  9 | 
 10 |       "License" shall mean the terms and conditions for use, reproduction,
 11 |       and distribution as defined by Sections 1 through 9 of this document.
 12 | 
 13 |       "Licensor" shall mean the copyright owner or entity authorized by
 14 |       the copyright owner that is granting the License.
 15 | 
 16 |       "Legal Entity" shall mean the union of the acting entity and all
 17 |       other entities that control, are controlled by, or are under common
 18 |       control with that entity. For the purposes of this definition,
 19 |       "control" means (i) the power, direct or indirect, to cause the
 20 |       direction or management of such entity, whether by contract or
 21 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 22 |       outstanding shares, or (iii) beneficial ownership of such entity.
 23 | 
 24 |       "You" (or "Your") shall mean an individual or Legal Entity
 25 |       exercising permissions granted by this License.
 26 | 
 27 |       "Source" form shall mean the preferred form for making modifications,
 28 |       including but not limited to software source code, documentation
 29 |       source, and configuration files.
 30 | 
 31 |       "Object" form shall mean any form resulting from mechanical
 32 |       transformation or translation of a Source form, including but
 33 |       not limited to compiled object code, generated documentation,
 34 |       and conversions to other media types.
 35 | 
 36 |       "Work" shall mean the work of authorship, whether in Source or
 37 |       Object form, made available under the License, as indicated by a
 38 |       copyright notice that is included in or attached to the work
 39 |       (an example is provided in the Appendix below).
 40 | 
 41 |       "Derivative Works" shall mean any work, whether in Source or Object
 42 |       form, that is based on (or derived from) the Work and for which the
 43 |       editorial revisions, annotations, elaborations, or other modifications
 44 |       represent, as a whole, an original work of authorship. For the purposes
 45 |       of this License, Derivative Works shall not include works that remain
 46 |       separable from, or merely link (or bind by name) to the interfaces of,
 47 |       the Work and Derivative Works thereof.
 48 | 
 49 |       "Contribution" shall mean any work of authorship, including
 50 |       the original version of the Work and any modifications or additions
 51 |       to that Work or Derivative Works thereof, that is intentionally
 52 |       submitted to Licensor for inclusion in the Work by the copyright owner
 53 |       or by an individual or Legal Entity authorized to submit on behalf of
 54 |       the copyright owner. For the purposes of this definition, "submitted"
 55 |       means any form of electronic, verbal, or written communication sent
 56 |       to the Licensor or its representatives, including but not limited to
 57 |       communication on electronic mailing lists, source code control systems,
 58 |       and issue tracking systems that are managed by, or on behalf of, the
 59 |       Licensor for the purpose of discussing and improving the Work, but
 60 |       excluding communication that is conspicuously marked or otherwise
 61 |       designated in writing by the copyright owner as "Not a Contribution."
 62 | 
 63 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 64 |       on behalf of whom a Contribution has been received by Licensor and
 65 |       subsequently incorporated within the Work.
 66 | 
 67 |    2. Grant of Copyright License. Subject to the terms and conditions of
 68 |       this License, each Contributor hereby grants to You a perpetual,
 69 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 70 |       copyright license to reproduce, prepare Derivative Works of,
 71 |       publicly display, publicly perform, sublicense, and distribute the
 72 |       Work and such Derivative Works in Source or Object form.
 73 | 
 74 |    3. Grant of Patent License. Subject to the terms and conditions of
 75 |       this License, each Contributor hereby grants to You a perpetual,
 76 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 77 |       (except as stated in this section) patent license to make, have made,
 78 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 79 |       where such license applies only to those patent claims licensable
 80 |       by such Contributor that are necessarily infringed by their
 81 |       Contribution(s) alone or by combination of their Contribution(s)
 82 |       with the Work to which such Contribution(s) was submitted. If You
 83 |       institute patent litigation against any entity (including a
 84 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 85 |       or a Contribution incorporated within the Work constitutes direct
 86 |       or contributory patent infringement, then any patent licenses
 87 |       granted to You under this License for that Work shall terminate
 88 |       as of the date such litigation is filed.
 89 | 
 90 |    4. Redistribution. You may reproduce and distribute copies of the
 91 |       Work or Derivative Works thereof in any medium, with or without
 92 |       modifications, and in Source or Object form, provided that You
 93 |       meet the following conditions:
 94 | 
 95 |       (a) You must give any other recipients of the Work or
 96 |           Derivative Works a copy of this License; and
 97 | 
 98 |       (b) You must cause any modified files to carry prominent notices
 99 |           stating that You changed the files; and
100 | 
101 |       (c) You must retain, in the Source form of any Derivative Works
102 |           that You distribute, all copyright, patent, trademark, and
103 |           attribution notices from the Source form of the Work,
104 |           excluding those notices that do not pertain to any part of
105 |           the Derivative Works; and
106 | 
107 |       (d) If the Work includes a "NOTICE" text file as part of its
108 |           distribution, then any Derivative Works that You distribute must
109 |           include a readable copy of the attribution notices contained
110 |           within such NOTICE file, excluding those notices that do not
111 |           pertain to any part of the Derivative Works, in at least one
112 |           of the following places: within a NOTICE text file distributed
113 |           as part of the Derivative Works; within the Source form or
114 |           documentation, if provided along with the Derivative Works; or,
115 |           within a display generated by the Derivative Works, if and
116 |           wherever such third-party notices normally appear. The contents
117 |           of the NOTICE file are for informational purposes only and
118 |           do not modify the License. You may add Your own attribution
119 |           notices within Derivative Works that You distribute, alongside
120 |           or as an addendum to the NOTICE text from the Work, provided
121 |           that such additional attribution notices cannot be construed
122 |           as modifying the License.
123 | 
124 |       You may add Your own copyright statement to Your modifications and
125 |       may provide additional or different license terms and conditions
126 |       for use, reproduction, or distribution of Your modifications, or
127 |       for any such Derivative Works as a whole, provided Your use,
128 |       reproduction, and distribution of the Work otherwise complies with
129 |       the conditions stated in this License.
130 | 
131 |    5. Submission of Contributions. Unless You explicitly state otherwise,
132 |       any Contribution intentionally submitted for inclusion in the Work
133 |       by You to the Licensor shall be under the terms and conditions of
134 |       this License, without any additional terms or conditions.
135 |       Notwithstanding the above, nothing herein shall supersede or modify
136 |       the terms of any separate license agreement you may have executed
137 |       with Licensor regarding such Contributions.
138 | 
139 |    6. Trademarks. This License does not grant permission to use the trade
140 |       names, trademarks, service marks, or product names of the Licensor,
141 |       except as required for reasonable and customary use in describing the
142 |       origin of the Work and reproducing the content of the NOTICE file.
143 | 
144 |    7. Disclaimer of Warranty. Unless required by applicable law or
145 |       agreed to in writing, Licensor provides the Work (and each
146 |       Contributor provides its Contributions) on an "AS IS" BASIS,
147 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 |       implied, including, without limitation, any warranties or conditions
149 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 |       PARTICULAR PURPOSE. You are solely responsible for determining the
151 |       appropriateness of using or redistributing the Work and assume any
152 |       risks associated with Your exercise of permissions under this License.
153 | 
154 |    8. Limitation of Liability. In no event and under no legal theory,
155 |       whether in tort (including negligence), contract, or otherwise,
156 |       unless required by applicable law (such as deliberate and grossly
157 |       negligent acts) or agreed to in writing, shall any Contributor be
158 |       liable to You for damages, including any direct, indirect, special,
159 |       incidental, or consequential damages of any character arising as a
160 |       result of this License or out of the use or inability to use the
161 |       Work (including but not limited to damages for loss of goodwill,
162 |       work stoppage, computer failure or malfunction, or any and all
163 |       other commercial damages or losses), even if such Contributor
164 |       has been advised of the possibility of such damages.
165 | 
166 |    9. Accepting Warranty or Additional Liability. While redistributing
167 |       the Work or Derivative Works thereof, You may choose to offer,
168 |       and charge a fee for, acceptance of support, warranty, indemnity,
169 |       or other liability obligations and/or rights consistent with this
170 |       License. However, in accepting such obligations, You may act only
171 |       on Your own behalf and on Your sole responsibility, not on behalf
172 |       of any other Contributor, and only if You agree to indemnify,
173 |       defend, and hold each Contributor harmless for any liability
174 |       incurred by, or claims asserted against, such Contributor by reason
175 |       of your accepting any such warranty or additional liability.
176 | 
177 |    END OF TERMS AND CONDITIONS
178 | 
179 |    APPENDIX: How to apply the Apache License to your work.
180 | 
181 |       To apply the Apache License to your work, attach the following
182 |       boilerplate notice, with the fields enclosed by brackets "[]"
183 |       replaced with your own identifying information. (Don't include
184 |       the brackets!)  The text should be enclosed in the appropriate
185 |       comment syntax for the file format. We also recommend that a
186 |       file or class name and description of purpose be included on the
187 |       same "printed page" as the copyright notice for easier
188 |       identification within third-party archives.
189 | 
190 |    Copyright [yyyy] [name of copyright owner]
191 | 
192 |    Licensed under the Apache License, Version 2.0 (the "License");
193 |    you may not use this file except in compliance with the License.
194 |    You may obtain a copy of the License at
195 | 
196 |        http://www.apache.org/licenses/LICENSE-2.0
197 | 
198 |    Unless required by applicable law or agreed to in writing, software
199 |    distributed under the License is distributed on an "AS IS" BASIS,
200 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 |    See the License for the specific language governing permissions and
202 |    limitations under the License.
203 | 


--------------------------------------------------------------------------------