├── examples
    ├── video_00.gif
    ├── video_00.jpg
    └── video_sum.mp4
├── docker
    ├── build.sh
    ├── base.Dockerfile
    └── README.md
├── Makefile.config
├── include
    └── hecate
    │   ├── knapsack.hpp
    │   ├── time.hpp
    │   ├── sort.hpp
    │   ├── shot_range.hpp
    │   ├── gapstat.hpp
    │   ├── ffmpeg_helper.hpp
    │   ├── file_helper.hpp
    │   ├── video_parser.hpp
    │   ├── cc_parser.hpp
    │   ├── hist_opencv.hpp
    │   ├── gflseg.hpp
    │   ├── hecate.hpp
    │   └── image_metrics.hpp
├── src
    ├── tools
    │   └── hecate.cpp
    └── hecate
    │   ├── hecate_main.cpp
    │   ├── hecate_thumbnail.cpp
    │   ├── gflseg.cpp
    │   ├── hecate_highlight.cpp
    │   └── video_parser.cpp
├── Makefile
├── README.md
└── LICENSE


/examples/video_00.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/yahoo/hecate/HEAD/examples/video_00.gif


--------------------------------------------------------------------------------
/examples/video_00.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/yahoo/hecate/HEAD/examples/video_00.jpg


--------------------------------------------------------------------------------
/examples/video_sum.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/yahoo/hecate/HEAD/examples/video_sum.mp4


--------------------------------------------------------------------------------
/docker/build.sh:
--------------------------------------------------------------------------------
1 | # Builds images
2 | docker build -f base.Dockerfile -t hecate-core .
3 | docker run --name hecate -it hecate-core


--------------------------------------------------------------------------------
/Makefile.config:
--------------------------------------------------------------------------------
 1 | # Directories containing ffmpeg and opencv
 2 | INCLUDE_DIRS := /usr/local/include
 3 | LIBRARY_DIRS := /usr/local/lib
 4 | 
 5 | # Comment if you're using OpenCV 2.4
 6 | OPENCV_VERSION := 3
 7 | 
 8 | BUILD_DIR := build
 9 | DISTRIBUTE_DIR := distribute
10 | 
11 | # Uncomment for debugging.
12 | # DEBUG := 1
13 | 
14 | # enable pretty build (comment to see full commands)
15 | Q ?= @
16 | 


--------------------------------------------------------------------------------
/include/hecate/knapsack.hpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * 0/1 Knapsack solver
 3 |  *
 4 |  * Copyright 2016 Yahoo Inc.
 5 |  * Licensed under the terms of the Apache 2.0 License.
 6 |  * See LICENSE file in the project root for terms.
 7 |  *
 8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
 9 |  */
10 | 
11 | #ifndef HECATE_KNAPSACK_HPP
12 | #define HECATE_KNAPSACK_HPP
13 | 
14 | #include <stdio.h>
15 | 
16 | namespace hecate {
17 |   
18 |   template <typename T>
19 |   inline T Tmax(T a, T b) {return (a>b) ? a : b; };
20 |   
21 |   template <typename T>
22 |   inline void solve_01knapsack( const vector<T>& value, const vector<int>& weight,
23 |                                const int budget, vector<bool>& solution)
24 |   {
25 |     int n = (int)value.size();
26 |     solution.resize(n, false);
27 |     
28 |     // recursion
29 |     vector<vector<T> > V( n+1, vector<T>(budget+1,0) );
30 |     for( int i=1; i<=n; i++ )
31 |       for( int w=1; w<=budget; w++ )
32 |         V[i][w] = ( weight[i-1]>w )
33 |         ? V[i-1][w] : Tmax( V[i-1][w], V[i-1][w-weight[i-1]] + value[i-1] );
34 |     
35 |     // backtrack
36 |     int w = budget;
37 |     for( int i=n; i>0; i-- ) {
38 |       if( V[i][w]!=V[i-1][w] && V[i][w]==V[i-1][w-weight[i-1]]+value[i-1] ) {
39 |         solution[i-1] = true;
40 |         w -= weight[i-1];
41 |       }
42 |     }
43 |   }
44 |   
45 | }
46 | #endif
47 | 
48 | 
49 | 


--------------------------------------------------------------------------------
/include/hecate/time.hpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Time util
 3 |  *
 4 |  * Copyright 2016 Yahoo Inc.
 5 |  * Licensed under the terms of the Apache 2.0 License.
 6 |  * See LICENSE file in the project root for terms.
 7 |  *
 8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
 9 |  */
10 | 
11 | #ifndef HECATE_TIME_HPP
12 | #define HECATE_TIME_HPP
13 | 
14 | #include <stdio.h>
15 | 
16 | #include <ctime>      // display date and time
17 | #include <cmath>      // floor
18 | #include <chrono>
19 | #include <string>
20 | 
21 | namespace hecate {
22 |   
23 |   typedef std::chrono::high_resolution_clock Clock;
24 |   typedef std::chrono::milliseconds milliseconds;
25 |   
26 |   inline double elapsed_time_ms( Clock::time_point from)
27 |   {
28 |     Clock::time_point now = Clock::now();
29 |     milliseconds ms = std::chrono::duration_cast<milliseconds>( now - from );
30 |     return (double)ms.count();
31 |   }
32 |   
33 |   inline double print_elapsed_time( Clock::time_point from, const char* prefix)
34 |   {
35 |     int msec = (int) elapsed_time_ms(from);
36 |     int sec = msec/1000;
37 |     printf("%s: Elapsed time %02d:%02d:%04d\n", prefix, sec/60, sec%60, msec%1000);
38 |     
39 |     return msec / 1000.0;
40 |   }
41 |   
42 |   inline std::string second2string(double sec, const std::string& format) {
43 |     char buf[128];
44 |     int hh, mm, ss, mss;
45 |     hh  = (int) floor( sec/3600 );
46 |     mm  = (int) floor( sec/60 ) % 60;
47 |     ss  = (int) sec % 60;
48 |     mss = (int) 10000*(sec - floor(sec));
49 |     if( format == "hh:mm:ss.mss" ) {
50 |       sprintf( buf, "%02d:%02d:%02d.%04d", hh,mm,ss,mss );
51 |     }
52 |     else if( format == "mm:ss.mss" ) {
53 |       sprintf( buf, "%02d:%02d.%04d", 60*hh+mm,ss,mss );
54 |     }
55 |     std::string ret = buf;
56 |     return ret;
57 |   }
58 | }
59 | #endif
60 | 
61 | 


--------------------------------------------------------------------------------
/src/tools/hecate.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * HECATE Yahoo Video Processing Library - Binary
 3 |  *
 4 |  * Copyright 2016 Yahoo Inc.
 5 |  * Licensed under the terms of the Apache 2.0 License.
 6 |  * See LICENSE file in the project root for terms.
 7 |  *
 8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
 9 |  */
10 | 
11 | #include "hecate/hecate.hpp"
12 | 
13 | using namespace std;
14 | using namespace cv;
15 | 
16 | int main( int argc, char** argv )
17 | {
18 |   hecate_copyright();
19 |   if( argc<3 )
20 |     hecate_usage();
21 |   
22 |   // Read input params
23 |   hecate_params opt;
24 |   hecate_parse_params( argc, argv, opt );
25 |   
26 |   // Run VIDSUM
27 |   vector<int> v_thumb_idx;
28 |   vector<hecate::Range> v_gif_range;
29 |   vector<hecate::Range> v_mov_range;
30 |   run_hecate( opt, v_thumb_idx, v_gif_range, v_mov_range );
31 |   
32 |   // Print debugging info
33 |   if( opt.debug ) {
34 |     if( opt.jpg ) {
35 |       printf("hecate: thumbnail indices: [ ");
36 |       for(size_t i=0; i<v_thumb_idx.size(); i++)
37 |         printf("%d ", v_thumb_idx[i]);
38 |       printf("]\n");
39 |     }
40 |     if( opt.gif ) {
41 |       printf("hecate: gif shot ranges:\n");
42 |       for(int i=0; i<v_gif_range.size(); i++)
43 |         v_gif_range[i].print();
44 |     }
45 |     if( opt.mov ) {
46 |       printf("hecate: mov shot ranges:\n");
47 |       for(int i=0; i<v_mov_range.size(); i++)
48 |         v_mov_range[i].print();
49 |     }
50 |   }
51 |   
52 |   // Produce results
53 |   if( opt.jpg ) {
54 |     generate_thumbnails( opt, v_thumb_idx );
55 |   }
56 |   
57 |   if( opt.gif ) {
58 |     bool mov = opt.mov;
59 |     opt.mov = false;
60 |     generate_highlight_clips( opt, v_gif_range );
61 |     opt.mov = mov;
62 |   }
63 |   
64 |   if( opt.mov ) {
65 |     bool gif = opt.gif;
66 |     opt.gif = false;
67 |     generate_highlight_clips( opt, v_mov_range );
68 |     opt.gif = gif;
69 |   }
70 | }
71 | 


--------------------------------------------------------------------------------
/docker/base.Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM debian:buster-slim
 2 | 
 3 | # System packages
 4 | RUN apt-get clean && apt-get update -y && apt-get upgrade -y
 5 | 
 6 | # install dependencies
 7 | RUN apt-get install -y git wget vim build-essential cmake pkg-config libavcodec-dev libavformat-dev libswscale-dev libv4l-dev libxvidcore-dev libx264-dev libgtk-3-dev libatlas-base-dev gfortran ffmpeg
 8 | 
 9 | # install opencv_contrib
10 | RUN git clone https://github.com/opencv/opencv_contrib.git && \
11 |     cd opencv_contrib && \
12 |     git checkout 3.3.1 && \
13 |     cd ..
14 | 
15 | # install opencv
16 | RUN git clone https://github.com/opencv/opencv.git && \
17 |     cd opencv && \
18 |     git checkout 3.3.1 && \
19 |     sed -i "43i #define AV_CODEC_FLAG_GLOBAL_HEADER (1 << 22)\n#define CODEC_FLAG_GLOBAL_HEADER AV_CODEC_FLAG_GLOBAL_HEADER\n#define AVFMT_RAWPICTURE 0x0020" modules/videoio/src/cap_ffmpeg_impl.hpp && \
20 |     mkdir build
21 |     
22 | RUN cd /opencv/build && \
23 |     cmake -D CMAKE_BUILD_TYPE=RELEASE \
24 |     -D CMAKE_INSTALL_PREFIX=/usr/local \
25 |     -D INSTALL_C_EXAMPLES=ON \
26 |     -D OPENCV_EXTRA_MODULES_PATH=/opencv_contrib/modules \
27 |     -D BUILD_EXAMPLES=ON \
28 |     -D WITH_FFMPEG=1 \
29 |     -D ENABLE_FAST_MATH=1 \
30 |     -D WITH_LAPACK=OFF .. && \
31 |     make -j2 && \
32 |     make install && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib>>~/.bashrc
33 | 
34 | # install hecate
35 | RUN git clone https://github.com/yahoo/hecate.git && \
36 |     cd hecate && \
37 |     make all && \
38 |     make distribute
39 | 
40 | RUN echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib">>~/.bashrc
41 | 
42 | # references
43 | # # ref: https://cerebrumedge.com/blog/entry/compiling-opencv-with-cuda-and-ffmpeg-on-ubuntu-16.04#:~:text=FFMpeg%20and%20OpenCV,OPENCV_SOURCE_CODE%2F3rdparty%2Fffmpeg%2F.
44 | # https://stackoverflow.com/questions/46884682/error-in-building-opencv-with-ffmpeg
45 | # https://stackoverflow.com/questions/12335848/opencv-program-compile-error-libopencv-core-so-2-4-cannot-open-shared-object-f


--------------------------------------------------------------------------------
/docker/README.md:
--------------------------------------------------------------------------------
 1 | # HECATE - DOCKER INSTALLATION
 2 | 
 3 | 
 4 | ## Installation
 5 | 
 6 | First you need to install docker on your local computer, see following [tutorial](https://docs.docker.com/install/linux/docker-ce/ubuntu/#set-up-the-repository). Note, for running the docker properly you have be logged as superuser otherwise you will face many partial issues which sometimes does not make much sense.
 7 | 
 8 | Check your installation using
 9 | ```
10 | docker -v
11 | ```
12 | 
13 | Once you install the docker correctly, follow the instruction below:
14 | ```
15 | $ git clone https://github.com/yahoo/hecate.git
16 | $ cd hecate/docker
17 | $ ./build.sh
18 | ```
19 | 
20 | Now the hecate CLI should open up for you or else you may need to:
21 | ```
22 | docker exec -it hecate bash
23 | ```
24 | 
25 | Once you've successfully compiled hecate, it will generate a binary executable under `distribute/bin/`. Run the following command to check if everything works properly:
26 | ```
27 | $ cd hecate
28 | $ ./distribute/bin/hecate
29 | 
30 | -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
31 |  HECATE Copyright 2016 Yahoo Inc.
32 |    Licensed under the terms of the Apache 2.0 License.
33 |    Developed by : Yale Song (yalesong@yahoo-inc.com)
34 |    Built on  : 11:46:03 Aug 11 2016
35 | -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
36 | USAGE: hecate -i infile [options]
37 | 
38 |   -i  --in_video      (string)    Input video file
39 |   -o  --out_dir       (string)    Output directory (./output)
40 |   -s  --step          (int)       Frame subsampling step size (1)
41 |   -n  --njpg          (int)       Number of thumbnails to be generated (5)
42 |   -q  --ngif          (int)       Number of GIFs to be generated (5)
43 |   -r  --lmov          (int)       Length of video summary to be generated (in seconds) (15)
44 |   -u  --jpg_width_px  (int)       Pixel width of thumbnail images (360)
45 |   -v  --gif_width_px  (int)       Pixel width of animated GIFs (360)
46 |   -w  --mov_width_px  (int)       Pixel width of summary video (360)
47 |   --generate_jpg                  Generate thumbnail images
48 |   --generate_gif                  Generate animated GIFs
49 |   --generate_mov                  Generate a summary video
50 |   --generate_gifsum               Generate animated GIFs summary
51 |   --generate_gifall               Generate all possible animated GIFs
52 |   --print_shot_info               Print shot boundary detection results
53 |   --print_keyfrm_info             Print keyframe indices
54 | ```
55 | 
56 | Congratulations! You have successfully installed hecate!
57 | 
58 | 
59 | ## Docker Developer
60 | 
61 | Sayantan Mandal: [github](https://github.com/smandal047)
62 | 
63 | ## 


--------------------------------------------------------------------------------
/include/hecate/sort.hpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * MATLAB-like sort
 3 |  *
 4 |  * Copyright 2016 Yahoo Inc.
 5 |  * Licensed under the terms of the Apache 2.0 License.
 6 |  * See LICENSE file in the project root for terms.
 7 |  *
 8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
 9 |  */
10 | 
11 | #ifndef HECATE_SORT_HPP
12 | #define HECATE_SORT_HPP
13 | 
14 | #include <vector>
15 | #include <algorithm>
16 | 
17 | namespace hecate {
18 |   
19 |   // Act like matlab's [Y,I] = SORT(X)
20 |   // Input:
21 |   //   unsorted  unsorted vector
22 |   // Output:
23 |   //   sorted     sorted vector, allowed to be same as unsorted
24 |   //   index_map  an index map such that sorted[i] = unsorted[index_map[i]]
25 |   template <class T>
26 |   void sort(
27 |             std::vector<T> &unsorted,
28 |             std::vector<T> &sorted,
29 |             std::vector<size_t> &index_map);
30 |   
31 |   // Act like matlab's Y = X[I]
32 |   // where I contains a vector of indices so that after,
33 |   // Y[j] = X[I[j]] for index j
34 |   // this implies that Y.size() == I.size()
35 |   // X and Y are allowed to be the same reference
36 |   template< class T >
37 |   void reorder(
38 |                std::vector<T> & unordered,
39 |                std::vector<size_t> const & index_map,
40 |                std::vector<T> & ordered);
41 |   
42 |   ////////////////////////////////////////////////////////////////////////////////
43 |   // Implementation
44 |   ////////////////////////////////////////////////////////////////////////////////
45 |   
46 |   
47 |   // Comparison struct used by sort
48 |   template<class T> struct index_cmp
49 |   {
50 |     index_cmp(const T arr) : arr(arr) {}
51 |     bool operator()(const size_t a, const size_t b) const
52 |     {
53 |       return arr[a] < arr[b];
54 |     }
55 |     const T arr;
56 |   };
57 |   
58 |   template <class T>
59 |   inline void sort(
60 |             std::vector<T> & unsorted,
61 |             std::vector<T> & sorted,
62 |             std::vector<size_t> & index_map)
63 |   {
64 |     // Original unsorted index map
65 |     index_map.resize(unsorted.size());
66 |     for(size_t i=0;i<unsorted.size();i++)
67 |     {
68 |       index_map[i] = i;
69 |     }
70 |     // Sort the index map, using unsorted for comparison
71 |     sort(
72 |          index_map.begin(),
73 |          index_map.end(),
74 |          index_cmp<std::vector<T>& >(unsorted));
75 |     
76 |     sorted.resize(unsorted.size());
77 |     reorder(unsorted,index_map,sorted);
78 |   }
79 |   
80 |   // This implementation is O(n), but also uses O(n) extra memory
81 |   template< class T >
82 |   inline void reorder(
83 |                std::vector<T> & unordered,
84 |                std::vector<size_t> const & index_map,
85 |                std::vector<T> & ordered)
86 |   {
87 |     // copy for the reorder according to index_map, because unsorted may also be
88 |     // sorted
89 |     std::vector<T> copy = unordered;
90 |     ordered.resize(index_map.size());
91 |     for(unsigned int i = 0; i<index_map.size();i++)
92 |     {
93 |       ordered[i] = copy[index_map[i]];
94 |     }
95 |   }
96 |   
97 | }
98 | #endif
99 | 


--------------------------------------------------------------------------------
/include/hecate/shot_range.hpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Shot range definition
  3 |  *
  4 |  * Copyright 2016 Yahoo Inc.
  5 |  * Licensed under the terms of the Apache 2.0 License.
  6 |  * See LICENSE file in the project root for terms.
  7 |  *
  8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
  9 |  */
 10 | 
 11 | #ifndef HECATE_SHOT_RANGE_HPP
 12 | #define HECATE_SHOT_RANGE_HPP
 13 | 
 14 | #include <string>
 15 | #include <vector>
 16 | 
 17 | #include <hecate/sort.hpp>
 18 | 
 19 | namespace hecate {
 20 |   
 21 |   /*
 22 |    */
 23 |   class Range {
 24 |   public:
 25 |     int start;
 26 |     int end;
 27 |     vector<int> v_idx; // store indices (keyframe index)
 28 |     
 29 |   public:
 30 |     Range(int s, int e): start(s), end(e) {};
 31 |     
 32 |     inline int length() const { return end-start+1; };
 33 |     
 34 |     inline void print() const {
 35 |       printf("range(%d:%d) (%d) [", start, end, length() );
 36 |       for(size_t i=0; i<v_idx.size(); i++) {
 37 |         printf("%d", v_idx[i]);
 38 |         if( i+1<v_idx.size() )
 39 |           printf(",");
 40 |       }
 41 |       printf("]\n");
 42 |     };
 43 |   };
 44 |   
 45 |   /*
 46 |    */
 47 |   class ShotRange: public Range {
 48 |   public:
 49 |     vector<Range> v_range;
 50 |     
 51 |   public:
 52 |     ShotRange(int s, int e): Range(s,e) {};
 53 |     
 54 |     inline void print() const {
 55 |       printf("shot_range(%d:%d) (%d) [", start, end, length() );
 56 |       for(size_t i=0; i<v_idx.size(); i++) {
 57 |         printf("%d", v_idx[i]);
 58 |         if( i+1<v_idx.size() )
 59 |           printf(",");
 60 |       }
 61 |       printf("]\n");
 62 |       
 63 |       for(size_t i=0; i<v_range.size(); i++) {
 64 |         printf("  sub_");
 65 |         v_range[i].print();
 66 |       }
 67 |     };
 68 |   };
 69 |   
 70 |   /*
 71 |    */
 72 |   class Tag {
 73 |   public:
 74 |     string label;
 75 |     double score;
 76 |     
 77 |   public:
 78 |     Tag(string& l, double s): label(l), score(s) {};
 79 |     
 80 |     inline void print() const {
 81 |       printf("(%s,%f)", label.c_str(), score);
 82 |     }
 83 |   };
 84 |   
 85 |   /*
 86 |    */
 87 |   class RangeTag: public Range {
 88 |   public:
 89 |     vector<Tag> v_tag;
 90 |   
 91 |   public:
 92 |     RangeTag(int s, int e): Range(s,e) {};
 93 |     RangeTag(Range r): Range(r.start,r.end) {
 94 |       v_idx=r.v_idx;
 95 |     };
 96 |     
 97 |     inline void print() const {
 98 |       printf("range(%d:%d) (%d) [", start, end, length() );
 99 |       for(size_t i=0; i<v_idx.size(); i++) {
100 |         printf("%d", v_idx[i]);
101 |         if( i+1<v_idx.size() )
102 |           printf(",");
103 |       }
104 |       printf("], tags: ");
105 |       for(size_t i=0; i<v_tag.size(); i++)
106 |         v_tag[i].print();
107 |       printf("\n");
108 |     };
109 |     
110 |     inline void sort() {
111 |       vector<double> v_scores;
112 |       for(size_t i=0; i<v_tag.size(); i++)
113 |         v_scores.push_back( v_tag[i].score );
114 |       vector<double> v_srtval;
115 |       vector<size_t> v_srtidx;
116 |       hecate::sort( v_scores, v_srtval, v_srtidx );
117 |       vector<Tag> v_tag_new;
118 |       for(size_t i=0; i<v_srtidx.size(); i++)
119 |         v_tag_new.push_back( v_tag[v_srtidx[v_srtidx.size()-1-i]] );
120 |       v_tag = v_tag_new;
121 |     }
122 |   };
123 |   
124 |   /*
125 |    */
126 |   class ShotRangeTag: public Range {
127 |   public:
128 |     vector<RangeTag> v_range_tag;
129 |     
130 |   public:
131 |     ShotRangeTag(int s, int e): Range(s,e) {};
132 |     ShotRangeTag(ShotRange& sr): Range(sr.start,sr.end) {
133 |       v_idx = sr.v_idx;
134 |       for(size_t i=0; i<sr.v_range.size(); i++) {
135 |         RangeTag rt( sr.v_range[i] );
136 |         v_range_tag.push_back( rt );
137 |       }
138 |     };
139 |     
140 |     inline void print() {
141 |       printf("shot_range_tag(%d:%d) (%d) [", start, end, length() );
142 |       for(size_t i=0; i<v_idx.size(); i++) {
143 |         printf("%d", v_idx[i]);
144 |         if( i+1<v_idx.size() )
145 |           printf(",");
146 |       }
147 |       printf("]\n");
148 |       
149 |       for(size_t i=0; i<v_range_tag.size(); i++) {
150 |         printf("  sub_");
151 |         v_range_tag[i].print();
152 |       }
153 |     };
154 |     
155 |     inline void sort() {
156 |       for(size_t i=0; i<v_range_tag.size(); i++)
157 |         v_range_tag[i].sort();
158 |     };
159 |   };
160 |   
161 | }
162 | #endif
163 | 
164 | 
165 | 


--------------------------------------------------------------------------------
/include/hecate/gapstat.hpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Gap statistics
  3 |  *
  4 |  * Copyright 2016 Yahoo Inc.
  5 |  * Licensed under the terms of the Apache 2.0 License.
  6 |  * See LICENSE file in the project root for terms.
  7 |  *
  8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
  9 |  */
 10 | 
 11 | 
 12 | #ifndef HECATE_GAPSTAT_HPP
 13 | #define HECATE_GAPSTAT_HPP
 14 | 
 15 | #include <stdio.h>
 16 | #include <stdlib.h>
 17 | 
 18 | #include <iostream>   // for standard I/O
 19 | #include <limits>
 20 | #include <numeric>
 21 | 
 22 | #include <opencv2/opencv.hpp>
 23 | #include <opencv2/core/core.hpp>
 24 | #include <opencv2/imgproc/imgproc.hpp>
 25 | #include <opencv2/highgui/highgui.hpp>
 26 | 
 27 | #if CV_MAJOR_VERSION > 2
 28 | #include <opencv2/core/core_c.h>
 29 | #include <opencv2/imgproc/types_c.h>
 30 | #endif
 31 | 
 32 | using namespace std;
 33 | using namespace cv;
 34 | 
 35 | namespace hecate {
 36 |   
 37 |   // Compute mean-squared error (mse)
 38 |   inline double calc_mse( const Mat& km_data, const Mat& km_lbl, const Mat& km_ctr )
 39 |   {
 40 |     double compactness = 0;
 41 |     
 42 |     int nsamples = km_lbl.rows;
 43 |     int nclusters  = km_ctr.rows;
 44 |     
 45 |     for( int k=0; k<nclusters; k++ ) {
 46 |       int nk = 0;
 47 |       double err = 0;
 48 |       for( int i=0; i<nsamples; i++ ) {
 49 |         if( km_lbl.at<int>(i)==k ) {
 50 |           nk++;
 51 |           err += pow(cv::norm(km_data.row(i)-km_ctr.row(k)),2.0);
 52 |         }
 53 |       }
 54 |       compactness += 0.5 * err / nk;
 55 |     }
 56 |     
 57 |     return compactness;
 58 |   }
 59 |   
 60 |   inline void find_bounds( const Mat& km_data, Mat& bounds )
 61 |   {
 62 |     int ndim = km_data.cols;
 63 |     bounds = Mat(2,ndim,CV_64F);
 64 |     
 65 |     double minval, maxval;
 66 |     for( int d=0; d<ndim; d++ ){
 67 |       minMaxLoc( km_data.col(d), &minval, &maxval );
 68 |       bounds.at<double>(0,d) = minval;
 69 |       bounds.at<double>(1,d) = maxval;
 70 |     }
 71 |   }
 72 |   
 73 |   inline void randu_bound( Mat& X, const Mat& bounds, int nsamples, int ndims )
 74 |   {
 75 |     X = Mat(nsamples, ndims, CV_32F);
 76 |     randu( X, Scalar::all(0.0), Scalar::all(1.0) );
 77 |     for(int c=0; c<X.cols; c++) {
 78 |       double lb = bounds.at<double>(0,c);
 79 |       double ub = bounds.at<double>(1,c);
 80 |       X.col(c) = (ub-lb)*X.col(c) + lb;
 81 |     }
 82 |   }
 83 |   
 84 |   //fix error when there is only 1 data point: http://docs.opencv.org/2.4/modules/core/doc/clustering.html
 85 |   inline void perform_kmeans(const Mat& km_data, Mat& km_lbl, Mat& km_ctr, int ncluster,
 86 |                              int km_attempts=1, int km_max_cnt=1000, double km_eps=0.0001)
 87 |   {
 88 |       if(km_data.rows==1){
 89 |           km_lbl = Mat::zeros(1,1, km_lbl.type());
 90 |           cv::reduce( km_data, km_ctr, 0, CV_REDUCE_AVG );
 91 |       }
 92 |       else{
 93 |         int km_k = min(ncluster, km_data.rows);
 94 |         TermCriteria km_opt = TermCriteria(CV_TERMCRIT_ITER|CV_TERMCRIT_EPS, km_max_cnt, km_eps);
 95 |         kmeans( km_data, km_k, km_lbl, km_opt, km_attempts, KMEANS_PP_CENTERS, km_ctr );
 96 |       }
 97 | }
 98 |   
 99 |   inline int perform_kmeans_gs( const Mat& km_data, Mat& km_lbl, Mat& km_ctr,
100 |                                vector<int> K, int B=10, int N=500 )
101 |   {
102 |     int ndims = km_data.cols;
103 |     
104 |     // Generate null reference dataset
105 |     vector<Mat> v_Xb;
106 |     Mat bounds; find_bounds( km_data, bounds );
107 |     for(int b=0; b<B; b++) {
108 |       Mat Xb;
109 |       randu_bound( Xb, bounds, N, ndims );
110 |       v_Xb.push_back( Xb );
111 |     }
112 |     
113 |     // Compute Gap Statistics
114 |     vector<double> gap(K.size(), 0.0);    // gap statistics = E[logWk] - logWk
115 |     vector<double> ElogWk(K.size(), 0.0); // compactness of the null reference distribution
116 |     vector<double> logWk(K.size(), 0.0);  // compactness of the given data
117 |     vector<double> Sk(K.size(), 0.0);   // standard deviations
118 |     
119 |     for( size_t i=0; i<K.size(); i++ )
120 |     {
121 |       Mat lbl, ctr;
122 |       perform_kmeans( km_data, lbl, ctr, K[i] );
123 |       logWk[i] = log(calc_mse( km_data, lbl, ctr ));
124 |       
125 |       ElogWk[i] = 0.0;
126 |       vector<double> logWkb(B, 0.0);
127 |       for(int b=0; b<B; b++)
128 |       {
129 |         Mat lbl_b, ctr_b;
130 |         perform_kmeans( v_Xb[b], lbl_b, ctr_b, K[i], 100, 1000, 0.001 );
131 |         logWkb[b] = log(calc_mse( v_Xb[b], lbl_b, ctr_b ));
132 |         ElogWk[i] += logWkb[b];
133 |       }
134 |       ElogWk[i] /= B;
135 |       
136 |       Sk[i] = 0.0;
137 |       for(int b=0; b<B; b++)
138 |         Sk[i] += (logWkb[b]-ElogWk[i])*(logWkb[b]-ElogWk[i]);
139 |       Sk[i] = sqrt(1.0+1.0/B) * sqrt( Sk[i] / B );
140 |       
141 |       gap[i] = ElogWk[i] - logWk[i];
142 |       printf("\tgapstat: k=%d, logWk=%f, ElogWk=%f, Sk=%f, Gap=%f\n",
143 |              K[i], logWk[i], ElogWk[i], Sk[i], gap[i]);
144 |     }
145 |     
146 |     // find the smallest k such that gap(k) >= gap(k+1) - S[k+1]
147 |     size_t kstar=0;
148 |     for(size_t i=0; i<K.size()-1; i++) {
149 |       if( gap[i] >= gap[i+1] - Sk[i+1] ) {
150 |         kstar = i;
151 |         break;
152 |       }
153 |     }
154 |     printf("\tgapstat: Optimal k=%d [%d:%d]\n", K[kstar], K[0], K[K.size()-1]);
155 |     
156 |     // return results
157 |     perform_kmeans( km_data, km_lbl, km_ctr, K[kstar] );
158 |     return K[kstar];
159 |     
160 |   }
161 | }
162 | 
163 | #endif
164 | 
165 | 


--------------------------------------------------------------------------------
/include/hecate/ffmpeg_helper.hpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Various helper functions using ffmpeg
  3 |  *
  4 |  * Copyright 2016 Yahoo Inc.
  5 |  * Licensed under the terms of the Apache 2.0 License.
  6 |  * See LICENSE file in the project root for terms.
  7 |  *
  8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
  9 |  */
 10 | 
 11 | 
 12 | #ifndef HECATE_FFMPEG_HPP
 13 | #define HECATE_FFMPEG_HPP
 14 | 
 15 | #include <stdio.h>
 16 | #include <string.h>
 17 | 
 18 | #include <iostream>
 19 | #include <fstream>
 20 | #include <string>
 21 | #include <vector>
 22 | 
 23 | #include "hecate/file_helper.hpp"
 24 | 
 25 | #define BUF_S 256
 26 | #define BUF_M 512
 27 | #define BUF_L 1024
 28 | 
 29 | using namespace std;
 30 | 
 31 | namespace hecate {
 32 |   const string _ffmpeg = hecate::which("ffmpeg");
 33 |   const string _ffmarg = "-loglevel quiet -y";
 34 |   const int _lfade = 8; // fade-in/out duration (unit: frame)
 35 |   
 36 |   // Crop video segment & resize
 37 |   static inline void ffmpeg_video_crop(string in_file,
 38 |                                        string out_file,
 39 |                                        string start_time,
 40 |                                        string duration,
 41 |                                        int out_width_px)
 42 |   {
 43 |     in_file = escape_space(in_file);
 44 |     
 45 |     char filter[BUF_S];
 46 |     sprintf( filter, "'scale=%d:trunc(ow/a/2)*2'", out_width_px);
 47 |     
 48 |     char cmd[BUF_L];
 49 |     sprintf( cmd, "%s -ss %s -t %s -i %s -strict -2 -vf %s %s %s",
 50 |             _ffmpeg.c_str(), start_time.c_str(), duration.c_str(),
 51 |             in_file.c_str(), filter, out_file.c_str(), _ffmarg.c_str());
 52 |     
 53 |     system( cmd );
 54 |   };
 55 |   
 56 |   static inline void ffmpeg_video_concat(string in_filelist,
 57 |                                          string out_file)
 58 |   {
 59 |     char cmd[BUF_L];
 60 |     sprintf( cmd, "%s -f concat -i %s -c copy %s %s",
 61 |             _ffmpeg.c_str(), in_filelist.c_str(), out_file.c_str(),
 62 |             _ffmarg.c_str() );
 63 |     system( cmd );
 64 |   };
 65 |   
 66 |   // Audio fade in/out
 67 |   static inline void ffmpeg_audio_fade(string in_file,
 68 |                                        string out_file,
 69 |                                        double video_duration_sec,
 70 |                                        double video_fps)
 71 |   {
 72 |     in_file = escape_space(in_file);
 73 |     
 74 |     const double afade_sec = (double)2*_lfade/video_fps;
 75 |     const double afade_msec = (int)10000*(afade_sec-floor(afade_sec));
 76 |     
 77 |     double afos  = video_duration_sec - afade_sec; // audio fade-out start
 78 |     int afos_ss  = (int) afos; // NOTE: we don't compute modulo this time
 79 |     int afos_mss = (int) 10000*(afos - floor(afos));
 80 |     
 81 |     char filter[BUF_S];
 82 |     sprintf( filter, "'afade=t=in:ss=0:d=0.%04d,"
 83 |                       "afade=t=out:st=%d.%04d:d=0.%04d'",
 84 |             (int)afade_msec, afos_ss, afos_mss, (int)afade_msec);
 85 | 
 86 |     char cmd[BUF_L];
 87 |     sprintf( cmd, "%s -i %s -af %s %s %s", _ffmpeg.c_str(),
 88 |             in_file.c_str(), filter, out_file.c_str(), _ffmarg.c_str() );
 89 |     system( cmd );
 90 | 
 91 |   };
 92 |   
 93 |   // Video fade in/out
 94 |   static inline void ffmpeg_video_fade(string in_file,
 95 |                                        string out_file,
 96 |                                        int video_duration,
 97 |                                        bool out_only=false)
 98 |   {
 99 |     in_file = escape_space(in_file);
100 |     
101 |     char filter[BUF_S];
102 |     if( out_only ) {
103 |       sprintf( filter, "'fade=out:%d:%d'",
104 |               video_duration-_lfade, _lfade);
105 |     }
106 |     else {
107 |       sprintf( filter, "'fade=in:0:%d,fade=out:%d:%d'",
108 |               _lfade, video_duration-_lfade, _lfade);
109 |     }
110 |     
111 |     char cmd[BUF_L];
112 |     sprintf( cmd, "%s -i %s -vf %s %s %s", _ffmpeg.c_str(),
113 |             in_file.c_str(), filter, out_file.c_str(), _ffmarg.c_str());
114 |     system( cmd );
115 |   };
116 |   
117 |   
118 |   // Based on http://blog.pkh.me/p/21-high-quality-gif-with-ffmpeg.html
119 |   static inline void ffmpeg_video2gif(string in_file,
120 |                                       string out_file,
121 |                                       string start_time,
122 |                                       string duration,
123 |                                       int out_fps,
124 |                                       int out_width_px)
125 |   {
126 |     in_file = escape_space(in_file);
127 |     
128 |     string out_dir = hecate::get_dir( std::string(out_file) );
129 |     
130 |     char filter[BUF_S];
131 |     sprintf( filter, "fps=%d,scale=%d:-1:flags=lanczos", out_fps, out_width_px );
132 |     
133 |     char palette[BUF_M];
134 |     sprintf( palette, "%s/palatte.png", out_dir.c_str() );
135 |     
136 |     char time_setup[BUF_S] = "";
137 |     if( !start_time.empty() && !duration.empty() ) {
138 |       sprintf( time_setup, "-ss %s -t %s",
139 |               start_time.c_str(), duration.c_str() );
140 |     }
141 | 
142 |     char cmd[BUF_L];
143 |     
144 |     // Create a palatte
145 |     sprintf( cmd, "%s %s -i %s -vf '%s,palettegen=stats_mode=diff' %s %s",
146 |             _ffmpeg.c_str(), time_setup,  in_file.c_str(), filter,
147 |             _ffmarg.c_str(), palette );
148 |     system( cmd );
149 |     
150 |     // Convert segment to gif
151 |     sprintf( cmd, "%s %s -i %s -i %s -lavfi '%s [x]; [x][1:v] paletteuse' %s %s",
152 |             _ffmpeg.c_str(), time_setup, in_file.c_str(), palette, filter,
153 |             _ffmarg.c_str(), out_file.c_str());
154 |     system( cmd );
155 |     
156 |     // Delete palette
157 |     sprintf( cmd, "rm %s", palette );
158 |     system( cmd );
159 |   };
160 |   
161 |   
162 |   
163 | }
164 | #endif
165 | 
166 | 


--------------------------------------------------------------------------------
/include/hecate/file_helper.hpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Various helper functions for filesystem
  3 |  *
  4 |  * Copyright 2016 Yahoo Inc.
  5 |  * Licensed under the terms of the Apache 2.0 License.
  6 |  * See LICENSE file in the project root for terms.
  7 |  *
  8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
  9 |  */
 10 | 
 11 | #ifndef HECATE_FILE_HPP
 12 | #define HECATE_FILE_HPP
 13 | 
 14 | #include <stdio.h>
 15 | #include <string.h>
 16 | 
 17 | #include <iostream>
 18 | #include <fstream>
 19 | #include <algorithm>
 20 | #include <functional>
 21 | #include <cctype>
 22 | #include <locale>
 23 | #include <string>
 24 | #include <vector>
 25 | 
 26 | namespace hecate {
 27 |   
 28 |   // GET FILENAME WITHOUT EXTENSION AND TRAILING DIRECTORIES
 29 |   struct FileParts
 30 |   {
 31 |     std::string path;
 32 |     std::string name;
 33 |     std::string ext;
 34 |   };
 35 |   
 36 |   static inline FileParts fileparts(std::string filename)
 37 |   {
 38 |     int idx0 = filename.rfind("/");
 39 |     int idx1 = filename.rfind(".");
 40 |     
 41 |     if( idx1 == (int) std::string::npos )
 42 |       idx1 = filename.length();
 43 |     
 44 |     FileParts fp;
 45 |     fp.path = filename.substr(0,idx0+1);
 46 |     fp.name = filename.substr(idx0+1,idx1-idx0-1);
 47 |     fp.ext  = filename.substr(idx1);
 48 |     
 49 |     return fp;
 50 |   };
 51 |   
 52 |   static inline std::string get_dir( std::string filepath ) {
 53 |     hecate::FileParts fp = hecate::fileparts( filepath );
 54 |     std::string dir = fp.path;
 55 |     return dir;
 56 |   };
 57 |   
 58 |   static inline std::string escape_space( std::string s ) {
 59 |     std::string out;
 60 |     for( size_t i=0; i<s.size(); i++) {
 61 |       if( s[i] == ' ' )
 62 |         out += '\\';
 63 |       out += s[i];
 64 |     }
 65 |     return out;
 66 |   }
 67 |   
 68 |   static inline std::string get_filename( std::string filepath ) {
 69 |     hecate::FileParts fp = hecate::fileparts( filepath );
 70 |     std::string filename = fp.name;
 71 |     replace( filename.begin(), filename.end(), ' ', '_' );
 72 |     return filename;
 73 |   };
 74 |   
 75 |   // TRIM STRING. USEFUL FOR PROCESSING STRING FILENAMES
 76 |   // trim from start
 77 |   static inline std::string &ltrim(std::string &s) {
 78 |     s.erase(s.begin(), std::find_if(s.begin(), s.end(), std::not1(std::ptr_fun<int, int>(std::isspace))));
 79 |     return s;
 80 |   };
 81 |   // trim from end
 82 |   static inline std::string &rtrim(std::string &s) {
 83 |     s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun<int, int>(std::isspace))).base(), s.end());
 84 |     return s;
 85 |   };
 86 |   // trim from both ends
 87 |   static inline std::string &trim(std::string &s) {
 88 |     return ltrim(rtrim(s));
 89 |   };
 90 |   
 91 |   
 92 |   static inline char *trim(char *str)
 93 |   {
 94 |     size_t len = 0;
 95 |     char *frontp = str;
 96 |     char *endp = NULL;
 97 |     
 98 |     if( str == NULL ) { return NULL; }
 99 |     if( str[0] == '\0' ) { return str; }
100 |     
101 |     len = strlen(str);
102 |     endp = str + len;
103 |     
104 |     /* Move the front and back pointers to address the first non-whitespace
105 |      * characters from each end.
106 |      */
107 |     while( isspace(*frontp) ) { ++frontp; }
108 |     if( endp != frontp )
109 |     {
110 |       while( isspace(*(--endp)) && endp != frontp ) {}
111 |     }
112 |     
113 |     if( str + len - 1 != endp )
114 |       *(endp + 1) = '\0';
115 |     else if( frontp != str &&  endp == frontp )
116 |       *str = '\0';
117 |     
118 |     /* Shift the string so that it starts at str so that if it's dynamically
119 |      * allocated, we can still free it on the returned pointer.  Note the reuse
120 |      * of endp to mean the front of the string buffer now.
121 |      */
122 |     endp = str;
123 |     if( frontp != str )
124 |     {
125 |       while( *frontp ) { *endp++ = *frontp++; }
126 |       *endp = '\0';
127 |     }
128 |     return str;
129 |   };
130 |   
131 |   
132 |   // CHECK IF FILE EXISTS
133 |   static inline bool file_exists (const std::string& name) {
134 |     if (FILE *file = fopen(name.c_str(), "r")) {
135 |       fclose(file);
136 |       return true;
137 |     } else {
138 |       return false;
139 |     }
140 |   };
141 |   
142 |   // READ TEXT FILE INTO A VECTOR
143 |   static inline void read_textlist( const std::string& in,
144 |                                    std::vector<std::string>& out )
145 |   {
146 |     std::string str;
147 |     std::ifstream file( in );
148 |     while( std::getline( file, str ) ) {
149 |       str = hecate::trim(str);
150 |       if( str.length()==0 ) continue;
151 |       if( str.at(0)=='#' ) continue;
152 |       out.push_back( str );
153 |     }
154 |   };
155 |   
156 |   // READ TEXT FILE CONTAINING A LIST OF FILEPATHS
157 |   static inline void read_filelist( const std::string& in,
158 |                                    std::vector<std::string>& out )
159 |   {
160 |     std::string str;
161 |     std::ifstream file( in );
162 |     while( std::getline( file, str ) ) {
163 |       str = hecate::trim(str);
164 |       if( str.length()==0 ) continue;
165 |       if( str.at(0)=='#' ) continue;
166 |       
167 |       if( hecate::file_exists( str ) )
168 |         out.push_back( str );
169 |       else
170 |         fprintf( stderr, "File doesn't exist: %s\n", str.c_str() );
171 |       
172 |     }
173 |   };
174 |   
175 |   static inline void split_string( char* in,
176 |                                   std::vector<std::string>& out,
177 |                                   const char* delimiter)
178 |   {
179 |     out.clear();
180 |     char *token = in;
181 |     while( (token=strsep(&in,delimiter)) != NULL )
182 |       out.push_back( hecate::trim(token) );
183 |   };
184 |   
185 |   static inline std::string exec(const char* cmd) {
186 |     FILE* pipe = popen(cmd, "r");
187 |     if (!pipe) return "ERROR";
188 |     char buffer[1024];
189 |     std::string result = "";
190 |     while (!feof(pipe)) {
191 |       if (fgets(buffer, 1024, pipe) != NULL)
192 |         result += buffer;
193 |     }
194 |     pclose(pipe);
195 |     return result;
196 |   };
197 |   
198 |   static inline std::string which(const char* bin) {
199 |     char buf[1024];
200 |     sprintf( buf, "echo `which %s`", bin );
201 |     std::string result = hecate::exec(buf);
202 |     result.erase( result.find_last_not_of(" \n\r\t")+1);
203 |     return result;
204 |   }
205 | }
206 | #endif
207 | 
208 | 


--------------------------------------------------------------------------------
/src/hecate/hecate_main.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * HECTATE Video Processing Library - Main
  3 |  *
  4 |  * Copyright 2016 Yahoo Inc.
  5 |  * Licensed under the terms of the Apache 2.0 License.
  6 |  * See LICENSE file in the project root for terms.
  7 |  *
  8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
  9 |  */
 10 | 
 11 | #include "hecate/hecate.hpp"
 12 | 
 13 | using namespace std;
 14 | using namespace cv;
 15 | 
 16 | void run_hecate( hecate_params& opt, vector<int>& v_thumb_idx,
 17 |                  vector<hecate::Range>& v_gif_range,
 18 |                  vector<hecate::Range>& v_mov_range)
 19 | {
 20 |   if( !hecate::file_exists(opt.in_video) ) {
 21 |     fprintf(stderr, "File not exist: %s\n", opt.in_video.c_str());
 22 |     return;
 23 |   }
 24 |   
 25 |   v_thumb_idx.clear();
 26 |   v_gif_range.clear();
 27 |   v_mov_range.clear();
 28 | 
 29 |   hecate::Clock::time_point t0;
 30 |   hecate::VideoParser parser;
 31 |   
 32 |   vector<hecate::ShotRange> v_shot_range;
 33 |   Mat histo_features;
 34 |   Mat diff_features;
 35 |   
 36 |   
 37 |   ////////////////////////////////////////////////////////////////////////////
 38 |   //
 39 |   // Parse video
 40 |   //
 41 |   ////////////////////////////////////////////////////////////////////////////
 42 |   
 43 |   if( opt.debug ) {
 44 |     printf("run_hecate: Video segmentation and keyframe extraction\n");
 45 |     t0 = hecate::Clock::now();
 46 |   }
 47 | 
 48 |   hecate::parser_params parser_opt;
 49 |   parser_opt.step_sz = opt.step_sz;
 50 |   parser_opt.gfl = opt.gfl;
 51 |   parser_opt.fltr_begin_sec = ( opt.fltr_begin_sec<0 )
 52 |       ? max(0.5, 0.05 * parser.meta.duration) : opt.fltr_begin_sec;
 53 |   parser_opt.fltr_end_sec = ( opt.fltr_end_sec<0 )
 54 |       ? max(0.5, 0.10 * parser.meta.duration) : opt.fltr_end_sec;
 55 |   parser_opt.max_duration = opt.max_duration;
 56 |   parser_opt.ignore_rest = (opt.max_duration>0); // ignore parts after max_nfrms
 57 |   parser_opt.debug = opt.debug;
 58 |   
 59 |   // PARSE
 60 |   v_shot_range = parser.parse_video( opt.in_video, parser_opt );
 61 |   if( v_shot_range.empty() ) {
 62 |     fprintf(stderr, "run_hecate: Failed to parse the video\n");
 63 |     return;
 64 |   }
 65 |   
 66 |   histo_features = parser.get_frame_features();
 67 |   diff_features  = parser.get_frame_diff_features();
 68 |   opt.step_sz    = parser.get_effective_step_size();
 69 |   
 70 |   // If video is shorter than desired summary length
 71 |   if( opt.mov && opt.lmov >= parser.meta.duration ) {
 72 |     fprintf( stderr, "run_hecate: Video duration is %.2f seconds, "
 73 |             "shorter than the requested summary of length %.2f seconds.\n"
 74 |             "\tVideo summarization is disabled.",
 75 |             parser.meta.duration, (double)opt.lmov);
 76 |     opt.mov = false;
 77 |   }
 78 |   
 79 |   // Check desired resolution of output
 80 |   if( opt.jpg_width_px<0 || opt.jpg_width_px > parser.meta.width ) {
 81 |     //fprintf( stderr, "run_hecate: Forcing jpg_width_px to %d\n",parser.meta.width);
 82 |     opt.jpg_width_px = parser.meta.width;
 83 |   }
 84 |   if( opt.gif_width_px<0 || opt.gif_width_px > parser.meta.width ) {
 85 |     //fprintf( stderr, "run_hecate: Forcing gif_width_px to %d\n",parser.meta.width);
 86 |     opt.gif_width_px = parser.meta.width;
 87 |   }
 88 |   if( opt.mov_width_px<0 || opt.mov_width_px > parser.meta.width ) {
 89 |     //fprintf( stderr, "run_hecate: Forcing mov_width_px to %d\n",parser.meta.width);
 90 |     opt.mov_width_px = parser.meta.width;
 91 |   }
 92 |   
 93 |   if( opt.debug ) {
 94 |     hecate::print_elapsed_time( t0, "run_hecate" );
 95 |     hecate::print_video_metadata( opt.in_video, parser.meta );
 96 |   }
 97 | 
 98 |   
 99 |   
100 |   ////////////////////////////////////////////////////////////////////////////
101 |   //
102 |   // Analyze video
103 |   //
104 |   ////////////////////////////////////////////////////////////////////////////
105 |   
106 |   // Print shot info
107 |   if( opt.info_shot ) {
108 |     printf("shots: ");
109 |     for(size_t i=0; i<v_shot_range.size(); i++) {
110 |       printf("[%d:%d]", v_shot_range[i].start, v_shot_range[i].end);
111 |       if( i<v_shot_range.size()-1 )
112 |         printf(",");
113 |     }
114 |     printf("\n");
115 |   }
116 |   
117 |   // Print keyframe indices
118 |   if( opt.info_keyfrm ) {
119 |     vector<int> keyfrms;
120 |     for(size_t i=0; i<v_shot_range.size(); i++) {
121 |       for(size_t j=0; j<v_shot_range[i].v_idx.size(); j++) {
122 |         keyfrms.push_back(v_shot_range[i].v_idx[j]);
123 |       }
124 |     }
125 |     
126 |     printf("keyframes: [");
127 |     for(size_t i=0; i<keyfrms.size(); i++) {
128 |       printf("%d", keyfrms[i]);
129 |       if( i<keyfrms.size()-1 )
130 |         printf(",");
131 |     }
132 |     printf("]\n");
133 |   }
134 |   
135 |   // Thumbnail extraction module
136 |   if( opt.jpg ) {
137 |     if( opt.debug ) {
138 |       printf("run_hecate: Video keyframe detection\n");
139 |       t0 = hecate::Clock::now();
140 |     }
141 |     
142 |     detect_thumbnail_frames( opt, parser.meta, v_shot_range,
143 |                              histo_features, diff_features,
144 |                              v_thumb_idx);
145 |     
146 |     if( opt.debug ) {
147 |       hecate::print_elapsed_time( t0, "run_hecate" );
148 |     }
149 |   }
150 |   
151 |   // GIF generation module
152 |   if( opt.gif ) {
153 |     if( opt.debug ) {
154 |       printf("run_hecate: Video highlight detection for GIF creation\n");
155 |       t0 = hecate::Clock::now();
156 |     }
157 |     
158 |     bool mov = opt.mov;
159 |     opt.mov = false;
160 |     detect_highlight_shots( opt, parser.meta, v_shot_range,
161 |                             histo_features, diff_features, v_gif_range );
162 |     opt.mov = mov;
163 |     if( opt.debug ) {
164 |       hecate::print_elapsed_time( t0, "run_hecate" );
165 |     }
166 |   }
167 |   
168 |   // Video summarization module
169 |   if( opt.mov ) {
170 |     if( opt.debug ) {
171 |       printf("run_hecate: Video highlight detection for summarization\n");
172 |       t0 = hecate::Clock::now();
173 |     }
174 |     
175 |     bool gif = opt.gif;
176 |     opt.gif = false;
177 |     detect_highlight_shots( opt, parser.meta, v_shot_range,
178 |                             histo_features, diff_features, v_mov_range );
179 |     opt.gif = gif;
180 |     if( opt.debug ) {
181 |       hecate::print_elapsed_time( t0, "run_hecate" );
182 |     }
183 |   }
184 | }
185 | 
186 | 
187 | 
188 | 
189 | 
190 | 


--------------------------------------------------------------------------------
/src/hecate/hecate_thumbnail.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * HECATE Video Processing Library - Thumbnail
  3 |  *
  4 |  * Copyright 2016 Yahoo Inc.
  5 |  * Licensed under the terms of the Apache 2.0 License.
  6 |  * See LICENSE file in the project root for terms.
  7 |  *
  8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
  9 |  */
 10 | 
 11 | #include "hecate/hecate.hpp"
 12 | 
 13 | using namespace std;
 14 | using namespace cv;
 15 | 
 16 | ////////////////////////////////////////////////////////////////////////////
 17 | //
 18 | // Select most representative frames (thumbnails)
 19 | //
 20 | // Perform k-means++ (k=#thumbnails) over valid frames
 21 | // Thumbnails are selected by sorting clusters by their size and
 22 | //   selecting a frame from the longest subshot
 23 | //
 24 | ////////////////////////////////////////////////////////////////////////////
 25 | 
 26 | void detect_thumbnail_frames( hecate_params& opt, hecate::video_metadata& meta,
 27 |                               const vector<hecate::ShotRange>& v_shot_range,
 28 |                               const Mat& X, const Mat& diff,
 29 |                               vector<int>& v_thumb_idx)
 30 | {
 31 |   v_thumb_idx.clear();
 32 | 
 33 |   const int minK = 5;   // min #clusters
 34 |   const int maxK = 30;  // max #clusters
 35 |   const int nfrm = meta.nframes;
 36 | 
 37 |   vector<bool> v_frm_valid(nfrm,false);
 38 |   for(size_t i=0; i<v_shot_range.size(); i++) {
 39 |     for(size_t j=0; j<v_shot_range[i].v_idx.size(); j++) {
 40 |       v_frm_valid[v_shot_range[i].v_idx[j]] = true;
 41 |     }
 42 |   }
 43 | 
 44 |   int nfrm_valid = accumulate(v_frm_valid.begin(),v_frm_valid.end(),0);
 45 |   if( nfrm_valid <= 1 ) {
 46 |     // If there's no valid frame, pick one most still frame
 47 |     int minidx=-1;
 48 |     double minval=numeric_limits<double>::max();
 49 |     for( int i=0; i<nfrm; i++ ) {
 50 |       double val = diff.at<double>(i);
 51 |       if( val<minval ) {
 52 |         minval = val;
 53 |         minidx = i;
 54 |       }
 55 |     }
 56 |     v_thumb_idx.push_back( minidx );
 57 |   }
 58 |   else if( nfrm_valid <= opt.njpg ) {
 59 |     // If not enough frames are left,
 60 |     // include all remaining keyframes sorted by shot length
 61 |     vector<int> v_shot_len;
 62 |     vector<int> v_keyfrm_idx;
 63 |     for(size_t i=0; i<v_shot_range.size(); i++) {
 64 |       int max_subshot_id=-1;
 65 |       int max_subshot_len=-1;
 66 |       for(size_t j=0; j<v_shot_range[i].v_range.size(); j++) {
 67 |         int shotlen = v_shot_range[i].v_range[j].length();
 68 |         if( shotlen>max_subshot_len ) {
 69 |           max_subshot_len = shotlen;
 70 |           max_subshot_id = j;
 71 |         }
 72 |       }
 73 |       v_shot_len.push_back(max_subshot_len);
 74 |       v_keyfrm_idx.push_back(v_shot_range[i].v_range[max_subshot_id].v_idx[0]);
 75 |     }
 76 | 
 77 |     // Include keyframes sorted by shot length
 78 |     vector<size_t> v_srt_idx;  // contains sorted indices
 79 |     vector<int> v_srt_val;     // contains sorted values
 80 |     hecate::sort( v_shot_len, v_srt_val, v_srt_idx );
 81 |     for(size_t i=0; i<v_srt_idx.size(); i++)
 82 |       v_thumb_idx.push_back( v_keyfrm_idx[v_srt_idx[v_srt_idx.size()-1-i]] );
 83 |   }
 84 |   else {
 85 |     vector<int> v_valid_frm_idx;
 86 |     vector<int> v_valid_frm_shotlen;
 87 |     for(size_t i=0; i<v_shot_range.size(); i++) {
 88 |       for(size_t j=0; j<v_shot_range[i].v_range.size(); j++) {
 89 |         v_valid_frm_idx.push_back( v_shot_range[i].v_range[j].v_idx[0] );
 90 |         v_valid_frm_shotlen.push_back( v_shot_range[i].v_range[j].length() );
 91 |       }
 92 |     }
 93 | 
 94 |     Mat km_data(nfrm_valid, X.cols, X.type());
 95 |     for(size_t i=0; i<v_valid_frm_idx.size(); i++) {
 96 |       X.row(v_valid_frm_idx[i]).copyTo( km_data.row(i) );
 97 |     }
 98 | 
 99 |     // Perform k-means (repeat 5 times)
100 |     Mat km_lbl; // integer row vector; stores cluster IDs for every sample.
101 |     Mat km_ctr; // one row per each cluster center.
102 |     int km_k = min(maxK, min(nfrm_valid, max(minK, opt.njpg)));
103 |     hecate::perform_kmeans( km_data, km_lbl, km_ctr, km_k, 5 );
104 | 
105 |     // For k-means with gap statistics
106 |     //vector<int> Kset;
107 |     //for(int i=km_k; i<=min(nfrm_valid,(int)(2*km_k)); i++)
108 |     //  Kset.push_back( i );
109 |     //km_k = hecate::perform_kmeans_gs( km_data, km_lbl, km_ctr, Kset, 3, 500 );
110 | 
111 |     // measure cluster size
112 |     vector<int> clust_sz(km_k,0);
113 |     for(int i=0; i<km_lbl.rows; i++)
114 |       clust_sz[ km_lbl.at<int>(i) ] += v_valid_frm_shotlen[i];
115 | 
116 |     // sort wrt cluster size in an ascending order
117 |     vector<size_t> v_srt_idx; // contains cluster id
118 |     vector<int> v_srt_val;    // contains cluster size
119 |     hecate::sort( clust_sz, v_srt_val, v_srt_idx );
120 | 
121 |     // obtain thumbnails -- the most still frame per cluster
122 |     for(int i=0; i<km_k; i++) {
123 |       int diff_min_idx = -1;
124 |       double diff_min_val = numeric_limits<double>::max();
125 |       for(int j=0; j<km_lbl.rows; j++) {
126 |         if( km_lbl.at<int>(j) == v_srt_idx[km_k-i-1] ) {
127 |           double mean_diff_j = diff.at<double>(v_valid_frm_idx[j]);
128 |           if( mean_diff_j<diff_min_val ) {
129 |             diff_min_idx = j;
130 |             diff_min_val = mean_diff_j;
131 |           }
132 |         }
133 |       }
134 |       // Convert back to the real index
135 |       v_thumb_idx.push_back( v_valid_frm_idx[diff_min_idx] );
136 |     }
137 |   }
138 | 
139 |   for(size_t i=0; i<v_thumb_idx.size(); i++)
140 |     v_thumb_idx[i] *= opt.step_sz;
141 | }
142 | 
143 | 
144 | 
145 | ////////////////////////////////////////////////////////////////////////////
146 | //
147 | // Thumbnail generation
148 | //
149 | ////////////////////////////////////////////////////////////////////////////
150 | void generate_thumbnails( hecate_params& opt, vector<int>& v_thumb_idx )
151 | {
152 |   char strbuf[256];
153 |   int njpg_cnt = 0;
154 |   int frm_idx = 0;
155 | 
156 |   string filename = hecate::get_filename( std::string(opt.in_video) );
157 | 
158 |   VideoCapture vr( opt.in_video );
159 |   double rsz_ratio = (double)(2+opt.jpg_width_px)/vr.get(CV_CAP_PROP_FRAME_WIDTH);
160 |   while( njpg_cnt < (int)v_thumb_idx.size() )
161 |   {
162 |     Mat frm; vr>>frm;
163 |     if( frm.empty() ) break;
164 | 
165 |     // Check if the current frame is in the selected thumbnail list
166 |     int rank = -1;
167 |     for( size_t i=0; i<v_thumb_idx.size(); i++ ) {
168 |       if( frm_idx==v_thumb_idx[i] ) {
169 |         rank = (int)i;
170 |         break;
171 |       }
172 |     }
173 | 
174 |     // Save that thumbnail
175 |     if( rank>=0 && rank<opt.njpg ) {
176 |       resize( frm, frm, Size(), rsz_ratio, rsz_ratio, CV_INTER_LINEAR );
177 |       frm = frm(Rect(0,0,frm.cols-2,frm.rows));
178 |       sprintf( strbuf, "%s/%s_%02d.jpg",
179 |               opt.out_dir.c_str(), filename.c_str(), rank );
180 |       imwrite( strbuf, frm );
181 |       njpg_cnt++;
182 |     }
183 | 
184 |     if (njpg_cnt >= opt.njpg) {
185 |         break;
186 |     }
187 | 
188 |     frm_idx++;
189 |   }
190 |   vr.release();
191 | }
192 | 
193 | 


--------------------------------------------------------------------------------
/include/hecate/video_parser.hpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Video Parser
  3 |  *
  4 |  * Copyright 2016 Yahoo Inc.
  5 |  * Licensed under the terms of the Apache 2.0 License.
  6 |  * See LICENSE file in the project root for terms.
  7 |  *
  8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
  9 |  */
 10 | 
 11 | #ifndef HECATE_VIDEO_PARSER_HPP
 12 | #define HECATE_VIDEO_PARSER_HPP
 13 | 
 14 | #include <stdio.h>
 15 | #include <stdlib.h>
 16 | #include <dirent.h>
 17 | #include <getopt.h>
 18 | #include <sys/stat.h> // mkdir
 19 | #include <unistd.h>   // access() function
 20 | 
 21 | #include <ctime>      // display date and time
 22 | #include <iostream>   // for standard I/O
 23 | #include <fstream>    // for file I/O
 24 | #include <string>     // for strings
 25 | #include <iomanip>    // for controlling float print precision
 26 | #include <sstream>    // string to number conversion
 27 | #include <chrono>
 28 | #include <limits>
 29 | #include <numeric>
 30 | 
 31 | // OpenMP library
 32 | #if defined(_OPENMP)
 33 | #include <omp.h>
 34 | #endif
 35 | 
 36 | // OpenCV library
 37 | #include <opencv2/opencv.hpp>
 38 | #include <opencv2/core/core.hpp>
 39 | #include <opencv2/imgproc/imgproc.hpp>
 40 | #include <opencv2/highgui/highgui.hpp>
 41 | 
 42 | #if CV_MAJOR_VERSION > 3
 43 | #include <opencv2/videoio/legacy/constants_c.h>
 44 | #endif
 45 | 
 46 | #include "hecate/sort.hpp"
 47 | #include "hecate/gflseg.hpp"
 48 | #include "hecate/gapstat.hpp"
 49 | #include "hecate/shot_range.hpp"
 50 | #include "hecate/hist_opencv.hpp"
 51 | #include "hecate/image_metrics.hpp"
 52 | 
 53 | using namespace std;
 54 | using namespace cv;
 55 | 
 56 | namespace hecate {
 57 |   
 58 |   // video metadata struct
 59 |   struct video_metadata {
 60 |     int nframes;
 61 |     int width;
 62 |     int height;
 63 |     double duration; // in seconds
 64 |     double fps;
 65 |   };
 66 |   
 67 |   struct parser_params {
 68 |     int step_sz;
 69 |     double fltr_begin_sec;
 70 |     double fltr_end_sec;
 71 |     double max_duration; // in seconds
 72 |     bool gfl;      // use group-fused lasso to refine boundaries
 73 |     bool fltr_lq;  // filter low-quality frames
 74 |     bool fltr_rdt; // filter redundant frames
 75 |     bool debug;
 76 |     bool ignore_rest; // if video is too long, ignore the rest (true) or
 77 |                       // adjust step_sz (false). for vidtag this should be
 78 |                       // false, while for hecate this should be true
 79 |     
 80 |     parser_params():
 81 |     step_sz(1),
 82 |     fltr_begin_sec(0),
 83 |     fltr_end_sec(0),
 84 |     max_duration(-1),
 85 |     gfl(false),
 86 |     fltr_lq(true),
 87 |     fltr_rdt(true),
 88 |     debug(false),
 89 |     ignore_rest(false)
 90 |     {};
 91 |   };
 92 |   
 93 |   inline void print_video_metadata( const string filename,
 94 |                                     const hecate::video_metadata m ) {
 95 |     printf("%s\n  seconds=%.2f, nframes=%d, fps=%.2f, resolution=%dx%d\n",
 96 |            filename.c_str(), m.duration, m.nframes, m.fps, m.width, m.height);
 97 |   };
 98 |   
 99 |   class VideoParser {
100 |   public:
101 |     VideoParser();
102 |     
103 |     /* Perform video parsing */
104 |     // max_duration is there to handle too long videos
105 |     //   (100K frms ~= 1 hr long with 30 fps)
106 |     // filter_first and filter_last is there to filter out
107 |     //   a few frames from the beginning and at the end
108 |     //   to manually handle logos, ending credits, etc.
109 |     vector<hecate::ShotRange> parse_video(const string& in_video,
110 |                                        hecate::parser_params opt);
111 |     
112 |     /* Get the number of valid frames */
113 |     int get_nfrm_valid();
114 |     
115 |     /* Display video with shot boundary information */
116 |     void play_video_filtered(const string& in_video,
117 |                              int step_sz=1,
118 |                              int max_frm_len=360);
119 |     
120 |     /* Get a vector of booleans representing valid frames */
121 |     inline void get_frm_valid(vector<bool>& vec) {vec=_v_frm_valid;}
122 |     
123 |     /* Get effective step size */
124 |     inline int get_effective_step_size() {return _step_sz;}
125 |     
126 |     /* Get extracted features */
127 |     inline const Mat get_frame_features() {return _X_feat;}
128 |     inline const Mat get_frame_diff_features() {return _X_diff;}
129 |     
130 |     
131 |   private:
132 |     /* Read video into the memory */
133 |     int read_video(const string& in_video,
134 |                    int step_sz=1,
135 |                    double max_duration=30*60,
136 |                    bool ignore_rest=false,
137 |                    int max_frm_len=160);
138 |     
139 |     /* Filter first end last few frames */
140 |     void filter_heuristic(double fltr_begin_sec=0,
141 |                           double fltr_end_sec=0);
142 |     
143 |     /* Filter out low-quality frames */
144 |     void filter_low_quality(double thrsh_bright=0.075,
145 |                             double thrsh_sharp=0.08,
146 |                             double thrsh_uniform=0.80 );
147 |     
148 |     /* Filter out frames during transition */
149 |     void filter_transition(double thrsh_diff=0.50,
150 |                            double thrsh_ecr=0.10 );
151 |     
152 |     /* Filter out redundant frames */
153 |     void filter_redundant_and_obtain_subshots();
154 |     
155 |     /* Update shot ranges, filter out shots if too short */
156 |     void update_shot_ranges(int min_shot_len=5);
157 | 
158 |     /* Perform post processing. Break up shots if too long */
159 |     void post_process(double min_shot_sec=2.0, bool gfl=false);
160 |     
161 |     /* Perform SBD using heuristics: add next big diff if
162 |      the new shot length is longer than min_shot_len */
163 |     void sbd_heuristic(vector<double> v_diff, vector<int>& jump,
164 |                        int njumps, int min_shot_len );
165 |     
166 |     // Helper functions
167 |     
168 |     /* Extract pyramid of histogram features */
169 |     void extract_histo_features(int pyr_level=2,
170 |                                 bool omit_filtered=true,
171 |                                 int nbin_color_hist=128,
172 |                                 int nbin_edge_ori_hist=8,
173 |                                 int nbin_edge_mag_hist=8);
174 |     
175 |     void mark_invalid( vector<bool>& vec, int idx, int wnd_sz=0 );
176 |     void mark_invalid( vector<bool>& vec, vector<string>& vec2,
177 |                       int idx, const string msg, int wnd_sz=0 );
178 |     
179 |     void release_memory();
180 |     
181 |     
182 |   public:
183 |     hecate::video_metadata meta;
184 |     vector<bool> _v_frm_valid;    // filtered frames
185 |     vector<string> _v_frm_log;    // filtered frames msgs (debug)
186 |     
187 |   private:
188 |     bool _debug;
189 |     bool _display;
190 |     int _step_sz;
191 |     int _nfrm_total;   // number of frames BEFORE sampling
192 |     int _nfrm_given;   // number of frames AFTER sampling
193 |     int _video_width;
194 |     int _video_height;
195 |     double _video_fps;
196 |     double _video_sec;
197 |     
198 |     Mat _X_feat; // frame-wise feature representation
199 |     Mat _X_diff; // n-by-1 frame-by-frame difference first-order derivative
200 |     Mat _X_ecr;  // n-by-1 ecr first-order derivative
201 |     
202 |     vector<Mat> _v_frm_rgb;       // rgb frames
203 |     vector<Mat> _v_frm_gray;      // gray-scale frames
204 |     
205 |     vector<hecate::ShotRange> _v_shot_ranges;
206 |   };
207 | }
208 | 
209 | 
210 | #endif
211 | 


--------------------------------------------------------------------------------
/include/hecate/cc_parser.hpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Video caption parser (VTT and TTML)
  3 |  *
  4 |  * Copyright 2016 Yahoo Inc.
  5 |  * Licensed under the terms of the Apache 2.0 License.
  6 |  * See LICENSE file in the project root for terms.
  7 |  *
  8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
  9 |  */
 10 | 
 11 | #ifndef HECATE_CC_PARSER_HPP
 12 | #define HECATE_CC_PARSER_HPP
 13 | 
 14 | #include <vector>
 15 | #include <string>
 16 | #include <iostream>
 17 | #include <sstream>
 18 | #include <fstream>
 19 | #include <cstring>
 20 | #include <algorithm>
 21 | 
 22 | #include "hecate/time.hpp"
 23 | 
 24 | using namespace std;
 25 | 
 26 | namespace hecate {
 27 |   
 28 |   struct tcc {
 29 |     double start; // milli-seconds
 30 |     double end;   // milli-seconds
 31 |     string text;
 32 |   };
 33 |   
 34 |   inline double time_str2num( const string str_time )
 35 |   {
 36 |     int hh,mm,ss,ms;
 37 |     
 38 |     hh = stoi( str_time.substr(0,2) );
 39 |     mm = stoi( str_time.substr(3,2) );
 40 |     ss = stoi( str_time.substr(6,2) );
 41 |     ms = stoi( str_time.substr(9,3) );
 42 |     
 43 |     return 60*60*hh + 60*mm + ss + (double)ms/1000.0;
 44 |   }
 45 |   
 46 |   
 47 |   inline string truncate_caption( const string src, const int maxlen )
 48 |   {
 49 |     string dst = src;
 50 |     while( (int)dst.length()>maxlen )
 51 |     {
 52 |       size_t pos = dst.find_last_of(" ");
 53 |       dst = dst.substr(0,pos);
 54 |     }
 55 |     return dst;
 56 |   }
 57 |   
 58 |   inline string clean_caption( const string src )
 59 |   {
 60 |     string dst = src;
 61 |     size_t pos, pos2;
 62 |     
 63 |     // Remove special characters
 64 |     char chars[] = ".,;:@#$%^&*\"><_+=~`/";
 65 |     for(unsigned i=0; i<strlen(chars); i++)
 66 |       dst.erase( remove(dst.begin(), dst.end(), chars[i]), dst.end() );
 67 |     
 68 |     // Remove leading & trailing spaces
 69 |     dst.erase(dst.begin(), find_if(dst.begin(), dst.end(), bind1st(not_equal_to<char>(), ' ')));
 70 |     dst.erase(find_if(dst.rbegin(), dst.rend(), bind1st(not_equal_to<char>(), ' ')).base(), dst.end());
 71 |     
 72 |     // Remove leadning words
 73 |     vector<string> lwords = {"And ","But ","Or ","and ","Then ","then "};
 74 |     for(size_t i=0; i<lwords.size(); i++)
 75 |     {
 76 |       pos = dst.find(lwords[i]);
 77 |       if( pos==0 ) {
 78 |         //printf("<<[%s>> [%s] -> [%s]\n", lwords[i].c_str(), dst.c_str(), dst.substr(lwords[i].length(),string::npos).c_str());
 79 |         dst = dst.substr(lwords[i].length(),string::npos);
 80 |       }
 81 |     }
 82 |     
 83 |     // Remove trailing words
 84 |     dst = dst + " ";
 85 |     vector<string> words = {" and "," or "," but "," however "," to "," of "," so ",
 86 |       " because "," who "," when "," where "," what "," how ", " why ",
 87 |       " while "," which "," with "," the "};
 88 |     for(size_t i=0; i<words.size(); i++)
 89 |     {
 90 |       pos = dst.substr(0,dst.find_last_of(" ")).find_last_of(" ");
 91 |       if( pos==string::npos )
 92 |         break;
 93 |       pos2 = dst.substr(pos,string::npos).find(words[i]);
 94 |       if( pos2!=string::npos ) {
 95 |         //printf("<<%s]>> [%s] -> [%s]\n", words[i].c_str(), dst.c_str(), dst.substr(0,pos).c_str());
 96 |         dst = dst.substr(0,pos) + " ";
 97 |       }
 98 |     }
 99 |     
100 |     // Remove annotation inside a bracket, e.g., [UNKNOWN], [LAUGH], [MUSIC], etc.
101 |     pos = dst.find("[");
102 |     while( pos!=string::npos )
103 |     {
104 |       pos2 = dst.find("]");
105 |       dst = dst.substr(0,pos) + dst.substr(pos2+1,string::npos);
106 |       pos = dst.find("[");
107 |     }
108 |     
109 |     // Remove double ??
110 |     pos = dst.find("??");
111 |     while( pos!=string::npos )
112 |     {
113 |       dst = dst.substr(0,pos) + dst.substr(pos+2,string::npos);
114 |       pos = dst.find("??");
115 |     }
116 |     
117 |     // Remove double space
118 |     pos = dst.find("  ");
119 |     while( pos!=string::npos )
120 |     {
121 |       dst = dst.substr(0,pos) + dst.substr(pos+1,string::npos);
122 |       pos = dst.find("  ");
123 |     }
124 |     
125 |     // Remove leading & trailing spaces
126 |     dst.erase(dst.begin(), find_if(dst.begin(), dst.end(), bind1st(not_equal_to<char>(), ' ')));
127 |     dst.erase(find_if(dst.rbegin(), dst.rend(), bind1st(not_equal_to<char>(), ' ')).base(), dst.end());
128 |     
129 |     
130 |     return dst;
131 |   }
132 |   
133 |   
134 |   
135 |   inline void print_closed_caption( vector<tcc>& cc )
136 |   {
137 |     for(size_t i=0; i<cc.size(); i++ )
138 |       printf("%.2f  -->  %.2f: [%s]\n",cc[i].start,cc[i].end,cc[i].text.c_str());
139 |   }
140 |   
141 |   
142 |   
143 |   inline void parse_vtt( const string filename, vector<tcc>& vtt )
144 |   {
145 |     string line;
146 |     
147 |     ifstream fid;
148 |     fid.open( filename );
149 |     getline( fid, line );
150 |     while( fid )
151 |     {
152 |       size_t pos = line.find(" --> ");
153 |       if( pos!=string::npos )
154 |       {
155 |         tcc cc;
156 |         
157 |         // time
158 |         cc.start = time_str2num( line.substr( 0, pos ) );
159 |         cc.end = time_str2num( line.substr( pos+5, string::npos) );
160 |         
161 |         // text
162 |         getline( fid, line );
163 |         line = clean_caption( line );
164 |         
165 |         pos = line.find("???");
166 |         if( pos==string::npos && line.length()>3 ) {
167 |           cc.text = line;
168 |           vtt.push_back( cc );
169 |         }
170 |       }
171 |       getline( fid, line );
172 |     }
173 |   }
174 |   
175 |   inline void parse_ttml( const string filename, vector<tcc>& ttml )
176 |   {
177 |     size_t pos1, pos2, pos3;
178 |     string line, item;
179 |     
180 |     string p_start = "<p ";
181 |     string p_end   = "</p>";
182 |     
183 |     ifstream fid;
184 |     fid.open( filename );
185 |     getline( fid, line );
186 |     
187 |     // Check the language
188 |     pos1 = line.find("<language>en-US</language>");
189 |     if( pos1==string::npos )
190 |       return;
191 |     
192 |     // Parse TTML
193 |     while( fid )
194 |     {
195 |       pos1 = line.find(p_start);
196 |       while( pos1!=string::npos )
197 |       {
198 |         pos2 = line.find(p_end);
199 |         item = line.substr(pos1, pos2-pos1);
200 |         line = line.substr( pos2+3, string::npos );
201 |         
202 |         pos1 = item.find("begin=");
203 |         pos2 = item.find("end=");
204 |         pos3 = item.find("\">");
205 |         
206 |         tcc cc;
207 |         cc.start = time_str2num( item.substr(pos1+7,12) );
208 |         cc.end   = time_str2num( item.substr(pos2+5,12) );
209 |         cc.text  = clean_caption( item.substr(pos3+2,string::npos) );
210 |         ttml.push_back( cc );
211 |         
212 |         pos1 = line.find(p_start);
213 |       }
214 |       getline( fid, line );
215 |     }
216 |     
217 |     //print_closed_caption( ttml );
218 |   }
219 |   
220 |   //
221 |   inline string encode_vtt( int index, float start_sec, float end_sec,
222 |                            vector<string>& v_msg )
223 |   {
224 |     char buf[512];
225 |     string start_sec_str = hecate::second2string( start_sec, "hh:mm:ss.mss" );
226 |     string end_sec_str = hecate::second2string( end_sec, "hh:mm:ss.mss" );
227 |     sprintf( buf, "%d\n%s --> %s\n", index, start_sec_str.c_str(),
228 |             end_sec_str.c_str());
229 | 
230 |     for(size_t i=0; i<v_msg.size(); i++) {
231 |       sprintf( buf, "%s- %s\n", buf, v_msg[i].c_str());
232 |     }
233 |     
234 |     string ret = buf;
235 |     return ret;
236 |   }
237 |   
238 | }
239 | #endif
240 | 
241 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
  1 | PROJECT := hecate
  2 | 
  3 | CONFIG_FILE := Makefile.config
  4 | 
  5 | # Explicitly check for the config file, otherwise make -k will proceed anyway.
  6 | ifeq ($(wildcard $(CONFIG_FILE)),)
  7 | 	$(error $(CONFIG_FILE) not found. See $(CONFIG_FILE).example.)
  8 | endif
  9 | include $(CONFIG_FILE)
 10 | 
 11 | BUILD_DIR_LINK := $(BUILD_DIR)
 12 | ifeq ($(RELEASE_BUILD_DIR),)
 13 | 	RELEASE_BUILD_DIR := .$(BUILD_DIR)_release
 14 | endif
 15 | ifeq ($(DEBUG_BUILD_DIR),)
 16 | 	DEBUG_BUILD_DIR := .$(BUILD_DIR)_debug
 17 | endif
 18 | 
 19 | DEBUG ?= 0
 20 | ifeq ($(DEBUG), 1)
 21 | 	BUILD_DIR := $(DEBUG_BUILD_DIR)
 22 | 	OTHER_BUILD_DIR := $(RELEASE_BUILD_DIR)
 23 | else
 24 | 	BUILD_DIR := $(RELEASE_BUILD_DIR)
 25 | 	OTHER_BUILD_DIR := $(DEBUG_BUILD_DIR)
 26 | endif
 27 | 
 28 | # All of the directories containing code.
 29 | SRC_DIRS := $(shell find * -type d -exec bash -c "find {} -maxdepth 1 \
 30 | 	\( -name '*.cpp' \) | grep -q ." \; -print)
 31 | 
 32 | # The target shared library name
 33 | LIB_BUILD_DIR := $(BUILD_DIR)/lib
 34 | STATIC_NAME := $(LIB_BUILD_DIR)/lib$(PROJECT).a
 35 | DYNAMIC_NAME := $(LIB_BUILD_DIR)/lib$(PROJECT).so
 36 | 
 37 | ##############################
 38 | # Get all source files
 39 | ##############################
 40 | # CXX_SRCS are the source files
 41 | ifeq ($(BUILD_TEXT), 1)
 42 |   CXX_SRCS := $(shell find src/$(PROJECT) -name "*.cpp")
 43 | else
 44 |   CXX_SRCS := $(filter-out src/$(PROJECT)/text.cpp, $(shell find src/$(PROJECT) -name "*.cpp"))
 45 | endif
 46 | 
 47 | # TOOL_SRCS are the source files for the tool binaries
 48 | ifeq ($(BUILD_TEXT), 1)
 49 |   TOOL_SRCS := $(shell find src/tools -name "*.cpp")
 50 | else
 51 |   TOOL_SRCS := $(filter-out src/tools/aesthetic_feature_extractor.cpp, $(shell find src/tools -name "*.cpp"))
 52 | endif
 53 | 
 54 | # BUILD_INCLUDE_DIR contains any generated header files we want to include.
 55 | BUILD_INCLUDE_DIR := $(BUILD_DIR)/src
 56 | 
 57 | 
 58 | ##############################
 59 | # Derive generated files
 60 | ##############################
 61 | # The generated files for protocol buffers
 62 | # The objects corresponding to the source files
 63 | # These objects will be linked into the final shared library, so we
 64 | # exclude the tool and test objects.
 65 | CXX_OBJS := $(addprefix $(BUILD_DIR)/, ${CXX_SRCS:.cpp=.o})
 66 | OBJS := $(CXX_OBJS)
 67 | 
 68 | # tool objects
 69 | TOOL_OBJS := $(addprefix $(BUILD_DIR)/, ${TOOL_SRCS:.cpp=.o})
 70 | TOOL_BUILD_DIR := $(BUILD_DIR)/tools
 71 | 
 72 | # Output files for automatic dependency generation
 73 | DEPS := ${CXX_OBJS:.o=.d}
 74 | 
 75 | # tool bins
 76 | TOOL_BINS := ${TOOL_OBJS:.o=}
 77 | 
 78 | 
 79 | ##############################
 80 | # Derive compiler warning dump locations
 81 | ##############################
 82 | WARNS_EXT := warnings.txt
 83 | CXX_WARNS := $(addprefix $(BUILD_DIR)/, ${CXX_SRCS:.cpp=.o.$(WARNS_EXT)})
 84 | TOOL_WARNS := $(addprefix $(BUILD_DIR)/, ${TOOL_SRCS:.cpp=.o.$(WARNS_EXT)})
 85 | ALL_CXX_WARNS := $(CXX_WARNS) $(TOOL_WARNS)
 86 | ALL_WARNS := $(ALL_CXX_WARNS)
 87 | NONEMPTY_WARN_REPORT := $(BUILD_DIR)/$(WARNS_EXT)
 88 | 
 89 | 
 90 | ##############################
 91 | # Derive include and lib directories
 92 | ##############################
 93 | INCLUDE_DIRS += $(BUILD_INCLUDE_DIR) ./include
 94 | 
 95 | LIBRARIES += opencv_core opencv_highgui opencv_imgproc
 96 | ifeq ($(OPENCV_VERSION), 3)
 97 | 	LIBRARIES += opencv_imgcodecs opencv_videoio
 98 | endif
 99 | 
100 | WARNINGS := -Wall -Wno-sign-compare
101 | 
102 | 
103 | ##############################
104 | # Set build directories
105 | ##############################
106 | 
107 | DISTRIBUTE_DIR ?= distribute
108 | DISTRIBUTE_SUBDIRS := $(DISTRIBUTE_DIR)/bin $(DISTRIBUTE_DIR)/lib
109 | DIST_ALIASES := dist
110 | ifneq ($(strip $(DISTRIBUTE_DIR)),distribute)
111 | 		DIST_ALIASES += distribute
112 | endif
113 | 
114 | ALL_BUILD_DIRS := $(sort $(BUILD_DIR) $(addprefix $(BUILD_DIR)/, \
115 | $(SRC_DIRS)) $(LIB_BUILD_DIR) $(DISTRIBUTE_SUBDIRS))
116 | 
117 | 
118 | ##############################
119 | # Configure build
120 | ##############################
121 | 
122 | # Determine platform
123 | UNAME := $(shell uname -s)
124 | ifeq ($(UNAME), Linux)
125 | 	LINUX := 1
126 | else ifeq ($(UNAME), Darwin)
127 | 	OSX := 1
128 | endif
129 | 
130 | # Linux
131 | ifeq ($(LINUX), 1)
132 | 	CXX ?= /usr/bin/g++
133 | 	GCCVERSION := $(shell $(CXX) -dumpversion | cut -f1,2 -d.)
134 | 	# older versions of gcc are too dumb to build boost with -Wuninitalized
135 | 	ifeq ($(shell echo | awk '{exit $(GCCVERSION) < 4.6;}'), 1)
136 | 		WARNINGS += -Wno-uninitialized
137 | 	endif
138 | 	# We will explicitly add stdc++ to the link target.
139 | 	LIBRARIES += stdc++
140 | 	COMMON_FLAGS += -fopenmp
141 | endif
142 | 
143 | # OS X:
144 | # clang++ instead of g++
145 | ifeq ($(OSX), 1)
146 | 	CXX := /usr/bin/clang++
147 | 	# we need to explicitly ask for the rpath to be obeyed
148 | 	DYNAMIC_FLAGS := -install_name @rpath/lib$(PROJECT).so
149 | 	ORIGIN := @loader_path
150 | else
151 | 	ORIGIN := \$$ORIGIN
152 | endif
153 | 
154 | # Custom compiler
155 | ifdef CUSTOM_CXX
156 | 	CXX := $(CUSTOM_CXX)
157 | endif
158 | 
159 | # Debugging
160 | ifeq ($(DEBUG), 1)
161 | 	COMMON_FLAGS += -std=c++0x -DDEBUG -g -O0
162 | else
163 | 	COMMON_FLAGS += -std=c++0x -DNDEBUG -O2
164 | endif
165 | 
166 | LIBRARY_DIRS += $(LIB_BUILD_DIR)
167 | 
168 | # Automatic dependency generation
169 | CXXFLAGS += -MMD -MP
170 | 
171 | # Complete build flags.
172 | COMMON_FLAGS += $(foreach includedir,$(INCLUDE_DIRS),-I$(includedir))
173 | CXXFLAGS += -pthread -fPIC $(COMMON_FLAGS) $(WARNINGS)
174 | LINKFLAGS += -pthread -fPIC $(COMMON_FLAGS) $(WARNINGS)
175 | 
176 | LDFLAGS += $(foreach librarydir,$(LIBRARY_DIRS),-L$(librarydir)) $(PKG_CONFIG) \
177 | 		$(foreach library,$(LIBRARIES),-l$(library))
178 | 
179 | 
180 | ##############################
181 | # Define build targets
182 | ##############################
183 | .PHONY: all lib tools clean $(DIST_ALIASES)
184 | 
185 | all: lib tools
186 | 
187 | lib: $(STATIC_NAME) $(DYNAMIC_NAME)
188 | 
189 | tools: $(TOOL_BINS)
190 | 
191 | $(ALL_WARNS): %.o.$(WARNS_EXT) : %.o
192 | 
193 | $(BUILD_DIR_LINK): $(BUILD_DIR)/.linked
194 | 
195 | # Create a target ".linked" in this BUILD_DIR to tell Make that the "build" link
196 | # is currently correct, then delete the one in the OTHER_BUILD_DIR in case it
197 | # exists and $(DEBUG) is toggled later.
198 | $(BUILD_DIR)/.linked:
199 | 	@ mkdir -p $(BUILD_DIR)
200 | 	@ $(RM) $(OTHER_BUILD_DIR)/.linked
201 | 	@ $(RM) -r $(BUILD_DIR_LINK)
202 | 	@ ln -s $(BUILD_DIR) $(BUILD_DIR_LINK)
203 | 	@ touch $@
204 | 
205 | $(ALL_BUILD_DIRS): | $(BUILD_DIR_LINK)
206 | 	@ mkdir -p $@
207 | 
208 | $(DYNAMIC_NAME): $(OBJS) | $(LIB_BUILD_DIR)
209 | 	@ echo LD -o $@
210 | 	$(Q)$(CXX) -shared -o $@ $(OBJS) $(LINKFLAGS) $(LDFLAGS) $(DYNAMIC_FLAGS)
211 | 
212 | $(STATIC_NAME): $(OBJS) | $(LIB_BUILD_DIR)
213 | 	@ echo AR -o $@
214 | 	$(Q)ar rcs $@ $(OBJS)
215 | 
216 | $(BUILD_DIR)/%.o: %.cpp | $(ALL_BUILD_DIRS)
217 | 	@ echo CXX $<
218 | 	$(Q)$(CXX) $< $(CXXFLAGS) -c -o $@ 2> $@.$(WARNS_EXT) \
219 | 		|| (cat $@.$(WARNS_EXT); exit 1)
220 | 	@ cat $@.$(WARNS_EXT)
221 | 
222 | # Target for extension-less tool binaries
223 | $(TOOL_BUILD_DIR)/%: $(TOOL_BUILD_DIR)/% | $(TOOL_BUILD_DIR)
224 | 	@ $(RM) $@
225 | 	@ ln -s $(abspath $<) $@
226 | 
227 | $(TOOL_BINS): % : %.o | $(DYNAMIC_NAME)
228 | 	@ echo CXX/LD -o $@
229 | 	$(Q)$(CXX) $< -o $@ $(LINKFLAGS) $(LDFLAGS) -l$(PROJECT)\
230 | 		-Wl,-rpath,$(ORIGIN)/../lib
231 | 
232 | clean:
233 | 	@- $(RM) -rf $(ALL_BUILD_DIRS)
234 | 	@- $(RM) -rf $(OTHER_BUILD_DIR)
235 | 	@- $(RM) -rf $(BUILD_DIR_LINK)
236 | 	@- $(RM) -rf $(DISTRIBUTE_DIR)
237 | 
238 | $(DIST_ALIASES): $(DISTRIBUTE_DIR)
239 | 
240 | $(DISTRIBUTE_DIR): all | $(DISTRIBUTE_SUBDIRS)
241 | 	# add include
242 | 	cp -r include $(DISTRIBUTE_DIR)/
243 | 	# add tool binaries
244 | 	cp $(TOOL_BINS) $(DISTRIBUTE_DIR)/bin
245 | 	# add libraries
246 | 	cp $(STATIC_NAME) $(DISTRIBUTE_DIR)/lib
247 | 	install -m 644 $(DYNAMIC_NAME) $(DISTRIBUTE_DIR)/lib
248 | 
249 | -include $(DEPS)
250 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # HECATE
  2 | Hecate [hek-uh-tee] is a video processing library that auto-magically generates thumbnails, animated GIFs, and video summaries from videos. This library is developed and maintained by Yahoo Research, New York.
  3 | 
  4 | The source code is Copyright 2016 Yahoo Inc. and is licensed under the terms of the Apache 2.0 License. See the [LICENSE](https://github.com/yahoo/hecate/blob/master/LICENSE) in the project root file for terms.
  5 | 
  6 | The technology behind this library is based on our research work. If you find this library useful in your work, we ask you to cite our research paper:
  7 | ```
  8 | "To Click or Not To Click: Automatic Selection of Beautiful Thumbnails from Videos."
  9 | Yale Song, Miriam Redi, Jordi Vallmitjana, Alejandro Jaimes, 
 10 | Proceedings of the 25th ACM International on Conference on Information and Knowledge Management, CIKM 2016
 11 | ```
 12 | 
 13 | ## Installation
 14 | Hecate has one dependency: [OpenCV library](https://github.com/opencv/opencv) with an [FFMPEG](https://github.com/FFmpeg/FFmpeg) support. You will need to install the library properly before trying out Hecate!
 15 | 
 16 | Once you install the dependenct library correctly, follow the instruction below:
 17 | ```
 18 | $ git clone https://github.com/yahoo/hecate.git
 19 | $ cd hecate
 20 | $ vim Makefile.config
 21 |  - Set INCLUDE_DIRS and LIBRARY_DIRS to where your 
 22 |    opencv library is installed. Usually under /usr/local.
 23 |  - If your OpenCV version is 2.4.x, comment out the line 
 24 |    OPENCV_VERSION := 3
 25 |  - Save and exit
 26 | $ make all
 27 | $ make distribute
 28 | ```
 29 | 
 30 | Once you've successfully compiled hecate, it will generate a binary executable under `distribute/bin/`. Run the following command to check if everything works properly:
 31 | ```
 32 | $ ./distribute/bin/hecate
 33 | 
 34 | -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
 35 |  HECATE Copyright 2016 Yahoo Inc.
 36 |    Licensed under the terms of the Apache 2.0 License.
 37 |    Developed by : Yale Song (yalesong@yahoo-inc.com)
 38 |    Built on  : 11:46:03 Aug 11 2016
 39 | -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
 40 | USAGE: hecate -i infile [options]
 41 | 
 42 |   -i  --in_video      (string)    Input video file
 43 |   -o  --out_dir       (string)    Output directory (./output)
 44 |   -s  --step          (int)       Frame subsampling step size (1)
 45 |   -n  --njpg          (int)       Number of thumbnails to be generated (5)
 46 |   -q  --ngif          (int)       Number of GIFs to be generated (5)
 47 |   -r  --lmov          (int)       Length of video summary to be generated (in seconds) (15)
 48 |   -u  --jpg_width_px  (int)       Pixel width of thumbnail images (360)
 49 |   -v  --gif_width_px  (int)       Pixel width of animated GIFs (360)
 50 |   -w  --mov_width_px  (int)       Pixel width of summary video (360)
 51 |   --generate_jpg                  Generate thumbnail images
 52 |   --generate_gif                  Generate animated GIFs
 53 |   --generate_mov                  Generate a summary video
 54 |   --generate_gifsum               Generate animated GIFs summary
 55 |   --generate_gifall               Generate all possible animated GIFs
 56 |   --print_shot_info               Print shot boundary detection results
 57 |   --print_keyfrm_info             Print keyframe indices
 58 | ```
 59 | 
 60 | Congratulations! You have successfully installed hecate!
 61 | 
 62 | 
 63 | ## Get started
 64 | In order to get started, we will need a video file to play with. In this example, we will use the video ["The Spirit of '43" by Walt Disney](https://archive.org/details/TheSpiritOf43_56) from [The Internet Archive](https://archive.org). 
 65 | 
 66 | Let's download the video and save it as `examples/video.mp4`:
 67 | ```
 68 | $ wget https://archive.org/download/TheSpiritOf43_56/The_Spirit_of__43_512kb.mp4 \
 69 |   --output-document examples/video.mp4 --no-check-certificate
 70 | ```
 71 | 
 72 | Hecate provides three main functionalities through a binary executable `hecate`: Thumbnail extraction, GIF generation, and video summarization. There are various other functionalities the library provides, such as shot boundary detection and keyframe extraction. 
 73 | 
 74 | We will explain each case below.
 75 | 
 76 | ### Shot boundary detection and keyframe extraction
 77 | Shot boundary detection and keyframe extraction are often the first steps towards various video processing methods. With Hecate, obtaining shot and keyframe information is easier than ever! Simply run the following command to get the result:
 78 | ```
 79 | $ ./distribute/bin/hecate -i examples/video.mp4 --print_shot_info  --print_keyfrm_info
 80 | ```
 81 | 
 82 | Below is the results we obtained on our dev machine (OS X 10.10 with OpenCV v3.1):
 83 | ```
 84 | shots: [0:81],[84:93],[96:102],[108:270],[272:418],...,[9966:10131],[10135:10164]
 85 | keyframes: [52,85,98,128,165,208,242,259,265,273,...,10127,10141]
 86 | ```
 87 | The units are frame indices (zero-based). You will notice that shot ranges are non-continuous; there are "gaps" between shots, e.g., two frames are missing between the first two shots [0:81] and [84:93]. This is normal and intentional: Hecate discards low-quality frames that aren't ideal in producing nicely looking thumbnails, animated GIFs, and video summaries. We refer to our CIKM 2016 paper for the rational behind our reason to invalidate low-quality frames.
 88 | 
 89 | ### Thumbnail generation
 90 | Hecate uses computer vision to determine frames that are most "suitable" as video thumbnails. By suitable, we mean a frame that is the most relevant to the video content and that is the most beautiful in terms of computational aesthetics; technical details are explained in our CIKM 2016 paper.
 91 | 
 92 | You can generate thumbnail images using Hecate. Run the following command to generate one thumbnail image from the video.
 93 | ```
 94 | $ ./distribute/bin/hecate -i examples/video.mp4 --generate_jpg --njpg 1
 95 | ```
 96 | You will see the generated thumbnail image under the output directory (set as `output` by default; you can change this using the option `--out_dir YOUR_DIRECTORY`). On our dev machine we get this thumbnail image:
 97 | 
 98 | ![alt text](https://github.com/yahoo/hecate/blob/master/examples/video_00.jpg "Hecate Thumbnail Image")
 99 | 
100 | In the above example, we generated only one thumbnail image. Are you not satisfied with the thumbnail image? Hecate can generate any number of thunbmail images! Let's generate five thumbnail images.
101 | ```
102 | $ ./distribute/bin/hecate -i examples/video.mp4 --generate_jpg --njpg 3
103 | ```
104 | 
105 | The output files are named `<video_filename>_<rank>.jpg`. The files are ranked by their quality (rank 0 means it's the best one).
106 | 
107 | ### Animated GIF generation
108 | Do you want to create animated GIFs from a video without the hassle of using manual tools? Hecate can automatically create them for you! Run the following command to create one animated GIF from the video.
109 | ```
110 | $ ./distribute/bin/hecate -i examples/video.mp4 --generate_gif --ngif 1
111 | ```
112 | On our dev machine, we get this animated GIF:
113 | 
114 | ![alt text](https://github.com/yahoo/hecate/blob/master/examples/video_00.gif "Hecate Animated GIF")
115 | 
116 | You can, of course, create more than just one GIF by setting the paramter `--ngif N` with an appropriate number N. When there are multiple GIFs, you can also generate a "summary GIF" by concatenating them, using this command:
117 | ```
118 | $ ./distribute/bin/hecate -i examples/video.mp4 --generate_gif --ngif 3 --generate_gifsum
119 | ```
120 | 
121 | If you'd rather want to obtain all available GIFs from the video, use the following command: 
122 | ```
123 | $ ./distribute/bin/hecate -i examples/video.mp4 --generate_gifall
124 | ```
125 | 
126 | ### Video summary generation
127 | Last but not least, Hecate can summarize a video! Run the following command to create a video summary of length 15 seconds.
128 | ```
129 | $ ./distribute/bin/hecate -i examples/video.mp4 --generate_mov --lmov 15
130 | ```
131 | We included the video summary generated on our dev machine here: 
132 | [https://github.com/yahoo/hecate/blob/master/examples/video_sum.mp4](https://github.com/yahoo/hecate/blob/master/examples/video_sum.mp4)
133 | 
134 | 
135 | ## Developer
136 | 
137 | Yale Song: [github](https://github.com/yalesong), [website](http://people.csail.mit.edu/yalesong)
138 | 


--------------------------------------------------------------------------------
/include/hecate/hist_opencv.hpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Compute histogram of [x] using OpenCV
  3 |  *
  4 |  * Copyright 2016 Yahoo Inc.
  5 |  * Licensed under the terms of the Apache 2.0 License.
  6 |  * See LICENSE file in the project root for terms.
  7 |  *
  8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
  9 |  */
 10 | 
 11 | #ifndef HECATE_HIST_OPENCV_HPP
 12 | #define HECATE_HIST_OPENCV_HPP
 13 | 
 14 | #include <opencv2/imgproc/imgproc.hpp>
 15 | #include <opencv2/highgui/highgui.hpp>
 16 | 
 17 | #if CV_MAJOR_VERSION > 2
 18 | #include <opencv2/imgproc/types_c.h>
 19 | #endif
 20 | 
 21 | using namespace cv;
 22 | using namespace std;
 23 | 
 24 | namespace hecate {
 25 |   
 26 |   
 27 |   inline void print( const Mat x, const string filename )
 28 |   {
 29 |     ofstream myfile;
 30 |     myfile.open( filename );
 31 |     myfile << x;
 32 |     myfile.close();
 33 |   }
 34 |   
 35 |   inline void calc_gray_hist( const Mat& img, Mat& hist, int nbins=128 )
 36 |   {
 37 |     // Gray histogram
 38 |     float _range_val_gry[] = {0,256};
 39 |     const float* _range_gry = {_range_val_gry};
 40 |     
 41 |     calcHist( &img, 1, 0, Mat(), hist, 1, &nbins, &_range_gry, true, false);
 42 |     normalize( hist, hist );
 43 |   }
 44 |   
 45 |   inline void calc_color_hist( const Mat& img, Mat& hist, int nbins=128 )
 46 |   {
 47 |     // Color histogram
 48 |     float _range_val_clr[] = {0,256};
 49 |     const float* _range_clr = {_range_val_clr};
 50 |     
 51 |     // CV_BGR2HSV, CV_BGR2HLS, CV_BGR2YCrCb
 52 |     Mat img_cvt;
 53 |     cvtColor( img, img_cvt, CV_BGR2HSV );
 54 |     
 55 |     vector<Mat> planes;
 56 |     split( img_cvt, planes );
 57 |     
 58 |     if( !hist.data )
 59 |       hist = Mat( 3*nbins, 1, CV_32F, Scalar(0,0,0)[0] );
 60 |     
 61 |     Mat hist0( hist, Rect(0,0*nbins,1,nbins) );
 62 |     Mat hist1( hist, Rect(0,1*nbins,1,nbins) );
 63 |     Mat hist2( hist, Rect(0,2*nbins,1,nbins) );
 64 |     
 65 |     calcHist( &planes[0], 1, 0, Mat(), hist0, 1, &nbins, &_range_clr, true, false);
 66 |     calcHist( &planes[1], 1, 0, Mat(), hist1, 1, &nbins, &_range_clr, true, false);
 67 |     calcHist( &planes[2], 1, 0, Mat(), hist2, 1, &nbins, &_range_clr, true, false);
 68 |     
 69 |     normalize( hist0, hist0 );
 70 |     normalize( hist1, hist1 );
 71 |     normalize( hist2, hist2 );
 72 |   }
 73 |   
 74 |   
 75 |   inline void orientation( const Mat& Gx, const Mat& Gy, Mat& ori )
 76 |   {
 77 |     double alpha = 180.0 / 3.14159265358979323846264338327950288419716939937510;
 78 |     
 79 |     ori = Mat( Gx.rows, Gx.cols, CV_32F, Scalar(0,0,0)[0] );
 80 |     for(int r=0; r<Gx.rows; r++) {
 81 |       for(int c=0; c<Gx.cols; c++) {
 82 |         double deg = atan2(Gy.at<float>(r,c), Gx.at<float>(r,c)) * alpha;
 83 |         deg = (deg>=0) ? deg : deg + 360.0;
 84 |         deg = (deg<180.0) ? deg : deg - 180.0;
 85 |         ori.at<float>(r,c) = deg;
 86 |       }
 87 |     }
 88 |   }
 89 |   
 90 |   
 91 |   inline void calc_edge_hist( const Mat& Gx, const Mat& Gy, Mat& hist, int nbins_ori=16, int nbins_mag=16 )
 92 |   {
 93 |     // Edge histogram
 94 |     float _range_val_ori[] = {0,180};
 95 |     float _range_val_mag[] = {0,256};
 96 |     const float* _range_ori = {_range_val_ori};
 97 |     const float* _range_mag = {_range_val_mag};
 98 |     
 99 |     Mat ori, mag;
100 |     orientation( Gx, Gy, ori );
101 |     cv::magnitude( Gx, Gy, mag );
102 |     
103 |     if( !hist.data )
104 |       hist = Mat( nbins_ori+nbins_mag, 1, CV_32F, Scalar(0,0,0)[0] );
105 |     
106 |     Mat hist_ori( hist, Rect(0,0,1,nbins_ori) );
107 |     Mat hist_mag( hist, Rect(0,nbins_ori,1,nbins_mag) );
108 |     
109 |     calcHist( &ori, 1, 0, Mat(), hist_ori, 1, &nbins_ori, &_range_ori, true, false );
110 |     calcHist( &mag, 1, 0, Mat(), hist_mag, 1, &nbins_mag, &_range_mag, true, false );
111 |     
112 |     normalize( hist_ori, hist_ori );
113 |     normalize( hist_mag, hist_mag );
114 |   }
115 |   
116 |   inline void calc_edge_hist( const Mat& img_gray, Mat& hist, int nbins_ori=16, int nbins_mag=16 )
117 |   {
118 |     Mat Gx, Gy;
119 |     Scharr( img_gray, Gx, CV_32F, 1, 0 ); // ddepth, dx, dy
120 |     Scharr( img_gray, Gy, CV_32F, 0, 1 );
121 |     
122 |     calc_edge_hist( Gx, Gy, hist, nbins_ori, nbins_mag );
123 |   }
124 |   
125 |   
126 |   inline void calc_pyr_gray_hist( const Mat& img, Mat& hist, int nbins=128, int level=2)
127 |   {
128 |     int w = img.cols;
129 |     int h = img.rows;
130 |     
131 |     int npatches = 0;
132 |     for(int i=0; i<level; i++)
133 |       npatches += pow(4,i);
134 |     
135 |     int hist_sz = nbins;
136 |     
137 |     if( !hist.data )
138 |       hist = Mat( hist_sz*npatches, 1, CV_32F, Scalar(0,0,0)[0] );
139 |     
140 |     int patch = 0;
141 |     for(int l=0; l<level; l++) {
142 |       for(int x=0; x<pow(2,l); x++) {
143 |         for(int y=0; y<pow(2,l); y++) {
144 |           int p_width  = floor( w/pow(2,l) );
145 |           int p_height = floor( h/pow(2,l) );
146 |           int p_x = x*p_width;
147 |           int p_y = y*p_height;
148 |           Mat patch_img( img, Rect(p_x,p_y,p_width,p_height) );
149 |           Mat patch_hist( hist, Rect(0,patch*hist_sz,1,hist_sz) );
150 |           calc_gray_hist( patch_img, patch_hist, nbins );
151 |           patch++;
152 |         }
153 |       }
154 |     }
155 |   }
156 |   
157 |   
158 |   inline void calc_pyr_color_hist( const Mat& img, Mat& hist, int nbins=128, int level=2)
159 |   {
160 |     int w = img.cols;
161 |     int h = img.rows;
162 |     
163 |     int npatches = 0;
164 |     for(int i=0; i<level; i++)
165 |       npatches += pow(4,i);
166 |     
167 |     int hist_sz = 3 * nbins;
168 |     
169 |     if( !hist.data )
170 |       hist = Mat( hist_sz*npatches, 1, CV_32F, Scalar(0,0,0)[0] );
171 |     
172 |     int patch = 0;
173 |     for(int l=0; l<level; l++) {
174 |       for(int x=0; x<pow(2,l); x++) {
175 |         for(int y=0; y<pow(2,l); y++) {
176 |           int p_width  = floor( w/pow(2,l) );
177 |           int p_height = floor( h/pow(2,l) );
178 |           int p_x = x*p_width;
179 |           int p_y = y*p_height;
180 |           Mat patch_img( img, Rect(p_x,p_y,p_width,p_height) );
181 |           Mat patch_hist( hist, Rect(0,patch*hist_sz,1,hist_sz) );
182 |           calc_color_hist( patch_img, patch_hist, nbins );
183 |           patch++;
184 |         }
185 |       }
186 |     }
187 |   }
188 |   
189 |   
190 |   inline void calc_pyr_edge_hist( const Mat& img, Mat& hist, int nbins_ori=16, int nbins_mag=16, int level=2)
191 |   {
192 |     int w = img.cols;
193 |     int h = img.rows;
194 |     
195 |     int npatches = 0;
196 |     for(int i=0; i<level; i++)
197 |       npatches += pow(4,i);
198 |     
199 |     int hist_sz = nbins_ori + nbins_mag;
200 |     
201 |     if( !hist.data )
202 |       hist = Mat( hist_sz*npatches, 1, CV_32F, Scalar(0,0,0)[0] );
203 |     
204 |     int patch = 0;
205 |     for(int l=0; l<level; l++) {
206 |       for(int x=0; x<pow(2,l); x++) {
207 |         for(int y=0; y<pow(2,l); y++) {
208 |           int p_width  = floor( w/pow(2,l) );
209 |           int p_height = floor( h/pow(2,l) );
210 |           int p_x = x*p_width;
211 |           int p_y = y*p_height;
212 |           Mat patch_img( img, Rect(p_x,p_y,p_width,p_height) );
213 |           Mat patch_hist( hist, Rect(0,patch*hist_sz,1,hist_sz) );
214 |           calc_edge_hist( patch_img, patch_hist, nbins_ori, nbins_mag );
215 |           patch++;
216 |         }
217 |       }
218 |     }
219 |   }
220 |   
221 |   
222 |   inline void calc_pyr_edge_hist( const Mat& Gx, const Mat& Gy, Mat& hist, int nbins_ori=16, int nbins_mag=16, int level=2)
223 |   {
224 |     int w = Gx.cols;
225 |     int h = Gx.rows;
226 |     
227 |     int npatches = 0;
228 |     for(int i=0; i<level; i++)
229 |       npatches += pow(4,i);
230 |     
231 |     int hist_sz = nbins_ori + nbins_mag;
232 |     
233 |     if( !hist.data )
234 |       hist = Mat( hist_sz*npatches, 1, CV_32F, Scalar(0,0,0)[0] );
235 |     
236 |     int patch = 0;
237 |     for(int l=0; l<level; l++) {
238 |       for(int x=0; x<pow(2,l); x++) {
239 |         for(int y=0; y<pow(2,l); y++) {
240 |           int p_width  = floor( w/pow(2,l) );
241 |           int p_height = floor( h/pow(2,l) );
242 |           int p_x = x*p_width;
243 |           int p_y = y*p_height;
244 |           Mat patch_Gx( Gx, Rect(p_x,p_y,p_width,p_height) );
245 |           Mat patch_Gy( Gy, Rect(p_x,p_y,p_width,p_height) );
246 |           Mat patch_hist( hist, Rect(0,patch*hist_sz,1,hist_sz) );
247 |           calc_edge_hist( patch_Gx, patch_Gy, patch_hist, nbins_ori, nbins_mag );
248 |           patch++;
249 |         }
250 |       }
251 |     }
252 |   }
253 |   
254 | }
255 | 
256 | #endif
257 | 


--------------------------------------------------------------------------------
/include/hecate/gflseg.hpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * GFLSEG Group Fused LASSO Change Point Detection solver
  3 |  *
  4 |  * Copyright 2016 Yahoo Inc.
  5 |  * Licensed under the terms of the Apache 2.0 License.
  6 |  * See LICENSE file in the project root for terms.
  7 |  *
  8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
  9 |  *
 10 |  * Implementation is based on:
 11 |  *   Kevin Bleakley and Jean-Philippe Vert.
 12 |  *     "The group fused lasso for multiple change-point detection."
 13 |  *   arXiv preprint arXiv:1106.4199 (2011).
 14 |  */
 15 | 
 16 | 
 17 | #ifndef HECATE_GFLSEG_HPP
 18 | #define HECATE_GFLSEG_HPP
 19 | 
 20 | #include <opencv2/opencv.hpp>
 21 | #include <opencv2/core/core.hpp>
 22 | 
 23 | #if CV_MAJOR_VERSION > 2
 24 | #include <opencv2/core/core_c.h>
 25 | #endif
 26 | 
 27 | #include "hecate/sort.hpp"
 28 | 
 29 | using namespace std;
 30 | using namespace cv;
 31 | 
 32 | namespace hecate {
 33 |   
 34 |   class Segmenter {
 35 |   public:
 36 |     /*
 37 |      Automatic multiple change-point detection
 38 |      
 39 |      This function is a wrapper for multi-dimensional signal segmentation into at
 40 |      least k change-points by the LARS (function gflars), followed by change-point
 41 |      selection with a dynamic programming optimization (function dpseg).
 42 |      
 43 |      INPUT
 44 |      X      : n-by-p matrix to be segmented (n: # obs, p: # dim).
 45 |      k      : the number of change points to find in the first segmentation step
 46 |      theta  : stopping criteria, see dpseg. (default=0.1)
 47 |      forcek : k is set as the hard maximum number of change points
 48 |      
 49 |      OUTPUT
 50 |      jumps  : vector of change-points
 51 |      val    : matrix of the value on each interval
 52 |      updown : the up/down statistics on each interval
 53 |      */
 54 |     void gflseg(const Mat& X, vector<int>& jumps, int k, double theta=0.1);
 55 |     
 56 |     
 57 |     
 58 |     /*
 59 |      Segmentation of a multi-dimensional signal with dynamic programming
 60 |      http://pbil.univ-lyon1.fr/members/fpicard/franckpicard_fichiers/pdf/aCGH-stat.pdf
 61 |      Improvement is made by penalizing with the optimal segment length,
 62 |      lambda=sqrt(lstar-l) where lstar = n/k (typically chosen as 2-second long)
 63 |      
 64 |      INPUT
 65 |      X     : n-by-p matrix to be segmented (n: # obs, p: # dim).
 66 |      cp    : candidate change points (default: [0:n-2])
 67 |      kmax  : maximum number of change-points to test (default: length(cp))
 68 |      theta : stopping criteria. Typically chonse to be in the interval (0 0.5].
 69 |      The smaller the threshold, the higher the tendency to keep more
 70 |      breakpoints. The criteria is based on the method found in
 71 |      'Picard et al (2005)' "A statistical approach for array CGH data
 72 |      analysis" (BMC Bioinformatics). (default=0.5)
 73 |      
 74 |      OUTPUT
 75 |      jump  : j-by-1 vector of change-point positions for the i-th lambda value
 76 |      (j depends on lambda). i varies between 1 and kmax
 77 |      rse   : (kmax+1)-by-1 vector of residual squared error
 78 |      kbest : the number of selected change-points
 79 |      */
 80 |     void dpseg(const Mat& X, const vector<int>& cp, vector<vector<int> >& jumps,
 81 |                Mat& rse, int& kbest, double theta, int kmax=-1 );
 82 |     
 83 |     
 84 |     private:
 85 |     /*
 86 |      Segmentation of a multi-dimensional signal with the group fused LARS.
 87 |      
 88 |      INPUT
 89 |      X       : n-by-p matrix to be segmented (n: # obs, p: # dim).
 90 |      k       : the number of change points to find.
 91 |      epsilon : values smaller than epsilon are considered null.
 92 |      weights : (n-1)*1 vector of weights for the weighted graph fused LASSO.
 93 |      
 94 |      OUTPUT
 95 |      lambda     : estimated lambda values for each change-point
 96 |      jump       : successive change-point positions (1 x k)
 97 |      meansignal : mean signal per column (1 x p vector)
 98 |      */
 99 |     void gflars(const Mat& X, vector<int>& jumps, vector<double>& lambda,
100 |                 const int k, const double epsilon );
101 |     
102 |     
103 |     /*
104 |      Fast computation of Y' * X
105 |      
106 |      Compute R = Y'*X, where Y is the n-by-(n-1) design matrix for the weighted
107 |      group fused lasso, with weights defined by the vector w, and X is a n-by-p
108 |      data matrix. The computation is done in O(np).
109 |      
110 |      INPUT
111 |      X : n-by-p matrix
112 |      w : (n-1)-by-1 vector of weights
113 |      
114 |      OUTPUT
115 |      R : (n-1)-by-p matrix equal to W' * X
116 |      */
117 |     void leftmultiplybyXt(const Mat& X, const Mat& w, Mat& R);
118 |     
119 |     
120 |     /*
121 |      Fast computation of inv(W'*W)*X
122 |      
123 |      Compute R = inv(W(:,ind)'*W(:,ind))*X, where W is the n-by-(n-1) design matrix
124 |      
125 |      INPUT
126 |      X   : a-by-p matrix
127 |      ind : a-by-1 vector of indices between 1 and n-1, sorted in an ascending order
128 |      w   : (n-1)-by-1 vector of weights
129 |      n   : the size of X is n-by-(n-1)
130 |      
131 |      OUTPUT
132 |      R   : a-by-p matrix equal to inv(W'*W)*X
133 |      */
134 |     void leftmultiplybyinvXAtXA(const Mat& X, const vector<int>& ind,
135 |                                 const Mat& w, const int n, Mat& R);
136 |     
137 |     
138 |     /*
139 |      Fast computation of W' * W * X when X is sparse
140 |      
141 |      Compute R = W'*W*X, where X is a wor-sparse (n-1)-by-p matrix and W is the
142 |      n-by-(n-1) design matrix
143 |      
144 |      INPUT
145 |      X   : a-by-p matrix whose rows are the non-zero rows of the original X
146 |      (same order as ind)
147 |      ind : a-by-1 vector of indices of the non-zero rows of X (each in [1,n-1])
148 |      w   : (n-1)-by-1 vector of weights
149 |      n   : size of the problem
150 |      
151 |      OUTPUT
152 |      R   : (n-1)-by-p matrix equal to W'*W*X
153 |      */
154 |     void multiplyXtXbysparse(const Mat& X, const vector<int>& ind,
155 |                              const Mat& w, const int n, Mat& R);
156 |     
157 |     
158 |     /* UTILITY FUNCTIONS */
159 |     
160 |     // dst = src(I)
161 |     inline void get_subvector(const Mat& src, Mat& dst, const vector<int>& I)
162 |     {
163 |       dst = Mat( I.size(), 1, src.type() );
164 |       for( size_t i=0; i<I.size(); i++ )
165 |         dst.at<double>(i) = src.at<double>(I[i]);
166 |     };
167 |     
168 |     // dst = src(I,:)
169 |     inline void get_submatrix_row(const Mat& src, Mat& dst, const vector<int>& I)
170 |     {
171 |       dst = Mat( I.size(), src.cols, src.type() );
172 |       for( size_t i=0; i<I.size(); i++ )
173 |         src.row(I[i]).copyTo( dst.row(i) );
174 |     };
175 |     
176 |     // dst = cumsum(src)
177 |     inline void cumsum(const Mat& src, Mat& dst)
178 |     {
179 |       if( !dst.data )
180 |         dst = Mat( src.rows, src.cols, src.type() );
181 |       
182 |       src.row(0).copyTo( dst.row(0) );
183 |       for( int r=1; r<src.rows; r++)
184 |         dst.row(r) = dst.row(r-1) + src.row(r);
185 |     };
186 |     
187 |     // dst = sum(src.^2,2)
188 |     inline void norm2sq( const Mat& src, Mat& dst, int dim)
189 |     {
190 |       if( dim==0 ) {// column-wise
191 |         dst = Mat( 1, src.cols, src.type() );
192 |         for(int c=0; c<src.cols; c++ )
193 |           dst.at<double>(c) = cv::sum(src.col(c).mul(src.col(c)))[0];
194 |       }
195 |       else {
196 |         dst = Mat( src.rows, 1, src.type() );
197 |         for(int r=0; r<src.rows; r++)
198 |           dst.at<double>(r) = cv::sum(src.row(r).mul(src.row(r)))[0];
199 |       }
200 |     }
201 |     
202 |     inline void check_nan(vector<double>& v) {
203 |       for( size_t i=0; i<v.size(); i++ )
204 |         if( v[i]!=v[i] )
205 |           v[i]=0.0;
206 |     };
207 |     
208 |     inline void print(Mat& X, string name)
209 |     {
210 |       cout << name << "_c=[" << endl;
211 |       
212 |       for(int r=0; r<X.rows; r++) {
213 |         for( int c=0; c<X.cols; c++) {
214 |           if( X.type()==CV_32F || X.type()==CV_64F )
215 |             printf("%f ", X.at<double>(r,c));
216 |           else
217 |             printf("%d ", X.at<int>(r,c));
218 |         }
219 |         printf("\n");
220 |       }
221 |       printf("];\n\n");
222 |     }
223 |     
224 |     inline void print(vector<int> v, string name)
225 |     {
226 |       cout << name << "_c=[";
227 |       for(size_t i=0; i<v.size(); i++)
228 |         printf("%d ", v[i]);
229 |       printf("];\n\n");
230 |     }
231 |     
232 |     inline void print(vector<double> v, string name)
233 |     {
234 |       cout << name << "_c=[";
235 |       for(size_t i=0; i<v.size(); i++)
236 |         printf("%f,", v[i]);
237 |       printf("];\n\n");
238 |     }
239 |   };
240 |   
241 | }
242 | 
243 | #endif
244 | 
245 | 
246 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 | Apache License
  2 | Version 2.0, January 2004
  3 | 
  4 | http://www.apache.org/licenses/
  5 | 
  6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  7 | 
  8 | 1. Definitions.
  9 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by 
 10 | Sections 1 through 9 of this document.
 11 | 
 12 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting 
 13 | the License.
 14 | 
 15 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are 
 16 | controlled by, or are under common control with that entity. For the purposes of this definition, 
 17 | "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, 
 18 | whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding 
 19 | shares, or (iii) beneficial ownership of such entity.
 20 | 
 21 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
 22 | 
 23 | "Source" form shall mean the preferred form for making modifications, including but not limited to software 
 24 | source code, documentation source, and configuration files.
 25 | 
 26 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, 
 27 | including but not limited to compiled object code, generated documentation, and conversions to other media types.
 28 | 
 29 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, 
 30 | as indicated by a copyright notice that is included in or attached to the work (an example is provided in 
 31 | the Appendix below).
 32 | 
 33 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) 
 34 | the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, 
 35 | as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not 
 36 | include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work 
 37 | and Derivative Works thereof.
 38 | 
 39 | "Contribution" shall mean any work of authorship, including the original version of the Work and any 
 40 | modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to 
 41 | Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to 
 42 | submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of 
 43 | electronic, verbal, or written communication sent to the Licensor or its representatives, including but not 
 44 | limited to communication on electronic mailing lists, source code control systems, and issue tracking systems 
 45 | that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but 
 46 | excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner 
 47 | as "Not a Contribution."
 48 | 
 49 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been 
 50 | received by Licensor and subsequently incorporated within the Work.
 51 | 
 52 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby 
 53 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license 
 54 | to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute 
 55 | the Work and such Derivative Works in Source or Object form.
 56 | 
 57 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby 
 58 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated 
 59 | in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer 
 60 | the Work, where such license applies only to those patent claims licensable by such Contributor that are 
 61 | necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work 
 62 | to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including 
 63 | a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the 
 64 | Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under 
 65 | this License for that Work shall terminate as of the date such litigation is filed.
 66 | 
 67 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any 
 68 | medium, with or without modifications, and in Source or Object form, provided that You meet the following 
 69 | conditions:
 70 | 
 71 | You must give any other recipients of the Work or Derivative Works a copy of this License; and
 72 | You must cause any modified files to carry prominent notices stating that You changed the files; and
 73 | You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, 
 74 | trademark, and attribution notices from the Source form of the Work, excluding those notices that do not 
 75 | pertain to any part of the Derivative Works; and
 76 | If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You 
 77 | distribute must include a readable copy of the attribution notices contained within such NOTICE file, 
 78 | excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the 
 79 | following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source 
 80 | form or documentation, if provided along with the Derivative Works; or, within a display generated by the 
 81 | Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file 
 82 | are for informational purposes only and do not modify the License. You may add Your own attribution notices 
 83 | within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, 
 84 | provided that such additional attribution notices cannot be construed as modifying the License. 
 85 | 
 86 | You may add Your own copyright statement to Your modifications and may provide additional or different 
 87 | license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such 
 88 | Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise 
 89 | complies with the conditions stated in this License.
 90 | 
 91 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally 
 92 | submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this 
 93 | License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall 
 94 | supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding 
 95 | such Contributions.
 96 | 
 97 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or 
 98 | product names of the Licensor, except as required for reasonable and customary use in describing the origin 
 99 | of the Work and reproducing the content of the NOTICE file.
100 | 
101 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the 
102 | Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
103 | OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, 
104 | NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for 
105 | determining the appropriateness of using or redistributing the Work and assume any risks associated with Your 
106 | exercise of permissions under this License.
107 | 
108 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), 
109 | contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) 
110 | or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, 
111 | special, incidental, or consequential damages of any character arising as a result of this License or out 
112 | of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work 
113 | stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such 
114 | Contributor has been advised of the possibility of such damages.
115 | 
116 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, 
117 | You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other 
118 | liability obligations and/or rights consistent with this License. However, in accepting such obligations, 
119 | You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, 
120 | and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred 
121 | by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional 
122 | liability.
123 | 
124 | END OF TERMS AND CONDITIONS
125 | 


--------------------------------------------------------------------------------
/include/hecate/hecate.hpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * HECTATE Video Processing Library
  3 |  *
  4 |  * Copyright 2016 Yahoo Inc.
  5 |  * Licensed under the terms of the Apache 2.0 License.
  6 |  * See LICENSE file in the project root for terms.
  7 |  *
  8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
  9 |  */
 10 | 
 11 | #ifndef HECATE_HPP
 12 | #define HECATE_HPP
 13 | 
 14 | #include <stdio.h>
 15 | #include <stdlib.h>
 16 | #include <dirent.h>
 17 | #include <getopt.h>
 18 | #include <sys/stat.h> // mkdir
 19 | #include <unistd.h>   // access() function
 20 | 
 21 | #include <ctime>      // display date and time
 22 | #include <iostream>   // for standard I/O
 23 | #include <fstream>    // for file I/O
 24 | #include <string>     // for strings
 25 | #include <iomanip>    // for controlling float print precision
 26 | #include <sstream>    // string to number conversion
 27 | #include <chrono>
 28 | #include <limits>
 29 | #include <numeric>
 30 | 
 31 | // OpenMP library
 32 | #if defined(_OPENMP)
 33 | #include <omp.h>
 34 | #endif
 35 | 
 36 | // Hecate headers
 37 | #include "hecate/sort.hpp"
 38 | #include "hecate/time.hpp"
 39 | #include "hecate/gapstat.hpp"
 40 | #include "hecate/knapsack.hpp"
 41 | #include "hecate/cc_parser.hpp"
 42 | #include "hecate/file_helper.hpp"
 43 | #include "hecate/video_parser.hpp"
 44 | #include "hecate/ffmpeg_helper.hpp"
 45 | 
 46 | // OpenCV library
 47 | #include <opencv2/objdetect/objdetect.hpp>
 48 | #include <opencv2/ml/ml.hpp>
 49 | 
 50 | using namespace std;
 51 | 
 52 | // program options
 53 | struct hecate_params {
 54 |   string in_video;
 55 |   string out_dir;
 56 |   string caption;
 57 |   int step_sz;          // frame subsampling step size
 58 |   int njpg;             // number of thumbnail images
 59 |   int ngif;             // number of GIFs
 60 |   int lmov;             // length of video summary (in seconds)
 61 |   int gif_fps;          // gif play speed
 62 |   int jpg_width_px;     // thumbnail image width
 63 |   int gif_width_px;     // animated GIF width
 64 |   int mov_width_px;     // summary video width
 65 |   int max_duration;     // maximum length of video to process (in seconds)
 66 |   double fltr_begin_sec;// always filter out x-second frames at the beginning
 67 |   double fltr_end_sec;  // always filter out x-second frames at the end
 68 |   double invalid_wnd;   // window for dropping neighbor frames of low-quality ones
 69 |   bool jpg;             // generate thumbnail jpg
 70 |   bool gif;             // generate summary gif
 71 |   bool mov;             // generate summary movie
 72 |   bool gifsum;          // if enabled, combine individual GIFs into one summary
 73 |   bool gifall;          // generate all possible gifs (for debugging purpose)
 74 |   bool info_shot;       // print shot boundary info
 75 |   bool info_keyfrm;     // print key frame indices
 76 |   bool prefer_dynamic;  // if enabled, prefer dynamic scene in highlight
 77 |   bool gfl;             // run group-fused lasso as part of shot segmentation
 78 |   bool fade;            // disable fade-in/out during shot transition
 79 |   bool debug;
 80 |   bool display;
 81 |   
 82 |   hecate_params():
 83 |   out_dir("./output"),
 84 |   caption(""),
 85 |   step_sz(1),
 86 |   njpg(5),
 87 |   ngif(5),
 88 |   lmov(15),
 89 |   gif_fps(8),
 90 |   jpg_width_px(360),
 91 |   gif_width_px(360),
 92 |   mov_width_px(360),
 93 |   max_duration(-1),
 94 |   fltr_begin_sec(-1.0),
 95 |   fltr_end_sec(-1.0),
 96 |   invalid_wnd(0.15),
 97 |   jpg(false),
 98 |   gif(false),
 99 |   mov(false),
100 |   gifsum(false),
101 |   gifall(false),
102 |   info_shot(false),
103 |   info_keyfrm(false),
104 |   prefer_dynamic(true),
105 |   gfl(false),
106 |   fade(false),
107 |   debug(false),
108 |   display(false)
109 |   {};
110 | };
111 | 
112 | inline void hecate_parse_params(int argc, char** argv, hecate_params& opt)
113 | {
114 |   static struct option long_options[] = {
115 |     {"in_video",        required_argument, 0, 'i'},
116 |     {"out_dir",         required_argument, 0, 'o'},
117 |     {"step",            required_argument, 0, 's'},
118 |     {"njpg",            required_argument, 0, 'n'},
119 |     {"ngif",            required_argument, 0, 'q'},
120 |     {"lmov",            required_argument, 0, 'l'},
121 |     {"gif_fps",         required_argument, 0, 'f'},
122 |     {"jpg_width_px",    required_argument, 0, 'u'},
123 |     {"gif_width_px",    required_argument, 0, 'v'},
124 |     {"mov_width_px",    required_argument, 0, 'w'},
125 |     {"max_duration",    required_argument, 0, 'd'},
126 |     {"fltr_begin_sec",  required_argument, 0, 'a'},
127 |     {"fltr_end_sec",    required_argument, 0, 'b'},
128 |     {"invalid_wnd",     required_argument, 0, 'k'},
129 |     {"generate_jpg",      no_argument, 0, 'J'},
130 |     {"generate_gif",      no_argument, 0, 'G'},
131 |     {"generate_mov",      no_argument, 0, 'M'},
132 |     {"optimize_gif",      no_argument, 0, 'O'},
133 |     {"generate_gifsum",   no_argument, 0, 'S'},
134 |     {"generate_gifall",   no_argument, 0, 'A'},
135 |     {"print_shot_info",   no_argument, 0, 'T'},
136 |     {"print_keyfrm_info", no_argument, 0, 'K'},
137 |     {"prefer_dynamic",    no_argument, 0, 'V'},
138 |     {"gfl",               no_argument, 0, 'B'},
139 |     {"fade",              no_argument, 0, 'F'},
140 |     {"debug",             no_argument, 0, 'D'},
141 |     {"display",           no_argument, 0, 'C'},
142 |     {0,0,0,0}
143 |   };
144 |   
145 |   while( true ) {
146 |     int opt_idx=0;
147 |     int c = getopt_long( argc, argv,
148 |                         "0:1:2:3:4:5:6:7:8:9:"
149 |                         "a:b:c:d:e:f:g:h:i:j:k:l:m:n:o:p:q:r:s:t:u:v:w:x:y:z:"
150 |                         "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z",
151 |                         long_options, &opt_idx);
152 |     if( c==-1 ) break;
153 |     switch( c ) {
154 |       case 'i': opt.in_video         = optarg; break;
155 |       case 'o': opt.out_dir          = optarg; break;
156 |       case 's': opt.step_sz          = atoi(optarg); break;
157 |       case 'n': opt.njpg             = atoi(optarg); break;
158 |       case 'q': opt.ngif             = atoi(optarg); break;
159 |       case 'l': opt.lmov             = atoi(optarg); break;
160 |       case 'f': opt.gif_fps          = atoi(optarg); break;
161 |       case 'u': opt.jpg_width_px     = atoi(optarg); break;
162 |       case 'v': opt.gif_width_px     = atoi(optarg); break;
163 |       case 'w': opt.mov_width_px     = atoi(optarg); break;
164 |       case 'd': opt.max_duration     = atof(optarg); break;
165 |       case 'a': opt.fltr_begin_sec   = atof(optarg); break;
166 |       case 'b': opt.fltr_end_sec     = atof(optarg); break;
167 |       case 'k': opt.invalid_wnd      = atof(optarg); break;
168 |       case 'J': opt.jpg              = true; break;
169 |       case 'G': opt.gif              = true; break;
170 |       case 'M': opt.mov              = true; break;
171 |       case 'S': opt.gifsum = opt.gif = true; break;
172 |       case 'A': opt.gifall = opt.gif = true; break;
173 |       case 'T': opt.info_shot        = true; break;
174 |       case 'K': opt.info_keyfrm      = true; break;
175 |       case 'V': opt.prefer_dynamic   = true; break;
176 |       case 'B': opt.gfl              = true; break;
177 |       case 'F': opt.fade             = true; break;
178 |       case 'D': opt.debug            = true; break;
179 |       case 'C': opt.display          = true; break;
180 |     }
181 |   }
182 |   
183 |   // Create output dir (silently fails if dir already exists)
184 |   mkdir( opt.out_dir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH );
185 | }
186 | 
187 | 
188 | 
189 | inline void hecate_copyright()
190 | {
191 |   printf("\n");
192 |   printf("-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\n");
193 |   printf(" HECATE Copyright 2016 Yahoo Inc.\n");
194 |   printf("   Licensed under the terms of the Apache 2.0 License.\n");
195 |   printf("   Developed by : Yale Song (yalesong@yahoo-inc.com)\n");
196 |   printf("   Built on  : %s %s\n", __TIME__, __DATE__ );
197 |   printf("-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\n");
198 | }
199 | 
200 | inline void hecate_usage()
201 | {
202 |   hecate_params opt;
203 |   printf("USAGE: hecate -i infile [options]\n");
204 |   printf("\n");
205 |   printf("  -i  --in_video      (string)    Input video file\n");
206 |   printf("  -o  --out_dir       (string)    Output directory (%s)\n", opt.out_dir.c_str());
207 |   printf("  -s  --step          (int)       Frame subsampling step size (%d)\n", opt.step_sz);
208 |   printf("  -n  --njpg          (int)       Number of thumbnails to be generated (%d)\n", opt.njpg);
209 |   printf("  -q  --ngif          (int)       Number of GIFs to be generated (%d)\n", opt.ngif);
210 |   printf("  -r  --lmov          (int)       Length of video summary to be generated (in seconds) (%d)\n", opt.lmov);
211 |   printf("  -u  --jpg_width_px  (int)       Pixel width of thumbnail images (%d)\n", opt.jpg_width_px);
212 |   printf("  -v  --gif_width_px  (int)       Pixel width of animated GIFs (%d)\n", opt.gif_width_px);
213 |   printf("  -w  --mov_width_px  (int)       Pixel width of summary video (%d)\n", opt.mov_width_px);
214 |   printf("  --generate_jpg                  Generate thumbnail images\n");
215 |   printf("  --generate_gif                  Generate animated GIFs\n");
216 |   printf("  --generate_mov                  Generate a summary video\n");
217 |   printf("  --generate_gifsum               Generate animated GIFs summary\n");
218 |   printf("  --generate_gifall               Generate all possible animated GIFs\n");
219 |   printf("  --print_shot_info               Print valid shot ranges\n");
220 |   printf("  --print_keyfrm_info             Print keyframe indices\n");
221 |   
222 |   exit(-1);
223 | }
224 | 
225 | 
226 | 
227 | /*******************************************************************************
228 |  
229 |  HECTATE INTERFACE
230 |  
231 |  INPUT:
232 |  hecate_params opt : input option paramaters
233 |  
234 |  OUTPUT:
235 |  vector<int> v_thumb_idx : thumbnail frame indices
236 |  vector<hecate::Range> v_gif_range : highlight shot ranges for GIF creation
237 |  vector<hecate::Range> v_mov_range : highlight shot ranges for video summarization
238 |  
239 |  *******************************************************************************/
240 | void run_hecate( hecate_params& opt,
241 |                  vector<int>& v_thumb_idx,
242 |                  vector<hecate::Range>& v_gif_range,
243 |                  vector<hecate::Range>& v_mov_range );
244 | 
245 | inline void run_hecate( hecate_params& opt, vector<int>& v_thumb_idx) {
246 |   vector<hecate::Range> v_gif_range, v_mov_range;
247 |   run_hecate( opt, v_thumb_idx, v_gif_range, v_mov_range );
248 | }
249 | 
250 | inline void run_hecate( hecate_params& opt, vector<hecate::Range>& v_range) {
251 |   vector<int> v_thumb_idx;
252 |   vector<hecate::Range> v_xxx_range;
253 |   
254 |   if( opt.gif )
255 |     run_hecate( opt, v_thumb_idx, v_range, v_xxx_range );
256 |   else if( opt.mov )
257 |     run_hecate( opt, v_thumb_idx, v_xxx_range, v_range );
258 | }
259 | 
260 | 
261 | /*******************************************************************************
262 |  
263 |  THUMBNAIL IMAGE GENERATION MODULE
264 |  
265 |  INPUT:
266 |  hecate_params opt        : input option paramaters
267 |  video_metadata meta      : video metadata
268 |  vector<hecate::ShotRanges> v_shot_range
269 |                           : shot boundaries
270 |  const Mat& X             : input features
271 |  const Mat& diff          : n-by-1 vector of frame-by-frame difference scores
272 |  
273 |  OUTPUT:
274 |  vector<int> v_thumb_idx  : vector of frame index numbers for thumbnails
275 |  
276 |  *******************************************************************************/
277 | 
278 | void detect_thumbnail_frames( hecate_params& opt,
279 |                               hecate::video_metadata& meta,
280 |                               const vector<hecate::ShotRange>& v_shot_range,
281 |                               const Mat& X,
282 |                               const Mat& diff,
283 |                               vector<int>& v_thumb_idx);
284 | 
285 | void generate_thumbnails( hecate_params& opt, vector<int>& v_thumb_idx );
286 | 
287 | 
288 | 
289 | 
290 | /*******************************************************************************
291 |  
292 |  VIDEO SUMMARIZATION MODULE
293 |  
294 |  INPUT:
295 |  hecate_params opt        : input option paramaters
296 |  video_metadata meta      : video metadata
297 |  vector<hecate::ShotRanges> v_shot_range
298 |                           : shot boundaries
299 |  const Mat& X             : input features
300 |  const Mat& diff          : n-by-1 vector of frame-by-frame difference scores
301 |  
302 |  OUTPUT:
303 |  vector<hecate::Ranges> v_highlight_range
304 |                           : vector of highlight shot ranges
305 |  
306 |  *******************************************************************************/
307 | 
308 | void detect_highlight_shots( hecate_params& opt,
309 |                              hecate::video_metadata& meta,
310 |                              const vector<hecate::ShotRange>& v_shot_range,
311 |                              const Mat& X,
312 |                              const Mat& diff,
313 |                              vector<hecate::Range>& v_highlight_range );
314 | 
315 | void generate_highlight_clips( hecate_params& opt,
316 |                                vector<hecate::Range>& v_highlight_range );
317 | 
318 | 
319 | ////////////////////////////////////////////////////////////////////////////////
320 | //
321 | // VARIOUS HELPER FUNCTIONS
322 | //
323 | ////////////////////////////////////////////////////////////////////////////////
324 | 
325 | 
326 | inline void mark_invalid( vector<bool>& vec, int idx, int wnd_sz=0 )
327 | {
328 |   int vec_len = (int)vec.size();
329 |   for(int i=max(0,idx-wnd_sz); i<=min(vec_len-1,idx+wnd_sz); i++)
330 |     vec[i] = false;
331 | }
332 | 
333 | inline void expand_invalid_frms( vector<bool>& valid, int k )
334 | {
335 |   vector<bool> valid_new = valid;
336 |   
337 |   int nfrm = (int) valid.size();
338 |   for( int pos=1; pos<nfrm-1; pos++ )
339 |   {
340 |     if( !valid[pos] )
341 |       for(int i=max(0,pos-k); i<=min(nfrm-1,pos+k); i++)
342 |         valid_new[i] = false;
343 |   }
344 |   
345 |   valid = valid_new;
346 | }
347 | 
348 | 
349 | #endif
350 | 
351 | 


--------------------------------------------------------------------------------
/include/hecate/image_metrics.hpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Various image metrics
  3 |  *
  4 |  * Copyright 2016 Yahoo Inc.
  5 |  * Licensed under the terms of the Apache 2.0 License.
  6 |  * See LICENSE file in the project root for terms.
  7 |  *
  8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
  9 |  */
 10 | 
 11 | #ifndef HECATE_IMAGE_METRICS_HPP
 12 | #define HECATE_IMAGE_METRICS_HPP
 13 | 
 14 | #include <math.h>
 15 | #include <stdio.h>
 16 | #include <cmath>
 17 | #include <string>
 18 | #include <vector>
 19 | 
 20 | #include <opencv2/imgproc/imgproc.hpp>
 21 | #include <opencv2/highgui/highgui.hpp>
 22 | 
 23 | #include <hecate/hist_opencv.hpp>
 24 | 
 25 | using namespace cv;
 26 | using namespace std;
 27 | 
 28 | namespace hecate {
 29 |   
 30 |   
 31 | #define VALIDATE(x) ( std::isnan(x) ) ? 0 : x
 32 |   
 33 |   /*------------------------------------------------------------------------
 34 |    Contrast is measured as the standard deviation of the pixel intensities.
 35 |    RMS contrast does not depend on the angular frequency content or the
 36 |    spatial distribution of contrast in the image
 37 |    https://en.wikipedia.org/wiki/Contrast_(vision)#RMS_contrast
 38 |    ------------------------------------------------------------------------*/
 39 |   inline double calc_rms_contrast( const Mat& gray_img )
 40 |   /*-----------------------------------------------------------------------*/
 41 |   
 42 |   {
 43 |     Mat I;
 44 |     gray_img.convertTo( I, CV_32FC1 );
 45 |     Scalar mu = cv::mean( I );
 46 |     I = I - mu[0];
 47 |     return VALIDATE(cv::norm(I)/sqrt(I.rows*I.cols));
 48 |   }
 49 |   
 50 |   /*------------------------------------------------------------------------
 51 |    Sharpness is measured as the sum of magnitude in frequency domain
 52 |    ------------------------------------------------------------------------*/
 53 |   inline double calc_sharpness( const Mat& gray_img )
 54 |   /*-----------------------------------------------------------------------*/
 55 |   {
 56 |     Mat img;
 57 |     gray_img.convertTo( img, CV_32FC1 );
 58 |     img *= 1./255;
 59 |     
 60 |     Mat dx, dy;
 61 |     Sobel( img, dx, img.type(), 1, 0, 3 );
 62 |     Sobel( img, dy, img.type(), 0, 1, 3 );
 63 |     magnitude( dx, dy, dx );
 64 |     
 65 |     int npixels = gray_img.rows * gray_img.cols;
 66 |     return VALIDATE(cv::sum(dx)[0] / npixels);
 67 |   }
 68 |   
 69 |   
 70 |   /*------------------------------------------------------------------------
 71 |    Brightness is measured as the relative luminance in colorimetric spaces
 72 |    https://en.wikipedia.org/wiki/Relative_luminance
 73 |    ------------------------------------------------------------------------*/
 74 |   inline double calc_brightness( const Mat& img )
 75 |   /*-----------------------------------------------------------------------*/
 76 |   {
 77 |     vector<Mat> bgr;
 78 |     split( img, bgr );
 79 |     for(size_t j=0; j<bgr.size(); j++)
 80 |       bgr[j].convertTo(bgr[j],CV_32F);
 81 |     
 82 |     // compute perceived brightness
 83 |     Mat img_pb = (0.2126*bgr[2] + 0.7152*bgr[1] + 0.0722*bgr[0])/255.0;
 84 |     return VALIDATE(mean( img_pb )[0]);
 85 |   }
 86 |   
 87 |   /*------------------------------------------------------------------------
 88 |    Uniformness is measured as the ratio between the 5% percentile pixel
 89 |    intensity histograms and the rest.
 90 |    ------------------------------------------------------------------------*/
 91 |   inline double calc_uniformity( const Mat& gray_img, int nbins=128 )
 92 |   /*-----------------------------------------------------------------------*/
 93 |   {
 94 |     double val = 0.0;
 95 |     
 96 |     Mat ghist;
 97 |     hecate::calc_gray_hist( gray_img, ghist, nbins );
 98 |     if( cv::sum(ghist)[0] == 0 ) {
 99 |       val = 1.0;
100 |     }
101 |     else {
102 |       Mat hist_sorted;
103 |       cv::sort( ghist, hist_sorted, CV_SORT_EVERY_COLUMN|CV_SORT_DESCENDING );
104 |       hist_sorted /= cv::sum( hist_sorted )[0];
105 |       for(int j=0; j<(int)(0.05*nbins); j++)
106 |         val += (double) hist_sorted.at<float>(j);
107 |     }
108 |     return VALIDATE(val);
109 |   }
110 |   
111 |   /*------------------------------------------------------------------------
112 |    Symmetry is measured as the difference between edge orientation & magnitude
113 |    histograms of the left and right halves.
114 |    ------------------------------------------------------------------------*/
115 |   inline double calc_asymmetry( const Mat& gray_img )
116 |   /*-----------------------------------------------------------------------*/
117 |   {
118 |     Mat img_l( gray_img, Rect(0,0,gray_img.cols/2,gray_img.rows) );
119 |     Mat img_r( gray_img, Rect(gray_img.cols/2,0,gray_img.cols/2,gray_img.rows) );
120 |     
121 |     Mat hist_l, hist_r;
122 |     hecate::calc_edge_hist( img_l, hist_l );
123 |     hecate::calc_edge_hist( img_r, hist_r );
124 |     
125 |     return VALIDATE((double)cv::norm( hist_l-hist_r ));
126 |   }
127 |   
128 |   /*------------------------------------------------------------------------
129 |    Third Saliency is measured as the residual saliency on the resized image,
130 |    then collects the average saliency for each sub-window. It's an indicator
131 |    of the informativeness of each window, but also of the general distribution
132 |    of the content in the image.
133 |    ------------------------------------------------------------------------*/
134 |   inline vector<double> calc_third_saliency( const Mat& gray_img )
135 |   {
136 |     const int w = 3; // divide 128-by-128 image into 3-by-3 patches
137 |     
138 |     // 0-1 normalize & resize to 128-by-128
139 |     Mat img;
140 |     gray_img.convertTo( img, CV_32FC1 ); img *= 1./255;
141 |     cv::resize( img, img, cv::Size(128,128), 0, 0, cv::INTER_AREA );
142 |     
143 |     //real is img image, im is zero
144 |     Mat planes[]  = {Mat_<double>(img),               Mat::zeros(img.size(),CV_32FC1)};
145 |     Mat planes2[] = {Mat::zeros(img.size(),CV_32FC1), Mat::zeros(img.size(),CV_32FC1)};
146 |     Mat planesr[] = {Mat::zeros(img.size(),CV_32FC1), Mat::zeros(img.size(),CV_32FC1)};
147 |     
148 |     Mat complexI, magI, phasem;
149 |     merge( planes, 2, complexI ); // Merges for FT
150 |     dft( complexI, complexI );    //Forward DFT
151 |     split( complexI, planes );    // planes[0] = Re(DFT(I), planes[1] = Im(DFT(I))
152 |     magnitude( planes[0], planes[1], magI );    //DFT magnitude (amplitude)
153 |     phase( planes[0], planes[1], phasem );      //DFT phase
154 |     
155 |     //log spectral amplitude
156 |     cv::log( magI, magI );
157 |     
158 |     Mat smooth, kernel;
159 |     smooth = Mat::zeros( img.size(), CV_32FC1 );
160 |     kernel = Mat::zeros( Size(10,10), CV_32FC1 );
161 |     kernel += Scalar::all( 0.01 ); //kernel for average filtering ( 0.01=1/(10*10) )
162 |     
163 |     //smoothed spectrum with average filtering
164 |     cv::filter2D( magI, smooth, CV_32FC1, kernel, Point(-1,-1), 0, BORDER_REPLICATE );
165 |     
166 |     Mat diff;
167 |     cv::subtract( magI, smooth, diff ); //spectral residual (log domain)
168 |     cv::exp( diff, diff );              //spectral residual (real domain)
169 |     
170 |     //recover real and im part of the DFT after the residual
171 |     polarToCart( diff, phasem, planes2[0], planes2[1] );
172 |     
173 |     //invert the DFT (we are back in the pixel domain! We have just created a Saliency Component)
174 |     Mat result;
175 |     merge( planes2, 2, complexI );
176 |     dft( complexI, result, DFT_INVERSE+DFT_SCALE );
177 |     split( result, planesr );                       // planes[0] = Re(DFT(I), planes[1] = Im(DFT(I))
178 |     magnitude( planesr[0], planesr[1], result );	// sqrt(Re(DFT(I))^2 + Im(DFT(I))^2)
179 |     
180 |     normalize( result, result, 0, 2, cv::NORM_MINMAX );
181 |     
182 |     //compute the window dimensions
183 |     float cx = result.cols/w;
184 |     float cy = result.rows/w;
185 |     vector<double> features;
186 |     
187 |     //local mean for each window
188 |     for( int a=0; a<w; ++a ) {
189 |       for( int b=0; b<w; ++b ) {
190 |         Mat temp = result(Rect(cx*a, cy*b, cx, cy));
191 |         double val = mean( temp )[0];
192 |         features.push_back( VALIDATE(val) );
193 |       }
194 |     }
195 |     
196 |     return features;
197 |   }
198 |   /*-----------------------------------------------------------------------*/
199 |   
200 |   /*------------------------------------------------------------------------
201 |    Entropy is measured -sum(p.*log2(p)), which is a statistical measure of
202 |    randomness that can be used to characterize the texture of the imput image.
203 |    http://www.mathworks.com/help/images/ref/entropy.html?refresh=true
204 |    ------------------------------------------------------------------------*/
205 |   inline double calc_entropy( const Mat& gray_img, int nbins=256 )
206 |   /*-----------------------------------------------------------------------*/
207 |   {
208 |     double val=0.0;
209 |     
210 |     Mat ghist;
211 |     hecate::calc_gray_hist( gray_img, ghist, nbins );
212 |     for(int i=0; i<nbins; i++) {
213 |       if( ghist.at<float>(i)!=0 )
214 |         val -= ghist.at<float>(i) * log2(ghist.at<float>(i));
215 |     }
216 |     return VALIDATE(val);
217 |   }
218 |   
219 |   
220 |   /*------------------------------------------------------------------------
221 |    Contrast balance is measured as the l2 distance between an original
222 |    image and contrast-equalized image.
223 |    ------------------------------------------------------------------------*/
224 |   inline double calc_contrast_balance( const Mat& img )
225 |   /*-----------------------------------------------------------------------*/
226 |   {
227 |     double res = 0;
228 |     
229 |     vector<Mat> hsv_planes;
230 |     split( img, hsv_planes );
231 |     
232 |     //for each channel
233 |     for (int pl=0; pl < 3; ++pl)
234 |     {
235 |       //equalize the histogram
236 |       Mat img_hist_equalized;
237 |       equalizeHist( hsv_planes[pl], img_hist_equalized );
238 |       
239 |       //compute the diff between original and equalized histograms
240 |       res += cv::norm( hsv_planes[pl], img_hist_equalized ) /
241 |       (double)(img.rows*img.cols);
242 |     }
243 |     
244 |     //the larger the difference, the larger the contrast error. so we negate
245 |     return VALIDATE(-res);
246 |   }
247 |   
248 |   
249 |   /*------------------------------------------------------------------------
250 |    Exposure balance is measured as an absolute value of the skewness
251 |    of the luminance histogram.
252 |    ------------------------------------------------------------------------*/
253 |   inline double calc_exposure_balance( const Mat& img )
254 |   /*-----------------------------------------------------------------------*/
255 |   {
256 |     vector<Mat> hsv_planes;
257 |     split( img, hsv_planes );
258 |     
259 |     /// Establish the number of bins
260 |     const int histSize = 256;
261 |     
262 |     /// Set the ranges (for B,G,R)
263 |     float range[] = { 0, 256 };
264 |     const float *histRange = { range };
265 |     
266 |     cv::Mat hist;
267 |     double res = 0;
268 |     for (int pl=0; pl < 3; ++pl)
269 |     {
270 |       /// Compute the histograms:
271 |       cv::calcHist( &hsv_planes[pl], 1, 0, cv::Mat(), hist, 1, &histSize, &histRange,
272 |                    /*uniform: */true, /*accumulate: */false );
273 |       cv::Scalar sump = cv::sum( hist );
274 |       
275 |       double mean = (double) sump[0] / hist.rows;
276 |       
277 |       cv::Mat prod;
278 |       cv::multiply( (hist-mean), (hist-mean), prod);
279 |       cv::Scalar sums = cv::sum( prod );
280 |       
281 |       cv::multiply( prod, (hist-mean), prod);
282 |       cv::Scalar sumskew = cv::sum( prod );
283 |       
284 |       double result[2];
285 |       result[0] = sqrt( (double) sums[0] / hist.rows );
286 |       result[1] = (double) sumskew[0] / (hist.rows * result[0]*result[0]*result[0]);
287 |       
288 |       //absolute value of the skewness gives an idea of how much
289 |       //the histogram of colors is incorrect
290 |       res += std::abs( result[1] );
291 |     }
292 |     
293 |     // the larger the skew, the larger the exposure error
294 |     return VALIDATE(-res / 3);
295 |   }
296 |   
297 |   /*------------------------------------------------------------------------
298 |    JPEG quality is measured using the no-reference quality estimation of
299 |    Z. Wang, H. R. Sheikh, and A. C. Bovik. No-reference perceptual quality
300 |    assessment of jpeg compressed images. In ICIP, 2002.
301 |    ------------------------------------------------------------------------*/
302 |   inline double calc_jpeg_quality( const Mat& gray_img )
303 |   /*-----------------------------------------------------------------------*/
304 |   {
305 |     // Convert to grayscale image with 64bit doubles
306 |     cv::Mat img;
307 |     gray_img.convertTo( img, CV_32FC1 );
308 |     
309 |     const int m = img.rows;
310 |     const int n = img.cols;
311 |     
312 |     if (m<16 || n<16)
313 |     {
314 |       return -2.0;
315 |     }
316 |     
317 |     // feature extraction: horizontal features
318 |     Mat dh, dh1, dh2;
319 |     dh1 = img( Rect(1, 0, n-1, m) );
320 |     dh2 = img( Rect(0, 0, n-1, m) );
321 |     dh = dh1 - dh2;
322 |     
323 |     double sum = 0;
324 |     int count = 0;
325 |     double sumz = 0;
326 |     for (int i=0; i < m; ++i) {
327 |       for (int j=0; j < n-2; ++j) {
328 |         if ((j+1)%8==0 && j>0 && (j+1)<8*floor(n/8))
329 |         {
330 |           sum += std::abs( dh.at<double>(i,j) );
331 |           count++;
332 |         }
333 |         double signval = copysign( 1.0, dh.at<double>(i,j) ) *
334 |         copysign( 1.0, dh.at<double>(i,j+1) );
335 |         if (signval < 0)
336 |           sumz += 1;
337 |       }
338 |     }
339 |     
340 |     double bh = sum / count;
341 |     double ah = (8.0 * mean( cv::abs(dh) )[0] - bh) / 7;
342 |     double zh = sumz / (m * (n-2));
343 |     
344 |     // feature extraction: vertical features
345 |     Mat dv1, dv2, dv;
346 |     dv1 = img( Rect(0, 1, n, m-1) );
347 |     dv2 = img( Rect(0, 0, n, m-1) );
348 |     dv = dv1 - dv2;
349 |     
350 |     sum = 0;
351 |     count = 0;
352 |     sumz = 0;
353 |     for (int i=0; i < m-2; ++i) {
354 |       for (int j=0; j < n; ++j) {
355 |         if ((i+1)%8==0 && i>0 && (i+1)<8*floor(m/8))
356 |         {
357 |           sum += std::abs( dv.at<double>(i,j) );
358 |           count++;
359 |         }
360 |         double signval = copysign( 1.0, dv.at<double>(i,j) ) *
361 |         copysign( 1.0, dv.at<double>(i,j+1) );
362 |         if (signval < 0)
363 |           sumz += 1;
364 |       }
365 |     }
366 |     
367 |     double bv = sum / count;
368 |     double av = (8.0 * mean( cv::abs(dv) )[0] - bv) / 7;
369 |     double zv = sumz / (n * (m-2));
370 |     
371 |     //combined features
372 |     double B = (bh + bv) / 2;
373 |     double A = (ah + av) / 2;
374 |     double Z = (zh + zv) / 2;
375 |     
376 |     // Quality Prediction
377 |     const double alpha = -245.8909;
378 |     const double beta = 261.9373;
379 |     const double gamma1 = -239.8886;
380 |     const double gamma2 = 160.1664;
381 |     const double gamma3 = 64.2859;
382 |     
383 |     double score = alpha + beta *
384 |     std::pow( B, (gamma1/10000) ) *
385 |     std::pow( A, (gamma2/10000) ) *
386 |     std::pow( Z, (gamma3/10000) );
387 |     
388 |     return VALIDATE(score);
389 |   }
390 |   
391 | }
392 | #endif
393 | 


--------------------------------------------------------------------------------
/src/hecate/gflseg.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * GFLSEG Group Fused LASSO Change Point Detection solver
  3 |  *
  4 |  * Copyright 2016 Yahoo Inc.
  5 |  * Licensed under the terms of the Apache 2.0 License.
  6 |  * See LICENSE file in the project root for terms.
  7 |  *
  8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
  9 |  *
 10 |  * Implementation is based on:
 11 |  *   Kevin Bleakley and Jean-Philippe Vert.
 12 |  *     "The group fused lasso for multiple change-point detection."
 13 |  *   arXiv preprint arXiv:1106.4199 (2011).
 14 |  */
 15 | 
 16 | #include "hecate/gflseg.hpp"
 17 | 
 18 | using namespace std;
 19 | using namespace cv;
 20 | using namespace hecate;
 21 | 
 22 | void Segmenter::gflseg(const Mat& X, vector<int>& jumps, int k, double theta)
 23 | {
 24 |   double epsilon = 1e-9;
 25 |   
 26 |   // group fused LASSO solver
 27 |   vector<int> jmp1;
 28 |   vector<double> lambda;
 29 |   gflars( X, jmp1, lambda, k, epsilon );
 30 |   
 31 |   // DP optimizer
 32 |   vector<vector<int> > jmp2;
 33 |   Mat rse;
 34 |   int kbest = 0;
 35 |   dpseg( X, jmp1, jmp2, rse, kbest, theta, -1 );
 36 |   
 37 |   // return the optimal solution
 38 |   jumps.resize( (int)jmp2[kbest].size(), 0 );
 39 |   for( size_t i=0; i<jmp2[kbest].size(); i++)
 40 |     jumps[i] = jmp2[kbest][i];
 41 |   //jumps[jmp2[kbest].size()] = X.rows-1; // last index
 42 | };
 43 | 
 44 | 
 45 | void Segmenter::dpseg(const Mat& X, const vector<int>& cp, vector<vector<int> >& jumps,
 46 |                       Mat& rse, int& kbest, double theta, int kmax )
 47 | {
 48 |   int n = X.rows; // the length of the signal X
 49 |   int p = X.cols; // the dimension of the signal X
 50 |   
 51 |   // Prevent the likelihood from diverging too much
 52 |   if( kmax<0 )
 53 |     kmax = min((int)cp.size(), (int)floor(n/10));
 54 |   
 55 |   vector<int>::iterator it;
 56 |   vector<int> cp_srt = cp;
 57 |   std::sort( cp_srt.begin(), cp_srt.end() );
 58 |   
 59 |   // Compute boundaries of the smallest intervals considered
 60 |   // b = sort(union([0],union([n],option.candidatechangepoints)));
 61 |   vector<int> b(2+cp.size());
 62 |   vector<int> edge = {-1, n-1};
 63 |   it = set_union( edge.begin(), edge.end(), cp_srt.begin(), cp_srt.end(), b.begin() );
 64 |   b.resize( it-b.begin() );
 65 |   std::sort( b.begin(), b.end() );
 66 |   
 67 |   
 68 |   // k is the number of such intervals
 69 |   int k = (int)b.size()-1;
 70 |   
 71 |   
 72 |   // Compute the k-by-k matrix J such that J(i,j) for i<=j is the RSE when
 73 |   // intervals i-to-j are merged.
 74 |   //
 75 |   // According to Picard,
 76 |   // "J(i,j) is the cost of the path connecting i-to-j in k steps (k segments)"
 77 |   // http://pbil.univ-lyon1.fr/members/fpicard/franckpicard_fichiers/pdf/aCGH-stat.pdf
 78 |   Mat J( k,   k, X.type(), Scalar(0,0,0)[0] );
 79 |   Mat S( n+1, p, X.type(), Scalar(0,0,0)[0] ); // cumsum of the rows of X
 80 |   Mat v( n+1, 1, X.type(), Scalar(0,0,0)[0] ); // cumsum of squared norm of the rows of X
 81 |   
 82 |   // S = [zeros(1,size(X,2));cumsum(X)]; % cumsum of the rows
 83 |   Mat Ssub = S( Rect(0,1,p,n) ); cumsum( X, Ssub );
 84 |   
 85 |   // v = [0;cumsum(sum(X.*X,2))]; % cumsum of squared norm of the rows
 86 |   Mat vsub = v( Rect(0,1,1,n) );
 87 |   cv::reduce( X.mul(X), vsub, 1, CV_REDUCE_SUM );
 88 |   cumsum( vsub, vsub );
 89 |   
 90 |   for( int i=0; i<k; i++ ) {
 91 |     for( int j=i; j<k; j++ ) {
 92 |       int Istart = b[i]+1;
 93 |       int Iend = b[j+1];
 94 |       // Regularization term: 1/length * ||s[end]-s[start]||^2_2
 95 |       double penalty = pow(cv::norm(S.row(Iend+1)-S.row(Istart)),2.0)/(Iend-Istart+1);
 96 |       J.at<double>(i,j) = v.at<double>(Iend+1)-v.at<double>(Istart)-penalty;
 97 |     }
 98 |   }
 99 |   
100 |   //
101 |   // DP recursion
102 |   //
103 |   
104 |   // V(i,j) is the best RSE for segmenting intervals 1 to j with at most i-1 change-points
105 |   Mat V( kmax+1, k, X.type(), Scalar(0,0,0)[0] );
106 |   Mat jumpmat( kmax, k, CV_32S, Scalar(-1,0,0)[0] );
107 |   
108 |   // With no change-points, V(1,j) is just the precomputed RSE for intervals 1 to j
109 |   J.row(0).copyTo( V.row(0) );
110 |   
111 |   // The recursive formula
112 |   // NOTE: minidx/maxidx in minMaxIdx are very, very confusing to use!
113 |   int minidx[2]; double minval;
114 |   for( int ki=0; ki<kmax; ki++ ) {
115 |     for( int j=ki+1; j<k; j++ ) {
116 |       Mat tmp = V(Rect(ki,ki,j-ki,1)) + J(Rect(j,ki+1,1,j-ki)).t();
117 |       minMaxIdx( tmp, &minval, 0, minidx, 0 ); // row matrix
118 |       V.at<double>(ki+1,j) = minval;
119 |       jumpmat.at<int>(ki,j) = minidx[1]+ki;
120 |     }
121 |   }
122 |   
123 |   
124 |   // Optimal segmentations
125 |   for( int ki=0; ki<kmax; ki++ )
126 |   {
127 |     vector<int> jump_ki(ki+1, 0);
128 |     jump_ki[ki] = jumpmat.at<int>( ki,k-1 );
129 |     for( int i=ki-1; i>=0; i-- )
130 |       jump_ki[i] = jumpmat.at<int>(i,jump_ki[i+1]);
131 |     jumps.push_back( jump_ki );
132 |   }
133 |   
134 |   
135 |   // Convert interval index back to the last position before the jump
136 |   for( int ki=0; ki<kmax; ki++ )
137 |     for( size_t i=0; i<jumps[ki].size(); i++ )
138 |       jumps[ki][i] = b[ 1+jumps[ki][i] ];
139 |   
140 |   
141 |   // RSE as a fuction of the number of change-points
142 |   V.col(k-1).copyTo( rse );
143 |   
144 |   
145 |   //
146 |   // Based on DP table, find the optimal number of change-points
147 |   //
148 |   
149 |   // log-likelihood is -n/2*(J-log(n)+1+log(2*pi));
150 |   Mat JJ; cv::log( rse, JJ );
151 |   
152 |   // normalize
153 |   // >> JJtild = (JJ(Km)-JJ) / (JJ(Km)-JJ(1)) * (Km-1) + 1;
154 |   int Km = JJ.rows;
155 |   Mat JJtild = (JJ.at<double>(Km-1)-JJ) / (JJ.at<double>(Km-1)-JJ.at<double>(0)) * (Km-1) + 1;
156 |   
157 |   
158 |   // find the inflexion point
159 |   // >> res.kbest = max(find(diff(diff(Jtild))>option.threshold))+1;
160 |   Mat dJJtild(Km-1,1,X.type(),Scalar(0,0,0)[0]);
161 |   Mat ddJJtild(Km-2,1,X.type(),Scalar(0,0,0)[0]);
162 |   for(int i=0; i<Km-1; i++)
163 |     dJJtild.at<double>(i) = JJtild.at<double>(i+1) - JJtild.at<double>(i);
164 |   for(int i=0; i<Km-2; i++)
165 |     ddJJtild.at<double>(i) = dJJtild.at<double>(i+1) - dJJtild.at<double>(i);
166 |   
167 |   kbest = 0;
168 |   for( int i=0; i<Km-2; i++ )
169 |     if( ddJJtild.at<double>(i)>theta && i>kbest )
170 |       kbest = i;
171 | };
172 | 
173 | 
174 | void Segmenter::gflars(const Mat& X, vector<int>& jumps,
175 |                        vector<double>& lambda, const int k, const double epsilon)
176 | {
177 |   int n = X.rows; // the length of the signal X
178 |   int p = X.cols; // the dimension of the signal X
179 |   
180 |   jumps.clear();
181 |   lambda.clear();
182 |   
183 |   // Default weight w(i) = sqrt(n/(i*(n-i)))
184 |   Mat weights( n-1, 1, X.type() );
185 |   for( int i=0; i<n-1; i++ )
186 |     weights.at<double>(i)= sqrt((double)n/((i+1)*(n-i-1)));
187 |   
188 |   // Auxilary variable to use MATLAB-like sort function. The variable vjmps
189 |   // should always be synced with vector<int> jumps.
190 |   vector<int> vsrtval;    // contains sorted values
191 |   vector<size_t> vsrtidx; // contains sorted indices
192 |   
193 |   //
194 |   // Initialize cHat = W'*X
195 |   Mat cHat;
196 |   leftmultiplybyXt(X,weights,cHat);
197 |   
198 |   vector<int> A; // Active set indices (sorted in an ascending order)
199 |   Mat cHatSub; // Used to access cHat(A,:)
200 |   
201 |   //
202 |   // Main loop to find the successive jumps
203 |   for( int iter=0; iter<k; iter++ )
204 |   {
205 |     // Compute row-wise norms of cHat
206 |     // >> cHatSquareNorm = sum(cHat.^2,2);
207 |     Mat cHatSquareNorm;
208 |     norm2sq( cHat, cHatSquareNorm, 1 ); // 0:col, 1:row-wise
209 |     
210 |     // >> [bigcHat,besti]=max(cHatSquareNorm);
211 |     int besti[2];
212 |     double bigcHat;
213 |     minMaxIdx( cHatSquareNorm, 0, &bigcHat, 0, besti ); // col matrix
214 |     
215 |     //
216 |     // In the first iteration, we add the most correlated feature to the
217 |     // active set. For the other iterations, this is already done at the
218 |     // end of the previous iteration
219 |     if( iter==0 ) {
220 |       jumps.push_back( besti[0] );
221 |     }
222 |     
223 |     // Resize active set vector and cHatSub matrix
224 |     A.resize( iter+1, 0 );
225 |     cHatSub = Mat( iter+1, p, X.type(), Scalar(0,0,0)[0] );
226 |     
227 |     
228 |     //
229 |     // Compute the descent direction W = inv(X(:,A)'*X(:,A))*cHat(A,:)
230 |     Mat W; // size of (iter+1)-by-p
231 |     
232 |     // >> [A,I]=sort(res.jump(1:iter));
233 |     hecate::sort( jumps, A, vsrtidx );
234 |     get_submatrix_row( cHat, cHatSub, A );
235 |     
236 |     // >> w = leftmultiplybyinvXAtXA(n,A,cHat(A,:),weights);
237 |     leftmultiplybyinvXAtXA(cHatSub,A,weights,n,W);
238 |     
239 |     // >> B = multiplyXtXbysparse(n,A,w,weights);
240 |     Mat B;
241 |     multiplyXtXbysparse(W,A,weights,n,B);
242 |     
243 |     //
244 |     // Compute the descent step
245 |     //   For each i we find the largest possible step alpha by solving:
246 |     //      norm(cHat(i,:)-alpha*B(i,:)) = norm(cHat(j,:)-alpha*B(j,:))
247 |     //      where j is in the active set.
248 |     //   We write it as a second order polynomial
249 |     //      a1(i)*alpha^2 - 2* a2(i)*alpha + a3(i)
250 |     
251 |     Mat a1,a2,a3;
252 |     Mat a1sub, a2sub, a3sub;
253 |     Mat tmp1, tmp2, tmp3;
254 |     vector<int> subset;
255 |     
256 |     // >> a1 = bigcHat - sum(B.^2,2);
257 |     cv::reduce( B.mul(B), a1, 1, CV_REDUCE_SUM );
258 |     a1 = bigcHat - a1;
259 |     
260 |     // >> a2 = bigcHat - sum(B.*cHat,2);
261 |     cv::reduce( B.mul(cHat), a2, 1, CV_REDUCE_SUM );
262 |     a2 = bigcHat - a2;
263 |     
264 |     // >> a3 = bigcHat - cHatSquareNorm;
265 |     a3 = bigcHat - cHatSquareNorm;
266 |     
267 |     //
268 |     // Now we solve it
269 |     // >> gammaTemp = zeros(2*(n-1),1);
270 |     Mat gammaTemp( 2*(n-1), 1, X.type(), Scalar(0,0,0)[0] );
271 |     
272 |     // First, those where we really have a second-order polynomial
273 |     // >> subset = find(a1 > EPSILON);
274 |     for( int i=0; i<a1.rows; i++ )
275 |       if( a1.at<double>(i)>epsilon )
276 |         subset.push_back(i);
277 |     
278 |     if( !subset.empty() )
279 |     {
280 |       get_subvector( a1, a1sub, subset );
281 |       get_subvector( a2, a2sub, subset );
282 |       get_subvector( a3, a3sub, subset );
283 |       
284 |       // tmp1 = sqrt( a2(subset).^2 - a1(subset).*a3(subset) )
285 |       cv::sqrt( a2sub.mul(a2sub)-a1sub.mul(a3sub), tmp1);
286 |       
287 |       // >> gammaTemp(subset)
288 |       // = (a2(subset) + sqrt( a2(subset).^2 - a1(subset).*a3(subset) )) ./ a1(subset);
289 |       cv::divide( a2sub + tmp1, a1sub, tmp2);
290 |       for( size_t i=0; i<subset.size(); i++ )
291 |         gammaTemp.at<double>(subset[i]) = tmp2.at<double>(i);
292 |       
293 |       // >> gammaTemp(subset+n-1)
294 |       // = (a2(subset) - sqrt( a2(subset).^2 - a1(subset).*a3(subset) )) ./ a1(subset);
295 |       cv::divide( a2sub - tmp1, a1sub, tmp2);
296 |       for( size_t i=0; i<subset.size(); i++ )
297 |         gammaTemp.at<double>(subset[i]+n-1) = tmp2.at<double>(i);
298 |       
299 |       subset.clear();
300 |     }
301 |     
302 |     //
303 |     // then those where the quadratic term vanishes and we have a
304 |     // first-order polynomial
305 |     // >> subset = find((a1 <= EPSILON) & (a2 > EPSILON));
306 |     for( int i=0; i<a1.rows; i++ )
307 |       if( a1.at<double>(i)<=epsilon && a2.at<double>(i)>epsilon )
308 |         subset.push_back(i);
309 |     
310 |     if( !subset.empty() )
311 |     {
312 |       get_subvector( a1, a1sub, subset );
313 |       get_subvector( a2, a2sub, subset );
314 |       get_subvector( a3, a3sub, subset );
315 |       
316 |       // >> gammaTemp(subset)     = a3(subset) ./ (2*a2(subset));
317 |       // >> gammaTemp(subset+n-1) = a3(subset) ./ (2*a2(subset));
318 |       cv::divide( a3sub, 2*a2sub, tmp2 );
319 |       for( size_t i=0; i<subset.size(); i++ )
320 |         gammaTemp.at<double>(subset[i]) = gammaTemp.at<double>(subset[i]+n-1) = tmp2.at<double>(i);
321 |       
322 |       subset.clear();
323 |     }
324 |     
325 |     
326 |     //
327 |     // Finally the active set should not be taken into account, as well as
328 |     // those for which the computation gives dummy solutions
329 |     // >> maxg=max(gammaTemp)+1;
330 |     double maxg; minMaxIdx(gammaTemp, 0, &maxg); maxg+=1.0;
331 |     
332 |     // >> subset = find((a1 <= EPSILON) & (a2 <= EPSILON));
333 |     for( int i=0; i<a1.rows; i++ )
334 |       if( a1.at<double>(i)<=epsilon && a2.at<double>(i)<=epsilon )
335 |         subset.push_back(i);
336 |     
337 |     if( !subset.empty() )
338 |     {
339 |       // >> gammaTemp(subset) = maxg;
340 |       // >> gammaTemp(n+subset) = maxg;
341 |       for( size_t i=0; i<subset.size(); i++ )
342 |         gammaTemp.at<double>(subset[i]) = gammaTemp.at<double>(subset[i]+n) = maxg;
343 |       subset.clear();
344 |     }
345 |     
346 |     // >> gammaTemp(A) = maxg;
347 |     // >> gammaTemp(n+A-1) = maxg;
348 |     for( size_t i=0; i<A.size(); i++ )
349 |       gammaTemp.at<double>(A[i]) = gammaTemp.at<double>(A[i]+n-1) = maxg;
350 |     
351 |     // >> gammaTemp(gammaTemp<=0)=maxg;
352 |     // >> gammaTemp(imag(gammaTemp)~=0) = maxg;
353 |     for( int i=0; i<gammaTemp.rows; i++ )
354 |       if( gammaTemp.at<double>(i)<=0 || gammaTemp.at<double>(i)!=gammaTemp.at<double>(i) )
355 |         gammaTemp.at<double>(i) = maxg;
356 |     
357 |     //
358 |     // Now we can take the minimum
359 |     // >> [gamma,nexttoadd]=min(gammaTemp);
360 |     double gamma;
361 |     int nexttoadd[2];
362 |     minMaxIdx( gammaTemp, &gamma, 0, nexttoadd, 0); // col matrix
363 |     
364 |     //
365 |     // Update
366 |     // >> res.value{iter} = zeros(iter,p);
367 |     // >> res.value{iter}(I,:) = gamma*w;
368 |     // >> if iter>1
369 |     // >>     res.value{iter}(1:(iter-1),:) = res.value{iter}(1:(iter-1),:) + res.value{iter-1};
370 |     // >> end
371 |     
372 |     // >> res.lambda(iter)=sqrt(bigcHat);
373 |     lambda.push_back( sqrt(bigcHat) );
374 |     
375 |     // >> if iter<k
376 |     // >>     res.jump(iter+1) = 1+mod(nexttoadd-1,n-1);
377 |     // >>     cHat = cHat-gamma*a;
378 |     // >> end
379 |     if( iter+1<k ) {
380 |       jumps.push_back( nexttoadd[0]%(n-1) );
381 |       cHat = cHat - gamma*B;
382 |     }
383 |   }
384 | };
385 | 
386 | 
387 | // X is n-by-p, w is (n-1)-by-1, R is (n-1)-by-p
388 | void Segmenter::leftmultiplybyXt(const Mat& X, const Mat& w, Mat& R)
389 | {
390 |   int n = X.rows;
391 |   int p = X.cols;
392 |   
393 |   R = Mat( n-1, p, X.type() );
394 |   
395 |   // R = ([1:n-1]'*U(end,:)/n - U(1:end-1,:)) .* w(:,ones(1,p));
396 |   Mat U; cumsum( X, U );
397 |   
398 |   // llt = [1:n-1]'
399 |   Mat llt( n-1, 1, X.type() );
400 |   for( int r=0; r<n-1; r++ )
401 |     llt.at<double>(r) = (double)(r+1);
402 |   
403 |   Mat lt = llt*U.row(n-1)/n - U(Rect(0,0,p,n-1));
404 |   for( int c=0; c<p; c++ )
405 |     R.col(c) = lt.col(c).mul(w);
406 | };
407 | 
408 | 
409 | 
410 | // X is a-by-p, w is (n-1)-by-1, R is a-by-p, ind is a-by-1, where 1<=a<=(n-1)
411 | void Segmenter::leftmultiplybyinvXAtXA(const Mat& X, const vector<int>& ind,
412 |                                        const Mat& w, const int n, Mat& R)
413 | {
414 |   int a = X.rows;
415 |   int p = X.cols;
416 |   R = Mat( a, p, X.type(), Scalar(0,0,0)[0] );
417 |   
418 |   if( a>0 )
419 |   {
420 |     // >> u = diff([0;ind;n])
421 |     // Note: we convert C++ index system (zero-base) to MATLAB (one-base)
422 |     Mat u( a+1, 1, X.type() );
423 |     u.at<double>(0) = ind[0]+1; // [2 0 1] becomes [3 1 2]
424 |     u.at<double>(a) = n - (ind[a-1]+1);
425 |     for( int i=1; i<a; i++ )
426 |       u.at<double>(i) = ind[i] - ind[i-1];
427 |     
428 |     // >> val = val ./ w(ind,ones(1,p))
429 |     Mat val;  X.copyTo(val);
430 |     Mat wsub; get_subvector(w, wsub, ind);
431 |     for( int c=0; c<p; c++ )
432 |       cv::divide( val.col(c), wsub, val.col(c) );
433 |     
434 |     // >> delta = diff( [zeros(1,p); val; zeros(1,p)] ) ./ u(:,ones(1,p))
435 |     Mat delta( a+1, p, X.type(), Scalar(0,0,0)[0] );
436 |     delta.row(0) = val.row(0)+0.0;
437 |     delta.row(a) = -val.row(a-1);
438 |     for( int r=1; r<a; r++ )
439 |       delta.row(r) = val.row(r) - val.row(r-1);
440 |     for( int c=0; c<p; c++ )
441 |       cv::divide( delta.col(c), u, delta.col(c) );
442 |     
443 |     // >> R = - diff( delta )
444 |     for(int r=0; r<a; r++)
445 |       R.row(r) = delta.row(r) - delta.row(r+1);
446 |     
447 |     // >> R = R ./ w(ind,ones(1,p))
448 |     for(int c=0; c<p; c++)
449 |       cv::divide( R.col(c), wsub, R.col(c) );
450 |   }
451 | };
452 | 
453 | 
454 | 
455 | void Segmenter::multiplyXtXbysparse(const Mat& X, const vector<int>& ind,
456 |                                     const Mat& w, const int n, Mat& R)
457 | {
458 |   int a = X.rows;
459 |   int p = X.cols;
460 |   
461 |   R = Mat( n-1, p, X.type(), Scalar(0,0,0)[0] );
462 |   if( a>0 )
463 |   {
464 |     Mat wsub; get_subvector(w, wsub, ind);
465 |     Mat val; X.copyTo(val);
466 |     
467 |     // First multiply beta by the weights
468 |     // >> val = val .* w(ind,ones(1,p));
469 |     for(int c=0; c<p; c++)
470 |       val.col(c) = val.col(c).mul(wsub);
471 |     
472 |     //  compute the matrix s of increments of r
473 |     // >> S = zeros(n-1,p);
474 |     Mat S( n-1, p, X.type(), Scalar(0,0,0)[0] );
475 |     
476 |     // >> S(ind,:) = val;
477 |     for( size_t i=0; i<ind.size(); i++ )
478 |       val.row(i).copyTo(S.row(ind[i]));
479 |     
480 |     // >> S = flipud(cumsum(flipud(S)));
481 |     flip(S,S,0); cumsum(S,S); flip(S,S,0);
482 |     
483 |     // >> u = ind' * val; // don't forget to add ones
484 |     Mat indv( 1, a, X.type() );
485 |     for( size_t i=0; i<ind.size(); i++ )
486 |       indv.at<double>(i) = (double)(ind[i]+1);
487 |     Mat u = indv * val;
488 |     
489 |     // >> S = S - u(ones(n-1,1),:)/n;
490 |     for( int r=0; r<n-1; r++ )
491 |       S.row(r) = S.row(r) - u/n;
492 |     
493 |     // then make the cumsum
494 |     // >> R = cumsum(S);
495 |     cumsum( S, R );
496 |     
497 |     // then multiply the rows by the weights
498 |     // >> R = R .* w(:,ones(1,p));
499 |     for( int c=0; c<p; c++ )
500 |       R.col(c) = R.col(c).mul(w);
501 |   }
502 | };
503 | 
504 | 
505 | 


--------------------------------------------------------------------------------
/src/hecate/hecate_highlight.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * HECTATE Video Processing Library - Highlight
  3 |  *
  4 |  * Copyright 2016 Yahoo Inc.
  5 |  * Licensed under the terms of the Apache 2.0 License.
  6 |  * See LICENSE file in the project root for terms.
  7 |  *
  8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
  9 |  */
 10 | 
 11 | #include "hecate/hecate.hpp"
 12 | 
 13 | using namespace std;
 14 | using namespace cv;
 15 | 
 16 | 
 17 | void detect_highlight_shots( hecate_params& opt, hecate::video_metadata& meta,
 18 |                             const vector<hecate::ShotRange>& v_shot_range,
 19 |                             const Mat& X, const Mat& diff,
 20 |                             vector<hecate::Range>& v_highlight_range)
 21 | {
 22 |   if( opt.gif && opt.mov ) {
 23 |     fprintf( stderr, "Fatal Error in detect_highlight_shots():"
 24 |             "opt.gif and opt.mov cannot be true at the same time.\n" );
 25 |     exit(-1);
 26 |   }
 27 |   
 28 |   v_highlight_range.clear();
 29 | 
 30 |   const int minK = 5;
 31 |   const int maxK = 30;
 32 | 
 33 |   ////////////////////////////////////////////////////////////////////////////
 34 |   //
 35 |   // Discard shots that are too short or too static
 36 |   //
 37 |   ////////////////////////////////////////////////////////////////////////////
 38 | 
 39 |   int min_shot_len, max_shot_len, min_num_shot;
 40 |   if( opt.mov ) {
 41 |     double addrate_min = 0.5 * sqrt(max(0.0,(double)opt.lmov-15.0)/45.0);
 42 |     double addrate_max = sqrt(max(0.0,(double)opt.lmov-15.0)/45.0);
 43 |     min_shot_len = floor((3.0+addrate_min) * meta.fps / opt.step_sz);
 44 |     max_shot_len = round((1.0+addrate_max) * min_shot_len);
 45 |     min_num_shot = ceil( opt.lmov / 3.0 );
 46 |     if( opt.debug ) {
 47 |       printf("detect_highlight_shots(): "
 48 |              "min_shot_len=%d, max_shot_len=%d, min_num_shot=%d, "
 49 |              "addrate_min=%f, addrate_max=%f\n",
 50 |              min_shot_len, max_shot_len, min_num_shot,
 51 |              addrate_min, addrate_max);
 52 |     }
 53 |   }
 54 |   else {
 55 |     min_shot_len = floor(1.5 * meta.fps / opt.step_sz);
 56 |     max_shot_len = ceil (4.5 * meta.fps / opt.step_sz);
 57 |     min_num_shot = opt.ngif;
 58 |   }
 59 | 
 60 |   // Active set containing candidates for highlight shots
 61 |   vector<hecate::Range> v_candidates;
 62 |   for(size_t i=0; i<v_shot_range.size(); i++) {
 63 |     hecate::Range shot(v_shot_range[i].start, v_shot_range[i].end);
 64 |     shot.v_idx = v_shot_range[i].v_idx;
 65 |     int shot_len = shot.length();
 66 |     double avg_diff=0;
 67 |     for(int i=shot.start; i<=shot.end; i++)
 68 |       avg_diff += diff.at<double>(i);
 69 |     avg_diff/=shot.length();
 70 |     
 71 |     // Discard if too short.
 72 |     if( shot_len < min_shot_len ) {
 73 |       //printf("Discard SHORT (shot_len=%d, avg_diff=%f) ", shot_len, avg_diff); shot.print();
 74 |       continue;
 75 |     }
 76 |     
 77 |     // Discard if too static
 78 |     if( avg_diff < 0.05 ) {
 79 |       //printf("Discard STATIC (shot_len=%d, avg_diff=%f) ", shot_len, avg_diff); shot.print();
 80 |       continue;
 81 |     }
 82 |     
 83 |     // Add the shot
 84 |     v_candidates.push_back( shot );
 85 |   }
 86 | 
 87 |   // If there's not enough shots, merged adjacent shots and add them
 88 |   if( (int)v_candidates.size() < min_num_shot ) {
 89 |     if( opt.debug ) {
 90 |       printf("detect_highlight_shots(): "
 91 |              "Not enough candidate shots (current=%d/%d, min=%d)\n",
 92 |              (int)v_candidates.size(), (int)v_shot_range.size(), min_num_shot);
 93 |     }
 94 |     for(size_t i=0; i<v_shot_range.size(); i++) {
 95 |       // Merge shots until it's longer than min_shot_len
 96 |       size_t j=i;
 97 |       int cumlen=0;
 98 |       while( (j+1)<v_shot_range.size() ) {
 99 |         cumlen += v_shot_range[j].length();
100 |         if( cumlen<min_shot_len ) { j++; } else { break; }
101 |       }
102 |       // Construct a shot
103 |       hecate::Range shot(v_shot_range[i].start, v_shot_range[j].end);
104 |       shot.v_idx.clear();
105 |       for(size_t k=i; k<=j; k++)
106 |         shot.v_idx.insert( shot.v_idx.end(),
107 |             v_shot_range[k].v_idx.cbegin(), v_shot_range[k].v_idx.cend());
108 | 
109 |       // Check if this shot is already in v_candidates
110 |       bool exists = false;
111 |       for( size_t k=0; k<v_candidates.size(); k++ ) {
112 |         if( v_candidates[k].start >= shot.start &&
113 |            v_candidates[k].start <= shot.end ) {
114 |           exists = true; break;
115 |         }
116 |       }
117 |       if( !exists && shot.length() >= min_shot_len ) {
118 |         v_candidates.push_back( shot );
119 |       }
120 |       i = j+1;
121 |     }
122 | 
123 |     // Sort candidate shots in a chronogical order
124 |     vector<hecate::Range> v_candidates_tmp;
125 |     vector<int> v_shot_nsrt; // shots in non-sorted order
126 |     for( size_t i=0; i<v_candidates.size(); i++ )
127 |       v_shot_nsrt.push_back( v_candidates[i].start );
128 |     // sort wrt frame index in an ascending order
129 |     vector<size_t> v_srt_idx; // contains idx
130 |     vector<int> v_srt_val; // contains value (start frame idx)
131 |     hecate::sort( v_shot_nsrt, v_srt_val, v_srt_idx );
132 |     for(size_t i=0; i<v_srt_idx.size(); i++) {
133 |       v_candidates_tmp.push_back( v_candidates[v_srt_idx[i]] );
134 |     }
135 |     v_candidates = v_candidates_tmp;
136 |   }
137 |   
138 |   
139 |   // Shorten shots if too long.
140 |   // Prefer static shot; potentially reduces jitter
141 |   for(size_t i=0; i<v_candidates.size(); i++) {
142 |     hecate::Range shot = v_candidates[i];
143 |     int shot_len = shot.length();
144 |     if( shot_len > max_shot_len ) {
145 |       while( shot_len > max_shot_len ) {
146 |         Scalar mu, sigma, mu2, sigma2;
147 |         meanStdDev( diff(Rect(0,shot.start+1,1,shot.length()-1)), mu, sigma );
148 |         meanStdDev( diff(Rect(0,shot.start,  1,shot.length()-1)), mu2, sigma2 );
149 |         if( sigma[0] < sigma2[0] )
150 |           shot.start++;
151 |         else
152 |           shot.end--;
153 |         shot_len = shot.length();
154 |       }
155 |     }
156 |     v_candidates[i] = shot;
157 |   }
158 |   
159 |   
160 |   ////////////////////////////////////////////////////////////////////////////
161 |   //
162 |   // Group visually simimlar shots
163 |   //
164 |   // Represent each shot as a piece-wise constant multi-dimensional signal
165 |   // Then run kmeans (k=#nshots_highlight).
166 |   //
167 |   ////////////////////////////////////////////////////////////////////////////
168 |   
169 |   // Prepare data for kmeans
170 |   // Compute piece-wise constant feature representation
171 |   Mat km_data( (int)v_candidates.size(), X.cols, X.type() );
172 |   for( size_t shotid=0; shotid<v_candidates.size(); shotid++ ) {
173 |     hecate::Range s = v_candidates[shotid];
174 |     Mat Xsub = X( Rect(0, s.start, X.cols, s.length()) );
175 |     cv::reduce( Xsub, km_data.row(shotid), 0, CV_REDUCE_AVG );
176 |   }
177 |   
178 |   // Perform k-means (repeat 5 times)
179 |   Mat km_lbl; // integer row vector; stores cluster IDs for every sample.
180 |   Mat km_ctr; // one row per each cluster center.
181 |   int km_k = min(maxK, min((int)v_candidates.size(), max(minK, min_num_shot)));
182 |   hecate::perform_kmeans( km_data, km_lbl, km_ctr, km_k, 5 );
183 |   
184 |   // measure cluster size
185 |   vector<int> v_shotlen;
186 |   for(size_t i=0; i<v_candidates.size(); i++) {
187 |     v_shotlen.push_back( v_candidates[i].length() );
188 |   }
189 |   vector<int> clust_sz(km_k,0);
190 |   for(int i=0; i<km_lbl.rows; i++)
191 |     clust_sz[ km_lbl.at<int>(i) ] += v_shotlen[i];
192 |   
193 |   // sort wrt cluster size in an ascending order
194 |   vector<size_t> v_srt_idx; // contains cluster id
195 |   vector<int> v_srt_val;    // contains cluster size
196 |   hecate::sort( clust_sz, v_srt_val, v_srt_idx );
197 |   
198 |   
199 |   ////////////////////////////////////////////////////////////////////////////
200 |   //
201 |   // Shot evaluation
202 |   //
203 |   // Criteria: prefer the most dynamic shot among others within the same cluster
204 |   //
205 |   ////////////////////////////////////////////////////////////////////////////
206 |   
207 |   // Pre-compute per-shot diff score avg & stddev
208 |   Mat shot_diff_avg(v_candidates.size(), 1, diff.type(), Scalar(0,0,0)[0]);
209 |   Mat shot_diff_std(v_candidates.size(), 1, diff.type(), Scalar(0,0,0)[0]);
210 |   for( size_t shotid=0; shotid<v_candidates.size(); shotid++ ) {
211 |     Scalar mu, sigma;
212 |     hecate::Range s = v_candidates[shotid];
213 |     meanStdDev( diff(Rect(0,s.start,1,s.length())), mu, sigma );
214 |     shot_diff_avg.at<double>(shotid) = (double) mu[0];
215 |     shot_diff_std.at<double>(shotid) = (double) sigma[0];
216 |   }
217 | 
218 |   vector<int> v_shot_len;
219 |   vector<double> v_shot_score;
220 |   for(size_t i=0; i<v_candidates.size(); i++) {
221 |     v_shot_len.push_back( v_candidates[i].length() );
222 |     v_shot_score.push_back( 0.0 );
223 |   }
224 | 
225 |   // Start with the largest cluster
226 |   for(int i=0; i<km_k; i++)
227 |   {
228 |     int k = v_srt_idx[km_k-i-1];
229 |     hecate::Range best_range(-1,-1);
230 |     double best_val = -numeric_limits<double>::max();
231 |     
232 |     // Collect IDs & scores of the shots in the k-th cluster
233 |     vector<int> v_tmp_shotid;
234 |     vector<double> v_tmp_score;
235 |     for(int shotid=0; shotid<km_lbl.rows; shotid++) {
236 |       int lbl = km_lbl.at<int>(shotid);
237 |       if( lbl==k ) {
238 |         double val = shot_diff_avg.at<double>(shotid);
239 |         if( val > best_val ) {
240 |           best_range = v_candidates[shotid];
241 |           best_val = val;
242 |         }
243 |         v_tmp_shotid.push_back( shotid );
244 |         v_tmp_score.push_back( val );
245 |       }
246 |     }
247 |     
248 |     // If GIF mode, add shots here
249 |     if( opt.gif ) {
250 |       if( best_range.start>=0 && best_range.length()>0 )
251 |         v_highlight_range.push_back( best_range );
252 |       if( (int)v_highlight_range.size() >= opt.ngif )
253 |         break;
254 |     }
255 |     
256 |     // If MOV mode, store shot scores
257 |     if( opt.mov ) {
258 |       // Sort wrt avg_diff score, ascending order
259 |       vector<size_t> v_srt_idx2;
260 |       vector<double> v_srt_val2;
261 |       hecate::sort( v_tmp_score, v_srt_val2, v_srt_idx2 );
262 |       
263 |       // Record normalized scores
264 |       for( size_t i=0; i<v_srt_idx2.size(); i++ ) {
265 |         // if prefer dynamic, higher avg_diff gets priority
266 |         int order = (opt.prefer_dynamic) ? v_srt_idx2.size()-1-i : i;
267 |         v_shot_score[v_tmp_shotid[v_srt_idx2[order]]] = 100.0/(1.0+(double(i)));
268 |       }
269 |     }
270 |   }
271 |   
272 |   
273 |   ////////////////////////////////////////////////////////////////////////////
274 |   //
275 |   // Shot selection
276 |   //
277 |   // Criteria: prefer the most dynamic shot among others within the same cluster
278 |   //
279 |   ////////////////////////////////////////////////////////////////////////////
280 |   
281 |   // Return all available segments
282 |   if ( opt.gif ) {
283 |     if( opt.gifall ) {
284 |       v_highlight_range = v_candidates;
285 |     }
286 |   }
287 |   
288 |   if( opt.mov ) {
289 |     int budget = floor(opt.lmov * meta.fps / opt.step_sz);
290 |     budget += (int)(0.5 * max_shot_len); // buffer
291 |     
292 |     // solve 0/1 knapsack
293 |     vector<bool> sol;
294 |     hecate::solve_01knapsack( v_shot_score, v_shot_len, budget, sol );
295 |     
296 |     // merge shots if only they are only half a second apart
297 |     int merge_thrsh = ceil(0.5 * meta.fps / opt.step_sz);
298 |     for( size_t i=0; i<sol.size(); i++ ) {
299 |       if( sol[i] ) {
300 |         hecate::Range r = v_candidates[i];
301 |         if( v_highlight_range.empty() ) {
302 |           v_highlight_range.push_back( v_candidates[i] );
303 |         }
304 |         else {
305 |           int dist = r.start - v_highlight_range[v_highlight_range.size()-1].end + 1;
306 |           if( dist < merge_thrsh )
307 |             v_highlight_range[v_highlight_range.size()-1].end = r.end;
308 |           else
309 |             v_highlight_range.push_back( v_candidates[i] );
310 |         }
311 |       }
312 |     }
313 |   }
314 | 
315 |   for(size_t i=0; i<v_highlight_range.size(); i++) {
316 |     v_highlight_range[i].start *= opt.step_sz;
317 |     v_highlight_range[i].end *= opt.step_sz;
318 |   }
319 |   
320 |   
321 |   ////////////////////////////////////////////////////////////////////////////
322 |   //
323 |   // Post Processing
324 |   //
325 |   // Ensures summary video length is exactly the same as required
326 |   //
327 |   ////////////////////////////////////////////////////////////////////////////
328 |   
329 |   // Ensures highlight ranges sums up to exactly target length
330 |   if( opt.mov ) {
331 |     int target = round(opt.lmov * meta.fps);
332 |     int curlen = 0;
333 |     for(size_t i=0; i<v_highlight_range.size(); i++) {
334 |       curlen += v_highlight_range[i].length();
335 |     }
336 |     //printf("target=%d, cur=%d\n", target, curlen);
337 |     if( curlen!=target ) {
338 |       // If summary is too long
339 |       while( curlen>target ) {
340 |         // Pick a random shot
341 |         int idx = rand() % (int) v_highlight_range.size();
342 |         hecate::Range shot = v_highlight_range[idx];
343 |         // Remove frame that reduces stddev of the shot by the most
344 |         Scalar mu, sigma, mu2, sigma2;
345 |         meanStdDev( diff(Rect(0,shot.start+1,1,shot.length()-1)), mu, sigma );
346 |         meanStdDev( diff(Rect(0,shot.start,1,shot.length()-1)), mu2, sigma2 );
347 |         if( sigma[0] < sigma2[0] )
348 |           v_highlight_range[idx].start++;
349 |         else
350 |           v_highlight_range[idx].end--;
351 |         curlen--;
352 |       }
353 |       // If video is longer than target, but summary is too short
354 |       int shot_edge_buf = 10;
355 |       int failed_attempt = 0;
356 |       while( curlen<target ) {
357 |         // Pick a random shot
358 |         int idx = rand() % (int) v_highlight_range.size();
359 |         hecate::Range shot = v_highlight_range[idx];
360 |         
361 |         // Add frame that increases stddev of the shot by the least
362 |         bool failed = false;
363 |         if( v_highlight_range[idx].start<shot_edge_buf ) {
364 |           if( idx+1<v_highlight_range.size() && v_highlight_range[idx+1].start >
365 |              v_highlight_range[idx].end+1+shot_edge_buf ) {
366 |               v_highlight_range[idx].end++;
367 |           }
368 |           else {
369 |             failed = true;
370 |           }
371 |         }
372 |         else if( v_highlight_range[idx].end>meta.nframes-shot_edge_buf ) {
373 |           if( idx>0 && v_highlight_range[idx-1].end <
374 |              v_highlight_range[idx].start-1-shot_edge_buf ) {
375 |             v_highlight_range[idx].start--;
376 |           }
377 |           else {
378 |             failed = true;
379 |           }
380 |         }
381 |         else {
382 |           Scalar mu, sigma, mu2, sigma2;
383 |           meanStdDev( diff(Rect(0,shot.start-1,1,shot.length()+1)), mu, sigma );
384 |           meanStdDev( diff(Rect(0,shot.start,1,shot.length()+1)), mu2, sigma2 );
385 |           if( sigma[0] < sigma2[0] ) {
386 |             if( v_highlight_range[idx-1].end <
387 |                v_highlight_range[idx].start-1-shot_edge_buf ) {
388 |               v_highlight_range[idx].start--;
389 |             }
390 |             else {
391 |               failed = true;
392 |             }
393 |           }
394 |           else {
395 |             if( v_highlight_range[idx+1].start >
396 |                v_highlight_range[idx].end+1+shot_edge_buf) {
397 |               v_highlight_range[idx].end++;
398 |             }
399 |             else {
400 |               failed = true;
401 |             }
402 |           }
403 |         }
404 |         if( !failed )
405 |           curlen++;
406 |         else
407 |           failed_attempt++;
408 |         if( failed_attempt>10*(int)v_highlight_range.size() ) {
409 |           if( opt.debug ) {
410 |             printf("Failed to meet the target length in video summary\n");
411 |           }
412 |           break;
413 |         }
414 |       }
415 |     }
416 |     //curlen = 0;
417 |     //for(size_t i=0; i<v_highlight_range.size(); i++)
418 |     //  curlen += v_highlight_range[i].length();
419 |     //printf("target=%d, cur=%d\n", target, curlen);
420 |   }
421 | }
422 | 
423 | 
424 | ////////////////////////////////////////////////////////////////////////////
425 | //
426 | // Highlight clip generation
427 | //
428 | ////////////////////////////////////////////////////////////////////////////
429 | 
430 | void generate_highlight_clips( hecate_params& opt, vector<hecate::Range>& v_highlight_range )
431 | {
432 |   if( opt.gif && opt.mov ) {
433 |     fprintf( stderr, "Fatal Error in generate_highlight_clips():"
434 |             "opt.gif and opt.mov cannot be true at the same time.\n" );
435 |     exit(-1);
436 |   }
437 |   
438 |   // prefix for hiddden files generated during execution
439 |   const char *cdot = "__tmp__";
440 |   
441 |   string filename = hecate::get_filename( std::string(opt.in_video) );
442 |   
443 |   VideoCapture vr( opt.in_video );
444 |   double fps = vr.get(CV_CAP_PROP_FPS);
445 |   vr.release();
446 |   
447 |   // Sort shots in chronological order
448 |   vector<int> v_shot_nsrt; // shots in non-sorted order
449 |   for( size_t i=0; i<v_highlight_range.size(); i++ )
450 |     v_shot_nsrt.push_back( v_highlight_range[i].start );
451 |   
452 |   // sort wrt frame index in an ascending order
453 |   vector<size_t> v_srt_idx; // contains idx
454 |   vector<int> v_srt_val; // contains value (start frame idx)
455 |   hecate::sort( v_shot_nsrt, v_srt_val, v_srt_idx );
456 |   
457 |   // For concatenating vieos clips
458 |   char filelist[512];
459 |   sprintf( filelist, "%s/%s%s_seg.txt",
460 |           opt.out_dir.c_str(), cdot, filename.c_str() );
461 |   FILE *ptr_filelist = fopen( filelist, "w+" );
462 |   
463 |   // Parse ttml-format caption, if provided
464 |   vector<hecate::tcc> ttml;
465 |   if( !opt.caption.empty() ) {
466 |     parse_ttml( opt.caption, ttml );
467 |     if( ttml.empty() ) {
468 |       fprintf( stderr, "generate_animated_gifs: "
469 |               "Caption file %s cannot be read\n", opt.caption.c_str() );
470 |     }
471 |     // Convert second to frame index
472 |     for(size_t i=0; i<ttml.size(); i++) {
473 |       ttml[i].start *= fps;
474 |       ttml[i].end *= fps;
475 |     }
476 |   }
477 |   
478 |   // GENERATE CLIPS
479 |   char cmd[1024];
480 |   char infile[512];
481 |   char outfile[512];
482 |   
483 |   for( size_t i=0; i<v_highlight_range.size(); i++ )
484 |   {
485 |     int shotid = v_srt_idx[i];
486 |     hecate::Range r = v_highlight_range[shotid];
487 |     double sec_from = (double) (r.start+1) / fps;
488 |     double sec_duration = (double) (r.end-r.start) / fps;
489 |     string start_pos = hecate::second2string( sec_from, "hh:mm:ss.mss" );
490 |     string duration = hecate::second2string( sec_duration, "hh:mm:ss.mss" );
491 |     
492 |     // --------------------- VIDEO CLIP GENERATOR ---------------------
493 |     if( opt.mov )
494 |     {
495 |       // Crop video segment
496 |       sprintf( outfile, "%s/%s%s_seg%03d.mp4",
497 |               opt.out_dir.c_str(), cdot, filename.c_str(), shotid);
498 |       hecate::ffmpeg_video_crop( opt.in_video, std::string(outfile),
499 |                              start_pos, duration, opt.mov_width_px );
500 |       
501 |       // Apply video fade-in/out
502 |       sprintf( infile, "%s/%s%s_seg%03d.mp4",
503 |               opt.out_dir.c_str(), cdot, filename.c_str(), shotid);
504 |       sprintf( outfile, "%s/%s%s_segV%03d.mp4",
505 |               opt.out_dir.c_str(), cdot, filename.c_str(), shotid);
506 |       if( opt.fade ) {
507 |         hecate::ffmpeg_video_fade( std::string(infile), std::string(outfile),
508 |                                r.end-r.start+1,i==0 );
509 |       }
510 |       else {
511 |         sprintf( cmd, "mv %s %s", infile, outfile ); system( cmd );
512 |       }
513 |       
514 |       // Apply audio fade-in/out
515 |       sprintf( infile, "%s/%s%s_segV%03d.mp4",
516 |               opt.out_dir.c_str(), cdot, filename.c_str(), shotid);
517 |       sprintf( outfile, "%s/%s%s_segAV%03d.mp4",
518 |               opt.out_dir.c_str(), cdot, filename.c_str(), shotid);
519 |       hecate::ffmpeg_audio_fade( std::string(infile), std::string(outfile),
520 |                              sec_duration, fps );
521 |       
522 |       // Log filename for concat
523 |       fprintf( ptr_filelist, "file %s%s_segAV%03d.mp4\n",
524 |               cdot, filename.c_str(), shotid);
525 |     }
526 |     // --------------------- VIDEO CLIP GENERATOR ---------------------
527 |     
528 |     // --------------------- GIF CLIP GENERATOR ---------------------
529 |     if( opt.gif ) {
530 |       if( opt.gifsum )
531 |       {
532 |         // Crop video segment
533 |         sprintf( outfile, "%s/%s%s_seg%03d.mp4",
534 |                 opt.out_dir.c_str(), cdot, filename.c_str(), shotid);
535 |         hecate::ffmpeg_video_crop( opt.in_video, std::string(outfile),
536 |                                start_pos, duration, opt.mov_width_px );
537 |         
538 |         // Convert video to gif
539 |         sprintf( infile, "%s", outfile );
540 |         sprintf( outfile, "%s/%s_%02d.gif",
541 |                 opt.out_dir.c_str(), filename.c_str(), shotid);
542 |         hecate::ffmpeg_video2gif( std::string(infile), std::string(outfile),
543 |                               "", "", opt.gif_fps, opt.gif_width_px );
544 |         
545 |         // Log filename for concat
546 |         fprintf( ptr_filelist, "file %s%s_seg%03d.mp4\n",
547 |                 cdot, filename.c_str(), shotid);
548 |       }
549 |       else
550 |       {
551 |         // Convert video to gif, direct memory access without cropping
552 |         sprintf( outfile, "%s/%s_%02d.gif",
553 |                 opt.out_dir.c_str(), filename.c_str(), shotid);
554 |         hecate::ffmpeg_video2gif( opt.in_video, std::string(outfile),
555 |                               start_pos, duration, opt.gif_fps,
556 |                               opt.gif_width_px );
557 |       }
558 |     }
559 |     // --------------------- GIF CLIP GENERATOR ---------------------
560 |   }
561 |   
562 |   // Close filelist
563 |   fclose( ptr_filelist );
564 |   
565 |   if( opt.mov )
566 |   {
567 |     // Concatenate segments
568 |     sprintf( outfile, "%s/%s_sum.mp4",
569 |             opt.out_dir.c_str(), filename.c_str() );
570 |     hecate::ffmpeg_video_concat( filelist, outfile );
571 |   }
572 |   
573 |   if( opt.gif && opt.gifsum )
574 |   {
575 |     // Concatenate segments
576 |     sprintf( outfile, "%s/%s%s_segsum.mp4",
577 |             opt.out_dir.c_str(), cdot, filename.c_str() );
578 |     hecate::ffmpeg_video_concat( filelist, outfile );
579 |     
580 |     // Convert video to gif
581 |     sprintf( infile, "%s", outfile );
582 |     sprintf( outfile, "%s/%s_sum.gif",
583 |             opt.out_dir.c_str(), filename.c_str());
584 |     hecate::ffmpeg_video2gif( std::string(infile), std::string(outfile),
585 |                           "", "", opt.gif_fps, opt.gif_width_px );
586 |   }
587 |   
588 |   // Clean up
589 |   sprintf( cmd, "rm %s/%s%s_seg*.*",
590 |           opt.out_dir.c_str(), cdot, filename.c_str() );
591 |   system( cmd );
592 | }
593 | 
594 | 


--------------------------------------------------------------------------------
/src/hecate/video_parser.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Video Parser
  3 |  *
  4 |  * Copyright 2016 Yahoo Inc.
  5 |  * Licensed under the terms of the Apache 2.0 License.
  6 |  * See LICENSE file in the project root for terms.
  7 |  *
  8 |  * Developer: Yale Song (yalesong@yahoo-inc.com)
  9 |  */
 10 | 
 11 | #include "hecate/video_parser.hpp"
 12 | 
 13 | using namespace std;
 14 | using namespace cv;
 15 | using namespace hecate;
 16 | 
 17 | /*-----------------------------------------------------------------------*/
 18 | VideoParser::VideoParser()
 19 | /*-----------------------------------------------------------------------*/
 20 | {
 21 |   _debug = false;
 22 |   _display = false;
 23 |   
 24 |   _nfrm_total = 0;
 25 |   _nfrm_given = 0;
 26 |   
 27 |   _v_shot_ranges.clear();
 28 | }
 29 | 
 30 | /*-----------------------------------------------------------------------*/
 31 | void VideoParser::release_memory()
 32 | /*-----------------------------------------------------------------------*/
 33 | {
 34 |   _v_frm_rgb.clear();
 35 |   _v_frm_gray.clear();
 36 | }
 37 | 
 38 | /*-----------------------------------------------------------------------*/
 39 | vector<hecate::ShotRange> VideoParser::parse_video(const string& in_video,
 40 |                                                 hecate::parser_params opt)
 41 | /*-----------------------------------------------------------------------*/
 42 | {
 43 |   _debug = opt.debug;
 44 |   
 45 |   int ret = read_video( in_video, opt.step_sz, opt.max_duration,
 46 |                        opt.ignore_rest );
 47 |   if( ret<0 ) {
 48 |     fprintf( stderr, "VideoParser: Failed to open input video: %s\n",
 49 |             in_video.c_str());
 50 |     return vector<hecate::ShotRange>();
 51 |   }
 52 |   
 53 |   // Record metadata
 54 |   meta.nframes  = _nfrm_total;
 55 |   meta.width    = _video_width;
 56 |   meta.height   = _video_height;
 57 |   meta.fps      = _video_fps;
 58 |   meta.duration = _video_sec;
 59 |   
 60 |   // Frame filtering
 61 |   if( opt.fltr_begin_sec>.0 || opt.fltr_end_sec>.0 )
 62 |     filter_heuristic(opt.fltr_begin_sec, opt.fltr_end_sec);
 63 |   
 64 |   if( opt.fltr_lq )
 65 |     filter_low_quality();
 66 |   
 67 |   filter_transition();
 68 |   
 69 |   // Extract feature representation
 70 |   extract_histo_features();
 71 |   
 72 |   // Post-process (break up shots if too long)
 73 |   double min_shot_len_sec = 2.0;
 74 |   post_process(min_shot_len_sec, opt.gfl);
 75 |   
 76 |   release_memory();
 77 |   
 78 |   // Store shot information
 79 |   update_shot_ranges();
 80 |   
 81 |   // Subshot detection; subshot info is stored during this step
 82 |   if( opt.fltr_rdt )
 83 |     filter_redundant_and_obtain_subshots();
 84 |   
 85 |   if( _display )
 86 |     play_video_filtered( in_video, 2, 360 );
 87 |   
 88 |   // Produce result
 89 |   return _v_shot_ranges;
 90 | }
 91 | 
 92 | /*------------------------------------------------------------------------
 93 |  Read input video and store as RGB and GRAY formats.
 94 |  ------------------------------------------------------------------------*/
 95 | int VideoParser::read_video( const string& in_video, int step_sz,
 96 |                             double max_duration, bool ignore_rest,
 97 |                             int max_frm_len)
 98 | /*-----------------------------------------------------------------------*/
 99 | {
100 |   if( _debug )
101 |     printf("VideoParser: read_video(\"%s\", \n\tstep_sz=%d, "
102 |            "max_duration=%.2f, max_frm_len=%d, ignore_rest=%d)\n",
103 |            in_video.c_str(), step_sz, max_duration, max_frm_len, ignore_rest);
104 |   
105 |   VideoCapture vr( in_video );
106 |   if( !vr.isOpened() ) {
107 |     return -1;
108 |   }
109 |   
110 |   _nfrm_total   = vr.get(CV_CAP_PROP_FRAME_COUNT);
111 |   _video_width  = vr.get(CV_CAP_PROP_FRAME_WIDTH);
112 |   _video_height = vr.get(CV_CAP_PROP_FRAME_HEIGHT);
113 |   _video_fps    = max(1.0, vr.get(CV_CAP_PROP_FPS));
114 |   if( _video_fps!=_video_fps )
115 |     _video_fps = 29.97;
116 |   
117 |   int max_nfrms = (max_duration<0)
118 |   ? _nfrm_total : round(max_duration*_video_fps);
119 |   
120 |   // need to store _step_sz for computing min_shot_len in sbg_gflseg()
121 |   _step_sz = step_sz;
122 |   if( step_sz>10 ) {
123 |     _step_sz = 10;
124 |     fprintf( stderr, "VideoParser: The maximum step size is 10"
125 |             " (provided is %d)\n", step_sz );
126 |   }
127 |   // if video is too long, and ignore_rest is false, adjust step size
128 |   if( !ignore_rest && max_nfrms > 0 && _nfrm_total > max_nfrms ) {
129 |     _step_sz = ceil( _nfrm_total/max_nfrms );
130 |     fprintf( stderr, "VideoParser: Video too long (%d frames),"
131 |             "increasing step size to %d\n", _nfrm_total, step_sz );
132 |   }
133 |   _step_sz = max(1, _step_sz);
134 |   
135 |   int maxlen = max( _video_width, _video_height );
136 |   double rsz_ratio = (maxlen>max_frm_len)
137 |   ? (double) max_frm_len / maxlen : -1.0;
138 |   
139 |   // Read RGB frames
140 |   _nfrm_total=0; // reset because vr.get is known to be inaccurate
141 |   while( true )
142 |   {
143 |     Mat frm;
144 |     vr >> frm;
145 |     if( frm.empty() ) break;
146 |     
147 |     if( _nfrm_total % _step_sz == 0 ) {
148 |       if( rsz_ratio>0 )
149 |         resize( frm, frm, Size(), rsz_ratio, rsz_ratio, CV_INTER_LINEAR );
150 |       _v_frm_rgb.push_back( frm );
151 |       
152 |       // if video is too long, and ignore_rest is true, cut the rest
153 |       if( ignore_rest && _nfrm_total>=max_nfrms ) {
154 |         if( _debug )
155 |           printf("VideoParser: video too long, "
156 |                  "cutting at (%d)-th frame\n", _nfrm_total);
157 |         break;
158 |       }
159 |     }
160 |     _nfrm_total++;
161 |   }
162 |   vr.release();
163 |   
164 |   _nfrm_given = (int) _v_frm_rgb.size();
165 |   _video_sec  = (double)_nfrm_total/_video_fps;
166 |   
167 |   // RGB2GRAY
168 |   _v_frm_gray.assign( _nfrm_given, Mat() );
169 |   
170 | #pragma omp parallel for
171 |   for( int i=0; i<_nfrm_given; i++ )
172 |   {
173 |     Mat frm_gray;
174 |     cvtColor( _v_frm_rgb[i], frm_gray, CV_BGR2GRAY );
175 |     GaussianBlur( frm_gray, frm_gray, Size(3,3), 0, 0 );
176 |     frm_gray.copyTo( _v_frm_gray[i] );
177 |   }
178 |   
179 |   _v_frm_valid.assign( _nfrm_given, true );
180 |   _v_frm_log.assign( _nfrm_given, " " );
181 |   
182 |   _X_diff = Mat( _nfrm_given, 1, CV_64F, Scalar(0,0,0)[0] );
183 |   _X_ecr  = Mat( _nfrm_given, 1, CV_64F, Scalar(0,0,0)[0] );
184 |   
185 |   return 0;
186 | }
187 | 
188 | 
189 | /*------------------------------------------------------------------------
190 |  Manually invalidate first and last frames (heuristic)
191 |  ------------------------------------------------------------------------*/
192 | void VideoParser::filter_heuristic(double fltr_begin_sec, double fltr_end_sec)
193 | /*-----------------------------------------------------------------------*/
194 | {
195 |   int fltr_begin_nfrms = ceil(fltr_begin_sec * _video_fps / (double)_step_sz);
196 |   int fltr_end_nfrms = ceil(fltr_end_sec * _video_fps / (double)_step_sz);
197 |   
198 |   for(int i=0; i<fltr_begin_nfrms; i++)
199 |     mark_invalid(_v_frm_valid, _v_frm_log, i, "[Begin]");
200 |   for(int i=0; i<fltr_end_nfrms; i++)
201 |     mark_invalid(_v_frm_valid, _v_frm_log, _nfrm_total-1-i, "[End]");
202 | }
203 | 
204 | 
205 | /*------------------------------------------------------------------------
206 |  Use a collection of heuristics to filter out low-quality frames.
207 |  ------------------------------------------------------------------------*/
208 | void VideoParser::filter_low_quality( double thrsh_bright,
209 |                                      double thrsh_sharp,
210 |                                      double thrsh_uniform )
211 | /*-----------------------------------------------------------------------*/
212 | {
213 |   // filter at most n percent of the total frames
214 |   int nfrm_nperc = (int)(0.15*_nfrm_given);
215 |   
216 |   vector<double> v_brightness(_nfrm_given,0.0);
217 |   vector<double> v_sharpness(_nfrm_given,0.0);
218 |   vector<double> v_uniformity(_nfrm_given,0.0);
219 |   
220 | #pragma omp parallel for
221 |   for( int i=0; i<_nfrm_given; i++ ) {
222 |     v_brightness[i] = hecate::calc_brightness( _v_frm_rgb[i]  );
223 |     v_sharpness[i]  = hecate::calc_sharpness(  _v_frm_gray[i] );
224 |     v_uniformity[i] = hecate::calc_uniformity( _v_frm_gray[i] );
225 |   }
226 |   
227 |   vector<size_t> v_srt_idx; // contains sorted indices
228 |   vector<double> v_srt_val; // contains sorted values
229 |   
230 |   // DARK frame detection
231 |   hecate::sort( v_brightness, v_srt_val, v_srt_idx );
232 |   for( int i=0; i<nfrm_nperc; i++ )
233 |     if( v_srt_val[i] <= thrsh_bright )
234 |       mark_invalid(_v_frm_valid, _v_frm_log, v_srt_idx[i], "[Dark]");
235 |   
236 |   // BLURRY frame detection
237 |   hecate::sort( v_sharpness, v_srt_val, v_srt_idx );
238 |   for( int i=0; i<nfrm_nperc; i++ )
239 |     if( v_srt_val[i] <= thrsh_sharp )
240 |       mark_invalid(_v_frm_valid, _v_frm_log, v_srt_idx[i], "[Blurry]");
241 |   
242 |   // UNIFORM frame detection
243 |   hecate::sort( v_uniformity, v_srt_val, v_srt_idx );
244 |   for( int i=0; i<nfrm_nperc; i++ )
245 |     if( v_srt_val[_nfrm_given-i-1] >= thrsh_uniform )
246 |       mark_invalid(_v_frm_valid, _v_frm_log, v_srt_idx[_nfrm_given-i-1], "[Uniform]");
247 | }
248 | 
249 | 
250 | /*------------------------------------------------------------------------
251 |  Frames around shot boundaries are usually low-quality, filter them out
252 |  Use two SBD methods: frame-by-frame difference and ECR
253 |  ------------------------------------------------------------------------*/
254 | void VideoParser::filter_transition( double thrsh_diff, double thrsh_ecr )
255 | /*-----------------------------------------------------------------------*/
256 | {
257 |   int nfrm_nperc = (int)(0.10*_nfrm_given); // n percent of the total frames
258 |   
259 |   vector<double> v_diff(_nfrm_given, 0.0);
260 |   vector<double> v_ecr(_nfrm_given, 0.0);
261 |   
262 |   // sort wrt cluster size in an ascending order
263 |   vector<size_t> v_srt_idx; // contains sorted indices
264 |   vector<double> v_srt_val;    // contains sorted values
265 |   
266 |   // compute the first-order derivative frame-by-frame difference
267 |   int img_sz = _v_frm_gray[0].cols * _v_frm_gray[0].rows;
268 | #pragma omp parallel for
269 |   for( int i=1; i<_nfrm_given-1; i++ ) {
270 |     v_diff[i] = (double) (cv::norm( (_v_frm_rgb[i]-_v_frm_rgb[i-1]) )
271 |                           + cv::norm( (_v_frm_rgb[i]-_v_frm_rgb[i+1]) )) / (2.0*img_sz);
272 |     _X_diff.at<double>(i) = v_diff[i];
273 |   }
274 |   
275 |   // compute edge-change-ratio (ECR)
276 |   {
277 |     int dl_sz = 5; // dilute size
278 |     Mat dl_elm = getStructuringElement(MORPH_CROSS,
279 |                                        Size(2*dl_sz+1, 2*dl_sz+1),
280 |                                        Point(dl_sz, dl_sz));
281 |     
282 |     // Pre-compute edge & edge dilation
283 |     vector<Mat> v_edge( _nfrm_given, Mat() );    // edge images
284 |     vector<Mat> v_edge_dl( _nfrm_given, Mat() ); // edge-diluted images
285 |     
286 | #pragma omp parallel for
287 |     for(int i=0; i<_nfrm_given; i++)
288 |     {
289 |       Mat tmp;
290 |       double theta = threshold(_v_frm_gray[i],tmp,0,255,CV_THRESH_BINARY|CV_THRESH_OTSU);
291 |       Canny( _v_frm_gray[i], v_edge[i], theta, 1.2*theta);
292 |       dilate( v_edge[i], v_edge_dl[i], dl_elm );
293 |       v_edge[i] -= 254; v_edge_dl[i] -= 254;
294 |     }
295 |     
296 |     // Transition detection using ECR (edge change ratio)
297 | #pragma omp parallel for
298 |     for(int i=1; i<_nfrm_given; i++)
299 |     {
300 |       double rho_out, rho_in;
301 |       rho_out = 1.0 - min(1.0,sum(v_edge[i-1].mul(v_edge_dl[i]))[0]/max(1e-6,sum(v_edge[i-1])[0]));
302 |       rho_in  = 1.0 - min(1.0,sum(v_edge_dl[i-1].mul(v_edge[i]))[0]/max(1e-6,sum(v_edge[i-1])[0]));
303 |       
304 |       v_ecr[i] = max(rho_out,rho_in); // edge change ratio
305 |       _X_ecr.at<double>(i) = v_ecr[i];
306 |     }
307 |   }
308 |   
309 |   // CUT detection
310 |   hecate::sort( v_diff, v_srt_val, v_srt_idx );
311 |   for( int i=0; i<nfrm_nperc; i++ )
312 |     if( v_srt_val[_nfrm_given-i-1] >= thrsh_diff )
313 |       mark_invalid(_v_frm_valid, _v_frm_log, v_srt_idx[_nfrm_given-i-1], "[Cut]" );
314 |   
315 |   // TRANSITION detection (cut, fade, dissolve, wipe)
316 |   hecate::sort( v_ecr, v_srt_val, v_srt_idx );
317 |   for( int i=0; i<nfrm_nperc; i++ )
318 |     if( v_srt_val[_nfrm_given-i-1] >= thrsh_ecr )
319 |       mark_invalid(_v_frm_valid, _v_frm_log, v_srt_idx[_nfrm_given-i-1], "[ECR]" );
320 | }
321 | 
322 | 
323 | 
324 | /*------------------------------------------------------------------------
325 |  Perform change point detection either by solving group-fused LASSO of
326 |  "The group fused Lasso for multiple change-point detection" (2011) by
327 |  Kevin Bleakley and Jean-Philippe Vert, or by using heuristics
328 |  -----------------------------------------------------------------------*/
329 | void VideoParser::post_process(double min_shot_sec, bool gfl)
330 | /*-----------------------------------------------------------------------*/
331 | {
332 |   hecate::Segmenter seg;
333 |   int start_idx=-1, end_idx=-1, shotlen=-1;
334 |   int min_shot_len = min_shot_sec * _video_fps / _step_sz;
335 |   int max_shot_len = 3 * min_shot_len;
336 |   double thrsh_gfl = 0.25;
337 |   
338 |   for( size_t i=0; i<_v_frm_valid.size(); i++ )
339 |   {
340 |     if( start_idx<0 && _v_frm_valid[i] ) {
341 |       start_idx = i;
342 |     }
343 |     if( start_idx>=0 && (!_v_frm_valid[i] || i+1==_v_frm_valid.size()) )
344 |     {
345 |       end_idx = i;
346 |       shotlen = end_idx-start_idx+1;
347 |       if( shotlen >= max_shot_len )
348 |       {
349 |         int njumps = floor(shotlen/min_shot_len);
350 |         vector<int> jump;
351 |         
352 |         // Solve group-fused LASSO
353 |         if( gfl ) {
354 |           Mat Xsub( shotlen, _X_feat.cols, _X_feat.type() );
355 |           for( int r=start_idx; r<=end_idx; r++ ) {
356 |             _X_feat.row(r).copyTo( Xsub.row(r-start_idx) );
357 |           }
358 |           Xsub.convertTo(Xsub,CV_64F);
359 |           
360 |           seg.gflseg( Xsub, jump, njumps, thrsh_gfl );
361 |         }
362 |         // Use heuristics
363 |         else {
364 |           vector<double> v_diff;
365 |           for(int i=start_idx; i<=end_idx; i++) {
366 |             v_diff.push_back( _X_diff.at<double>(i) );
367 |           }
368 |           sbd_heuristic( v_diff, jump, njumps, min_shot_len );
369 |         }
370 |         
371 |         for(size_t k=0; k<jump.size(); k++) {
372 |           mark_invalid(_v_frm_valid, _v_frm_log, start_idx+jump[k]-1, "[GFL]" );
373 |           mark_invalid(_v_frm_valid, _v_frm_log, start_idx+jump[k], "[GFL]" );
374 |         }
375 |         
376 |         if( _debug )
377 |           printf("segmenter: %s (seqlen=%d (%d:%d),"
378 |                  "nJumpsEst=%d, nJumps=%d, theta=%.2f)\n",
379 |                  (gfl) ? "gflseg" : "heuristic",  shotlen, start_idx, end_idx,
380 |                  njumps, (int)jump.size(), thrsh_gfl);
381 |         
382 |       }
383 |       start_idx = end_idx = -1;
384 |     }
385 |   }
386 | }
387 | 
388 | /*-----------------------------------------------------------------------*/
389 | void VideoParser::sbd_heuristic(vector<double> v_diff, vector<int>& jump,
390 |                                 int njumps, int min_shot_len )
391 | /*-----------------------------------------------------------------------*/
392 | {
393 |   vector<size_t> v_srt_idx;
394 |   vector<double> v_srt_val; // contains sorted values
395 |   hecate::sort( v_diff, v_srt_val, v_srt_idx );
396 |   for(int i=(int)v_srt_val.size()-1; i>=0; i--) {
397 |     bool add = true;
398 |     if (((unsigned) v_srt_idx[i] + 1 < min_shot_len) ||
399 |         ((unsigned) v_diff.size() - v_srt_idx[i] < min_shot_len)) {
400 |       add = false;
401 |     } else {
402 |       for (size_t j = 0; j < jump.size(); j++) {
403 |         int len = abs(jump[j] - (int) v_srt_idx[i]) + 1;
404 |         if (len < min_shot_len) {
405 |           add = false;
406 |           break;
407 |         }
408 |       }
409 |     }
410 |     if( add ) {
411 |       jump.push_back( (int)v_srt_idx[i] );
412 |     }
413 |     if( (int)jump.size() == njumps ) {
414 |       break;
415 |     }
416 |   }
417 | };
418 | 
419 | /*-----------------------------------------------------------------------*/
420 | void VideoParser::extract_histo_features(int pyr_level, bool omit_filtered,
421 |                                          int nbin_color, int nbin_edge_ori,
422 |                                          int nbin_edge_mag)
423 | /*-----------------------------------------------------------------------*/
424 | {
425 |   int npatches = 0;
426 |   for(int l=0; l<pyr_level; l++)
427 |     npatches += pow(4,l);
428 |   
429 |   int nbin_edge = nbin_edge_ori + nbin_edge_mag;
430 |   Mat X_color_hist = Mat( npatches*3*nbin_color, _nfrm_given, CV_32F, Scalar(0,0,0)[0] );
431 |   Mat X_edge_hist  = Mat( npatches*nbin_edge, _nfrm_given, CV_32F, Scalar(0,0,0)[0] );
432 |   
433 | #pragma omp parallel for
434 |   for(int i=0; i<_nfrm_given; i++)
435 |   {
436 |     if( omit_filtered && !_v_frm_valid[i] ) continue;
437 |     
438 |     Mat color_hist;
439 |     hecate::calc_pyr_color_hist( _v_frm_rgb[i], color_hist, nbin_color, pyr_level );
440 |     color_hist.copyTo( X_color_hist.col(i) );
441 |     
442 |     Mat edge_hist;
443 |     hecate::calc_pyr_edge_hist( _v_frm_gray[i], edge_hist, nbin_edge_ori, nbin_edge_mag, pyr_level );
444 |     edge_hist.copyTo( X_edge_hist.col(i) );
445 |     
446 |   }
447 |   
448 |   // Transpose X_gray/edge_hist, X_hist = horzcat( X_color_hist, X_edge_hist )
449 |   X_color_hist = X_color_hist.t();
450 |   X_edge_hist = X_edge_hist.t();
451 |   hconcat( X_color_hist, X_edge_hist, _X_feat );
452 | }
453 | 
454 | 
455 | /*-----------------------------------------------------------------------*/
456 | void VideoParser::filter_redundant_and_obtain_subshots()
457 | /*-----------------------------------------------------------------------*/
458 | {
459 |   if( _v_shot_ranges.empty() )
460 |     update_shot_ranges();
461 |   
462 |   // Generate data matrix for kmeans, consider only valid frames
463 |   int nfrm_valid = get_nfrm_valid();
464 |   if( nfrm_valid==0 ) {
465 |     return;
466 |   }
467 | 
468 |   Mat km_data( nfrm_valid, _X_feat.cols, _X_feat.type() );
469 |   vector<int> v_idxmap( nfrm_valid, 0 );
470 |   
471 |   int row=0;
472 |   for(int i=0; i<_nfrm_given; i++)
473 |   {
474 |     if( _v_frm_valid[i] ) {
475 |       _X_feat.row(i).copyTo(km_data.row(row));
476 |       v_idxmap[row] = i;
477 |       row++;
478 |     }
479 |   }
480 |   
481 |   //
482 |   // Perform k-means
483 |   int ncluster = min(nfrm_valid/2, (int)_v_shot_ranges.size());
484 |   Mat km_lbl; // integer row vector; stores cluster IDs for every sample.
485 |   Mat km_ctr; // One row per each cluster center.
486 |   hecate::perform_kmeans( km_data, km_lbl, km_ctr, ncluster );
487 |   
488 |   //
489 |   // convert km_lbl to v_frm_clusterid
490 |   vector<int> v_frm_clusterid(_nfrm_given,-1);
491 |   for(int i=0; i<km_lbl.rows; i++)
492 |     v_frm_clusterid[ v_idxmap[i] ] = km_lbl.at<int>(i);
493 |   
494 |   //
495 |   // Pick the most "still" frame from every sub-shot within each shot
496 |   for(size_t shotid=0; shotid<_v_shot_ranges.size(); shotid++)
497 |   {
498 |     int sb0 = _v_shot_ranges[shotid].start;
499 |     int sb1 = _v_shot_ranges[shotid].end;
500 |     
501 |     // Identify sub-shots (per kmeans result)
502 |     int ssb0=-1, ssb1=-1, lbl=-1;
503 |     for( int j=sb0; j<=sb1; j++ )
504 |     {
505 |       if( _v_frm_valid[j] ) {
506 |         if( ssb0<0 ) { // enter a new sub-shot area
507 |           ssb0 = j;
508 |           lbl = v_frm_clusterid[j];
509 |         }
510 |         ssb1 = j;
511 |       }
512 |       if( ssb0>=0 && (v_frm_clusterid[j]!=lbl || j==sb1) )  // exit the current sub-shot area
513 |       {
514 |         int diff_min_idx=-1;
515 |         double diff_min_val = numeric_limits<double>::max();
516 |         for( int k=ssb0; k<=ssb1; k++ )
517 |         {
518 |           double diff_k = _X_diff.at<double>(k);
519 |           if( diff_k < diff_min_val )
520 |           {
521 |             diff_min_idx = k;
522 |             diff_min_val = diff_k;
523 |           }
524 |         }
525 |         
526 |         // Store subshot with keyframe index
527 |         hecate::Range r(ssb0, ssb1);
528 |         r.v_idx.push_back( diff_min_idx );
529 |         _v_shot_ranges[shotid].v_idx.push_back( diff_min_idx );
530 |         _v_shot_ranges[shotid].v_range.push_back( r );
531 |         
532 |         // Filter out redundant frames
533 |         for( int k=ssb0; k<=ssb1; k++ )
534 |           if( k!=diff_min_idx )
535 |             mark_invalid(_v_frm_valid, _v_frm_log, k, "[Redundant]");
536 |         
537 |         // reset sub-shot area
538 |         ssb0 = ssb1 = lbl = -1;
539 |       }
540 |     }
541 |     
542 |     // reset shot area
543 |     sb0 = sb1 = -1;
544 |   }
545 | }
546 | 
547 | 
548 | /*-----------------------------------------------------------------------*/
549 | void VideoParser::update_shot_ranges( int min_shot_len )
550 | /*-----------------------------------------------------------------------*/
551 | {
552 |   _v_shot_ranges.clear();
553 |   
554 |   int sb0=0, sb1=-1;
555 |   for( int i=0; i<_nfrm_given; i++ )
556 |   {
557 |     if( _v_frm_valid[i] ) {
558 |       // enter a new shot area
559 |       if( sb0<0 ) sb0 = i;
560 |       sb1 = i;
561 |     }
562 |     
563 |     // exit the current shot area
564 |     if( sb0>=0 && sb1>=0 && (!_v_frm_valid[i] || i+1==_nfrm_given) )
565 |     {
566 |       hecate::ShotRange r( sb0, sb1 );
567 |       if( r.length()>min_shot_len ) {
568 |         _v_shot_ranges.push_back( r );
569 |       }
570 |       else {
571 |         for(int j=sb0; j<=sb1; j++ )
572 |           mark_invalid( _v_frm_valid, _v_frm_log, j, "[Short]" );
573 |       }
574 |       sb0 = sb1 = -1;
575 |     }
576 |   }
577 | }
578 | 
579 | 
580 | /*-----------------------------------------------------------------------*/
581 | int VideoParser::get_nfrm_valid()
582 | /*-----------------------------------------------------------------------*/
583 | {
584 |   return (int) accumulate( _v_frm_valid.begin(), _v_frm_valid.end(), 0 );
585 | }
586 | 
587 | 
588 | /*-----------------------------------------------------------------------*/
589 | void VideoParser::mark_invalid( vector<bool>& vec, int idx, int wnd_sz )
590 | /*-----------------------------------------------------------------------*/
591 | {
592 |   int vec_len = (int)vec.size();
593 |   for(int i=max(0,idx-wnd_sz); i<=min(vec_len-1,idx+wnd_sz); i++) {
594 |     vec[i] = false;
595 |   }
596 | }
597 | 
598 | /*-----------------------------------------------------------------------*/
599 | void VideoParser::mark_invalid( vector<bool>& vec, vector<string>& vec2,
600 |                                int idx, const string msg, int wnd_sz )
601 | /*-----------------------------------------------------------------------*/
602 | {
603 |   int vec_len = (int)vec.size();
604 |   for(int i=max(0,idx-wnd_sz); i<=min(vec_len-1,idx+wnd_sz); i++) {
605 |     vec[i] = false;
606 |     vec2[i] = msg;
607 |   }
608 | }
609 | 
610 | 
611 | 
612 | /*-----------------------------------------------------------------------*/
613 | void VideoParser::play_video_filtered( const string& in_video,
614 |                                       int step_sz, int max_frm_len )
615 | /*-----------------------------------------------------------------------*/
616 | {
617 |   printf("\nDebug mode: Displaying shot segmentation results...\n");
618 |   
619 |   // Debug visualization
620 |   const char* WND = "DEBUG";
621 |   namedWindow( WND ,1 );
622 |   
623 |   Mat frm;
624 |   double debug_rsz_ratio = (double)max_frm_len/_video_width;
625 |   
626 |   VideoCapture vr( in_video );
627 |   vr >> frm;
628 |   resize( frm, frm, Size(), debug_rsz_ratio, debug_rsz_ratio, CV_INTER_LINEAR );
629 |   vr.set( CV_CAP_PROP_POS_FRAMES, 0 );
630 |   
631 |   Size sz = frm.size();
632 |   Mat frm_lr( sz.height, 2*sz.width, CV_8UC3 );
633 |   Mat frm_l(frm_lr, Rect(0,0,sz.width,sz.height));
634 |   Mat frm_r(frm_lr, Rect(sz.width,0,sz.width,sz.height));
635 |   frm_lr.setTo(Scalar(0));
636 |   
637 |   for( int i=0; i<_nfrm_given; i++)
638 |   {
639 |     for( int j=0; j<step_sz; j++ ) {
640 |       vr >> frm;
641 |       if( frm.empty() ) break;
642 |     }
643 |     if( frm.empty() ) break;
644 |     resize( frm, frm, Size(), debug_rsz_ratio, debug_rsz_ratio, CV_INTER_LINEAR );
645 |     
646 |     std::stringstream s;
647 |     s << _v_frm_log[i];
648 |     putText(frm, s.str(), Point2f(5,30), FONT_HERSHEY_PLAIN, 1.3, Scalar(255,0,255,255), 2);
649 |     
650 |     if( _v_frm_log[i].length()<2 ) {
651 |       frm.copyTo( frm_l );
652 |     }
653 |     else {
654 |       frm.copyTo( frm_r );
655 |     }
656 |     
657 |     imshow( WND, frm_lr );
658 |     if( waitKey(_video_fps)>=0 ) break;
659 |   }
660 |   vr.release();
661 |   destroyWindow( WND );
662 | }
663 | 
664 | 


--------------------------------------------------------------------------------