├── LICENSE ├── Makefile ├── README.md ├── examples ├── computeColor.m ├── extract_mpeg_flow.sh ├── flowToColor.m ├── mpi_sintel_final_alley_1.avi ├── mpi_sintel_final_alley_1_vis_hue_examples │ ├── 000003.png │ ├── 000015.png │ ├── 000050.png │ ├── gt_frame_0002.png │ ├── gt_frame_0014.png │ └── gt_frame_0049.png ├── vis_hue.m └── vis_mpeg_flow.sh ├── mpegflow.cpp └── vis.cpp /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Vadim Kantorov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CFLAGS = -O3 -D__STDC_CONSTANT_MACROS 2 | LDFLAGS = -lswscale -lavdevice -lavformat -lavcodec -lswresample -lavutil -lpthread -lbz2 -lz -lc -lrt 3 | INSTALLED_DEPS = -Idependencies/include -Ldependencies/lib 4 | 5 | mpegflow: mpegflow.cpp 6 | g++ $< -o $@ $(CFLAGS) $(LDFLAGS) $(INSTALLED_DEPS) 7 | 8 | vis: vis.cpp 9 | g++ $< -o $@ $(CFLAGS) -lopencv_highgui -lopencv_videoio -lopencv_imgproc -lopencv_imgcodecs -lopencv_core -lpng $(LDFLAGS) $(INSTALLED_DEPS) 10 | 11 | mpegflow.exe : mpegflow.cpp 12 | cl $? /MT /EHsc /I$(FFMPEG_DIR)\include /link avcodec.lib avformat.lib avutil.lib swscale.lib swresample.lib /LIBPATH:$(FFMPEG_DIR)\lib /OUT:$@ 13 | for %I in ($(FFMPEG_DIR:dev=shared)\bin\avutil-*.dll $(FFMPEG_DIR:dev=shared)\bin\avformat-*.dll $(FFMPEG_DIR:dev=shared)\bin\avcodec-*.dll $(FFMPEG_DIR:dev=shared)\bin\swresample-*.dll) do copy %I $(MAKEDIR) 14 | 15 | vis.exe: vis.cpp 16 | cl $? /MT /EHsc /I$(OPENCV_DIR)\..\..\include /link opencv_world320.lib /LIBPATH:$(OPENCV_DIR)\lib /OUT:$@ 17 | for %I in ($(OPENCV_DIR)\bin\opencv_world320.dll $(OPENCV_DIR)\bin\opencv_world320_64.dll) do copy %I $(MAKEDIR) 18 | 19 | clean: 20 | $(OS:Windows_NT=del) rm mpegflow vis *.exe *.obj *.dll 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Information & Contact 2 | 3 | We release **mpegflow** for easy extraction of MPEG-flow (motion vectors) from video files along with a visualization tool **vis**, both under the [MIT license](http://github.com/vadimkantorov/mpegflow/blob/master/LICENSE). We provide Makefiles for Linux / Windows and distribute [binary releases](http://github.com/vadimkantorov/mpegflow/releases). 4 | 5 | Please submit bugs on [GitHub](http://github.com/vadimkantorov/mpeflow/issues) directly. For any other question, please contact Vadim Kantorov at vadim.kantorov@inria.fr or vadim.kantorov@gmail.com. 6 | 7 | If you use this code, please cite our work: 8 | 9 | > @inproceedings{kantorov2014, 10 |       author = {Kantorov, V. and Laptev, I.}, 11 |       title = {Efficient feature extraction, encoding and classification for action recognition}, 12 |       booktitle = {Proc. Computer Vision and Pattern Recognition (CVPR), IEEE, 2014}, 13 |       year = {2014} 14 | } 15 | 16 | 17 | Below is a visualization of some flow maps of MPI Sintel's *alley_1* sequence: 18 | 19 | Source | 3 | 15 | 50 20 | --- | --- | --- | --- 21 | mpegflow | | | 22 | Ground truth | ![](https://raw.githubusercontent.com/vadimkantorov/mpegflow/master/examples/mpi_sintel_final_alley_1_vis_hue_examples/gt_frame_0002.png) | ![](https://raw.githubusercontent.com/vadimkantorov/mpegflow/master/examples/mpi_sintel_final_alley_1_vis_hue_examples/gt_frame_0014.png) | ![](https://raw.githubusercontent.com/vadimkantorov/mpegflow/master/examples/mpi_sintel_final_alley_1_vis_hue_examples/gt_frame_0049.png) 23 | 24 | # mpegflow 25 | The tool accepts a video file path as command-line argument and writes MPEG-flow (motion vectors) to standard output. By default, the tool outputs the motion vectors arranged in two matrices - dx and dy. 26 | 27 | ##### Command-line options: 28 | 29 | Option | Description 30 | --- | --- 31 | --help or -h | will output the usage info 32 | --raw | will prevent motion vectors from being arranged in matrices 33 | --grid8x8 | will force fine 8x8 grid 34 | --occupancy | will append occupancy matrix after motion vector matrices 35 | --quiet | will suppress debug output 36 | 37 | # vis 38 | The tool accepts a video file path and a dump directory as command-line arguments, output of **mpegflow** on standard input and saveson disk the visualization of motion vectors overlayed on video frames. 39 | 40 | ##### Command-line options: 41 | 42 | Option | Description 43 | --- | --- 44 | --help or -h | will output the usage info 45 | --occupancy | will expect occupancy information from **mpegflow** and will visualize it as well 46 | 47 | # Examples 48 | 49 | - To extract motion vectors: 50 | > ./mpegflow examples/mpi_sintel_final_alley_1.avi > examples/alley_1.txt 51 | 52 | - To visualize motion vectors: 53 | > mkdir -p examples/vis_dump && ./mpegflow examples/mpi_sintel_final_alley_1.avi | ./vis examples/mpi_sintel_final_alley_1.avi examples/vis_dump 54 | 55 | Feel free to copy-paste and run the examples above. More runnable examples are in ```examples/extract_motion_vectors.sh``` and ```examples/vis_motion_vectors.sh```. Feel free to use ```vis.cpp``` and ```examples/vis_hue.m``` as examples of parsing **mpegflow** output. ```examples/vis_hue``` can also be used to produce hue flow visualizations like above. 56 | 57 | # Building from source 58 | **mpegflow** depends only on a recent FFmpeg, **vis** depends on FFmpeg, OpenCV and libpng. The tools are known to work with FFmpeg 3.1 and OpenCV 3.1. 59 | 60 | Once the dependencies are visible to g++, run: 61 | ```bash 62 | make # to build mpegflow 63 | make vis # to build vis 64 | ``` 65 | 66 | You will probably end up with a shared build; for a static build, please feel free to play with Makefile. 67 | 68 | To build the tools on Windows: 69 | 70 | 1. Create directory `dependencies` 71 | 2. Extract FFmpeg dev and shared [builds](http://ffmpeg.zeranoe.com/builds/) to the `dependencies` directory (for mpegflow and vis) 72 | 3. Extract an OpenCV 3.x build to the `dependencies` directory (for vis) 73 | 4. Open VS2015 x64 Native Tools Command Prompt (VS2015 Community Edition will work) and run: 74 | 75 | ```shell 76 | # fix the paths and versions before running 77 | 78 | nmake mpegflow.exe FFMPEG_DIR=dependencies\ffmpeg-3.0.1-win64-dev\ffmpeg-3.0.1-win64-dev 79 | # nmake vis.exe OPENCV_DIR=dependencies\opencv-3.1.0\opencv\build\x64\vc14 80 | ``` 81 | 82 | 6. The Windows build is not fully static. You need to keep `avutil-54.dll`, `avformat-56.dll`, `avcodec-56.dll`, `swresample-2.dll` (for **mpegflow**) and `opencv_world310.dll` (for **vis**) in the same directory as the binary. Note that the instructions and the Makefile assume x64 machine architecture. 83 | -------------------------------------------------------------------------------- /examples/computeColor.m: -------------------------------------------------------------------------------- 1 | function img = computeColor(u,v) 2 | 3 | % computeColor color codes flow field U, V 4 | 5 | % According to the c++ source code of Daniel Scharstein 6 | % Contact: schar@middlebury.edu 7 | 8 | % Author: Deqing Sun, Department of Computer Science, Brown University 9 | % Contact: dqsun@cs.brown.edu 10 | % $Date: 2007-10-31 21:20:30 (Wed, 31 Oct 2006) $ 11 | 12 | % Copyright 2007, Deqing Sun. 13 | % 14 | % All Rights Reserved 15 | % 16 | % Permission to use, copy, modify, and distribute this software and its 17 | % documentation for any purpose other than its incorporation into a 18 | % commercial product is hereby granted without fee, provided that the 19 | % above copyright notice appear in all copies and that both that 20 | % copyright notice and this permission notice appear in supporting 21 | % documentation, and that the name of the author and Brown University not be used in 22 | % advertising or publicity pertaining to distribution of the software 23 | % without specific, written prior permission. 24 | % 25 | % THE AUTHOR AND BROWN UNIVERSITY DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, 26 | % INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ANY 27 | % PARTICULAR PURPOSE. IN NO EVENT SHALL THE AUTHOR OR BROWN UNIVERSITY BE LIABLE FOR 28 | % ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 29 | % WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 30 | % ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 31 | % OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 32 | 33 | nanIdx = isnan(u) | isnan(v); 34 | u(nanIdx) = 0; 35 | v(nanIdx) = 0; 36 | 37 | colorwheel = makeColorwheel(); 38 | ncols = size(colorwheel, 1); 39 | 40 | rad = sqrt(u.^2+v.^2); 41 | 42 | a = atan2(-v, -u)/pi; 43 | 44 | fk = (a+1) /2 * (ncols-1) + 1; % -1~1 maped to 1~ncols 45 | 46 | k0 = floor(fk); % 1, 2, ..., ncols 47 | 48 | k1 = k0+1; 49 | k1(k1==ncols+1) = 1; 50 | 51 | f = fk - k0; 52 | 53 | for i = 1:size(colorwheel,2) 54 | tmp = colorwheel(:,i); 55 | col0 = tmp(k0)/255; 56 | col1 = tmp(k1)/255; 57 | col = (1-f).*col0 + f.*col1; 58 | 59 | idx = rad <= 1; 60 | col(idx) = 1-rad(idx).*(1-col(idx)); % increase saturation with radius 61 | 62 | col(~idx) = col(~idx)*0.75; % out of range 63 | 64 | img(:,:, i) = uint8(floor(255*col.*(1-nanIdx))); 65 | end; 66 | 67 | %% 68 | function colorwheel = makeColorwheel() 69 | 70 | % color encoding scheme 71 | 72 | % adapted from the color circle idea described at 73 | % http://members.shaw.ca/quadibloc/other/colint.htm 74 | 75 | 76 | RY = 15; 77 | YG = 6; 78 | GC = 4; 79 | CB = 11; 80 | BM = 13; 81 | MR = 6; 82 | 83 | ncols = RY + YG + GC + CB + BM + MR; 84 | 85 | colorwheel = zeros(ncols, 3); % r g b 86 | 87 | col = 0; 88 | %RY 89 | colorwheel(1:RY, 1) = 255; 90 | colorwheel(1:RY, 2) = floor(255*(0:RY-1)/RY)'; 91 | col = col+RY; 92 | 93 | %YG 94 | colorwheel(col+(1:YG), 1) = 255 - floor(255*(0:YG-1)/YG)'; 95 | colorwheel(col+(1:YG), 2) = 255; 96 | col = col+YG; 97 | 98 | %GC 99 | colorwheel(col+(1:GC), 2) = 255; 100 | colorwheel(col+(1:GC), 3) = floor(255*(0:GC-1)/GC)'; 101 | col = col+GC; 102 | 103 | %CB 104 | colorwheel(col+(1:CB), 2) = 255 - floor(255*(0:CB-1)/CB)'; 105 | colorwheel(col+(1:CB), 3) = 255; 106 | col = col+CB; 107 | 108 | %BM 109 | colorwheel(col+(1:BM), 3) = 255; 110 | colorwheel(col+(1:BM), 1) = floor(255*(0:BM-1)/BM)'; 111 | col = col+BM; 112 | 113 | %MR 114 | colorwheel(col+(1:MR), 3) = 255 - floor(255*(0:MR-1)/MR)'; 115 | colorwheel(col+(1:MR), 1) = 255; -------------------------------------------------------------------------------- /examples/extract_mpeg_flow.sh: -------------------------------------------------------------------------------- 1 | # extract motion vectors 2 | ../mpegflow mpi_sintel_final_alley_1.avi > alley_1.txt 3 | 4 | # extract motion vectors and occupancy info 5 | ../mpegflow --occupancy mpi_sintel_final_alley_1.avi > alley_1_occupancy.txt 6 | 7 | # extract motion vectors in raw format 8 | ../mpegflow --raw mpi_sintel_final_alley_1.avi > alley_1_raw.txt 9 | 10 | # extract motion vectors with enforced 8x8 grid 11 | ../mpegflow --grid8x8 mpi_sintel_final_alley_1.avi > alley_1_grid8x8.txt 12 | -------------------------------------------------------------------------------- /examples/flowToColor.m: -------------------------------------------------------------------------------- 1 | function img = flowToColor(flow, varargin) 2 | 3 | % flowToColor(flow, maxFlow) flowToColor color codes flow field, normalize 4 | % based on specified value, 5 | % 6 | % flowToColor(flow) flowToColor color codes flow field, normalize 7 | % based on maximum flow present otherwise 8 | 9 | % According to the c++ source code of Daniel Scharstein 10 | % Contact: schar@middlebury.edu 11 | 12 | % Author: Deqing Sun, Department of Computer Science, Brown University 13 | % Contact: dqsun@cs.brown.edu 14 | % $Date: 2007-10-31 18:33:30 (Wed, 31 Oct 2006) $ 15 | 16 | % Copyright 2007, Deqing Sun. 17 | % 18 | % All Rights Reserved 19 | % 20 | % Permission to use, copy, modify, and distribute this software and its 21 | % documentation for any purpose other than its incorporation into a 22 | % commercial product is hereby granted without fee, provided that the 23 | % above copyright notice appear in all copies and that both that 24 | % copyright notice and this permission notice appear in supporting 25 | % documentation, and that the name of the author and Brown University not be used in 26 | % advertising or publicity pertaining to distribution of the software 27 | % without specific, written prior permission. 28 | % 29 | % THE AUTHOR AND BROWN UNIVERSITY DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, 30 | % INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ANY 31 | % PARTICULAR PURPOSE. IN NO EVENT SHALL THE AUTHOR OR BROWN UNIVERSITY BE LIABLE FOR 32 | % ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 33 | % WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 34 | % ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 35 | % OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 36 | 37 | UNKNOWN_FLOW_THRESH = 1e9; 38 | UNKNOWN_FLOW = 1e10; % 39 | 40 | [height widht nBands] = size(flow); 41 | 42 | if nBands ~= 2 43 | error('flowToColor: image must have two bands'); 44 | end; 45 | 46 | u = flow(:,:,1); 47 | v = flow(:,:,2); 48 | 49 | maxu = -999; 50 | maxv = -999; 51 | 52 | minu = 999; 53 | minv = 999; 54 | maxrad = -1; 55 | 56 | % fix unknown flow 57 | idxUnknown = (abs(u)> UNKNOWN_FLOW_THRESH) | (abs(v)> UNKNOWN_FLOW_THRESH) ; 58 | u(idxUnknown) = 0; 59 | v(idxUnknown) = 0; 60 | 61 | maxu = max(maxu, max(u(:))); 62 | minu = min(minu, min(u(:))); 63 | 64 | maxv = max(maxv, max(v(:))); 65 | minv = min(minv, min(v(:))); 66 | 67 | rad = sqrt(u.^2+v.^2); 68 | maxrad = max(maxrad, max(rad(:))); 69 | 70 | fprintf('max flow: %.4f flow range: u = %.3f .. %.3f; v = %.3f .. %.3f\n', maxrad, minu, maxu, minv, maxv); 71 | 72 | if isempty(varargin) ==0 73 | maxFlow = varargin{1}; 74 | if maxFlow > 0 75 | maxrad = maxFlow; 76 | end; 77 | end; 78 | 79 | u = u/(maxrad+eps); 80 | v = v/(maxrad+eps); 81 | 82 | % compute color 83 | 84 | img = computeColor(u, v); 85 | 86 | % unknown flow 87 | IDX = repmat(idxUnknown, [1 1 3]); 88 | img(IDX) = 0; -------------------------------------------------------------------------------- /examples/mpi_sintel_final_alley_1.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vadimkantorov/mpegflow/27662e31458f4cb1654bbd30817e416be23f8ae4/examples/mpi_sintel_final_alley_1.avi -------------------------------------------------------------------------------- /examples/mpi_sintel_final_alley_1_vis_hue_examples/000003.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vadimkantorov/mpegflow/27662e31458f4cb1654bbd30817e416be23f8ae4/examples/mpi_sintel_final_alley_1_vis_hue_examples/000003.png -------------------------------------------------------------------------------- /examples/mpi_sintel_final_alley_1_vis_hue_examples/000015.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vadimkantorov/mpegflow/27662e31458f4cb1654bbd30817e416be23f8ae4/examples/mpi_sintel_final_alley_1_vis_hue_examples/000015.png -------------------------------------------------------------------------------- /examples/mpi_sintel_final_alley_1_vis_hue_examples/000050.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vadimkantorov/mpegflow/27662e31458f4cb1654bbd30817e416be23f8ae4/examples/mpi_sintel_final_alley_1_vis_hue_examples/000050.png -------------------------------------------------------------------------------- /examples/mpi_sintel_final_alley_1_vis_hue_examples/gt_frame_0002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vadimkantorov/mpegflow/27662e31458f4cb1654bbd30817e416be23f8ae4/examples/mpi_sintel_final_alley_1_vis_hue_examples/gt_frame_0002.png -------------------------------------------------------------------------------- /examples/mpi_sintel_final_alley_1_vis_hue_examples/gt_frame_0014.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vadimkantorov/mpegflow/27662e31458f4cb1654bbd30817e416be23f8ae4/examples/mpi_sintel_final_alley_1_vis_hue_examples/gt_frame_0014.png -------------------------------------------------------------------------------- /examples/mpi_sintel_final_alley_1_vis_hue_examples/gt_frame_0049.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vadimkantorov/mpegflow/27662e31458f4cb1654bbd30817e416be23f8ae4/examples/mpi_sintel_final_alley_1_vis_hue_examples/gt_frame_0049.png -------------------------------------------------------------------------------- /examples/vis_hue.m: -------------------------------------------------------------------------------- 1 | % The script requires the training sequences of the MPI Sintel dataset (http://sintel.is.tue.mpg.de/). 2 | % flowToColor and computeColor are originally part of the MPI Sintel's bundler. 3 | % The script has originally been used to produce visualizations for https://github.com/vadimkantorov/mpegflow/blob/master/README.md, but you could use it on other videos by adjusting VIDEO_PATH 4 | 5 | VIDEO_PATH = 'mpi_sintel_final_alley_1.avi'; 6 | 7 | % system(sprintf('ffmpeg -i training/final/alley_1/frame_%%04d.png -q:v 1.0 %s', VIDEO_PATH)); 8 | 9 | system(sprintf('../mpegflow "%s" > vis_hue_flow.txt', VIDEO_PATH)); 10 | system('mkdir -p vis_hue_dump'); 11 | 12 | f = fopen('vis_hue_flow.txt', 'r'); 13 | while true 14 | frameIndex_rows_cols = fscanf(f, '# pts=%*d frame_index=%d pict_type=%*c output_type=%*s shape=%dx%d origin=%*s\n'); 15 | if isempty(frameIndex_rows_cols) 16 | break 17 | end 18 | 19 | dxdy = fscanf(f, '%d ', fliplr(frameIndex_rows_cols(2:end)'))'; 20 | flow = cat(3, dxdy(1:size(dxdy, 1) / 2, :), dxdy(size(dxdy, 1) / 2 + 1 : end, :)); 21 | img = flowToColor(flow); 22 | imwrite(img, sprintf('vis_hue_dump/%06d.png', frameIndex_rows_cols(1))); 23 | end 24 | fclose(f); 25 | -------------------------------------------------------------------------------- /examples/vis_mpeg_flow.sh: -------------------------------------------------------------------------------- 1 | # visualize motion vectors 2 | mkdir -p vis_dump 3 | ../mpegflow mpi_sintel_final_alley_1.avi | ../vis mpi_sintel_final_alley_1.avi vis_dump 4 | 5 | # visualize motion vectors and occupancy info 6 | mkdir -p vis_dump 7 | ../mpegflow --occupancy mpi_sintel_final_alley_1.avi | ../vis --occupancy mpi_sintel_final_alley.avi vis_dump 8 | -------------------------------------------------------------------------------- /mpegflow.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | extern "C" 8 | { 9 | #include 10 | #include 11 | #include 12 | #include 13 | } 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | using namespace std; 21 | 22 | AVFrame* ffmpeg_pFrame; 23 | AVFormatContext* ffmpeg_pFormatCtx; 24 | AVStream* ffmpeg_pVideoStream; 25 | int ffmpeg_videoStreamIndex; 26 | size_t ffmpeg_frameWidth, ffmpeg_frameHeight; 27 | 28 | bool ARG_OUTPUT_RAW_MOTION_VECTORS, ARG_FORCE_GRID_8, ARG_FORCE_GRID_16, ARG_OUTPUT_OCCUPANCY, ARG_QUIET, ARG_HELP; 29 | const char* ARG_VIDEO_PATH; 30 | 31 | void ffmpeg_print_error(int err) // copied from cmdutils.c, originally called print_error 32 | { 33 | char errbuf[128]; 34 | const char *errbuf_ptr = errbuf; 35 | 36 | if (av_strerror(err, errbuf, sizeof(errbuf)) < 0) 37 | errbuf_ptr = strerror(AVUNERROR(err)); 38 | av_log(NULL, AV_LOG_ERROR, "ffmpeg_print_error: %s\n", errbuf_ptr); 39 | } 40 | 41 | void ffmpeg_log_callback_null(void *ptr, int level, const char *fmt, va_list vl) 42 | { 43 | } 44 | 45 | void ffmpeg_init() 46 | { 47 | av_register_all(); 48 | 49 | if(ARG_QUIET) 50 | { 51 | av_log_set_level(AV_LOG_ERROR); 52 | av_log_set_callback(ffmpeg_log_callback_null); 53 | } 54 | 55 | ffmpeg_pFrame = av_frame_alloc(); 56 | ffmpeg_pFormatCtx = avformat_alloc_context(); 57 | ffmpeg_videoStreamIndex = -1; 58 | 59 | int err = 0; 60 | 61 | if ((err = avformat_open_input(&ffmpeg_pFormatCtx, ARG_VIDEO_PATH, NULL, NULL)) != 0) 62 | { 63 | ffmpeg_print_error(err); 64 | throw std::runtime_error("Couldn't open file. Possibly it doesn't exist."); 65 | } 66 | 67 | if ((err = avformat_find_stream_info(ffmpeg_pFormatCtx, NULL)) < 0) 68 | { 69 | ffmpeg_print_error(err); 70 | throw std::runtime_error("Stream information not found."); 71 | } 72 | 73 | for(int i = 0; i < ffmpeg_pFormatCtx->nb_streams; i++) 74 | { 75 | AVCodecContext *enc = ffmpeg_pFormatCtx->streams[i]->codec; 76 | if( AVMEDIA_TYPE_VIDEO == enc->codec_type && ffmpeg_videoStreamIndex < 0 ) 77 | { 78 | AVCodec *pCodec = avcodec_find_decoder(enc->codec_id); 79 | AVDictionary *opts = NULL; 80 | av_dict_set(&opts, "flags2", "+export_mvs", 0); 81 | if (!pCodec || avcodec_open2(enc, pCodec, &opts) < 0) 82 | throw std::runtime_error("Codec not found or cannot open codec."); 83 | 84 | ffmpeg_videoStreamIndex = i; 85 | ffmpeg_pVideoStream = ffmpeg_pFormatCtx->streams[i]; 86 | ffmpeg_frameWidth = enc->width; 87 | ffmpeg_frameHeight = enc->height; 88 | 89 | break; 90 | } 91 | } 92 | 93 | if(ffmpeg_videoStreamIndex == -1) 94 | throw std::runtime_error("Video stream not found."); 95 | } 96 | 97 | bool process_frame(AVPacket *pkt) 98 | { 99 | av_frame_unref(ffmpeg_pFrame); 100 | 101 | int got_frame = 0; 102 | int ret = avcodec_decode_video2(ffmpeg_pVideoStream->codec, ffmpeg_pFrame, &got_frame, pkt); 103 | if (ret < 0) 104 | return false; 105 | 106 | ret = FFMIN(ret, pkt->size); /* guard against bogus return values */ 107 | pkt->data += ret; 108 | pkt->size -= ret; 109 | return got_frame > 0; 110 | } 111 | 112 | bool read_packets() 113 | { 114 | static bool initialized = false; 115 | static AVPacket pkt, pktCopy; 116 | 117 | while(true) 118 | { 119 | if(initialized) 120 | { 121 | if(process_frame(&pktCopy)) 122 | return true; 123 | else 124 | { 125 | av_packet_unref(&pkt); 126 | initialized = false; 127 | } 128 | } 129 | 130 | int ret = av_read_frame(ffmpeg_pFormatCtx, &pkt); 131 | if(ret != 0) 132 | break; 133 | 134 | initialized = true; 135 | pktCopy = pkt; 136 | if(pkt.stream_index != ffmpeg_videoStreamIndex) 137 | { 138 | av_packet_unref(&pkt); 139 | initialized = false; 140 | continue; 141 | } 142 | } 143 | 144 | return process_frame(&pkt); 145 | } 146 | 147 | bool read_frame(int64_t& pts, char& pictType, vector& motion_vectors) 148 | { 149 | if(!read_packets()) 150 | return false; 151 | 152 | pictType = av_get_picture_type_char(ffmpeg_pFrame->pict_type); 153 | // fragile, consult fresh f_select.c and ffprobe.c when updating ffmpeg 154 | pts = ffmpeg_pFrame->pkt_pts != AV_NOPTS_VALUE ? ffmpeg_pFrame->pkt_pts : (ffmpeg_pFrame->pkt_dts != AV_NOPTS_VALUE ? ffmpeg_pFrame->pkt_dts : pts + 1); 155 | bool noMotionVectors = av_frame_get_side_data(ffmpeg_pFrame, AV_FRAME_DATA_MOTION_VECTORS) == NULL; 156 | if(!noMotionVectors) 157 | { 158 | // reading motion vectors, see ff_print_debug_info2 in ffmpeg's libavcodec/mpegvideo.c for reference and a fresh doc/examples/extract_mvs.c 159 | AVFrameSideData* sd = av_frame_get_side_data(ffmpeg_pFrame, AV_FRAME_DATA_MOTION_VECTORS); 160 | AVMotionVector* mvs = (AVMotionVector*)sd->data; 161 | int mvcount = sd->size / sizeof(AVMotionVector); 162 | motion_vectors = vector(mvs, mvs + mvcount); 163 | } 164 | else 165 | { 166 | motion_vectors = vector(); 167 | } 168 | 169 | return true; 170 | } 171 | 172 | struct FrameInfo 173 | { 174 | const static size_t MAX_GRID_SIZE = 512; 175 | 176 | size_t GridStep; 177 | pair Shape; 178 | 179 | int dx[MAX_GRID_SIZE][MAX_GRID_SIZE]; 180 | int dy[MAX_GRID_SIZE][MAX_GRID_SIZE]; 181 | uint8_t occupancy[MAX_GRID_SIZE][MAX_GRID_SIZE]; 182 | int64_t Pts; 183 | int FrameIndex; 184 | char PictType; 185 | const char* Origin; 186 | bool Empty; 187 | bool Printed; 188 | 189 | FrameInfo() 190 | { 191 | memset(dx, 0, sizeof(dx)); 192 | memset(dy, 0, sizeof(dy)); 193 | memset(occupancy, 0, sizeof(occupancy)); 194 | Empty = true; 195 | Printed = false; 196 | PictType = '?'; 197 | FrameIndex = -1; 198 | Pts = -1; 199 | Origin = ""; 200 | } 201 | 202 | void InterpolateFlow(FrameInfo& prev, FrameInfo& next) 203 | { 204 | for(int i = 0; i < Shape.first; i++) 205 | { 206 | for(int j = 0; j < Shape.second; j++) 207 | { 208 | dx[i][j] = (prev.dx[i][j] + next.dx[i][j]) / 2; 209 | dy[i][j] = (prev.dy[i][j] + next.dy[i][j]) / 2; 210 | } 211 | } 212 | Empty = false; 213 | Origin = "interpolated"; 214 | } 215 | 216 | void FillInSomeMissingVectorsInGrid8() 217 | { 218 | for(int k = 0; k < 2; k++) 219 | { 220 | for(int i = 1; i < Shape.first - 1; i++) 221 | { 222 | for(int j = 1; j < Shape.second - 1; j++) 223 | { 224 | if(occupancy[i][j] == 0) 225 | { 226 | if(occupancy[i][j - 1] != 0 && occupancy[i][j + 1] != 0) 227 | { 228 | dx[i][j] = (dx[i][j -1] + dx[i][j + 1]) / 2; 229 | dy[i][j] = (dy[i][j -1] + dy[i][j + 1]) / 2; 230 | occupancy[i][j] = 2; 231 | } 232 | else if(occupancy[i - 1][j] != 0 && occupancy[i + 1][j] != 0) 233 | { 234 | dx[i][j] = (dx[i - 1][j] + dx[i + 1][j]) / 2; 235 | dy[i][j] = (dy[i - 1][j] + dy[i + 1][j]) / 2; 236 | occupancy[i][j] = 2; 237 | } 238 | } 239 | } 240 | } 241 | } 242 | } 243 | 244 | void PrintIfNotPrinted() 245 | { 246 | static int64_t FirstPts = -1; 247 | 248 | if(Printed) 249 | return; 250 | 251 | if(FirstPts == -1) 252 | FirstPts = Pts; 253 | 254 | printf("# pts=%lld frame_index=%d pict_type=%c output_type=arranged shape=%zux%zu origin=%s\n", (long long) Pts - FirstPts, FrameIndex, PictType, (ARG_OUTPUT_OCCUPANCY ? 3 : 2) * Shape.first, Shape.second, Origin); 255 | for(int i = 0; i < Shape.first; i++) 256 | { 257 | for(int j = 0; j < Shape.second; j++) 258 | { 259 | printf("%d\t", dx[i][j]); 260 | } 261 | printf("\n"); 262 | } 263 | for(int i = 0; i < Shape.first; i++) 264 | { 265 | for(int j = 0; j < Shape.second; j++) 266 | { 267 | printf("%d\t", dy[i][j]); 268 | } 269 | printf("\n"); 270 | } 271 | 272 | if(ARG_OUTPUT_OCCUPANCY) 273 | { 274 | for(int i = 0; i < Shape.first; i++) 275 | { 276 | for(int j = 0; j < Shape.second; j++) 277 | { 278 | printf("%d\t", occupancy[i][j]); 279 | } 280 | printf("\n"); 281 | } 282 | } 283 | 284 | Printed = true; 285 | } 286 | }; 287 | 288 | const size_t FrameInfo::MAX_GRID_SIZE; 289 | 290 | void output_vectors_raw(int frameIndex, int64_t pts, char pictType, vector& motionVectors) 291 | { 292 | printf("# pts=%lld frame_index=%d pict_type=%c output_type=raw shape=%zux4\n", (long long) pts, frameIndex, pictType, motionVectors.size()); 293 | for(int i = 0; i < motionVectors.size(); i++) 294 | { 295 | AVMotionVector& mv = motionVectors[i]; 296 | int mvdx = mv.dst_x - mv.src_x; 297 | int mvdy = mv.dst_y - mv.src_y; 298 | 299 | printf("%d\t%d\t%d\t%d\n", mv.dst_x, mv.dst_y, mvdx, mvdy); 300 | } 301 | } 302 | 303 | void output_vectors_std(int frameIndex, int64_t pts, char pictType, vector& motionVectors) 304 | { 305 | static vector prev; 306 | 307 | size_t gridStep = ARG_FORCE_GRID_8 ? 8 : 16; 308 | pair shape = make_pair(min(ffmpeg_frameHeight / gridStep, FrameInfo::MAX_GRID_SIZE), min(ffmpeg_frameWidth / gridStep, FrameInfo::MAX_GRID_SIZE)); 309 | 310 | if(!prev.empty() && pts != prev.back().Pts + 1) 311 | { 312 | for(int64_t dummy_pts = prev.back().Pts + 1; dummy_pts < pts; dummy_pts++) 313 | { 314 | FrameInfo dummy; 315 | dummy.FrameIndex = -1; 316 | dummy.Pts = dummy_pts; 317 | dummy.Origin = "dummy"; 318 | dummy.PictType = '?'; 319 | dummy.GridStep = gridStep; 320 | dummy.Shape = shape; 321 | prev.push_back(dummy); 322 | } 323 | } 324 | 325 | FrameInfo cur; 326 | cur.FrameIndex = frameIndex; 327 | cur.Pts = pts; 328 | cur.Origin = "video"; 329 | cur.PictType = pictType; 330 | cur.GridStep = gridStep; 331 | cur.Shape = shape; 332 | 333 | for(int i = 0; i < motionVectors.size(); i++) 334 | { 335 | AVMotionVector& mv = motionVectors[i]; 336 | int mvdx = mv.dst_x - mv.src_x; 337 | int mvdy = mv.dst_y - mv.src_y; 338 | 339 | size_t i_clipped = max(size_t(0), min(mv.dst_y / cur.GridStep, cur.Shape.first - 1)); 340 | size_t j_clipped = max(size_t(0), min(mv.dst_x / cur.GridStep, cur.Shape.second - 1)); 341 | 342 | cur.Empty = false; 343 | cur.dx[i_clipped][j_clipped] = mvdx; 344 | cur.dy[i_clipped][j_clipped] = mvdy; 345 | cur.occupancy[i_clipped][j_clipped] = true; 346 | } 347 | 348 | if(cur.GridStep == 8) 349 | cur.FillInSomeMissingVectorsInGrid8(); 350 | 351 | if(frameIndex == -1) 352 | { 353 | for(int i = 0; i < prev.size(); i++) 354 | prev[i].PrintIfNotPrinted(); 355 | } 356 | else if(!motionVectors.empty()) 357 | { 358 | if(prev.size() == 2 && prev.front().Empty == false) 359 | { 360 | prev.back().InterpolateFlow(prev.front(), cur); 361 | prev.back().PrintIfNotPrinted(); 362 | } 363 | else 364 | { 365 | for(int i = 0; i < prev.size(); i++) 366 | prev[i].PrintIfNotPrinted(); 367 | } 368 | prev.clear(); 369 | cur.PrintIfNotPrinted(); 370 | } 371 | 372 | prev.push_back(cur); 373 | } 374 | 375 | void parse_options(int argc, const char* argv[]) 376 | { 377 | for(int i = 1; i < argc; i++) 378 | { 379 | if(strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) 380 | ARG_HELP = true; 381 | else if(strcmp(argv[i], "--raw") == 0) 382 | ARG_OUTPUT_RAW_MOTION_VECTORS = true; 383 | else if(strcmp(argv[i], "--grid8x8") == 0) 384 | ARG_FORCE_GRID_8 = true; 385 | else if(strcmp(argv[i], "--occupancy") == 0) 386 | ARG_OUTPUT_OCCUPANCY = true; 387 | else if(strcmp(argv[i], "-q") == 0 || strcmp(argv[i], "--quiet") == 0) 388 | ARG_QUIET = true; 389 | else 390 | ARG_VIDEO_PATH = argv[i]; 391 | } 392 | if(ARG_HELP || ARG_VIDEO_PATH == NULL) 393 | { 394 | fprintf(stderr, "Usage: mpegflow [--raw | [[--grid8x8] [--occupancy]]] videoPath\n --help and -h will output this help message.\n --raw will prevent motion vectors from being arranged in matrices.\n --grid8x8 will force fine 8x8 grid.\n --occupancy will append occupancy matrix after motion vector matrices.\n --quiet will suppress debug output.\n"); 395 | exit(1); 396 | } 397 | } 398 | 399 | int main(int argc, const char* argv[]) 400 | { 401 | parse_options(argc, argv); 402 | ffmpeg_init(); 403 | 404 | int64_t pts, prev_pts = -1; 405 | char pictType; 406 | vector motionVectors; 407 | 408 | for(int frameIndex = 1; read_frame(pts, pictType, motionVectors); frameIndex++) 409 | { 410 | if(pts <= prev_pts && prev_pts != -1) 411 | { 412 | if(!ARG_QUIET) 413 | fprintf(stderr, "Skipping frame %d (frame with pts %d already processed).\n", int(frameIndex), int(pts)); 414 | continue; 415 | } 416 | 417 | if(ARG_OUTPUT_RAW_MOTION_VECTORS) 418 | output_vectors_raw(frameIndex, pts, pictType, motionVectors); 419 | else 420 | output_vectors_std(frameIndex, pts, pictType, motionVectors); 421 | 422 | prev_pts = pts; 423 | } 424 | if(ARG_OUTPUT_RAW_MOTION_VECTORS == false) 425 | output_vectors_std(-1, pts, pictType, motionVectors); 426 | } 427 | -------------------------------------------------------------------------------- /vis.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | using namespace std; 16 | using namespace cv; 17 | 18 | const char* ARG_VIDEO_PATH = NULL; 19 | const char* ARG_DUMP_DIR = NULL; 20 | bool ARG_HELP, ARG_OCCUPANCY; 21 | 22 | void draw_arrow(Mat img, Point pStart, Point pEnd, double len, double alphaDegrees, Scalar lineColor, Scalar startColor) 23 | { 24 | const double PI = acos(-1); 25 | const int lineThickness = 1; 26 | const int lineType = CV_AA; 27 | 28 | double angle = atan2((double)(pStart.y - pEnd.y), (double)(pStart.x - pEnd.x)); 29 | line(img, pStart, pEnd, lineColor, lineThickness, lineType); 30 | img.at(pStart) = Vec3b(startColor[0], startColor[1], startColor[2]); 31 | if(len > 0) 32 | { 33 | for(int k = 0; k < 2; k++) 34 | { 35 | int sign = k == 1 ? 1 : -1; 36 | Point arrow(pEnd.x + len * cos(angle + sign * PI * alphaDegrees / 180), pEnd.y + len * sin(angle + sign * PI * alphaDegrees / 180)); 37 | line(img, pEnd, arrow, lineColor, lineThickness, lineType); 38 | } 39 | } 40 | } 41 | 42 | void vis_flow(pair flow, Mat frame, const char* dumpDir) 43 | { 44 | Mat flowComponents[3]; 45 | split(flow.first, flowComponents); 46 | int rows = flowComponents[0].rows; 47 | int cols = flowComponents[0].cols; 48 | 49 | Mat img = frame.clone(); 50 | for(int i = 0; i < rows; i++) 51 | { 52 | for(int j = 0; j < cols; j++) 53 | { 54 | int dx = flowComponents[0].at(i, j); 55 | int dy = flowComponents[1].at(i, j); 56 | int occupancy = flowComponents[2].at(i, j); 57 | 58 | Point start(double(j) / cols * img.cols + img.cols / cols / 2, double(i) / rows * img.rows + img.rows / rows / 2); 59 | Point end(start.x + dx, start.y + dy); 60 | 61 | draw_arrow(img, start, end, 2.0, 20.0, CV_RGB(255, 0, 0), (occupancy == 1 || occupancy == 2) ? CV_RGB(0, 255, 0) : CV_RGB(0, 255, 255)); 62 | } 63 | } 64 | 65 | stringstream s; 66 | s << dumpDir << "/" << setfill('0') << setw(6) << flow.second << ".png"; 67 | imwrite(s.str(), img); 68 | } 69 | 70 | pair read_flow() 71 | { 72 | int rows, cols, frameIndex; 73 | bool ok = scanf("# pts=%*d frame_index=%d pict_type=%*c output_type=%*s shape=%dx%d origin=%*s\n", &frameIndex, &rows, &cols) == 3; 74 | int D = ARG_OCCUPANCY ? 3 : 2; 75 | 76 | if(!(ok && rows % D == 0)) 77 | { 78 | return make_pair(Mat(), -1); 79 | } 80 | 81 | rows /= D; 82 | 83 | Mat_ dx(rows, cols), dy(rows, cols), occupancy(rows, cols); 84 | occupancy = 1; 85 | Mat flowComponents[] = {dx, dy, occupancy}; 86 | for(int k = 0; k < D; k++) 87 | for(int i = 0; i < rows; i++) 88 | for(int j = 0; j < cols; j++) 89 | assert(scanf("%d ", &flowComponents[k].at(i, j)) == 1); 90 | 91 | Mat flow; 92 | merge(flowComponents, 3, flow); 93 | 94 | return make_pair(flow, frameIndex); 95 | } 96 | 97 | void parse_options(int argc, const char* argv[]) 98 | { 99 | for(int i = 1; i < argc; i++) 100 | { 101 | if(strcmp(argv[i], "--occupancy") == 0) 102 | ARG_OCCUPANCY = true; 103 | else if(strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) 104 | ARG_HELP = true; 105 | else if(i == argc - 1) 106 | ARG_DUMP_DIR = argv[i]; 107 | else 108 | ARG_VIDEO_PATH = argv[i]; 109 | } 110 | if(ARG_HELP || ARG_VIDEO_PATH == NULL || ARG_DUMP_DIR == NULL) 111 | { 112 | fprintf(stderr, "Usage: cat mpegflow.txt | ./vis [--occupancy] videoPath dumpDir\n --help and -h will output this help message.\n dumpDir specifies the directory to save the visualization images\n --occupancy will expect --occupancy option used for the mpegflow call and will visualize occupancy grid\n"); 113 | exit(1); 114 | } 115 | } 116 | 117 | int main(int argc, const char* argv[]) 118 | { 119 | parse_options(argc, argv); 120 | 121 | pair flow = read_flow(); 122 | 123 | VideoCapture in(ARG_VIDEO_PATH); 124 | Mat frame; 125 | assert(in.read(frame)); 126 | for(int opencvFrameIndex = 1; in.read(frame); opencvFrameIndex++) 127 | { 128 | if(opencvFrameIndex == flow.second) 129 | { 130 | vis_flow(flow, frame, ARG_DUMP_DIR); 131 | flow = read_flow(); 132 | } 133 | else 134 | fprintf(stderr, "Skipping frame %d.\n", int(opencvFrameIndex)); 135 | } 136 | } 137 | --------------------------------------------------------------------------------