├── utils ├── __init__.py ├── heatmap.py ├── visualize.py └── dataset.py ├── docs └── intro.png ├── notebooks └── _init_paths.py ├── get_dataset.sh ├── LICENSE ├── .gitignore ├── README.md ├── download.py └── segment.py /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/intro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuanfang/opra/HEAD/docs/intro.png -------------------------------------------------------------------------------- /notebooks/_init_paths.py: -------------------------------------------------------------------------------- 1 | """Set up paths.""" 2 | 3 | import os.path 4 | import sys 5 | 6 | 7 | def add_path(path): 8 | if path not in sys.path: 9 | sys.path.insert(0, path) 10 | 11 | 12 | this_dir = os.path.dirname(__file__) 13 | 14 | 15 | # Add lib to PYTHONPATH. 16 | lib_path = os.path.join(this_dir, '..') 17 | add_path(lib_path) 18 | -------------------------------------------------------------------------------- /get_dataset.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DATA_DIR='./data' 4 | 5 | # Download and unzip the data folder. 6 | wget ftp://cs.stanford.edu/cs/cvgl/OPRA/data.zip 7 | unzip data.zip 8 | 9 | # Download product review videos from YouTube. 10 | python download.py --playlist ${DATA_DIR}'/playlists' --output ${DATA_DIR}'/raw_videos/' 11 | 12 | # Segment the videos according to the annotations. 13 | python segment.py --annotations ${DATA_DIR}'/annotations/train.txt' --raw ${DATA_DIR}'/raw_videos/' --output ${DATA_DIR}'/clips' 14 | python segment.py --annotations ${DATA_DIR}'/annotations/test.txt' --raw ${DATA_DIR}'/raw_videos/' --output ${DATA_DIR}'/clips' 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | Copyright (c) 2018 Kuan Fang, Te-Lin Wu 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all 12 | copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | SOFTWARE. 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Self-defined 2 | *.pyc 3 | data 4 | archived 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | env/ 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | .hypothesis/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # SageMath parsed files 84 | *.sage.py 85 | 86 | # dotenv 87 | .env 88 | 89 | # virtualenv 90 | .venv 91 | venv/ 92 | ENV/ 93 | 94 | # Spyder project settings 95 | .spyderproject 96 | .spyproject 97 | 98 | # Rope project settings 99 | .ropeproject 100 | 101 | # mkdocs documentation 102 | /site 103 | 104 | # mypy 105 | .mypy_cache/ 106 | 107 | -------------------------------------------------------------------------------- /utils/heatmap.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------- 2 | # OPRA Dataset 3 | # Copyright 2018 University of Southern California, Stanford University 4 | # Licensed under The MIT License [see LICENSE for details] 5 | # Written by Kuan Fang, Te-Lin Wu 6 | # --------------------------------------------------------------------- 7 | 8 | """Heatmap utilities. 9 | """ 10 | 11 | import numpy as np 12 | import cv2 13 | 14 | 15 | def compute_heatmap(points, image_size, crop=None, scale=None, k_ratio=1.0): 16 | """Compute the heatmap from annotated points. 17 | 18 | Args: 19 | points: The annotated points. 20 | image_size: The size of the image. 21 | crop: If not None, crop the generated heatmap array. 22 | scale: If not None, scale the genearted heatmap array. 23 | k_ratio: The kernal size of Gaussian blur. 24 | 25 | Returns: 26 | The heatmap array. 27 | """ 28 | heatmap = np.zeros((image_size[0], image_size[1]), dtype=np.float32) 29 | n_points = points.shape[0] 30 | 31 | for i in xrange(n_points): 32 | x = points[i, 0] 33 | y = points[i, 1] 34 | row = int(y) 35 | col = int(x) 36 | heatmap[row, col] += 1.0 37 | 38 | # Compute kernel size of the Gaussian filter. The kernel size must be odd. 39 | k_size = int(np.sqrt(image_size[0] * image_size[1]) / k_ratio) 40 | 41 | if k_size % 2 == 0: 42 | k_size += 1 43 | 44 | # Compute the heatmap using the Gaussian filter. 45 | heatmap = cv2.GaussianBlur(heatmap, (k_size, k_size), 0) 46 | 47 | if crop: 48 | heatmap = heatmap[crop[0]:crop[0]+crop[2], crop[1]:crop[1]+crop[3]] 49 | 50 | if scale: 51 | heatmap = cv2.resize(heatmap, None, None, fx=scale, fy=scale, 52 | interpolation=cv2.INTER_LINEAR) 53 | 54 | heatmap /= np.sum(heatmap) 55 | 56 | return heatmap 57 | 58 | -------------------------------------------------------------------------------- /utils/visualize.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------- 2 | # OPRA Dataset 3 | # Copyright 2018 University of Southern California, Stanford University 4 | # Licensed under The MIT License [see LICENSE for details] 5 | # Written by Kuan Fang, Te-Lin Wu 6 | # --------------------------------------------------------------------- 7 | 8 | """Visualization utilities. 9 | """ 10 | 11 | import numpy as np 12 | import cv2 13 | import matplotlib.pyplot as plt 14 | import io 15 | import base64 16 | 17 | 18 | def plot_annotation(image, points, heatmap, alpha=0.7): 19 | """Plot the heatmap on the target image. 20 | 21 | Args: 22 | image: The target image. 23 | points: The annotated points. 24 | heatmap: The generated heatmap. 25 | alpha: The alpha value of the overlay image. 26 | """ 27 | fig = plt.figure(figsize=(15, 5)) 28 | ax1 = fig.add_subplot(1, 3, 1) 29 | ax2 = fig.add_subplot(1, 3, 2) 30 | ax3 = fig.add_subplot(1, 3, 3) 31 | 32 | # Plot the image. 33 | image = cv2.cvtColor(image.astype('uint8'), cv2.COLOR_BGR2RGB) 34 | ax1.imshow(image) 35 | 36 | # Plot the points. 37 | ax1.scatter(points[:, 0], points[:, 1], c='r') 38 | ax1.set_title('points annotation\nnum_points = {}'.format(points.shape[0])) 39 | 40 | # Plot the heatmap. 41 | ax2.imshow(heatmap, cmap='gray', interpolation='nearest') 42 | ax2.set_title('heatmap\nnp.sum(heatmap) = {}'.format(np.sum(heatmap))) 43 | 44 | # Plot the overlay of heatmap on the target image. 45 | processed_heatmap = heatmap * 255 / np.max(heatmap) 46 | processed_heatmap = np.tile(processed_heatmap[:, :, np.newaxis], (1, 1, 3)) 47 | processed_heatmap = processed_heatmap.astype('uint8') 48 | assert processed_heatmap.shape == image.shape 49 | overlay = cv2.addWeighted(processed_heatmap, alpha, image, 1-alpha, 0) 50 | ax3.imshow(overlay, interpolation='nearest') 51 | ax3.set_title('heatmap overlay\nalpha = {}'.format(alpha)) 52 | 53 | 54 | def plot_video(video_path): 55 | """Plot the video clip in the Ipython notebook. 56 | 57 | Args: 58 | video_path: Path to the video. 59 | """ 60 | from IPython.display import HTML 61 | 62 | video = io.open(video_path, 'rb').read() 63 | encoded = base64.b64encode(video) 64 | video_tag = ''''''.format(encoded.decode('ascii')) 67 | 68 | return HTML(data=video_tag) 69 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OPRA Dataset: Online Product Reviews for Affordances 2 | 3 | **[Kuan Fang*](https://ai.stanford.edu/~kuanfang/), [Te-Lin Wu*](mailto:telinwu@usc.edu), [Daniel Yang](https://www.linkedin.com/in/dxyang/), [Silvio Savarese](http://svl.stanford.edu/people) and [Joseph J. Lim](http://www-bcf.usc.edu/~limjj/)** 4 | 5 | For questions regarding OPRA Dataset please contact Kuan Fang and Te-Lin Wu. 6 | 7 | ### Introduction 8 | The OPRA Dataset was introduced in our [Demo2Vec paper](http://ai.stanford.edu/~kuanfang/pdf/demo2vec2018cvpr) for reasoning object affordances from online demonstration videos. It contains 11,505 demonstration clips and 2,512 object images scraped from 6 popular YouTube product review channels along with the corresponding affordance annotations. More details can be found on our [website](https://sites.google.com/view/demo2vec/). 9 | 10 |

11 | 12 | ### Citation 13 | ``` 14 | @inproceedings{demo2vec2018cvpr, 15 | author = {Fang, Kuan and Wu, Te-Lin and Yang, Daniel and Savarese, Silvio and Lim, Joseph J.}, 16 | title = {Demo2Vec: Reasoning Object Affordances From Online Videos}, 17 | booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, 18 | month = {June}, 19 | year = {2018} 20 | } 21 | ``` 22 | 23 | ### Requirements 24 | 25 | Install [youtube-dl](https://github.com/rg3/youtube-dl): 26 | ```Shell 27 | sudo -H pip install --upgrade youtube-dl 28 | ``` 29 | 30 | Install [ffmpeg](https://www.ffmpeg.org/): 31 | ```Shell 32 | sudo add-apt-repository ppa:mc3man/trusty-media 33 | sudo apt-get update 34 | sudo apt-get install ffmpeg 35 | ``` 36 | 37 | To visualize the dataset using Jupyter notebooks, these packages need to be installed: [NumPy](https://scipy.org/install.html), [OpenCV](https://docs.opencv.org/3.4.1/d2/de6/tutorial_py_setup_in_ubuntu.html), [Matplotlib](https://matplotlib.org/users/installing.html), [Jupyter](http://jupyter.org/install). 38 | 39 | ### Usage 40 | 41 | One can follow the step-by-step instructions below or just run the bash script `bash ./get_dataset.sh`. 42 | 43 | Download and unzip `data/` (around 500MB). The folder contains playlists of YouTube product review videos (`playlists/`), product images (`images/`), and human annotations of the video segmentation and interactiion regions (`annotations/`). 44 | ```Shell 45 | wget ftp://cs.stanford.edu/cs/cvgl/OPRA/data.zip 46 | unzip data.zip 47 | ``` 48 | 49 | Download the product review videos from YouTube (around 51GB). (Note that some of the url may be no longer valid when you run the script, because they have been deleted from the playlist or due to other technical issues.) 50 | ```Shell 51 | python download.py --playlist data/playlists --output data/raw_videos/ 52 | ``` 53 | 54 | Segment the videos according to the annotations (around 13GB). After segmentation, raw videos will be no longer needed and can be removed. 55 | ```Shell 56 | python segment.py --annotations data/annotations/train.txt --raw data/raw_videos/ --output data/clips 57 | python segment.py --annotations data/annotations/test.txt --raw data/raw_videos/ --output data/clips 58 | ``` 59 | 60 | ### Demo 61 | 62 | We provide a simple demo for loading and visualizing the dataset. Please run the Jupyter notebook: 63 | ```Shell 64 | jupyter notebook 65 | 66 | # Or run remotely. 67 | jupyter notebook --no-browser --port=8888 68 | ``` 69 | 70 | Open `./notebooks/visualize_annotations.ipynb` in the browser (Chrome and FireFox are recommended) at this [link](http://localhost:8888/notebooks/notebooks/visualize_annotations.ipynb). The notebook provide a basic demo of loading the dataset and visualizing the video clip, the target image, and the annotation. 71 | -------------------------------------------------------------------------------- /download.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # --------------------------------------------------------------------- 4 | # OPRA Dataset 5 | # Copyright 2018 University of Southern California, Stanford University 6 | # Licensed under The MIT License [see LICENSE for details] 7 | # Written by Kuan Fang, Te-Lin Wu 8 | # --------------------------------------------------------------------- 9 | 10 | """Download videos in the OPRA Dataset from YouTube. 11 | """ 12 | 13 | from __future__ import absolute_import 14 | from __future__ import division 15 | from __future__ import print_function 16 | 17 | import argparse 18 | import csv 19 | import glob 20 | import os 21 | 22 | 23 | def parse_args(): 24 | """Parse arguments. 25 | 26 | Returns: 27 | args: The parsed arguments. 28 | """ 29 | parser = argparse.ArgumentParser() 30 | 31 | parser.add_argument( 32 | '--playlist', 33 | dest='playlist_dir', 34 | help='The playlist directory.', 35 | type=str, 36 | default='data/playlists') 37 | 38 | parser.add_argument( 39 | '--output', 40 | dest='output_dir', 41 | help='The output directory.', 42 | type=str, 43 | default='data/raw_videos') 44 | 45 | args = parser.parse_args() 46 | 47 | return args 48 | 49 | 50 | def read_playlist(filename): 51 | """Read the playlist from file. 52 | 53 | Args: 54 | filename: Path to the playlist file. 55 | 56 | Returns: 57 | playlist: A dict of the playlist. 58 | """ 59 | playlist = [] 60 | 61 | with open(filename) as fin: 62 | num_videos = 0 63 | for line in csv.reader(fin, delimiter=','): 64 | if 'http' not in line[0]: 65 | continue 66 | entry = { 67 | 'index': num_videos + 1, 68 | 'url': line[0], 69 | 'channel': line[1], 70 | 'title': line[2], 71 | 'description': line[3], 72 | } 73 | playlist.append(entry) 74 | num_videos += 1 75 | 76 | return playlist 77 | 78 | 79 | def download_videos_in_playlist(playlist, output_dir): 80 | """Download videos in the playlist. 81 | 82 | Args: 83 | playlist: A dict of the playlist. 84 | output_dir: The directory to save the downloaded videos. 85 | """ 86 | if not os.path.exists(output_dir): 87 | os.makedirs(output_dir) 88 | 89 | num_videos = len(playlist) 90 | 91 | for i, entry in enumerate(playlist): 92 | filename = str(entry['index']) 93 | output_path = os.path.join(output_dir, '%s.mp4' % (filename)) 94 | command = 'youtube-dl -U {} -f mp4 -o {}'.format( 95 | entry['url'], output_path) 96 | print('Downloading video (%d / %d) from %s...' % 97 | (i, num_videos, entry['url'])) 98 | os.system(command) 99 | 100 | 101 | def main(): 102 | args = parse_args() 103 | 104 | if args.output_dir is not None: 105 | if not os.path.exists(args.output_dir): 106 | os.makedirs(args.output_dir) 107 | 108 | for filename in glob.glob(os.path.join(args.playlist_dir, '*', '*.csv')): 109 | playlist = read_playlist(filename) 110 | print('Playlist %s contains %d videos.' % (filename, len(playlist))) 111 | 112 | items = filename.strip('.csv').split('/') 113 | output_dir = os.path.join(args.output_dir, items[-2], items[-1]) 114 | download_videos_in_playlist(playlist, output_dir) 115 | 116 | 117 | if __name__ == '__main__': 118 | main() 119 | -------------------------------------------------------------------------------- /segment.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # --------------------------------------------------------------------- 4 | # OPRA Dataset 5 | # Copyright 2018 University of Southern California, Stanford University 6 | # Licensed under The MIT License [see LICENSE for details] 7 | # Written by Kuan Fang, Te-Lin Wu 8 | # --------------------------------------------------------------------- 9 | 10 | """Segment videos in the OPRA Dataset according to the annotations. 11 | """ 12 | 13 | from __future__ import absolute_import 14 | from __future__ import division 15 | from __future__ import print_function 16 | 17 | import argparse 18 | import os 19 | 20 | 21 | def parse_args(): 22 | """Parse arguments. 23 | 24 | Returns: 25 | args: The parsed arguments. 26 | """ 27 | parser = argparse.ArgumentParser() 28 | 29 | parser.add_argument( 30 | '--annotations', 31 | dest='annotations', 32 | help='Path of the annotations.', 33 | type=str, 34 | required=True) 35 | 36 | parser.add_argument( 37 | '--raw', 38 | dest='raw_video_dir', 39 | help='Directory of raw videos', 40 | type=str, 41 | default='data/playlists') 42 | 43 | parser.add_argument( 44 | '--output', 45 | dest='output_dir', 46 | help='The output directory.', 47 | type=str, 48 | default='data/clips') 49 | 50 | args = parser.parse_args() 51 | 52 | return args 53 | 54 | 55 | def read_annotations(filename, num_points=10): 56 | """Read the annotations from file. 57 | 58 | Args: 59 | filename: Path to the annotations. 60 | num_points: Number of annotated points. 61 | 62 | Returns: 63 | annotations: The dict of annotations. 64 | """ 65 | annotations = [] 66 | 67 | with open(filename, 'r') as fin: 68 | for line in fin: 69 | items = line.split(' ') 70 | 71 | points = [] 72 | for i in xrange(num_points): 73 | x = float(items[8 + 2*i]) 74 | y = float(items[8 + 2*i + 1]) 75 | points.append([x, y]) 76 | 77 | entry = { 78 | 'channel': items[0], 79 | 'playlist': items[1], 80 | 'video': items[2], 81 | 'start_time': items[3], 82 | 'duration': items[4], 83 | 'image': items[5], 84 | 'image_shape': (float(items[6]), float(items[7])), 85 | 'points': points, 86 | } 87 | annotations.append(entry) 88 | 89 | return annotations 90 | 91 | 92 | def segment_videos(annotations, raw_video_dir, output_dir): 93 | """Segment the videos. 94 | 95 | Args: 96 | annotations: The dict of the annotations. 97 | raw_video_dir: Directory of raw videos. 98 | 99 | Returns: 100 | output_dir: Output directory. 101 | """ 102 | if not os.path.exists(output_dir): 103 | os.makedirs(output_dir) 104 | 105 | num_annotations = len(annotations) 106 | for i, entry in enumerate(annotations): 107 | input_path = os.path.join( 108 | raw_video_dir, entry['channel'], entry['playlist'], 109 | '%s.mp4' % (entry['video'])) 110 | video_output_dir = os.path.join(output_dir, entry['channel'], 111 | entry['playlist'], entry['video']) 112 | output_path = os.path.join( 113 | video_output_dir, 114 | '%s_%s.mp4' % (entry['start_time'], entry['duration'])) 115 | 116 | if not os.path.exists(video_output_dir): 117 | os.makedirs(video_output_dir) 118 | 119 | command = ('ffmpeg -y -ss {} -i {} -t {} -c copy {}').format( 120 | entry['start_time'], input_path, entry['duration'], output_path) 121 | print('Segmenting video (%d / %d) by calling [%s] ...' % 122 | (i, num_annotations, command)) 123 | os.system(command) 124 | 125 | 126 | def main(): 127 | args = parse_args() 128 | 129 | if args.output_dir is not None: 130 | if not os.path.exists(args.output_dir): 131 | os.makedirs(args.output_dir) 132 | 133 | annotations = read_annotations(args.annotations) 134 | segment_videos(annotations, args.raw_video_dir, args.output_dir) 135 | 136 | 137 | if __name__ == '__main__': 138 | main() 139 | -------------------------------------------------------------------------------- /utils/dataset.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------- 2 | # OPRA Dataset 3 | # Copyright 2018 University of Southern California, Stanford University 4 | # Licensed under The MIT License [see LICENSE for details] 5 | # Written by Kuan Fang, Te-Lin Wu 6 | # --------------------------------------------------------------------- 7 | 8 | """OPRA Dataset. 9 | """ 10 | 11 | import os.path 12 | 13 | import numpy as np 14 | import cv2 15 | 16 | 17 | def _resize_points(points, src_shape, dst_shape): 18 | """Resize the points.""" 19 | xs = points[:, 0] 20 | ys = points[:, 1] 21 | new_xs = xs * dst_shape[0] / src_shape[0] 22 | new_ys = ys * dst_shape[1] / src_shape[1] 23 | new_points = np.hstack([new_xs[:, np.newaxis], new_ys[:, np.newaxis]]) 24 | return new_points 25 | 26 | 27 | class Dataset(object): 28 | 29 | def __init__(self, 30 | data_dir, 31 | annotation_path, 32 | num_points=10): 33 | """Initialize. 34 | 35 | Args: 36 | data_dir: Dataset directory. 37 | annotation_path: Path to the annotation file. 38 | num_points: Number of annotated points. 39 | """ 40 | self._data_dir = data_dir 41 | self._num_points = num_points 42 | self._annotation_path = annotation_path 43 | 44 | self._load() 45 | self.resize_points() 46 | 47 | @ property 48 | def data_dir(self): 49 | """Dataset directory.""" 50 | return self._data_dir 51 | 52 | @ property 53 | def num_points(self): 54 | """Number of annotated points.""" 55 | return self._num_points 56 | 57 | @property 58 | def data(self): 59 | """Data entries.""" 60 | return self._data 61 | 62 | def _load(self): 63 | """Load the dataset from file.""" 64 | self._data = [] 65 | 66 | with open(self._annotation_path, 'r') as fin: 67 | for line in fin: 68 | items = line.split(' ') 69 | 70 | points = [] 71 | for i in xrange(self.num_points): 72 | x = float(items[9 + 2 * i]) 73 | y = float(items[9 + 2 * i + 1]) 74 | points.append([x, y]) 75 | 76 | points = np.array(points, dtype=np.float32) 77 | 78 | height = float(items[6]) 79 | width = float(items[7]) 80 | action = int(items[8]) 81 | image_shape = (int(height), int(width)) 82 | 83 | entry = { 84 | 'channel': items[0], 85 | 'playlist': items[1], 86 | 'video': items[2], 87 | 'start_time': items[3], 88 | 'duration': items[4], 89 | 'image': items[5], 90 | 'image_shape': image_shape, 91 | 'points': np.array(points, dtype=np.float32), 92 | 'action': action, 93 | } 94 | 95 | self._data.append(entry) 96 | 97 | def resize_points(self): 98 | """Resize the annotated points. 99 | 100 | The points might be annotated with respect to a different image 101 | resolution. The point coordinates need to be resized after being loaded. 102 | """ 103 | for index in range(len(self.data)): 104 | entry = self.data[index] 105 | image_path = self.get_image_path(index) 106 | im = cv2.imread(image_path) 107 | 108 | image_shape = entry['image_shape'] 109 | if (im.shape[0] != image_shape[0]) or ( 110 | im.shape[1] != image_shape[1]): 111 | entry['points'] = _resize_points( 112 | entry['points'], image_shape, im.shape) 113 | entry['image_shape'] = (im.shape[0], im.shape[1]) 114 | 115 | def get_image_path(self, index): 116 | """Get the path of the target image of the entry. 117 | 118 | Args: 119 | index: The index of the entry. 120 | 121 | Returns: 122 | The file path. 123 | """ 124 | entry = self.data[index] 125 | 126 | image_name = os.path.join( 127 | entry['channel'], entry['playlist'], entry['video'], 128 | entry['image']) 129 | 130 | return os.path.join(self._data_dir, 'images', image_name) 131 | 132 | def get_video_path(self, index): 133 | """Get the path of the segmented video clip of the entry. 134 | 135 | Args: 136 | index: The index of the entry. 137 | 138 | Returns: 139 | The file path. 140 | """ 141 | entry = self.data[index] 142 | 143 | video_name = os.path.join( 144 | entry['channel'], entry['playlist'], entry['video'], 145 | '%s_%s.mp4' % (entry['start_time'], entry['duration'])) 146 | 147 | return os.path.join(self._data_dir, 'clips', video_name) 148 | --------------------------------------------------------------------------------