├── datasets └── .gitkeep ├── results └── .gitkeep ├── tools ├── __init__.py ├── funcs.py └── engine.py ├── utils ├── __init__.py ├── io.py ├── utils_.py ├── knapsack.py ├── sys_utils.py └── vsum_tools.py ├── configs ├── summe_can.py ├── tvsum_can.py ├── tvsum_aug.py └── summe_trans.py ├── README.md ├── splits ├── summe_splits.json ├── summe_trans_splits.json ├── tvsum_trans_splits.json ├── tvsum_splits.json ├── summe_aug_splits.json └── tvsum_aug_splits.json ├── main.py ├── models └── reccotrans.py └── cluster.json /datasets/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /results/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tools/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/io.py: -------------------------------------------------------------------------------- 1 | import hdf5storage 2 | import json 3 | import numpy as np 4 | import os 5 | import scipy.io as sio 6 | import json 7 | 8 | 9 | def load_sumres(dirname): 10 | with open(dirname, "r") as f: 11 | data = json.load(f) 12 | 13 | data_list = {} 14 | for key, val in data.items(): 15 | data_list[key] = np.array([val["summary"]]).T 16 | return data_list 17 | 18 | 19 | def load_summe_mat(dirname): 20 | mat_list = os.listdir(dirname) 21 | 22 | data_list = [] 23 | for mat in mat_list: 24 | data = sio.loadmat(os.path.join(dirname, mat)) 25 | 26 | item_dict = { 27 | "video": mat[:-4], 28 | "length": data["video_duration"], 29 | "nframes": data["nFrames"], 30 | "user_anno": data["user_score"], 31 | "gt_score": data["gt_score"], 32 | } 33 | 34 | data_list.append((item_dict)) 35 | 36 | return data_list 37 | 38 | 39 | def load_tvsum_mat(filename): 40 | data = hdf5storage.loadmat(filename, variable_names=["tvsum50"]) 41 | data = data["tvsum50"].ravel() 42 | 43 | data_list = [] 44 | for item in data: 45 | video, category, title, length, nframes, user_anno, gt_score = item 46 | 47 | item_dict = { 48 | "video": video[0, 0], 49 | "category": category[0, 0], 50 | "title": title[0, 0], 51 | "length": length[0, 0], 52 | "nframes": nframes[0, 0], 53 | "user_anno": user_anno, 54 | "gt_score": gt_score, 55 | } 56 | 57 | data_list.append((item_dict)) 58 | 59 | return data_list 60 | -------------------------------------------------------------------------------- /configs/summe_can.py: -------------------------------------------------------------------------------- 1 | from torch.autograd import Variable 2 | 3 | 4 | class HParameters: 5 | 6 | def __init__(self): 7 | 8 | # models 9 | self.engine = 'engine' 10 | self.model_name = 'reccotrans' 11 | self.nhead1 = 2 12 | self.nhid1 = 4096 13 | self.nlayers1 = 2 14 | self.shot_dim = 512 15 | self.dropout1 = 0 16 | 17 | self.nhead2 = 2 18 | self.nhid2 = 2048 19 | self.nlayers2 = 3 20 | self.dropout2 = 0 21 | 22 | # training 23 | self.l2_req = 0.000001 24 | self.lr_epochs = [0,30] 25 | self.lr = [0.00001, 0.000001] 26 | self.epochs_max = 60 27 | self.regular = True 28 | self.eval_freq = 1 29 | self.rec_factor = 1 30 | self.supervised_factor = 1 31 | 32 | # data 33 | self.datasets=['datasets/eccv16_dataset_summe_google_pool5.h5', 34 | 'datasets/eccv16_dataset_tvsum_google_pool5.h5'] 35 | 36 | # self.splits = ['splits/summe_splits.json', 37 | # 'splits/tvsum_splits.json'] 38 | self.splits = ['splits/summe_splits.json'] 39 | 40 | # general 41 | self.rnd_seed = 0 42 | self.verbose = False 43 | self.use_cuda = True 44 | self.cuda_device = 0 45 | self.max_summary_length = 0.15 46 | self.rand_choice = True 47 | 48 | return 49 | 50 | 51 | def get_dataset_by_name(self, dataset_name): 52 | for d in self.datasets: 53 | if dataset_name in d: 54 | return [d] 55 | return None 56 | 57 | def load_from_args(self, args): 58 | for key in args: 59 | val = args[key] 60 | if val is not None: 61 | if hasattr(self, key) and isinstance(getattr(self, key), list): 62 | val = val.split() 63 | 64 | setattr(self, key, val) 65 | 66 | def __str__(self): 67 | vars = [attr for attr in dir(self) if not callable(getattr(self,attr)) and not (attr.startswith("__") or attr.startswith("_"))] 68 | 69 | info_str = '' 70 | for i, var in enumerate(vars): 71 | val = getattr(self, var) 72 | if isinstance(val, Variable): 73 | val = val.data.cpu().numpy().tolist()[0] 74 | info_str += '['+str(i)+'] '+var+': '+str(val)+'\n' 75 | 76 | return info_str 77 | -------------------------------------------------------------------------------- /tools/funcs.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | import json 4 | import hdf5storage 5 | import numpy as np 6 | import torch.nn.init as init 7 | from utils.sys_utils import * 8 | from utils.vsum_tools import * 9 | 10 | 11 | def get_vidnames(dset): 12 | if dset == "summe": 13 | path = "rethinking_data/data/raw/summe/GT" 14 | vids = sorted(os.listdir(path)) 15 | name = [vid.replace(".mat", "") for vid in vids] 16 | if dset == "tvsum": 17 | filename = "rethinking_data/data/raw/tvsum/ydata-tvsum50.mat" 18 | data = hdf5storage.loadmat(filename, variable_names=["tvsum50"]) 19 | data = data["tvsum50"].ravel() 20 | name = [i[0][0, 0] for i in data] 21 | return name 22 | 23 | 24 | def weights_init(m): 25 | classname = m.__class__.__name__ 26 | if classname == "Linear": 27 | init.xavier_uniform_(m.weight, gain=np.sqrt(2.0)) 28 | if m.bias is not None: 29 | init.constant_(m.bias, 0.1) 30 | 31 | 32 | def parse_splits_filename(splits_filename): 33 | # Parse split file and count number of k_folds 34 | spath, sfname = os.path.split(splits_filename) 35 | sfname, _ = os.path.splitext(sfname) 36 | dataset_name = sfname.split("_")[0] # Get dataset name e.g. tvsum 37 | dataset_type = sfname.split("_")[1] # augmentation type e.g. aug 38 | 39 | # The keyword 'splits' is used as the filename fields terminator from historical reasons. 40 | if dataset_type == "splits": 41 | # Split type is not present 42 | dataset_type = "" 43 | 44 | # Get number of discrete splits within each split json file 45 | with open(splits_filename, "r") as sf: 46 | splits = json.load(sf) 47 | 48 | return dataset_name, dataset_type, splits 49 | 50 | 51 | def lookup_weights_splits_file(path, dataset_name, dataset_type, split_id): 52 | dataset_type_str = "" if dataset_type == "" else dataset_type + "_" 53 | weights_filename = path + "/models/{}_{}splits_{}_*.tar.pth".format( 54 | dataset_name, dataset_type_str, split_id 55 | ) 56 | weights_filename = glob.glob(weights_filename) 57 | if len(weights_filename) == 0: 58 | print("Couldn't find model weights: ", weights_filename) 59 | return "" 60 | 61 | # Get the first weights file in the dir 62 | weights_filename = weights_filename[0] 63 | splits_file = path + "/splits/{}_{}splits.json".format( 64 | dataset_name, dataset_type_str 65 | ) 66 | 67 | return weights_filename, splits_file 68 | -------------------------------------------------------------------------------- /configs/tvsum_can.py: -------------------------------------------------------------------------------- 1 | from torch.autograd import Variable 2 | 3 | 4 | class HParameters: 5 | def __init__(self): 6 | 7 | # models 8 | self.engine = "engine" 9 | self.model_name = "reccotrans" 10 | self.nhead1 = 2 11 | self.nhid1 = 4096 12 | self.nlayers1 = 2 13 | self.shot_dim = 512 14 | self.dropout1 = 0 15 | 16 | self.nhead2 = 2 17 | self.nhid2 = 2048 18 | self.nlayers2 = 3 19 | self.dropout2 = 0 20 | 21 | # training 22 | self.l2_req = 0.00001 23 | self.lr_epochs = [0, 30] 24 | self.lr = [0.00001, 0.000001] 25 | self.epochs_max = 60 26 | self.regular = True 27 | self.eval_freq = 1 28 | self.rec_factor = 0.01 29 | self.supervised_factor = 1 30 | 31 | # data 32 | self.datasets = [ 33 | "datasets/eccv16_dataset_summe_google_pool5.h5", 34 | "datasets/eccv16_dataset_tvsum_google_pool5.h5", 35 | ] 36 | 37 | # self.splits = ['splits/summe_splits.json', 38 | # 'splits/tvsum_splits.json'] 39 | self.splits = ["splits/tvsum_splits.json"] 40 | 41 | # general 42 | self.rnd_seed = 1234 43 | self.verbose = False 44 | self.use_cuda = True 45 | self.cuda_device = 0 46 | self.max_summary_length = 0.15 47 | self.rand_choice = False 48 | 49 | return 50 | 51 | def get_dataset_by_name(self, dataset_name): 52 | for d in self.datasets: 53 | if dataset_name in d: 54 | return [d] 55 | return None 56 | 57 | def load_from_args(self, args): 58 | for key in args: 59 | val = args[key] 60 | if val is not None: 61 | if hasattr(self, key) and isinstance(getattr(self, key), list): 62 | val = val.split() 63 | 64 | setattr(self, key, val) 65 | 66 | def __str__(self): 67 | vars = [ 68 | attr 69 | for attr in dir(self) 70 | if not callable(getattr(self, attr)) 71 | and not (attr.startswith("__") or attr.startswith("_")) 72 | ] 73 | 74 | info_str = "" 75 | for i, var in enumerate(vars): 76 | val = getattr(self, var) 77 | if isinstance(val, Variable): 78 | val = val.data.cpu().numpy().tolist()[0] 79 | info_str += "[" + str(i) + "] " + var + ": " + str(val) + "\n" 80 | 81 | return info_str 82 | -------------------------------------------------------------------------------- /utils/utils_.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import os 3 | import sys 4 | import errno 5 | import json 6 | import os.path as osp 7 | 8 | import torch 9 | 10 | 11 | def mkdir_if_missing(directory): 12 | if not osp.exists(directory): 13 | try: 14 | os.makedirs(directory) 15 | except OSError as e: 16 | if e.errno != errno.EEXIST: 17 | raise 18 | 19 | 20 | class AverageMeter(object): 21 | """Computes and stores the average and current value. 22 | 23 | Code imported from https://github.com/pytorch/examples/blob/master/imagenet/main.py#L247-L262 24 | """ 25 | 26 | def __init__(self): 27 | self.reset() 28 | 29 | def reset(self): 30 | self.val = 0 31 | self.avg = 0 32 | self.sum = 0 33 | self.count = 0 34 | 35 | def update(self, val, n=1): 36 | self.val = val 37 | self.sum += val * n 38 | self.count += n 39 | self.avg = self.sum / self.count 40 | 41 | 42 | def save_checkpoint(state, fpath="checkpoint.pth.tar"): 43 | mkdir_if_missing(osp.dirname(fpath)) 44 | torch.save(state, fpath) 45 | 46 | 47 | class Logger(object): 48 | """ 49 | Write console output to external text file. 50 | Code imported from https://github.com/Cysu/open-reid/blob/master/reid/utils/logging.py. 51 | """ 52 | 53 | def __init__(self, fpath=None): 54 | self.console = sys.stdout 55 | self.file = None 56 | if fpath is not None: 57 | mkdir_if_missing(os.path.dirname(fpath)) 58 | self.file = open(fpath, "w") 59 | 60 | def __del__(self): 61 | self.close() 62 | 63 | def __enter__(self): 64 | pass 65 | 66 | def __exit__(self, *args): 67 | self.close() 68 | 69 | def write(self, msg): 70 | self.console.write(msg) 71 | if self.file is not None: 72 | self.file.write(msg) 73 | 74 | def flush(self): 75 | self.console.flush() 76 | if self.file is not None: 77 | self.file.flush() 78 | os.fsync(self.file.fileno()) 79 | 80 | def close(self): 81 | self.console.close() 82 | if self.file is not None: 83 | self.file.close() 84 | 85 | 86 | def read_json(fpath): 87 | with open(fpath, "r") as f: 88 | obj = json.load(f) 89 | return obj 90 | 91 | 92 | def write_json(obj, fpath): 93 | mkdir_if_missing(osp.dirname(fpath)) 94 | with open(fpath, "w") as f: 95 | json.dump(obj, f, indent=4, separators=(",", ": ")) 96 | -------------------------------------------------------------------------------- /configs/tvsum_aug.py: -------------------------------------------------------------------------------- 1 | from torch.autograd import Variable 2 | 3 | 4 | class HParameters: 5 | 6 | def __init__(self): 7 | 8 | # models 9 | self.engine = 'engine' 10 | self.model_name = 'reccotrans' 11 | self.nhead1 = 2 12 | self.nhid1 = 4096 13 | self.nlayers1 = 2 14 | self.shot_dim = 512 15 | self.dropout1 = 0 16 | 17 | self.nhead2 = 2 18 | self.nhid2 = 2048 19 | self.nlayers2 = 3 20 | self.dropout2 = 0 21 | 22 | # training 23 | self.l2_req = 0.00001 24 | self.lr_epochs = [0,30] 25 | self.lr = [0.00001, 0.000001] 26 | self.epochs_max = 60 27 | self.regular = True 28 | self.eval_freq = 1 29 | self.rec_factor = 0.01 30 | self.supervised_factor = 1 31 | 32 | # data 33 | self.datasets = ['datasets/eccv16_dataset_summe_google_pool5.h5', 34 | 'datasets/eccv16_dataset_tvsum_google_pool5.h5', 35 | 'datasets/eccv16_dataset_ovp_google_pool5.h5', 36 | 'datasets/eccv16_dataset_youtube_google_pool5.h5'] 37 | 38 | # self.splits = ['splits/summe_splits.json', 39 | # 'splits/tvsum_splits.json'] 40 | self.splits = ['splits/tvsum_aug_splits.json'] 41 | 42 | # general 43 | self.rnd_seed = 1234 44 | self.verbose = False 45 | self.use_cuda = True 46 | self.cuda_device = 0 47 | self.max_summary_length = 0.15 48 | self.rand_choice = False 49 | 50 | return 51 | 52 | 53 | def get_dataset_by_name(self, dataset_name): 54 | for d in self.datasets: 55 | if dataset_name in d: 56 | return [d] 57 | return None 58 | 59 | def load_from_args(self, args): 60 | for key in args: 61 | val = args[key] 62 | if val is not None: 63 | if hasattr(self, key) and isinstance(getattr(self, key), list): 64 | val = val.split() 65 | 66 | setattr(self, key, val) 67 | 68 | def __str__(self): 69 | vars = [attr for attr in dir(self) if not callable(getattr(self,attr)) and not (attr.startswith("__") or attr.startswith("_"))] 70 | 71 | info_str = '' 72 | for i, var in enumerate(vars): 73 | val = getattr(self, var) 74 | if isinstance(val, Variable): 75 | val = val.data.cpu().numpy().tolist()[0] 76 | info_str += '['+str(i)+'] '+var+': '+str(val)+'\n' 77 | 78 | return info_str 79 | -------------------------------------------------------------------------------- /configs/summe_trans.py: -------------------------------------------------------------------------------- 1 | from torch.autograd import Variable 2 | 3 | 4 | class HParameters: 5 | def __init__(self): 6 | 7 | # models 8 | self.engine = "engine" 9 | self.model_name = "reccotrans" 10 | self.nhead1 = 2 11 | self.nhid1 = 4096 12 | self.nlayers1 = 2 13 | self.shot_dim = 512 14 | self.dropout1 = 0 15 | 16 | self.nhead2 = 2 17 | self.nhid2 = 2048 18 | self.nlayers2 = 3 19 | self.dropout2 = 0 20 | 21 | # training 22 | self.l2_req = 0.000001 23 | self.lr_epochs = [0, 30] 24 | self.lr = [0.00001, 0.000001] 25 | self.epochs_max = 60 26 | self.regular = True 27 | self.eval_freq = 1 28 | self.rec_factor = 1 29 | self.supervised_factor = 1 30 | 31 | # data 32 | self.datasets = [ 33 | "datasets/eccv16_dataset_summe_google_pool5.h5", 34 | "datasets/eccv16_dataset_tvsum_google_pool5.h5", 35 | "datasets/eccv16_dataset_ovp_google_pool5.h5", 36 | "datasets/eccv16_dataset_youtube_google_pool5.h5", 37 | ] 38 | 39 | # self.splits = ['splits/summe_splits.json', 40 | # 'splits/tvsum_splits.json'] 41 | self.splits = ["splits/summe_trans_splits.json"] 42 | 43 | # general 44 | self.rnd_seed = 0 45 | self.verbose = False 46 | self.use_cuda = True 47 | self.cuda_device = 0 48 | self.max_summary_length = 0.15 49 | self.rand_choice = False 50 | 51 | return 52 | 53 | def get_dataset_by_name(self, dataset_name): 54 | for d in self.datasets: 55 | if dataset_name in d: 56 | return [d] 57 | return None 58 | 59 | def load_from_args(self, args): 60 | for key in args: 61 | val = args[key] 62 | if val is not None: 63 | if hasattr(self, key) and isinstance(getattr(self, key), list): 64 | val = val.split() 65 | 66 | setattr(self, key, val) 67 | 68 | def __str__(self): 69 | vars = [ 70 | attr 71 | for attr in dir(self) 72 | if not callable(getattr(self, attr)) 73 | and not (attr.startswith("__") or attr.startswith("_")) 74 | ] 75 | 76 | info_str = "" 77 | for i, var in enumerate(vars): 78 | val = getattr(self, var) 79 | if isinstance(val, Variable): 80 | val = val.data.cpu().numpy().tolist()[0] 81 | info_str += "[" + str(i) + "] " + var + ": " + str(val) + "\n" 82 | 83 | return info_str 84 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Video Joint Modelling Based on Hierarchical Transformer for Co-summarization (VJMHT) 2 | [Paper](https://arxiv.org/pdf/2112.13478.pdf) 3 | 4 | Haopeng Li, Qiuhong Ke, [Mingming Gong](https://mingming-gong.github.io/), [Rui Zhang](https://www.ruizhang.info/) 5 | 6 | IEEE Transactions on Pattern Analysis and Machine Intelligence 7 | 8 | 9 | ## Introduction 10 | We propose **V**ideo **J**oint **M**odelling based on **H**ierarchical **T**ransformer (**VJMHT**) for co-summarization, which takes into consideration the semantic dependencies across videos. 11 | 12 | VJMHT consists of two layers of Transformer: the first layer extracts semantic representation from individual shots of similar videos, while the second layer performs shot-level video joint modelling to aggregate cross-video semantic information. By this means, complete cross-video high-level patterns are explicitly modelled and learned for the summarization of individual videos. 13 | 14 | 15 | Moreover, Transformer-based video representation reconstruction is introduced to maximize the high-level similarity between the summary and the original video. 16 | 17 | 18 | ## Requirements and Dependencies 19 | 20 | - Python=3.8.5 21 | - PyTorch=1.9, ortools=8.1.8487 22 | 23 | ## Data Preparation 24 | 25 | Download the [datasets](https://figshare.com/articles/dataset/datasets_zip/29974894) to ``datasets/``. 26 | 27 | ## Evaluation 28 | 29 | Download [our models](https://figshare.com/articles/journal_contribution/VJMHT_trained_Models/29974912?file=57382336) to ``results/``. 30 | 31 | Run the following command to test our models. 32 | 33 | ``` 34 | $ python main.py -c configs/dataset_setting.py --eval 35 | ``` 36 | 37 | where ``dataset_setting.py`` is the configuration file that can be found in ``configs/``. The results are saved in ``results/DATASET_SETTING/``. 38 | 39 | Example for testing the model trained on TVSum in the canonical setting: 40 | 41 | ``` 42 | $ python main.py -c configs/tvsum_can.py --eval 43 | ``` 44 | 45 | The results are saved in ``results/TVSUM_CAN``. 46 | 47 | ## Training 48 | Run the following command to train the model: 49 | 50 | ``` 51 | $ python main.py -c configs/dataset_setting.py 52 | ``` 53 | 54 | Example for training the model on TVSum in the canonical setting: 55 | 56 | ``` 57 | $ python main.py -c configs/tvsum_can.py 58 | ``` 59 | 60 | The trained models and results are saved in ``results/TVSUM_CAN``. 61 | 62 | 63 | ## License and Citation 64 | 65 | The use of this code is RESTRICTED to **non-commercial research and educational purposes**. 66 | 67 | If you use this code or reference our paper in your work please cite this publication as: 68 | 69 | ``` 70 | @article{li2022video, 71 | title={Video Joint Modelling Based on Hierarchical Transformer for Co-summarization}, 72 | author={Li, Haopeng and Ke, Qiuhong and Gong, Mingming and Zhang, Rui}, 73 | journal={IEEE Transactions on Pattern Analysis and Machine Intelligence}, 74 | year={2022}, 75 | publisher={IEEE} 76 | } 77 | ``` 78 | ## Acknowledgement 79 | 80 | The code is developed based on [VASNet](https://github.com/ok1zjf/VASNet). 81 | 82 | -------------------------------------------------------------------------------- /splits/summe_splits.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "train_keys": [ 4 | "video_24", 5 | "video_14", 6 | "video_4", 7 | "video_2", 8 | "video_25", 9 | "video_9", 10 | "video_6", 11 | "video_16", 12 | "video_15", 13 | "video_12", 14 | "video_20", 15 | "video_11", 16 | "video_19", 17 | "video_17", 18 | "video_3", 19 | "video_21", 20 | "video_5", 21 | "video_10", 22 | "video_18", 23 | "video_23" 24 | ], 25 | "test_keys": [ 26 | "video_8", 27 | "video_7", 28 | "video_1", 29 | "video_13", 30 | "video_22" 31 | ] 32 | }, 33 | { 34 | "train_keys": [ 35 | "video_8", 36 | "video_7", 37 | "video_1", 38 | "video_13", 39 | "video_22", 40 | "video_9", 41 | "video_6", 42 | "video_16", 43 | "video_15", 44 | "video_12", 45 | "video_20", 46 | "video_11", 47 | "video_19", 48 | "video_17", 49 | "video_3", 50 | "video_21", 51 | "video_5", 52 | "video_10", 53 | "video_18", 54 | "video_23" 55 | ], 56 | "test_keys": [ 57 | "video_24", 58 | "video_14", 59 | "video_4", 60 | "video_2", 61 | "video_25" 62 | ] 63 | }, 64 | { 65 | "train_keys": [ 66 | "video_8", 67 | "video_7", 68 | "video_1", 69 | "video_13", 70 | "video_22", 71 | "video_24", 72 | "video_14", 73 | "video_4", 74 | "video_2", 75 | "video_25", 76 | "video_20", 77 | "video_11", 78 | "video_19", 79 | "video_17", 80 | "video_3", 81 | "video_21", 82 | "video_5", 83 | "video_10", 84 | "video_18", 85 | "video_23" 86 | ], 87 | "test_keys": [ 88 | "video_9", 89 | "video_6", 90 | "video_16", 91 | "video_15", 92 | "video_12" 93 | ] 94 | }, 95 | { 96 | "train_keys": [ 97 | "video_8", 98 | "video_7", 99 | "video_1", 100 | "video_13", 101 | "video_22", 102 | "video_24", 103 | "video_14", 104 | "video_4", 105 | "video_2", 106 | "video_25", 107 | "video_9", 108 | "video_6", 109 | "video_16", 110 | "video_15", 111 | "video_12", 112 | "video_21", 113 | "video_5", 114 | "video_10", 115 | "video_18", 116 | "video_23" 117 | ], 118 | "test_keys": [ 119 | "video_20", 120 | "video_11", 121 | "video_19", 122 | "video_17", 123 | "video_3" 124 | ] 125 | }, 126 | { 127 | "train_keys": [ 128 | "video_8", 129 | "video_7", 130 | "video_1", 131 | "video_13", 132 | "video_22", 133 | "video_24", 134 | "video_14", 135 | "video_4", 136 | "video_2", 137 | "video_25", 138 | "video_9", 139 | "video_6", 140 | "video_16", 141 | "video_15", 142 | "video_12", 143 | "video_20", 144 | "video_11", 145 | "video_19", 146 | "video_17", 147 | "video_3" 148 | ], 149 | "test_keys": [ 150 | "video_21", 151 | "video_5", 152 | "video_10", 153 | "video_18", 154 | "video_23" 155 | ] 156 | } 157 | ] -------------------------------------------------------------------------------- /utils/knapsack.py: -------------------------------------------------------------------------------- 1 | # A Dynamic Programming based Python Program for 0-1 Knapsack problem 2 | # Returns the maximum value that can be put in a knapsack of capacity W 3 | import numpy as np 4 | from ortools.algorithms import pywrapknapsack_solver 5 | 6 | 7 | def knapsack(W, wt, val, n): 8 | K = [[0 for x in range(W+1)] for x in range(n+1)] 9 | 10 | # Build table K[][] in bottom up manner 11 | for i in range(n+1): 12 | for w in range(W+1): 13 | if i==0 or w==0: 14 | K[i][w] = 0 15 | elif wt[i-1] <= w: 16 | K[i][w] = max(val[i-1] + K[i-1][w-wt[i-1]], K[i-1][w]) 17 | else: 18 | K[i][w] = K[i-1][w] 19 | 20 | 21 | best = K[n][W] 22 | 23 | amount = np.zeros(n) 24 | a = best 25 | j = n 26 | Y = W 27 | 28 | # j = j + 1; 29 | # 30 | # amount(j) = 1; 31 | # Y = Y - weights(j); 32 | # j = j - 1; 33 | # a = A(j + 1, Y + 1); 34 | 35 | while a > 0: 36 | while K[j][Y] == a: 37 | j = j - 1 38 | 39 | j = j + 1 40 | amount[j-1] = 1 41 | Y = Y - wt[j-1] 42 | j = j - 1 43 | a = K[j][Y] 44 | 45 | return amount 46 | 47 | 48 | def test_knapsack(): 49 | weights = [1 ,1 ,1, 1 ,2 ,2 ,3] 50 | values = [1 ,1 ,2 ,3, 1, 3 ,5] 51 | best = 13 52 | print(knapsack(7, weights, values, 7)) 53 | 54 | #=========================================== 55 | ''' 56 | ------------------------------------------------ 57 | Use dynamic programming (DP) to solve 0/1 knapsack problem 58 | Time complexity: O(nW), where n is number of items and W is capacity 59 | 60 | Author: Kaiyang Zhou 61 | Website: https://kaiyangzhou.github.io/ 62 | ------------------------------------------------ 63 | knapsack_dp(values,weights,n_items,capacity,return_all=False) 64 | 65 | Input arguments: 66 | 1. values: a list of numbers in either int or float, specifying the values of items 67 | 2. weights: a list of int numbers specifying weights of items 68 | 3. n_items: an int number indicating number of items 69 | 4. capacity: an int number indicating the knapsack capacity 70 | 5. return_all: whether return all info, defaulty is False (optional) 71 | 72 | Return: 73 | 1. picks: a list of numbers storing the positions of selected items 74 | 2. max_val: maximum value (optional) 75 | ------------------------------------------------ 76 | ''' 77 | def knapsack_dp(values,weights,n_items,capacity,return_all=False): 78 | check_inputs(values,weights,n_items,capacity) 79 | 80 | table = np.zeros((n_items+1,capacity+1),dtype=np.float32) 81 | keep = np.zeros((n_items+1,capacity+1),dtype=np.float32) 82 | 83 | for i in range(1,n_items+1): 84 | for w in range(0,capacity+1): 85 | wi = weights[i-1] # weight of current item 86 | vi = values[i-1] # value of current item 87 | if (wi <= w) and (vi + table[i-1,w-wi] > table[i-1,w]): 88 | table[i,w] = vi + table[i-1,w-wi] 89 | keep[i,w] = 1 90 | else: 91 | table[i,w] = table[i-1,w] 92 | 93 | picks = [] 94 | K = capacity 95 | 96 | for i in range(n_items,0,-1): 97 | if keep[i,K] == 1: 98 | picks.append(i) 99 | K -= weights[i-1] 100 | 101 | picks.sort() 102 | picks = [x-1 for x in picks] # change to 0-index 103 | 104 | if return_all: 105 | max_val = table[n_items,capacity] 106 | return picks,max_val 107 | return picks 108 | 109 | def check_inputs(values,weights,n_items,capacity): 110 | # check variable type 111 | assert(isinstance(values,list)) 112 | assert(isinstance(weights,list)) 113 | assert(isinstance(n_items,int)) 114 | assert(isinstance(capacity,int)) 115 | # check value type 116 | assert(all(isinstance(val,int) or isinstance(val,float) for val in values)) 117 | assert(all(isinstance(val,int) for val in weights)) 118 | # check validity of value 119 | assert(all(val >= 0 for val in weights)) 120 | assert(n_items > 0) 121 | assert(capacity > 0) 122 | 123 | def test_knapsack_dp(): 124 | values = [2,3,4] 125 | weights = [1,2,3] 126 | n_items = 3 127 | capacity = 3 128 | picks = knapsack_dp(values,weights,n_items,capacity) 129 | print (picks) 130 | 131 | 132 | 133 | osolver = pywrapknapsack_solver.KnapsackSolver( 134 | # pywrapknapsack_solver.KnapsackSolver.KNAPSACK_MULTIDIMENSION_BRANCH_AND_BOUND_SOLVER, 135 | pywrapknapsack_solver.KnapsackSolver.KNAPSACK_DYNAMIC_PROGRAMMING_SOLVER, 136 | 'test') 137 | 138 | def knapsack_ortools(values, weights, items, capacity ): 139 | scale = 1000 140 | values = np.array(values) 141 | weights = np.array(weights) 142 | values = (values * scale).astype(np.int) 143 | weights = (weights).astype(np.int) 144 | capacity = capacity 145 | 146 | osolver.Init(values.tolist(), [weights.tolist()], [capacity]) 147 | computed_value = osolver.Solve() 148 | packed_items = [x for x in range(0, len(weights)) 149 | if osolver.BestSolutionContains(x)] 150 | 151 | return packed_items 152 | 153 | 154 | if __name__ == "__main__": 155 | test_knapsack_dp() 156 | test_knapsack() -------------------------------------------------------------------------------- /utils/sys_utils.py: -------------------------------------------------------------------------------- 1 | __author__ = 'Jiri Fajtl' 2 | __email__ = 'ok1zjf@gmail.com' 3 | __version__= '3.6' 4 | __status__ = "Research" 5 | __date__ = "1/12/2018" 6 | __license__= "MIT License" 7 | 8 | 9 | import os 10 | import numpy as np 11 | import subprocess 12 | import platform 13 | import sys 14 | import pkg_resources 15 | import torch 16 | import h5py 17 | import json 18 | import ortools 19 | from torch.nn.modules.module import _addindent 20 | # import PIL as Image 21 | # import cv2 22 | 23 | def list_files(path, extensions=[], sort=True, max_len=-1): 24 | if os.path.isdir(path): 25 | filenames = [os.path.join(path, fn) for fn in os.listdir(path) if 26 | any([fn.lower().endswith(ext) for ext in extensions])] 27 | else: 28 | print("ERROR. ", path,' is not a directory!') 29 | return [] 30 | 31 | if sort: 32 | filenames.sort() 33 | 34 | if max_len>-1: 35 | filenames = filenames[:max_len] 36 | 37 | return filenames 38 | 39 | 40 | def del_file(filename): 41 | try: 42 | os.remove(filename) 43 | except: 44 | pass 45 | return 46 | 47 | def get_video_list(video_path, max_len=-1, extensions=['avi', 'flv', 'mpg', 'mp4']): 48 | return list_files(video_path, extensions=extensions , sort=True, max_len=max_len) 49 | 50 | def get_image_list(video_path, max_len=-1): 51 | return list_files(video_path, extensions=['jpg', 'jpeg', 'png'], sort=True, max_len=max_len) 52 | 53 | def run_command(command): 54 | p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 55 | return '\n'.join([ '\t'+line.decode("utf-8").strip() for line in p.stdout.readlines()]) 56 | 57 | def ge_pkg_versions(): 58 | dep_versions = {} 59 | dep_versions['display'] = run_command('cat /proc/driver/nvidia/version') 60 | 61 | dep_versions['cuda'] = 'NA' 62 | cuda_home = '/usr/local/cuda/' 63 | if 'CUDA_HOME' in os.environ: 64 | cuda_home = os.environ['CUDA_HOME'] 65 | 66 | cmd = cuda_home+'/version.txt' 67 | if os.path.isfile(cmd): 68 | dep_versions['cuda'] = run_command('cat '+cmd) 69 | 70 | dep_versions['cudnn'] = torch.backends.cudnn.version() 71 | dep_versions['platform'] = platform.platform() 72 | dep_versions['python'] = sys.version_info[:3] 73 | dep_versions['torch'] = torch.__version__ 74 | dep_versions['numpy'] = np.__version__ 75 | dep_versions['h5py'] = h5py.__version__ 76 | dep_versions['json'] = json.__version__ 77 | dep_versions['ortools'] = ortools.__version__ 78 | dep_versions['torchvision'] = pkg_resources.get_distribution("torchvision").version 79 | 80 | # dep_versions['PIL'] = Image.VERSION 81 | # dep_versions['OpenCV'] = 'NA' 82 | # if 'cv2' in sys.modules: 83 | # dep_versions['OpenCV'] = cv2.__version__ 84 | 85 | 86 | return dep_versions 87 | 88 | 89 | def print_pkg_versions(): 90 | print("Packages & system versions:") 91 | print("----------------------------------------------------------------------") 92 | versions = ge_pkg_versions() 93 | for key, val in versions.items(): 94 | print(key,": ",val) 95 | print("") 96 | return 97 | 98 | 99 | def torch_summarize(model, show_weights=True, show_parameters=True): 100 | """Summarizes torch model by showing trainable parameters and weights.""" 101 | tmpstr = model.__class__.__name__ + ' (\n' 102 | parameters = 0 103 | convs = 0 104 | for key, module in model._modules.items(): 105 | # if it contains layers let call it recursively to get params and weights 106 | if type(module) in [torch.nn.modules.container.Container, torch.nn.modules.container.Sequential]: 107 | modstr, p, cnvs = torch_summarize(module) 108 | parameters += p 109 | convs += cnvs 110 | else: 111 | modstr = module.__repr__() 112 | convs += len(modstr.split('Conv2d')) - 1 113 | 114 | modstr = _addindent(modstr, 2) 115 | # if 'conv' in key: 116 | # convs += 1 117 | 118 | params = sum([np.prod(p.size()) for p in module.parameters()]) 119 | parameters += params 120 | weights = tuple([tuple(p.size()) for p in module.parameters()]) 121 | 122 | tmpstr += ' (' + key + '): ' + modstr 123 | if show_weights: 124 | tmpstr += ', weights={}'.format(weights) 125 | if show_parameters: 126 | tmpstr += ', parameters={} / {}'.format(params, parameters) 127 | tmpstr += ', convs={}'.format(convs) 128 | tmpstr += '\n' 129 | 130 | tmpstr = tmpstr + ')' 131 | return tmpstr, parameters, convs 132 | 133 | 134 | def print_table(table, cell_width=[3,35,8]): 135 | slen=sum(cell_width)+len(cell_width)*2+2 136 | print('-'*slen) 137 | header = table.pop(0) 138 | for i, head in enumerate(header): 139 | print(' {name: <{alignment}}'.format(name=head, alignment=cell_width[i]), end='') 140 | 141 | print('') 142 | print('='*slen) 143 | for row in table: 144 | for i, val in enumerate(row): 145 | print(' {val: <{alignment}}'.format(val=val, alignment=cell_width[i]), end='') 146 | print('') 147 | print('-'*slen) 148 | 149 | 150 | if __name__ == "__main__": 151 | # Tests 152 | print_pkg_versions() 153 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from utils.sys_utils import * 3 | from utils.vsum_tools import * 4 | from tools.funcs import * 5 | import importlib 6 | import time 7 | 8 | 9 | def eval_split(hps, splits_filename, data_dir="test"): 10 | 11 | print("\n") 12 | model = engine.sumnet(hps) 13 | model.initialize() 14 | model.load_datasets() 15 | model.load_split_file(splits_filename) 16 | 17 | val_fscores = [] 18 | for split_id in range(len(model.splits)): 19 | model.select_split(split_id) 20 | weights_filename = model.lookup_weights_file(data_dir) 21 | print("Loading model:", weights_filename) 22 | model.load_model(weights_filename) 23 | val_fscore, video_scores = model.eval(model.test_keys) 24 | val_fscores.append(val_fscore) 25 | val_fscore_avg = np.mean(val_fscores) 26 | 27 | if hps.verbose: 28 | video_scores = [["No.", "Video", "F-score"]] + video_scores 29 | print_table(video_scores, cell_width=[4, 45, 5]) 30 | 31 | print("Avg F-score: ", val_fscore) 32 | print("") 33 | 34 | print("Total AVG F-score: ", val_fscore_avg) 35 | return val_fscore_avg 36 | 37 | 38 | def train(hps): 39 | os.makedirs(hps.output_dir, exist_ok=True) 40 | os.makedirs(os.path.join(hps.output_dir, "models"), exist_ok=True) 41 | 42 | # Create a file to collect results from all splits 43 | f = open(hps.output_dir + "/results.txt", "wt") 44 | 45 | for split_filename in hps.splits: 46 | dataset_name, dataset_type, splits = parse_splits_filename(split_filename) 47 | 48 | # For no augmentation use only a dataset corresponding to the split file 49 | datasets = None 50 | if dataset_type == "": 51 | datasets = hps.get_dataset_by_name(dataset_name) 52 | 53 | if datasets is None: 54 | datasets = hps.datasets 55 | 56 | f_avg = 0 57 | n_folds = len(splits) 58 | for split_id in range(n_folds): 59 | model = engine.sumnet(hps) 60 | model.initialize() 61 | model.load_datasets(datasets=datasets) 62 | model.load_split_file(splits_file=split_filename) 63 | model.select_split(split_id=split_id) 64 | 65 | fscore, fscore_epoch = model.train(output_dir=hps.output_dir) 66 | f_avg += fscore 67 | 68 | # Log F-score for this split_id 69 | f.write( 70 | split_filename 71 | + ", " 72 | + str(split_id) 73 | + ", " 74 | + str(fscore) 75 | + ", " 76 | + str(fscore_epoch) 77 | + "\n" 78 | ) 79 | f.flush() 80 | 81 | # Save model with the highest F score 82 | _, log_file = os.path.split(split_filename) 83 | log_dir, _ = os.path.splitext(log_file) 84 | log_dir += "_" + str(split_id) 85 | log_file = os.path.join(hps.output_dir, "models", log_dir) + ".tar.pth" 86 | 87 | os.makedirs( 88 | os.path.join( 89 | hps.output_dir, 90 | "models", 91 | ), 92 | exist_ok=True, 93 | ) 94 | os.system( 95 | "mv " 96 | + hps.output_dir 97 | + "/models_temp/" 98 | + log_dir 99 | + "/" 100 | + str(fscore_epoch) 101 | + "_*.pth.tar " 102 | + log_file 103 | ) 104 | os.system("rm -rf " + hps.output_dir + "/models_temp") 105 | 106 | print( 107 | "Split: {0:} Best F-score: {1:0.5f} Model: {2:}".format( 108 | split_filename, fscore, log_file 109 | ) 110 | ) 111 | 112 | # Write average F-score for all splits to the results.txt file 113 | f_avg /= n_folds 114 | f.write(split_filename + ", " + str("avg") + ", " + str(f_avg) + "\n") 115 | f.flush() 116 | 117 | f.close() 118 | 119 | 120 | if __name__ == "__main__": 121 | print_pkg_versions() 122 | 123 | parser = argparse.ArgumentParser("PyTorch implementation of VJMHT") 124 | parser.add_argument( 125 | "-d", 126 | "--datasets", 127 | type=str, 128 | help="Path to a comma separated list of h5 datasets", 129 | ) 130 | parser.add_argument( 131 | "-s", "--splits", type=str, help="Comma separated list of split files." 132 | ) 133 | parser.add_argument("-e", "--eval", action="store_true", help="eval") 134 | parser.add_argument( 135 | "-v", "--verbose", action="store_true", help="Prints out more messages" 136 | ) 137 | parser.add_argument("-c", "--config", type=str, default="cfg", help="config file") 138 | args = parser.parse_args() 139 | 140 | # MAIN 141 | # ====================== 142 | args.config = args.config.replace("/", ".").replace(".py", "") 143 | config = importlib.import_module(args.config) 144 | EXP = args.config.split(".")[-1].upper() 145 | args.output_dir = os.path.join("results", EXP) 146 | os.makedirs(args.output_dir, exist_ok=True) 147 | hps = config.HParameters() 148 | engine = importlib.import_module("tools." + hps.engine) 149 | hps.load_from_args(args.__dict__) 150 | 151 | print("Parameters:") 152 | print("----------------------------------------------------------------------") 153 | print(hps) 154 | 155 | if not hps.eval: 156 | train(hps) 157 | 158 | results = [["No", "Split", "Mean F-score"]] 159 | 160 | start = time.time() 161 | for i, split_filename in enumerate(hps.splits): 162 | f_score = eval_split(hps, split_filename, data_dir=hps.output_dir) 163 | results.append([i + 1, split_filename, str(round(f_score * 100.0, 3)) + "%"]) 164 | end = time.time() 165 | print(end - start) 166 | 167 | print("\nFinal Results:") 168 | print_table(results) 169 | sys.exit(0) 170 | -------------------------------------------------------------------------------- /models/reccotrans.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import math 4 | from torch.nn import TransformerEncoder, TransformerEncoderLayer 5 | from torch.nn.utils.rnn import pad_sequence 6 | 7 | __all__ = ["DSN"] 8 | 9 | 10 | class PositionalEncoding(nn.Module): 11 | def __init__(self, d_model, dropout=0.1, max_len=5000): 12 | super(PositionalEncoding, self).__init__() 13 | self.dropout = nn.Dropout(p=dropout) 14 | 15 | pe = torch.zeros(max_len, d_model) 16 | position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) 17 | div_term = torch.exp( 18 | torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model) 19 | ) 20 | pe[:, 0::2] = torch.sin(position * div_term) 21 | pe[:, 1::2] = torch.cos(position * div_term) 22 | pe = pe.unsqueeze(0).transpose(0, 1) 23 | self.register_buffer("pe", pe) 24 | 25 | def forward(self, x): 26 | x = x + self.pe[: x.size(0), :] 27 | return self.dropout(x) 28 | 29 | 30 | class TransformerModel(nn.Module): 31 | def __init__( 32 | self, out_dim=512, in_dim=1024, nhead=4, nhid=2048, nlayers=3, dropout=0.1 33 | ): 34 | super(TransformerModel, self).__init__() 35 | 36 | self.pos_encoder = PositionalEncoding(in_dim, dropout) 37 | encoder_layers = TransformerEncoderLayer(in_dim, nhead, nhid, dropout) 38 | self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers) 39 | self.decoder = nn.Linear(in_dim, out_dim) 40 | 41 | # self.init_weights() 42 | # 43 | # 44 | # def init_weights(self): 45 | # initrange = 0.1 46 | # self.decoder.bias.data.zero_() 47 | # self.decoder.weight.data.uniform_(-initrange, initrange) 48 | 49 | def forward(self, src, src_mask=None): 50 | src = self.pos_encoder(src) 51 | feature = self.transformer_encoder(src, mask=src_mask) 52 | output = self.decoder(feature) 53 | return output, feature 54 | 55 | 56 | class DSN(nn.Module): 57 | """Deep Summarization Network""" 58 | 59 | def __init__(self, hps): 60 | super(DSN, self).__init__() 61 | self.trans1 = TransformerModel( 62 | out_dim=hps.shot_dim, 63 | in_dim=1024, 64 | nhead=hps.nhead1, 65 | nhid=hps.nhid1, 66 | nlayers=hps.nlayers1, 67 | dropout=hps.dropout1, 68 | ) 69 | self.trans2 = TransformerModel( 70 | out_dim=hps.shot_dim, 71 | in_dim=hps.shot_dim, 72 | nhead=hps.nhead2, 73 | nhid=hps.nhid2, 74 | nlayers=hps.nlayers2, 75 | dropout=hps.dropout2, 76 | ) 77 | 78 | self.fused = nn.Linear(hps.shot_dim * 2, 1) 79 | self.extra = nn.Embedding(1, 1024) 80 | self.extra_shot = nn.Embedding(1, hps.shot_dim) 81 | 82 | def postprocess(self, p, boundary): 83 | if len(p) == 0: 84 | return 85 | temp = [ 86 | p[i : i + 1].expand(boundary[i, 1] - boundary[i, 0] + 1) 87 | for i in range(p.shape[0]) 88 | ] 89 | probs = torch.cat(temp, 0) 90 | return probs[::15] 91 | 92 | def forward(self, inputs1, inputs2=None): 93 | if self.training: 94 | shots1, boundary1 = inputs1["shots"], inputs1["boundary"] 95 | shots2, boundary2 = inputs2["shots"], inputs2["boundary"] 96 | n_shots1 = len(shots1) 97 | shots = shots1 + shots2 98 | shots = [torch.cat([self.extra.weight, shot], 0) for shot in shots] 99 | shots = pad_sequence(shots) 100 | 101 | h, _ = self.trans1(shots) 102 | h = h[0, :, :].unsqueeze(1) 103 | seq1, seq2 = h[:n_shots1], h[n_shots1:] 104 | token1 = token2 = self.extra_shot.weight.unsqueeze(0) 105 | seq = torch.cat([token1, seq1, seq2, token2], 0) 106 | src_mask = torch.zeros((len(seq), len(seq))) 107 | src_mask[0, n_shots1 + 1 :] = 1 108 | src_mask[-1, 0 : n_shots1 + 1] = 1 109 | src_mask = src_mask.bool().to(seq.device) 110 | decoded, feature = self.trans2(seq, src_mask=src_mask) 111 | decoded_seq1, decoded_seq2 = ( 112 | decoded[1 : n_shots1 + 1], 113 | decoded[n_shots1 + 1 : -1], 114 | ) 115 | decoded_token1, decoded_token2 = decoded[0:1].expand_as( 116 | decoded_seq1 117 | ), decoded[-1:].expand_as(decoded_seq2) 118 | decoded_seq1, decoded_seq2 = torch.cat( 119 | (decoded_seq1, decoded_token1), 2 120 | ), torch.cat((decoded_seq2, decoded_token2), 2) 121 | decoded_seq = torch.cat((decoded_seq1, decoded_seq2), 0) 122 | probs = self.fused(decoded_seq).squeeze() 123 | p1, p2 = probs[:n_shots1], probs[n_shots1:] 124 | feature_video1 = feature[0] 125 | weighted_shots1 = seq1 * p1.unsqueeze(1).unsqueeze(1) 126 | weighted_shots = torch.cat( 127 | [self.extra_shot.weight.unsqueeze(0), weighted_shots1], 0 128 | ) 129 | _, feature_key = self.trans2(weighted_shots) 130 | feature_summary = feature_key[0] 131 | rec_loss = (feature_video1 - feature_summary).pow(2).mean() 132 | p1, p2 = self.postprocess(p1, boundary1), self.postprocess(p2, boundary2) 133 | return p1, p2, rec_loss 134 | else: 135 | shots, boundary1 = inputs1["shots"], inputs1["boundary"] 136 | n_shots1 = len(shots) 137 | shots = [torch.cat([self.extra.weight, shot], 0) for shot in shots] 138 | shots = pad_sequence(shots) 139 | h, _ = self.trans1(shots) 140 | h = h[0, :, :].unsqueeze(1) 141 | seq1 = h[:n_shots1] 142 | seq = torch.cat([self.extra_shot.weight.unsqueeze(0), seq1], 0) 143 | decoded, _ = self.trans2(seq) 144 | decoded_seq1 = decoded[1 : n_shots1 + 1] 145 | decoded_token1 = decoded[0:1].expand_as(decoded_seq1) 146 | decoded_seq1 = torch.cat((decoded_seq1, decoded_token1), 2) 147 | probs = self.fused(decoded_seq1).squeeze() 148 | p1 = self.postprocess(probs, boundary1) 149 | return p1 150 | -------------------------------------------------------------------------------- /splits/summe_trans_splits.json: -------------------------------------------------------------------------------- 1 | [{"train_keys": ["eccv16_dataset_youtube_google_pool5/video_11", "eccv16_dataset_youtube_google_pool5/video_12", "eccv16_dataset_youtube_google_pool5/video_13", "eccv16_dataset_youtube_google_pool5/video_14", "eccv16_dataset_youtube_google_pool5/video_15", "eccv16_dataset_youtube_google_pool5/video_16", "eccv16_dataset_youtube_google_pool5/video_17", "eccv16_dataset_youtube_google_pool5/video_18", "eccv16_dataset_youtube_google_pool5/video_19", "eccv16_dataset_youtube_google_pool5/video_20", "eccv16_dataset_youtube_google_pool5/video_21", "eccv16_dataset_youtube_google_pool5/video_23", "eccv16_dataset_youtube_google_pool5/video_24", "eccv16_dataset_youtube_google_pool5/video_25", "eccv16_dataset_youtube_google_pool5/video_26", "eccv16_dataset_youtube_google_pool5/video_27", "eccv16_dataset_youtube_google_pool5/video_28", "eccv16_dataset_youtube_google_pool5/video_29", "eccv16_dataset_youtube_google_pool5/video_30", "eccv16_dataset_youtube_google_pool5/video_31", "eccv16_dataset_youtube_google_pool5/video_32", "eccv16_dataset_youtube_google_pool5/video_33", "eccv16_dataset_youtube_google_pool5/video_34", "eccv16_dataset_youtube_google_pool5/video_35", "eccv16_dataset_youtube_google_pool5/video_36", "eccv16_dataset_youtube_google_pool5/video_37", "eccv16_dataset_youtube_google_pool5/video_38", "eccv16_dataset_youtube_google_pool5/video_39", "eccv16_dataset_youtube_google_pool5/video_40", "eccv16_dataset_youtube_google_pool5/video_41", "eccv16_dataset_youtube_google_pool5/video_42", "eccv16_dataset_youtube_google_pool5/video_43", "eccv16_dataset_youtube_google_pool5/video_44", "eccv16_dataset_youtube_google_pool5/video_45", "eccv16_dataset_youtube_google_pool5/video_46", "eccv16_dataset_youtube_google_pool5/video_47", "eccv16_dataset_youtube_google_pool5/video_48", "eccv16_dataset_youtube_google_pool5/video_49", "eccv16_dataset_youtube_google_pool5/video_50", "eccv16_dataset_ovp_google_pool5/video_1", "eccv16_dataset_ovp_google_pool5/video_10", "eccv16_dataset_ovp_google_pool5/video_11", "eccv16_dataset_ovp_google_pool5/video_12", "eccv16_dataset_ovp_google_pool5/video_13", "eccv16_dataset_ovp_google_pool5/video_14", "eccv16_dataset_ovp_google_pool5/video_15", "eccv16_dataset_ovp_google_pool5/video_16", "eccv16_dataset_ovp_google_pool5/video_17", "eccv16_dataset_ovp_google_pool5/video_18", "eccv16_dataset_ovp_google_pool5/video_19", "eccv16_dataset_ovp_google_pool5/video_2", "eccv16_dataset_ovp_google_pool5/video_20", "eccv16_dataset_ovp_google_pool5/video_21", "eccv16_dataset_ovp_google_pool5/video_22", "eccv16_dataset_ovp_google_pool5/video_23", "eccv16_dataset_ovp_google_pool5/video_24", "eccv16_dataset_ovp_google_pool5/video_25", "eccv16_dataset_ovp_google_pool5/video_26", "eccv16_dataset_ovp_google_pool5/video_27", "eccv16_dataset_ovp_google_pool5/video_28", "eccv16_dataset_ovp_google_pool5/video_29", "eccv16_dataset_ovp_google_pool5/video_3", "eccv16_dataset_ovp_google_pool5/video_30", "eccv16_dataset_ovp_google_pool5/video_31", "eccv16_dataset_ovp_google_pool5/video_32", "eccv16_dataset_ovp_google_pool5/video_33", "eccv16_dataset_ovp_google_pool5/video_34", "eccv16_dataset_ovp_google_pool5/video_35", "eccv16_dataset_ovp_google_pool5/video_36", "eccv16_dataset_ovp_google_pool5/video_37", "eccv16_dataset_ovp_google_pool5/video_38", "eccv16_dataset_ovp_google_pool5/video_39", "eccv16_dataset_ovp_google_pool5/video_4", "eccv16_dataset_ovp_google_pool5/video_40", "eccv16_dataset_ovp_google_pool5/video_41", "eccv16_dataset_ovp_google_pool5/video_42", "eccv16_dataset_ovp_google_pool5/video_43", "eccv16_dataset_ovp_google_pool5/video_44", "eccv16_dataset_ovp_google_pool5/video_45", "eccv16_dataset_ovp_google_pool5/video_46", "eccv16_dataset_ovp_google_pool5/video_47", "eccv16_dataset_ovp_google_pool5/video_48", "eccv16_dataset_ovp_google_pool5/video_49", "eccv16_dataset_ovp_google_pool5/video_5", "eccv16_dataset_ovp_google_pool5/video_50", "eccv16_dataset_ovp_google_pool5/video_6", "eccv16_dataset_ovp_google_pool5/video_7", "eccv16_dataset_ovp_google_pool5/video_8", "eccv16_dataset_ovp_google_pool5/video_9", "eccv16_dataset_tvsum_google_pool5/video_1", "eccv16_dataset_tvsum_google_pool5/video_10", "eccv16_dataset_tvsum_google_pool5/video_11", "eccv16_dataset_tvsum_google_pool5/video_12", "eccv16_dataset_tvsum_google_pool5/video_13", "eccv16_dataset_tvsum_google_pool5/video_14", "eccv16_dataset_tvsum_google_pool5/video_15", "eccv16_dataset_tvsum_google_pool5/video_16", "eccv16_dataset_tvsum_google_pool5/video_17", "eccv16_dataset_tvsum_google_pool5/video_18", "eccv16_dataset_tvsum_google_pool5/video_19", "eccv16_dataset_tvsum_google_pool5/video_2", "eccv16_dataset_tvsum_google_pool5/video_20", "eccv16_dataset_tvsum_google_pool5/video_21", "eccv16_dataset_tvsum_google_pool5/video_22", "eccv16_dataset_tvsum_google_pool5/video_23", "eccv16_dataset_tvsum_google_pool5/video_24", "eccv16_dataset_tvsum_google_pool5/video_25", "eccv16_dataset_tvsum_google_pool5/video_26", "eccv16_dataset_tvsum_google_pool5/video_27", "eccv16_dataset_tvsum_google_pool5/video_28", "eccv16_dataset_tvsum_google_pool5/video_29", "eccv16_dataset_tvsum_google_pool5/video_3", "eccv16_dataset_tvsum_google_pool5/video_30", "eccv16_dataset_tvsum_google_pool5/video_31", "eccv16_dataset_tvsum_google_pool5/video_32", "eccv16_dataset_tvsum_google_pool5/video_33", "eccv16_dataset_tvsum_google_pool5/video_34", "eccv16_dataset_tvsum_google_pool5/video_35", "eccv16_dataset_tvsum_google_pool5/video_36", "eccv16_dataset_tvsum_google_pool5/video_37", "eccv16_dataset_tvsum_google_pool5/video_38", "eccv16_dataset_tvsum_google_pool5/video_39", "eccv16_dataset_tvsum_google_pool5/video_4", "eccv16_dataset_tvsum_google_pool5/video_40", "eccv16_dataset_tvsum_google_pool5/video_41", "eccv16_dataset_tvsum_google_pool5/video_42", "eccv16_dataset_tvsum_google_pool5/video_43", "eccv16_dataset_tvsum_google_pool5/video_44", "eccv16_dataset_tvsum_google_pool5/video_45", "eccv16_dataset_tvsum_google_pool5/video_46", "eccv16_dataset_tvsum_google_pool5/video_47", "eccv16_dataset_tvsum_google_pool5/video_48", "eccv16_dataset_tvsum_google_pool5/video_49", "eccv16_dataset_tvsum_google_pool5/video_5", "eccv16_dataset_tvsum_google_pool5/video_50", "eccv16_dataset_tvsum_google_pool5/video_6", "eccv16_dataset_tvsum_google_pool5/video_7", "eccv16_dataset_tvsum_google_pool5/video_8", "eccv16_dataset_tvsum_google_pool5/video_9"], "test_keys": ["eccv16_dataset_summe_google_pool5/video_1", "eccv16_dataset_summe_google_pool5/video_10", "eccv16_dataset_summe_google_pool5/video_11", "eccv16_dataset_summe_google_pool5/video_12", "eccv16_dataset_summe_google_pool5/video_13", "eccv16_dataset_summe_google_pool5/video_14", "eccv16_dataset_summe_google_pool5/video_15", "eccv16_dataset_summe_google_pool5/video_16", "eccv16_dataset_summe_google_pool5/video_17", "eccv16_dataset_summe_google_pool5/video_18", "eccv16_dataset_summe_google_pool5/video_19", "eccv16_dataset_summe_google_pool5/video_2", "eccv16_dataset_summe_google_pool5/video_20", "eccv16_dataset_summe_google_pool5/video_21", "eccv16_dataset_summe_google_pool5/video_22", "eccv16_dataset_summe_google_pool5/video_23", "eccv16_dataset_summe_google_pool5/video_24", "eccv16_dataset_summe_google_pool5/video_25", "eccv16_dataset_summe_google_pool5/video_3", "eccv16_dataset_summe_google_pool5/video_4", "eccv16_dataset_summe_google_pool5/video_5", "eccv16_dataset_summe_google_pool5/video_6", "eccv16_dataset_summe_google_pool5/video_7", "eccv16_dataset_summe_google_pool5/video_8", "eccv16_dataset_summe_google_pool5/video_9"]}] -------------------------------------------------------------------------------- /splits/tvsum_trans_splits.json: -------------------------------------------------------------------------------- 1 | [{"train_keys": ["eccv16_dataset_summe_google_pool5/video_1", "eccv16_dataset_summe_google_pool5/video_10", "eccv16_dataset_summe_google_pool5/video_11", "eccv16_dataset_summe_google_pool5/video_12", "eccv16_dataset_summe_google_pool5/video_13", "eccv16_dataset_summe_google_pool5/video_14", "eccv16_dataset_summe_google_pool5/video_15", "eccv16_dataset_summe_google_pool5/video_16", "eccv16_dataset_summe_google_pool5/video_17", "eccv16_dataset_summe_google_pool5/video_18", "eccv16_dataset_summe_google_pool5/video_19", "eccv16_dataset_summe_google_pool5/video_2", "eccv16_dataset_summe_google_pool5/video_20", "eccv16_dataset_summe_google_pool5/video_21", "eccv16_dataset_summe_google_pool5/video_22", "eccv16_dataset_summe_google_pool5/video_23", "eccv16_dataset_summe_google_pool5/video_24", "eccv16_dataset_summe_google_pool5/video_25", "eccv16_dataset_summe_google_pool5/video_3", "eccv16_dataset_summe_google_pool5/video_4", "eccv16_dataset_summe_google_pool5/video_5", "eccv16_dataset_summe_google_pool5/video_6", "eccv16_dataset_summe_google_pool5/video_7", "eccv16_dataset_summe_google_pool5/video_8", "eccv16_dataset_summe_google_pool5/video_9", "eccv16_dataset_youtube_google_pool5/video_11", "eccv16_dataset_youtube_google_pool5/video_12", "eccv16_dataset_youtube_google_pool5/video_13", "eccv16_dataset_youtube_google_pool5/video_14", "eccv16_dataset_youtube_google_pool5/video_15", "eccv16_dataset_youtube_google_pool5/video_16", "eccv16_dataset_youtube_google_pool5/video_17", "eccv16_dataset_youtube_google_pool5/video_18", "eccv16_dataset_youtube_google_pool5/video_19", "eccv16_dataset_youtube_google_pool5/video_20", "eccv16_dataset_youtube_google_pool5/video_21", "eccv16_dataset_youtube_google_pool5/video_23", "eccv16_dataset_youtube_google_pool5/video_24", "eccv16_dataset_youtube_google_pool5/video_25", "eccv16_dataset_youtube_google_pool5/video_26", "eccv16_dataset_youtube_google_pool5/video_27", "eccv16_dataset_youtube_google_pool5/video_28", "eccv16_dataset_youtube_google_pool5/video_29", "eccv16_dataset_youtube_google_pool5/video_30", "eccv16_dataset_youtube_google_pool5/video_31", "eccv16_dataset_youtube_google_pool5/video_32", "eccv16_dataset_youtube_google_pool5/video_33", "eccv16_dataset_youtube_google_pool5/video_34", "eccv16_dataset_youtube_google_pool5/video_35", "eccv16_dataset_youtube_google_pool5/video_36", "eccv16_dataset_youtube_google_pool5/video_37", "eccv16_dataset_youtube_google_pool5/video_38", "eccv16_dataset_youtube_google_pool5/video_39", "eccv16_dataset_youtube_google_pool5/video_40", "eccv16_dataset_youtube_google_pool5/video_41", "eccv16_dataset_youtube_google_pool5/video_42", "eccv16_dataset_youtube_google_pool5/video_43", "eccv16_dataset_youtube_google_pool5/video_44", "eccv16_dataset_youtube_google_pool5/video_45", "eccv16_dataset_youtube_google_pool5/video_46", "eccv16_dataset_youtube_google_pool5/video_47", "eccv16_dataset_youtube_google_pool5/video_48", "eccv16_dataset_youtube_google_pool5/video_49", "eccv16_dataset_youtube_google_pool5/video_50", "eccv16_dataset_ovp_google_pool5/video_1", "eccv16_dataset_ovp_google_pool5/video_10", "eccv16_dataset_ovp_google_pool5/video_11", "eccv16_dataset_ovp_google_pool5/video_12", "eccv16_dataset_ovp_google_pool5/video_13", "eccv16_dataset_ovp_google_pool5/video_14", "eccv16_dataset_ovp_google_pool5/video_15", "eccv16_dataset_ovp_google_pool5/video_16", "eccv16_dataset_ovp_google_pool5/video_17", "eccv16_dataset_ovp_google_pool5/video_18", "eccv16_dataset_ovp_google_pool5/video_19", "eccv16_dataset_ovp_google_pool5/video_2", "eccv16_dataset_ovp_google_pool5/video_20", "eccv16_dataset_ovp_google_pool5/video_21", "eccv16_dataset_ovp_google_pool5/video_22", "eccv16_dataset_ovp_google_pool5/video_23", "eccv16_dataset_ovp_google_pool5/video_24", "eccv16_dataset_ovp_google_pool5/video_25", "eccv16_dataset_ovp_google_pool5/video_26", "eccv16_dataset_ovp_google_pool5/video_27", "eccv16_dataset_ovp_google_pool5/video_28", "eccv16_dataset_ovp_google_pool5/video_29", "eccv16_dataset_ovp_google_pool5/video_3", "eccv16_dataset_ovp_google_pool5/video_30", "eccv16_dataset_ovp_google_pool5/video_31", "eccv16_dataset_ovp_google_pool5/video_32", "eccv16_dataset_ovp_google_pool5/video_33", "eccv16_dataset_ovp_google_pool5/video_34", "eccv16_dataset_ovp_google_pool5/video_35", "eccv16_dataset_ovp_google_pool5/video_36", "eccv16_dataset_ovp_google_pool5/video_37", "eccv16_dataset_ovp_google_pool5/video_38", "eccv16_dataset_ovp_google_pool5/video_39", "eccv16_dataset_ovp_google_pool5/video_4", "eccv16_dataset_ovp_google_pool5/video_40", "eccv16_dataset_ovp_google_pool5/video_41", "eccv16_dataset_ovp_google_pool5/video_42", "eccv16_dataset_ovp_google_pool5/video_43", "eccv16_dataset_ovp_google_pool5/video_44", "eccv16_dataset_ovp_google_pool5/video_45", "eccv16_dataset_ovp_google_pool5/video_46", "eccv16_dataset_ovp_google_pool5/video_47", "eccv16_dataset_ovp_google_pool5/video_48", "eccv16_dataset_ovp_google_pool5/video_49", "eccv16_dataset_ovp_google_pool5/video_5", "eccv16_dataset_ovp_google_pool5/video_50", "eccv16_dataset_ovp_google_pool5/video_6", "eccv16_dataset_ovp_google_pool5/video_7", "eccv16_dataset_ovp_google_pool5/video_8", "eccv16_dataset_ovp_google_pool5/video_9"], "test_keys": ["eccv16_dataset_tvsum_google_pool5/video_1", "eccv16_dataset_tvsum_google_pool5/video_10", "eccv16_dataset_tvsum_google_pool5/video_11", "eccv16_dataset_tvsum_google_pool5/video_12", "eccv16_dataset_tvsum_google_pool5/video_13", "eccv16_dataset_tvsum_google_pool5/video_14", "eccv16_dataset_tvsum_google_pool5/video_15", "eccv16_dataset_tvsum_google_pool5/video_16", "eccv16_dataset_tvsum_google_pool5/video_17", "eccv16_dataset_tvsum_google_pool5/video_18", "eccv16_dataset_tvsum_google_pool5/video_19", "eccv16_dataset_tvsum_google_pool5/video_2", "eccv16_dataset_tvsum_google_pool5/video_20", "eccv16_dataset_tvsum_google_pool5/video_21", "eccv16_dataset_tvsum_google_pool5/video_22", "eccv16_dataset_tvsum_google_pool5/video_23", "eccv16_dataset_tvsum_google_pool5/video_24", "eccv16_dataset_tvsum_google_pool5/video_25", "eccv16_dataset_tvsum_google_pool5/video_26", "eccv16_dataset_tvsum_google_pool5/video_27", "eccv16_dataset_tvsum_google_pool5/video_28", "eccv16_dataset_tvsum_google_pool5/video_29", "eccv16_dataset_tvsum_google_pool5/video_3", "eccv16_dataset_tvsum_google_pool5/video_30", "eccv16_dataset_tvsum_google_pool5/video_31", "eccv16_dataset_tvsum_google_pool5/video_32", "eccv16_dataset_tvsum_google_pool5/video_33", "eccv16_dataset_tvsum_google_pool5/video_34", "eccv16_dataset_tvsum_google_pool5/video_35", "eccv16_dataset_tvsum_google_pool5/video_36", "eccv16_dataset_tvsum_google_pool5/video_37", "eccv16_dataset_tvsum_google_pool5/video_38", "eccv16_dataset_tvsum_google_pool5/video_39", "eccv16_dataset_tvsum_google_pool5/video_4", "eccv16_dataset_tvsum_google_pool5/video_40", "eccv16_dataset_tvsum_google_pool5/video_41", "eccv16_dataset_tvsum_google_pool5/video_42", "eccv16_dataset_tvsum_google_pool5/video_43", "eccv16_dataset_tvsum_google_pool5/video_44", "eccv16_dataset_tvsum_google_pool5/video_45", "eccv16_dataset_tvsum_google_pool5/video_46", "eccv16_dataset_tvsum_google_pool5/video_47", "eccv16_dataset_tvsum_google_pool5/video_48", "eccv16_dataset_tvsum_google_pool5/video_49", "eccv16_dataset_tvsum_google_pool5/video_5", "eccv16_dataset_tvsum_google_pool5/video_50", "eccv16_dataset_tvsum_google_pool5/video_6", "eccv16_dataset_tvsum_google_pool5/video_7", "eccv16_dataset_tvsum_google_pool5/video_8", "eccv16_dataset_tvsum_google_pool5/video_9"]}] -------------------------------------------------------------------------------- /utils/vsum_tools.py: -------------------------------------------------------------------------------- 1 | """' 2 | Courtesy of KaiyangZhou 3 | https://github.com/KaiyangZhou/pytorch-vsumm-reinforce 4 | 5 | @article{zhou2017reinforcevsumm, 6 | title={Deep Reinforcement Learning for Unsupervised Video Summarization with Diversity-Representativeness Reward}, 7 | author={Zhou, Kaiyang and Qiao, Yu and Xiang, Tao}, 8 | journal={arXiv:1801.00054}, 9 | year={2017} 10 | } 11 | 12 | Modifications by Jiri Fajtl 13 | - knapsack replaced with knapsack_ortools 14 | - added evaluate_user_summaries() for user summaries ground truth evaluation 15 | """ 16 | 17 | import numpy as np 18 | from utils.knapsack import knapsack_ortools 19 | import math 20 | 21 | 22 | def generate_summary( 23 | ypred, cps, n_frames, nfps, positions, proportion=0.15, method="knapsack" 24 | ): 25 | """Generate keyshot-based video summary i.e. a binary vector. 26 | Args: 27 | --------------------------------------------- 28 | - ypred: predicted importance scores. 29 | - cps: change points, 2D matrix, each row contains a segment. 30 | - n_frames: original number of frames. 31 | - nfps: number of frames per segment. 32 | - positions: positions of subsampled frames in the original video. 33 | - proportion: length of video summary (compared to original video length). 34 | - method: defines how shots are selected, ['knapsack', 'rank']. 35 | """ 36 | n_segs = cps.shape[0] 37 | frame_scores = np.zeros((n_frames), dtype=np.float32) 38 | if positions.dtype != int: 39 | positions = positions.astype(np.int32) 40 | if positions[-1] != n_frames: 41 | positions = np.concatenate([positions, [n_frames]]) 42 | for i in range(len(positions) - 1): 43 | pos_left, pos_right = positions[i], positions[i + 1] 44 | if i == len(ypred): 45 | frame_scores[pos_left:pos_right] = 0 46 | else: 47 | frame_scores[pos_left:pos_right] = ypred[i] 48 | 49 | seg_score = [] 50 | for seg_idx in range(n_segs): 51 | start, end = int(cps[seg_idx, 0]), int(cps[seg_idx, 1] + 1) 52 | scores = frame_scores[start:end] 53 | seg_score.append(float(scores.mean())) 54 | 55 | limits = int(math.floor(n_frames * proportion)) 56 | 57 | if method == "knapsack": 58 | # picks = knapsack_dp(seg_score, nfps, n_segs, limits) 59 | picks = knapsack_ortools(seg_score, nfps, n_segs, limits) 60 | elif method == "rank": 61 | order = np.argsort(seg_score)[::-1].tolist() 62 | picks = [] 63 | total_len = 0 64 | for i in order: 65 | if total_len + nfps[i] < limits: 66 | picks.append(i) 67 | total_len += nfps[i] 68 | else: 69 | raise KeyError("Unknown method {}".format(method)) 70 | 71 | summary = np.zeros((1), dtype=np.float32) # this element should be deleted 72 | for seg_idx in range(n_segs): 73 | nf = nfps[seg_idx] 74 | if seg_idx in picks: 75 | tmp = np.ones((nf), dtype=np.float32) 76 | else: 77 | tmp = np.zeros((nf), dtype=np.float32) 78 | summary = np.concatenate((summary, tmp)) 79 | 80 | summary = np.delete(summary, 0) # delete the first element 81 | return summary 82 | 83 | 84 | def evaluate_summary(machine_summary, user_summary, eval_metric="avg"): 85 | """Compare machine summary with user summary (keyshot-based). 86 | Args: 87 | -------------------------------- 88 | machine_summary and user_summary should be binary vectors of ndarray type. 89 | eval_metric = {'avg', 'max'} 90 | 'avg' averages results of comparing multiple human summaries. 91 | 'max' takes the maximum (best) out of multiple comparisons. 92 | """ 93 | machine_summary = machine_summary.astype(np.float32) 94 | user_summary = user_summary.astype(np.float32) 95 | # pdb.set_trace() 96 | n_users, n_frames = user_summary.shape 97 | 98 | # binarization 99 | machine_summary[machine_summary > 0] = 1 100 | user_summary[user_summary > 0] = 1 101 | 102 | if len(machine_summary) > n_frames: 103 | machine_summary = machine_summary[:n_frames] 104 | elif len(machine_summary) < n_frames: 105 | zero_padding = np.zeros((n_frames - len(machine_summary))) 106 | machine_summary = np.concatenate([machine_summary, zero_padding]) 107 | 108 | f_scores = [] 109 | prec_arr = [] 110 | rec_arr = [] 111 | 112 | for user_idx in range(n_users): 113 | gt_summary = user_summary[user_idx, :] 114 | overlap_duration = (machine_summary * gt_summary).sum() 115 | precision = overlap_duration / (machine_summary.sum() + 1e-8) 116 | recall = overlap_duration / (gt_summary.sum() + 1e-8) 117 | if precision == 0 and recall == 0: 118 | f_score = 0.0 119 | else: 120 | f_score = (2 * precision * recall) / (precision + recall) 121 | f_scores.append(f_score) 122 | prec_arr.append(precision) 123 | rec_arr.append(recall) 124 | if eval_metric == "avg": 125 | final_f_score = np.mean(f_scores) 126 | final_prec = np.mean(prec_arr) 127 | final_rec = np.mean(rec_arr) 128 | elif eval_metric == "max": 129 | final_f_score = np.max(f_scores) 130 | max_idx = np.argmax(f_scores) 131 | final_prec = prec_arr[max_idx] 132 | final_rec = rec_arr[max_idx] 133 | 134 | return final_f_score, final_prec, final_rec 135 | 136 | 137 | def evaluate_user_summaries(user_summary, eval_metric="avg"): 138 | """Compare machine summary with user summary (keyshot-based). 139 | Args: 140 | -------------------------------- 141 | machine_summary and user_summary should be binary vectors of ndarray type. 142 | eval_metric = {'avg', 'max'} 143 | 'avg' averages results of comparing multiple human summaries. 144 | 'max' takes the maximum (best) out of multiple comparisons. 145 | """ 146 | user_summary = user_summary.astype(np.float32) 147 | n_users, n_frames = user_summary.shape 148 | 149 | # binarization 150 | user_summary[user_summary > 0] = 1 151 | 152 | f_scores = [] 153 | prec_arr = [] 154 | rec_arr = [] 155 | 156 | for user_idx in range(n_users): 157 | gt_summary = user_summary[user_idx, :] 158 | for other_user_idx in range(user_idx + 1, n_users): 159 | other_gt_summary = user_summary[other_user_idx, :] 160 | overlap_duration = (other_gt_summary * gt_summary).sum() 161 | precision = overlap_duration / (other_gt_summary.sum() + 1e-8) 162 | recall = overlap_duration / (gt_summary.sum() + 1e-8) 163 | # pdb.set_trace() 164 | if precision == 0 and recall == 0: 165 | f_score = 0.0 166 | else: 167 | f_score = (2 * precision * recall) / (precision + recall) 168 | f_scores.append(f_score) 169 | prec_arr.append(precision) 170 | rec_arr.append(recall) 171 | 172 | if eval_metric == "avg": 173 | final_f_score = np.mean(f_scores) 174 | final_prec = np.mean(prec_arr) 175 | final_rec = np.mean(rec_arr) 176 | elif eval_metric == "max": 177 | final_f_score = np.max(f_scores) 178 | max_idx = np.argmax(f_scores) 179 | final_prec = prec_arr[max_idx] 180 | final_rec = rec_arr[max_idx] 181 | 182 | return final_f_score, final_prec, final_rec 183 | -------------------------------------------------------------------------------- /splits/tvsum_splits.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "train_keys": [ 4 | "video_40", 5 | "video_21", 6 | "video_33", 7 | "video_17", 8 | "video_15", 9 | "video_24", 10 | "video_25", 11 | "video_2", 12 | "video_5", 13 | "video_34", 14 | "video_43", 15 | "video_14", 16 | "video_13", 17 | "video_47", 18 | "video_7", 19 | "video_28", 20 | "video_9", 21 | "video_41", 22 | "video_39", 23 | "video_12", 24 | "video_38", 25 | "video_44", 26 | "video_1", 27 | "video_27", 28 | "video_29", 29 | "video_4", 30 | "video_31", 31 | "video_11", 32 | "video_8", 33 | "video_20", 34 | "video_16", 35 | "video_32", 36 | "video_48", 37 | "video_35", 38 | "video_22", 39 | "video_10", 40 | "video_50", 41 | "video_49", 42 | "video_19", 43 | "video_30" 44 | ], 45 | "test_keys": [ 46 | "video_6", 47 | "video_42", 48 | "video_46", 49 | "video_18", 50 | "video_26", 51 | "video_3", 52 | "video_45", 53 | "video_37", 54 | "video_23", 55 | "video_36" 56 | ] 57 | }, 58 | { 59 | "train_keys": [ 60 | "video_6", 61 | "video_42", 62 | "video_46", 63 | "video_18", 64 | "video_26", 65 | "video_3", 66 | "video_45", 67 | "video_37", 68 | "video_23", 69 | "video_36", 70 | "video_43", 71 | "video_14", 72 | "video_13", 73 | "video_47", 74 | "video_7", 75 | "video_28", 76 | "video_9", 77 | "video_41", 78 | "video_39", 79 | "video_12", 80 | "video_38", 81 | "video_44", 82 | "video_1", 83 | "video_27", 84 | "video_29", 85 | "video_4", 86 | "video_31", 87 | "video_11", 88 | "video_8", 89 | "video_20", 90 | "video_16", 91 | "video_32", 92 | "video_48", 93 | "video_35", 94 | "video_22", 95 | "video_10", 96 | "video_50", 97 | "video_49", 98 | "video_19", 99 | "video_30" 100 | ], 101 | "test_keys": [ 102 | "video_40", 103 | "video_21", 104 | "video_33", 105 | "video_17", 106 | "video_15", 107 | "video_24", 108 | "video_25", 109 | "video_2", 110 | "video_5", 111 | "video_34" 112 | ] 113 | }, 114 | { 115 | "train_keys": [ 116 | "video_6", 117 | "video_42", 118 | "video_46", 119 | "video_18", 120 | "video_26", 121 | "video_3", 122 | "video_45", 123 | "video_37", 124 | "video_23", 125 | "video_36", 126 | "video_40", 127 | "video_21", 128 | "video_33", 129 | "video_17", 130 | "video_15", 131 | "video_24", 132 | "video_25", 133 | "video_2", 134 | "video_5", 135 | "video_34", 136 | "video_38", 137 | "video_44", 138 | "video_1", 139 | "video_27", 140 | "video_29", 141 | "video_4", 142 | "video_31", 143 | "video_11", 144 | "video_8", 145 | "video_20", 146 | "video_16", 147 | "video_32", 148 | "video_48", 149 | "video_35", 150 | "video_22", 151 | "video_10", 152 | "video_50", 153 | "video_49", 154 | "video_19", 155 | "video_30" 156 | ], 157 | "test_keys": [ 158 | "video_43", 159 | "video_14", 160 | "video_13", 161 | "video_47", 162 | "video_7", 163 | "video_28", 164 | "video_9", 165 | "video_41", 166 | "video_39", 167 | "video_12" 168 | ] 169 | }, 170 | { 171 | "train_keys": [ 172 | "video_6", 173 | "video_42", 174 | "video_46", 175 | "video_18", 176 | "video_26", 177 | "video_3", 178 | "video_45", 179 | "video_37", 180 | "video_23", 181 | "video_36", 182 | "video_40", 183 | "video_21", 184 | "video_33", 185 | "video_17", 186 | "video_15", 187 | "video_24", 188 | "video_25", 189 | "video_2", 190 | "video_5", 191 | "video_34", 192 | "video_43", 193 | "video_14", 194 | "video_13", 195 | "video_47", 196 | "video_7", 197 | "video_28", 198 | "video_9", 199 | "video_41", 200 | "video_39", 201 | "video_12", 202 | "video_16", 203 | "video_32", 204 | "video_48", 205 | "video_35", 206 | "video_22", 207 | "video_10", 208 | "video_50", 209 | "video_49", 210 | "video_19", 211 | "video_30" 212 | ], 213 | "test_keys": [ 214 | "video_38", 215 | "video_44", 216 | "video_1", 217 | "video_27", 218 | "video_29", 219 | "video_4", 220 | "video_31", 221 | "video_11", 222 | "video_8", 223 | "video_20" 224 | ] 225 | }, 226 | { 227 | "train_keys": [ 228 | "video_6", 229 | "video_42", 230 | "video_46", 231 | "video_18", 232 | "video_26", 233 | "video_3", 234 | "video_45", 235 | "video_37", 236 | "video_23", 237 | "video_36", 238 | "video_40", 239 | "video_21", 240 | "video_33", 241 | "video_17", 242 | "video_15", 243 | "video_24", 244 | "video_25", 245 | "video_2", 246 | "video_5", 247 | "video_34", 248 | "video_43", 249 | "video_14", 250 | "video_13", 251 | "video_47", 252 | "video_7", 253 | "video_28", 254 | "video_9", 255 | "video_41", 256 | "video_39", 257 | "video_12", 258 | "video_38", 259 | "video_44", 260 | "video_1", 261 | "video_27", 262 | "video_29", 263 | "video_4", 264 | "video_31", 265 | "video_11", 266 | "video_8", 267 | "video_20" 268 | ], 269 | "test_keys": [ 270 | "video_16", 271 | "video_32", 272 | "video_48", 273 | "video_35", 274 | "video_22", 275 | "video_10", 276 | "video_50", 277 | "video_49", 278 | "video_19", 279 | "video_30" 280 | ] 281 | } 282 | ] -------------------------------------------------------------------------------- /cluster.json: -------------------------------------------------------------------------------- 1 | { 2 | "3": [ 3 | "eccv16_dataset_ovp_google_pool5/video_1", 4 | "eccv16_dataset_ovp_google_pool5/video_13", 5 | "eccv16_dataset_ovp_google_pool5/video_16", 6 | "eccv16_dataset_ovp_google_pool5/video_26", 7 | "eccv16_dataset_ovp_google_pool5/video_28", 8 | "eccv16_dataset_ovp_google_pool5/video_32", 9 | "eccv16_dataset_ovp_google_pool5/video_38", 10 | "eccv16_dataset_ovp_google_pool5/video_49", 11 | "eccv16_dataset_youtube_google_pool5/video_27", 12 | "eccv16_dataset_youtube_google_pool5/video_42" 13 | ], 14 | "15": [ 15 | "eccv16_dataset_ovp_google_pool5/video_10", 16 | "eccv16_dataset_ovp_google_pool5/video_2", 17 | "eccv16_dataset_ovp_google_pool5/video_3", 18 | "eccv16_dataset_ovp_google_pool5/video_35", 19 | "eccv16_dataset_ovp_google_pool5/video_48", 20 | "eccv16_dataset_youtube_google_pool5/video_23", 21 | "eccv16_dataset_youtube_google_pool5/video_25" 22 | ], 23 | "0": [ 24 | "eccv16_dataset_ovp_google_pool5/video_11", 25 | "eccv16_dataset_ovp_google_pool5/video_20", 26 | "eccv16_dataset_ovp_google_pool5/video_21", 27 | "eccv16_dataset_ovp_google_pool5/video_25", 28 | "eccv16_dataset_ovp_google_pool5/video_31", 29 | "eccv16_dataset_ovp_google_pool5/video_41", 30 | "eccv16_dataset_ovp_google_pool5/video_46", 31 | "eccv16_dataset_ovp_google_pool5/video_47", 32 | "eccv16_dataset_ovp_google_pool5/video_9", 33 | "eccv16_dataset_youtube_google_pool5/video_16", 34 | "eccv16_dataset_youtube_google_pool5/video_40" 35 | ], 36 | "22": [ 37 | "eccv16_dataset_ovp_google_pool5/video_12", 38 | "eccv16_dataset_ovp_google_pool5/video_22", 39 | "eccv16_dataset_ovp_google_pool5/video_24", 40 | "eccv16_dataset_ovp_google_pool5/video_27", 41 | "eccv16_dataset_ovp_google_pool5/video_40", 42 | "eccv16_dataset_summe_google_pool5/video_15", 43 | "eccv16_dataset_summe_google_pool5/video_23", 44 | "eccv16_dataset_youtube_google_pool5/video_33", 45 | "eccv16_dataset_youtube_google_pool5/video_36", 46 | "eccv16_dataset_youtube_google_pool5/video_38", 47 | "eccv16_dataset_youtube_google_pool5/video_41", 48 | "eccv16_dataset_youtube_google_pool5/video_49" 49 | ], 50 | "7": [ 51 | "eccv16_dataset_ovp_google_pool5/video_14", 52 | "eccv16_dataset_ovp_google_pool5/video_19", 53 | "eccv16_dataset_ovp_google_pool5/video_23", 54 | "eccv16_dataset_ovp_google_pool5/video_30", 55 | "eccv16_dataset_ovp_google_pool5/video_37", 56 | "eccv16_dataset_ovp_google_pool5/video_44", 57 | "eccv16_dataset_youtube_google_pool5/video_17", 58 | "eccv16_dataset_youtube_google_pool5/video_20", 59 | "eccv16_dataset_youtube_google_pool5/video_28", 60 | "eccv16_dataset_youtube_google_pool5/video_30", 61 | "eccv16_dataset_youtube_google_pool5/video_31" 62 | ], 63 | "12": [ 64 | "eccv16_dataset_ovp_google_pool5/video_15", 65 | "eccv16_dataset_ovp_google_pool5/video_34", 66 | "eccv16_dataset_ovp_google_pool5/video_36", 67 | "eccv16_dataset_ovp_google_pool5/video_39", 68 | "eccv16_dataset_ovp_google_pool5/video_4", 69 | "eccv16_dataset_ovp_google_pool5/video_42", 70 | "eccv16_dataset_ovp_google_pool5/video_43", 71 | "eccv16_dataset_ovp_google_pool5/video_45", 72 | "eccv16_dataset_youtube_google_pool5/video_24" 73 | ], 74 | "14": [ 75 | "eccv16_dataset_ovp_google_pool5/video_17" 76 | ], 77 | "24": [ 78 | "eccv16_dataset_ovp_google_pool5/video_18" 79 | ], 80 | "10": [ 81 | "eccv16_dataset_ovp_google_pool5/video_29", 82 | "eccv16_dataset_ovp_google_pool5/video_7", 83 | "eccv16_dataset_ovp_google_pool5/video_8", 84 | "eccv16_dataset_tvsum_google_pool5/video_1", 85 | "eccv16_dataset_youtube_google_pool5/video_29", 86 | "eccv16_dataset_youtube_google_pool5/video_32", 87 | "eccv16_dataset_youtube_google_pool5/video_35", 88 | "eccv16_dataset_youtube_google_pool5/video_39", 89 | "eccv16_dataset_youtube_google_pool5/video_46" 90 | ], 91 | "2": [ 92 | "eccv16_dataset_ovp_google_pool5/video_33", 93 | "eccv16_dataset_ovp_google_pool5/video_5", 94 | "eccv16_dataset_ovp_google_pool5/video_50", 95 | "eccv16_dataset_youtube_google_pool5/video_19", 96 | "eccv16_dataset_youtube_google_pool5/video_48" 97 | ], 98 | "18": [ 99 | "eccv16_dataset_ovp_google_pool5/video_6", 100 | "eccv16_dataset_youtube_google_pool5/video_15", 101 | "eccv16_dataset_youtube_google_pool5/video_21", 102 | "eccv16_dataset_youtube_google_pool5/video_43", 103 | "eccv16_dataset_youtube_google_pool5/video_44", 104 | "eccv16_dataset_youtube_google_pool5/video_45", 105 | "eccv16_dataset_youtube_google_pool5/video_47" 106 | ], 107 | "4": [ 108 | "eccv16_dataset_summe_google_pool5/video_1", 109 | "eccv16_dataset_summe_google_pool5/video_16", 110 | "eccv16_dataset_summe_google_pool5/video_2", 111 | "eccv16_dataset_summe_google_pool5/video_24", 112 | "eccv16_dataset_summe_google_pool5/video_3", 113 | "eccv16_dataset_summe_google_pool5/video_4", 114 | "eccv16_dataset_tvsum_google_pool5/video_26", 115 | "eccv16_dataset_tvsum_google_pool5/video_48", 116 | "eccv16_dataset_tvsum_google_pool5/video_5" 117 | ], 118 | "5": [ 119 | "eccv16_dataset_summe_google_pool5/video_10", 120 | "eccv16_dataset_summe_google_pool5/video_21", 121 | "eccv16_dataset_summe_google_pool5/video_7", 122 | "eccv16_dataset_tvsum_google_pool5/video_12", 123 | "eccv16_dataset_tvsum_google_pool5/video_18", 124 | "eccv16_dataset_tvsum_google_pool5/video_27", 125 | "eccv16_dataset_tvsum_google_pool5/video_28", 126 | "eccv16_dataset_tvsum_google_pool5/video_36", 127 | "eccv16_dataset_tvsum_google_pool5/video_40", 128 | "eccv16_dataset_tvsum_google_pool5/video_6", 129 | "eccv16_dataset_tvsum_google_pool5/video_8" 130 | ], 131 | "16": [ 132 | "eccv16_dataset_summe_google_pool5/video_11", 133 | "eccv16_dataset_summe_google_pool5/video_13", 134 | "eccv16_dataset_summe_google_pool5/video_14", 135 | "eccv16_dataset_tvsum_google_pool5/video_30", 136 | "eccv16_dataset_tvsum_google_pool5/video_34", 137 | "eccv16_dataset_tvsum_google_pool5/video_44", 138 | "eccv16_dataset_tvsum_google_pool5/video_45" 139 | ], 140 | "9": [ 141 | "eccv16_dataset_summe_google_pool5/video_12" 142 | ], 143 | "6": [ 144 | "eccv16_dataset_summe_google_pool5/video_17", 145 | "eccv16_dataset_tvsum_google_pool5/video_17", 146 | "eccv16_dataset_tvsum_google_pool5/video_20", 147 | "eccv16_dataset_tvsum_google_pool5/video_22", 148 | "eccv16_dataset_tvsum_google_pool5/video_25", 149 | "eccv16_dataset_tvsum_google_pool5/video_4", 150 | "eccv16_dataset_tvsum_google_pool5/video_42", 151 | "eccv16_dataset_tvsum_google_pool5/video_49", 152 | "eccv16_dataset_tvsum_google_pool5/video_50", 153 | "eccv16_dataset_tvsum_google_pool5/video_9" 154 | ], 155 | "17": [ 156 | "eccv16_dataset_summe_google_pool5/video_18", 157 | "eccv16_dataset_summe_google_pool5/video_25", 158 | "eccv16_dataset_summe_google_pool5/video_8", 159 | "eccv16_dataset_tvsum_google_pool5/video_13", 160 | "eccv16_dataset_tvsum_google_pool5/video_19", 161 | "eccv16_dataset_tvsum_google_pool5/video_7" 162 | ], 163 | "21": [ 164 | "eccv16_dataset_summe_google_pool5/video_19", 165 | "eccv16_dataset_summe_google_pool5/video_5", 166 | "eccv16_dataset_tvsum_google_pool5/video_10", 167 | "eccv16_dataset_tvsum_google_pool5/video_31", 168 | "eccv16_dataset_tvsum_google_pool5/video_43" 169 | ], 170 | "8": [ 171 | "eccv16_dataset_summe_google_pool5/video_20", 172 | "eccv16_dataset_tvsum_google_pool5/video_23" 173 | ], 174 | "11": [ 175 | "eccv16_dataset_summe_google_pool5/video_22", 176 | "eccv16_dataset_tvsum_google_pool5/video_15", 177 | "eccv16_dataset_tvsum_google_pool5/video_2", 178 | "eccv16_dataset_tvsum_google_pool5/video_24", 179 | "eccv16_dataset_tvsum_google_pool5/video_32", 180 | "eccv16_dataset_tvsum_google_pool5/video_35", 181 | "eccv16_dataset_tvsum_google_pool5/video_38", 182 | "eccv16_dataset_tvsum_google_pool5/video_39" 183 | ], 184 | "1": [ 185 | "eccv16_dataset_summe_google_pool5/video_6", 186 | "eccv16_dataset_tvsum_google_pool5/video_11", 187 | "eccv16_dataset_tvsum_google_pool5/video_37" 188 | ], 189 | "13": [ 190 | "eccv16_dataset_summe_google_pool5/video_9", 191 | "eccv16_dataset_tvsum_google_pool5/video_47" 192 | ], 193 | "19": [ 194 | "eccv16_dataset_tvsum_google_pool5/video_14" 195 | ], 196 | "20": [ 197 | "eccv16_dataset_tvsum_google_pool5/video_16", 198 | "eccv16_dataset_tvsum_google_pool5/video_21", 199 | "eccv16_dataset_tvsum_google_pool5/video_29", 200 | "eccv16_dataset_tvsum_google_pool5/video_3", 201 | "eccv16_dataset_tvsum_google_pool5/video_33", 202 | "eccv16_dataset_tvsum_google_pool5/video_41", 203 | "eccv16_dataset_tvsum_google_pool5/video_46" 204 | ], 205 | "23": [ 206 | "eccv16_dataset_youtube_google_pool5/video_11", 207 | "eccv16_dataset_youtube_google_pool5/video_12", 208 | "eccv16_dataset_youtube_google_pool5/video_13", 209 | "eccv16_dataset_youtube_google_pool5/video_14", 210 | "eccv16_dataset_youtube_google_pool5/video_18", 211 | "eccv16_dataset_youtube_google_pool5/video_26", 212 | "eccv16_dataset_youtube_google_pool5/video_34", 213 | "eccv16_dataset_youtube_google_pool5/video_37", 214 | "eccv16_dataset_youtube_google_pool5/video_50" 215 | ] 216 | } -------------------------------------------------------------------------------- /tools/engine.py: -------------------------------------------------------------------------------- 1 | # hierachical co-transformer with cluster 2 | 3 | import os 4 | import h5py 5 | import glob 6 | import json 7 | import random 8 | import importlib 9 | import numpy as np 10 | import torch 11 | from utils.sys_utils import * 12 | from utils.vsum_tools import * 13 | from tools.funcs import * 14 | 15 | 16 | class sumnet: 17 | def __init__(self, hps): 18 | self.hps = hps 19 | self.model = None 20 | self.log_file = None 21 | self.verbose = hps.verbose 22 | 23 | def fix_keys(self, keys, dataset_name=None): 24 | """ 25 | :param keys: 26 | :return: 27 | """ 28 | # dataset_name = None 29 | if len(self.datasets) == 1: 30 | dataset_name = next(iter(self.datasets)) 31 | 32 | keys_out = [] 33 | for key in keys: 34 | t = key.split("/") 35 | if len(t) != 2: 36 | assert ( 37 | dataset_name is not None 38 | ), "ERROR dataset name in some keys is missing but there are multiple dataset {} to choose from".format( 39 | len(self.datasets) 40 | ) 41 | 42 | key_name = dataset_name + "/" + key 43 | keys_out.append(key_name) 44 | else: 45 | keys_out.append(key) 46 | 47 | return keys_out 48 | 49 | def load_datasets(self, datasets=None): 50 | """ 51 | Loads all h5 datasets from the datasets list into a dictionary self.dataset 52 | referenced by their base filename 53 | :param datasets: List of dataset filenames 54 | :return: 55 | """ 56 | if datasets is None: 57 | datasets = self.hps.datasets 58 | 59 | datasets_dict = {} 60 | for dataset in datasets: 61 | _, base_filename = os.path.split(dataset) 62 | base_filename, _ = os.path.splitext(base_filename) 63 | print("Loading:", dataset) 64 | datasets_dict[base_filename] = h5py.File(dataset, "r") 65 | 66 | self.datasets = datasets_dict 67 | return datasets_dict 68 | 69 | def load_split_file(self, splits_file): 70 | 71 | self.dataset_name, self.dataset_type, self.splits = parse_splits_filename( 72 | splits_file 73 | ) 74 | n_folds = len(self.splits) 75 | self.split_file = splits_file 76 | print("Loading splits from: ", splits_file) 77 | 78 | return n_folds 79 | 80 | def select_split(self, split_id): 81 | print("Selecting split: ", split_id) 82 | 83 | self.split_id = split_id 84 | n_folds = len(self.splits) 85 | assert self.split_id < n_folds, "split_id (got {}) exceeds {}".format( 86 | self.split_id, n_folds 87 | ) 88 | 89 | split = self.splits[self.split_id] 90 | self.train_keys = split["train_keys"] 91 | self.test_keys = split["test_keys"] 92 | 93 | dataset_filename = self.hps.get_dataset_by_name(self.dataset_name)[0] 94 | _, dataset_filename = os.path.split(dataset_filename) 95 | dataset_filename, _ = os.path.splitext(dataset_filename) 96 | self.train_keys = self.fix_keys(self.train_keys, dataset_filename) 97 | self.test_keys = self.fix_keys(self.test_keys, dataset_filename) 98 | return 99 | 100 | def load_model(self, model_filename): 101 | self.model.load_state_dict( 102 | torch.load(model_filename, map_location=lambda storage, loc: storage) 103 | ) 104 | return 105 | 106 | def initialize(self, cuda_device=None): 107 | rnd_seed = self.hps.rnd_seed 108 | random.seed(rnd_seed) 109 | np.random.seed(rnd_seed) 110 | torch.manual_seed(rnd_seed) 111 | model_module = importlib.import_module("models." + self.hps.model_name) 112 | self.model = model_module.DSN(self.hps) 113 | self.model.eval() 114 | self.model.apply(weights_init) 115 | 116 | cuda_device = cuda_device or self.hps.cuda_device 117 | 118 | if self.hps.use_cuda: 119 | print("Setting CUDA device: ", cuda_device) 120 | torch.cuda.set_device(cuda_device) 121 | torch.cuda.manual_seed(rnd_seed) 122 | 123 | if self.hps.use_cuda: 124 | self.model.cuda() 125 | 126 | return 127 | 128 | def get_data(self, key): 129 | key_parts = key.split("/") 130 | assert len(key_parts) == 2, "ERROR. Wrong key name: " + key 131 | dataset, key = key_parts 132 | # if not key in self.datasets[dataset].keys(): 133 | # pdb.set_trace() 134 | return self.datasets[dataset][key] 135 | 136 | def lookup_weights_file(self, data_path): 137 | dataset_type_str = "" if self.dataset_type == "" else self.dataset_type + "_" 138 | weights_filename = data_path + "/models/{}_{}splits_{}.tar.pth".format( 139 | self.dataset_name, dataset_type_str, self.split_id 140 | ) 141 | weights_filename = glob.glob(weights_filename) 142 | if len(weights_filename) == 0: 143 | print("Couldn't find model weights: ", weights_filename) 144 | return "" 145 | 146 | # Get the first weights filename in the dir 147 | weights_filename = weights_filename[0] 148 | 149 | return weights_filename 150 | 151 | def preprocess(self, key): 152 | dataset = self.get_data(key) 153 | seq = dataset["features"][...] # sequence of features, (seq_len, dim) 154 | gtscore = dataset["gtscore"][...] 155 | gtscore -= gtscore.min() 156 | gtscore /= gtscore.max() 157 | 158 | if "change_points" in dataset.keys(): 159 | boundary = dataset["change_points"][...] 160 | else: 161 | l_shot = 60 162 | n_frames = len(seq) * 15 163 | boundary = [ 164 | [i * l_shot, (i + 1) * l_shot - 1] 165 | for i in range(int(np.ceil(n_frames / l_shot))) 166 | ] 167 | boundary[-1][-1] = n_frames - 1 168 | boundary = np.array(boundary) 169 | shots = [] 170 | for i in range(len(boundary)): 171 | start = int(boundary[i, 0] / 15) 172 | end = int(boundary[i, 1] / 15) + 1 173 | shot = seq[start:end, :] 174 | shot = torch.from_numpy(shot) 175 | if self.hps.use_cuda: 176 | shot = shot.cuda() 177 | shots.append(shot) 178 | return { 179 | "shots": shots, 180 | "gtscore": gtscore, 181 | "boundary": boundary, 182 | "n_seq": len(gtscore), 183 | } 184 | 185 | def train(self, output_dir="EX-0"): 186 | 187 | print("Initializing model and optimizer...") 188 | 189 | criterion = torch.nn.MSELoss() 190 | 191 | if self.hps.use_cuda: 192 | criterion = criterion.cuda() 193 | 194 | parameters = filter(lambda p: p.requires_grad, self.model.parameters()) 195 | self.optimizer = torch.optim.Adam( 196 | parameters, lr=self.hps.lr[0], weight_decay=self.hps.l2_req 197 | ) 198 | 199 | print("Starting training...") 200 | 201 | max_val_fscore = 0 202 | max_val_fscore_epoch = 0 203 | train_keys = self.train_keys[:] 204 | 205 | cluster_res_file = "cluster.json" 206 | with open(cluster_res_file, "r") as f: 207 | cluster = json.load(f) 208 | 209 | temp = {} 210 | for key, value in cluster.items(): 211 | temp[key] = [v for v in value if v in train_keys] 212 | 213 | vid2cls = {} 214 | for key, value in cluster.items(): 215 | for v in value: 216 | vid2cls[v] = key 217 | 218 | for epoch in range(self.hps.epochs_max): 219 | self.model.train() 220 | 221 | print( 222 | "Epoch: {0:6}".format(str(epoch) + "/" + str(self.hps.epochs_max)), 223 | end="", 224 | ) 225 | avg_loss = [] 226 | 227 | random.shuffle(train_keys) 228 | 229 | for idx, key in enumerate(train_keys): 230 | c = vid2cls[key] 231 | inter_cls_vid = temp[c] 232 | n = len(inter_cls_vid) 233 | 234 | if n == 0: 235 | key2 = key 236 | else: 237 | if self.hps.rand_choice: 238 | key2 = random.choice(inter_cls_vid) 239 | else: 240 | i = (inter_cls_vid.index(key) + 1) % n 241 | key2 = inter_cls_vid[i] 242 | 243 | inputs1 = self.preprocess(key) 244 | inputs2 = self.preprocess(key2) 245 | probs1, probs2, rec_loss = self.model(inputs1, inputs2) 246 | gtscore1 = inputs1["gtscore"] 247 | gtscore2 = inputs2["gtscore"] 248 | 249 | cost = ( 250 | criterion(probs1, torch.from_numpy(gtscore1).cuda()) 251 | + criterion(probs2, torch.from_numpy(gtscore2).cuda()) 252 | ) / 2 * self.hps.supervised_factor + rec_loss * self.hps.rec_factor 253 | 254 | if self.hps.regular: 255 | cost += ( 256 | 0.1 * (probs1.mean() - 0.5) ** 2 257 | + 0.1 * (probs2.mean() - 0.5) ** 2 258 | ) / 2 259 | 260 | self.optimizer.zero_grad() 261 | 262 | cost.backward() 263 | torch.nn.utils.clip_grad_norm_(self.model.parameters(), 5.0) 264 | self.optimizer.step() 265 | avg_loss.append(float(cost)) 266 | 267 | # Evaluate test dataset 268 | if (epoch + 1) % self.hps.eval_freq == 0: 269 | val_fscore, video_scores = self.eval(self.test_keys) 270 | if max_val_fscore < val_fscore: 271 | max_val_fscore = val_fscore 272 | max_val_fscore_epoch = epoch 273 | print( 274 | " Test F-score avg/max: {0:0.5}/{1:0.5}".format( 275 | val_fscore, max_val_fscore 276 | ) 277 | ) 278 | 279 | avg_loss = np.array(avg_loss) 280 | print(" Train loss: {0:.05f}\n".format(np.mean(avg_loss)), end="") 281 | 282 | if self.verbose: 283 | video_scores = [["No", "Video", "F-score"]] + video_scores 284 | print_table(video_scores, cell_width=[3, 40, 8]) 285 | 286 | # Save model weights 287 | if (epoch + 1) % self.hps.eval_freq == 0: 288 | path, filename = os.path.split(self.split_file) 289 | base_filename, _ = os.path.splitext(filename) 290 | path = os.path.join( 291 | output_dir, "models_temp", base_filename + "_" + str(self.split_id) 292 | ) 293 | os.makedirs(path, exist_ok=True) 294 | filename = ( 295 | str(epoch) + "_" + str(round(val_fscore * 100, 3)) + ".pth.tar" 296 | ) 297 | torch.save(self.model.state_dict(), os.path.join(path, filename)) 298 | 299 | return max_val_fscore, max_val_fscore_epoch 300 | 301 | def eval(self, keys, results_filename=None): 302 | 303 | self.model.eval() 304 | summary = {} 305 | att_vecs = {} 306 | with torch.no_grad(): 307 | for i, key in enumerate(keys): 308 | inputs = self.preprocess(key) 309 | probs = self.model(inputs) 310 | probs = probs.data.cpu().numpy() 311 | summary[key] = probs 312 | 313 | f_score, video_scores = self.eval_summary( 314 | summary, 315 | keys, 316 | metric=self.dataset_name, 317 | results_filename=results_filename, 318 | ) 319 | 320 | return f_score, video_scores 321 | 322 | def eval_summary( 323 | self, 324 | machine_summary_activations, 325 | test_keys, 326 | results_filename=None, 327 | metric="tvsum", 328 | ): 329 | 330 | eval_metric = "avg" if metric == "tvsum" else "max" 331 | 332 | if results_filename is not None: 333 | h5_res = h5py.File(results_filename, "w") 334 | 335 | fms = [] 336 | video_scores = [] 337 | for key_idx, key in enumerate(test_keys): 338 | d = self.get_data(key) 339 | probs = machine_summary_activations[key] 340 | 341 | if "change_points" not in d: 342 | print("ERROR: No change points in dataset/video ", key) 343 | 344 | cps = d["change_points"][...] 345 | num_frames = d["n_frames"][()] 346 | nfps = d["n_frame_per_seg"][...].tolist() 347 | positions = d["picks"][...] 348 | user_summary = d["user_summary"][...] 349 | 350 | machine_summary = generate_summary(probs, cps, num_frames, nfps, positions) 351 | fm, _, _ = evaluate_summary(machine_summary, user_summary, eval_metric) 352 | fms.append(fm) 353 | 354 | # Reporting & logging 355 | video_scores.append([key_idx + 1, key, "{:.1%}".format(fm)]) 356 | 357 | if results_filename: 358 | gt = d["gtscore"][...] 359 | h5_res.create_dataset(key + "/score", data=probs) 360 | h5_res.create_dataset(key + "/machine_summary", data=machine_summary) 361 | h5_res.create_dataset(key + "/gtscore", data=gt) 362 | h5_res.create_dataset(key + "/fm", data=fm) 363 | h5_res.create_dataset(key + "/picks", data=positions) 364 | 365 | video_name = key.split("/")[1] 366 | if "video_name" in d: 367 | video_name = d["video_name"][...] 368 | h5_res.create_dataset(key + "/video_name", data=video_name) 369 | 370 | mean_fm = np.mean(fms) 371 | 372 | # Reporting & logging 373 | if results_filename is not None: 374 | h5_res.close() 375 | 376 | return mean_fm, video_scores 377 | -------------------------------------------------------------------------------- /splits/summe_aug_splits.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "train_keys": [ 4 | "eccv16_dataset_youtube_google_pool5/video_28", 5 | "eccv16_dataset_tvsum_google_pool5/video_36", 6 | "eccv16_dataset_tvsum_google_pool5/video_21", 7 | "eccv16_dataset_ovp_google_pool5/video_16", 8 | "eccv16_dataset_youtube_google_pool5/video_20", 9 | "eccv16_dataset_tvsum_google_pool5/video_43", 10 | "eccv16_dataset_ovp_google_pool5/video_31", 11 | "eccv16_dataset_youtube_google_pool5/video_32", 12 | "eccv16_dataset_ovp_google_pool5/video_5", 13 | "eccv16_dataset_youtube_google_pool5/video_41", 14 | "eccv16_dataset_ovp_google_pool5/video_49", 15 | "eccv16_dataset_ovp_google_pool5/video_40", 16 | "eccv16_dataset_ovp_google_pool5/video_37", 17 | "eccv16_dataset_tvsum_google_pool5/video_27", 18 | "eccv16_dataset_tvsum_google_pool5/video_8", 19 | "eccv16_dataset_summe_google_pool5/video_4", 20 | "eccv16_dataset_summe_google_pool5/video_9", 21 | "eccv16_dataset_tvsum_google_pool5/video_1", 22 | "eccv16_dataset_ovp_google_pool5/video_34", 23 | "eccv16_dataset_summe_google_pool5/video_17", 24 | "eccv16_dataset_ovp_google_pool5/video_27", 25 | "eccv16_dataset_ovp_google_pool5/video_2", 26 | "eccv16_dataset_tvsum_google_pool5/video_22", 27 | "eccv16_dataset_ovp_google_pool5/video_4", 28 | "eccv16_dataset_youtube_google_pool5/video_12", 29 | "eccv16_dataset_tvsum_google_pool5/video_17", 30 | "eccv16_dataset_tvsum_google_pool5/video_35", 31 | "eccv16_dataset_tvsum_google_pool5/video_11", 32 | "eccv16_dataset_summe_google_pool5/video_23", 33 | "eccv16_dataset_ovp_google_pool5/video_11", 34 | "eccv16_dataset_ovp_google_pool5/video_48", 35 | "eccv16_dataset_youtube_google_pool5/video_18", 36 | "eccv16_dataset_summe_google_pool5/video_20", 37 | "eccv16_dataset_tvsum_google_pool5/video_29", 38 | "eccv16_dataset_ovp_google_pool5/video_43", 39 | "eccv16_dataset_tvsum_google_pool5/video_14", 40 | "eccv16_dataset_youtube_google_pool5/video_24", 41 | "eccv16_dataset_youtube_google_pool5/video_44", 42 | "eccv16_dataset_summe_google_pool5/video_16", 43 | "eccv16_dataset_youtube_google_pool5/video_48", 44 | "eccv16_dataset_tvsum_google_pool5/video_45", 45 | "eccv16_dataset_ovp_google_pool5/video_1", 46 | "eccv16_dataset_ovp_google_pool5/video_50", 47 | "eccv16_dataset_youtube_google_pool5/video_42", 48 | "eccv16_dataset_tvsum_google_pool5/video_44", 49 | "eccv16_dataset_summe_google_pool5/video_24", 50 | "eccv16_dataset_youtube_google_pool5/video_31", 51 | "eccv16_dataset_summe_google_pool5/video_1", 52 | "eccv16_dataset_tvsum_google_pool5/video_34", 53 | "eccv16_dataset_tvsum_google_pool5/video_32", 54 | "eccv16_dataset_youtube_google_pool5/video_21", 55 | "eccv16_dataset_ovp_google_pool5/video_41", 56 | "eccv16_dataset_ovp_google_pool5/video_39", 57 | "eccv16_dataset_tvsum_google_pool5/video_5", 58 | "eccv16_dataset_youtube_google_pool5/video_33", 59 | "eccv16_dataset_ovp_google_pool5/video_10", 60 | "eccv16_dataset_youtube_google_pool5/video_23", 61 | "eccv16_dataset_ovp_google_pool5/video_12", 62 | "eccv16_dataset_youtube_google_pool5/video_43", 63 | "eccv16_dataset_tvsum_google_pool5/video_3", 64 | "eccv16_dataset_youtube_google_pool5/video_16", 65 | "eccv16_dataset_ovp_google_pool5/video_45", 66 | "eccv16_dataset_summe_google_pool5/video_5", 67 | "eccv16_dataset_ovp_google_pool5/video_30", 68 | "eccv16_dataset_tvsum_google_pool5/video_41", 69 | "eccv16_dataset_youtube_google_pool5/video_39", 70 | "eccv16_dataset_summe_google_pool5/video_12", 71 | "eccv16_dataset_ovp_google_pool5/video_3", 72 | "eccv16_dataset_ovp_google_pool5/video_17", 73 | "eccv16_dataset_ovp_google_pool5/video_18", 74 | "eccv16_dataset_ovp_google_pool5/video_44", 75 | "eccv16_dataset_tvsum_google_pool5/video_25", 76 | "eccv16_dataset_tvsum_google_pool5/video_23", 77 | "eccv16_dataset_ovp_google_pool5/video_19", 78 | "eccv16_dataset_youtube_google_pool5/video_15", 79 | "eccv16_dataset_summe_google_pool5/video_2", 80 | "eccv16_dataset_tvsum_google_pool5/video_39", 81 | "eccv16_dataset_ovp_google_pool5/video_29", 82 | "eccv16_dataset_tvsum_google_pool5/video_7", 83 | "eccv16_dataset_youtube_google_pool5/video_35", 84 | "eccv16_dataset_tvsum_google_pool5/video_12", 85 | "eccv16_dataset_youtube_google_pool5/video_38", 86 | "eccv16_dataset_summe_google_pool5/video_15", 87 | "eccv16_dataset_tvsum_google_pool5/video_31", 88 | "eccv16_dataset_ovp_google_pool5/video_42", 89 | "eccv16_dataset_tvsum_google_pool5/video_46", 90 | "eccv16_dataset_ovp_google_pool5/video_23", 91 | "eccv16_dataset_tvsum_google_pool5/video_37", 92 | "eccv16_dataset_summe_google_pool5/video_11", 93 | "eccv16_dataset_youtube_google_pool5/video_36", 94 | "eccv16_dataset_summe_google_pool5/video_10", 95 | "eccv16_dataset_youtube_google_pool5/video_14", 96 | "eccv16_dataset_tvsum_google_pool5/video_19", 97 | "eccv16_dataset_ovp_google_pool5/video_35", 98 | "eccv16_dataset_ovp_google_pool5/video_15", 99 | "eccv16_dataset_ovp_google_pool5/video_6", 100 | "eccv16_dataset_tvsum_google_pool5/video_4", 101 | "eccv16_dataset_summe_google_pool5/video_7", 102 | "eccv16_dataset_ovp_google_pool5/video_33", 103 | "eccv16_dataset_youtube_google_pool5/video_34", 104 | "eccv16_dataset_tvsum_google_pool5/video_48", 105 | "eccv16_dataset_tvsum_google_pool5/video_38", 106 | "eccv16_dataset_summe_google_pool5/video_19", 107 | "eccv16_dataset_tvsum_google_pool5/video_42", 108 | "eccv16_dataset_ovp_google_pool5/video_21", 109 | "eccv16_dataset_ovp_google_pool5/video_32", 110 | "eccv16_dataset_youtube_google_pool5/video_27", 111 | "eccv16_dataset_ovp_google_pool5/video_26", 112 | "eccv16_dataset_summe_google_pool5/video_21", 113 | "eccv16_dataset_youtube_google_pool5/video_13", 114 | "eccv16_dataset_youtube_google_pool5/video_11", 115 | "eccv16_dataset_youtube_google_pool5/video_37", 116 | "eccv16_dataset_youtube_google_pool5/video_30", 117 | "eccv16_dataset_ovp_google_pool5/video_38", 118 | "eccv16_dataset_summe_google_pool5/video_6", 119 | "eccv16_dataset_tvsum_google_pool5/video_16", 120 | "eccv16_dataset_tvsum_google_pool5/video_6", 121 | "eccv16_dataset_youtube_google_pool5/video_47", 122 | "eccv16_dataset_youtube_google_pool5/video_29", 123 | "eccv16_dataset_ovp_google_pool5/video_24", 124 | "eccv16_dataset_tvsum_google_pool5/video_13", 125 | "eccv16_dataset_tvsum_google_pool5/video_18", 126 | "eccv16_dataset_ovp_google_pool5/video_25", 127 | "eccv16_dataset_tvsum_google_pool5/video_24", 128 | "eccv16_dataset_tvsum_google_pool5/video_49", 129 | "eccv16_dataset_youtube_google_pool5/video_25", 130 | "eccv16_dataset_tvsum_google_pool5/video_26", 131 | "eccv16_dataset_tvsum_google_pool5/video_10", 132 | "eccv16_dataset_tvsum_google_pool5/video_2", 133 | "eccv16_dataset_ovp_google_pool5/video_20", 134 | "eccv16_dataset_tvsum_google_pool5/video_33", 135 | "eccv16_dataset_ovp_google_pool5/video_9", 136 | "eccv16_dataset_youtube_google_pool5/video_49", 137 | "eccv16_dataset_tvsum_google_pool5/video_28", 138 | "eccv16_dataset_ovp_google_pool5/video_36", 139 | "eccv16_dataset_ovp_google_pool5/video_8", 140 | "eccv16_dataset_youtube_google_pool5/video_26", 141 | "eccv16_dataset_ovp_google_pool5/video_22", 142 | "eccv16_dataset_tvsum_google_pool5/video_40", 143 | "eccv16_dataset_tvsum_google_pool5/video_50", 144 | "eccv16_dataset_ovp_google_pool5/video_7", 145 | "eccv16_dataset_youtube_google_pool5/video_46", 146 | "eccv16_dataset_youtube_google_pool5/video_17", 147 | "eccv16_dataset_ovp_google_pool5/video_47", 148 | "eccv16_dataset_tvsum_google_pool5/video_15", 149 | "eccv16_dataset_youtube_google_pool5/video_40", 150 | "eccv16_dataset_ovp_google_pool5/video_14", 151 | "eccv16_dataset_tvsum_google_pool5/video_47", 152 | "eccv16_dataset_youtube_google_pool5/video_19", 153 | "eccv16_dataset_tvsum_google_pool5/video_9", 154 | "eccv16_dataset_ovp_google_pool5/video_46", 155 | "eccv16_dataset_youtube_google_pool5/video_50", 156 | "eccv16_dataset_youtube_google_pool5/video_45", 157 | "eccv16_dataset_ovp_google_pool5/video_13", 158 | "eccv16_dataset_ovp_google_pool5/video_28", 159 | "eccv16_dataset_tvsum_google_pool5/video_20", 160 | "eccv16_dataset_summe_google_pool5/video_8", 161 | "eccv16_dataset_tvsum_google_pool5/video_30", 162 | "eccv16_dataset_summe_google_pool5/video_3" 163 | ], 164 | "test_keys": [ 165 | "eccv16_dataset_summe_google_pool5/video_22", 166 | "eccv16_dataset_summe_google_pool5/video_18", 167 | "eccv16_dataset_summe_google_pool5/video_14", 168 | "eccv16_dataset_summe_google_pool5/video_25", 169 | "eccv16_dataset_summe_google_pool5/video_13" 170 | ] 171 | }, 172 | { 173 | "train_keys": [ 174 | "eccv16_dataset_ovp_google_pool5/video_29", 175 | "eccv16_dataset_youtube_google_pool5/video_46", 176 | "eccv16_dataset_summe_google_pool5/video_21", 177 | "eccv16_dataset_ovp_google_pool5/video_34", 178 | "eccv16_dataset_ovp_google_pool5/video_36", 179 | "eccv16_dataset_youtube_google_pool5/video_28", 180 | "eccv16_dataset_tvsum_google_pool5/video_1", 181 | "eccv16_dataset_summe_google_pool5/video_15", 182 | "eccv16_dataset_tvsum_google_pool5/video_37", 183 | "eccv16_dataset_youtube_google_pool5/video_42", 184 | "eccv16_dataset_youtube_google_pool5/video_40", 185 | "eccv16_dataset_summe_google_pool5/video_12", 186 | "eccv16_dataset_youtube_google_pool5/video_17", 187 | "eccv16_dataset_youtube_google_pool5/video_20", 188 | "eccv16_dataset_tvsum_google_pool5/video_4", 189 | "eccv16_dataset_tvsum_google_pool5/video_13", 190 | "eccv16_dataset_ovp_google_pool5/video_21", 191 | "eccv16_dataset_tvsum_google_pool5/video_26", 192 | "eccv16_dataset_ovp_google_pool5/video_33", 193 | "eccv16_dataset_summe_google_pool5/video_1", 194 | "eccv16_dataset_tvsum_google_pool5/video_44", 195 | "eccv16_dataset_summe_google_pool5/video_2", 196 | "eccv16_dataset_ovp_google_pool5/video_26", 197 | "eccv16_dataset_ovp_google_pool5/video_32", 198 | "eccv16_dataset_ovp_google_pool5/video_13", 199 | "eccv16_dataset_youtube_google_pool5/video_29", 200 | "eccv16_dataset_youtube_google_pool5/video_38", 201 | "eccv16_dataset_tvsum_google_pool5/video_7", 202 | "eccv16_dataset_ovp_google_pool5/video_7", 203 | "eccv16_dataset_youtube_google_pool5/video_25", 204 | "eccv16_dataset_summe_google_pool5/video_6", 205 | "eccv16_dataset_ovp_google_pool5/video_27", 206 | "eccv16_dataset_ovp_google_pool5/video_35", 207 | "eccv16_dataset_summe_google_pool5/video_13", 208 | "eccv16_dataset_summe_google_pool5/video_7", 209 | "eccv16_dataset_youtube_google_pool5/video_23", 210 | "eccv16_dataset_ovp_google_pool5/video_8", 211 | "eccv16_dataset_ovp_google_pool5/video_41", 212 | "eccv16_dataset_youtube_google_pool5/video_47", 213 | "eccv16_dataset_ovp_google_pool5/video_17", 214 | "eccv16_dataset_ovp_google_pool5/video_38", 215 | "eccv16_dataset_tvsum_google_pool5/video_32", 216 | "eccv16_dataset_youtube_google_pool5/video_44", 217 | "eccv16_dataset_tvsum_google_pool5/video_50", 218 | "eccv16_dataset_tvsum_google_pool5/video_25", 219 | "eccv16_dataset_ovp_google_pool5/video_2", 220 | "eccv16_dataset_ovp_google_pool5/video_28", 221 | "eccv16_dataset_youtube_google_pool5/video_39", 222 | "eccv16_dataset_youtube_google_pool5/video_16", 223 | "eccv16_dataset_tvsum_google_pool5/video_19", 224 | "eccv16_dataset_tvsum_google_pool5/video_42", 225 | "eccv16_dataset_ovp_google_pool5/video_15", 226 | "eccv16_dataset_tvsum_google_pool5/video_31", 227 | "eccv16_dataset_ovp_google_pool5/video_4", 228 | "eccv16_dataset_youtube_google_pool5/video_27", 229 | "eccv16_dataset_ovp_google_pool5/video_9", 230 | "eccv16_dataset_tvsum_google_pool5/video_24", 231 | "eccv16_dataset_youtube_google_pool5/video_33", 232 | "eccv16_dataset_ovp_google_pool5/video_50", 233 | "eccv16_dataset_tvsum_google_pool5/video_27", 234 | "eccv16_dataset_ovp_google_pool5/video_3", 235 | "eccv16_dataset_tvsum_google_pool5/video_18", 236 | "eccv16_dataset_tvsum_google_pool5/video_40", 237 | "eccv16_dataset_youtube_google_pool5/video_36", 238 | "eccv16_dataset_tvsum_google_pool5/video_46", 239 | "eccv16_dataset_tvsum_google_pool5/video_39", 240 | "eccv16_dataset_ovp_google_pool5/video_18", 241 | "eccv16_dataset_ovp_google_pool5/video_47", 242 | "eccv16_dataset_summe_google_pool5/video_18", 243 | "eccv16_dataset_youtube_google_pool5/video_12", 244 | "eccv16_dataset_ovp_google_pool5/video_31", 245 | "eccv16_dataset_ovp_google_pool5/video_20", 246 | "eccv16_dataset_ovp_google_pool5/video_44", 247 | "eccv16_dataset_tvsum_google_pool5/video_12", 248 | "eccv16_dataset_ovp_google_pool5/video_40", 249 | "eccv16_dataset_ovp_google_pool5/video_24", 250 | "eccv16_dataset_tvsum_google_pool5/video_10", 251 | "eccv16_dataset_tvsum_google_pool5/video_36", 252 | "eccv16_dataset_youtube_google_pool5/video_26", 253 | "eccv16_dataset_youtube_google_pool5/video_11", 254 | "eccv16_dataset_tvsum_google_pool5/video_35", 255 | "eccv16_dataset_tvsum_google_pool5/video_6", 256 | "eccv16_dataset_summe_google_pool5/video_19", 257 | "eccv16_dataset_tvsum_google_pool5/video_16", 258 | "eccv16_dataset_youtube_google_pool5/video_24", 259 | "eccv16_dataset_youtube_google_pool5/video_14", 260 | "eccv16_dataset_ovp_google_pool5/video_39", 261 | "eccv16_dataset_youtube_google_pool5/video_35", 262 | "eccv16_dataset_ovp_google_pool5/video_22", 263 | "eccv16_dataset_ovp_google_pool5/video_1", 264 | "eccv16_dataset_summe_google_pool5/video_3", 265 | "eccv16_dataset_ovp_google_pool5/video_48", 266 | "eccv16_dataset_tvsum_google_pool5/video_15", 267 | "eccv16_dataset_summe_google_pool5/video_22", 268 | "eccv16_dataset_summe_google_pool5/video_24", 269 | "eccv16_dataset_youtube_google_pool5/video_48", 270 | "eccv16_dataset_ovp_google_pool5/video_23", 271 | "eccv16_dataset_youtube_google_pool5/video_30", 272 | "eccv16_dataset_tvsum_google_pool5/video_34", 273 | "eccv16_dataset_tvsum_google_pool5/video_2", 274 | "eccv16_dataset_tvsum_google_pool5/video_38", 275 | "eccv16_dataset_tvsum_google_pool5/video_22", 276 | "eccv16_dataset_youtube_google_pool5/video_18", 277 | "eccv16_dataset_tvsum_google_pool5/video_17", 278 | "eccv16_dataset_tvsum_google_pool5/video_33", 279 | "eccv16_dataset_summe_google_pool5/video_5", 280 | "eccv16_dataset_tvsum_google_pool5/video_9", 281 | "eccv16_dataset_youtube_google_pool5/video_19", 282 | "eccv16_dataset_summe_google_pool5/video_14", 283 | "eccv16_dataset_tvsum_google_pool5/video_45", 284 | "eccv16_dataset_tvsum_google_pool5/video_5", 285 | "eccv16_dataset_tvsum_google_pool5/video_41", 286 | "eccv16_dataset_ovp_google_pool5/video_11", 287 | "eccv16_dataset_summe_google_pool5/video_17", 288 | "eccv16_dataset_youtube_google_pool5/video_34", 289 | "eccv16_dataset_tvsum_google_pool5/video_47", 290 | "eccv16_dataset_tvsum_google_pool5/video_29", 291 | "eccv16_dataset_youtube_google_pool5/video_43", 292 | "eccv16_dataset_youtube_google_pool5/video_13", 293 | "eccv16_dataset_ovp_google_pool5/video_43", 294 | "eccv16_dataset_tvsum_google_pool5/video_21", 295 | "eccv16_dataset_youtube_google_pool5/video_32", 296 | "eccv16_dataset_ovp_google_pool5/video_25", 297 | "eccv16_dataset_youtube_google_pool5/video_45", 298 | "eccv16_dataset_tvsum_google_pool5/video_28", 299 | "eccv16_dataset_summe_google_pool5/video_11", 300 | "eccv16_dataset_youtube_google_pool5/video_41", 301 | "eccv16_dataset_youtube_google_pool5/video_50", 302 | "eccv16_dataset_ovp_google_pool5/video_42", 303 | "eccv16_dataset_tvsum_google_pool5/video_11", 304 | "eccv16_dataset_tvsum_google_pool5/video_49", 305 | "eccv16_dataset_tvsum_google_pool5/video_20", 306 | "eccv16_dataset_ovp_google_pool5/video_37", 307 | "eccv16_dataset_ovp_google_pool5/video_6", 308 | "eccv16_dataset_ovp_google_pool5/video_12", 309 | "eccv16_dataset_youtube_google_pool5/video_37", 310 | "eccv16_dataset_tvsum_google_pool5/video_23", 311 | "eccv16_dataset_ovp_google_pool5/video_49", 312 | "eccv16_dataset_tvsum_google_pool5/video_43", 313 | "eccv16_dataset_tvsum_google_pool5/video_14", 314 | "eccv16_dataset_ovp_google_pool5/video_45", 315 | "eccv16_dataset_ovp_google_pool5/video_19", 316 | "eccv16_dataset_youtube_google_pool5/video_21", 317 | "eccv16_dataset_summe_google_pool5/video_16", 318 | "eccv16_dataset_youtube_google_pool5/video_31", 319 | "eccv16_dataset_summe_google_pool5/video_20", 320 | "eccv16_dataset_ovp_google_pool5/video_16", 321 | "eccv16_dataset_youtube_google_pool5/video_49", 322 | "eccv16_dataset_tvsum_google_pool5/video_3", 323 | "eccv16_dataset_summe_google_pool5/video_8", 324 | "eccv16_dataset_ovp_google_pool5/video_46", 325 | "eccv16_dataset_tvsum_google_pool5/video_30", 326 | "eccv16_dataset_ovp_google_pool5/video_10", 327 | "eccv16_dataset_youtube_google_pool5/video_15", 328 | "eccv16_dataset_ovp_google_pool5/video_5", 329 | "eccv16_dataset_tvsum_google_pool5/video_8", 330 | "eccv16_dataset_tvsum_google_pool5/video_48", 331 | "eccv16_dataset_ovp_google_pool5/video_14", 332 | "eccv16_dataset_ovp_google_pool5/video_30" 333 | ], 334 | "test_keys": [ 335 | "eccv16_dataset_summe_google_pool5/video_25", 336 | "eccv16_dataset_summe_google_pool5/video_23", 337 | "eccv16_dataset_summe_google_pool5/video_9", 338 | "eccv16_dataset_summe_google_pool5/video_10", 339 | "eccv16_dataset_summe_google_pool5/video_4" 340 | ] 341 | }, 342 | { 343 | "train_keys": [ 344 | "eccv16_dataset_tvsum_google_pool5/video_24", 345 | "eccv16_dataset_tvsum_google_pool5/video_8", 346 | "eccv16_dataset_tvsum_google_pool5/video_38", 347 | "eccv16_dataset_tvsum_google_pool5/video_26", 348 | "eccv16_dataset_tvsum_google_pool5/video_19", 349 | "eccv16_dataset_youtube_google_pool5/video_29", 350 | "eccv16_dataset_youtube_google_pool5/video_19", 351 | "eccv16_dataset_ovp_google_pool5/video_46", 352 | "eccv16_dataset_tvsum_google_pool5/video_40", 353 | "eccv16_dataset_ovp_google_pool5/video_6", 354 | "eccv16_dataset_tvsum_google_pool5/video_22", 355 | "eccv16_dataset_youtube_google_pool5/video_24", 356 | "eccv16_dataset_tvsum_google_pool5/video_9", 357 | "eccv16_dataset_summe_google_pool5/video_20", 358 | "eccv16_dataset_tvsum_google_pool5/video_25", 359 | "eccv16_dataset_summe_google_pool5/video_23", 360 | "eccv16_dataset_ovp_google_pool5/video_42", 361 | "eccv16_dataset_summe_google_pool5/video_22", 362 | "eccv16_dataset_youtube_google_pool5/video_31", 363 | "eccv16_dataset_ovp_google_pool5/video_8", 364 | "eccv16_dataset_ovp_google_pool5/video_48", 365 | "eccv16_dataset_tvsum_google_pool5/video_2", 366 | "eccv16_dataset_ovp_google_pool5/video_33", 367 | "eccv16_dataset_tvsum_google_pool5/video_31", 368 | "eccv16_dataset_ovp_google_pool5/video_12", 369 | "eccv16_dataset_youtube_google_pool5/video_20", 370 | "eccv16_dataset_ovp_google_pool5/video_44", 371 | "eccv16_dataset_ovp_google_pool5/video_45", 372 | "eccv16_dataset_youtube_google_pool5/video_33", 373 | "eccv16_dataset_youtube_google_pool5/video_12", 374 | "eccv16_dataset_ovp_google_pool5/video_25", 375 | "eccv16_dataset_ovp_google_pool5/video_9", 376 | "eccv16_dataset_ovp_google_pool5/video_38", 377 | "eccv16_dataset_youtube_google_pool5/video_44", 378 | "eccv16_dataset_ovp_google_pool5/video_14", 379 | "eccv16_dataset_ovp_google_pool5/video_3", 380 | "eccv16_dataset_youtube_google_pool5/video_14", 381 | "eccv16_dataset_youtube_google_pool5/video_16", 382 | "eccv16_dataset_youtube_google_pool5/video_27", 383 | "eccv16_dataset_youtube_google_pool5/video_37", 384 | "eccv16_dataset_youtube_google_pool5/video_43", 385 | "eccv16_dataset_ovp_google_pool5/video_28", 386 | "eccv16_dataset_tvsum_google_pool5/video_48", 387 | "eccv16_dataset_tvsum_google_pool5/video_27", 388 | "eccv16_dataset_ovp_google_pool5/video_50", 389 | "eccv16_dataset_summe_google_pool5/video_11", 390 | "eccv16_dataset_summe_google_pool5/video_3", 391 | "eccv16_dataset_tvsum_google_pool5/video_4", 392 | "eccv16_dataset_tvsum_google_pool5/video_21", 393 | "eccv16_dataset_summe_google_pool5/video_2", 394 | "eccv16_dataset_tvsum_google_pool5/video_6", 395 | "eccv16_dataset_tvsum_google_pool5/video_47", 396 | "eccv16_dataset_youtube_google_pool5/video_23", 397 | "eccv16_dataset_youtube_google_pool5/video_46", 398 | "eccv16_dataset_ovp_google_pool5/video_4", 399 | "eccv16_dataset_ovp_google_pool5/video_35", 400 | "eccv16_dataset_tvsum_google_pool5/video_1", 401 | "eccv16_dataset_summe_google_pool5/video_13", 402 | "eccv16_dataset_tvsum_google_pool5/video_5", 403 | "eccv16_dataset_summe_google_pool5/video_19", 404 | "eccv16_dataset_tvsum_google_pool5/video_15", 405 | "eccv16_dataset_tvsum_google_pool5/video_16", 406 | "eccv16_dataset_youtube_google_pool5/video_18", 407 | "eccv16_dataset_tvsum_google_pool5/video_20", 408 | "eccv16_dataset_ovp_google_pool5/video_16", 409 | "eccv16_dataset_ovp_google_pool5/video_49", 410 | "eccv16_dataset_ovp_google_pool5/video_1", 411 | "eccv16_dataset_tvsum_google_pool5/video_14", 412 | "eccv16_dataset_tvsum_google_pool5/video_11", 413 | "eccv16_dataset_ovp_google_pool5/video_23", 414 | "eccv16_dataset_youtube_google_pool5/video_21", 415 | "eccv16_dataset_summe_google_pool5/video_18", 416 | "eccv16_dataset_tvsum_google_pool5/video_18", 417 | "eccv16_dataset_tvsum_google_pool5/video_30", 418 | "eccv16_dataset_youtube_google_pool5/video_48", 419 | "eccv16_dataset_ovp_google_pool5/video_7", 420 | "eccv16_dataset_youtube_google_pool5/video_30", 421 | "eccv16_dataset_tvsum_google_pool5/video_36", 422 | "eccv16_dataset_summe_google_pool5/video_17", 423 | "eccv16_dataset_youtube_google_pool5/video_45", 424 | "eccv16_dataset_tvsum_google_pool5/video_37", 425 | "eccv16_dataset_youtube_google_pool5/video_40", 426 | "eccv16_dataset_youtube_google_pool5/video_38", 427 | "eccv16_dataset_tvsum_google_pool5/video_10", 428 | "eccv16_dataset_youtube_google_pool5/video_13", 429 | "eccv16_dataset_youtube_google_pool5/video_28", 430 | "eccv16_dataset_ovp_google_pool5/video_30", 431 | "eccv16_dataset_ovp_google_pool5/video_43", 432 | "eccv16_dataset_tvsum_google_pool5/video_33", 433 | "eccv16_dataset_youtube_google_pool5/video_42", 434 | "eccv16_dataset_ovp_google_pool5/video_15", 435 | "eccv16_dataset_ovp_google_pool5/video_40", 436 | "eccv16_dataset_youtube_google_pool5/video_17", 437 | "eccv16_dataset_ovp_google_pool5/video_34", 438 | "eccv16_dataset_ovp_google_pool5/video_22", 439 | "eccv16_dataset_ovp_google_pool5/video_5", 440 | "eccv16_dataset_summe_google_pool5/video_14", 441 | "eccv16_dataset_ovp_google_pool5/video_27", 442 | "eccv16_dataset_youtube_google_pool5/video_36", 443 | "eccv16_dataset_ovp_google_pool5/video_20", 444 | "eccv16_dataset_youtube_google_pool5/video_11", 445 | "eccv16_dataset_ovp_google_pool5/video_13", 446 | "eccv16_dataset_tvsum_google_pool5/video_41", 447 | "eccv16_dataset_ovp_google_pool5/video_17", 448 | "eccv16_dataset_youtube_google_pool5/video_50", 449 | "eccv16_dataset_youtube_google_pool5/video_26", 450 | "eccv16_dataset_ovp_google_pool5/video_47", 451 | "eccv16_dataset_ovp_google_pool5/video_2", 452 | "eccv16_dataset_summe_google_pool5/video_25", 453 | "eccv16_dataset_summe_google_pool5/video_24", 454 | "eccv16_dataset_summe_google_pool5/video_10", 455 | "eccv16_dataset_summe_google_pool5/video_1", 456 | "eccv16_dataset_summe_google_pool5/video_5", 457 | "eccv16_dataset_summe_google_pool5/video_4", 458 | "eccv16_dataset_tvsum_google_pool5/video_42", 459 | "eccv16_dataset_tvsum_google_pool5/video_13", 460 | "eccv16_dataset_summe_google_pool5/video_6", 461 | "eccv16_dataset_ovp_google_pool5/video_32", 462 | "eccv16_dataset_youtube_google_pool5/video_25", 463 | "eccv16_dataset_tvsum_google_pool5/video_43", 464 | "eccv16_dataset_ovp_google_pool5/video_11", 465 | "eccv16_dataset_tvsum_google_pool5/video_3", 466 | "eccv16_dataset_ovp_google_pool5/video_10", 467 | "eccv16_dataset_ovp_google_pool5/video_41", 468 | "eccv16_dataset_tvsum_google_pool5/video_12", 469 | "eccv16_dataset_tvsum_google_pool5/video_7", 470 | "eccv16_dataset_youtube_google_pool5/video_41", 471 | "eccv16_dataset_ovp_google_pool5/video_21", 472 | "eccv16_dataset_youtube_google_pool5/video_35", 473 | "eccv16_dataset_ovp_google_pool5/video_39", 474 | "eccv16_dataset_tvsum_google_pool5/video_17", 475 | "eccv16_dataset_summe_google_pool5/video_16", 476 | "eccv16_dataset_summe_google_pool5/video_8", 477 | "eccv16_dataset_tvsum_google_pool5/video_50", 478 | "eccv16_dataset_ovp_google_pool5/video_37", 479 | "eccv16_dataset_tvsum_google_pool5/video_45", 480 | "eccv16_dataset_youtube_google_pool5/video_39", 481 | "eccv16_dataset_youtube_google_pool5/video_15", 482 | "eccv16_dataset_tvsum_google_pool5/video_35", 483 | "eccv16_dataset_ovp_google_pool5/video_29", 484 | "eccv16_dataset_tvsum_google_pool5/video_44", 485 | "eccv16_dataset_ovp_google_pool5/video_18", 486 | "eccv16_dataset_youtube_google_pool5/video_34", 487 | "eccv16_dataset_tvsum_google_pool5/video_29", 488 | "eccv16_dataset_tvsum_google_pool5/video_32", 489 | "eccv16_dataset_youtube_google_pool5/video_47", 490 | "eccv16_dataset_tvsum_google_pool5/video_23", 491 | "eccv16_dataset_ovp_google_pool5/video_19", 492 | "eccv16_dataset_ovp_google_pool5/video_24", 493 | "eccv16_dataset_ovp_google_pool5/video_36", 494 | "eccv16_dataset_tvsum_google_pool5/video_34", 495 | "eccv16_dataset_tvsum_google_pool5/video_49", 496 | "eccv16_dataset_tvsum_google_pool5/video_28", 497 | "eccv16_dataset_ovp_google_pool5/video_26", 498 | "eccv16_dataset_youtube_google_pool5/video_49", 499 | "eccv16_dataset_tvsum_google_pool5/video_46", 500 | "eccv16_dataset_ovp_google_pool5/video_31", 501 | "eccv16_dataset_youtube_google_pool5/video_32", 502 | "eccv16_dataset_tvsum_google_pool5/video_39" 503 | ], 504 | "test_keys": [ 505 | "eccv16_dataset_summe_google_pool5/video_9", 506 | "eccv16_dataset_summe_google_pool5/video_15", 507 | "eccv16_dataset_summe_google_pool5/video_12", 508 | "eccv16_dataset_summe_google_pool5/video_21", 509 | "eccv16_dataset_summe_google_pool5/video_7" 510 | ] 511 | }, 512 | { 513 | "train_keys": [ 514 | "eccv16_dataset_tvsum_google_pool5/video_25", 515 | "eccv16_dataset_tvsum_google_pool5/video_37", 516 | "eccv16_dataset_summe_google_pool5/video_2", 517 | "eccv16_dataset_tvsum_google_pool5/video_48", 518 | "eccv16_dataset_tvsum_google_pool5/video_27", 519 | "eccv16_dataset_ovp_google_pool5/video_14", 520 | "eccv16_dataset_ovp_google_pool5/video_7", 521 | "eccv16_dataset_summe_google_pool5/video_4", 522 | "eccv16_dataset_youtube_google_pool5/video_39", 523 | "eccv16_dataset_ovp_google_pool5/video_3", 524 | "eccv16_dataset_tvsum_google_pool5/video_39", 525 | "eccv16_dataset_summe_google_pool5/video_11", 526 | "eccv16_dataset_ovp_google_pool5/video_9", 527 | "eccv16_dataset_youtube_google_pool5/video_30", 528 | "eccv16_dataset_tvsum_google_pool5/video_9", 529 | "eccv16_dataset_summe_google_pool5/video_13", 530 | "eccv16_dataset_ovp_google_pool5/video_4", 531 | "eccv16_dataset_youtube_google_pool5/video_26", 532 | "eccv16_dataset_youtube_google_pool5/video_31", 533 | "eccv16_dataset_summe_google_pool5/video_17", 534 | "eccv16_dataset_tvsum_google_pool5/video_42", 535 | "eccv16_dataset_youtube_google_pool5/video_36", 536 | "eccv16_dataset_ovp_google_pool5/video_20", 537 | "eccv16_dataset_tvsum_google_pool5/video_18", 538 | "eccv16_dataset_ovp_google_pool5/video_42", 539 | "eccv16_dataset_ovp_google_pool5/video_36", 540 | "eccv16_dataset_ovp_google_pool5/video_34", 541 | "eccv16_dataset_tvsum_google_pool5/video_12", 542 | "eccv16_dataset_youtube_google_pool5/video_29", 543 | "eccv16_dataset_ovp_google_pool5/video_27", 544 | "eccv16_dataset_youtube_google_pool5/video_24", 545 | "eccv16_dataset_ovp_google_pool5/video_23", 546 | "eccv16_dataset_youtube_google_pool5/video_45", 547 | "eccv16_dataset_ovp_google_pool5/video_41", 548 | "eccv16_dataset_ovp_google_pool5/video_12", 549 | "eccv16_dataset_youtube_google_pool5/video_15", 550 | "eccv16_dataset_summe_google_pool5/video_8", 551 | "eccv16_dataset_summe_google_pool5/video_24", 552 | "eccv16_dataset_tvsum_google_pool5/video_44", 553 | "eccv16_dataset_youtube_google_pool5/video_33", 554 | "eccv16_dataset_tvsum_google_pool5/video_23", 555 | "eccv16_dataset_summe_google_pool5/video_3", 556 | "eccv16_dataset_youtube_google_pool5/video_16", 557 | "eccv16_dataset_youtube_google_pool5/video_37", 558 | "eccv16_dataset_summe_google_pool5/video_18", 559 | "eccv16_dataset_ovp_google_pool5/video_10", 560 | "eccv16_dataset_tvsum_google_pool5/video_41", 561 | "eccv16_dataset_youtube_google_pool5/video_14", 562 | "eccv16_dataset_tvsum_google_pool5/video_13", 563 | "eccv16_dataset_ovp_google_pool5/video_35", 564 | "eccv16_dataset_tvsum_google_pool5/video_1", 565 | "eccv16_dataset_summe_google_pool5/video_20", 566 | "eccv16_dataset_tvsum_google_pool5/video_49", 567 | "eccv16_dataset_youtube_google_pool5/video_48", 568 | "eccv16_dataset_tvsum_google_pool5/video_24", 569 | "eccv16_dataset_youtube_google_pool5/video_27", 570 | "eccv16_dataset_youtube_google_pool5/video_25", 571 | "eccv16_dataset_ovp_google_pool5/video_25", 572 | "eccv16_dataset_tvsum_google_pool5/video_11", 573 | "eccv16_dataset_tvsum_google_pool5/video_22", 574 | "eccv16_dataset_tvsum_google_pool5/video_40", 575 | "eccv16_dataset_tvsum_google_pool5/video_45", 576 | "eccv16_dataset_ovp_google_pool5/video_43", 577 | "eccv16_dataset_ovp_google_pool5/video_16", 578 | "eccv16_dataset_tvsum_google_pool5/video_21", 579 | "eccv16_dataset_tvsum_google_pool5/video_31", 580 | "eccv16_dataset_ovp_google_pool5/video_32", 581 | "eccv16_dataset_ovp_google_pool5/video_6", 582 | "eccv16_dataset_ovp_google_pool5/video_1", 583 | "eccv16_dataset_youtube_google_pool5/video_28", 584 | "eccv16_dataset_ovp_google_pool5/video_2", 585 | "eccv16_dataset_youtube_google_pool5/video_35", 586 | "eccv16_dataset_youtube_google_pool5/video_19", 587 | "eccv16_dataset_youtube_google_pool5/video_17", 588 | "eccv16_dataset_tvsum_google_pool5/video_14", 589 | "eccv16_dataset_tvsum_google_pool5/video_46", 590 | "eccv16_dataset_summe_google_pool5/video_1", 591 | "eccv16_dataset_tvsum_google_pool5/video_3", 592 | "eccv16_dataset_ovp_google_pool5/video_24", 593 | "eccv16_dataset_youtube_google_pool5/video_13", 594 | "eccv16_dataset_tvsum_google_pool5/video_43", 595 | "eccv16_dataset_ovp_google_pool5/video_19", 596 | "eccv16_dataset_ovp_google_pool5/video_22", 597 | "eccv16_dataset_tvsum_google_pool5/video_33", 598 | "eccv16_dataset_summe_google_pool5/video_7", 599 | "eccv16_dataset_youtube_google_pool5/video_41", 600 | "eccv16_dataset_tvsum_google_pool5/video_34", 601 | "eccv16_dataset_youtube_google_pool5/video_23", 602 | "eccv16_dataset_tvsum_google_pool5/video_16", 603 | "eccv16_dataset_youtube_google_pool5/video_47", 604 | "eccv16_dataset_summe_google_pool5/video_21", 605 | "eccv16_dataset_tvsum_google_pool5/video_36", 606 | "eccv16_dataset_ovp_google_pool5/video_46", 607 | "eccv16_dataset_tvsum_google_pool5/video_8", 608 | "eccv16_dataset_summe_google_pool5/video_15", 609 | "eccv16_dataset_ovp_google_pool5/video_21", 610 | "eccv16_dataset_ovp_google_pool5/video_15", 611 | "eccv16_dataset_tvsum_google_pool5/video_6", 612 | "eccv16_dataset_ovp_google_pool5/video_26", 613 | "eccv16_dataset_ovp_google_pool5/video_45", 614 | "eccv16_dataset_youtube_google_pool5/video_49", 615 | "eccv16_dataset_tvsum_google_pool5/video_35", 616 | "eccv16_dataset_tvsum_google_pool5/video_7", 617 | "eccv16_dataset_ovp_google_pool5/video_37", 618 | "eccv16_dataset_youtube_google_pool5/video_46", 619 | "eccv16_dataset_youtube_google_pool5/video_20", 620 | "eccv16_dataset_summe_google_pool5/video_10", 621 | "eccv16_dataset_tvsum_google_pool5/video_32", 622 | "eccv16_dataset_ovp_google_pool5/video_49", 623 | "eccv16_dataset_tvsum_google_pool5/video_10", 624 | "eccv16_dataset_ovp_google_pool5/video_33", 625 | "eccv16_dataset_youtube_google_pool5/video_50", 626 | "eccv16_dataset_youtube_google_pool5/video_40", 627 | "eccv16_dataset_ovp_google_pool5/video_48", 628 | "eccv16_dataset_youtube_google_pool5/video_38", 629 | "eccv16_dataset_ovp_google_pool5/video_39", 630 | "eccv16_dataset_tvsum_google_pool5/video_38", 631 | "eccv16_dataset_tvsum_google_pool5/video_20", 632 | "eccv16_dataset_tvsum_google_pool5/video_17", 633 | "eccv16_dataset_summe_google_pool5/video_9", 634 | "eccv16_dataset_youtube_google_pool5/video_32", 635 | "eccv16_dataset_ovp_google_pool5/video_44", 636 | "eccv16_dataset_ovp_google_pool5/video_13", 637 | "eccv16_dataset_youtube_google_pool5/video_18", 638 | "eccv16_dataset_tvsum_google_pool5/video_5", 639 | "eccv16_dataset_ovp_google_pool5/video_31", 640 | "eccv16_dataset_youtube_google_pool5/video_42", 641 | "eccv16_dataset_ovp_google_pool5/video_8", 642 | "eccv16_dataset_youtube_google_pool5/video_11", 643 | "eccv16_dataset_tvsum_google_pool5/video_30", 644 | "eccv16_dataset_tvsum_google_pool5/video_47", 645 | "eccv16_dataset_summe_google_pool5/video_16", 646 | "eccv16_dataset_youtube_google_pool5/video_44", 647 | "eccv16_dataset_ovp_google_pool5/video_5", 648 | "eccv16_dataset_youtube_google_pool5/video_34", 649 | "eccv16_dataset_summe_google_pool5/video_22", 650 | "eccv16_dataset_tvsum_google_pool5/video_29", 651 | "eccv16_dataset_ovp_google_pool5/video_29", 652 | "eccv16_dataset_youtube_google_pool5/video_43", 653 | "eccv16_dataset_ovp_google_pool5/video_40", 654 | "eccv16_dataset_summe_google_pool5/video_6", 655 | "eccv16_dataset_ovp_google_pool5/video_50", 656 | "eccv16_dataset_tvsum_google_pool5/video_15", 657 | "eccv16_dataset_ovp_google_pool5/video_38", 658 | "eccv16_dataset_tvsum_google_pool5/video_26", 659 | "eccv16_dataset_tvsum_google_pool5/video_2", 660 | "eccv16_dataset_tvsum_google_pool5/video_28", 661 | "eccv16_dataset_ovp_google_pool5/video_30", 662 | "eccv16_dataset_ovp_google_pool5/video_18", 663 | "eccv16_dataset_tvsum_google_pool5/video_4", 664 | "eccv16_dataset_tvsum_google_pool5/video_19", 665 | "eccv16_dataset_youtube_google_pool5/video_21", 666 | "eccv16_dataset_ovp_google_pool5/video_11", 667 | "eccv16_dataset_tvsum_google_pool5/video_50", 668 | "eccv16_dataset_ovp_google_pool5/video_28", 669 | "eccv16_dataset_youtube_google_pool5/video_12", 670 | "eccv16_dataset_ovp_google_pool5/video_47", 671 | "eccv16_dataset_ovp_google_pool5/video_17", 672 | "eccv16_dataset_summe_google_pool5/video_25" 673 | ], 674 | "test_keys": [ 675 | "eccv16_dataset_summe_google_pool5/video_12", 676 | "eccv16_dataset_summe_google_pool5/video_5", 677 | "eccv16_dataset_summe_google_pool5/video_19", 678 | "eccv16_dataset_summe_google_pool5/video_14", 679 | "eccv16_dataset_summe_google_pool5/video_23" 680 | ] 681 | }, 682 | { 683 | "train_keys": [ 684 | "eccv16_dataset_youtube_google_pool5/video_14", 685 | "eccv16_dataset_youtube_google_pool5/video_32", 686 | "eccv16_dataset_youtube_google_pool5/video_41", 687 | "eccv16_dataset_ovp_google_pool5/video_8", 688 | "eccv16_dataset_youtube_google_pool5/video_24", 689 | "eccv16_dataset_tvsum_google_pool5/video_14", 690 | "eccv16_dataset_ovp_google_pool5/video_32", 691 | "eccv16_dataset_youtube_google_pool5/video_37", 692 | "eccv16_dataset_summe_google_pool5/video_3", 693 | "eccv16_dataset_ovp_google_pool5/video_16", 694 | "eccv16_dataset_tvsum_google_pool5/video_15", 695 | "eccv16_dataset_summe_google_pool5/video_20", 696 | "eccv16_dataset_ovp_google_pool5/video_33", 697 | "eccv16_dataset_summe_google_pool5/video_11", 698 | "eccv16_dataset_youtube_google_pool5/video_13", 699 | "eccv16_dataset_youtube_google_pool5/video_16", 700 | "eccv16_dataset_tvsum_google_pool5/video_27", 701 | "eccv16_dataset_youtube_google_pool5/video_43", 702 | "eccv16_dataset_youtube_google_pool5/video_11", 703 | "eccv16_dataset_ovp_google_pool5/video_25", 704 | "eccv16_dataset_tvsum_google_pool5/video_23", 705 | "eccv16_dataset_tvsum_google_pool5/video_1", 706 | "eccv16_dataset_tvsum_google_pool5/video_19", 707 | "eccv16_dataset_youtube_google_pool5/video_46", 708 | "eccv16_dataset_tvsum_google_pool5/video_5", 709 | "eccv16_dataset_youtube_google_pool5/video_45", 710 | "eccv16_dataset_youtube_google_pool5/video_38", 711 | "eccv16_dataset_youtube_google_pool5/video_15", 712 | "eccv16_dataset_summe_google_pool5/video_25", 713 | "eccv16_dataset_youtube_google_pool5/video_50", 714 | "eccv16_dataset_youtube_google_pool5/video_19", 715 | "eccv16_dataset_ovp_google_pool5/video_6", 716 | "eccv16_dataset_ovp_google_pool5/video_49", 717 | "eccv16_dataset_tvsum_google_pool5/video_9", 718 | "eccv16_dataset_ovp_google_pool5/video_18", 719 | "eccv16_dataset_tvsum_google_pool5/video_32", 720 | "eccv16_dataset_tvsum_google_pool5/video_10", 721 | "eccv16_dataset_tvsum_google_pool5/video_44", 722 | "eccv16_dataset_ovp_google_pool5/video_50", 723 | "eccv16_dataset_tvsum_google_pool5/video_26", 724 | "eccv16_dataset_youtube_google_pool5/video_44", 725 | "eccv16_dataset_ovp_google_pool5/video_28", 726 | "eccv16_dataset_tvsum_google_pool5/video_16", 727 | "eccv16_dataset_tvsum_google_pool5/video_46", 728 | "eccv16_dataset_tvsum_google_pool5/video_34", 729 | "eccv16_dataset_tvsum_google_pool5/video_47", 730 | "eccv16_dataset_youtube_google_pool5/video_28", 731 | "eccv16_dataset_ovp_google_pool5/video_38", 732 | "eccv16_dataset_youtube_google_pool5/video_35", 733 | "eccv16_dataset_summe_google_pool5/video_15", 734 | "eccv16_dataset_tvsum_google_pool5/video_29", 735 | "eccv16_dataset_youtube_google_pool5/video_12", 736 | "eccv16_dataset_ovp_google_pool5/video_20", 737 | "eccv16_dataset_tvsum_google_pool5/video_18", 738 | "eccv16_dataset_summe_google_pool5/video_18", 739 | "eccv16_dataset_ovp_google_pool5/video_27", 740 | "eccv16_dataset_ovp_google_pool5/video_1", 741 | "eccv16_dataset_summe_google_pool5/video_4", 742 | "eccv16_dataset_summe_google_pool5/video_8", 743 | "eccv16_dataset_ovp_google_pool5/video_47", 744 | "eccv16_dataset_youtube_google_pool5/video_21", 745 | "eccv16_dataset_summe_google_pool5/video_6", 746 | "eccv16_dataset_youtube_google_pool5/video_18", 747 | "eccv16_dataset_ovp_google_pool5/video_7", 748 | "eccv16_dataset_youtube_google_pool5/video_23", 749 | "eccv16_dataset_youtube_google_pool5/video_20", 750 | "eccv16_dataset_tvsum_google_pool5/video_3", 751 | "eccv16_dataset_ovp_google_pool5/video_17", 752 | "eccv16_dataset_ovp_google_pool5/video_45", 753 | "eccv16_dataset_summe_google_pool5/video_22", 754 | "eccv16_dataset_tvsum_google_pool5/video_37", 755 | "eccv16_dataset_tvsum_google_pool5/video_43", 756 | "eccv16_dataset_ovp_google_pool5/video_19", 757 | "eccv16_dataset_youtube_google_pool5/video_25", 758 | "eccv16_dataset_tvsum_google_pool5/video_17", 759 | "eccv16_dataset_ovp_google_pool5/video_15", 760 | "eccv16_dataset_tvsum_google_pool5/video_11", 761 | "eccv16_dataset_youtube_google_pool5/video_42", 762 | "eccv16_dataset_tvsum_google_pool5/video_24", 763 | "eccv16_dataset_youtube_google_pool5/video_40", 764 | "eccv16_dataset_tvsum_google_pool5/video_40", 765 | "eccv16_dataset_youtube_google_pool5/video_29", 766 | "eccv16_dataset_youtube_google_pool5/video_31", 767 | "eccv16_dataset_tvsum_google_pool5/video_49", 768 | "eccv16_dataset_ovp_google_pool5/video_2", 769 | "eccv16_dataset_ovp_google_pool5/video_30", 770 | "eccv16_dataset_youtube_google_pool5/video_27", 771 | "eccv16_dataset_summe_google_pool5/video_9", 772 | "eccv16_dataset_youtube_google_pool5/video_17", 773 | "eccv16_dataset_tvsum_google_pool5/video_8", 774 | "eccv16_dataset_ovp_google_pool5/video_10", 775 | "eccv16_dataset_youtube_google_pool5/video_48", 776 | "eccv16_dataset_ovp_google_pool5/video_26", 777 | "eccv16_dataset_tvsum_google_pool5/video_31", 778 | "eccv16_dataset_tvsum_google_pool5/video_38", 779 | "eccv16_dataset_tvsum_google_pool5/video_39", 780 | "eccv16_dataset_ovp_google_pool5/video_24", 781 | "eccv16_dataset_ovp_google_pool5/video_31", 782 | "eccv16_dataset_summe_google_pool5/video_5", 783 | "eccv16_dataset_youtube_google_pool5/video_36", 784 | "eccv16_dataset_tvsum_google_pool5/video_28", 785 | "eccv16_dataset_ovp_google_pool5/video_48", 786 | "eccv16_dataset_ovp_google_pool5/video_43", 787 | "eccv16_dataset_tvsum_google_pool5/video_13", 788 | "eccv16_dataset_summe_google_pool5/video_1", 789 | "eccv16_dataset_youtube_google_pool5/video_33", 790 | "eccv16_dataset_ovp_google_pool5/video_12", 791 | "eccv16_dataset_ovp_google_pool5/video_46", 792 | "eccv16_dataset_ovp_google_pool5/video_29", 793 | "eccv16_dataset_summe_google_pool5/video_17", 794 | "eccv16_dataset_tvsum_google_pool5/video_2", 795 | "eccv16_dataset_ovp_google_pool5/video_9", 796 | "eccv16_dataset_tvsum_google_pool5/video_4", 797 | "eccv16_dataset_youtube_google_pool5/video_47", 798 | "eccv16_dataset_ovp_google_pool5/video_34", 799 | "eccv16_dataset_ovp_google_pool5/video_42", 800 | "eccv16_dataset_ovp_google_pool5/video_39", 801 | "eccv16_dataset_ovp_google_pool5/video_22", 802 | "eccv16_dataset_tvsum_google_pool5/video_45", 803 | "eccv16_dataset_ovp_google_pool5/video_4", 804 | "eccv16_dataset_ovp_google_pool5/video_36", 805 | "eccv16_dataset_tvsum_google_pool5/video_21", 806 | "eccv16_dataset_ovp_google_pool5/video_11", 807 | "eccv16_dataset_tvsum_google_pool5/video_22", 808 | "eccv16_dataset_ovp_google_pool5/video_14", 809 | "eccv16_dataset_tvsum_google_pool5/video_20", 810 | "eccv16_dataset_summe_google_pool5/video_10", 811 | "eccv16_dataset_ovp_google_pool5/video_23", 812 | "eccv16_dataset_tvsum_google_pool5/video_33", 813 | "eccv16_dataset_summe_google_pool5/video_21", 814 | "eccv16_dataset_youtube_google_pool5/video_26", 815 | "eccv16_dataset_tvsum_google_pool5/video_35", 816 | "eccv16_dataset_tvsum_google_pool5/video_48", 817 | "eccv16_dataset_tvsum_google_pool5/video_30", 818 | "eccv16_dataset_tvsum_google_pool5/video_7", 819 | "eccv16_dataset_ovp_google_pool5/video_21", 820 | "eccv16_dataset_tvsum_google_pool5/video_41", 821 | "eccv16_dataset_summe_google_pool5/video_2", 822 | "eccv16_dataset_ovp_google_pool5/video_40", 823 | "eccv16_dataset_ovp_google_pool5/video_41", 824 | "eccv16_dataset_tvsum_google_pool5/video_6", 825 | "eccv16_dataset_youtube_google_pool5/video_39", 826 | "eccv16_dataset_ovp_google_pool5/video_5", 827 | "eccv16_dataset_youtube_google_pool5/video_30", 828 | "eccv16_dataset_ovp_google_pool5/video_35", 829 | "eccv16_dataset_youtube_google_pool5/video_34", 830 | "eccv16_dataset_tvsum_google_pool5/video_36", 831 | "eccv16_dataset_ovp_google_pool5/video_3", 832 | "eccv16_dataset_tvsum_google_pool5/video_50", 833 | "eccv16_dataset_youtube_google_pool5/video_49", 834 | "eccv16_dataset_ovp_google_pool5/video_44", 835 | "eccv16_dataset_tvsum_google_pool5/video_12", 836 | "eccv16_dataset_ovp_google_pool5/video_37", 837 | "eccv16_dataset_summe_google_pool5/video_16", 838 | "eccv16_dataset_tvsum_google_pool5/video_25", 839 | "eccv16_dataset_ovp_google_pool5/video_13", 840 | "eccv16_dataset_summe_google_pool5/video_24", 841 | "eccv16_dataset_summe_google_pool5/video_13", 842 | "eccv16_dataset_tvsum_google_pool5/video_42" 843 | ], 844 | "test_keys": [ 845 | "eccv16_dataset_summe_google_pool5/video_7", 846 | "eccv16_dataset_summe_google_pool5/video_12", 847 | "eccv16_dataset_summe_google_pool5/video_19", 848 | "eccv16_dataset_summe_google_pool5/video_23", 849 | "eccv16_dataset_summe_google_pool5/video_14" 850 | ] 851 | } 852 | ] -------------------------------------------------------------------------------- /splits/tvsum_aug_splits.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "test_keys": [ 4 | "eccv16_dataset_tvsum_google_pool5/video_41", 5 | "eccv16_dataset_tvsum_google_pool5/video_31", 6 | "eccv16_dataset_tvsum_google_pool5/video_12", 7 | "eccv16_dataset_tvsum_google_pool5/video_2", 8 | "eccv16_dataset_tvsum_google_pool5/video_36", 9 | "eccv16_dataset_tvsum_google_pool5/video_37", 10 | "eccv16_dataset_tvsum_google_pool5/video_21", 11 | "eccv16_dataset_tvsum_google_pool5/video_35", 12 | "eccv16_dataset_tvsum_google_pool5/video_14", 13 | "eccv16_dataset_tvsum_google_pool5/video_11" 14 | ], 15 | "train_keys": [ 16 | "eccv16_dataset_tvsum_google_pool5/video_42", 17 | "eccv16_dataset_ovp_google_pool5/video_10", 18 | "eccv16_dataset_ovp_google_pool5/video_26", 19 | "eccv16_dataset_ovp_google_pool5/video_21", 20 | "eccv16_dataset_summe_google_pool5/video_22", 21 | "eccv16_dataset_youtube_google_pool5/video_49", 22 | "eccv16_dataset_youtube_google_pool5/video_36", 23 | "eccv16_dataset_summe_google_pool5/video_1", 24 | "eccv16_dataset_ovp_google_pool5/video_20", 25 | "eccv16_dataset_youtube_google_pool5/video_28", 26 | "eccv16_dataset_summe_google_pool5/video_15", 27 | "eccv16_dataset_youtube_google_pool5/video_15", 28 | "eccv16_dataset_youtube_google_pool5/video_33", 29 | "eccv16_dataset_ovp_google_pool5/video_46", 30 | "eccv16_dataset_youtube_google_pool5/video_39", 31 | "eccv16_dataset_youtube_google_pool5/video_46", 32 | "eccv16_dataset_tvsum_google_pool5/video_6", 33 | "eccv16_dataset_youtube_google_pool5/video_27", 34 | "eccv16_dataset_youtube_google_pool5/video_16", 35 | "eccv16_dataset_youtube_google_pool5/video_34", 36 | "eccv16_dataset_youtube_google_pool5/video_11", 37 | "eccv16_dataset_ovp_google_pool5/video_15", 38 | "eccv16_dataset_youtube_google_pool5/video_23", 39 | "eccv16_dataset_tvsum_google_pool5/video_9", 40 | "eccv16_dataset_tvsum_google_pool5/video_46", 41 | "eccv16_dataset_tvsum_google_pool5/video_30", 42 | "eccv16_dataset_ovp_google_pool5/video_43", 43 | "eccv16_dataset_summe_google_pool5/video_2", 44 | "eccv16_dataset_tvsum_google_pool5/video_3", 45 | "eccv16_dataset_ovp_google_pool5/video_25", 46 | "eccv16_dataset_tvsum_google_pool5/video_43", 47 | "eccv16_dataset_ovp_google_pool5/video_45", 48 | "eccv16_dataset_tvsum_google_pool5/video_26", 49 | "eccv16_dataset_youtube_google_pool5/video_50", 50 | "eccv16_dataset_ovp_google_pool5/video_12", 51 | "eccv16_dataset_tvsum_google_pool5/video_24", 52 | "eccv16_dataset_youtube_google_pool5/video_13", 53 | "eccv16_dataset_ovp_google_pool5/video_39", 54 | "eccv16_dataset_ovp_google_pool5/video_7", 55 | "eccv16_dataset_youtube_google_pool5/video_38", 56 | "eccv16_dataset_youtube_google_pool5/video_24", 57 | "eccv16_dataset_summe_google_pool5/video_11", 58 | "eccv16_dataset_summe_google_pool5/video_6", 59 | "eccv16_dataset_tvsum_google_pool5/video_28", 60 | "eccv16_dataset_ovp_google_pool5/video_19", 61 | "eccv16_dataset_youtube_google_pool5/video_14", 62 | "eccv16_dataset_youtube_google_pool5/video_26", 63 | "eccv16_dataset_youtube_google_pool5/video_17", 64 | "eccv16_dataset_youtube_google_pool5/video_45", 65 | "eccv16_dataset_summe_google_pool5/video_14", 66 | "eccv16_dataset_ovp_google_pool5/video_13", 67 | "eccv16_dataset_summe_google_pool5/video_8", 68 | "eccv16_dataset_youtube_google_pool5/video_47", 69 | "eccv16_dataset_summe_google_pool5/video_10", 70 | "eccv16_dataset_summe_google_pool5/video_9", 71 | "eccv16_dataset_summe_google_pool5/video_25", 72 | "eccv16_dataset_tvsum_google_pool5/video_50", 73 | "eccv16_dataset_tvsum_google_pool5/video_4", 74 | "eccv16_dataset_tvsum_google_pool5/video_27", 75 | "eccv16_dataset_youtube_google_pool5/video_35", 76 | "eccv16_dataset_ovp_google_pool5/video_5", 77 | "eccv16_dataset_summe_google_pool5/video_16", 78 | "eccv16_dataset_ovp_google_pool5/video_31", 79 | "eccv16_dataset_tvsum_google_pool5/video_29", 80 | "eccv16_dataset_youtube_google_pool5/video_41", 81 | "eccv16_dataset_ovp_google_pool5/video_42", 82 | "eccv16_dataset_ovp_google_pool5/video_40", 83 | "eccv16_dataset_tvsum_google_pool5/video_34", 84 | "eccv16_dataset_tvsum_google_pool5/video_38", 85 | "eccv16_dataset_tvsum_google_pool5/video_47", 86 | "eccv16_dataset_youtube_google_pool5/video_42", 87 | "eccv16_dataset_tvsum_google_pool5/video_33", 88 | "eccv16_dataset_ovp_google_pool5/video_1", 89 | "eccv16_dataset_tvsum_google_pool5/video_13", 90 | "eccv16_dataset_ovp_google_pool5/video_11", 91 | "eccv16_dataset_ovp_google_pool5/video_3", 92 | "eccv16_dataset_ovp_google_pool5/video_49", 93 | "eccv16_dataset_summe_google_pool5/video_17", 94 | "eccv16_dataset_youtube_google_pool5/video_20", 95 | "eccv16_dataset_youtube_google_pool5/video_44", 96 | "eccv16_dataset_summe_google_pool5/video_7", 97 | "eccv16_dataset_youtube_google_pool5/video_32", 98 | "eccv16_dataset_summe_google_pool5/video_19", 99 | "eccv16_dataset_youtube_google_pool5/video_19", 100 | "eccv16_dataset_summe_google_pool5/video_4", 101 | "eccv16_dataset_ovp_google_pool5/video_48", 102 | "eccv16_dataset_ovp_google_pool5/video_50", 103 | "eccv16_dataset_summe_google_pool5/video_12", 104 | "eccv16_dataset_tvsum_google_pool5/video_23", 105 | "eccv16_dataset_ovp_google_pool5/video_24", 106 | "eccv16_dataset_youtube_google_pool5/video_31", 107 | "eccv16_dataset_tvsum_google_pool5/video_40", 108 | "eccv16_dataset_ovp_google_pool5/video_23", 109 | "eccv16_dataset_ovp_google_pool5/video_34", 110 | "eccv16_dataset_tvsum_google_pool5/video_15", 111 | "eccv16_dataset_youtube_google_pool5/video_25", 112 | "eccv16_dataset_ovp_google_pool5/video_16", 113 | "eccv16_dataset_tvsum_google_pool5/video_10", 114 | "eccv16_dataset_tvsum_google_pool5/video_17", 115 | "eccv16_dataset_ovp_google_pool5/video_28", 116 | "eccv16_dataset_ovp_google_pool5/video_9", 117 | "eccv16_dataset_summe_google_pool5/video_13", 118 | "eccv16_dataset_tvsum_google_pool5/video_48", 119 | "eccv16_dataset_summe_google_pool5/video_3", 120 | "eccv16_dataset_youtube_google_pool5/video_12", 121 | "eccv16_dataset_tvsum_google_pool5/video_20", 122 | "eccv16_dataset_tvsum_google_pool5/video_1", 123 | "eccv16_dataset_tvsum_google_pool5/video_8", 124 | "eccv16_dataset_youtube_google_pool5/video_40", 125 | "eccv16_dataset_summe_google_pool5/video_24", 126 | "eccv16_dataset_youtube_google_pool5/video_43", 127 | "eccv16_dataset_ovp_google_pool5/video_6", 128 | "eccv16_dataset_summe_google_pool5/video_23", 129 | "eccv16_dataset_ovp_google_pool5/video_29", 130 | "eccv16_dataset_ovp_google_pool5/video_30", 131 | "eccv16_dataset_ovp_google_pool5/video_2", 132 | "eccv16_dataset_tvsum_google_pool5/video_18", 133 | "eccv16_dataset_ovp_google_pool5/video_35", 134 | "eccv16_dataset_ovp_google_pool5/video_38", 135 | "eccv16_dataset_ovp_google_pool5/video_32", 136 | "eccv16_dataset_ovp_google_pool5/video_17", 137 | "eccv16_dataset_tvsum_google_pool5/video_39", 138 | "eccv16_dataset_ovp_google_pool5/video_14", 139 | "eccv16_dataset_ovp_google_pool5/video_44", 140 | "eccv16_dataset_ovp_google_pool5/video_37", 141 | "eccv16_dataset_ovp_google_pool5/video_4", 142 | "eccv16_dataset_summe_google_pool5/video_18", 143 | "eccv16_dataset_tvsum_google_pool5/video_32", 144 | "eccv16_dataset_ovp_google_pool5/video_8", 145 | "eccv16_dataset_youtube_google_pool5/video_21", 146 | "eccv16_dataset_youtube_google_pool5/video_18", 147 | "eccv16_dataset_tvsum_google_pool5/video_49", 148 | "eccv16_dataset_ovp_google_pool5/video_36", 149 | "eccv16_dataset_tvsum_google_pool5/video_7", 150 | "eccv16_dataset_ovp_google_pool5/video_18", 151 | "eccv16_dataset_ovp_google_pool5/video_47", 152 | "eccv16_dataset_tvsum_google_pool5/video_25", 153 | "eccv16_dataset_youtube_google_pool5/video_48", 154 | "eccv16_dataset_tvsum_google_pool5/video_45", 155 | "eccv16_dataset_summe_google_pool5/video_20", 156 | "eccv16_dataset_ovp_google_pool5/video_22", 157 | "eccv16_dataset_ovp_google_pool5/video_27", 158 | "eccv16_dataset_summe_google_pool5/video_5", 159 | "eccv16_dataset_tvsum_google_pool5/video_5", 160 | "eccv16_dataset_youtube_google_pool5/video_30", 161 | "eccv16_dataset_tvsum_google_pool5/video_19", 162 | "eccv16_dataset_summe_google_pool5/video_21", 163 | "eccv16_dataset_ovp_google_pool5/video_33", 164 | "eccv16_dataset_youtube_google_pool5/video_37", 165 | "eccv16_dataset_tvsum_google_pool5/video_44", 166 | "eccv16_dataset_ovp_google_pool5/video_41", 167 | "eccv16_dataset_youtube_google_pool5/video_29", 168 | "eccv16_dataset_tvsum_google_pool5/video_16", 169 | "eccv16_dataset_tvsum_google_pool5/video_22" 170 | ] 171 | }, 172 | { 173 | "test_keys": [ 174 | "eccv16_dataset_tvsum_google_pool5/video_13", 175 | "eccv16_dataset_tvsum_google_pool5/video_32", 176 | "eccv16_dataset_tvsum_google_pool5/video_28", 177 | "eccv16_dataset_tvsum_google_pool5/video_24", 178 | "eccv16_dataset_tvsum_google_pool5/video_30", 179 | "eccv16_dataset_tvsum_google_pool5/video_12", 180 | "eccv16_dataset_tvsum_google_pool5/video_27", 181 | "eccv16_dataset_tvsum_google_pool5/video_43", 182 | "eccv16_dataset_tvsum_google_pool5/video_35", 183 | "eccv16_dataset_tvsum_google_pool5/video_6" 184 | ], 185 | "train_keys": [ 186 | "eccv16_dataset_tvsum_google_pool5/video_18", 187 | "eccv16_dataset_summe_google_pool5/video_15", 188 | "eccv16_dataset_ovp_google_pool5/video_47", 189 | "eccv16_dataset_tvsum_google_pool5/video_23", 190 | "eccv16_dataset_summe_google_pool5/video_24", 191 | "eccv16_dataset_tvsum_google_pool5/video_14", 192 | "eccv16_dataset_youtube_google_pool5/video_24", 193 | "eccv16_dataset_summe_google_pool5/video_7", 194 | "eccv16_dataset_ovp_google_pool5/video_19", 195 | "eccv16_dataset_tvsum_google_pool5/video_34", 196 | "eccv16_dataset_ovp_google_pool5/video_12", 197 | "eccv16_dataset_summe_google_pool5/video_22", 198 | "eccv16_dataset_youtube_google_pool5/video_29", 199 | "eccv16_dataset_youtube_google_pool5/video_40", 200 | "eccv16_dataset_ovp_google_pool5/video_15", 201 | "eccv16_dataset_ovp_google_pool5/video_9", 202 | "eccv16_dataset_tvsum_google_pool5/video_42", 203 | "eccv16_dataset_youtube_google_pool5/video_28", 204 | "eccv16_dataset_ovp_google_pool5/video_1", 205 | "eccv16_dataset_ovp_google_pool5/video_50", 206 | "eccv16_dataset_ovp_google_pool5/video_48", 207 | "eccv16_dataset_ovp_google_pool5/video_42", 208 | "eccv16_dataset_tvsum_google_pool5/video_50", 209 | "eccv16_dataset_ovp_google_pool5/video_13", 210 | "eccv16_dataset_tvsum_google_pool5/video_10", 211 | "eccv16_dataset_ovp_google_pool5/video_11", 212 | "eccv16_dataset_ovp_google_pool5/video_37", 213 | "eccv16_dataset_youtube_google_pool5/video_30", 214 | "eccv16_dataset_youtube_google_pool5/video_21", 215 | "eccv16_dataset_summe_google_pool5/video_12", 216 | "eccv16_dataset_summe_google_pool5/video_3", 217 | "eccv16_dataset_youtube_google_pool5/video_13", 218 | "eccv16_dataset_youtube_google_pool5/video_33", 219 | "eccv16_dataset_ovp_google_pool5/video_44", 220 | "eccv16_dataset_tvsum_google_pool5/video_45", 221 | "eccv16_dataset_ovp_google_pool5/video_29", 222 | "eccv16_dataset_ovp_google_pool5/video_18", 223 | "eccv16_dataset_summe_google_pool5/video_6", 224 | "eccv16_dataset_tvsum_google_pool5/video_1", 225 | "eccv16_dataset_summe_google_pool5/video_8", 226 | "eccv16_dataset_tvsum_google_pool5/video_22", 227 | "eccv16_dataset_summe_google_pool5/video_1", 228 | "eccv16_dataset_ovp_google_pool5/video_7", 229 | "eccv16_dataset_ovp_google_pool5/video_35", 230 | "eccv16_dataset_tvsum_google_pool5/video_37", 231 | "eccv16_dataset_ovp_google_pool5/video_34", 232 | "eccv16_dataset_summe_google_pool5/video_13", 233 | "eccv16_dataset_summe_google_pool5/video_11", 234 | "eccv16_dataset_youtube_google_pool5/video_11", 235 | "eccv16_dataset_ovp_google_pool5/video_6", 236 | "eccv16_dataset_youtube_google_pool5/video_42", 237 | "eccv16_dataset_tvsum_google_pool5/video_17", 238 | "eccv16_dataset_youtube_google_pool5/video_48", 239 | "eccv16_dataset_tvsum_google_pool5/video_44", 240 | "eccv16_dataset_summe_google_pool5/video_5", 241 | "eccv16_dataset_tvsum_google_pool5/video_33", 242 | "eccv16_dataset_youtube_google_pool5/video_15", 243 | "eccv16_dataset_ovp_google_pool5/video_24", 244 | "eccv16_dataset_youtube_google_pool5/video_27", 245 | "eccv16_dataset_ovp_google_pool5/video_25", 246 | "eccv16_dataset_youtube_google_pool5/video_37", 247 | "eccv16_dataset_ovp_google_pool5/video_20", 248 | "eccv16_dataset_youtube_google_pool5/video_12", 249 | "eccv16_dataset_tvsum_google_pool5/video_38", 250 | "eccv16_dataset_ovp_google_pool5/video_40", 251 | "eccv16_dataset_tvsum_google_pool5/video_31", 252 | "eccv16_dataset_ovp_google_pool5/video_28", 253 | "eccv16_dataset_youtube_google_pool5/video_19", 254 | "eccv16_dataset_summe_google_pool5/video_19", 255 | "eccv16_dataset_ovp_google_pool5/video_45", 256 | "eccv16_dataset_ovp_google_pool5/video_30", 257 | "eccv16_dataset_tvsum_google_pool5/video_25", 258 | "eccv16_dataset_ovp_google_pool5/video_4", 259 | "eccv16_dataset_summe_google_pool5/video_16", 260 | "eccv16_dataset_tvsum_google_pool5/video_40", 261 | "eccv16_dataset_tvsum_google_pool5/video_39", 262 | "eccv16_dataset_summe_google_pool5/video_23", 263 | "eccv16_dataset_youtube_google_pool5/video_38", 264 | "eccv16_dataset_ovp_google_pool5/video_49", 265 | "eccv16_dataset_summe_google_pool5/video_18", 266 | "eccv16_dataset_ovp_google_pool5/video_43", 267 | "eccv16_dataset_summe_google_pool5/video_17", 268 | "eccv16_dataset_ovp_google_pool5/video_2", 269 | "eccv16_dataset_summe_google_pool5/video_10", 270 | "eccv16_dataset_youtube_google_pool5/video_14", 271 | "eccv16_dataset_ovp_google_pool5/video_16", 272 | "eccv16_dataset_ovp_google_pool5/video_38", 273 | "eccv16_dataset_youtube_google_pool5/video_32", 274 | "eccv16_dataset_tvsum_google_pool5/video_46", 275 | "eccv16_dataset_summe_google_pool5/video_4", 276 | "eccv16_dataset_ovp_google_pool5/video_3", 277 | "eccv16_dataset_youtube_google_pool5/video_25", 278 | "eccv16_dataset_ovp_google_pool5/video_36", 279 | "eccv16_dataset_ovp_google_pool5/video_22", 280 | "eccv16_dataset_youtube_google_pool5/video_17", 281 | "eccv16_dataset_ovp_google_pool5/video_10", 282 | "eccv16_dataset_summe_google_pool5/video_2", 283 | "eccv16_dataset_tvsum_google_pool5/video_11", 284 | "eccv16_dataset_summe_google_pool5/video_25", 285 | "eccv16_dataset_youtube_google_pool5/video_31", 286 | "eccv16_dataset_youtube_google_pool5/video_36", 287 | "eccv16_dataset_tvsum_google_pool5/video_5", 288 | "eccv16_dataset_ovp_google_pool5/video_32", 289 | "eccv16_dataset_tvsum_google_pool5/video_7", 290 | "eccv16_dataset_youtube_google_pool5/video_18", 291 | "eccv16_dataset_youtube_google_pool5/video_46", 292 | "eccv16_dataset_tvsum_google_pool5/video_15", 293 | "eccv16_dataset_tvsum_google_pool5/video_16", 294 | "eccv16_dataset_ovp_google_pool5/video_33", 295 | "eccv16_dataset_tvsum_google_pool5/video_4", 296 | "eccv16_dataset_tvsum_google_pool5/video_19", 297 | "eccv16_dataset_youtube_google_pool5/video_43", 298 | "eccv16_dataset_youtube_google_pool5/video_20", 299 | "eccv16_dataset_youtube_google_pool5/video_26", 300 | "eccv16_dataset_ovp_google_pool5/video_23", 301 | "eccv16_dataset_youtube_google_pool5/video_50", 302 | "eccv16_dataset_ovp_google_pool5/video_39", 303 | "eccv16_dataset_youtube_google_pool5/video_41", 304 | "eccv16_dataset_tvsum_google_pool5/video_36", 305 | "eccv16_dataset_summe_google_pool5/video_21", 306 | "eccv16_dataset_youtube_google_pool5/video_34", 307 | "eccv16_dataset_ovp_google_pool5/video_14", 308 | "eccv16_dataset_youtube_google_pool5/video_45", 309 | "eccv16_dataset_tvsum_google_pool5/video_41", 310 | "eccv16_dataset_tvsum_google_pool5/video_48", 311 | "eccv16_dataset_tvsum_google_pool5/video_29", 312 | "eccv16_dataset_ovp_google_pool5/video_21", 313 | "eccv16_dataset_youtube_google_pool5/video_44", 314 | "eccv16_dataset_ovp_google_pool5/video_26", 315 | "eccv16_dataset_youtube_google_pool5/video_49", 316 | "eccv16_dataset_ovp_google_pool5/video_27", 317 | "eccv16_dataset_tvsum_google_pool5/video_2", 318 | "eccv16_dataset_youtube_google_pool5/video_47", 319 | "eccv16_dataset_youtube_google_pool5/video_39", 320 | "eccv16_dataset_ovp_google_pool5/video_17", 321 | "eccv16_dataset_summe_google_pool5/video_9", 322 | "eccv16_dataset_tvsum_google_pool5/video_21", 323 | "eccv16_dataset_ovp_google_pool5/video_8", 324 | "eccv16_dataset_ovp_google_pool5/video_41", 325 | "eccv16_dataset_ovp_google_pool5/video_31", 326 | "eccv16_dataset_ovp_google_pool5/video_46", 327 | "eccv16_dataset_tvsum_google_pool5/video_49", 328 | "eccv16_dataset_tvsum_google_pool5/video_47", 329 | "eccv16_dataset_tvsum_google_pool5/video_8", 330 | "eccv16_dataset_tvsum_google_pool5/video_20", 331 | "eccv16_dataset_summe_google_pool5/video_14", 332 | "eccv16_dataset_summe_google_pool5/video_20", 333 | "eccv16_dataset_youtube_google_pool5/video_23", 334 | "eccv16_dataset_youtube_google_pool5/video_35", 335 | "eccv16_dataset_ovp_google_pool5/video_5", 336 | "eccv16_dataset_tvsum_google_pool5/video_26", 337 | "eccv16_dataset_tvsum_google_pool5/video_9", 338 | "eccv16_dataset_youtube_google_pool5/video_16", 339 | "eccv16_dataset_tvsum_google_pool5/video_3" 340 | ] 341 | }, 342 | { 343 | "test_keys": [ 344 | "eccv16_dataset_tvsum_google_pool5/video_39", 345 | "eccv16_dataset_tvsum_google_pool5/video_7", 346 | "eccv16_dataset_tvsum_google_pool5/video_16", 347 | "eccv16_dataset_tvsum_google_pool5/video_36", 348 | "eccv16_dataset_tvsum_google_pool5/video_35", 349 | "eccv16_dataset_tvsum_google_pool5/video_9", 350 | "eccv16_dataset_tvsum_google_pool5/video_46", 351 | "eccv16_dataset_tvsum_google_pool5/video_8", 352 | "eccv16_dataset_tvsum_google_pool5/video_1", 353 | "eccv16_dataset_tvsum_google_pool5/video_22" 354 | ], 355 | "train_keys": [ 356 | "eccv16_dataset_tvsum_google_pool5/video_6", 357 | "eccv16_dataset_tvsum_google_pool5/video_2", 358 | "eccv16_dataset_tvsum_google_pool5/video_4", 359 | "eccv16_dataset_tvsum_google_pool5/video_32", 360 | "eccv16_dataset_youtube_google_pool5/video_35", 361 | "eccv16_dataset_ovp_google_pool5/video_5", 362 | "eccv16_dataset_ovp_google_pool5/video_36", 363 | "eccv16_dataset_tvsum_google_pool5/video_33", 364 | "eccv16_dataset_ovp_google_pool5/video_27", 365 | "eccv16_dataset_summe_google_pool5/video_19", 366 | "eccv16_dataset_youtube_google_pool5/video_40", 367 | "eccv16_dataset_ovp_google_pool5/video_43", 368 | "eccv16_dataset_tvsum_google_pool5/video_42", 369 | "eccv16_dataset_ovp_google_pool5/video_31", 370 | "eccv16_dataset_ovp_google_pool5/video_18", 371 | "eccv16_dataset_tvsum_google_pool5/video_15", 372 | "eccv16_dataset_summe_google_pool5/video_5", 373 | "eccv16_dataset_ovp_google_pool5/video_35", 374 | "eccv16_dataset_summe_google_pool5/video_14", 375 | "eccv16_dataset_ovp_google_pool5/video_50", 376 | "eccv16_dataset_youtube_google_pool5/video_30", 377 | "eccv16_dataset_youtube_google_pool5/video_41", 378 | "eccv16_dataset_ovp_google_pool5/video_22", 379 | "eccv16_dataset_ovp_google_pool5/video_37", 380 | "eccv16_dataset_summe_google_pool5/video_15", 381 | "eccv16_dataset_youtube_google_pool5/video_33", 382 | "eccv16_dataset_ovp_google_pool5/video_45", 383 | "eccv16_dataset_ovp_google_pool5/video_21", 384 | "eccv16_dataset_youtube_google_pool5/video_19", 385 | "eccv16_dataset_tvsum_google_pool5/video_13", 386 | "eccv16_dataset_ovp_google_pool5/video_46", 387 | "eccv16_dataset_tvsum_google_pool5/video_49", 388 | "eccv16_dataset_ovp_google_pool5/video_23", 389 | "eccv16_dataset_summe_google_pool5/video_25", 390 | "eccv16_dataset_summe_google_pool5/video_10", 391 | "eccv16_dataset_ovp_google_pool5/video_29", 392 | "eccv16_dataset_youtube_google_pool5/video_14", 393 | "eccv16_dataset_youtube_google_pool5/video_23", 394 | "eccv16_dataset_tvsum_google_pool5/video_3", 395 | "eccv16_dataset_summe_google_pool5/video_16", 396 | "eccv16_dataset_tvsum_google_pool5/video_27", 397 | "eccv16_dataset_ovp_google_pool5/video_14", 398 | "eccv16_dataset_tvsum_google_pool5/video_23", 399 | "eccv16_dataset_tvsum_google_pool5/video_47", 400 | "eccv16_dataset_ovp_google_pool5/video_40", 401 | "eccv16_dataset_youtube_google_pool5/video_12", 402 | "eccv16_dataset_ovp_google_pool5/video_6", 403 | "eccv16_dataset_ovp_google_pool5/video_17", 404 | "eccv16_dataset_summe_google_pool5/video_7", 405 | "eccv16_dataset_ovp_google_pool5/video_13", 406 | "eccv16_dataset_tvsum_google_pool5/video_28", 407 | "eccv16_dataset_tvsum_google_pool5/video_5", 408 | "eccv16_dataset_youtube_google_pool5/video_13", 409 | "eccv16_dataset_ovp_google_pool5/video_49", 410 | "eccv16_dataset_youtube_google_pool5/video_28", 411 | "eccv16_dataset_ovp_google_pool5/video_33", 412 | "eccv16_dataset_tvsum_google_pool5/video_25", 413 | "eccv16_dataset_tvsum_google_pool5/video_26", 414 | "eccv16_dataset_ovp_google_pool5/video_26", 415 | "eccv16_dataset_tvsum_google_pool5/video_10", 416 | "eccv16_dataset_ovp_google_pool5/video_24", 417 | "eccv16_dataset_ovp_google_pool5/video_30", 418 | "eccv16_dataset_ovp_google_pool5/video_42", 419 | "eccv16_dataset_summe_google_pool5/video_18", 420 | "eccv16_dataset_youtube_google_pool5/video_25", 421 | "eccv16_dataset_youtube_google_pool5/video_21", 422 | "eccv16_dataset_tvsum_google_pool5/video_18", 423 | "eccv16_dataset_youtube_google_pool5/video_11", 424 | "eccv16_dataset_tvsum_google_pool5/video_29", 425 | "eccv16_dataset_ovp_google_pool5/video_25", 426 | "eccv16_dataset_youtube_google_pool5/video_26", 427 | "eccv16_dataset_summe_google_pool5/video_21", 428 | "eccv16_dataset_ovp_google_pool5/video_28", 429 | "eccv16_dataset_summe_google_pool5/video_22", 430 | "eccv16_dataset_ovp_google_pool5/video_16", 431 | "eccv16_dataset_tvsum_google_pool5/video_37", 432 | "eccv16_dataset_tvsum_google_pool5/video_19", 433 | "eccv16_dataset_summe_google_pool5/video_6", 434 | "eccv16_dataset_youtube_google_pool5/video_47", 435 | "eccv16_dataset_youtube_google_pool5/video_32", 436 | "eccv16_dataset_ovp_google_pool5/video_4", 437 | "eccv16_dataset_tvsum_google_pool5/video_40", 438 | "eccv16_dataset_ovp_google_pool5/video_39", 439 | "eccv16_dataset_youtube_google_pool5/video_43", 440 | "eccv16_dataset_summe_google_pool5/video_9", 441 | "eccv16_dataset_ovp_google_pool5/video_48", 442 | "eccv16_dataset_youtube_google_pool5/video_17", 443 | "eccv16_dataset_summe_google_pool5/video_4", 444 | "eccv16_dataset_ovp_google_pool5/video_9", 445 | "eccv16_dataset_summe_google_pool5/video_13", 446 | "eccv16_dataset_ovp_google_pool5/video_34", 447 | "eccv16_dataset_ovp_google_pool5/video_2", 448 | "eccv16_dataset_youtube_google_pool5/video_16", 449 | "eccv16_dataset_youtube_google_pool5/video_18", 450 | "eccv16_dataset_ovp_google_pool5/video_41", 451 | "eccv16_dataset_ovp_google_pool5/video_44", 452 | "eccv16_dataset_tvsum_google_pool5/video_20", 453 | "eccv16_dataset_youtube_google_pool5/video_42", 454 | "eccv16_dataset_youtube_google_pool5/video_48", 455 | "eccv16_dataset_youtube_google_pool5/video_34", 456 | "eccv16_dataset_tvsum_google_pool5/video_30", 457 | "eccv16_dataset_youtube_google_pool5/video_31", 458 | "eccv16_dataset_youtube_google_pool5/video_38", 459 | "eccv16_dataset_youtube_google_pool5/video_46", 460 | "eccv16_dataset_summe_google_pool5/video_20", 461 | "eccv16_dataset_youtube_google_pool5/video_15", 462 | "eccv16_dataset_youtube_google_pool5/video_36", 463 | "eccv16_dataset_ovp_google_pool5/video_38", 464 | "eccv16_dataset_ovp_google_pool5/video_47", 465 | "eccv16_dataset_tvsum_google_pool5/video_43", 466 | "eccv16_dataset_tvsum_google_pool5/video_48", 467 | "eccv16_dataset_tvsum_google_pool5/video_41", 468 | "eccv16_dataset_youtube_google_pool5/video_29", 469 | "eccv16_dataset_tvsum_google_pool5/video_17", 470 | "eccv16_dataset_tvsum_google_pool5/video_38", 471 | "eccv16_dataset_tvsum_google_pool5/video_45", 472 | "eccv16_dataset_youtube_google_pool5/video_50", 473 | "eccv16_dataset_youtube_google_pool5/video_49", 474 | "eccv16_dataset_ovp_google_pool5/video_10", 475 | "eccv16_dataset_summe_google_pool5/video_11", 476 | "eccv16_dataset_ovp_google_pool5/video_12", 477 | "eccv16_dataset_ovp_google_pool5/video_20", 478 | "eccv16_dataset_summe_google_pool5/video_2", 479 | "eccv16_dataset_youtube_google_pool5/video_37", 480 | "eccv16_dataset_ovp_google_pool5/video_15", 481 | "eccv16_dataset_youtube_google_pool5/video_45", 482 | "eccv16_dataset_summe_google_pool5/video_12", 483 | "eccv16_dataset_ovp_google_pool5/video_1", 484 | "eccv16_dataset_summe_google_pool5/video_17", 485 | "eccv16_dataset_ovp_google_pool5/video_19", 486 | "eccv16_dataset_youtube_google_pool5/video_20", 487 | "eccv16_dataset_tvsum_google_pool5/video_31", 488 | "eccv16_dataset_summe_google_pool5/video_23", 489 | "eccv16_dataset_tvsum_google_pool5/video_44", 490 | "eccv16_dataset_tvsum_google_pool5/video_24", 491 | "eccv16_dataset_tvsum_google_pool5/video_21", 492 | "eccv16_dataset_youtube_google_pool5/video_27", 493 | "eccv16_dataset_youtube_google_pool5/video_44", 494 | "eccv16_dataset_youtube_google_pool5/video_24", 495 | "eccv16_dataset_ovp_google_pool5/video_3", 496 | "eccv16_dataset_ovp_google_pool5/video_11", 497 | "eccv16_dataset_tvsum_google_pool5/video_11", 498 | "eccv16_dataset_ovp_google_pool5/video_8", 499 | "eccv16_dataset_tvsum_google_pool5/video_14", 500 | "eccv16_dataset_tvsum_google_pool5/video_50", 501 | "eccv16_dataset_summe_google_pool5/video_3", 502 | "eccv16_dataset_summe_google_pool5/video_8", 503 | "eccv16_dataset_youtube_google_pool5/video_39", 504 | "eccv16_dataset_tvsum_google_pool5/video_12", 505 | "eccv16_dataset_summe_google_pool5/video_1", 506 | "eccv16_dataset_ovp_google_pool5/video_32", 507 | "eccv16_dataset_tvsum_google_pool5/video_34", 508 | "eccv16_dataset_ovp_google_pool5/video_7", 509 | "eccv16_dataset_summe_google_pool5/video_24" 510 | ] 511 | }, 512 | { 513 | "train_keys": [ 514 | "eccv16_dataset_youtube_google_pool5/video_17", 515 | "eccv16_dataset_ovp_google_pool5/video_8", 516 | "eccv16_dataset_youtube_google_pool5/video_18", 517 | "eccv16_dataset_tvsum_google_pool5/video_46", 518 | "eccv16_dataset_ovp_google_pool5/video_43", 519 | "eccv16_dataset_ovp_google_pool5/video_10", 520 | "eccv16_dataset_tvsum_google_pool5/video_43", 521 | "eccv16_dataset_youtube_google_pool5/video_12", 522 | "eccv16_dataset_ovp_google_pool5/video_47", 523 | "eccv16_dataset_tvsum_google_pool5/video_23", 524 | "eccv16_dataset_youtube_google_pool5/video_20", 525 | "eccv16_dataset_youtube_google_pool5/video_31", 526 | "eccv16_dataset_youtube_google_pool5/video_26", 527 | "eccv16_dataset_tvsum_google_pool5/video_24", 528 | "eccv16_dataset_tvsum_google_pool5/video_49", 529 | "eccv16_dataset_ovp_google_pool5/video_40", 530 | "eccv16_dataset_youtube_google_pool5/video_49", 531 | "eccv16_dataset_tvsum_google_pool5/video_22", 532 | "eccv16_dataset_tvsum_google_pool5/video_25", 533 | "eccv16_dataset_summe_google_pool5/video_16", 534 | "eccv16_dataset_summe_google_pool5/video_17", 535 | "eccv16_dataset_tvsum_google_pool5/video_13", 536 | "eccv16_dataset_summe_google_pool5/video_21", 537 | "eccv16_dataset_ovp_google_pool5/video_25", 538 | "eccv16_dataset_youtube_google_pool5/video_13", 539 | "eccv16_dataset_tvsum_google_pool5/video_36", 540 | "eccv16_dataset_tvsum_google_pool5/video_7", 541 | "eccv16_dataset_summe_google_pool5/video_24", 542 | "eccv16_dataset_tvsum_google_pool5/video_21", 543 | "eccv16_dataset_summe_google_pool5/video_13", 544 | "eccv16_dataset_summe_google_pool5/video_5", 545 | "eccv16_dataset_youtube_google_pool5/video_21", 546 | "eccv16_dataset_tvsum_google_pool5/video_42", 547 | "eccv16_dataset_ovp_google_pool5/video_19", 548 | "eccv16_dataset_youtube_google_pool5/video_45", 549 | "eccv16_dataset_ovp_google_pool5/video_46", 550 | "eccv16_dataset_ovp_google_pool5/video_20", 551 | "eccv16_dataset_tvsum_google_pool5/video_6", 552 | "eccv16_dataset_tvsum_google_pool5/video_11", 553 | "eccv16_dataset_ovp_google_pool5/video_4", 554 | "eccv16_dataset_youtube_google_pool5/video_24", 555 | "eccv16_dataset_tvsum_google_pool5/video_41", 556 | "eccv16_dataset_tvsum_google_pool5/video_50", 557 | "eccv16_dataset_summe_google_pool5/video_10", 558 | "eccv16_dataset_youtube_google_pool5/video_37", 559 | "eccv16_dataset_tvsum_google_pool5/video_10", 560 | "eccv16_dataset_summe_google_pool5/video_23", 561 | "eccv16_dataset_youtube_google_pool5/video_11", 562 | "eccv16_dataset_tvsum_google_pool5/video_15", 563 | "eccv16_dataset_youtube_google_pool5/video_38", 564 | "eccv16_dataset_ovp_google_pool5/video_23", 565 | "eccv16_dataset_youtube_google_pool5/video_42", 566 | "eccv16_dataset_tvsum_google_pool5/video_39", 567 | "eccv16_dataset_tvsum_google_pool5/video_27", 568 | "eccv16_dataset_ovp_google_pool5/video_16", 569 | "eccv16_dataset_tvsum_google_pool5/video_44", 570 | "eccv16_dataset_summe_google_pool5/video_7", 571 | "eccv16_dataset_ovp_google_pool5/video_42", 572 | "eccv16_dataset_tvsum_google_pool5/video_33", 573 | "eccv16_dataset_youtube_google_pool5/video_33", 574 | "eccv16_dataset_summe_google_pool5/video_25", 575 | "eccv16_dataset_ovp_google_pool5/video_21", 576 | "eccv16_dataset_youtube_google_pool5/video_47", 577 | "eccv16_dataset_youtube_google_pool5/video_39", 578 | "eccv16_dataset_youtube_google_pool5/video_28", 579 | "eccv16_dataset_tvsum_google_pool5/video_30", 580 | "eccv16_dataset_ovp_google_pool5/video_36", 581 | "eccv16_dataset_tvsum_google_pool5/video_48", 582 | "eccv16_dataset_ovp_google_pool5/video_11", 583 | "eccv16_dataset_youtube_google_pool5/video_14", 584 | "eccv16_dataset_ovp_google_pool5/video_18", 585 | "eccv16_dataset_summe_google_pool5/video_14", 586 | "eccv16_dataset_tvsum_google_pool5/video_2", 587 | "eccv16_dataset_ovp_google_pool5/video_41", 588 | "eccv16_dataset_tvsum_google_pool5/video_3", 589 | "eccv16_dataset_summe_google_pool5/video_18", 590 | "eccv16_dataset_ovp_google_pool5/video_35", 591 | "eccv16_dataset_tvsum_google_pool5/video_47", 592 | "eccv16_dataset_ovp_google_pool5/video_14", 593 | "eccv16_dataset_youtube_google_pool5/video_23", 594 | "eccv16_dataset_ovp_google_pool5/video_15", 595 | "eccv16_dataset_ovp_google_pool5/video_13", 596 | "eccv16_dataset_ovp_google_pool5/video_31", 597 | "eccv16_dataset_summe_google_pool5/video_9", 598 | "eccv16_dataset_ovp_google_pool5/video_38", 599 | "eccv16_dataset_summe_google_pool5/video_4", 600 | "eccv16_dataset_ovp_google_pool5/video_22", 601 | "eccv16_dataset_ovp_google_pool5/video_39", 602 | "eccv16_dataset_ovp_google_pool5/video_6", 603 | "eccv16_dataset_youtube_google_pool5/video_40", 604 | "eccv16_dataset_ovp_google_pool5/video_49", 605 | "eccv16_dataset_ovp_google_pool5/video_30", 606 | "eccv16_dataset_tvsum_google_pool5/video_16", 607 | "eccv16_dataset_tvsum_google_pool5/video_14", 608 | "eccv16_dataset_tvsum_google_pool5/video_40", 609 | "eccv16_dataset_summe_google_pool5/video_1", 610 | "eccv16_dataset_youtube_google_pool5/video_43", 611 | "eccv16_dataset_tvsum_google_pool5/video_29", 612 | "eccv16_dataset_youtube_google_pool5/video_15", 613 | "eccv16_dataset_tvsum_google_pool5/video_19", 614 | "eccv16_dataset_ovp_google_pool5/video_17", 615 | "eccv16_dataset_youtube_google_pool5/video_35", 616 | "eccv16_dataset_tvsum_google_pool5/video_5", 617 | "eccv16_dataset_ovp_google_pool5/video_44", 618 | "eccv16_dataset_youtube_google_pool5/video_27", 619 | "eccv16_dataset_youtube_google_pool5/video_36", 620 | "eccv16_dataset_ovp_google_pool5/video_12", 621 | "eccv16_dataset_ovp_google_pool5/video_1", 622 | "eccv16_dataset_tvsum_google_pool5/video_9", 623 | "eccv16_dataset_ovp_google_pool5/video_33", 624 | "eccv16_dataset_youtube_google_pool5/video_46", 625 | "eccv16_dataset_ovp_google_pool5/video_50", 626 | "eccv16_dataset_tvsum_google_pool5/video_4", 627 | "eccv16_dataset_summe_google_pool5/video_22", 628 | "eccv16_dataset_ovp_google_pool5/video_24", 629 | "eccv16_dataset_summe_google_pool5/video_11", 630 | "eccv16_dataset_summe_google_pool5/video_15", 631 | "eccv16_dataset_youtube_google_pool5/video_44", 632 | "eccv16_dataset_ovp_google_pool5/video_27", 633 | "eccv16_dataset_youtube_google_pool5/video_50", 634 | "eccv16_dataset_youtube_google_pool5/video_32", 635 | "eccv16_dataset_tvsum_google_pool5/video_37", 636 | "eccv16_dataset_ovp_google_pool5/video_45", 637 | "eccv16_dataset_ovp_google_pool5/video_32", 638 | "eccv16_dataset_ovp_google_pool5/video_2", 639 | "eccv16_dataset_summe_google_pool5/video_12", 640 | "eccv16_dataset_youtube_google_pool5/video_19", 641 | "eccv16_dataset_youtube_google_pool5/video_16", 642 | "eccv16_dataset_tvsum_google_pool5/video_17", 643 | "eccv16_dataset_youtube_google_pool5/video_30", 644 | "eccv16_dataset_ovp_google_pool5/video_7", 645 | "eccv16_dataset_ovp_google_pool5/video_37", 646 | "eccv16_dataset_summe_google_pool5/video_3", 647 | "eccv16_dataset_ovp_google_pool5/video_26", 648 | "eccv16_dataset_tvsum_google_pool5/video_38", 649 | "eccv16_dataset_summe_google_pool5/video_8", 650 | "eccv16_dataset_ovp_google_pool5/video_34", 651 | "eccv16_dataset_youtube_google_pool5/video_25", 652 | "eccv16_dataset_ovp_google_pool5/video_5", 653 | "eccv16_dataset_summe_google_pool5/video_2", 654 | "eccv16_dataset_tvsum_google_pool5/video_1", 655 | "eccv16_dataset_tvsum_google_pool5/video_20", 656 | "eccv16_dataset_youtube_google_pool5/video_48", 657 | "eccv16_dataset_ovp_google_pool5/video_3", 658 | "eccv16_dataset_ovp_google_pool5/video_48", 659 | "eccv16_dataset_summe_google_pool5/video_19", 660 | "eccv16_dataset_summe_google_pool5/video_20", 661 | "eccv16_dataset_youtube_google_pool5/video_34", 662 | "eccv16_dataset_ovp_google_pool5/video_29", 663 | "eccv16_dataset_summe_google_pool5/video_6", 664 | "eccv16_dataset_ovp_google_pool5/video_9", 665 | "eccv16_dataset_youtube_google_pool5/video_41", 666 | "eccv16_dataset_ovp_google_pool5/video_28", 667 | "eccv16_dataset_youtube_google_pool5/video_29" 668 | ], 669 | "test_keys": [ 670 | "eccv16_dataset_tvsum_google_pool5/video_12", 671 | "eccv16_dataset_tvsum_google_pool5/video_31", 672 | "eccv16_dataset_tvsum_google_pool5/video_32", 673 | "eccv16_dataset_tvsum_google_pool5/video_28", 674 | "eccv16_dataset_tvsum_google_pool5/video_35", 675 | "eccv16_dataset_tvsum_google_pool5/video_26", 676 | "eccv16_dataset_tvsum_google_pool5/video_8", 677 | "eccv16_dataset_tvsum_google_pool5/video_18", 678 | "eccv16_dataset_tvsum_google_pool5/video_45", 679 | "eccv16_dataset_tvsum_google_pool5/video_34" 680 | ] 681 | }, 682 | { 683 | "test_keys": [ 684 | "eccv16_dataset_tvsum_google_pool5/video_34", 685 | "eccv16_dataset_tvsum_google_pool5/video_22", 686 | "eccv16_dataset_tvsum_google_pool5/video_17", 687 | "eccv16_dataset_tvsum_google_pool5/video_11", 688 | "eccv16_dataset_tvsum_google_pool5/video_3", 689 | "eccv16_dataset_tvsum_google_pool5/video_9", 690 | "eccv16_dataset_tvsum_google_pool5/video_28", 691 | "eccv16_dataset_tvsum_google_pool5/video_27", 692 | "eccv16_dataset_tvsum_google_pool5/video_44", 693 | "eccv16_dataset_tvsum_google_pool5/video_50" 694 | ], 695 | "train_keys": [ 696 | "eccv16_dataset_ovp_google_pool5/video_26", 697 | "eccv16_dataset_tvsum_google_pool5/video_21", 698 | "eccv16_dataset_youtube_google_pool5/video_14", 699 | "eccv16_dataset_summe_google_pool5/video_13", 700 | "eccv16_dataset_youtube_google_pool5/video_12", 701 | "eccv16_dataset_summe_google_pool5/video_6", 702 | "eccv16_dataset_summe_google_pool5/video_23", 703 | "eccv16_dataset_youtube_google_pool5/video_24", 704 | "eccv16_dataset_tvsum_google_pool5/video_24", 705 | "eccv16_dataset_youtube_google_pool5/video_26", 706 | "eccv16_dataset_tvsum_google_pool5/video_47", 707 | "eccv16_dataset_ovp_google_pool5/video_16", 708 | "eccv16_dataset_tvsum_google_pool5/video_46", 709 | "eccv16_dataset_youtube_google_pool5/video_28", 710 | "eccv16_dataset_youtube_google_pool5/video_19", 711 | "eccv16_dataset_tvsum_google_pool5/video_18", 712 | "eccv16_dataset_ovp_google_pool5/video_1", 713 | "eccv16_dataset_ovp_google_pool5/video_23", 714 | "eccv16_dataset_ovp_google_pool5/video_9", 715 | "eccv16_dataset_ovp_google_pool5/video_45", 716 | "eccv16_dataset_tvsum_google_pool5/video_20", 717 | "eccv16_dataset_youtube_google_pool5/video_49", 718 | "eccv16_dataset_youtube_google_pool5/video_41", 719 | "eccv16_dataset_tvsum_google_pool5/video_33", 720 | "eccv16_dataset_tvsum_google_pool5/video_43", 721 | "eccv16_dataset_ovp_google_pool5/video_36", 722 | "eccv16_dataset_ovp_google_pool5/video_27", 723 | "eccv16_dataset_youtube_google_pool5/video_40", 724 | "eccv16_dataset_ovp_google_pool5/video_11", 725 | "eccv16_dataset_tvsum_google_pool5/video_32", 726 | "eccv16_dataset_tvsum_google_pool5/video_8", 727 | "eccv16_dataset_summe_google_pool5/video_9", 728 | "eccv16_dataset_tvsum_google_pool5/video_26", 729 | "eccv16_dataset_tvsum_google_pool5/video_16", 730 | "eccv16_dataset_summe_google_pool5/video_10", 731 | "eccv16_dataset_ovp_google_pool5/video_24", 732 | "eccv16_dataset_ovp_google_pool5/video_13", 733 | "eccv16_dataset_ovp_google_pool5/video_33", 734 | "eccv16_dataset_ovp_google_pool5/video_32", 735 | "eccv16_dataset_ovp_google_pool5/video_48", 736 | "eccv16_dataset_youtube_google_pool5/video_30", 737 | "eccv16_dataset_ovp_google_pool5/video_7", 738 | "eccv16_dataset_ovp_google_pool5/video_30", 739 | "eccv16_dataset_ovp_google_pool5/video_29", 740 | "eccv16_dataset_tvsum_google_pool5/video_36", 741 | "eccv16_dataset_summe_google_pool5/video_3", 742 | "eccv16_dataset_tvsum_google_pool5/video_15", 743 | "eccv16_dataset_ovp_google_pool5/video_44", 744 | "eccv16_dataset_summe_google_pool5/video_4", 745 | "eccv16_dataset_summe_google_pool5/video_16", 746 | "eccv16_dataset_ovp_google_pool5/video_18", 747 | "eccv16_dataset_youtube_google_pool5/video_15", 748 | "eccv16_dataset_youtube_google_pool5/video_34", 749 | "eccv16_dataset_tvsum_google_pool5/video_1", 750 | "eccv16_dataset_youtube_google_pool5/video_27", 751 | "eccv16_dataset_tvsum_google_pool5/video_37", 752 | "eccv16_dataset_tvsum_google_pool5/video_45", 753 | "eccv16_dataset_summe_google_pool5/video_21", 754 | "eccv16_dataset_ovp_google_pool5/video_49", 755 | "eccv16_dataset_youtube_google_pool5/video_45", 756 | "eccv16_dataset_summe_google_pool5/video_19", 757 | "eccv16_dataset_summe_google_pool5/video_15", 758 | "eccv16_dataset_ovp_google_pool5/video_37", 759 | "eccv16_dataset_tvsum_google_pool5/video_12", 760 | "eccv16_dataset_ovp_google_pool5/video_41", 761 | "eccv16_dataset_tvsum_google_pool5/video_4", 762 | "eccv16_dataset_summe_google_pool5/video_8", 763 | "eccv16_dataset_youtube_google_pool5/video_37", 764 | "eccv16_dataset_tvsum_google_pool5/video_39", 765 | "eccv16_dataset_tvsum_google_pool5/video_48", 766 | "eccv16_dataset_ovp_google_pool5/video_5", 767 | "eccv16_dataset_tvsum_google_pool5/video_29", 768 | "eccv16_dataset_ovp_google_pool5/video_10", 769 | "eccv16_dataset_summe_google_pool5/video_2", 770 | "eccv16_dataset_tvsum_google_pool5/video_19", 771 | "eccv16_dataset_youtube_google_pool5/video_44", 772 | "eccv16_dataset_youtube_google_pool5/video_18", 773 | "eccv16_dataset_ovp_google_pool5/video_25", 774 | "eccv16_dataset_youtube_google_pool5/video_43", 775 | "eccv16_dataset_tvsum_google_pool5/video_41", 776 | "eccv16_dataset_summe_google_pool5/video_17", 777 | "eccv16_dataset_ovp_google_pool5/video_6", 778 | "eccv16_dataset_youtube_google_pool5/video_16", 779 | "eccv16_dataset_summe_google_pool5/video_11", 780 | "eccv16_dataset_summe_google_pool5/video_5", 781 | "eccv16_dataset_ovp_google_pool5/video_21", 782 | "eccv16_dataset_summe_google_pool5/video_20", 783 | "eccv16_dataset_ovp_google_pool5/video_15", 784 | "eccv16_dataset_youtube_google_pool5/video_36", 785 | "eccv16_dataset_ovp_google_pool5/video_40", 786 | "eccv16_dataset_ovp_google_pool5/video_8", 787 | "eccv16_dataset_ovp_google_pool5/video_22", 788 | "eccv16_dataset_tvsum_google_pool5/video_14", 789 | "eccv16_dataset_summe_google_pool5/video_1", 790 | "eccv16_dataset_tvsum_google_pool5/video_35", 791 | "eccv16_dataset_youtube_google_pool5/video_39", 792 | "eccv16_dataset_summe_google_pool5/video_12", 793 | "eccv16_dataset_tvsum_google_pool5/video_42", 794 | "eccv16_dataset_ovp_google_pool5/video_35", 795 | "eccv16_dataset_tvsum_google_pool5/video_13", 796 | "eccv16_dataset_youtube_google_pool5/video_35", 797 | "eccv16_dataset_ovp_google_pool5/video_2", 798 | "eccv16_dataset_ovp_google_pool5/video_19", 799 | "eccv16_dataset_tvsum_google_pool5/video_25", 800 | "eccv16_dataset_tvsum_google_pool5/video_5", 801 | "eccv16_dataset_tvsum_google_pool5/video_6", 802 | "eccv16_dataset_youtube_google_pool5/video_50", 803 | "eccv16_dataset_ovp_google_pool5/video_34", 804 | "eccv16_dataset_summe_google_pool5/video_24", 805 | "eccv16_dataset_tvsum_google_pool5/video_10", 806 | "eccv16_dataset_youtube_google_pool5/video_29", 807 | "eccv16_dataset_ovp_google_pool5/video_20", 808 | "eccv16_dataset_youtube_google_pool5/video_23", 809 | "eccv16_dataset_youtube_google_pool5/video_48", 810 | "eccv16_dataset_youtube_google_pool5/video_11", 811 | "eccv16_dataset_youtube_google_pool5/video_32", 812 | "eccv16_dataset_ovp_google_pool5/video_3", 813 | "eccv16_dataset_summe_google_pool5/video_7", 814 | "eccv16_dataset_summe_google_pool5/video_18", 815 | "eccv16_dataset_ovp_google_pool5/video_38", 816 | "eccv16_dataset_summe_google_pool5/video_22", 817 | "eccv16_dataset_tvsum_google_pool5/video_2", 818 | "eccv16_dataset_ovp_google_pool5/video_42", 819 | "eccv16_dataset_tvsum_google_pool5/video_30", 820 | "eccv16_dataset_ovp_google_pool5/video_28", 821 | "eccv16_dataset_youtube_google_pool5/video_33", 822 | "eccv16_dataset_tvsum_google_pool5/video_38", 823 | "eccv16_dataset_ovp_google_pool5/video_12", 824 | "eccv16_dataset_ovp_google_pool5/video_31", 825 | "eccv16_dataset_youtube_google_pool5/video_46", 826 | "eccv16_dataset_youtube_google_pool5/video_42", 827 | "eccv16_dataset_youtube_google_pool5/video_21", 828 | "eccv16_dataset_youtube_google_pool5/video_47", 829 | "eccv16_dataset_tvsum_google_pool5/video_40", 830 | "eccv16_dataset_ovp_google_pool5/video_4", 831 | "eccv16_dataset_tvsum_google_pool5/video_7", 832 | "eccv16_dataset_ovp_google_pool5/video_43", 833 | "eccv16_dataset_tvsum_google_pool5/video_49", 834 | "eccv16_dataset_tvsum_google_pool5/video_23", 835 | "eccv16_dataset_youtube_google_pool5/video_17", 836 | "eccv16_dataset_youtube_google_pool5/video_25", 837 | "eccv16_dataset_summe_google_pool5/video_14", 838 | "eccv16_dataset_summe_google_pool5/video_25", 839 | "eccv16_dataset_youtube_google_pool5/video_13", 840 | "eccv16_dataset_tvsum_google_pool5/video_31", 841 | "eccv16_dataset_ovp_google_pool5/video_47", 842 | "eccv16_dataset_youtube_google_pool5/video_38", 843 | "eccv16_dataset_ovp_google_pool5/video_39", 844 | "eccv16_dataset_ovp_google_pool5/video_17", 845 | "eccv16_dataset_youtube_google_pool5/video_31", 846 | "eccv16_dataset_ovp_google_pool5/video_14", 847 | "eccv16_dataset_youtube_google_pool5/video_20", 848 | "eccv16_dataset_ovp_google_pool5/video_50", 849 | "eccv16_dataset_ovp_google_pool5/video_46" 850 | ] 851 | } 852 | ] 853 | --------------------------------------------------------------------------------